Articles as the IndieWeb defines them: a name separate from the content.
- Migration 023 adds notes.title and notes.summary. A one-time Python data
step (migrations.DATA_MIGRATIONS) moves each note's '# Heading' first line
into title, rewrites the file and hash, empties FTS for a rebuild, and puts
files back if the migration fails. Trashed notes included.
- Micropub: name -> title, summary stored when sent with content, summary
updatable, q=source returns both. MCP tools take a summary.
- Feeds: titles only for articles (RSS/JSON none, Atom empty); Atom and
JSON Feed summaries; the title is no longer repeated in the content.
- Pages: p-name/p-summary on the article page. Homepage and tag pages show
articles as title, summary and "Read more" without images (hidden u-photo
keeps the post type for parsers).
- Crossposts: articles go out as title, summary, link and hashtags, no
images; the Bluesky card is the article. Notes unchanged.
- Editor: Title and Summary fields; an article may have no body.
- Fix: a line starting with a hashtag ('#indieweb ...') was rendered as a
heading and crossposted without its '#' (HashtagLineExtension).
- Fix: excerpts showed raw Markdown; they are plain text now.
Back up the database and the data folder before deploying: the migration
rewrites note files.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
416 lines
17 KiB
Python
416 lines
17 KiB
Python
"""
|
|
Tests for crossposting to Bluesky (starpunk/syndication.py, v2.2.0)
|
|
|
|
A fake PDS (httpx.MockTransport) records what StarPunk sends.
|
|
"""
|
|
|
|
import io
|
|
import json
|
|
|
|
import httpx
|
|
import pytest
|
|
from PIL import Image
|
|
|
|
from starpunk import create_app, syndication
|
|
from starpunk.notes import create_note, delete_note, update_note
|
|
from starpunk.syndication import (
|
|
BlueskyClient,
|
|
bluesky_facets,
|
|
get_note_syndications,
|
|
plain_text,
|
|
process_due,
|
|
select_targets,
|
|
)
|
|
|
|
SERVICE = "https://bsky.example"
|
|
PDS = "https://pds.example"
|
|
DID = "did:plc:abc123"
|
|
|
|
|
|
class FakePDS:
|
|
def __init__(self):
|
|
self.records = {}
|
|
self.blobs = []
|
|
self.calls = []
|
|
self.logins = 0
|
|
self.expire_next = False
|
|
self.lose_next_reply = False
|
|
self.fail_next = None
|
|
|
|
def _json(self, code, body):
|
|
return httpx.Response(code, json=body)
|
|
|
|
def __call__(self, request: httpx.Request) -> httpx.Response:
|
|
nsid = request.url.path.removeprefix("/xrpc/")
|
|
self.calls.append((request.url.host, nsid))
|
|
if nsid == "com.atproto.server.createSession":
|
|
body = json.loads(request.content)
|
|
if body["password"] != "app-pass":
|
|
return self._json(401, {"error": "AuthenticationRequired"})
|
|
self.logins += 1
|
|
return self._json(200, {
|
|
"did": DID, "handle": "phil.example", "accessJwt": f"access{self.logins}",
|
|
"refreshJwt": "refresh",
|
|
"didDoc": {"service": [{"id": "#atproto_pds", "serviceEndpoint": PDS}]},
|
|
})
|
|
if nsid == "com.atproto.server.refreshSession":
|
|
self.logins += 1
|
|
return self._json(200, {"did": DID, "handle": "phil.example",
|
|
"accessJwt": f"access{self.logins}", "refreshJwt": "refresh"})
|
|
assert request.url.host == "pds.example"
|
|
if request.headers.get("Authorization") != f"Bearer access{self.logins}" or self.expire_next:
|
|
self.expire_next = False
|
|
return self._json(400, {"error": "ExpiredToken"})
|
|
if self.fail_next:
|
|
code, self.fail_next = self.fail_next, None
|
|
return self._json(code, {"error": "Nope"})
|
|
if nsid == "com.atproto.repo.uploadBlob":
|
|
self.blobs.append(len(request.content))
|
|
assert len(request.content) <= 1_000_000
|
|
return self._json(200, {"blob": {"$type": "blob", "ref": {"$link": f"b{len(self.blobs)}"},
|
|
"mimeType": request.headers["Content-Type"],
|
|
"size": len(request.content)}})
|
|
if nsid == "com.atproto.repo.createRecord":
|
|
body = json.loads(request.content)
|
|
uri = f"at://{DID}/app.bsky.feed.post/r{len(self.records) + 1}"
|
|
self.records[uri] = body["record"]
|
|
if self.lose_next_reply:
|
|
self.lose_next_reply = False
|
|
raise httpx.ReadTimeout("lost reply")
|
|
return self._json(200, {"uri": uri, "cid": "c"})
|
|
if nsid == "com.atproto.repo.listRecords":
|
|
return self._json(200, {"records": [{"uri": u, "value": v} for u, v in self.records.items()]})
|
|
if nsid == "com.atproto.repo.deleteRecord":
|
|
body = json.loads(request.content)
|
|
self.records.pop(f"at://{body['repo']}/{body['collection']}/{body['rkey']}", None)
|
|
return self._json(200, {})
|
|
return self._json(500, {"error": "unknown"})
|
|
|
|
|
|
@pytest.fixture
|
|
def pds():
|
|
return FakePDS()
|
|
|
|
|
|
class FakeWeb:
|
|
"""Other people's pages, for link cards: url -> (status, headers, body)"""
|
|
def __init__(self):
|
|
self.pages = {}
|
|
self.fetched = []
|
|
|
|
def __call__(self, request):
|
|
# Pinned requests go to the checked IP; the Host header names the site
|
|
url = f"{request.url.scheme}://{request.headers['host']}{request.url.raw_path.decode()}"
|
|
self.fetched.append(url)
|
|
status, headers, body = self.pages.get(url, (404, {}, b"nope"))
|
|
return httpx.Response(status, headers=headers, content=body)
|
|
|
|
|
|
@pytest.fixture
|
|
def web():
|
|
return FakeWeb()
|
|
|
|
|
|
def routed(pds, web):
|
|
return httpx.MockTransport(
|
|
lambda r: pds(r) if r.headers.get("host", r.url.host) in ("bsky.example", "pds.example") else web(r))
|
|
|
|
|
|
def resolve(host, *args, **kwargs):
|
|
"""Every test host is public, except internal.example"""
|
|
ip = "10.0.0.5" if host == "internal.example" else "93.184.216.34"
|
|
return [(2, 1, 6, "", (ip, 0))]
|
|
|
|
|
|
@pytest.fixture
|
|
def app(tmp_path, pds, web, monkeypatch):
|
|
# Pages and images for link cards go through the shared SSRF-safe fetcher (v2.11.0)
|
|
monkeypatch.setattr("starpunk.webmention_ssrf.TRANSPORT", routed(pds, web))
|
|
monkeypatch.setattr("starpunk.webmention_ssrf.socket.getaddrinfo", resolve)
|
|
data = tmp_path / "data"
|
|
app = create_app({
|
|
"TESTING": True,
|
|
"SITE_URL": "https://blog.example.com",
|
|
"ADMIN_ME": "https://example.com",
|
|
"SESSION_SECRET": "test-secret",
|
|
"DATA_PATH": data,
|
|
"NOTES_PATH": data / "notes",
|
|
"DATABASE_PATH": data / "starpunk.db",
|
|
"BLUESKY_HANDLE": "phil.example",
|
|
"BLUESKY_APP_PASSWORD": "app-pass",
|
|
"BLUESKY_SERVICE": SERVICE,
|
|
})
|
|
syndication._client_cache[f"bluesky:{SERVICE}:phil.example"] = BlueskyClient(
|
|
SERVICE, "phil.example", "app-pass", http=httpx.Client(transport=routed(pds, web))
|
|
)
|
|
yield app
|
|
syndication._client_cache.clear()
|
|
|
|
|
|
def publish(app, content="Hello Bluesky", tags=None):
|
|
with app.test_request_context():
|
|
note = create_note(content=content, published=True, tags=tags)
|
|
select_targets(note.id, ["bluesky"])
|
|
process_due()
|
|
return note
|
|
|
|
|
|
def only_record(pds):
|
|
(record,) = pds.records.values()
|
|
return record
|
|
|
|
|
|
def row(app, note):
|
|
with app.app_context():
|
|
(r,) = get_note_syndications(note.id)
|
|
return r
|
|
|
|
|
|
class TestText:
|
|
def test_markdown_becomes_plain_text(self):
|
|
md = "# Title\n\nSome *emphasis* and a [link](https://x.example).\n\n- one\n- two\n\n"
|
|
assert plain_text(md) == "Title\n\nSome emphasis and a link.\n\n• one\n• two"
|
|
|
|
def test_facets_use_utf8_byte_offsets(self):
|
|
text = "Café ☕ https://blog.example.com/note/x #IndieWeb"
|
|
link, tag = bluesky_facets(text)
|
|
raw = text.encode()
|
|
assert raw[link["index"]["byteStart"]:link["index"]["byteEnd"]] == b"https://blog.example.com/note/x"
|
|
assert raw[tag["index"]["byteStart"]:tag["index"]["byteEnd"]] == b"#IndieWeb"
|
|
assert tag["features"][0]["tag"] == "IndieWeb"
|
|
|
|
def test_hashtag_at_line_start_stays_a_tag(self):
|
|
"""Not turned into a heading and lost (v2.16.0)"""
|
|
assert plain_text("#indieweb is fun") == "#indieweb is fun"
|
|
assert [f["features"][0]["tag"] for f in bluesky_facets(plain_text("#indieweb is fun"))] == ["indieweb"]
|
|
|
|
def test_fragment_in_url_is_not_a_tag(self):
|
|
facets = bluesky_facets("see https://x.example/page#section and #42 too")
|
|
assert [f["features"][0]["$type"] for f in facets] == ["app.bsky.richtext.facet#link"]
|
|
|
|
|
|
class TestLifecycle:
|
|
def test_article_posts_teaser_with_its_own_card(self, app, pds):
|
|
"""An article's card is the article, even when its text links elsewhere (v2.16.0)"""
|
|
with app.test_request_context():
|
|
note = create_note(content="See [this](https://elsewhere.example/x) for more.",
|
|
title="My Article", summary="The gist.", published=True)
|
|
select_targets(note.id, ["bluesky"])
|
|
process_due()
|
|
record = only_record(pds)
|
|
permalink = f"https://blog.example.com/note/{note.slug}"
|
|
assert record["text"] == f"My Article\n\nThe gist.\n\n{permalink}"
|
|
assert record["embed"]["external"] == {
|
|
"uri": permalink, "title": "My Article", "description": "The gist."}
|
|
|
|
def test_publish_posts_with_link_card(self, app, pds):
|
|
note = publish(app, content="Hello **Bluesky**", tags=["IndieWeb"])
|
|
record = only_record(pds)
|
|
permalink = f"https://blog.example.com/note/{note.slug}"
|
|
assert record["text"] == f"Hello Bluesky\n\n{permalink}\n\n#IndieWeb"
|
|
assert record["embed"]["external"]["uri"] == permalink
|
|
kinds = {f["features"][0]["$type"] for f in record["facets"]}
|
|
assert kinds == {"app.bsky.richtext.facet#link", "app.bsky.richtext.facet#tag"}
|
|
r = row(app, note)
|
|
assert r["state"] == "synced"
|
|
assert r["remote_id"] == f"at://{DID}/app.bsky.feed.post/r1"
|
|
assert r["remote_url"] == "https://bsky.app/profile/phil.example/post/r1"
|
|
|
|
def test_long_note_fits_300(self, app, pds):
|
|
note = publish(app, content="word " * 500)
|
|
text = only_record(pds)["text"]
|
|
assert len(text) <= 300
|
|
assert text.endswith(f"https://blog.example.com/note/{note.slug}")
|
|
|
|
def test_edit_leaves_post_alone(self, app, pds):
|
|
note = publish(app, content="first")
|
|
with app.test_request_context():
|
|
update_note(id=note.id, content="second")
|
|
process_due()
|
|
assert only_record(pds)["text"].startswith("first")
|
|
assert row(app, note)["state"] == "synced"
|
|
|
|
def test_delete_and_unpublish_remove_post(self, app, pds):
|
|
note = publish(app)
|
|
with app.test_request_context():
|
|
update_note(id=note.id, published=False)
|
|
process_due()
|
|
assert pds.records == {}
|
|
update_note(id=note.id, published=True)
|
|
process_due()
|
|
assert len(pds.records) == 1
|
|
delete_note(id=note.id)
|
|
process_due()
|
|
assert pds.records == {}
|
|
|
|
|
|
class TestSession:
|
|
def test_logs_in_once_and_uses_pds(self, app, pds):
|
|
publish(app, content="one")
|
|
publish(app, content="two")
|
|
assert pds.logins == 1
|
|
assert ("bsky.example", "com.atproto.server.createSession") in pds.calls
|
|
assert ("pds.example", "com.atproto.repo.createRecord") in pds.calls
|
|
|
|
def test_expired_token_is_refreshed(self, app, pds):
|
|
publish(app, content="one")
|
|
pds.expire_next = True
|
|
publish(app, content="two")
|
|
assert len(pds.records) == 2
|
|
assert ("bsky.example", "com.atproto.server.refreshSession") in pds.calls
|
|
|
|
def test_bad_password_fails_permanently(self, app, pds):
|
|
syndication._client_cache[f"bluesky:{SERVICE}:phil.example"].password = "wrong"
|
|
note = publish(app)
|
|
r = row(app, note)
|
|
assert r["state"] == "failed" and "401" in r["last_error"]
|
|
|
|
def test_lost_reply_does_not_double_post(self, app, pds):
|
|
pds.lose_next_reply = True
|
|
note = publish(app)
|
|
assert row(app, note)["state"] == "pending"
|
|
with app.test_request_context():
|
|
from starpunk.syndication import retry
|
|
retry(row(app, note)["id"])
|
|
# retry() resets attempts; simulate the scheduled retry instead
|
|
from starpunk.database import get_db
|
|
get_db(app).execute("UPDATE syndications SET attempts = 1")
|
|
get_db(app).commit()
|
|
process_due()
|
|
assert len(pds.records) == 1
|
|
assert row(app, note)["state"] == "synced"
|
|
|
|
def test_server_error_retries(self, app, pds):
|
|
publish(app, content="warm up") # log in first
|
|
pds.fail_next = 502
|
|
note = publish(app, content="later")
|
|
assert row(app, note)["state"] == "pending"
|
|
|
|
|
|
class TestMedia:
|
|
def test_large_image_is_shrunk_with_alt_and_aspect_ratio(self, app, pds):
|
|
from starpunk.media import attach_media_to_note, save_media
|
|
import os
|
|
|
|
img = Image.frombytes("RGB", (2400, 1800), os.urandom(2400 * 1800 * 3))
|
|
buf = io.BytesIO()
|
|
img.save(buf, format="PNG")
|
|
assert buf.tell() > 1_000_000
|
|
with app.test_request_context():
|
|
note = create_note(content="noisy", published=True)
|
|
media = save_media(buf.getvalue(), "noise.png")
|
|
attach_media_to_note(note.id, [media["id"]], ["Static"])
|
|
select_targets(note.id, ["bluesky"])
|
|
process_due()
|
|
record = only_record(pds)
|
|
(image,) = record["embed"]["images"]
|
|
assert record["embed"]["$type"] == "app.bsky.embed.images"
|
|
assert image["alt"] == "Static"
|
|
assert image["aspectRatio"]["width"] > 0
|
|
|
|
|
|
ARTICLE = "https://news.example/story"
|
|
ARTICLE_HTML = b"""<html><head><title>Fallback title</title>
|
|
<meta property="og:title" content="A Great Story">
|
|
<meta property="og:description" content="What happened, in short.">
|
|
<meta property="og:image" content="/img/story.png">
|
|
</head><body>...</body></html>"""
|
|
|
|
|
|
def png_bytes():
|
|
buf = io.BytesIO()
|
|
Image.new("RGB", (60, 40), (10, 120, 200)).save(buf, format="PNG")
|
|
return buf.getvalue()
|
|
|
|
|
|
class TestLinkCards:
|
|
def test_note_link_gets_the_card(self, app, pds, web):
|
|
web.pages[ARTICLE] = (200, {"content-type": "text/html"}, ARTICLE_HTML)
|
|
web.pages["https://news.example/img/story.png"] = (200, {"content-type": "image/png"}, png_bytes())
|
|
note = publish(app, content=f"Read [this story]({ARTICLE}) now")
|
|
record = only_record(pds)
|
|
external = record["embed"]["external"]
|
|
assert external["uri"] == ARTICLE
|
|
assert external["title"] == "A Great Story"
|
|
assert external["description"] == "What happened, in short."
|
|
assert external["thumb"]["$type"] == "blob"
|
|
# The link text is clickable and the note's permalink is still in the text
|
|
permalink = f"https://blog.example.com/note/{note.slug}"
|
|
assert record["text"].startswith("Read this story now") and permalink in record["text"]
|
|
raw = record["text"].encode()
|
|
links = {raw[f["index"]["byteStart"]:f["index"]["byteEnd"]].decode(): f["features"][0]["uri"]
|
|
for f in record["facets"]}
|
|
assert links["this story"] == ARTICLE
|
|
assert links[permalink] == permalink
|
|
|
|
def test_bare_url_gets_the_card(self, app, pds, web):
|
|
web.pages[ARTICLE] = (200, {"content-type": "text/html"}, ARTICLE_HTML)
|
|
publish(app, content=f"See {ARTICLE}.")
|
|
assert only_record(pds)["embed"]["external"]["uri"] == ARTICLE
|
|
|
|
def test_unreachable_page_still_gets_a_card(self, app, pds, web):
|
|
note = publish(app, content=f"See {ARTICLE}")
|
|
external = only_record(pds)["embed"]["external"]
|
|
assert external == {"uri": ARTICLE, "title": ARTICLE, "description": ""}
|
|
assert row(app, note)["state"] == "synced"
|
|
|
|
def test_redirect_to_private_address_is_not_followed(self, app, pds, web, monkeypatch):
|
|
web.pages[ARTICLE] = (302, {"location": "http://internal.example/admin"}, b"")
|
|
publish(app, content=f"See {ARTICLE}") # internal.example resolves to 10.0.0.5
|
|
assert "http://internal.example/admin" not in web.fetched
|
|
assert only_record(pds)["embed"]["external"]["title"] == ARTICLE
|
|
|
|
def test_images_win_over_link_card(self, app, pds, web):
|
|
from starpunk.media import attach_media_to_note, save_media
|
|
with app.test_request_context():
|
|
note = create_note(content=f"Look {ARTICLE}", published=True)
|
|
media = save_media(png_bytes(), "a.png")
|
|
attach_media_to_note(note.id, [media["id"]], ["pic"])
|
|
select_targets(note.id, ["bluesky"])
|
|
process_due()
|
|
assert only_record(pds)["embed"]["$type"] == "app.bsky.embed.images"
|
|
assert web.fetched == []
|
|
|
|
|
|
class TestLinkText:
|
|
def test_note_links_in_reading_order(self):
|
|
from starpunk.syndication import note_links
|
|
md = "a [x](https://one.example) b https://two.example/p. c <https://three.example> [rel](/local)"
|
|
assert note_links(md) == ["https://one.example", "https://two.example/p", "https://three.example"]
|
|
|
|
def test_link_modes(self):
|
|
from starpunk.syndication import plain_text_links
|
|
md = "[a story](https://s.example) and [https://u.example](https://u.example)"
|
|
assert plain_text_links(md)[0] == "a story and https://u.example"
|
|
assert plain_text_links(md, "inline")[0] == "a story (https://s.example) and https://u.example"
|
|
assert plain_text_links(md, "anchor") == ("a story and https://u.example",
|
|
[("a story", "https://s.example")])
|
|
|
|
def test_inline_url_budget(self):
|
|
from starpunk.syndication import plain_text_links
|
|
md = "https://bare.example then [one](https://1.example) [two](https://2.example) [one again](https://1.example)"
|
|
text = plain_text_links(md, "inline", max_urls=2)[0]
|
|
assert text == "https://bare.example then one (https://1.example) two one again (https://1.example)"
|
|
|
|
def test_anchor_facet_byte_offsets(self):
|
|
text = "Café ☕ this story and more"
|
|
(facet,) = bluesky_facets(text, [("this story", "https://s.example")])
|
|
raw = text.encode()
|
|
assert raw[facet["index"]["byteStart"]:facet["index"]["byteEnd"]] == b"this story"
|
|
|
|
|
|
def test_both_targets_listed_for_micropub(tmp_path):
|
|
data = tmp_path / "d"
|
|
app = create_app({
|
|
"TESTING": True, "SITE_URL": "https://b.example", "ADMIN_ME": "https://e.example",
|
|
"SESSION_SECRET": "x", "DATA_PATH": data, "NOTES_PATH": data / "notes",
|
|
"DATABASE_PATH": data / "s.db",
|
|
"GOTOSOCIAL_URL": "https://social.example", "GOTOSOCIAL_ACCESS_TOKEN": "t",
|
|
"BLUESKY_HANDLE": "phil.example", "BLUESKY_APP_PASSWORD": "p",
|
|
})
|
|
with app.app_context():
|
|
from starpunk.syndication import get_targets, resolve_targets
|
|
assert set(get_targets()) == {"gotosocial", "bluesky"}
|
|
assert resolve_targets(["https://bsky.app/profile/phil.example"]) == ["bluesky"]
|
|
syndication._client_cache.clear()
|