Replies to Mastodon posts showed only the server's domain: the page has no h-entry, so nothing was found and the host stood in for the author. - Context sources in order: Bluesky AppView API, microformats, the origin's Mastodon API or this site's GoToSocial instance, OpenGraph - Migration 024: author_handle, photo, photo_alt, source; rows without a source are fetched again once by the webmention worker - Note page: a card with avatar, name, handle, text, image and date; home/tag/likes: "In reply to <author>" with a short quote - Avatars and images saved locally (CSP allows only this site's images); avatar upkeep now covers reply contexts so their avatars aren't removed Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
271 lines
12 KiB
Python
271 lines
12 KiB
Python
"""
|
|
Reply contexts from fediverse, Bluesky and OpenGraph sources (v2.19.0)
|
|
|
|
Mastodon and GoToSocial pages have no h-entry, so a reply to one used to show
|
|
only the server's domain. The context now comes from the Mastodon API, this
|
|
site's GoToSocial instance, the Bluesky AppView API or OpenGraph tags, and the
|
|
note page shows it as a card; lists show the author and a short quote.
|
|
"""
|
|
|
|
import io
|
|
import json
|
|
import sqlite3
|
|
|
|
import httpx
|
|
import mf2py
|
|
import pytest
|
|
from PIL import Image
|
|
|
|
from starpunk import avatars, syndication
|
|
from starpunk.reply_context import fetch_reply_context, refresh_stale
|
|
from starpunk.syndication import MastodonClient
|
|
|
|
PUBLIC = [(2, 1, 6, "", ("93.184.216.34", 0))]
|
|
|
|
TOOT = "https://masto.example/@ada/117611186265622833"
|
|
GTS_POST = "https://gts.example/@bob/statuses/01M7BE51ABF9HP2G0CPH5DKKGG"
|
|
SKEET = "https://bsky.app/profile/carol.example/post/3l6oveex3ii2l"
|
|
ARTICLE = "https://news.example/2026/11/13/ground-stop"
|
|
|
|
# Mastodon's status page: an app shell with OpenGraph tags, no microformats
|
|
TOOT_PAGE = """<html><head>
|
|
<meta property="og:site_name" content="Masto Example">
|
|
<meta property="og:title" content="Ada (@ada@masto.example)">
|
|
<meta property="profile:username" content="ada@masto.example">
|
|
</head><body><div id="mastodon"></div></body></html>"""
|
|
|
|
STATUS = {
|
|
"id": "117611186265622833",
|
|
"created_at": "2026-11-13T21:11:22.624Z",
|
|
"spoiler_text": "",
|
|
"sensitive": False,
|
|
"content": '<p>All aircraft are <a href="https://x.example/" class="mention">ground</a> stopped.</p>'
|
|
'<script>alert(1)</script>',
|
|
"account": {"username": "ada", "acct": "ada", "display_name": "Ada Lovelace",
|
|
"url": "https://masto.example/@ada",
|
|
"avatar_static": "https://masto.example/avatars/ada.png"},
|
|
"media_attachments": [{"type": "image", "preview_url": "https://masto.example/media/small.png",
|
|
"description": "An advisory"}],
|
|
}
|
|
|
|
GTS_PAGE = """<html><head>
|
|
<meta property="og:title" content="Bob (@bob@gts.example)">
|
|
<meta property="og:profile:username" content="bob@gts.example">
|
|
</head><body><main><article class="status expanded">
|
|
<a class="p-author h-card" href="https://gts.example/@bob"><span class="p-name">Bob</span></a>
|
|
<div class="content e-content"><p>Posted on GoToSocial</p></div>
|
|
</article></main></body></html>"""
|
|
|
|
ARTICLE_PAGE = """<html><head><title>Ignored</title>
|
|
<meta property="og:site_name" content="News Example">
|
|
<meta property="og:title" content="The FAA stops all departures">
|
|
<meta property="og:description" content="Flights across the US were held on the ground on Friday.">
|
|
<meta property="og:image" content="/img/lead.png">
|
|
<meta property="og:image:alt" content="A departures board">
|
|
<meta property="article:published_time" content="2026-11-13T22:00:00Z">
|
|
<meta name="author" content="Dee Reporter">
|
|
</head><body><p>Story</p></body></html>"""
|
|
|
|
THREAD = {"thread": {"post": {
|
|
"author": {"handle": "carol.example", "displayName": "Carol",
|
|
"avatar": "https://cdn.example/avatar/carol.png"},
|
|
"record": {"text": "Hello <world>\n\nSecond paragraph", "createdAt": "2026-11-12T08:00:00.000Z"},
|
|
"embed": {"$type": "app.bsky.embed.images#view",
|
|
"images": [{"thumb": "https://cdn.example/thumb/1.png", "alt": "A cat"}]},
|
|
}}}
|
|
|
|
|
|
def png(color="red", size=(800, 400)):
|
|
buffer = io.BytesIO()
|
|
Image.new("RGB", size, color).save(buffer, "PNG")
|
|
return buffer.getvalue()
|
|
|
|
|
|
@pytest.fixture
|
|
def web(monkeypatch):
|
|
"""The mocked internet; tests change `routes` and read `seen`"""
|
|
seen = []
|
|
routes = {
|
|
TOOT: lambda r: httpx.Response(200, html=TOOT_PAGE),
|
|
"https://masto.example/api/v1/statuses/117611186265622833": lambda r: httpx.Response(200, json=STATUS),
|
|
GTS_POST: lambda r: httpx.Response(200, html=GTS_PAGE),
|
|
ARTICLE: lambda r: httpx.Response(200, html=ARTICLE_PAGE),
|
|
}
|
|
|
|
def handler(request):
|
|
# The SSRF-safe fetcher connects to the checked IP; the name is in Host
|
|
url = f"{request.url.scheme}://{request.headers['host']}{request.url.raw_path.decode()}"
|
|
seen.append(url)
|
|
if url.startswith("https://public.api.bsky.app/"):
|
|
return httpx.Response(200, json=THREAD)
|
|
if url.endswith(".png"):
|
|
return httpx.Response(200, content=png(), headers={"Content-Type": "image/png"})
|
|
route = routes.get(url)
|
|
return route(request) if route else httpx.Response(404)
|
|
|
|
monkeypatch.setattr("starpunk.webmention_ssrf.socket.getaddrinfo", lambda *a, **k: PUBLIC)
|
|
monkeypatch.setattr("starpunk.webmention_ssrf.TRANSPORT", httpx.MockTransport(handler))
|
|
return type("Web", (), {"routes": routes, "seen": seen})
|
|
|
|
|
|
def fetch(app, url, **kwargs):
|
|
with app.app_context():
|
|
return fetch_reply_context(app.config["DATABASE_PATH"], url, app.config["SITE_URL"], **kwargs)
|
|
|
|
|
|
def test_mastodon_post_from_its_public_api(app, web):
|
|
context = fetch(app, TOOT)
|
|
|
|
assert context["source"] == "fediverse"
|
|
assert context["author_name"] == "Ada Lovelace"
|
|
assert context["author_handle"] == "@ada@masto.example"
|
|
assert context["author_url"] == "https://masto.example/@ada"
|
|
assert "ground</a> stopped" in context["content_snippet"]
|
|
assert "script" not in context["content_snippet"]
|
|
assert context["photo"] == "https://masto.example/media/small.png"
|
|
assert context["photo_alt"] == "An advisory"
|
|
assert context["published_at"].startswith("2026-11-13T21:11:22")
|
|
# Avatar and image are saved locally (the CSP allows only this site's images)
|
|
data = app.config["DATA_PATH"]
|
|
assert avatars.cached_name(data, context["author_photo"])
|
|
preview = avatars.preview_folder(data) / avatars.cached_preview(data, context["photo"])
|
|
with Image.open(preview) as image:
|
|
assert image.width == avatars.PREVIEW_WIDTH
|
|
|
|
# Stored with its source, so it isn't fetched again
|
|
web.seen.clear()
|
|
assert fetch(app, TOOT)["author_name"] == "Ada Lovelace"
|
|
assert web.seen == []
|
|
|
|
|
|
def test_content_warning_is_shown_instead_of_the_post(app, web):
|
|
web.routes["https://masto.example/api/v1/statuses/117611186265622833"] = lambda r: httpx.Response(
|
|
200, json={**STATUS, "spoiler_text": "flight news", "sensitive": True})
|
|
context = fetch(app, TOOT)
|
|
assert context["content_snippet"] == "<p>CW: flight news</p>"
|
|
assert context["photo"] is None
|
|
|
|
|
|
def test_fediverse_post_through_own_instance(app, web):
|
|
"""GoToSocial has no anonymous API; this site's instance resolves the post"""
|
|
asked = []
|
|
|
|
def gts(request):
|
|
asked.append(request.url.params.get("q"))
|
|
status = {**STATUS, "content": "<p>Posted on GoToSocial</p>", "media_attachments": [],
|
|
"account": {**STATUS["account"], "acct": "bob@gts.example", "display_name": "Bob",
|
|
"url": "https://gts.example/@bob"}}
|
|
return httpx.Response(200, json={"statuses": [status]})
|
|
|
|
own = "https://own.example"
|
|
app.config.update(GOTOSOCIAL_URL=own, GOTOSOCIAL_ACCESS_TOKEN="t")
|
|
syndication._client_cache[own] = MastodonClient(
|
|
own, "t", http=httpx.Client(transport=httpx.MockTransport(gts)))
|
|
try:
|
|
with app.test_request_context():
|
|
context = fetch_reply_context(app.config["DATABASE_PATH"], GTS_POST, app.config["SITE_URL"])
|
|
finally:
|
|
syndication._client_cache.clear()
|
|
|
|
assert asked == [GTS_POST]
|
|
assert context["source"] == "fediverse"
|
|
assert context["author_name"] == "Bob"
|
|
assert context["author_handle"] == "@bob@gts.example"
|
|
assert "Posted on GoToSocial" in context["content_snippet"]
|
|
|
|
|
|
def test_fediverse_post_without_an_api_uses_opengraph(app, web):
|
|
context = fetch(app, GTS_POST)
|
|
assert context["source"] == "opengraph"
|
|
assert context["author_name"] == "Bob"
|
|
assert context["author_handle"] == "@bob@gts.example"
|
|
assert context["author_url"] == "https://gts.example/@bob"
|
|
assert context["title"] is None
|
|
|
|
|
|
def test_bluesky_post_from_the_public_api(app, web):
|
|
context = fetch(app, SKEET)
|
|
assert context["source"] == "bluesky"
|
|
assert context["author_name"] == "Carol"
|
|
assert context["author_handle"] == "@carol.example"
|
|
assert context["author_url"] == "https://bsky.app/profile/carol.example"
|
|
assert context["content_snippet"] == "<p>Hello <world></p><p>Second paragraph</p>"
|
|
assert (context["photo"], context["photo_alt"]) == ("https://cdn.example/thumb/1.png", "A cat")
|
|
assert any("uri=at%3A%2F%2Fcarol.example%2Fapp.bsky.feed.post%2F3l6oveex3ii2l" in u for u in web.seen)
|
|
|
|
|
|
def test_other_pages_use_opengraph(app, web):
|
|
context = fetch(app, ARTICLE)
|
|
assert context["source"] == "opengraph"
|
|
assert context["title"] == "The FAA stops all departures"
|
|
assert context["author_name"] == "Dee Reporter"
|
|
assert context["site_name"] == "News Example"
|
|
assert "held on the ground" in context["content_snippet"]
|
|
assert context["photo"] == "https://news.example/img/lead.png"
|
|
assert context["photo_alt"] == "A departures board"
|
|
|
|
|
|
def _old_row(app, url, **values):
|
|
"""A context stored before v2.19.0: no source, the domain as author"""
|
|
conn = sqlite3.connect(app.config["DATABASE_PATH"])
|
|
conn.execute("DELETE FROM reply_contexts WHERE url = ?", (url,))
|
|
conn.execute("INSERT INTO reply_contexts (url, author_name, fetch_status) VALUES (?, ?, 'fetched')",
|
|
(url, values.get("author_name", "masto.example")))
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
|
|
def _reply(app, url, content="Agreed"):
|
|
from starpunk.notes import create_note
|
|
with app.test_request_context():
|
|
return create_note(content, published=True, in_reply_to=url)
|
|
|
|
|
|
def test_contexts_from_before_are_fetched_again_once(app, web):
|
|
_reply(app, TOOT)
|
|
_old_row(app, TOOT)
|
|
|
|
with app.app_context():
|
|
assert refresh_stale(app.config["DATABASE_PATH"], app.config["SITE_URL"]) == 1
|
|
assert refresh_stale(app.config["DATABASE_PATH"], app.config["SITE_URL"]) == 0
|
|
assert fetch(app, TOOT)["author_name"] == "Ada Lovelace"
|
|
|
|
|
|
def test_failed_refetch_keeps_what_was_there(app, web):
|
|
_reply(app, TOOT)
|
|
_old_row(app, TOOT, author_name="Ada (old)")
|
|
web.routes.clear() # everything 404s now
|
|
|
|
with app.app_context():
|
|
refresh_stale(app.config["DATABASE_PATH"], app.config["SITE_URL"])
|
|
assert refresh_stale(app.config["DATABASE_PATH"], app.config["SITE_URL"]) == 0
|
|
context = fetch(app, TOOT)
|
|
assert (context["fetch_status"], context["author_name"]) == ("fetched", "Ada (old)")
|
|
|
|
|
|
def test_reply_page_and_home_page_show_the_context(app, web):
|
|
note = _reply(app, TOOT, "This cannot be good")
|
|
client = app.test_client()
|
|
|
|
page = client.get(f"/note/{note.slug}").data.decode()
|
|
assert "Ada Lovelace" in page and "@ada@masto.example" in page
|
|
assert "ground</a> stopped" in page
|
|
assert "/previews/" in page and 'alt="An advisory"' in page
|
|
assert "/avatars/" in page
|
|
|
|
home = client.get("/").data.decode()
|
|
assert "↩ In reply to" in home
|
|
assert ">Ada Lovelace</a>" in home
|
|
assert "All aircraft are ground stopped." in home
|
|
|
|
# Still a reply to the toot (Post Type Discovery), on both pages, and the
|
|
# card's author and content stay inside the h-cite
|
|
for html in (page, home):
|
|
items = mf2py.parse(doc=html, url="http://localhost:5000/")["items"]
|
|
entries = [i for i in items if "h-entry" in i["type"]] or [
|
|
c for i in items for c in i.get("children", []) if "h-entry" in c["type"]]
|
|
(entry,) = [e for e in entries if "This cannot be good" in json.dumps(e)]
|
|
reply_to = entry["properties"]["in-reply-to"][0]
|
|
assert (reply_to if isinstance(reply_to, str) else reply_to["properties"]["url"][0]) == TOOT
|
|
assert "Ada Lovelace" not in json.dumps(entry["properties"].get("author", ""))
|