Mastodon turns GIFs into silent looping MP4s (gifv); only the first-frame preview was kept. The MP4 is now saved locally and played muted and looping with the still as poster. Animated GIFs from other sites are saved as animated WebP. Migration 025 adds reply_contexts.video and has fediverse contexts with an image fetched again once. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
327 lines
14 KiB
Python
327 lines
14 KiB
Python
"""
|
|
Reply contexts from fediverse, Bluesky and OpenGraph sources (v2.19.0)
|
|
|
|
Mastodon and GoToSocial pages have no h-entry, so a reply to one used to show
|
|
only the server's domain. The context now comes from the Mastodon API, this
|
|
site's GoToSocial instance, the Bluesky AppView API or OpenGraph tags, and the
|
|
note page shows it as a card; lists show the author and a short quote.
|
|
"""
|
|
|
|
import io
|
|
import json
|
|
import sqlite3
|
|
|
|
import httpx
|
|
import mf2py
|
|
import pytest
|
|
from PIL import Image
|
|
|
|
from starpunk import avatars, syndication
|
|
from starpunk.reply_context import fetch_reply_context, refresh_stale
|
|
from starpunk.syndication import MastodonClient
|
|
|
|
PUBLIC = [(2, 1, 6, "", ("93.184.216.34", 0))]
|
|
|
|
TOOT = "https://masto.example/@ada/117611186265622833"
|
|
GTS_POST = "https://gts.example/@bob/statuses/01M7BE51ABF9HP2G0CPH5DKKGG"
|
|
SKEET = "https://bsky.app/profile/carol.example/post/3l6oveex3ii2l"
|
|
ARTICLE = "https://news.example/2026/11/13/ground-stop"
|
|
|
|
# Mastodon's status page: an app shell with OpenGraph tags, no microformats
|
|
TOOT_PAGE = """<html><head>
|
|
<meta property="og:site_name" content="Masto Example">
|
|
<meta property="og:title" content="Ada (@ada@masto.example)">
|
|
<meta property="profile:username" content="ada@masto.example">
|
|
</head><body><div id="mastodon"></div></body></html>"""
|
|
|
|
STATUS = {
|
|
"id": "117611186265622833",
|
|
"created_at": "2026-11-13T21:11:22.624Z",
|
|
"spoiler_text": "",
|
|
"sensitive": False,
|
|
"content": '<p>All aircraft are <a href="https://x.example/" class="mention">ground</a> stopped.</p>'
|
|
'<script>alert(1)</script>',
|
|
"account": {"username": "ada", "acct": "ada", "display_name": "Ada Lovelace",
|
|
"url": "https://masto.example/@ada",
|
|
"avatar_static": "https://masto.example/avatars/ada.png"},
|
|
"media_attachments": [{"type": "image", "preview_url": "https://masto.example/media/small.png",
|
|
"description": "An advisory"}],
|
|
}
|
|
|
|
GTS_PAGE = """<html><head>
|
|
<meta property="og:title" content="Bob (@bob@gts.example)">
|
|
<meta property="og:profile:username" content="bob@gts.example">
|
|
</head><body><main><article class="status expanded">
|
|
<a class="p-author h-card" href="https://gts.example/@bob"><span class="p-name">Bob</span></a>
|
|
<div class="content e-content"><p>Posted on GoToSocial</p></div>
|
|
</article></main></body></html>"""
|
|
|
|
ARTICLE_PAGE = """<html><head><title>Ignored</title>
|
|
<meta property="og:site_name" content="News Example">
|
|
<meta property="og:title" content="The FAA stops all departures">
|
|
<meta property="og:description" content="Flights across the US were held on the ground on Friday.">
|
|
<meta property="og:image" content="/img/lead.png">
|
|
<meta property="og:image:alt" content="A departures board">
|
|
<meta property="article:published_time" content="2026-11-13T22:00:00Z">
|
|
<meta name="author" content="Dee Reporter">
|
|
</head><body><p>Story</p></body></html>"""
|
|
|
|
THREAD = {"thread": {"post": {
|
|
"author": {"handle": "carol.example", "displayName": "Carol",
|
|
"avatar": "https://cdn.example/avatar/carol.png"},
|
|
"record": {"text": "Hello <world>\n\nSecond paragraph", "createdAt": "2026-11-12T08:00:00.000Z"},
|
|
"embed": {"$type": "app.bsky.embed.images#view",
|
|
"images": [{"thumb": "https://cdn.example/thumb/1.png", "alt": "A cat"}]},
|
|
}}}
|
|
|
|
|
|
def png(color="red", size=(800, 400)):
|
|
buffer = io.BytesIO()
|
|
Image.new("RGB", size, color).save(buffer, "PNG")
|
|
return buffer.getvalue()
|
|
|
|
|
|
@pytest.fixture
|
|
def web(monkeypatch):
|
|
"""The mocked internet; tests change `routes` and read `seen`"""
|
|
seen = []
|
|
routes = {
|
|
TOOT: lambda r: httpx.Response(200, html=TOOT_PAGE),
|
|
"https://masto.example/api/v1/statuses/117611186265622833": lambda r: httpx.Response(200, json=STATUS),
|
|
GTS_POST: lambda r: httpx.Response(200, html=GTS_PAGE),
|
|
ARTICLE: lambda r: httpx.Response(200, html=ARTICLE_PAGE),
|
|
}
|
|
|
|
def handler(request):
|
|
# The SSRF-safe fetcher connects to the checked IP; the name is in Host
|
|
url = f"{request.url.scheme}://{request.headers['host']}{request.url.raw_path.decode()}"
|
|
seen.append(url)
|
|
if url.startswith("https://public.api.bsky.app/"):
|
|
return httpx.Response(200, json=THREAD)
|
|
if url.endswith(".png"):
|
|
return httpx.Response(200, content=png(), headers={"Content-Type": "image/png"})
|
|
route = routes.get(url)
|
|
return route(request) if route else httpx.Response(404)
|
|
|
|
monkeypatch.setattr("starpunk.webmention_ssrf.socket.getaddrinfo", lambda *a, **k: PUBLIC)
|
|
monkeypatch.setattr("starpunk.webmention_ssrf.TRANSPORT", httpx.MockTransport(handler))
|
|
return type("Web", (), {"routes": routes, "seen": seen})
|
|
|
|
|
|
def fetch(app, url, **kwargs):
|
|
with app.app_context():
|
|
return fetch_reply_context(app.config["DATABASE_PATH"], url, app.config["SITE_URL"], **kwargs)
|
|
|
|
|
|
def test_mastodon_post_from_its_public_api(app, web):
|
|
context = fetch(app, TOOT)
|
|
|
|
assert context["source"] == "fediverse"
|
|
assert context["author_name"] == "Ada Lovelace"
|
|
assert context["author_handle"] == "@ada@masto.example"
|
|
assert context["author_url"] == "https://masto.example/@ada"
|
|
assert "ground</a> stopped" in context["content_snippet"]
|
|
assert "script" not in context["content_snippet"]
|
|
assert context["photo"] == "https://masto.example/media/small.png"
|
|
assert context["photo_alt"] == "An advisory"
|
|
assert context["published_at"].startswith("2026-11-13T21:11:22")
|
|
# Avatar and image are saved locally (the CSP allows only this site's images)
|
|
data = app.config["DATA_PATH"]
|
|
assert avatars.cached_name(data, context["author_photo"])
|
|
preview = avatars.preview_folder(data) / avatars.cached_preview(data, context["photo"])
|
|
with Image.open(preview) as image:
|
|
assert image.width == avatars.PREVIEW_WIDTH
|
|
|
|
# Stored with its source, so it isn't fetched again
|
|
web.seen.clear()
|
|
assert fetch(app, TOOT)["author_name"] == "Ada Lovelace"
|
|
assert web.seen == []
|
|
|
|
|
|
def test_content_warning_is_shown_instead_of_the_post(app, web):
|
|
web.routes["https://masto.example/api/v1/statuses/117611186265622833"] = lambda r: httpx.Response(
|
|
200, json={**STATUS, "spoiler_text": "flight news", "sensitive": True})
|
|
context = fetch(app, TOOT)
|
|
assert context["content_snippet"] == "<p>CW: flight news</p>"
|
|
assert context["photo"] is None
|
|
|
|
|
|
def test_fediverse_post_through_own_instance(app, web):
|
|
"""GoToSocial has no anonymous API; this site's instance resolves the post"""
|
|
asked = []
|
|
|
|
def gts(request):
|
|
asked.append(request.url.params.get("q"))
|
|
status = {**STATUS, "content": "<p>Posted on GoToSocial</p>", "media_attachments": [],
|
|
"account": {**STATUS["account"], "acct": "bob@gts.example", "display_name": "Bob",
|
|
"url": "https://gts.example/@bob"}}
|
|
return httpx.Response(200, json={"statuses": [status]})
|
|
|
|
own = "https://own.example"
|
|
app.config.update(GOTOSOCIAL_URL=own, GOTOSOCIAL_ACCESS_TOKEN="t")
|
|
syndication._client_cache[own] = MastodonClient(
|
|
own, "t", http=httpx.Client(transport=httpx.MockTransport(gts)))
|
|
try:
|
|
with app.test_request_context():
|
|
context = fetch_reply_context(app.config["DATABASE_PATH"], GTS_POST, app.config["SITE_URL"])
|
|
finally:
|
|
syndication._client_cache.clear()
|
|
|
|
assert asked == [GTS_POST]
|
|
assert context["source"] == "fediverse"
|
|
assert context["author_name"] == "Bob"
|
|
assert context["author_handle"] == "@bob@gts.example"
|
|
assert "Posted on GoToSocial" in context["content_snippet"]
|
|
|
|
|
|
def test_fediverse_post_without_an_api_uses_opengraph(app, web):
|
|
context = fetch(app, GTS_POST)
|
|
assert context["source"] == "opengraph"
|
|
assert context["author_name"] == "Bob"
|
|
assert context["author_handle"] == "@bob@gts.example"
|
|
assert context["author_url"] == "https://gts.example/@bob"
|
|
assert context["title"] is None
|
|
|
|
|
|
def test_bluesky_post_from_the_public_api(app, web):
|
|
context = fetch(app, SKEET)
|
|
assert context["source"] == "bluesky"
|
|
assert context["author_name"] == "Carol"
|
|
assert context["author_handle"] == "@carol.example"
|
|
assert context["author_url"] == "https://bsky.app/profile/carol.example"
|
|
assert context["content_snippet"] == "<p>Hello <world></p><p>Second paragraph</p>"
|
|
assert (context["photo"], context["photo_alt"]) == ("https://cdn.example/thumb/1.png", "A cat")
|
|
assert any("uri=at%3A%2F%2Fcarol.example%2Fapp.bsky.feed.post%2F3l6oveex3ii2l" in u for u in web.seen)
|
|
|
|
|
|
def test_other_pages_use_opengraph(app, web):
|
|
context = fetch(app, ARTICLE)
|
|
assert context["source"] == "opengraph"
|
|
assert context["title"] == "The FAA stops all departures"
|
|
assert context["author_name"] == "Dee Reporter"
|
|
assert context["site_name"] == "News Example"
|
|
assert "held on the ground" in context["content_snippet"]
|
|
assert context["photo"] == "https://news.example/img/lead.png"
|
|
assert context["photo_alt"] == "A departures board"
|
|
|
|
|
|
def _old_row(app, url, **values):
|
|
"""A context stored before v2.19.0: no source, the domain as author"""
|
|
conn = sqlite3.connect(app.config["DATABASE_PATH"])
|
|
conn.execute("DELETE FROM reply_contexts WHERE url = ?", (url,))
|
|
conn.execute("INSERT INTO reply_contexts (url, author_name, fetch_status) VALUES (?, ?, 'fetched')",
|
|
(url, values.get("author_name", "masto.example")))
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
|
|
def _reply(app, url, content="Agreed"):
|
|
from starpunk.notes import create_note
|
|
with app.test_request_context():
|
|
return create_note(content, published=True, in_reply_to=url)
|
|
|
|
|
|
def test_contexts_from_before_are_fetched_again_once(app, web):
|
|
_reply(app, TOOT)
|
|
_old_row(app, TOOT)
|
|
|
|
with app.app_context():
|
|
assert refresh_stale(app.config["DATABASE_PATH"], app.config["SITE_URL"]) == 1
|
|
assert refresh_stale(app.config["DATABASE_PATH"], app.config["SITE_URL"]) == 0
|
|
assert fetch(app, TOOT)["author_name"] == "Ada Lovelace"
|
|
|
|
|
|
def test_failed_refetch_keeps_what_was_there(app, web):
|
|
_reply(app, TOOT)
|
|
_old_row(app, TOOT, author_name="Ada (old)")
|
|
web.routes.clear() # everything 404s now
|
|
|
|
with app.app_context():
|
|
refresh_stale(app.config["DATABASE_PATH"], app.config["SITE_URL"])
|
|
assert refresh_stale(app.config["DATABASE_PATH"], app.config["SITE_URL"]) == 0
|
|
context = fetch(app, TOOT)
|
|
assert (context["fetch_status"], context["author_name"]) == ("fetched", "Ada (old)")
|
|
|
|
|
|
def test_reply_page_and_home_page_show_the_context(app, web):
|
|
note = _reply(app, TOOT, "This cannot be good")
|
|
client = app.test_client()
|
|
|
|
page = client.get(f"/note/{note.slug}").data.decode()
|
|
assert "Ada Lovelace" in page and "@ada@masto.example" in page
|
|
assert "ground</a> stopped" in page
|
|
assert "/previews/" in page and 'alt="An advisory"' in page
|
|
assert "/avatars/" in page
|
|
|
|
home = client.get("/").data.decode()
|
|
assert "↩ In reply to" in home
|
|
assert ">Ada Lovelace</a>" in home
|
|
assert "All aircraft are ground stopped." in home
|
|
|
|
# Still a reply to the toot (Post Type Discovery), on both pages, and the
|
|
# card's author and content stay inside the h-cite
|
|
for html in (page, home):
|
|
items = mf2py.parse(doc=html, url="http://localhost:5000/")["items"]
|
|
entries = [i for i in items if "h-entry" in i["type"]] or [
|
|
c for i in items for c in i.get("children", []) if "h-entry" in c["type"]]
|
|
(entry,) = [e for e in entries if "This cannot be good" in json.dumps(e)]
|
|
reply_to = entry["properties"]["in-reply-to"][0]
|
|
assert (reply_to if isinstance(reply_to, str) else reply_to["properties"]["url"][0]) == TOOT
|
|
assert "Ada Lovelace" not in json.dumps(entry["properties"].get("author", ""))
|
|
|
|
|
|
# --- Moving pictures (v2.20.1) --------------------------------------------------
|
|
|
|
MP4 = b"\x00\x00\x00\x18ftypmp42" + b"\x00" * 64
|
|
|
|
|
|
def animated_gif():
|
|
buffer = io.BytesIO()
|
|
frames = [Image.new("RGB", (800, 400), color) for color in ("red", "green", "blue")]
|
|
frames[0].save(buffer, "GIF", save_all=True, append_images=frames[1:], duration=80, loop=0)
|
|
return buffer.getvalue()
|
|
|
|
|
|
def test_mastodon_gifv_is_kept_as_a_looping_video(app, web, monkeypatch):
|
|
status = {**STATUS, "media_attachments": [{
|
|
"type": "gifv", "url": "https://masto.example/media/original.mp4",
|
|
"preview_url": "https://masto.example/media/small.png", "description": "Engineering report"}]}
|
|
web.routes["https://masto.example/api/v1/statuses/117611186265622833"] = \
|
|
lambda r: httpx.Response(200, json=status)
|
|
web.routes["https://masto.example/media/original.mp4"] = \
|
|
lambda r: httpx.Response(200, content=MP4, headers={"Content-Type": "video/mp4"})
|
|
|
|
note = _reply(app, TOOT, "Same here")
|
|
context = fetch(app, TOOT)
|
|
assert context["video"] == "https://masto.example/media/original.mp4"
|
|
assert avatars.cached_video(app.config["DATA_PATH"], context["video"])
|
|
|
|
client = app.test_client()
|
|
page = client.get(f"/note/{note.slug}").data.decode()
|
|
assert "<video" in page and "autoplay loop muted playsinline" in page
|
|
assert 'aria-label="Engineering report"' in page
|
|
src = page.split('<video class="reply-photo" src="', 1)[1].split('"', 1)[0]
|
|
response = client.get(src)
|
|
assert response.status_code == 200 and response.mimetype == "video/mp4"
|
|
|
|
|
|
def test_animated_gif_stays_animated(app, web):
|
|
web.routes[ARTICLE] = lambda r: httpx.Response(200, html=ARTICLE_PAGE.replace("/img/lead.png", "/img/lead.gif"))
|
|
web.routes["https://news.example/img/lead.gif"] = \
|
|
lambda r: httpx.Response(200, content=animated_gif(), headers={"Content-Type": "image/gif"})
|
|
context = fetch(app, ARTICLE)
|
|
data = app.config["DATA_PATH"]
|
|
with Image.open(avatars.preview_folder(data) / avatars.cached_preview(data, context["photo"])) as image:
|
|
assert image.n_frames == 3
|
|
assert image.width == avatars.PREVIEW_WIDTH
|
|
|
|
|
|
def test_unused_videos_are_removed(app, web):
|
|
data = app.config["DATA_PATH"]
|
|
folder = avatars.preview_folder(data)
|
|
folder.mkdir(parents=True, exist_ok=True)
|
|
stray = folder / avatars.video_name_for("https://gone.example/x.mp4")
|
|
stray.write_bytes(MP4)
|
|
avatars.remove_unused(app.config["DATABASE_PATH"], data)
|
|
assert not stray.exists()
|