Files
StarPunk/tests/test_backfeed.py
T
PhilandClaude Opus 5.5 55ce6a1fdd feat: backfeed of responses to crossposts (v2.13.0)
Likes, reposts, quotes and replies to a note's GoToSocial, Bluesky and
Threads copies are read back from the silos and stored as webmentions
(ADR-073): age-based schedule to 90 days, pruning of withdrawn responses,
#nobridge/#nobot opt-outs, admin backfill and Check now, Threads counts,
a Quotes group, local avatar copies, and delete tombstones.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-10 12:19:57 -07:00

543 lines
24 KiB
Python

"""
Tests for backfeed of responses to crossposts (v2.13.0, ADR-073)
Silo clients are tested against fake APIs (httpx.MockTransport). The
worker, storage and pages are tested with a fake client that returns
whatever a test puts in it.
"""
import io
import json
from urllib.parse import parse_qs
import httpx
import pytest
from PIL import Image
from starpunk import avatars, backfeed, create_app, syndication
from starpunk.backfeed import Harvest, Response, facets_html, interval_for, text_html
from starpunk.database import get_db
from starpunk.notes import create_note
from starpunk.syndication import BlueskyClient, MastodonClient, SyndicationError, ThreadsClient
SITE = "https://blog.example.com"
GTS = "https://social.example"
COPY = f"{GTS}/@phil/statuses/01COPY"
# --- Fakes --------------------------------------------------------------------
class FakeClient:
"""Stands in for a silo client: returns the harvest a test sets"""
config_token = "token"
def __init__(self):
self.harvest = Harvest([])
self.error = None
self.calls = []
def responses(self, remote_id, remote_url, cap=1000):
self.calls.append(remote_id)
if self.error:
raise self.error
return self.harvest
def maintain(self):
pass
def like(n, bio="", **kw):
return Response(kind="like", source_url=f"{COPY}#liked-by-{n}", remote_id=str(n),
author_id=str(n), author_name=f"Liker {n}", author_url=f"{GTS}/@l{n}",
author_bio=bio, **kw)
def reply(n, text="Nice!"):
return Response(kind="reply", source_url=f"{GTS}/@r{n}/statuses/{n}", remote_id=str(n),
author_id=f"r{n}", author_name=f"Replier {n}", author_url=f"{GTS}/@r{n}",
content_html=f"<p>{text}</p>", published="2026-11-10T12:00:00Z")
def resolve(host, *args, **kwargs):
return [(2, 1, 6, "", ("93.184.216.34", 0))]
def png(color="red", size=(200, 120)):
buf = io.BytesIO()
Image.new("RGB", size, color).save(buf, "PNG")
return buf.getvalue()
class FakeWeb:
"""Avatar images by URL (requests are pinned to an IP; Host names the site)"""
def __init__(self):
self.files = {}
def __call__(self, request):
url = f"{request.url.scheme}://{request.headers['host']}{request.url.raw_path.decode()}"
if url in self.files:
return httpx.Response(200, headers={"content-type": "image/png"}, content=self.files[url])
return httpx.Response(404)
@pytest.fixture
def web(monkeypatch):
fake = FakeWeb()
monkeypatch.setattr("starpunk.webmention_ssrf.TRANSPORT", httpx.MockTransport(fake))
monkeypatch.setattr("starpunk.webmention_ssrf.socket.getaddrinfo", resolve)
return fake
@pytest.fixture
def client():
return FakeClient()
@pytest.fixture
def app(tmp_path, client, web):
data = tmp_path / "data"
app = create_app({
"TESTING": True,
"SITE_URL": SITE,
"ADMIN_ME": "https://example.com",
"SESSION_SECRET": "test-secret",
"DATA_PATH": data,
"NOTES_PATH": data / "notes",
"DATABASE_PATH": data / "starpunk.db",
"GOTOSOCIAL_URL": GTS,
"GOTOSOCIAL_ACCESS_TOKEN": "token",
"BACKFEED_ENABLED": True,
})
syndication._client_cache[GTS] = client
backfeed.denied.clear()
yield app
syndication._client_cache.clear()
backfeed.denied.clear()
def crossposted(app, age_days=0, content="Hello"):
"""A published note with a synced GoToSocial copy; returns (note, syndication id)"""
with app.app_context():
note = create_note(content=content, published=True)
db = get_db(app)
if age_days:
db.execute("UPDATE notes SET created_at = datetime('now', ?) WHERE id = ?",
(f"-{age_days} days", note.id))
cursor = db.execute(
"INSERT INTO syndications (note_id, target, enabled, remote_id, remote_url, state) "
"VALUES (?, 'gotosocial', 1, '01COPY', ?, 'synced')", (note.id, COPY))
db.commit()
return note, cursor.lastrowid
def rows(app, note_id):
with app.app_context():
return [dict(r) for r in get_db(app).execute(
"SELECT * FROM webmentions WHERE target_note_id = ? ORDER BY source_url", (note_id,))]
def run(app):
with app.app_context():
return backfeed.process_due()
# --- Schedule and text ------------------------------------------------------------
def test_schedule_slows_with_age_and_stops_at_cutoff():
day = 24 * 3600
assert interval_for(3600, 90) == 15 * 60
assert interval_for(2 * day, 90) == 3600
assert interval_for(10 * day, 90) == day
assert interval_for(60 * day, 90) == 7 * day
assert interval_for(91 * day, 90) is None
def test_text_and_facets_become_safe_html():
assert text_html("hi <b>\nsee https://a.example/x") == \
'<p>hi &lt;b&gt;<br>see <a href="https://a.example/x">https://a.example/x</a></p>'
text = "café: example.com/long…"
start = len("café: ".encode())
facets = [{"index": {"byteStart": start, "byteEnd": len(text.encode())},
"features": [{"$type": "app.bsky.richtext.facet#link", "uri": "https://example.com/long/path"}]}]
assert facets_html(text, facets) == \
'<p>café: <a href="https://example.com/long/path">example.com/long…</a></p>'
assert facets_html("", []) == ""
# --- Worker and storage -------------------------------------------------------------
def test_check_stores_responses_as_webmentions(app, client):
note, sid = crossposted(app)
client.harvest = Harvest([like(1), reply(2, "Great <script>x</script>post")])
assert run(app) == 1
stored = rows(app, note.id)
assert [r["mention_type"] for r in stored] == ["like", "reply"]
assert all(r["origin"] == "gotosocial" and r["validation_status"] == "valid"
and r["moderation_status"] == "approved" for r in stored)
assert stored[1]["content_snippet"] == "<p>Great post</p>"
assert stored[0]["target_url"] == f"{SITE}/note/{note.slug}"
with app.app_context():
s = get_db(app).execute("SELECT * FROM syndications WHERE id = ?", (sid,)).fetchone()
assert s["backfeed_checked_at"] and s["backfeed_next_at"] and s["backfeed_error"] is None
assert run(app) == 0, "not due again for 15 minutes"
def test_repeat_check_updates_rows_and_keeps_moderation(app, client):
note, sid = crossposted(app)
client.harvest = Harvest([like(1), like(2)])
run(app)
with app.app_context():
db = get_db(app)
db.execute("UPDATE webmentions SET moderation_status = 'hidden' WHERE source_url LIKE '%-1'")
db.execute("UPDATE syndications SET backfeed_next_at = datetime('now')")
db.commit()
client.harvest = Harvest([like(1), like(2)])
run(app)
stored = rows(app, note.id)
assert len(stored) == 2
assert [r["moderation_status"] for r in stored] == ["hidden", "approved"]
def test_withdrawn_responses_drop_off_and_come_back(app, client):
note, _ = crossposted(app)
client.harvest = Harvest([like(1), reply(2)])
run(app)
def recheck(harvest):
with app.app_context():
get_db(app).execute("UPDATE syndications SET backfeed_next_at = datetime('now')")
get_db(app).commit()
client.harvest = harvest
run(app)
return {r["source_url"]: (r["validation_status"], r["validation_error"]) for r in rows(app, note.id)}
# An incomplete list (cap reached, say) removes nothing
assert all(v == ("valid", None) for v in recheck(Harvest([like(1)], complete=False)).values())
state = recheck(Harvest([like(1)]))
assert state[reply(2).source_url] == ("invalid", "No longer on GoToSocial")
assert recheck(Harvest([like(1), reply(2)]))[reply(2).source_url] == ("valid", None)
def test_people_who_opt_out_are_skipped_and_removed(app, client):
note, _ = crossposted(app)
client.harvest = Harvest([like(1), like(2)])
run(app)
with app.app_context():
get_db(app).execute("UPDATE syndications SET backfeed_next_at = datetime('now')")
get_db(app).commit()
client.harvest = Harvest([like(1, bio="<p>Please #NoBridge thanks</p>"), like(2),
like(3, opted_out=True)])
run(app)
assert [r["source_url"] for r in rows(app, note.id)] == [like(2).source_url]
def test_admin_delete_is_a_tombstone_the_next_check_keeps(app, client):
from starpunk.webmention_queries import count_webmentions, delete_webmentions, list_webmentions
note, _ = crossposted(app)
client.harvest = Harvest([like(1)])
run(app)
with app.test_request_context():
wm_id = rows(app, note.id)[0]["id"]
assert delete_webmentions([wm_id]) == 1
get_db(app).execute("UPDATE syndications SET backfeed_next_at = datetime('now')")
get_db(app).commit()
run(app)
assert rows(app, note.id)[0]["moderation_status"] == "deleted"
with app.test_request_context():
assert list_webmentions()[0] == [] and count_webmentions() == 0
assert len(list_webmentions(status_filter="deleted")[0]) == 1
def test_old_notes_wait_for_a_backfill(app, client):
old, sid = crossposted(app, age_days=200)
client.harvest = Harvest([reply(1)])
assert run(app) == 0
with app.app_context():
assert backfeed.start_backfill() == 1
assert backfeed.status()["remaining"] == 1
assert run(app) == 1
assert len(rows(app, old.id)) == 1
with app.app_context():
status = backfeed.status()
s = get_db(app).execute("SELECT * FROM syndications WHERE id = ?", (sid,)).fetchone()
assert status["remaining"] == 0 and status["finished"]
assert s["backfeed_backfill"] == 0 and s["backfeed_next_at"] is None, "past the cutoff: no more checks"
def test_backfill_is_paced(app, client):
crossposted(app, age_days=200)
crossposted(app, age_days=300)
with app.app_context():
backfeed.start_backfill()
assert run(app) == 1
assert run(app) == 0, "the next backfill check waits a few seconds"
with app.app_context():
get_db(app).execute("UPDATE site_state SET value = datetime('now', '-10 seconds') "
"WHERE key = 'backfeed_backfill_last'")
get_db(app).commit()
assert run(app) == 1
def test_routine_checks_skip_likes_reposts_and_unpublished(app, client):
with app.app_context():
liked = create_note(content="", published=True, like_of="https://other.example/post")
draft = create_note(content="Draft", published=False)
db = get_db(app)
for note in (liked, draft):
db.execute("INSERT INTO syndications (note_id, target, enabled, remote_id, remote_url, state) "
"VALUES (?, 'gotosocial', 1, 'x', 'https://s.example/x', 'synced')", (note.id,))
db.commit()
assert run(app) == 0 and client.calls == []
def test_refused_credential_stops_that_silo(app, client):
crossposted(app)
error = SyndicationError("GET /api/v1/statuses/01COPY/favourited_by: HTTP 403 forbidden", permanent=True)
error.status = 403
client.error = error
run(app)
assert "gotosocial" in backfeed.denied and "read:statuses" in backfeed.denied["gotosocial"]
client.error = None
assert run(app) == 0, "not retried until restart"
def test_failures_back_off(app, client):
_, sid = crossposted(app)
client.error = SyndicationError("HTTP 502 bad gateway")
run(app)
with app.app_context():
s = get_db(app).execute("SELECT * FROM syndications WHERE id = ?", (sid,)).fetchone()
assert s["backfeed_attempts"] == 1 and "502" in s["backfeed_error"]
assert run(app) == 0
def test_disabled_does_nothing(app, client):
crossposted(app)
app.config["BACKFEED_ENABLED"] = False
assert run(app) == 0 and client.calls == []
# --- Pages ------------------------------------------------------------------------------
def test_note_page_shows_backfed_responses_quotes_and_counts(app, client, web):
note, sid = crossposted(app)
web.files["https://cdn.example/a.png"] = png()
quote = Response(kind="quote", source_url="https://bsky.app/profile/q.example/post/3k",
remote_id="at://did:plc:q/app.bsky.feed.post/3k", author_name="Quoter",
author_url="https://bsky.app/profile/q.example", content_html="<p>Look at this</p>")
client.harvest = Harvest([like(1, author_photo="https://cdn.example/a.png"), reply(2), quote])
run(app)
with app.app_context():
get_db(app).execute("UPDATE syndications SET likes_count = 12, reposts_count = 1 WHERE id = ?", (sid,))
get_db(app).commit()
page = app.test_client().get(f"/note/{note.slug}").get_data(as_text=True)
assert "via GoToSocial" in page
assert "1 Quote" in page and "Look at this" in page
assert 'class="silo-counts">On GoToSocial: 12 likes · 1 repost<' in page
assert 'src="/avatars/' in page and 'src="https://cdn.example/a.png"' not in page
def test_avatars_are_saved_served_and_cleaned_up(app, web):
url = "https://cdn.example/face.png"
web.files[url] = png(size=(300, 200))
data = app.config["DATA_PATH"]
assert avatars.cached_name(data, url) is None
with app.app_context():
assert avatars.fetch(data, url, SITE)
assert not avatars.fetch(data, "https://cdn.example/missing.png", SITE)
name = avatars.cached_name(data, url)
with Image.open(avatars.folder(data) / name) as image:
assert image.size == (96, 96) and image.format == "WEBP"
response = app.test_client().get(f"/avatars/{name}")
assert response.status_code == 200 and response.mimetype == "image/webp"
assert app.test_client().get("/avatars/../starpunk.db").status_code == 404
# Nothing uses it: the weekly cleanup removes it
assert avatars.remove_unused(app.config["DATABASE_PATH"], data) == 1
def test_webmention_cannot_overwrite_a_backfed_response(app, client):
note, _ = crossposted(app)
client.harvest = Harvest([reply(7)])
run(app)
response = app.test_client().post("/webmention", data={
"source": reply(7).source_url, "target": f"{SITE}/note/{note.slug}"})
assert response.status_code == 202
assert rows(app, note.id)[0]["validation_status"] == "valid"
# --- Silo clients ---------------------------------------------------------------------------
def mastodon_api(request):
path, query = request.url.path, parse_qs(request.url.query.decode())
account = lambda n, **kw: {"id": n, "display_name": f"User {n}", "url": f"{GTS}/@u{n}",
"avatar": f"{GTS}/a/{n}.png", "note": "", **kw}
if path == "/api/v1/accounts/verify_credentials":
return httpx.Response(200, json={"id": "me"})
if path == "/api/v1/statuses/01COPY/favourited_by":
if "max_id" in query:
return httpx.Response(200, json=[account("3")])
return httpx.Response(200, json=[account("1"), account("me")],
headers={"Link": f'<{GTS}/api/v1/statuses/01COPY/favourited_by?max_id=9>; rel="next"'})
if path == "/api/v1/statuses/01COPY/reblogged_by":
return httpx.Response(200, json=[account("2")])
if path == "/api/v1/statuses/01COPY/context":
status = lambda n, parent="01COPY", vis="public", who=None: {
"id": n, "url": f"{GTS}/@u/statuses/{n}", "uri": f"{GTS}/u/{n}", "in_reply_to_id": parent,
"visibility": vis, "content": f"<p>reply {n}</p>", "created_at": "2026-11-10T00:00:00Z",
"account": who or account("r" + n)}
return httpx.Response(200, json={"ancestors": [], "descendants": [
status("10"), status("11", vis="unlisted"), status("12", parent="10"),
status("13", who=account("me"))]})
return httpx.Response(404)
def test_mastodon_client_reads_likes_boosts_and_public_direct_replies():
gts = MastodonClient(GTS, "t", http=httpx.Client(transport=httpx.MockTransport(mastodon_api)))
harvest = gts.responses("01COPY", COPY)
assert harvest.complete
got = [(r.kind, r.source_url) for r in harvest.responses]
assert got == [("like", f"{COPY}#liked-by-1"), ("like", f"{COPY}#liked-by-3"),
("repost", f"{COPY}#reposted-by-2"), ("reply", f"{GTS}/@u/statuses/10")]
assert harvest.responses[-1].content_html == "<p>reply 10</p>"
assert gts.responses("01COPY", COPY, cap=1).complete is False
def test_bluesky_client_reads_likes_reposts_quotes_and_replies():
me, post = "did:plc:me", "at://did:plc:me/app.bsky.feed.post/abc"
def api(request):
nsid = request.url.path.removeprefix("/xrpc/")
query = parse_qs(request.url.query.decode())
profile = lambda d, **kw: {"did": d, "handle": f"{d[8:]}.example", "displayName": d[8:].title(), **kw}
view = lambda d, rkey, text: {"uri": f"at://{d}/app.bsky.feed.post/{rkey}", "author": profile(d),
"record": {"text": text, "createdAt": "2026-11-10T00:00:00Z"}}
if nsid == "com.atproto.server.createSession":
return httpx.Response(200, json={"did": me, "handle": "me.example", "accessJwt": "a", "refreshJwt": "r"})
if nsid == "app.bsky.feed.getLikes":
if query.get("cursor"):
return httpx.Response(200, json={"likes": [{"actor": profile("did:plc:carol")}]})
return httpx.Response(200, json={"cursor": "c1", "likes": [
{"actor": profile("did:plc:alice"), "createdAt": "2026-11-10T01:00:00Z"},
{"actor": profile("did:plc:shy", labels=[{"val": "!no-unauthenticated"}])},
{"actor": profile(me)}]})
if nsid == "app.bsky.feed.getRepostedBy":
return httpx.Response(200, json={"repostedBy": [profile("did:plc:bob")]})
if nsid == "app.bsky.feed.getQuotes":
return httpx.Response(200, json={"posts": [view("did:plc:dan", "q1", "Quoting this")]})
if nsid == "app.bsky.feed.getPostThread":
return httpx.Response(200, json={"thread": {"post": {}, "replies": [
{"post": view("did:plc:eve", "r1", "A reply")},
{"$type": "app.bsky.feed.defs#blockedPost", "uri": "at://x"}]}})
if nsid == "app.bsky.actor.getProfiles":
return httpx.Response(200, json={"profiles": [
profile("did:plc:dan", description="no thanks #nobot"), profile("did:plc:eve")]})
return httpx.Response(400, json={"error": "Unknown"})
bsky = BlueskyClient("https://bsky.example", "me.example", "pw",
http=httpx.Client(transport=httpx.MockTransport(api)))
harvest = bsky.responses(post, "https://bsky.app/profile/me.example/post/abc")
by = {(r.kind, r.author_id): r for r in harvest.responses}
assert set(by) == {("like", "did:plc:alice"), ("like", "did:plc:shy"), ("like", "did:plc:carol"),
("repost", "did:plc:bob"), ("quote", "did:plc:dan"), ("reply", "did:plc:eve")}
assert by[("like", "did:plc:shy")].opted_out
assert by[("quote", "did:plc:dan")].author_bio == "no thanks #nobot"
assert by[("reply", "did:plc:eve")].source_url == "https://bsky.app/profile/eve.example/post/r1"
assert by[("like", "did:plc:alice")].source_url.endswith("#liked-by-did:plc:alice")
assert harvest.complete
def test_threads_client_reads_replies_and_counts():
def api(request):
path = request.url.path
params = parse_qs(request.url.query.decode())
if path.endswith("/me"):
return httpx.Response(200, json={"id": "42", "username": "me"})
if path.endswith("/111/replies"):
assert "profile_picture_url" in params["fields"][0]
return httpx.Response(200, json={"data": [
{"id": "1", "text": "Hi", "username": "ann", "permalink": "https://www.threads.com/@ann/post/A",
"timestamp": "2026-11-10T00:00:00+0000", "hide_status": "NOT_HUSHED"},
{"id": "2", "text": "private"},
{"id": "3", "text": "hidden", "username": "x", "permalink": "p", "hide_status": "HIDDEN"},
{"id": "4", "text": "mine", "username": "me", "permalink": "p"}]})
if path.endswith("/111/insights"):
assert params["metric"] == ["likes,reposts,quotes"]
return httpx.Response(200, json={"data": [
{"name": "likes", "values": [{"value": 12}]}, {"name": "reposts", "values": [{"value": 3}]},
{"name": "quotes", "values": [{"value": 0}]}]})
return httpx.Response(404, json={"error": {"code": 100}})
threads = ThreadsClient("tok", http=httpx.Client(transport=httpx.MockTransport(api)))
harvest = threads.responses("111", "https://www.threads.com/@me/post/X")
assert [(r.kind, r.author_name, r.content_html) for r in harvest.responses] == [("reply", "ann", "<p>Hi</p>")]
assert harvest.counts == {"likes": 12, "reposts": 3, "quotes": 0}
# --- Migration ---------------------------------------------------------------------------
def test_migration_020_keeps_existing_webmentions(tmp_path):
import sqlite3
from pathlib import Path
from starpunk.database.schema import INITIAL_SCHEMA_SQL
from starpunk.migrations import execute_in_transaction
migrations = Path(__file__).parent.parent / "migrations"
conn = sqlite3.connect(tmp_path / "old.db")
conn.execute("PRAGMA foreign_keys=ON")
conn.executescript(INITIAL_SCHEMA_SQL)
for name in ("010_add_webmentions.sql", "012_add_syndications.sql", "014_webmention_source_details.sql"):
matches = list(migrations.glob(name.split("_")[0] + "_*.sql"))
conn.execute("BEGIN")
execute_in_transaction(conn, matches[0].read_text())
conn.commit()
conn.execute("INSERT INTO notes (slug, file_path, published) VALUES ('n', 'n.md', 1)")
conn.execute("INSERT INTO webmentions (source_url, target_url, target_note_id, mention_type, "
"validation_status, moderation_status, source_published) "
"VALUES ('https://a.example/r', 'https://b.example/note/n', 1, 'reply', 'valid', 'hidden', "
"'2026-01-01')")
conn.commit()
conn.execute("BEGIN")
execute_in_transaction(conn, (migrations / "020_backfeed.sql").read_text())
conn.commit()
row = conn.execute("SELECT mention_type, moderation_status, source_published, origin, target_note_id "
"FROM webmentions").fetchone()
assert row == ("reply", "hidden", "2026-01-01", "webmention", 1)
conn.execute("INSERT INTO webmentions (source_url, target_url, mention_type, origin) "
"VALUES ('https://q.example', 'https://b.example/note/n', 'quote', 'bluesky')")
columns = {r[1] for r in conn.execute("PRAGMA table_info(syndications)")}
assert {"backfeed_next_at", "backfeed_backfill", "likes_count"} <= columns
assert conn.execute("PRAGMA foreign_key_check").fetchall() == []
# --- Admin ----------------------------------------------------------------------------------
def test_admin_backfill_and_check_now(app, client):
from starpunk.auth import create_session
app.config.update(ADMIN_ME="https://example.com", CSRF_ENABLED=False)
note, sid = crossposted(app, age_days=200)
with app.test_request_context():
token = create_session("https://example.com")
browser = app.test_client()
browser.set_cookie("starpunk_session", token)
page = browser.get("/admin/webmentions").get_data(as_text=True)
assert "No backfill has been run" in page and "Backfill responses" in page
response = browser.post("/admin/backfeed/backfill", follow_redirects=True)
assert "Backfill started: 1 crosspost(s) to check" in response.get_data(as_text=True)
assert "1 crosspost(s) left to check" in browser.get("/admin/webmentions").get_data(as_text=True)
edit = browser.get(f"/admin/edit/{note.id}").get_data(as_text=True)
assert "Responses not checked yet" in edit and "Check now" in edit
browser.post(f"/admin/syndication/{sid}/check")
with app.app_context():
s = get_db(app).execute("SELECT backfeed_next_at FROM syndications WHERE id = ?", (sid,)).fetchone()
assert s["backfeed_next_at"]
backfeed.denied["gotosocial"] = "Needs a GoToSocial token with read:statuses"
assert "Needs a GoToSocial token" in browser.get("/admin/webmentions?origin=gotosocial").get_data(as_text=True)