""" Backfeed of responses to crossposts (v2.13.0, ADR-073) StarPunk asks each silo for the responses to its own copies of a note (likes, reposts, quotes and replies) and stores them in `webmentions`, so moderation, the Responses section and re-sending webmentions when a note's responses change (spec 3.1.4) work as they do for received webmentions. Each synced copy of a published note is checked on a schedule that slows as the note ages, and not at all after BACKFEED_MAX_AGE_DAYS. A backfill checks every copy once, whatever its age. Checks run in the syndication worker's loop (`process_due`), one copy at a time under a lease. A check that read the whole list marks responses that are no longer there as invalid, so withdrawn likes and deleted replies drop off the page. """ import html import random import re from dataclasses import dataclass, field from datetime import datetime, timezone from typing import Dict, List, Optional from flask import current_app from starpunk.database import get_db from starpunk.utils import post_path DAY = 24 * 3600 SCHEDULE = [ # (note younger than, check every), in seconds (DAY, 15 * 60), (7 * DAY, 3600), (30 * DAY, DAY), ] LATE_INTERVAL = 7 * DAY # from 30 days until the cutoff CLAIM_MINUTES = 5 MAX_ATTEMPTS = 8 BACKFILL_GAP = 5 # seconds between backfill checks, across workers SILO_NAMES = {"gotosocial": "GoToSocial", "bluesky": "Bluesky", "threads": "Threads"} OPT_OUT_RE = re.compile(r"#no(bridge|bot)\b", re.IGNORECASE) # Silos whose credential was refused this run: target id -> message denied: Dict[str, str] = {} @dataclass class Response: """One like, repost, quote or reply on a silo""" kind: str # like, repost, quote, reply source_url: str # its URL (likes/reposts: the copy + #liked-by-) remote_id: str # silo id of the response, or of the actor author_id: str = "" author_name: str = "" author_url: str = "" author_photo: Optional[str] = None author_bio: str = "" opted_out: bool = False # set by the client from silo-specific signals content_html: Optional[str] = None published: Optional[str] = None # Replies in a thread (v2.20.2): the silo id of the response this one # answers (None: our copy itself), and whether we wrote it parent_id: Optional[str] = None own: bool = False def thread_members(posts: List[dict], root: str) -> Dict[str, Optional[str]]: """ Which replies under one of our copies to keep, and what each answers (v2.20.2). `posts` are {'id', 'parent', 'own'} for every reply in the conversation, in any order. Kept: others' replies to our copy, and any reply (ours too) to a kept reply. Our own replies to our copy are the thread continuing, not responses. Returns {id: parent id or None}. """ kept: Dict[str, Optional[str]] = {} pending = list(posts) while True: rest = [] for post in pending: if post["parent"] == root and not post["own"]: kept[post["id"]] = None elif post["parent"] in kept and post["parent"] != post["id"]: kept[post["id"]] = post["parent"] else: rest.append(post) if len(rest) == len(pending): return kept pending = rest @dataclass class Harvest: """What one check found. complete=False: some list was cut short.""" responses: List[Response] complete: bool = True # Likes/reposts/quotes shown only as numbers: all of them on Threads # (its API doesn't say who); elsewhere, those by people who opted out counts: Optional[Dict[str, int]] = None # --- Text ----------------------------------------------------------------- def text_html(text: str) -> str: """Plain text as HTML: escaped, URLs linked, line breaks kept""" escaped = html.escape(text) linked = re.sub(r"https?://[^\s<>&\"]+", lambda m: f'{m.group(0)}', escaped) return "

" + linked.replace("\n", "
") + "

" if text.strip() else "" def facets_html(text: str, facets: list) -> str: """Bluesky post text as HTML, with its link facets (byte ranges) as links""" data = text.encode() links = [] for facet in facets: index = facet.get("index") or {} for feature in facet.get("features") or []: uri = feature.get("uri") or "" if feature.get("$type") == "app.bsky.richtext.facet#link" and uri.startswith(("http://", "https://")): links.append((index.get("byteStart", 0), index.get("byteEnd", 0), uri)) links.sort() parts, pos = [], 0 for start, end, uri in links: if start < pos or end > len(data) or start >= end: continue parts.append(html.escape(data[pos:start].decode(errors="ignore"))) label = html.escape(data[start:end].decode(errors="ignore")) parts.append(f'{label}') pos = end parts.append(html.escape(data[pos:].decode(errors="ignore"))) body = "".join(parts) return "

" + body.replace("\n", "
") + "

" if text.strip() else "" # --- Scheduling -------------------------------------------------------------- def interval_for(age_seconds: float, max_age_days: int) -> Optional[int]: """Seconds until a note of this age is checked again; None past the cutoff""" if age_seconds >= max_age_days * DAY: return None for younger_than, every in SCHEDULE: if age_seconds < younger_than: return every return LATE_INTERVAL def _age_seconds(created_at) -> float: if isinstance(created_at, str): created_at = datetime.fromisoformat(created_at.replace("Z", "+00:00")) if created_at.tzinfo is None: created_at = created_at.replace(tzinfo=timezone.utc) return (datetime.now(timezone.utc) - created_at).total_seconds() _ELIGIBLE = """ FROM syndications s JOIN notes n ON n.id = s.note_id WHERE s.state = 'synced' AND s.enabled = 1 AND s.remote_id IS NOT NULL AND n.published = 1 AND n.deleted_at IS NULL AND n.like_of IS NULL AND n.repost_of IS NULL """ def _claim(db, row_id: int) -> bool: cursor = db.execute( f""" UPDATE syndications SET backfeed_claimed_until = datetime('now', '+{CLAIM_MINUTES} minutes') WHERE id = ? AND (backfeed_claimed_until IS NULL OR backfeed_claimed_until < datetime('now')) """, (row_id,), ) db.commit() return cursor.rowcount == 1 def _backfill_turn(db) -> bool: """Take the next backfill slot: at most one check every BACKFILL_GAP seconds""" cursor = db.execute( f""" UPDATE site_state SET value = datetime('now') WHERE key = 'backfeed_backfill_last' AND value <= datetime('now', '-{BACKFILL_GAP} seconds') """ ) if cursor.rowcount == 0: cursor = db.execute( "INSERT OR IGNORE INTO site_state (key, value) VALUES ('backfeed_backfill_last', datetime('now'))") db.commit() return cursor.rowcount == 1 def _due(db, limit: int) -> list: """Routine checks due now; a backfill check only when none are""" max_age = int(current_app.config.get("BACKFEED_MAX_AGE_DAYS", 90)) lease = "(s.backfeed_claimed_until IS NULL OR s.backfeed_claimed_until < datetime('now'))" rows = db.execute( f""" SELECT s.*, n.created_at AS note_created, n.slug AS note_slug, n.title AS note_title {_ELIGIBLE} AND {lease} AND n.created_at >= datetime('now', ?) AND (s.backfeed_next_at IS NULL OR s.backfeed_next_at <= datetime('now')) ORDER BY s.backfeed_next_at IS NOT NULL, s.backfeed_next_at LIMIT ? """, (f"-{max_age} days", limit), ).fetchall() if rows: return rows backfill = db.execute( f""" SELECT s.*, n.created_at AS note_created, n.slug AS note_slug, n.title AS note_title {_ELIGIBLE} AND {lease} AND s.backfeed_backfill = 1 AND (s.backfeed_next_at IS NULL OR s.backfeed_next_at <= datetime('now')) ORDER BY s.id LIMIT 1 """ ).fetchall() if backfill and _backfill_turn(db): return backfill return [] def _finish(db, row) -> None: max_age = int(current_app.config.get("BACKFEED_MAX_AGE_DAYS", 90)) every = interval_for(_age_seconds(row["note_created"]), max_age) if every is None: next_at = None else: next_at = f"+{int(every * random.uniform(1.0, 1.1))} seconds" db.execute( """ UPDATE syndications SET backfeed_checked_at = datetime('now'), backfeed_claimed_until = NULL, backfeed_attempts = 0, backfeed_error = NULL, backfeed_backfill = 0, backfeed_next_at = CASE WHEN ? IS NULL THEN NULL ELSE datetime('now', ?) END WHERE id = ? """, (next_at, next_at, row["id"]), ) db.commit() def _fail(db, row, message: str) -> None: """Back off; after MAX_ATTEMPTS drop any backfill and try again in a month""" attempts = row["backfeed_attempts"] + 1 stop = attempts >= MAX_ATTEMPTS delay = 30 * DAY if stop else min(30 * (2 ** attempts), 6 * 3600) db.execute( f""" UPDATE syndications SET backfeed_attempts = ?, backfeed_error = ?, backfeed_claimed_until = NULL, backfeed_backfill = CASE WHEN ? THEN 0 ELSE backfeed_backfill END, backfeed_next_at = datetime('now', '+{delay} seconds') WHERE id = ? """, (attempts, message[:500], 1 if stop else 0, row["id"]), ) db.commit() current_app.logger.warning( f"Backfeed {row['target']} for note {row['note_id']} failed (attempt {attempts}): {message}") # --- Storing ----------------------------------------------------------------- def _opted_out(response: Response) -> bool: return response.opted_out or bool(OPT_OUT_RE.search(html.unescape(response.author_bio or ""))) def _display_state(db, note_id: int) -> list: return [tuple(r) for r in db.execute( """ SELECT id, mention_type, author_name, author_url, author_photo, content_snippet FROM webmentions WHERE target_note_id = ? AND validation_status = 'valid' AND moderation_status = 'approved' ORDER BY id """, (note_id,), )] def hidden_counts(harvest: Harvest) -> Dict[str, int]: """ Likes, reposts and quotes to show only as numbers: the silo's own counts when it gives no names (Threads), else those by people who opted out (v2.13.3: they're counted, not named) """ if harvest.counts is not None: return harvest.counts hidden = [r.kind for r in harvest.responses if _opted_out(r)] return {k: hidden.count(kind) for k, kind in (("likes", "like"), ("reposts", "repost"), ("quotes", "quote"))} def store(db, note_id: int, target_url: str, origin: str, harvest: Harvest, quiet: bool = False) -> None: """ Upsert a check's responses for one note and silo. With a complete harvest, that silo's responses for the note that weren't found are marked invalid. Responses from people who opted out are removed (they are counted instead, see `hidden_counts`). With `quiet`, new responses are stored as already notified (ADR-074 Decision 5). """ from starpunk.webmention_processor import _snippet moderation = "approved" if current_app.config.get("WEBMENTION_AUTO_APPROVE", True) else "pending" silo = SILO_NAMES.get(origin, origin) keep = [r for r in harvest.responses if not _opted_out(r)] dropped = [r.source_url for r in harvest.responses if _opted_out(r)] from starpunk.avatars import cached_name before = _display_state(db, note_id) for r in keep: snippet = _snippet(r.content_html) if r.content_html else None # Threads avatar URLs are signed and change on every request: keep # the stored URL while its saved copy exists, so it isn't refetched old = db.execute("SELECT author_photo FROM webmentions WHERE source_url = ? AND target_url = ?", (r.source_url, target_url)).fetchone() if old and old["author_photo"] and origin == "threads" \ and cached_name(current_app.config["DATA_PATH"], old["author_photo"]): r.author_photo = old["author_photo"] db.execute( """ INSERT INTO webmentions ( source_url, target_url, target_note_id, mention_type, author_name, author_url, author_photo, content_snippet, validation_status, moderation_status, validated_at, source_published, origin, remote_id, notified_at, parent_remote_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'valid', ?, CURRENT_TIMESTAMP, ?, ?, ?, CASE WHEN ? THEN CURRENT_TIMESTAMP END, ?) ON CONFLICT(source_url, target_url) DO UPDATE SET target_note_id = excluded.target_note_id, mention_type = excluded.mention_type, author_name = excluded.author_name, author_url = excluded.author_url, author_photo = excluded.author_photo, content_snippet = excluded.content_snippet, source_published = COALESCE(excluded.source_published, webmentions.source_published), origin = excluded.origin, remote_id = excluded.remote_id, parent_remote_id = excluded.parent_remote_id, validation_status = 'valid', validation_error = NULL, validated_at = CURRENT_TIMESTAMP, updated_at = CURRENT_TIMESTAMP """, (r.source_url, target_url, note_id, r.kind, r.author_name or None, r.author_url or None, r.author_photo or None, snippet, moderation, r.published, origin, r.remote_id, # Our own replies in a thread are no news to us 1 if quiet or r.own else 0, r.parent_id), ) if dropped: marks = ",".join("?" * len(dropped)) db.execute(f"DELETE FROM webmentions WHERE origin = ? AND target_note_id = ? AND source_url IN ({marks})", [origin, note_id] + dropped) if harvest.complete: found = [r.source_url for r in keep] marks = ",".join("?" * len(found)) or "''" db.execute( f""" UPDATE webmentions SET validation_status = 'invalid', validation_error = ?, updated_at = CURRENT_TIMESTAMP WHERE target_note_id = ? AND origin = ? AND validation_status = 'valid' AND source_url NOT IN ({marks}) """, [f"No longer on {silo}", note_id, origin] + found, ) db.commit() if _display_state(db, note_id) != before: try: from pathlib import Path from starpunk.webmention_sender import note_responses_changed note_responses_changed(Path(current_app.config["DATABASE_PATH"]), [note_id]) except Exception as e: current_app.logger.warning(f"Failed to re-send webmentions for note {note_id}: {e}") from starpunk import avatars for r in keep: avatars.fetch(current_app.config["DATA_PATH"], r.author_photo, current_app.config.get("SITE_URL", "")) def _store_counts(db, row_id: int, counts: Optional[dict]) -> None: if counts is None: return db.execute( "UPDATE syndications SET likes_count = ?, reposts_count = ?, quotes_count = ? WHERE id = ?", (counts.get("likes"), counts.get("reposts"), counts.get("quotes"), row_id), ) db.commit() # --- Checking --------------------------------------------------------------- def _is_denied(error) -> bool: """The silo refused the credential (missing scope or bad token)""" from starpunk.syndication import _GRAPH_AUTH return getattr(error, "status", None) in (401, 403) or getattr(error, "code", None) in _GRAPH_AUTH SCOPES = { "gotosocial": "a GoToSocial token with read:statuses and read:accounts (or read)", "bluesky": "a working Bluesky app password", "threads": "a Threads token with threads_read_replies and threads_manage_insights", } def check(row, target) -> None: """Check one copy now and store what it found (raises SyndicationError)""" db = get_db(current_app) site = current_app.config["SITE_URL"].rstrip("/") harvest = target.client.responses(row["remote_id"], row["remote_url"] or "") # No notifications for history: a backfill, or a copy's first check on an # older note (e.g. one linked after the fact). ADR-074 Decision 5. quiet = bool(row["backfeed_backfill"]) or ( row["backfeed_checked_at"] is None and _age_seconds(row["note_created"]) > DAY) store(db, row["note_id"], site + post_path(row["note_slug"], row["note_title"]), row["target"], harvest, quiet=quiet) _store_counts(db, row["id"], hidden_counts(harvest)) def process_due(limit: int = 3) -> int: """Check up to `limit` copies that are due. Returns how many were handled.""" if not current_app.config.get("BACKFEED_ENABLED", False): return 0 from starpunk.syndication import SyndicationError, get_targets targets = get_targets() db = get_db(current_app) handled = 0 for row in _due(db, limit): if row["target"] in denied or row["target"] not in targets: continue if not _claim(db, row["id"]): continue handled += 1 try: check(row, targets[row["target"]]) except SyndicationError as e: if _is_denied(e): denied[row["target"]] = f"Needs {SCOPES.get(row['target'], 'read access')}: {e}" current_app.logger.error(f"Backfeed for {row['target']} stopped: {denied[row['target']]}") from starpunk import notifications site = current_app.config["SITE_URL"].rstrip("/") notifications.problem(current_app.config, current_app.config["DATABASE_PATH"], f"Backfeed for {SILO_NAMES.get(row['target'], row['target'])} stopped", denied[row["target"]], f"{site}/admin/webmentions") db.execute("UPDATE syndications SET backfeed_claimed_until = NULL WHERE id = ?", (row["id"],)) db.commit() continue _fail(db, row, str(e)) continue except Exception as e: # one bad copy mustn't stop the worker _fail(db, row, f"unexpected error: {e}") continue _finish(db, row) _note_backfill_done(db) return handled # --- Backfill and admin --------------------------------------------------------- def _state(db, key: str) -> Optional[str]: row = db.execute("SELECT value FROM site_state WHERE key = ?", (key,)).fetchone() return row["value"] if row and row["value"] else None def _set_state(db, key: str, value: str) -> None: db.execute("INSERT INTO site_state (key, value) VALUES (?, ?) " "ON CONFLICT(key) DO UPDATE SET value = excluded.value", (key, value)) def start_backfill() -> int: """Mark every eligible copy for one check, whatever its age. Returns how many.""" db = get_db(current_app) cursor = db.execute( f""" UPDATE syndications SET backfeed_backfill = 1, backfeed_attempts = 0, backfeed_next_at = datetime('now') WHERE id IN (SELECT s.id {_ELIGIBLE}) """ ) now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S") _set_state(db, "backfeed_backfill_started", now) _set_state(db, "backfeed_backfill_finished", "") db.commit() return cursor.rowcount def _note_backfill_done(db) -> None: if _state(db, "backfeed_backfill_started") and not _state(db, "backfeed_backfill_finished"): left = db.execute("SELECT COUNT(*) FROM syndications WHERE backfeed_backfill = 1").fetchone()[0] if left == 0: _set_state(db, "backfeed_backfill_finished", datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S")) db.commit() def check_now(syndication_id: int) -> None: """Check this copy's responses at the next worker pass""" db = get_db(current_app) db.execute("UPDATE syndications SET backfeed_next_at = datetime('now'), backfeed_attempts = 0 " "WHERE id = ?", (syndication_id,)) db.commit() def status() -> dict: """For the admin pages: backfill progress and silos whose credential was refused""" db = get_db(current_app) return { "enabled": bool(current_app.config.get("BACKFEED_ENABLED", False)), "started": _state(db, "backfeed_backfill_started"), "finished": _state(db, "backfeed_backfill_finished"), "remaining": db.execute( "SELECT COUNT(*) FROM syndications WHERE backfeed_backfill = 1").fetchone()[0], "denied": dict(denied), "errors": db.execute( "SELECT COUNT(*) FROM syndications WHERE backfeed_error IS NOT NULL").fetchone()[0], }