Files
PhilandClaude Opus 5.5 bf6e0fdc0b fix: backfeed whole reply threads and nest them (v2.20.2)
Backfeed only read first-level replies (Threads /replies, GoToSocial
direct replies, Bluesky depth 1) and skipped the owner's own replies, so
conversations under crossposts lost everything after the first reply.

- Threads /conversation, all GoToSocial descendants, Bluesky depth 10
- backfeed.thread_members keeps others' replies to our copy and any reply
  (ours included) to a kept reply; own replies are stored as notified
- Migration 026: webmentions.parent_remote_id; replies render nested as
  comments of comments

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-13 20:43:30 -07:00

517 lines
21 KiB
Python

"""
Backfeed of responses to crossposts (v2.13.0, ADR-073)
StarPunk asks each silo for the responses to its own copies of a note
(likes, reposts, quotes and replies) and stores them in `webmentions`, so
moderation, the Responses section and re-sending webmentions when a note's
responses change (spec 3.1.4) work as they do for received webmentions.
Each synced copy of a published note is checked on a schedule that slows as
the note ages, and not at all after BACKFEED_MAX_AGE_DAYS. A backfill
checks every copy once, whatever its age. Checks run in the syndication
worker's loop (`process_due`), one copy at a time under a lease.
A check that read the whole list marks responses that are no longer there
as invalid, so withdrawn likes and deleted replies drop off the page.
"""
import html
import random
import re
from dataclasses import dataclass, field
from datetime import datetime, timezone
from typing import Dict, List, Optional
from flask import current_app
from starpunk.database import get_db
from starpunk.utils import post_path
DAY = 24 * 3600
SCHEDULE = [ # (note younger than, check every), in seconds
(DAY, 15 * 60),
(7 * DAY, 3600),
(30 * DAY, DAY),
]
LATE_INTERVAL = 7 * DAY # from 30 days until the cutoff
CLAIM_MINUTES = 5
MAX_ATTEMPTS = 8
BACKFILL_GAP = 5 # seconds between backfill checks, across workers
SILO_NAMES = {"gotosocial": "GoToSocial", "bluesky": "Bluesky", "threads": "Threads"}
OPT_OUT_RE = re.compile(r"#no(bridge|bot)\b", re.IGNORECASE)
# Silos whose credential was refused this run: target id -> message
denied: Dict[str, str] = {}
@dataclass
class Response:
"""One like, repost, quote or reply on a silo"""
kind: str # like, repost, quote, reply
source_url: str # its URL (likes/reposts: the copy + #liked-by-<id>)
remote_id: str # silo id of the response, or of the actor
author_id: str = ""
author_name: str = ""
author_url: str = ""
author_photo: Optional[str] = None
author_bio: str = ""
opted_out: bool = False # set by the client from silo-specific signals
content_html: Optional[str] = None
published: Optional[str] = None
# Replies in a thread (v2.20.2): the silo id of the response this one
# answers (None: our copy itself), and whether we wrote it
parent_id: Optional[str] = None
own: bool = False
def thread_members(posts: List[dict], root: str) -> Dict[str, Optional[str]]:
"""
Which replies under one of our copies to keep, and what each answers
(v2.20.2). `posts` are {'id', 'parent', 'own'} for every reply in the
conversation, in any order. Kept: others' replies to our copy, and any
reply (ours too) to a kept reply. Our own replies to our copy are the
thread continuing, not responses. Returns {id: parent id or None}.
"""
kept: Dict[str, Optional[str]] = {}
pending = list(posts)
while True:
rest = []
for post in pending:
if post["parent"] == root and not post["own"]:
kept[post["id"]] = None
elif post["parent"] in kept and post["parent"] != post["id"]:
kept[post["id"]] = post["parent"]
else:
rest.append(post)
if len(rest) == len(pending):
return kept
pending = rest
@dataclass
class Harvest:
"""What one check found. complete=False: some list was cut short."""
responses: List[Response]
complete: bool = True
# Likes/reposts/quotes shown only as numbers: all of them on Threads
# (its API doesn't say who); elsewhere, those by people who opted out
counts: Optional[Dict[str, int]] = None
# --- Text -----------------------------------------------------------------
def text_html(text: str) -> str:
"""Plain text as HTML: escaped, URLs linked, line breaks kept"""
escaped = html.escape(text)
linked = re.sub(r"https?://[^\s<>&\"]+",
lambda m: f'<a href="{m.group(0)}">{m.group(0)}</a>', escaped)
return "<p>" + linked.replace("\n", "<br>") + "</p>" if text.strip() else ""
def facets_html(text: str, facets: list) -> str:
"""Bluesky post text as HTML, with its link facets (byte ranges) as links"""
data = text.encode()
links = []
for facet in facets:
index = facet.get("index") or {}
for feature in facet.get("features") or []:
uri = feature.get("uri") or ""
if feature.get("$type") == "app.bsky.richtext.facet#link" and uri.startswith(("http://", "https://")):
links.append((index.get("byteStart", 0), index.get("byteEnd", 0), uri))
links.sort()
parts, pos = [], 0
for start, end, uri in links:
if start < pos or end > len(data) or start >= end:
continue
parts.append(html.escape(data[pos:start].decode(errors="ignore")))
label = html.escape(data[start:end].decode(errors="ignore"))
parts.append(f'<a href="{html.escape(uri, quote=True)}">{label}</a>')
pos = end
parts.append(html.escape(data[pos:].decode(errors="ignore")))
body = "".join(parts)
return "<p>" + body.replace("\n", "<br>") + "</p>" if text.strip() else ""
# --- Scheduling --------------------------------------------------------------
def interval_for(age_seconds: float, max_age_days: int) -> Optional[int]:
"""Seconds until a note of this age is checked again; None past the cutoff"""
if age_seconds >= max_age_days * DAY:
return None
for younger_than, every in SCHEDULE:
if age_seconds < younger_than:
return every
return LATE_INTERVAL
def _age_seconds(created_at) -> float:
if isinstance(created_at, str):
created_at = datetime.fromisoformat(created_at.replace("Z", "+00:00"))
if created_at.tzinfo is None:
created_at = created_at.replace(tzinfo=timezone.utc)
return (datetime.now(timezone.utc) - created_at).total_seconds()
_ELIGIBLE = """
FROM syndications s JOIN notes n ON n.id = s.note_id
WHERE s.state = 'synced' AND s.enabled = 1 AND s.remote_id IS NOT NULL
AND n.published = 1 AND n.deleted_at IS NULL
AND n.like_of IS NULL AND n.repost_of IS NULL
"""
def _claim(db, row_id: int) -> bool:
cursor = db.execute(
f"""
UPDATE syndications SET backfeed_claimed_until = datetime('now', '+{CLAIM_MINUTES} minutes')
WHERE id = ? AND (backfeed_claimed_until IS NULL OR backfeed_claimed_until < datetime('now'))
""",
(row_id,),
)
db.commit()
return cursor.rowcount == 1
def _backfill_turn(db) -> bool:
"""Take the next backfill slot: at most one check every BACKFILL_GAP seconds"""
cursor = db.execute(
f"""
UPDATE site_state SET value = datetime('now')
WHERE key = 'backfeed_backfill_last' AND value <= datetime('now', '-{BACKFILL_GAP} seconds')
"""
)
if cursor.rowcount == 0:
cursor = db.execute(
"INSERT OR IGNORE INTO site_state (key, value) VALUES ('backfeed_backfill_last', datetime('now'))")
db.commit()
return cursor.rowcount == 1
def _due(db, limit: int) -> list:
"""Routine checks due now; a backfill check only when none are"""
max_age = int(current_app.config.get("BACKFEED_MAX_AGE_DAYS", 90))
lease = "(s.backfeed_claimed_until IS NULL OR s.backfeed_claimed_until < datetime('now'))"
rows = db.execute(
f"""
SELECT s.*, n.created_at AS note_created, n.slug AS note_slug, n.title AS note_title {_ELIGIBLE}
AND {lease}
AND n.created_at >= datetime('now', ?)
AND (s.backfeed_next_at IS NULL OR s.backfeed_next_at <= datetime('now'))
ORDER BY s.backfeed_next_at IS NOT NULL, s.backfeed_next_at LIMIT ?
""",
(f"-{max_age} days", limit),
).fetchall()
if rows:
return rows
backfill = db.execute(
f"""
SELECT s.*, n.created_at AS note_created, n.slug AS note_slug, n.title AS note_title {_ELIGIBLE}
AND {lease} AND s.backfeed_backfill = 1
AND (s.backfeed_next_at IS NULL OR s.backfeed_next_at <= datetime('now'))
ORDER BY s.id LIMIT 1
"""
).fetchall()
if backfill and _backfill_turn(db):
return backfill
return []
def _finish(db, row) -> None:
max_age = int(current_app.config.get("BACKFEED_MAX_AGE_DAYS", 90))
every = interval_for(_age_seconds(row["note_created"]), max_age)
if every is None:
next_at = None
else:
next_at = f"+{int(every * random.uniform(1.0, 1.1))} seconds"
db.execute(
"""
UPDATE syndications
SET backfeed_checked_at = datetime('now'), backfeed_claimed_until = NULL,
backfeed_attempts = 0, backfeed_error = NULL, backfeed_backfill = 0,
backfeed_next_at = CASE WHEN ? IS NULL THEN NULL ELSE datetime('now', ?) END
WHERE id = ?
""",
(next_at, next_at, row["id"]),
)
db.commit()
def _fail(db, row, message: str) -> None:
"""Back off; after MAX_ATTEMPTS drop any backfill and try again in a month"""
attempts = row["backfeed_attempts"] + 1
stop = attempts >= MAX_ATTEMPTS
delay = 30 * DAY if stop else min(30 * (2 ** attempts), 6 * 3600)
db.execute(
f"""
UPDATE syndications
SET backfeed_attempts = ?, backfeed_error = ?, backfeed_claimed_until = NULL,
backfeed_backfill = CASE WHEN ? THEN 0 ELSE backfeed_backfill END,
backfeed_next_at = datetime('now', '+{delay} seconds')
WHERE id = ?
""",
(attempts, message[:500], 1 if stop else 0, row["id"]),
)
db.commit()
current_app.logger.warning(
f"Backfeed {row['target']} for note {row['note_id']} failed (attempt {attempts}): {message}")
# --- Storing -----------------------------------------------------------------
def _opted_out(response: Response) -> bool:
return response.opted_out or bool(OPT_OUT_RE.search(html.unescape(response.author_bio or "")))
def _display_state(db, note_id: int) -> list:
return [tuple(r) for r in db.execute(
"""
SELECT id, mention_type, author_name, author_url, author_photo, content_snippet
FROM webmentions
WHERE target_note_id = ? AND validation_status = 'valid' AND moderation_status = 'approved'
ORDER BY id
""",
(note_id,),
)]
def hidden_counts(harvest: Harvest) -> Dict[str, int]:
"""
Likes, reposts and quotes to show only as numbers: the silo's own counts
when it gives no names (Threads), else those by people who opted out
(v2.13.3: they're counted, not named)
"""
if harvest.counts is not None:
return harvest.counts
hidden = [r.kind for r in harvest.responses if _opted_out(r)]
return {k: hidden.count(kind) for k, kind in
(("likes", "like"), ("reposts", "repost"), ("quotes", "quote"))}
def store(db, note_id: int, target_url: str, origin: str, harvest: Harvest,
quiet: bool = False) -> None:
"""
Upsert a check's responses for one note and silo. With a complete
harvest, that silo's responses for the note that weren't found are
marked invalid. Responses from people who opted out are removed (they
are counted instead, see `hidden_counts`). With `quiet`, new responses
are stored as already notified (ADR-074 Decision 5).
"""
from starpunk.webmention_processor import _snippet
moderation = "approved" if current_app.config.get("WEBMENTION_AUTO_APPROVE", True) else "pending"
silo = SILO_NAMES.get(origin, origin)
keep = [r for r in harvest.responses if not _opted_out(r)]
dropped = [r.source_url for r in harvest.responses if _opted_out(r)]
from starpunk.avatars import cached_name
before = _display_state(db, note_id)
for r in keep:
snippet = _snippet(r.content_html) if r.content_html else None
# Threads avatar URLs are signed and change on every request: keep
# the stored URL while its saved copy exists, so it isn't refetched
old = db.execute("SELECT author_photo FROM webmentions WHERE source_url = ? AND target_url = ?",
(r.source_url, target_url)).fetchone()
if old and old["author_photo"] and origin == "threads" \
and cached_name(current_app.config["DATA_PATH"], old["author_photo"]):
r.author_photo = old["author_photo"]
db.execute(
"""
INSERT INTO webmentions (
source_url, target_url, target_note_id, mention_type, author_name, author_url,
author_photo, content_snippet, validation_status, moderation_status,
validated_at, source_published, origin, remote_id, notified_at, parent_remote_id)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'valid', ?, CURRENT_TIMESTAMP, ?, ?, ?,
CASE WHEN ? THEN CURRENT_TIMESTAMP END, ?)
ON CONFLICT(source_url, target_url) DO UPDATE SET
target_note_id = excluded.target_note_id,
mention_type = excluded.mention_type,
author_name = excluded.author_name,
author_url = excluded.author_url,
author_photo = excluded.author_photo,
content_snippet = excluded.content_snippet,
source_published = COALESCE(excluded.source_published, webmentions.source_published),
origin = excluded.origin,
remote_id = excluded.remote_id,
parent_remote_id = excluded.parent_remote_id,
validation_status = 'valid',
validation_error = NULL,
validated_at = CURRENT_TIMESTAMP,
updated_at = CURRENT_TIMESTAMP
""",
(r.source_url, target_url, note_id, r.kind, r.author_name or None, r.author_url or None,
r.author_photo or None, snippet, moderation, r.published, origin, r.remote_id,
# Our own replies in a thread are no news to us
1 if quiet or r.own else 0, r.parent_id),
)
if dropped:
marks = ",".join("?" * len(dropped))
db.execute(f"DELETE FROM webmentions WHERE origin = ? AND target_note_id = ? AND source_url IN ({marks})",
[origin, note_id] + dropped)
if harvest.complete:
found = [r.source_url for r in keep]
marks = ",".join("?" * len(found)) or "''"
db.execute(
f"""
UPDATE webmentions
SET validation_status = 'invalid', validation_error = ?, updated_at = CURRENT_TIMESTAMP
WHERE target_note_id = ? AND origin = ? AND validation_status = 'valid'
AND source_url NOT IN ({marks})
""",
[f"No longer on {silo}", note_id, origin] + found,
)
db.commit()
if _display_state(db, note_id) != before:
try:
from pathlib import Path
from starpunk.webmention_sender import note_responses_changed
note_responses_changed(Path(current_app.config["DATABASE_PATH"]), [note_id])
except Exception as e:
current_app.logger.warning(f"Failed to re-send webmentions for note {note_id}: {e}")
from starpunk import avatars
for r in keep:
avatars.fetch(current_app.config["DATA_PATH"], r.author_photo,
current_app.config.get("SITE_URL", ""))
def _store_counts(db, row_id: int, counts: Optional[dict]) -> None:
if counts is None:
return
db.execute(
"UPDATE syndications SET likes_count = ?, reposts_count = ?, quotes_count = ? WHERE id = ?",
(counts.get("likes"), counts.get("reposts"), counts.get("quotes"), row_id),
)
db.commit()
# --- Checking ---------------------------------------------------------------
def _is_denied(error) -> bool:
"""The silo refused the credential (missing scope or bad token)"""
from starpunk.syndication import _GRAPH_AUTH
return getattr(error, "status", None) in (401, 403) or getattr(error, "code", None) in _GRAPH_AUTH
SCOPES = {
"gotosocial": "a GoToSocial token with read:statuses and read:accounts (or read)",
"bluesky": "a working Bluesky app password",
"threads": "a Threads token with threads_read_replies and threads_manage_insights",
}
def check(row, target) -> None:
"""Check one copy now and store what it found (raises SyndicationError)"""
db = get_db(current_app)
site = current_app.config["SITE_URL"].rstrip("/")
harvest = target.client.responses(row["remote_id"], row["remote_url"] or "")
# No notifications for history: a backfill, or a copy's first check on an
# older note (e.g. one linked after the fact). ADR-074 Decision 5.
quiet = bool(row["backfeed_backfill"]) or (
row["backfeed_checked_at"] is None and _age_seconds(row["note_created"]) > DAY)
store(db, row["note_id"], site + post_path(row["note_slug"], row["note_title"]), row["target"], harvest, quiet=quiet)
_store_counts(db, row["id"], hidden_counts(harvest))
def process_due(limit: int = 3) -> int:
"""Check up to `limit` copies that are due. Returns how many were handled."""
if not current_app.config.get("BACKFEED_ENABLED", False):
return 0
from starpunk.syndication import SyndicationError, get_targets
targets = get_targets()
db = get_db(current_app)
handled = 0
for row in _due(db, limit):
if row["target"] in denied or row["target"] not in targets:
continue
if not _claim(db, row["id"]):
continue
handled += 1
try:
check(row, targets[row["target"]])
except SyndicationError as e:
if _is_denied(e):
denied[row["target"]] = f"Needs {SCOPES.get(row['target'], 'read access')}: {e}"
current_app.logger.error(f"Backfeed for {row['target']} stopped: {denied[row['target']]}")
from starpunk import notifications
site = current_app.config["SITE_URL"].rstrip("/")
notifications.problem(current_app.config, current_app.config["DATABASE_PATH"],
f"Backfeed for {SILO_NAMES.get(row['target'], row['target'])} stopped",
denied[row["target"]], f"{site}/admin/webmentions")
db.execute("UPDATE syndications SET backfeed_claimed_until = NULL WHERE id = ?", (row["id"],))
db.commit()
continue
_fail(db, row, str(e))
continue
except Exception as e: # one bad copy mustn't stop the worker
_fail(db, row, f"unexpected error: {e}")
continue
_finish(db, row)
_note_backfill_done(db)
return handled
# --- Backfill and admin ---------------------------------------------------------
def _state(db, key: str) -> Optional[str]:
row = db.execute("SELECT value FROM site_state WHERE key = ?", (key,)).fetchone()
return row["value"] if row and row["value"] else None
def _set_state(db, key: str, value: str) -> None:
db.execute("INSERT INTO site_state (key, value) VALUES (?, ?) "
"ON CONFLICT(key) DO UPDATE SET value = excluded.value", (key, value))
def start_backfill() -> int:
"""Mark every eligible copy for one check, whatever its age. Returns how many."""
db = get_db(current_app)
cursor = db.execute(
f"""
UPDATE syndications
SET backfeed_backfill = 1, backfeed_attempts = 0, backfeed_next_at = datetime('now')
WHERE id IN (SELECT s.id {_ELIGIBLE})
"""
)
now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
_set_state(db, "backfeed_backfill_started", now)
_set_state(db, "backfeed_backfill_finished", "")
db.commit()
return cursor.rowcount
def _note_backfill_done(db) -> None:
if _state(db, "backfeed_backfill_started") and not _state(db, "backfeed_backfill_finished"):
left = db.execute("SELECT COUNT(*) FROM syndications WHERE backfeed_backfill = 1").fetchone()[0]
if left == 0:
_set_state(db, "backfeed_backfill_finished",
datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S"))
db.commit()
def check_now(syndication_id: int) -> None:
"""Check this copy's responses at the next worker pass"""
db = get_db(current_app)
db.execute("UPDATE syndications SET backfeed_next_at = datetime('now'), backfeed_attempts = 0 "
"WHERE id = ?", (syndication_id,))
db.commit()
def status() -> dict:
"""For the admin pages: backfill progress and silos whose credential was refused"""
db = get_db(current_app)
return {
"enabled": bool(current_app.config.get("BACKFEED_ENABLED", False)),
"started": _state(db, "backfeed_backfill_started"),
"finished": _state(db, "backfeed_backfill_finished"),
"remaining": db.execute(
"SELECT COUNT(*) FROM syndications WHERE backfeed_backfill = 1").fetchone()[0],
"denied": dict(denied),
"errors": db.execute(
"SELECT COUNT(*) FROM syndications WHERE backfeed_error IS NOT NULL").fetchone()[0],
}