457 lines
18 KiB
Python
457 lines
18 KiB
Python
"""
|
|
Backfeed of responses to crossposts (v2.13.0, ADR-073)
|
|
|
|
StarPunk asks each silo for the responses to its own copies of a note
|
|
(likes, reposts, quotes and replies) and stores them in `webmentions`, so
|
|
moderation, the Responses section and re-sending webmentions when a note's
|
|
responses change (spec 3.1.4) work as they do for received webmentions.
|
|
|
|
Each synced copy of a published note is checked on a schedule that slows as
|
|
the note ages, and not at all after BACKFEED_MAX_AGE_DAYS. A backfill
|
|
checks every copy once, whatever its age. Checks run in the syndication
|
|
worker's loop (`process_due`), one copy at a time under a lease.
|
|
|
|
A check that read the whole list marks responses that are no longer there
|
|
as invalid, so withdrawn likes and deleted replies drop off the page.
|
|
"""
|
|
|
|
import html
|
|
import random
|
|
import re
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timezone
|
|
from typing import Dict, List, Optional
|
|
|
|
from flask import current_app
|
|
|
|
from starpunk.database import get_db
|
|
|
|
DAY = 24 * 3600
|
|
SCHEDULE = [ # (note younger than, check every), in seconds
|
|
(DAY, 15 * 60),
|
|
(7 * DAY, 3600),
|
|
(30 * DAY, DAY),
|
|
]
|
|
LATE_INTERVAL = 7 * DAY # from 30 days until the cutoff
|
|
CLAIM_MINUTES = 5
|
|
MAX_ATTEMPTS = 8
|
|
BACKFILL_GAP = 5 # seconds between backfill checks, across workers
|
|
SILO_NAMES = {"gotosocial": "GoToSocial", "bluesky": "Bluesky", "threads": "Threads"}
|
|
OPT_OUT_RE = re.compile(r"#no(bridge|bot)\b", re.IGNORECASE)
|
|
|
|
# Silos whose credential was refused this run: target id -> message
|
|
denied: Dict[str, str] = {}
|
|
|
|
|
|
@dataclass
|
|
class Response:
|
|
"""One like, repost, quote or reply on a silo"""
|
|
kind: str # like, repost, quote, reply
|
|
source_url: str # its URL (likes/reposts: the copy + #liked-by-<id>)
|
|
remote_id: str # silo id of the response, or of the actor
|
|
author_id: str = ""
|
|
author_name: str = ""
|
|
author_url: str = ""
|
|
author_photo: Optional[str] = None
|
|
author_bio: str = ""
|
|
opted_out: bool = False # set by the client from silo-specific signals
|
|
content_html: Optional[str] = None
|
|
published: Optional[str] = None
|
|
|
|
|
|
@dataclass
|
|
class Harvest:
|
|
"""What one check found. complete=False: some list was cut short."""
|
|
responses: List[Response]
|
|
complete: bool = True
|
|
counts: Optional[Dict[str, int]] = None # Threads likes/reposts/quotes
|
|
|
|
|
|
# --- Text -----------------------------------------------------------------
|
|
|
|
def text_html(text: str) -> str:
|
|
"""Plain text as HTML: escaped, URLs linked, line breaks kept"""
|
|
escaped = html.escape(text)
|
|
linked = re.sub(r"https?://[^\s<>&\"]+",
|
|
lambda m: f'<a href="{m.group(0)}">{m.group(0)}</a>', escaped)
|
|
return "<p>" + linked.replace("\n", "<br>") + "</p>" if text.strip() else ""
|
|
|
|
|
|
def facets_html(text: str, facets: list) -> str:
|
|
"""Bluesky post text as HTML, with its link facets (byte ranges) as links"""
|
|
data = text.encode()
|
|
links = []
|
|
for facet in facets:
|
|
index = facet.get("index") or {}
|
|
for feature in facet.get("features") or []:
|
|
uri = feature.get("uri") or ""
|
|
if feature.get("$type") == "app.bsky.richtext.facet#link" and uri.startswith(("http://", "https://")):
|
|
links.append((index.get("byteStart", 0), index.get("byteEnd", 0), uri))
|
|
links.sort()
|
|
parts, pos = [], 0
|
|
for start, end, uri in links:
|
|
if start < pos or end > len(data) or start >= end:
|
|
continue
|
|
parts.append(html.escape(data[pos:start].decode(errors="ignore")))
|
|
label = html.escape(data[start:end].decode(errors="ignore"))
|
|
parts.append(f'<a href="{html.escape(uri, quote=True)}">{label}</a>')
|
|
pos = end
|
|
parts.append(html.escape(data[pos:].decode(errors="ignore")))
|
|
body = "".join(parts)
|
|
return "<p>" + body.replace("\n", "<br>") + "</p>" if text.strip() else ""
|
|
|
|
|
|
# --- Scheduling --------------------------------------------------------------
|
|
|
|
def interval_for(age_seconds: float, max_age_days: int) -> Optional[int]:
|
|
"""Seconds until a note of this age is checked again; None past the cutoff"""
|
|
if age_seconds >= max_age_days * DAY:
|
|
return None
|
|
for younger_than, every in SCHEDULE:
|
|
if age_seconds < younger_than:
|
|
return every
|
|
return LATE_INTERVAL
|
|
|
|
|
|
def _age_seconds(created_at) -> float:
|
|
if isinstance(created_at, str):
|
|
created_at = datetime.fromisoformat(created_at.replace("Z", "+00:00"))
|
|
if created_at.tzinfo is None:
|
|
created_at = created_at.replace(tzinfo=timezone.utc)
|
|
return (datetime.now(timezone.utc) - created_at).total_seconds()
|
|
|
|
|
|
_ELIGIBLE = """
|
|
FROM syndications s JOIN notes n ON n.id = s.note_id
|
|
WHERE s.state = 'synced' AND s.enabled = 1 AND s.remote_id IS NOT NULL
|
|
AND n.published = 1 AND n.deleted_at IS NULL
|
|
AND n.like_of IS NULL AND n.repost_of IS NULL
|
|
"""
|
|
|
|
|
|
def _claim(db, row_id: int) -> bool:
|
|
cursor = db.execute(
|
|
f"""
|
|
UPDATE syndications SET backfeed_claimed_until = datetime('now', '+{CLAIM_MINUTES} minutes')
|
|
WHERE id = ? AND (backfeed_claimed_until IS NULL OR backfeed_claimed_until < datetime('now'))
|
|
""",
|
|
(row_id,),
|
|
)
|
|
db.commit()
|
|
return cursor.rowcount == 1
|
|
|
|
|
|
def _backfill_turn(db) -> bool:
|
|
"""Take the next backfill slot: at most one check every BACKFILL_GAP seconds"""
|
|
cursor = db.execute(
|
|
f"""
|
|
UPDATE site_state SET value = datetime('now')
|
|
WHERE key = 'backfeed_backfill_last' AND value <= datetime('now', '-{BACKFILL_GAP} seconds')
|
|
"""
|
|
)
|
|
if cursor.rowcount == 0:
|
|
cursor = db.execute(
|
|
"INSERT OR IGNORE INTO site_state (key, value) VALUES ('backfeed_backfill_last', datetime('now'))")
|
|
db.commit()
|
|
return cursor.rowcount == 1
|
|
|
|
|
|
def _due(db, limit: int) -> list:
|
|
"""Routine checks due now; a backfill check only when none are"""
|
|
max_age = int(current_app.config.get("BACKFEED_MAX_AGE_DAYS", 90))
|
|
lease = "(s.backfeed_claimed_until IS NULL OR s.backfeed_claimed_until < datetime('now'))"
|
|
rows = db.execute(
|
|
f"""
|
|
SELECT s.*, n.created_at AS note_created, n.slug AS note_slug {_ELIGIBLE}
|
|
AND {lease}
|
|
AND n.created_at >= datetime('now', ?)
|
|
AND (s.backfeed_next_at IS NULL OR s.backfeed_next_at <= datetime('now'))
|
|
ORDER BY s.backfeed_next_at IS NOT NULL, s.backfeed_next_at LIMIT ?
|
|
""",
|
|
(f"-{max_age} days", limit),
|
|
).fetchall()
|
|
if rows:
|
|
return rows
|
|
backfill = db.execute(
|
|
f"""
|
|
SELECT s.*, n.created_at AS note_created, n.slug AS note_slug {_ELIGIBLE}
|
|
AND {lease} AND s.backfeed_backfill = 1
|
|
AND (s.backfeed_next_at IS NULL OR s.backfeed_next_at <= datetime('now'))
|
|
ORDER BY s.id LIMIT 1
|
|
"""
|
|
).fetchall()
|
|
if backfill and _backfill_turn(db):
|
|
return backfill
|
|
return []
|
|
|
|
|
|
def _finish(db, row) -> None:
|
|
max_age = int(current_app.config.get("BACKFEED_MAX_AGE_DAYS", 90))
|
|
every = interval_for(_age_seconds(row["note_created"]), max_age)
|
|
if every is None:
|
|
next_at = None
|
|
else:
|
|
next_at = f"+{int(every * random.uniform(1.0, 1.1))} seconds"
|
|
db.execute(
|
|
"""
|
|
UPDATE syndications
|
|
SET backfeed_checked_at = datetime('now'), backfeed_claimed_until = NULL,
|
|
backfeed_attempts = 0, backfeed_error = NULL, backfeed_backfill = 0,
|
|
backfeed_next_at = CASE WHEN ? IS NULL THEN NULL ELSE datetime('now', ?) END
|
|
WHERE id = ?
|
|
""",
|
|
(next_at, next_at, row["id"]),
|
|
)
|
|
db.commit()
|
|
|
|
|
|
def _fail(db, row, message: str) -> None:
|
|
"""Back off; after MAX_ATTEMPTS drop any backfill and try again in a month"""
|
|
attempts = row["backfeed_attempts"] + 1
|
|
stop = attempts >= MAX_ATTEMPTS
|
|
delay = 30 * DAY if stop else min(30 * (2 ** attempts), 6 * 3600)
|
|
db.execute(
|
|
f"""
|
|
UPDATE syndications
|
|
SET backfeed_attempts = ?, backfeed_error = ?, backfeed_claimed_until = NULL,
|
|
backfeed_backfill = CASE WHEN ? THEN 0 ELSE backfeed_backfill END,
|
|
backfeed_next_at = datetime('now', '+{delay} seconds')
|
|
WHERE id = ?
|
|
""",
|
|
(attempts, message[:500], 1 if stop else 0, row["id"]),
|
|
)
|
|
db.commit()
|
|
current_app.logger.warning(
|
|
f"Backfeed {row['target']} for note {row['note_id']} failed (attempt {attempts}): {message}")
|
|
|
|
|
|
# --- Storing -----------------------------------------------------------------
|
|
|
|
def _opted_out(response: Response) -> bool:
|
|
return response.opted_out or bool(OPT_OUT_RE.search(html.unescape(response.author_bio or "")))
|
|
|
|
|
|
def _display_state(db, note_id: int) -> list:
|
|
return [tuple(r) for r in db.execute(
|
|
"""
|
|
SELECT id, mention_type, author_name, author_url, author_photo, content_snippet
|
|
FROM webmentions
|
|
WHERE target_note_id = ? AND validation_status = 'valid' AND moderation_status = 'approved'
|
|
ORDER BY id
|
|
""",
|
|
(note_id,),
|
|
)]
|
|
|
|
|
|
def store(db, note_id: int, target_url: str, origin: str, harvest: Harvest) -> None:
|
|
"""
|
|
Upsert a check's responses for one note and silo. With a complete
|
|
harvest, that silo's responses for the note that weren't found are
|
|
marked invalid. Responses from people who opted out are removed.
|
|
"""
|
|
from starpunk.webmention_processor import _snippet
|
|
|
|
moderation = "approved" if current_app.config.get("WEBMENTION_AUTO_APPROVE", True) else "pending"
|
|
silo = SILO_NAMES.get(origin, origin)
|
|
keep = [r for r in harvest.responses if not _opted_out(r)]
|
|
dropped = [r.source_url for r in harvest.responses if _opted_out(r)]
|
|
|
|
from starpunk.avatars import cached_name
|
|
|
|
before = _display_state(db, note_id)
|
|
for r in keep:
|
|
snippet = _snippet(r.content_html) if r.content_html else None
|
|
# Threads avatar URLs are signed and change on every request: keep
|
|
# the stored URL while its saved copy exists, so it isn't refetched
|
|
old = db.execute("SELECT author_photo FROM webmentions WHERE source_url = ? AND target_url = ?",
|
|
(r.source_url, target_url)).fetchone()
|
|
if old and old["author_photo"] and origin == "threads" \
|
|
and cached_name(current_app.config["DATA_PATH"], old["author_photo"]):
|
|
r.author_photo = old["author_photo"]
|
|
db.execute(
|
|
"""
|
|
INSERT INTO webmentions (
|
|
source_url, target_url, target_note_id, mention_type, author_name, author_url,
|
|
author_photo, content_snippet, validation_status, moderation_status,
|
|
validated_at, source_published, origin, remote_id)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'valid', ?, CURRENT_TIMESTAMP, ?, ?, ?)
|
|
ON CONFLICT(source_url, target_url) DO UPDATE SET
|
|
target_note_id = excluded.target_note_id,
|
|
mention_type = excluded.mention_type,
|
|
author_name = excluded.author_name,
|
|
author_url = excluded.author_url,
|
|
author_photo = excluded.author_photo,
|
|
content_snippet = excluded.content_snippet,
|
|
source_published = COALESCE(excluded.source_published, webmentions.source_published),
|
|
origin = excluded.origin,
|
|
remote_id = excluded.remote_id,
|
|
validation_status = 'valid',
|
|
validation_error = NULL,
|
|
validated_at = CURRENT_TIMESTAMP,
|
|
updated_at = CURRENT_TIMESTAMP
|
|
""",
|
|
(r.source_url, target_url, note_id, r.kind, r.author_name or None, r.author_url or None,
|
|
r.author_photo or None, snippet, moderation, r.published, origin, r.remote_id),
|
|
)
|
|
if dropped:
|
|
marks = ",".join("?" * len(dropped))
|
|
db.execute(f"DELETE FROM webmentions WHERE origin = ? AND target_note_id = ? AND source_url IN ({marks})",
|
|
[origin, note_id] + dropped)
|
|
if harvest.complete:
|
|
found = [r.source_url for r in keep]
|
|
marks = ",".join("?" * len(found)) or "''"
|
|
db.execute(
|
|
f"""
|
|
UPDATE webmentions
|
|
SET validation_status = 'invalid', validation_error = ?, updated_at = CURRENT_TIMESTAMP
|
|
WHERE target_note_id = ? AND origin = ? AND validation_status = 'valid'
|
|
AND source_url NOT IN ({marks})
|
|
""",
|
|
[f"No longer on {silo}", note_id, origin] + found,
|
|
)
|
|
db.commit()
|
|
|
|
if _display_state(db, note_id) != before:
|
|
try:
|
|
from pathlib import Path
|
|
|
|
from starpunk.webmention_sender import note_responses_changed
|
|
note_responses_changed(Path(current_app.config["DATABASE_PATH"]), [note_id])
|
|
except Exception as e:
|
|
current_app.logger.warning(f"Failed to re-send webmentions for note {note_id}: {e}")
|
|
|
|
from starpunk import avatars
|
|
for r in keep:
|
|
avatars.fetch(current_app.config["DATA_PATH"], r.author_photo,
|
|
current_app.config.get("SITE_URL", ""))
|
|
|
|
|
|
def _store_counts(db, row_id: int, counts: Optional[dict]) -> None:
|
|
if counts is None:
|
|
return
|
|
db.execute(
|
|
"UPDATE syndications SET likes_count = ?, reposts_count = ?, quotes_count = ? WHERE id = ?",
|
|
(counts.get("likes"), counts.get("reposts"), counts.get("quotes"), row_id),
|
|
)
|
|
db.commit()
|
|
|
|
|
|
# --- Checking ---------------------------------------------------------------
|
|
|
|
def _is_denied(error) -> bool:
|
|
"""The silo refused the credential (missing scope or bad token)"""
|
|
from starpunk.syndication import _GRAPH_AUTH
|
|
|
|
return getattr(error, "status", None) in (401, 403) or getattr(error, "code", None) in _GRAPH_AUTH
|
|
|
|
|
|
SCOPES = {
|
|
"gotosocial": "a GoToSocial token with read:statuses and read:accounts (or read)",
|
|
"bluesky": "a working Bluesky app password",
|
|
"threads": "a Threads token with threads_read_replies and threads_manage_insights",
|
|
}
|
|
|
|
|
|
def check(row, target) -> None:
|
|
"""Check one copy now and store what it found (raises SyndicationError)"""
|
|
db = get_db(current_app)
|
|
site = current_app.config["SITE_URL"].rstrip("/")
|
|
harvest = target.client.responses(row["remote_id"], row["remote_url"] or "")
|
|
store(db, row["note_id"], f"{site}/note/{row['note_slug']}", row["target"], harvest)
|
|
_store_counts(db, row["id"], harvest.counts)
|
|
|
|
|
|
def process_due(limit: int = 3) -> int:
|
|
"""Check up to `limit` copies that are due. Returns how many were handled."""
|
|
if not current_app.config.get("BACKFEED_ENABLED", False):
|
|
return 0
|
|
from starpunk.syndication import SyndicationError, get_targets
|
|
|
|
targets = get_targets()
|
|
db = get_db(current_app)
|
|
handled = 0
|
|
for row in _due(db, limit):
|
|
if row["target"] in denied or row["target"] not in targets:
|
|
continue
|
|
if not _claim(db, row["id"]):
|
|
continue
|
|
handled += 1
|
|
try:
|
|
check(row, targets[row["target"]])
|
|
except SyndicationError as e:
|
|
if _is_denied(e):
|
|
denied[row["target"]] = f"Needs {SCOPES.get(row['target'], 'read access')}: {e}"
|
|
current_app.logger.error(f"Backfeed for {row['target']} stopped: {denied[row['target']]}")
|
|
db.execute("UPDATE syndications SET backfeed_claimed_until = NULL WHERE id = ?", (row["id"],))
|
|
db.commit()
|
|
continue
|
|
_fail(db, row, str(e))
|
|
continue
|
|
except Exception as e: # one bad copy mustn't stop the worker
|
|
_fail(db, row, f"unexpected error: {e}")
|
|
continue
|
|
_finish(db, row)
|
|
_note_backfill_done(db)
|
|
return handled
|
|
|
|
|
|
# --- Backfill and admin ---------------------------------------------------------
|
|
|
|
def _state(db, key: str) -> Optional[str]:
|
|
row = db.execute("SELECT value FROM site_state WHERE key = ?", (key,)).fetchone()
|
|
return row["value"] if row and row["value"] else None
|
|
|
|
|
|
def _set_state(db, key: str, value: str) -> None:
|
|
db.execute("INSERT INTO site_state (key, value) VALUES (?, ?) "
|
|
"ON CONFLICT(key) DO UPDATE SET value = excluded.value", (key, value))
|
|
|
|
|
|
def start_backfill() -> int:
|
|
"""Mark every eligible copy for one check, whatever its age. Returns how many."""
|
|
db = get_db(current_app)
|
|
cursor = db.execute(
|
|
f"""
|
|
UPDATE syndications
|
|
SET backfeed_backfill = 1, backfeed_attempts = 0, backfeed_next_at = datetime('now')
|
|
WHERE id IN (SELECT s.id {_ELIGIBLE})
|
|
"""
|
|
)
|
|
now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
|
|
_set_state(db, "backfeed_backfill_started", now)
|
|
_set_state(db, "backfeed_backfill_finished", "")
|
|
db.commit()
|
|
return cursor.rowcount
|
|
|
|
|
|
def _note_backfill_done(db) -> None:
|
|
if _state(db, "backfeed_backfill_started") and not _state(db, "backfeed_backfill_finished"):
|
|
left = db.execute("SELECT COUNT(*) FROM syndications WHERE backfeed_backfill = 1").fetchone()[0]
|
|
if left == 0:
|
|
_set_state(db, "backfeed_backfill_finished",
|
|
datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S"))
|
|
db.commit()
|
|
|
|
|
|
def check_now(syndication_id: int) -> None:
|
|
"""Check this copy's responses at the next worker pass"""
|
|
db = get_db(current_app)
|
|
db.execute("UPDATE syndications SET backfeed_next_at = datetime('now'), backfeed_attempts = 0 "
|
|
"WHERE id = ?", (syndication_id,))
|
|
db.commit()
|
|
|
|
|
|
def status() -> dict:
|
|
"""For the admin pages: backfill progress and silos whose credential was refused"""
|
|
db = get_db(current_app)
|
|
return {
|
|
"enabled": bool(current_app.config.get("BACKFEED_ENABLED", False)),
|
|
"started": _state(db, "backfeed_backfill_started"),
|
|
"finished": _state(db, "backfeed_backfill_finished"),
|
|
"remaining": db.execute(
|
|
"SELECT COUNT(*) FROM syndications WHERE backfeed_backfill = 1").fetchone()[0],
|
|
"denied": dict(denied),
|
|
"errors": db.execute(
|
|
"SELECT COUNT(*) FROM syndications WHERE backfeed_error IS NOT NULL").fetchone()[0],
|
|
}
|