diff --git a/.env.example b/.env.example index 491a4c1..00d5ea5 100644 --- a/.env.example +++ b/.env.example @@ -165,7 +165,8 @@ WEBSUB_ENABLED=true # GoToSocial or any Mastodon-API server (v2.1.0, ADR-065). Create an app # token with scopes: write:statuses write:media, plus read:search -# write:favourites for native likes and boosts (v2.5.0) +# write:favourites for native likes and boosts (v2.5.0), plus read:statuses +# read:accounts (or read) for backfeed (v2.13.0) GOTOSOCIAL_URL= GOTOSOCIAL_ACCESS_TOKEN= # Name shown in the editor and Micropub apps (default: GoToSocial) @@ -185,7 +186,8 @@ BLUESKY_APP_PASSWORD= # BLUESKY_MAX_CHARS=300 # Threads (v2.6.0, ADR-069). A long-lived user token with threads_basic, -# threads_content_publish and threads_delete; StarPunk refreshes it daily and +# threads_content_publish and threads_delete (plus threads_read_replies and +# threads_manage_insights for backfeed, v2.13.0); StarPunk refreshes it daily and # keeps the current one in DATA_PATH/threads_token.json. # How to get one: docs/design/v2.6.0/2026-10-31-threads-crossposting.md THREADS_ACCESS_TOKEN= @@ -198,6 +200,12 @@ THREADS_USERNAME= # Seconds between syndication worker passes (default: 10) # SYNDICATION_POLL_INTERVAL=10 +# Backfeed (v2.13.0, ADR-073): bring likes, reposts, quotes and replies to the +# crossposts back to the note as responses. Needs the read scopes above. +# BACKFEED_ENABLED=false +# Stop checking a note's responses after this many days (default: 90) +# BACKFEED_MAX_AGE_DAYS=90 + # ============================================================================= # DEBUGGING # ============================================================================= diff --git a/CHANGELOG.md b/CHANGELOG.md index 2577ab1..e4b4275 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [2.13.0] - 2026-11-10 + +Backfeed: responses to crossposts come back to the note (ADR-073). + +### Added + +- Likes, reposts, quotes and replies to a note's crossposts on GoToSocial and Bluesky, and replies on Threads, are shown on the note as responses, labelled "via Bluesky" (or GoToSocial, Threads). StarPunk checks each crosspost itself: every 15 minutes on a note's first day, hourly in its first week, daily until 30 days, weekly until `BACKFEED_MAX_AGE_DAYS` (90). Withdrawn likes and deleted replies drop off at the next check +- Threads like, repost and quote counts, shown as a line such as "On Threads: 12 likes · 3 reposts" (the Threads API doesn't say who) +- A Quotes group in a note's responses +- **Backfill responses** on the admin Webmentions page checks every crosspost once, however old the note; each note's edit page shows when its responses were last checked, with **Check now** +- The admin Webmentions list can be filtered by where a response came from, and shows deleted ones under **Deleted** +- Settings `BACKFEED_ENABLED` (default off) and `BACKFEED_MAX_AGE_DAYS`. Backfeed needs read access: a GoToSocial token with `read:statuses` and `read:accounts` (or `read`), and a Threads token with `threads_read_replies` and `threads_manage_insights`. Bluesky app passwords already work + +### Changed + +- Response authors' avatars are saved on this site (96 px WebP in `DATA_PATH/avatars`) and shown from there, instead of being loaded from other sites. Until a copy is saved the initial is shown +- Deleting a webmention in the admin keeps it as a deleted record, so a re-sent webmention or the next check doesn't bring it back +- People with `#nobridge` or `#nobot` in their profile, and Bluesky accounts that ask not to be shown to logged-out visitors, are not backfed; only public responses are +- A webmention for a response that was already backfed (for example a reply also sent by Bridgy) is ignored, so it isn't shown twice +- The Threads client uses `graph.threads.com`, the host in Meta's current documentation + ## [2.12.1] - 2026-11-10 ### Fixed diff --git a/docs/architecture/syndication-architecture.md b/docs/architecture/syndication-architecture.md index c18c069..eceaa26 100644 --- a/docs/architecture/syndication-architecture.md +++ b/docs/architecture/syndication-architecture.md @@ -223,6 +223,49 @@ Webmention worker rejects a webmention whose source is one of the note's own copies (e.g. backfed by Bridgy), so a copy is never shown as a response to itself (v2.8.0). +## Backfeed (v2.13.0, ADR-073) + +`starpunk/backfeed.py` brings responses to the copies back to the note. +After each crosspost pass, the worker loop calls `backfeed.process_due()`, +which checks up to three synced copies of published notes (not likes or +reposts) that are due, each under a lease (`backfeed_claimed_until`): + +- **Reading.** Each client has `responses(remote_id, remote_url)`, which + returns a `Harvest`: GoToSocial `favourited_by`, `reblogged_by` and + public direct replies from `context`; Bluesky `getLikes`, + `getRepostedBy`, `getQuotes` and `getPostThread` (depth 1), plus + `getProfiles` for reply and quote authors' bios; Threads `replies` and + `insights` (like, repost and quote counts). Lists are paged up to 1,000 + items; a list cut short makes the harvest incomplete. +- **Storing.** Responses are upserted into `webmentions` with `origin` (the + target id) and `remote_id`, `validation_status = 'valid'`, keyed by + `(source_url, target_url)`. Likes and reposts use the copy's URL plus + `#liked-by-` / `#reposted-by-`. A new row gets the + `WEBMENTION_AUTO_APPROVE` default; an existing row keeps its moderation. + A complete harvest marks that silo's rows for the note that weren't + found `invalid` ("No longer on "). Authors with `#nobridge` or + `#nobot` in their bio, or Bluesky's `!no-unauthenticated` label, are + skipped and their rows deleted. If the note's shown responses changed, + its webmentions are re-sent (spec 3.1.4). Threads counts go in + `syndications.likes_count`, `reposts_count`, `quotes_count`. +- **Schedule.** `backfeed_next_at` follows the note's age: 15 minutes (day + 1), 1 hour (week 1), 1 day (to 30 days), 1 week (to + `BACKFEED_MAX_AGE_DAYS`), with up to 10% jitter; none after. Failures back + off like crossposts; after 8 they wait a month. +- **Backfill.** `start_backfill()` flags every eligible copy + (`backfeed_backfill = 1`). Flagged copies are checked only when no + routine check is due, at most one every 5 seconds across workers + (`site_state.backfeed_backfill_last`). Start and finish times are in + `site_state`. +- **Refused credentials.** A 401/403 (or Graph auth error) puts the target + in `backfeed.denied` until restart; the admin Webmentions page says which + scopes are needed. Crossposting is unaffected. + +Avatars of all responses (received and backfed) are saved by +`starpunk/avatars.py` and served from `/avatars/.webp`; the +webmention worker fills in missing ones every 10 minutes and removes unused +ones weekly. + ## Client sessions and tokens - **Bluesky**: `createSession` with the app password, then diff --git a/docs/decisions/ADR-073-backfeed.md b/docs/decisions/ADR-073-backfeed.md index a6e93bf..8e2a9aa 100644 --- a/docs/decisions/ADR-073-backfeed.md +++ b/docs/decisions/ADR-073-backfeed.md @@ -2,7 +2,20 @@ ## Status -Proposed (2026-11-10, for v2.13.0) +Accepted (v2.13.0, 2026-11-10) + +Implementation notes (v2.13.0): + +- Until an avatar has a local copy, pages show the initial and keep the + original URL only as `` for microformats parsers, + so browsers don't load it. +- Bluesky post views carry no bio, so reply and quote authors' profiles + are fetched (`getProfiles`, 25 at a time) to honour `#nobridge`. +- A Threads token without `threads_manage_insights` stops Threads + backfeed as a whole, as for any refused credential. +- Backfill checks take a shared slot in `site_state` + (`backfeed_backfill_last`), so all gunicorn workers together do at most + one every 5 seconds. ## Context diff --git a/docs/decisions/INDEX.md b/docs/decisions/INDEX.md index fee6e6e..99c2d29 100644 --- a/docs/decisions/INDEX.md +++ b/docs/decisions/INDEX.md @@ -85,7 +85,7 @@ See also [IndieWeb protocols](../architecture/indieweb-protocols.md). | [066](ADR-066-bluesky-syndication.md) | Crossposting to Bluesky | Accepted | Bluesky as a second crosspost target (v2.2.0). | | [069](ADR-069-threads-syndication.md) | Crossposting to Threads | Accepted | Threads as a third target, with automatic token refresh (v2.6.0). | | [070](ADR-070-crosspost-links-and-cards.md) | Links and link cards in crossposts | Accepted | Keep note links in crossposts and build link cards for the linked page (v2.6.3). | -| [073](ADR-073-backfeed.md) | Backfeed of responses to crossposts | Proposed | Poll GoToSocial, Bluesky and Threads for responses to crossposts and store them as webmentions; admin backfill (planned v2.13.0). | +| [073](ADR-073-backfeed.md) | Backfeed of responses to crossposts | Accepted | Poll GoToSocial, Bluesky and Threads for responses to crossposts and store them as webmentions; admin backfill (v2.13.0). | ### Authentication (current) diff --git a/docs/design/INDEX.md b/docs/design/INDEX.md index 3c2b462..cd0531d 100644 --- a/docs/design/INDEX.md +++ b/docs/design/INDEX.md @@ -21,7 +21,7 @@ is in `../architecture/`; decisions are in `../decisions/` (ADRs). ## Contents -[v1.0.0](#v100) · [v1.1.1](#v111) · [v1.1.2](#v112) · [v1.2.0](#v120) · [v1.3.0](#v130) · [v1.3.1](#v131) · [v1.4.0](#v140) · [v1.4.1](#v141) · [v1.4.2](#v142) · [v1.5.0](#v150) · [v1.6.0](#v160) · [v2.0.0](#v200) · [v2.0.1](#v201) · [v2.0.2](#v202) · [v2.1.0](#v210) · [v2.2.0](#v220) · [v2.3.0](#v230) · [v2.3.1](#v231) · [v2.3.2](#v232) · [v2.3.3](#v233) · [v2.4.0](#v240) · [v2.5.0](#v250) · [v2.6.0](#v260) · [v2.6.1](#v261) · [v2.6.2](#v262) · [v2.6.3](#v263) · [v2.6.4](#v264) · [v2.6.5](#v265) · [v2.6.6](#v266) · [v2.6.7](#v267) · [v2.6.8](#v268) · [v2.6.9](#v269) · [v2.7.0](#v270) · [v2.7.1](#v271) · [v2.7.2](#v272) · [v2.8.0](#v280) · [v2.8.1](#v281) · [v2.9.0](#v290) · [v2.10.0](#v2100) · [v2.10.1](#v2101) · [v2.10.2](#v2102) · [v2.11.0](#v2110) · [v2.12.0](#v2120) · [v2.12.1](#v2121) · [hotfix](#hotfix) +[v1.0.0](#v100) · [v1.1.1](#v111) · [v1.1.2](#v112) · [v1.2.0](#v120) · [v1.3.0](#v130) · [v1.3.1](#v131) · [v1.4.0](#v140) · [v1.4.1](#v141) · [v1.4.2](#v142) · [v1.5.0](#v150) · [v1.6.0](#v160) · [v2.0.0](#v200) · [v2.0.1](#v201) · [v2.0.2](#v202) · [v2.1.0](#v210) · [v2.2.0](#v220) · [v2.3.0](#v230) · [v2.3.1](#v231) · [v2.3.2](#v232) · [v2.3.3](#v233) · [v2.4.0](#v240) · [v2.5.0](#v250) · [v2.6.0](#v260) · [v2.6.1](#v261) · [v2.6.2](#v262) · [v2.6.3](#v263) · [v2.6.4](#v264) · [v2.6.5](#v265) · [v2.6.6](#v266) · [v2.6.7](#v267) · [v2.6.8](#v268) · [v2.6.9](#v269) · [v2.7.0](#v270) · [v2.7.1](#v271) · [v2.7.2](#v272) · [v2.8.0](#v280) · [v2.8.1](#v281) · [v2.9.0](#v290) · [v2.10.0](#v2100) · [v2.10.1](#v2101) · [v2.10.2](#v2102) · [v2.11.0](#v2110) · [v2.12.0](#v2120) · [v2.12.1](#v2121) · [v2.13.0](#v2130) · [hotfix](#hotfix) ## v1.0.0 @@ -422,6 +422,10 @@ OIDC admin login and built-in IndieAuth server. - [2026-11-10-category-unused-tags.md](v2.12.1/2026-11-10-category-unused-tags.md) - q=category lists only tags on posts that exist +## v2.13.0 + +- [2026-11-10-backfeed.md](v2.13.0/2026-11-10-backfeed.md) - Backfeed of responses to crossposts (ADR-073), avatar cache, delete tombstones + ## hotfix Not a release folder. diff --git a/docs/design/v2.13.0/2026-11-10-backfeed.md b/docs/design/v2.13.0/2026-11-10-backfeed.md new file mode 100644 index 0000000..cbd1604 --- /dev/null +++ b/docs/design/v2.13.0/2026-11-10-backfeed.md @@ -0,0 +1,52 @@ +# v2.13.0: Backfeed (implementation report) + +Implements ADR-073. + +## What changed + +- `migrations/020_backfeed.sql`: `webmentions` rebuilt with `origin`, + `remote_id` and the `quote` type (rows copied, indexes recreated, plus + `(target_note_id, origin)`); `syndications` gains the `backfeed_*` + columns and Threads counts. +- `starpunk/backfeed.py` (new): `Response`/`Harvest`, schedule, lease, + storing and pruning, opt-outs, backfill, `check_now`, `status`. +- `starpunk/syndication.py`: `responses()` on `MastodonClient`, + `BlueskyClient` and `ThreadsClient`; HTTP status kept on errors + (`exc.status`) so refused credentials can be told apart; Mastodon `_call` + accepts absolute URLs (pagination links); the worker loop runs backfeed + after crossposts; Threads host is `graph.threads.com`. +- `starpunk/avatars.py` (new) and `/avatars/`: local avatar copies + for every response; `local_avatar` template filter. +- Webmentions: Quotes group and "via" label; admin delete is a tombstone + (`moderation_status = 'deleted'`, hidden from the list unless filtered); + origin filter; a webmention can't reset a backfed row, and one whose + source is an already backfed response is rejected ("Already backfed"). +- Admin: Backfeed panel with **Backfill responses** on the Webmentions + page; **Check now** and last-checked time per crosspost on the edit page. +- Settings `BACKFEED_ENABLED` (default false), `BACKFEED_MAX_AGE_DAYS` (90). + +## Verification + +- `tests/test_backfeed.py` (21 tests): schedule; text/facet HTML; storing, + updating and moderation kept; pruning only on complete harvests and + restoring; opt-outs; tombstones; backfill and its pacing; likes/reposts + and drafts skipped; refused credentials; backoff; note page (via label, + quotes, Threads counts, local avatars); avatar save/serve/cleanup; + webmention can't overwrite a backfed row; the three clients against fake + APIs; migration 020 on a pre-020 database keeps rows and foreign keys; + the admin Backfeed panel, backfill and Check now. +- Tests for the touched areas (webmentions, syndication, microformats, + migrations, templates, routes): all pass. The full suite runs in CI. +- `test_microformats_v280` expects response authors' `u-photo`: kept via + `` until the avatar is saved. + +## Deploying + +1. Issue a new GoToSocial token with `read:statuses read:accounts` added + to the current scopes, and re-authorise the Threads app with + `threads_read_replies` and `threads_manage_insights`. Put both in the + Ansible vault. +2. Set `BACKFEED_ENABLED=true` in the StarPunk environment and deploy + (migration 020 runs at start: back up the database first). +3. Run **Backfill responses** once from Admin → Webmentions. Turn Bridgy + off for the Mastodon and Bluesky accounts. diff --git a/docs/projectplan/BACKLOG.md b/docs/projectplan/BACKLOG.md index 5cea18b..f7b3ff9 100644 --- a/docs/projectplan/BACKLOG.md +++ b/docs/projectplan/BACKLOG.md @@ -74,11 +74,6 @@ None open. - **Where**: `templates/base.html` - **Source**: Documentation refresh 2026-11-03 -### Backfeed responses to crossposts -- **Description**: Bring likes, reposts, quotes and replies to StarPunk's own crossposts back to the note as responses. Decided in ADR-073 (Proposed): native polling of GoToSocial, Bluesky and Threads into the `webmentions` table, Threads likes/reposts/quotes as counts, local avatar cache, admin backfill. Needs new GoToSocial and Threads tokens with read scopes. -- **Where**: `starpunk/syndication.py`, `starpunk/webmention_*.py`, `templates/partials/responses.html`, `starpunk/routes/admin.py` -- **Source**: v2.8.0 microformats2 review discussion; ADR-073 (2026-11-10) - ### Tag-filtered feeds - **Description**: Tag pages are h-feeds and WebSub topics, but there are no RSS, Atom or JSON feeds per tag (for example `/tag/python/feed.atom`). Tag pages also have no pagination, and the editor has no tag autocomplete. - **Where**: `starpunk/routes/public.py`, `starpunk/feeds/`, `templates/admin/` diff --git a/docs/standards/versioning-strategy.md b/docs/standards/versioning-strategy.md index 425204d..4261cff 100644 --- a/docs/standards/versioning-strategy.md +++ b/docs/standards/versioning-strategy.md @@ -4,7 +4,7 @@ **Last Updated**: 2026-11-03 (v2.10.1) StarPunk uses [Semantic Versioning 2.0.0](https://semver.org/): -`MAJOR.MINOR.PATCH`, currently **2.12.1**. Every release is a git tag +`MAJOR.MINOR.PATCH`, currently **2.13.0**. Every release is a git tag `vX.Y.Z` and a `CHANGELOG.md` entry. Where the number is stored and how to write the changelog entry is in [version-implementation-guide.md](version-implementation-guide.md); the diff --git a/migrations/020_backfeed.sql b/migrations/020_backfeed.sql new file mode 100644 index 0000000..952a414 --- /dev/null +++ b/migrations/020_backfeed.sql @@ -0,0 +1,66 @@ +-- Migration 020: Backfeed of responses to crossposts (v2.13.0, ADR-073) +-- +-- webmentions: where a response came from (origin, remote_id) and a new +-- 'quote' type. SQLite can't change a CHECK constraint, so the table is +-- rebuilt (as in migration 003). Nothing references webmentions. +-- +-- syndications: when each copy's responses were last checked and are next +-- due, a lease so one gunicorn worker checks a copy, and Threads counts. + +CREATE TABLE webmentions_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_url TEXT NOT NULL, + target_url TEXT NOT NULL, + target_note_id INTEGER, + mention_type TEXT NOT NULL DEFAULT 'mention' + CHECK (mention_type IN ('reply', 'like', 'repost', 'bookmark', 'mention', 'quote')), + author_name TEXT, + author_url TEXT, + author_photo TEXT, + content_snippet TEXT, + validation_status TEXT NOT NULL DEFAULT 'pending' + CHECK (validation_status IN ('pending', 'valid', 'invalid', 'error')), + moderation_status TEXT NOT NULL DEFAULT 'pending' + CHECK (moderation_status IN ('pending', 'approved', 'hidden', 'deleted')), + validation_error TEXT, + validated_at TIMESTAMP, + received_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + source_post_url TEXT, + source_published TIMESTAMP, + origin TEXT NOT NULL DEFAULT 'webmention', + remote_id TEXT, + FOREIGN KEY (target_note_id) REFERENCES notes(id) ON DELETE SET NULL, + UNIQUE(source_url, target_url) +); + +INSERT INTO webmentions_new ( + id, source_url, target_url, target_note_id, mention_type, author_name, author_url, + author_photo, content_snippet, validation_status, moderation_status, validation_error, + validated_at, received_at, updated_at, source_post_url, source_published +) +SELECT + id, source_url, target_url, target_note_id, mention_type, author_name, author_url, + author_photo, content_snippet, validation_status, moderation_status, validation_error, + validated_at, received_at, updated_at, source_post_url, source_published +FROM webmentions; + +DROP TABLE webmentions; + +ALTER TABLE webmentions_new RENAME TO webmentions; + +CREATE INDEX IF NOT EXISTS idx_webmentions_target_note ON webmentions(target_note_id); +CREATE INDEX IF NOT EXISTS idx_webmentions_moderation ON webmentions(moderation_status); +CREATE INDEX IF NOT EXISTS idx_webmentions_validation ON webmentions(validation_status); +CREATE INDEX IF NOT EXISTS idx_webmentions_received ON webmentions(received_at DESC); +CREATE INDEX IF NOT EXISTS idx_webmentions_note_origin ON webmentions(target_note_id, origin); + +ALTER TABLE syndications ADD COLUMN backfeed_next_at TIMESTAMP; +ALTER TABLE syndications ADD COLUMN backfeed_checked_at TIMESTAMP; +ALTER TABLE syndications ADD COLUMN backfeed_claimed_until TIMESTAMP; +ALTER TABLE syndications ADD COLUMN backfeed_attempts INTEGER NOT NULL DEFAULT 0; +ALTER TABLE syndications ADD COLUMN backfeed_error TEXT; +ALTER TABLE syndications ADD COLUMN backfeed_backfill INTEGER NOT NULL DEFAULT 0; +ALTER TABLE syndications ADD COLUMN likes_count INTEGER; +ALTER TABLE syndications ADD COLUMN reposts_count INTEGER; +ALTER TABLE syndications ADD COLUMN quotes_count INTEGER; diff --git a/starpunk/__init__.py b/starpunk/__init__.py index 207218f..3541b3c 100644 --- a/starpunk/__init__.py +++ b/starpunk/__init__.py @@ -217,6 +217,14 @@ def create_app(config=None): from starpunk.utils import iso_utc app.add_template_filter(iso_utc, 'iso_utc') + # Template filter - local copy of a response author's avatar, or None (v2.13.0) + @app.template_filter('local_avatar') + def local_avatar_filter(url): + from flask import url_for + from starpunk.avatars import cached_name + name = cached_name(app.config.get('DATA_PATH', 'data'), url) + return url_for('public.avatar', name=name) if name else None + # Template filter - URL to color for facepile (v1.6.0) @app.template_filter('url_to_color') def url_to_color_filter(url): @@ -457,5 +465,5 @@ def create_app(config=None): # Package version (Semantic Versioning 2.0.0) # See docs/standards/versioning-strategy.md for details -__version__ = "2.12.1" -__version_info__ = (2, 12, 1) +__version__ = "2.13.0" +__version_info__ = (2, 13, 0) diff --git a/starpunk/avatars.py b/starpunk/avatars.py new file mode 100644 index 0000000..e1fa719 --- /dev/null +++ b/starpunk/avatars.py @@ -0,0 +1,144 @@ +""" +Local copies of response authors' avatars (v2.13.0, ADR-073 Decision 6) + +Pages used to show other sites' avatar URLs directly, which tells those +sites who is reading, and Bluesky and Threads avatar URLs change or expire. +Each avatar is fetched once (SSRF-safe), shrunk to a 96 px WebP and saved +as DATA_PATH/avatars/.webp. Pages show the saved copy, +or the coloured initial until there is one. + +Paths and URLs are passed in, so the webmention worker can use it. Fetching +needs an app context (the SSRF-safe fetcher reads DEV_MODE). +""" + +import hashlib +import io +import logging +import re +import sqlite3 +import time +from pathlib import Path +from typing import Optional + +logger = logging.getLogger(__name__) + +SIZE = 96 +MAX_BYTES = 1024 * 1024 +NAME_RE = re.compile(r"^[0-9a-f]{64}\.webp$") +CLEANUP_EVERY = 7 * 24 * 3600 + + +def name_for(url: str) -> str: + return hashlib.sha256(url.encode()).hexdigest() + ".webp" + + +def folder(data_path) -> Path: + return Path(data_path) / "avatars" + + +def cached_name(data_path, url: Optional[str]) -> Optional[str]: + """File name of the saved copy of this avatar, if there is one""" + if not url or not url.startswith(("http://", "https://")): + return None + name = name_for(url) + return name if (folder(data_path) / name).is_file() else None + + +def fetch(data_path, url: Optional[str], site_url: str) -> bool: + """ + Save a copy of an avatar unless there is one. Returns True if one exists + afterwards. Failures are logged and the page keeps showing the initial. + """ + if not url or not url.startswith(("http://", "https://")): + return False + path = folder(data_path) / name_for(url) + if path.is_file(): + return True + from PIL import Image + + from starpunk.webmention_ssrf import safe_get, user_agent + + try: + result = safe_get(url, headers={"User-Agent": user_agent(site_url)}, + max_bytes=MAX_BYTES, max_redirects=5, timeout=10.0, truncate=False) + if result.status_code >= 400: + raise ValueError(f"HTTP {result.status_code}") + with Image.open(io.BytesIO(result.content)) as image: + image.load() + image = image.convert("RGBA" if "A" in image.getbands() else "RGB") + side = min(image.size) + left, top = (image.width - side) // 2, (image.height - side) // 2 + image = image.crop((left, top, left + side, top + side)) + image = image.resize((SIZE, SIZE), Image.LANCZOS) + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(".tmp") + image.save(tmp, "WEBP", quality=80) + tmp.replace(path) + return True + except Exception as e: + logger.info(f"Could not save avatar {url}: {type(e).__name__}: {e}") + return False + + +def fill_missing(db_path, data_path, site_url: str, limit: int = 20) -> int: + """Save avatars of shown responses that have no copy yet. Returns how many were tried.""" + conn = sqlite3.connect(db_path) + try: + urls = [row[0] for row in conn.execute( + """ + SELECT DISTINCT author_photo FROM webmentions + WHERE author_photo IS NOT NULL AND author_photo != '' + AND validation_status = 'valid' AND moderation_status = 'approved' + ORDER BY id DESC + """ + )] + finally: + conn.close() + tried = 0 + for url in urls: + if tried >= limit: + break + if cached_name(data_path, url) or not url.startswith(("http://", "https://")): + continue + fetch(data_path, url, site_url) + tried += 1 + return tried + + +def remove_unused(db_path, data_path) -> int: + """Delete saved avatars no stored response uses. Returns how many were removed.""" + directory = folder(data_path) + if not directory.is_dir(): + return 0 + conn = sqlite3.connect(db_path) + try: + used = {name_for(row[0]) for row in conn.execute( + "SELECT DISTINCT author_photo FROM webmentions WHERE author_photo IS NOT NULL")} + finally: + conn.close() + removed = 0 + for path in directory.iterdir(): + if NAME_RE.match(path.name) and path.name not in used: + path.unlink(missing_ok=True) + removed += 1 + return removed + + +class Upkeep: + """Runs fill_missing every `fill_every` seconds and remove_unused weekly""" + + def __init__(self, fill_every: int = 600): + self.fill_every = fill_every + self._filled_at = 0.0 + self._cleaned_at = 0.0 + + def run(self, db_path, data_path, site_url: str) -> None: + now = time.time() + if now - self._filled_at >= self.fill_every: + self._filled_at = now + fill_missing(db_path, data_path, site_url) + if now - self._cleaned_at >= CLEANUP_EVERY: + self._cleaned_at = now + removed = remove_unused(db_path, data_path) + if removed: + logger.info(f"Removed {removed} unused avatars") diff --git a/starpunk/backfeed.py b/starpunk/backfeed.py new file mode 100644 index 0000000..899b212 --- /dev/null +++ b/starpunk/backfeed.py @@ -0,0 +1,447 @@ +""" +Backfeed of responses to crossposts (v2.13.0, ADR-073) + +StarPunk asks each silo for the responses to its own copies of a note +(likes, reposts, quotes and replies) and stores them in `webmentions`, so +moderation, the Responses section and re-sending webmentions when a note's +responses change (spec 3.1.4) work as they do for received webmentions. + +Each synced copy of a published note is checked on a schedule that slows as +the note ages, and not at all after BACKFEED_MAX_AGE_DAYS. A backfill +checks every copy once, whatever its age. Checks run in the syndication +worker's loop (`process_due`), one copy at a time under a lease. + +A check that read the whole list marks responses that are no longer there +as invalid, so withdrawn likes and deleted replies drop off the page. +""" + +import html +import random +import re +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Dict, List, Optional + +from flask import current_app + +from starpunk.database import get_db + +DAY = 24 * 3600 +SCHEDULE = [ # (note younger than, check every), in seconds + (DAY, 15 * 60), + (7 * DAY, 3600), + (30 * DAY, DAY), +] +LATE_INTERVAL = 7 * DAY # from 30 days until the cutoff +CLAIM_MINUTES = 5 +MAX_ATTEMPTS = 8 +BACKFILL_GAP = 5 # seconds between backfill checks, across workers +SILO_NAMES = {"gotosocial": "GoToSocial", "bluesky": "Bluesky", "threads": "Threads"} +OPT_OUT_RE = re.compile(r"#no(bridge|bot)\b", re.IGNORECASE) + +# Silos whose credential was refused this run: target id -> message +denied: Dict[str, str] = {} + + +@dataclass +class Response: + """One like, repost, quote or reply on a silo""" + kind: str # like, repost, quote, reply + source_url: str # its URL (likes/reposts: the copy + #liked-by-) + remote_id: str # silo id of the response, or of the actor + author_id: str = "" + author_name: str = "" + author_url: str = "" + author_photo: Optional[str] = None + author_bio: str = "" + opted_out: bool = False # set by the client from silo-specific signals + content_html: Optional[str] = None + published: Optional[str] = None + + +@dataclass +class Harvest: + """What one check found. complete=False: some list was cut short.""" + responses: List[Response] + complete: bool = True + counts: Optional[Dict[str, int]] = None # Threads likes/reposts/quotes + + +# --- Text ----------------------------------------------------------------- + +def text_html(text: str) -> str: + """Plain text as HTML: escaped, URLs linked, line breaks kept""" + escaped = html.escape(text) + linked = re.sub(r"https?://[^\s<>&\"]+", + lambda m: f'{m.group(0)}', escaped) + return "

" + linked.replace("\n", "
") + "

" if text.strip() else "" + + +def facets_html(text: str, facets: list) -> str: + """Bluesky post text as HTML, with its link facets (byte ranges) as links""" + data = text.encode() + links = [] + for facet in facets: + index = facet.get("index") or {} + for feature in facet.get("features") or []: + uri = feature.get("uri") or "" + if feature.get("$type") == "app.bsky.richtext.facet#link" and uri.startswith(("http://", "https://")): + links.append((index.get("byteStart", 0), index.get("byteEnd", 0), uri)) + links.sort() + parts, pos = [], 0 + for start, end, uri in links: + if start < pos or end > len(data) or start >= end: + continue + parts.append(html.escape(data[pos:start].decode(errors="ignore"))) + label = html.escape(data[start:end].decode(errors="ignore")) + parts.append(f'{label}') + pos = end + parts.append(html.escape(data[pos:].decode(errors="ignore"))) + body = "".join(parts) + return "

" + body.replace("\n", "
") + "

" if text.strip() else "" + + +# --- Scheduling -------------------------------------------------------------- + +def interval_for(age_seconds: float, max_age_days: int) -> Optional[int]: + """Seconds until a note of this age is checked again; None past the cutoff""" + if age_seconds >= max_age_days * DAY: + return None + for younger_than, every in SCHEDULE: + if age_seconds < younger_than: + return every + return LATE_INTERVAL + + +def _age_seconds(created_at) -> float: + if isinstance(created_at, str): + created_at = datetime.fromisoformat(created_at.replace("Z", "+00:00")) + if created_at.tzinfo is None: + created_at = created_at.replace(tzinfo=timezone.utc) + return (datetime.now(timezone.utc) - created_at).total_seconds() + + +_ELIGIBLE = """ + FROM syndications s JOIN notes n ON n.id = s.note_id + WHERE s.state = 'synced' AND s.enabled = 1 AND s.remote_id IS NOT NULL + AND n.published = 1 AND n.deleted_at IS NULL + AND n.like_of IS NULL AND n.repost_of IS NULL +""" + + +def _claim(db, row_id: int) -> bool: + cursor = db.execute( + f""" + UPDATE syndications SET backfeed_claimed_until = datetime('now', '+{CLAIM_MINUTES} minutes') + WHERE id = ? AND (backfeed_claimed_until IS NULL OR backfeed_claimed_until < datetime('now')) + """, + (row_id,), + ) + db.commit() + return cursor.rowcount == 1 + + +def _backfill_turn(db) -> bool: + """Take the next backfill slot: at most one check every BACKFILL_GAP seconds""" + cursor = db.execute( + f""" + UPDATE site_state SET value = datetime('now') + WHERE key = 'backfeed_backfill_last' AND value <= datetime('now', '-{BACKFILL_GAP} seconds') + """ + ) + if cursor.rowcount == 0: + cursor = db.execute( + "INSERT OR IGNORE INTO site_state (key, value) VALUES ('backfeed_backfill_last', datetime('now'))") + db.commit() + return cursor.rowcount == 1 + + +def _due(db, limit: int) -> list: + """Routine checks due now; a backfill check only when none are""" + max_age = int(current_app.config.get("BACKFEED_MAX_AGE_DAYS", 90)) + lease = "(s.backfeed_claimed_until IS NULL OR s.backfeed_claimed_until < datetime('now'))" + rows = db.execute( + f""" + SELECT s.*, n.created_at AS note_created, n.slug AS note_slug {_ELIGIBLE} + AND {lease} + AND n.created_at >= datetime('now', ?) + AND (s.backfeed_next_at IS NULL OR s.backfeed_next_at <= datetime('now')) + ORDER BY s.backfeed_next_at IS NOT NULL, s.backfeed_next_at LIMIT ? + """, + (f"-{max_age} days", limit), + ).fetchall() + if rows: + return rows + backfill = db.execute( + f""" + SELECT s.*, n.created_at AS note_created, n.slug AS note_slug {_ELIGIBLE} + AND {lease} AND s.backfeed_backfill = 1 + AND (s.backfeed_next_at IS NULL OR s.backfeed_next_at <= datetime('now')) + ORDER BY s.id LIMIT 1 + """ + ).fetchall() + if backfill and _backfill_turn(db): + return backfill + return [] + + +def _finish(db, row) -> None: + max_age = int(current_app.config.get("BACKFEED_MAX_AGE_DAYS", 90)) + every = interval_for(_age_seconds(row["note_created"]), max_age) + if every is None: + next_at = None + else: + next_at = f"+{int(every * random.uniform(1.0, 1.1))} seconds" + db.execute( + """ + UPDATE syndications + SET backfeed_checked_at = datetime('now'), backfeed_claimed_until = NULL, + backfeed_attempts = 0, backfeed_error = NULL, backfeed_backfill = 0, + backfeed_next_at = CASE WHEN ? IS NULL THEN NULL ELSE datetime('now', ?) END + WHERE id = ? + """, + (next_at, next_at, row["id"]), + ) + db.commit() + + +def _fail(db, row, message: str) -> None: + """Back off; after MAX_ATTEMPTS drop any backfill and try again in a month""" + attempts = row["backfeed_attempts"] + 1 + stop = attempts >= MAX_ATTEMPTS + delay = 30 * DAY if stop else min(30 * (2 ** attempts), 6 * 3600) + db.execute( + f""" + UPDATE syndications + SET backfeed_attempts = ?, backfeed_error = ?, backfeed_claimed_until = NULL, + backfeed_backfill = CASE WHEN ? THEN 0 ELSE backfeed_backfill END, + backfeed_next_at = datetime('now', '+{delay} seconds') + WHERE id = ? + """, + (attempts, message[:500], 1 if stop else 0, row["id"]), + ) + db.commit() + current_app.logger.warning( + f"Backfeed {row['target']} for note {row['note_id']} failed (attempt {attempts}): {message}") + + +# --- Storing ----------------------------------------------------------------- + +def _opted_out(response: Response) -> bool: + return response.opted_out or bool(OPT_OUT_RE.search(html.unescape(response.author_bio or ""))) + + +def _display_state(db, note_id: int) -> list: + return [tuple(r) for r in db.execute( + """ + SELECT id, mention_type, author_name, author_url, author_photo, content_snippet + FROM webmentions + WHERE target_note_id = ? AND validation_status = 'valid' AND moderation_status = 'approved' + ORDER BY id + """, + (note_id,), + )] + + +def store(db, note_id: int, target_url: str, origin: str, harvest: Harvest) -> None: + """ + Upsert a check's responses for one note and silo. With a complete + harvest, that silo's responses for the note that weren't found are + marked invalid. Responses from people who opted out are removed. + """ + from starpunk.webmention_processor import _snippet + + moderation = "approved" if current_app.config.get("WEBMENTION_AUTO_APPROVE", True) else "pending" + silo = SILO_NAMES.get(origin, origin) + keep = [r for r in harvest.responses if not _opted_out(r)] + dropped = [r.source_url for r in harvest.responses if _opted_out(r)] + + before = _display_state(db, note_id) + for r in keep: + snippet = _snippet(r.content_html) if r.content_html else None + db.execute( + """ + INSERT INTO webmentions ( + source_url, target_url, target_note_id, mention_type, author_name, author_url, + author_photo, content_snippet, validation_status, moderation_status, + validated_at, source_published, origin, remote_id) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'valid', ?, CURRENT_TIMESTAMP, ?, ?, ?) + ON CONFLICT(source_url, target_url) DO UPDATE SET + target_note_id = excluded.target_note_id, + mention_type = excluded.mention_type, + author_name = excluded.author_name, + author_url = excluded.author_url, + author_photo = excluded.author_photo, + content_snippet = excluded.content_snippet, + source_published = COALESCE(excluded.source_published, webmentions.source_published), + origin = excluded.origin, + remote_id = excluded.remote_id, + validation_status = 'valid', + validation_error = NULL, + validated_at = CURRENT_TIMESTAMP, + updated_at = CURRENT_TIMESTAMP + """, + (r.source_url, target_url, note_id, r.kind, r.author_name or None, r.author_url or None, + r.author_photo or None, snippet, moderation, r.published, origin, r.remote_id), + ) + if dropped: + marks = ",".join("?" * len(dropped)) + db.execute(f"DELETE FROM webmentions WHERE origin = ? AND target_note_id = ? AND source_url IN ({marks})", + [origin, note_id] + dropped) + if harvest.complete: + found = [r.source_url for r in keep] + marks = ",".join("?" * len(found)) or "''" + db.execute( + f""" + UPDATE webmentions + SET validation_status = 'invalid', validation_error = ?, updated_at = CURRENT_TIMESTAMP + WHERE target_note_id = ? AND origin = ? AND validation_status = 'valid' + AND source_url NOT IN ({marks}) + """, + [f"No longer on {silo}", note_id, origin] + found, + ) + db.commit() + + if _display_state(db, note_id) != before: + try: + from pathlib import Path + + from starpunk.webmention_sender import note_responses_changed + note_responses_changed(Path(current_app.config["DATABASE_PATH"]), [note_id]) + except Exception as e: + current_app.logger.warning(f"Failed to re-send webmentions for note {note_id}: {e}") + + from starpunk import avatars + for r in keep: + avatars.fetch(current_app.config["DATA_PATH"], r.author_photo, + current_app.config.get("SITE_URL", "")) + + +def _store_counts(db, row_id: int, counts: Optional[dict]) -> None: + if counts is None: + return + db.execute( + "UPDATE syndications SET likes_count = ?, reposts_count = ?, quotes_count = ? WHERE id = ?", + (counts.get("likes"), counts.get("reposts"), counts.get("quotes"), row_id), + ) + db.commit() + + +# --- Checking --------------------------------------------------------------- + +def _is_denied(error) -> bool: + """The silo refused the credential (missing scope or bad token)""" + from starpunk.syndication import _GRAPH_AUTH + + return getattr(error, "status", None) in (401, 403) or getattr(error, "code", None) in _GRAPH_AUTH + + +SCOPES = { + "gotosocial": "a GoToSocial token with read:statuses and read:accounts (or read)", + "bluesky": "a working Bluesky app password", + "threads": "a Threads token with threads_read_replies and threads_manage_insights", +} + + +def check(row, target) -> None: + """Check one copy now and store what it found (raises SyndicationError)""" + db = get_db(current_app) + site = current_app.config["SITE_URL"].rstrip("/") + harvest = target.client.responses(row["remote_id"], row["remote_url"] or "") + store(db, row["note_id"], f"{site}/note/{row['note_slug']}", row["target"], harvest) + _store_counts(db, row["id"], harvest.counts) + + +def process_due(limit: int = 3) -> int: + """Check up to `limit` copies that are due. Returns how many were handled.""" + if not current_app.config.get("BACKFEED_ENABLED", False): + return 0 + from starpunk.syndication import SyndicationError, get_targets + + targets = get_targets() + db = get_db(current_app) + handled = 0 + for row in _due(db, limit): + if row["target"] in denied or row["target"] not in targets: + continue + if not _claim(db, row["id"]): + continue + handled += 1 + try: + check(row, targets[row["target"]]) + except SyndicationError as e: + if _is_denied(e): + denied[row["target"]] = f"Needs {SCOPES.get(row['target'], 'read access')}: {e}" + current_app.logger.error(f"Backfeed for {row['target']} stopped: {denied[row['target']]}") + db.execute("UPDATE syndications SET backfeed_claimed_until = NULL WHERE id = ?", (row["id"],)) + db.commit() + continue + _fail(db, row, str(e)) + continue + except Exception as e: # one bad copy mustn't stop the worker + _fail(db, row, f"unexpected error: {e}") + continue + _finish(db, row) + _note_backfill_done(db) + return handled + + +# --- Backfill and admin --------------------------------------------------------- + +def _state(db, key: str) -> Optional[str]: + row = db.execute("SELECT value FROM site_state WHERE key = ?", (key,)).fetchone() + return row["value"] if row and row["value"] else None + + +def _set_state(db, key: str, value: str) -> None: + db.execute("INSERT INTO site_state (key, value) VALUES (?, ?) " + "ON CONFLICT(key) DO UPDATE SET value = excluded.value", (key, value)) + + +def start_backfill() -> int: + """Mark every eligible copy for one check, whatever its age. Returns how many.""" + db = get_db(current_app) + cursor = db.execute( + f""" + UPDATE syndications + SET backfeed_backfill = 1, backfeed_attempts = 0, backfeed_next_at = datetime('now') + WHERE id IN (SELECT s.id {_ELIGIBLE}) + """ + ) + now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S") + _set_state(db, "backfeed_backfill_started", now) + _set_state(db, "backfeed_backfill_finished", "") + db.commit() + return cursor.rowcount + + +def _note_backfill_done(db) -> None: + if _state(db, "backfeed_backfill_started") and not _state(db, "backfeed_backfill_finished"): + left = db.execute("SELECT COUNT(*) FROM syndications WHERE backfeed_backfill = 1").fetchone()[0] + if left == 0: + _set_state(db, "backfeed_backfill_finished", + datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S")) + db.commit() + + +def check_now(syndication_id: int) -> None: + """Check this copy's responses at the next worker pass""" + db = get_db(current_app) + db.execute("UPDATE syndications SET backfeed_next_at = datetime('now'), backfeed_attempts = 0 " + "WHERE id = ?", (syndication_id,)) + db.commit() + + +def status() -> dict: + """For the admin pages: backfill progress and silos whose credential was refused""" + db = get_db(current_app) + return { + "enabled": bool(current_app.config.get("BACKFEED_ENABLED", False)), + "started": _state(db, "backfeed_backfill_started"), + "finished": _state(db, "backfeed_backfill_finished"), + "remaining": db.execute( + "SELECT COUNT(*) FROM syndications WHERE backfeed_backfill = 1").fetchone()[0], + "denied": dict(denied), + "errors": db.execute( + "SELECT COUNT(*) FROM syndications WHERE backfeed_error IS NOT NULL").fetchone()[0], + } diff --git a/starpunk/config.py b/starpunk/config.py index 04a5efa..adbaa74 100644 --- a/starpunk/config.py +++ b/starpunk/config.py @@ -146,6 +146,9 @@ def load_config(app, config_override=None): app.config["THREADS_MAX_CHARS"] = int(os.getenv("THREADS_MAX_CHARS", "500")) app.config["THREADS_MAX_MEDIA"] = int(os.getenv("THREADS_MAX_MEDIA", "4")) app.config["SYNDICATION_POLL_INTERVAL"] = int(os.getenv("SYNDICATION_POLL_INTERVAL", "10")) + # Backfeed of responses to crossposts (v2.13.0, ADR-073) + app.config["BACKFEED_ENABLED"] = os.getenv("BACKFEED_ENABLED", "false").lower() == "true" + app.config["BACKFEED_MAX_AGE_DAYS"] = int(os.getenv("BACKFEED_MAX_AGE_DAYS", "90")) # Apply overrides if provided # CSRF protection for admin and consent forms (v2.11.0); on by default. diff --git a/starpunk/routes/admin.py b/starpunk/routes/admin.py index 7201073..e0997e1 100644 --- a/starpunk/routes/admin.py +++ b/starpunk/routes/admin.py @@ -5,7 +5,7 @@ Handles authenticated admin functionality including dashboard, note creation, editing, and deletion. All routes require authentication. """ -from flask import Blueprint, flash, g, jsonify, redirect, render_template, request, url_for +from flask import Blueprint, current_app, flash, g, jsonify, redirect, render_template, request, url_for import logging import os import shutil @@ -259,6 +259,7 @@ def edit_note_form(note_id: int): syndication_targets=get_targets().values(), syndications=syndications, syndication_enabled=enabled, + backfeed_enabled=current_app.config.get("BACKFEED_ENABLED", False), ) @@ -496,18 +497,24 @@ def webmentions(): from starpunk.webmention_queries import list_webmentions status_filter = request.args.get('status') + origin_filter = request.args.get('origin') or None page = int(request.args.get('page', 1)) webmentions, total_pages = list_webmentions( status_filter=status_filter, page=page, - per_page=25 + per_page=25, + origin_filter=origin_filter, ) + from starpunk import backfeed + return render_template( "admin/webmentions.html", webmentions=webmentions, status_filter=status_filter, + origin_filter=origin_filter, + backfeed=backfeed.status(), page=page, total_pages=total_pages, user_me=g.me @@ -674,6 +681,28 @@ def revoke_token(token_id): return redirect(url_for("admin.tokens")) +@bp.route("/backfeed/backfill", methods=["POST"]) +@require_auth +def backfeed_backfill(): + """Check every crosspost once for responses, whatever its age (v2.13.0)""" + from starpunk import backfeed + + count = backfeed.start_backfill() + flash(f"Backfill started: {count} crosspost(s) to check", "success") + return redirect(url_for("admin.webmentions")) + + +@bp.route("/syndication//check", methods=["POST"]) +@require_auth +def check_syndication_responses(syndication_id): + """Check one crosspost's responses at the next worker pass (v2.13.0)""" + from starpunk import backfeed + + backfeed.check_now(syndication_id) + flash("Responses will be checked shortly", "success") + return redirect(request.referrer or url_for("admin.dashboard")) + + @bp.route("/syndication//retry", methods=["POST"]) @require_auth def retry_syndication(syndication_id): diff --git a/starpunk/routes/public.py b/starpunk/routes/public.py index 5cf4726..22a8db4 100644 --- a/starpunk/routes/public.py +++ b/starpunk/routes/public.py @@ -250,6 +250,20 @@ def media_file(path): return response +@bp.route("/avatars/") +def avatar(name: str): + """A saved copy of a response author's avatar (v2.13.0, ADR-073)""" + from pathlib import Path + from starpunk.avatars import NAME_RE, folder + + if not NAME_RE.match(name): + abort(404) + response = send_from_directory(folder(Path(current_app.config.get('DATA_PATH', 'data'))), name) + # Named by the hash of the original URL; a changed avatar gets a new name + response.headers['Cache-Control'] = 'public, max-age=604800' + return response + + @bp.route("/") def index(): """ diff --git a/starpunk/routes/webmention.py b/starpunk/routes/webmention.py index 5342953..c320571 100644 --- a/starpunk/routes/webmention.py +++ b/starpunk/routes/webmention.py @@ -258,7 +258,10 @@ def receive_webmention(): ON CONFLICT(source_url, target_url) DO UPDATE SET validation_status = 'pending', updated_at = CURRENT_TIMESTAMP + WHERE webmentions.origin = 'webmention' """, (source, target, target_note_id)) + # (A backfed response with the same URLs is StarPunk's own copy of + # what the silo said; a webmention doesn't overwrite it. v2.13.0) db.commit() current_app.logger.info( diff --git a/starpunk/syndication.py b/starpunk/syndication.py index 5ab9743..1a2aff8 100644 --- a/starpunk/syndication.py +++ b/starpunk/syndication.py @@ -67,8 +67,9 @@ class MastodonClient: def _call(self, method: str, path: str, **kwargs) -> httpx.Response: headers = {**self.headers, **kwargs.pop("headers", {})} + url = path if path.startswith(("http://", "https://")) else self.base_url + path try: - response = self.http.request(method, self.base_url + path, headers=headers, **kwargs) + response = self.http.request(method, url, headers=headers, **kwargs) except httpx.HTTPError as e: raise SyndicationError(f"{method} {path}: {e}") if response.status_code == 404: @@ -76,10 +77,12 @@ class MastodonClient: if response.status_code >= 400: # 4xx other than rate limiting won't fix itself permanent = 400 <= response.status_code < 500 and response.status_code != 429 - raise SyndicationError( + exc = SyndicationError( f"{method} {path}: HTTP {response.status_code} {response.text[:200]}", permanent=permanent, ) + exc.status = response.status_code + raise exc return response def upload_media(self, path: Path, mime_type: str, description: str) -> str: @@ -166,6 +169,61 @@ class MastodonClient: text, media_ids = self._prepare(post) return self.update_status(remote_id, text, media_ids) + # --- Backfeed (v2.13.0, ADR-073) --- + + def _account_id(self) -> str: + """Id of the account StarPunk posts as (its own responses are skipped)""" + if not getattr(self, "account_id", None): + self.account_id = self._call("GET", "/api/v1/accounts/verify_credentials").json()["id"] + return self.account_id + + def _list(self, path: str, cap: int) -> tuple: + """Items of a paginated list (Link: rel="next"). Returns (items, complete).""" + items, url, params = [], path, {"limit": 80} + while url: + response = self._call("GET", url, params=params) + items.extend(response.json() or []) + if len(items) >= cap: + return items[:cap], False + url, params = (response.links.get("next") or {}).get("url"), None + return items, True + + @staticmethod + def _author(account: dict) -> dict: + return { + "author_id": account.get("id") or "", + "author_name": account.get("display_name") or account.get("username") or account.get("acct") or "", + "author_url": account.get("url") or "", + "author_photo": account.get("avatar_static") or account.get("avatar"), + "author_bio": account.get("note") or "", + } + + def responses(self, remote_id: str, remote_url: str, cap: int = 1000): + """Likes, boosts and public direct replies of one of our statuses""" + from starpunk.backfeed import Harvest, Response + + me = self._account_id() + found, complete = [], True + for kind, path, fragment in (("like", "favourited_by", "liked-by"), + ("repost", "reblogged_by", "reposted-by")): + accounts, done = self._list(f"/api/v1/statuses/{remote_id}/{path}", cap) + complete = complete and done + for account in accounts: + if account.get("id") == me: + continue + found.append(Response(kind=kind, source_url=f"{remote_url}#{fragment}-{account['id']}", + remote_id=account["id"], **self._author(account))) + context = self._call("GET", f"/api/v1/statuses/{remote_id}/context").json() or {} + for status in context.get("descendants") or []: + account = status.get("account") or {} + if (status.get("in_reply_to_id") != remote_id or status.get("visibility") != "public" + or account.get("id") == me): + continue + found.append(Response(kind="reply", source_url=status.get("url") or status["uri"], + remote_id=status["id"], content_html=status.get("content") or "", + published=status.get("created_at"), **self._author(account))) + return Harvest(found, complete) + def delete(self, remote_id: str) -> None: action, _, status_id = remote_id.partition(":") if action in self.NATIVE and status_id: @@ -227,9 +285,11 @@ class BlueskyClient: if response.status_code < 400: return permanent = 400 <= response.status_code < 500 and response.status_code != 429 - raise SyndicationError( + exc = SyndicationError( f"{nsid}: HTTP {response.status_code} {response.text[:200]}", permanent=permanent ) + exc.status = response.status_code + raise exc def _login(self) -> None: response = None @@ -405,6 +465,94 @@ class BlueskyClient: def update(self, remote_id: str, post: "Post") -> dict: raise NotImplementedError("Bluesky posts can't be edited") + # --- Backfeed (v2.13.0, ADR-073) --- + + def _paged(self, nsid: str, params: dict, key: str, cap: int) -> tuple: + """Items of a cursor-paginated XRPC list. Returns (items, complete).""" + items, cursor = [], None + while True: + query = {**params, "limit": 100} + if cursor: + query["cursor"] = cursor + data = self._call("GET", nsid, params=query).json() + page = data.get(key) or [] + items.extend(page) + if len(items) >= cap: + return items[:cap], False + cursor = data.get("cursor") + if not cursor or not page: + return items, True + + @staticmethod + def _author(profile: dict) -> dict: + handle = profile.get("handle") or profile.get("did") or "" + return { + "author_id": profile.get("did") or "", + "author_name": profile.get("displayName") or handle, + "author_url": f"https://bsky.app/profile/{handle}", + "author_photo": profile.get("avatar"), + "author_bio": profile.get("description") or "", + # The author asked not to be shown to people who aren't logged in + "opted_out": any(label.get("val") == "!no-unauthenticated" + for label in profile.get("labels") or []), + } + + def _post_response(self, kind: str, view: dict): + from starpunk.backfeed import Response, facets_html + + author = view.get("author") or {} + record = view.get("record") or {} + rkey = view["uri"].rsplit("/", 1)[1] + return Response( + kind=kind, + source_url=f"https://bsky.app/profile/{author.get('handle') or author.get('did')}/post/{rkey}", + remote_id=view["uri"], + content_html=facets_html(record.get("text") or "", record.get("facets") or []), + published=record.get("createdAt") or view.get("indexedAt"), + **self._author(author), + ) + + def responses(self, remote_id: str, remote_url: str, cap: int = 1000): + """Likes, reposts, quotes and direct replies of one of our posts""" + from starpunk.backfeed import Harvest, Response + + if not self.did: + with self._lock: + self._login() + found, complete = [], True + likes, done = self._paged("app.bsky.feed.getLikes", {"uri": remote_id}, "likes", cap) + complete = complete and done + for like in likes: + actor = like.get("actor") or {} + found.append(Response(kind="like", source_url=f"{remote_url}#liked-by-{actor.get('did')}", + remote_id=actor.get("did") or "", published=like.get("createdAt"), + **self._author(actor))) + reposters, done = self._paged("app.bsky.feed.getRepostedBy", {"uri": remote_id}, + "repostedBy", cap) + complete = complete and done + for actor in reposters: + found.append(Response(kind="repost", source_url=f"{remote_url}#reposted-by-{actor.get('did')}", + remote_id=actor.get("did") or "", **self._author(actor))) + quotes, done = self._paged("app.bsky.feed.getQuotes", {"uri": remote_id}, "posts", cap) + complete = complete and done + posts = [self._post_response("quote", view) for view in quotes] + thread = self._call("GET", "app.bsky.feed.getPostThread", + params={"uri": remote_id, "depth": 1, "parentHeight": 0}).json() + for item in (thread.get("thread") or {}).get("replies") or []: + if item.get("post"): # skips blocked and deleted replies + posts.append(self._post_response("reply", item["post"])) + # Post views carry no bio; fetch them to honour #nobridge + dids = list({p.author_id for p in posts if p.author_id}) + bios = {} + for i in range(0, len(dids), 25): + profiles = self._call("GET", "app.bsky.actor.getProfiles", + params={"actors": dids[i:i + 25]}).json().get("profiles") or [] + bios.update({p["did"]: p.get("description") or "" for p in profiles}) + for post in posts: + post.author_bio = bios.get(post.author_id, "") + found.extend(posts) + return Harvest([r for r in found if r.author_id != self.did], complete) + def delete(self, remote_id: str) -> None: # at:///app.bsky.feed.post/; deleting a missing record succeeds repo, collection, rkey = remote_id[len("at://"):].split("/", 2) @@ -513,7 +661,7 @@ def page_card(http: httpx.Client, url: str) -> dict: return card -THREADS_API = "https://graph.threads.net/v1.0" +THREADS_API = "https://graph.threads.com/v1.0" # Meta's current host (v2.13.0) THREADS_IMAGE_TYPES = {"image/jpeg", "image/png"} THREADS_IMAGE_LIMIT = 8_000_000 THREADS_MAX_LINKS = 5 @@ -775,6 +923,57 @@ class ThreadsClient: def update(self, remote_id: str, post: "Post") -> dict: raise NotImplementedError("Threads posts can't be edited through the API") + # --- Backfeed (v2.13.0, ADR-073) --- + + REPLY_FIELDS = "id,text,username,permalink,timestamp,profile_picture_url,hide_status" + SHOWN = (None, "NOT_HUSHED", "UNHUSHED") # hide_status values of replies nobody hid + + def responses(self, remote_id: str, remote_url: str, cap: int = 1000): + """ + Direct replies to one of our posts, and its like/repost/quote counts. + The API doesn't say who liked, reposted or quoted (ADR-073 Decision 7). + """ + from starpunk.backfeed import Harvest, Response, text_html + + self._me() + replies, after, complete = [], None, True + while True: + params = {"fields": self.REPLY_FIELDS} + if after: + params["after"] = after + data = self._call("GET", f"{remote_id}/replies", **params) + page = data.get("data") or [] + replies.extend(page) + if len(replies) >= cap: + replies, complete = replies[:cap], False + break + paging = data.get("paging") or {} + after = (paging.get("cursors") or {}).get("after") + if not page or not after or not paging.get("next"): + break + found = [] + for reply in replies: + username = reply.get("username") + # Private accounts have no username or permalink: nothing public to show + if (not username or not reply.get("permalink") or username == self.username + or reply.get("hide_status") not in self.SHOWN): + continue + found.append(Response( + kind="reply", source_url=reply["permalink"], remote_id=reply["id"], + author_id=username, author_name=username, + author_url=f"https://www.threads.com/@{username}", + author_photo=reply.get("profile_picture_url"), + content_html=text_html(reply.get("text") or ""), + published=reply.get("timestamp"), + )) + insights = self._call("GET", f"{remote_id}/insights", metric="likes,reposts,quotes") + counts = {} + for metric in insights.get("data") or []: + values = metric.get("values") or [{}] + counts[metric.get("name")] = int((values[0] or {}).get("value") or 0) + return Harvest(found, complete, counts={k: counts.get(k, 0) + for k in ("likes", "reposts", "quotes")}) + def delete(self, remote_id: str) -> None: try: self._call("DELETE", remote_id) @@ -1381,12 +1580,19 @@ class SyndicationWorker(threading.Thread): def run(self): while not self._stop_event.is_set(): + handled = 0 try: with self.app.app_context(): handled = process_due() except Exception as e: self.app.logger.error(f"Syndication worker error: {e}", exc_info=True) - handled = 0 + # Responses to the copies (v2.13.0, ADR-073), after any crossposts + try: + from starpunk import backfeed + with self.app.app_context(): + handled += backfeed.process_due() + except Exception as e: + self.app.logger.error(f"Backfeed error: {e}", exc_info=True) if handled == 0: self._stop_event.wait(self.poll_interval) diff --git a/starpunk/webmention_queries.py b/starpunk/webmention_queries.py index 9315b68..f1f1636 100644 --- a/starpunk/webmention_queries.py +++ b/starpunk/webmention_queries.py @@ -39,7 +39,8 @@ def get_webmentions_for_note(note_id: int) -> Dict[str, List[Dict[str, Any]]]: note_id: Note database ID Returns: - Dict with keys: 'replies', 'likes', 'reposts', 'bookmarks', 'mentions' + Dict with keys: 'replies', 'likes', 'reposts', 'bookmarks', 'quotes' + (backfed quote posts, v2.13.0), 'mentions' Each value is a list of webmention dicts """ db = get_db() @@ -56,24 +57,34 @@ def get_webmentions_for_note(note_id: int) -> Dict[str, List[Dict[str, Any]]]: WHEN 'like' THEN 2 WHEN 'repost' THEN 3 WHEN 'bookmark' THEN 4 - WHEN 'mention' THEN 5 + WHEN 'quote' THEN 5 + WHEN 'mention' THEN 6 END, received_at ASC """ rows = db.execute(query, (note_id,)).fetchall() + # Backfed responses say which silo they came from (v2.13.0) + silo_names = {} + if any(row['origin'] != 'webmention' for row in rows): + from starpunk.backfeed import SILO_NAMES + from starpunk.syndication import get_targets + silo_names = {**SILO_NAMES, **{t.id: t.name for t in get_targets().values()}} + # Group by type grouped = { 'replies': [], 'likes': [], 'reposts': [], 'bookmarks': [], + 'quotes': [], 'mentions': [] } for row in rows: wm = dict(row) + wm['via'] = silo_names.get(wm['origin']) if wm['origin'] != 'webmention' else None # Parse timestamps (source_published may be missing or malformed) for key in ('received_at', 'source_published'): if isinstance(wm.get(key), str): @@ -91,6 +102,8 @@ def get_webmentions_for_note(note_id: int) -> Dict[str, List[Dict[str, Any]]]: grouped['reposts'].append(wm) elif mention_type == 'bookmark': grouped['bookmarks'].append(wm) + elif mention_type == 'quote': + grouped['quotes'].append(wm) else: grouped['mentions'].append(wm) @@ -100,15 +113,18 @@ def get_webmentions_for_note(note_id: int) -> Dict[str, List[Dict[str, Any]]]: def list_webmentions( status_filter: Optional[str] = None, page: int = 1, - per_page: int = 25 + per_page: int = 25, + origin_filter: Optional[str] = None, ) -> tuple[List[Dict[str, Any]], int]: """ List webmentions for admin with pagination and filtering. Args: - status_filter: Filter by moderation_status ('pending', 'approved', 'hidden') + status_filter: Filter by moderation_status ('pending', 'approved', 'hidden', + 'deleted'). Deleted ones are left out unless asked for (v2.13.0). page: Page number (1-indexed) per_page: Items per page (default: 25) + origin_filter: 'webmention' or a backfeed silo id (v2.13.0) Returns: Tuple of (webmentions list, total_pages) @@ -124,12 +140,16 @@ def list_webmentions( count_query = "SELECT COUNT(*) FROM webmentions" params = [] - where_clause = "" - + conditions = ["moderation_status = ?" if status_filter else "moderation_status != 'deleted'"] if status_filter: - where_clause = " WHERE w.moderation_status = ?" - count_query += " WHERE moderation_status = ?" params.append(status_filter) + if origin_filter: + conditions.append("origin = ?") + params.append(origin_filter) + where = " AND ".join(conditions) + where_clause = " WHERE " + where.replace("moderation_status", "w.moderation_status").replace( + "origin", "w.origin") + count_query += " WHERE " + where # Count total total_count = db.execute(count_query, params).fetchone()[0] @@ -206,7 +226,7 @@ def count_webmentions(status_filter: Optional[str] = None) -> int: query = "SELECT COUNT(*) FROM webmentions WHERE moderation_status = ?" result = db.execute(query, (status_filter,)).fetchone() else: - query = "SELECT COUNT(*) FROM webmentions" + query = "SELECT COUNT(*) FROM webmentions WHERE moderation_status != 'deleted'" result = db.execute(query).fetchone() return result[0] if result else 0 @@ -260,6 +280,9 @@ def delete_webmentions(webmention_ids: List[int]) -> int: """ Delete webmentions by IDs. + The row stays as a tombstone (moderation_status 'deleted', v2.13.0), so + a re-sent webmention or the next backfeed check doesn't bring it back. + Args: webmention_ids: List of webmention IDs to delete @@ -276,7 +299,8 @@ def delete_webmentions(webmention_ids: List[int]) -> int: ) placeholders = ','.join('?' * len(webmention_ids)) - query = f"DELETE FROM webmentions WHERE id IN ({placeholders})" + query = (f"UPDATE webmentions SET moderation_status = 'deleted', updated_at = CURRENT_TIMESTAMP " + f"WHERE id IN ({placeholders}) AND moderation_status != 'deleted'") cursor = db.execute(query, webmention_ids) db.commit() diff --git a/starpunk/webmention_worker.py b/starpunk/webmention_worker.py index 1042447..6c971d0 100644 --- a/starpunk/webmention_worker.py +++ b/starpunk/webmention_worker.py @@ -82,6 +82,8 @@ class WebmentionWorker(threading.Thread): self._stop_event = threading.Event() self._last_processed = None self._lock_file = None + from starpunk.avatars import Upkeep + self._avatar_upkeep = Upkeep() def run(self): """ @@ -113,6 +115,9 @@ class WebmentionWorker(threading.Thread): # Phase 3: Process outgoing webmentions processed += self._process_pending_sends() + # Local copies of response avatars (v2.13.0) + self._avatar_upkeep.run(self.db_path, self.app.config['DATA_PATH'], + self.site_url) # Update last processed time if processed > 0: @@ -281,6 +286,22 @@ class WebmentionWorker(threading.Thread): source_url, result.get('source_post_url'), result.get('source_uid')) if u} return bool(copies & source) + def _is_backfed(self, note_id: Optional[int], source_url: str, result: Dict[str, Any]) -> bool: + """True if the source is a response already stored by backfeed for this note""" + if note_id is None: + return False + conn = sqlite3.connect(self.db_path) + try: + backfed = {normalize_url(row[0]) for row in conn.execute( + "SELECT source_url FROM webmentions WHERE target_note_id = ? AND origin != 'webmention'", + (note_id,), + )} + finally: + conn.close() + source = {normalize_url(u) for u in ( + source_url, result.get('source_post_url'), result.get('source_uid')) if u} + return bool(backfed & source) + def _display_state(self, webmention_id: int) -> Optional[tuple]: """ What the note page shows for this webmention, or None if hidden @@ -328,6 +349,11 @@ class WebmentionWorker(threading.Thread): if self._is_own_syndicated_copy(target_note_id, source_url, result): raise ValidationError("Source is this note's own syndicated copy") + # A response StarPunk already backfed from the silo (e.g. a reply + # also sent by Bridgy) would show twice (v2.13.0, ADR-073) + if self._is_backfed(target_note_id, source_url, result): + raise ValidationError("Already backfed from the silo") + # Update database with successful validation conn = sqlite3.connect(self.db_path) cursor = conn.cursor() @@ -372,6 +398,10 @@ class WebmentionWorker(threading.Thread): conn.commit() conn.close() + # Pages show a local copy of the avatar (v2.13.0, ADR-073) + from starpunk import avatars + avatars.fetch(self.app.config['DATA_PATH'], result['author_photo'], self.site_url) + logger.info( f"Validated webmention: {source_url} -> {target_url} " f"(type={result['mention_type']})" @@ -422,7 +452,8 @@ class WebmentionWorker(threading.Thread): # All other errors are 'error' status if transient: status = 'error' - elif any(x in error.lower() for x in ['does not link', 'returned 4', 'own syndicated copy']): + elif any(x in error.lower() for x in ['does not link', 'returned 4', 'own syndicated copy', + 'already backfed']): status = 'invalid' else: status = 'error' diff --git a/static/css/style.css b/static/css/style.css index e66179d..d4baf49 100644 --- a/static/css/style.css +++ b/static/css/style.css @@ -343,6 +343,13 @@ small { display: block; margin-top: var(--spacing-xs); color: var(--color-text-l color: var(--color-text-light); } +/* Backfed responses and silo counts (v2.13.0) */ +.webmention-via, +.silo-counts { + font-size: 0.875rem; + color: var(--color-text-light); +} + /* Facepile */ .facepile { display: flex; diff --git a/templates/admin/edit.html b/templates/admin/edit.html index 5c19292..5b7e1b3 100644 --- a/templates/admin/edit.html +++ b/templates/admin/edit.html @@ -102,6 +102,12 @@ {% endif %} + {% if backfeed_enabled and s.state == 'synced' and s.remote_id %} +
Responses {% if s.backfeed_checked_at %}checked {{ s.backfeed_checked_at }} UTC{% else %}not checked yet{% endif %}{% if s.backfeed_error %}: {{ s.backfeed_error }}{% endif %} +
+ +
+ {% endif %} {% endfor %} diff --git a/templates/admin/webmention_detail.html b/templates/admin/webmention_detail.html index 9eabbf8..377dfd9 100644 --- a/templates/admin/webmention_detail.html +++ b/templates/admin/webmention_detail.html @@ -16,6 +16,7 @@
Source: {{ webmention.source_url }} + {% if webmention.origin and webmention.origin != 'webmention' %}(backfed from {{ webmention.origin }}){% endif %}
@@ -43,7 +44,9 @@ {% if webmention.author_photo %}
Author Photo: - {{ webmention.author_name or 'Author' }} + {% set photo = webmention.author_photo|local_avatar %} + {% if photo %}{{ webmention.author_name or 'Author' }}{% endif %} + {{ webmention.author_photo|truncate(60) }}
{% endif %} diff --git a/templates/admin/webmentions.html b/templates/admin/webmentions.html index c93f1a6..323fb25 100644 --- a/templates/admin/webmentions.html +++ b/templates/admin/webmentions.html @@ -12,6 +12,27 @@
+ {# Backfeed (v2.13.0, ADR-073) #} + {% if backfeed and backfeed.enabled %} +
+

Backfeed

+ {% for silo, message in backfeed.denied.items() %} +

{{ silo }}: {{ message }}

+ {% endfor %} + {% if backfeed.remaining %} +

Backfill started {{ backfeed.started }} UTC: {{ backfeed.remaining }} crosspost(s) left to check.

+ {% elif backfeed.finished %} +

Last backfill: started {{ backfeed.started }}, finished {{ backfeed.finished }} UTC.

+ {% else %} +

No backfill has been run. It checks every crosspost once for responses, however old the note.

+ {% endif %} + {% if backfeed.errors %}

{{ backfeed.errors }} crosspost(s) had an error at their last check (see each note's edit page).

{% endif %} +
+ +
+
+ {% endif %} + {# Filter tabs #} +
+ {% for value, label in [(None, 'Any source'), ('webmention', 'Webmentions'), ('gotosocial', 'GoToSocial'), ('bluesky', 'Bluesky'), ('threads', 'Threads')] %} + {{ label }} + {% endfor %}
{% if webmentions %} @@ -93,11 +122,11 @@ {% if total_pages > 1 %} {% endif %} diff --git a/templates/note.html b/templates/note.html index 05c6690..6948345 100644 --- a/templates/note.html +++ b/templates/note.html @@ -115,6 +115,13 @@ {% endif %} + {% if quotes %} +
+

{{ quotes|length }} {{ 'Quote' if quotes|length == 1 else 'Quotes' }}

+ {% for wm in quotes %}{{ responses.mention(wm) }}{% endfor %} +
+ {% endif %} + {% if mentions %}

{{ mentions|length }} {{ 'Mention' if mentions|length == 1 else 'Mentions' }}

@@ -123,5 +130,6 @@ {% endif %} {% endif %} + {{ responses.silo_counts(syndications) }} {% endblock %} diff --git a/templates/partials/responses.html b/templates/partials/responses.html index 9f09d7e..b7e9db6 100644 --- a/templates/partials/responses.html +++ b/templates/partials/responses.html @@ -12,11 +12,18 @@ {# The response's own URL: the source's u-url if it gave one #} {% macro response_url(wm) %}{{ wm.source_post_url or wm.source_url }}{% endmacro %} +{# Where a backfed response came from, e.g. "via Bluesky" (v2.13.0) #} +{% macro via(wm) %}{% if wm.via %}via {{ wm.via }}{% endif %}{% endmacro %} + {# Author h-card: name, URL and photo. u-url is on an inner link: on the - h-card element itself it would belong to the enclosing h-cite #} + h-card element itself it would belong to the enclosing h-cite. + The photo is this site's saved copy (v2.13.0); until there is one, the + initial is shown instead of loading it from the other site. #} {% macro author_card(wm, class='', photo_size=48, show_name=True) %} - - {%- if wm.author_photo %} +{% set photo = wm.author_photo|local_avatar %} + + {%- if photo %} + {%- elif wm.author_photo %}{% if not show_name %}{% endif %} {%- elif not show_name %} {%- endif %} {{ wm.author_name or 'Anonymous' }} @@ -30,7 +37,7 @@ {% if wm.content_snippet %}
{{ wm.content_snippet|safe }}
{% endif %} -
{{ response_time(wm) }} + {{ response_time(wm) }} {{ via(wm) }} {%- endmacro %} @@ -50,13 +57,23 @@
{%- endmacro %} -{# A post that mentions this note without replying #} +{# A post that mentions this note without replying; also quote posts (v2.13.0) #} {% macro mention(wm) %}
{{ author_card(wm, photo_size=24) }} {% if wm.content_snippet %}

{{ wm.content_snippet|striptags|truncate(200) }}

{% endif %} - {{ response_time(wm) }} + {{ response_time(wm) }} {{ via(wm) }}
{%- endmacro %} + +{# Like, repost and quote counts a silo reports without saying who (Threads, + v2.13.0). Plain text: these aren't people, so no u-like / h-cite. #} +{% macro silo_counts(syndications) %} +{% for s in syndications or [] if s.likes_count or s.reposts_count or s.quotes_count %} +{% set parts = [] %} +{% for n, one, many in [(s.likes_count, 'like', 'likes'), (s.reposts_count, 'repost', 'reposts'), (s.quotes_count, 'quote', 'quotes')] if n %}{% set _ = parts.append(n ~ ' ' ~ (one if n == 1 else many)) %}{% endfor %} +

On {{ s.name }}: {{ parts|join(' · ') }}

+{% endfor %} +{%- endmacro %} diff --git a/tests/test_backfeed.py b/tests/test_backfeed.py new file mode 100644 index 0000000..392023e --- /dev/null +++ b/tests/test_backfeed.py @@ -0,0 +1,542 @@ +""" +Tests for backfeed of responses to crossposts (v2.13.0, ADR-073) + +Silo clients are tested against fake APIs (httpx.MockTransport). The +worker, storage and pages are tested with a fake client that returns +whatever a test puts in it. +""" + +import io +import json +from urllib.parse import parse_qs + +import httpx +import pytest +from PIL import Image + +from starpunk import avatars, backfeed, create_app, syndication +from starpunk.backfeed import Harvest, Response, facets_html, interval_for, text_html +from starpunk.database import get_db +from starpunk.notes import create_note +from starpunk.syndication import BlueskyClient, MastodonClient, SyndicationError, ThreadsClient + +SITE = "https://blog.example.com" +GTS = "https://social.example" +COPY = f"{GTS}/@phil/statuses/01COPY" + + +# --- Fakes -------------------------------------------------------------------- + +class FakeClient: + """Stands in for a silo client: returns the harvest a test sets""" + config_token = "token" + + def __init__(self): + self.harvest = Harvest([]) + self.error = None + self.calls = [] + + def responses(self, remote_id, remote_url, cap=1000): + self.calls.append(remote_id) + if self.error: + raise self.error + return self.harvest + + def maintain(self): + pass + + +def like(n, bio="", **kw): + return Response(kind="like", source_url=f"{COPY}#liked-by-{n}", remote_id=str(n), + author_id=str(n), author_name=f"Liker {n}", author_url=f"{GTS}/@l{n}", + author_bio=bio, **kw) + + +def reply(n, text="Nice!"): + return Response(kind="reply", source_url=f"{GTS}/@r{n}/statuses/{n}", remote_id=str(n), + author_id=f"r{n}", author_name=f"Replier {n}", author_url=f"{GTS}/@r{n}", + content_html=f"

{text}

", published="2026-11-10T12:00:00Z") + + +def resolve(host, *args, **kwargs): + return [(2, 1, 6, "", ("93.184.216.34", 0))] + + +def png(color="red", size=(200, 120)): + buf = io.BytesIO() + Image.new("RGB", size, color).save(buf, "PNG") + return buf.getvalue() + + +class FakeWeb: + """Avatar images by URL (requests are pinned to an IP; Host names the site)""" + def __init__(self): + self.files = {} + + def __call__(self, request): + url = f"{request.url.scheme}://{request.headers['host']}{request.url.raw_path.decode()}" + if url in self.files: + return httpx.Response(200, headers={"content-type": "image/png"}, content=self.files[url]) + return httpx.Response(404) + + +@pytest.fixture +def web(monkeypatch): + fake = FakeWeb() + monkeypatch.setattr("starpunk.webmention_ssrf.TRANSPORT", httpx.MockTransport(fake)) + monkeypatch.setattr("starpunk.webmention_ssrf.socket.getaddrinfo", resolve) + return fake + + +@pytest.fixture +def client(): + return FakeClient() + + +@pytest.fixture +def app(tmp_path, client, web): + data = tmp_path / "data" + app = create_app({ + "TESTING": True, + "SITE_URL": SITE, + "ADMIN_ME": "https://example.com", + "SESSION_SECRET": "test-secret", + "DATA_PATH": data, + "NOTES_PATH": data / "notes", + "DATABASE_PATH": data / "starpunk.db", + "GOTOSOCIAL_URL": GTS, + "GOTOSOCIAL_ACCESS_TOKEN": "token", + "BACKFEED_ENABLED": True, + }) + syndication._client_cache[GTS] = client + backfeed.denied.clear() + yield app + syndication._client_cache.clear() + backfeed.denied.clear() + + +def crossposted(app, age_days=0, content="Hello"): + """A published note with a synced GoToSocial copy; returns (note, syndication id)""" + with app.app_context(): + note = create_note(content=content, published=True) + db = get_db(app) + if age_days: + db.execute("UPDATE notes SET created_at = datetime('now', ?) WHERE id = ?", + (f"-{age_days} days", note.id)) + cursor = db.execute( + "INSERT INTO syndications (note_id, target, enabled, remote_id, remote_url, state) " + "VALUES (?, 'gotosocial', 1, '01COPY', ?, 'synced')", (note.id, COPY)) + db.commit() + return note, cursor.lastrowid + + +def rows(app, note_id): + with app.app_context(): + return [dict(r) for r in get_db(app).execute( + "SELECT * FROM webmentions WHERE target_note_id = ? ORDER BY source_url", (note_id,))] + + +def run(app): + with app.app_context(): + return backfeed.process_due() + + +# --- Schedule and text ------------------------------------------------------------ + +def test_schedule_slows_with_age_and_stops_at_cutoff(): + day = 24 * 3600 + assert interval_for(3600, 90) == 15 * 60 + assert interval_for(2 * day, 90) == 3600 + assert interval_for(10 * day, 90) == day + assert interval_for(60 * day, 90) == 7 * day + assert interval_for(91 * day, 90) is None + + +def test_text_and_facets_become_safe_html(): + assert text_html("hi \nsee https://a.example/x") == \ + '

hi <b>
see https://a.example/x

' + text = "café: example.com/long…" + start = len("café: ".encode()) + facets = [{"index": {"byteStart": start, "byteEnd": len(text.encode())}, + "features": [{"$type": "app.bsky.richtext.facet#link", "uri": "https://example.com/long/path"}]}] + assert facets_html(text, facets) == \ + '

café: example.com/long…

' + assert facets_html("", []) == "" + + +# --- Worker and storage ------------------------------------------------------------- + +def test_check_stores_responses_as_webmentions(app, client): + note, sid = crossposted(app) + client.harvest = Harvest([like(1), reply(2, "Great post")]) + assert run(app) == 1 + stored = rows(app, note.id) + assert [r["mention_type"] for r in stored] == ["like", "reply"] + assert all(r["origin"] == "gotosocial" and r["validation_status"] == "valid" + and r["moderation_status"] == "approved" for r in stored) + assert stored[1]["content_snippet"] == "

Great post

" + assert stored[0]["target_url"] == f"{SITE}/note/{note.slug}" + with app.app_context(): + s = get_db(app).execute("SELECT * FROM syndications WHERE id = ?", (sid,)).fetchone() + assert s["backfeed_checked_at"] and s["backfeed_next_at"] and s["backfeed_error"] is None + assert run(app) == 0, "not due again for 15 minutes" + + +def test_repeat_check_updates_rows_and_keeps_moderation(app, client): + note, sid = crossposted(app) + client.harvest = Harvest([like(1), like(2)]) + run(app) + with app.app_context(): + db = get_db(app) + db.execute("UPDATE webmentions SET moderation_status = 'hidden' WHERE source_url LIKE '%-1'") + db.execute("UPDATE syndications SET backfeed_next_at = datetime('now')") + db.commit() + client.harvest = Harvest([like(1), like(2)]) + run(app) + stored = rows(app, note.id) + assert len(stored) == 2 + assert [r["moderation_status"] for r in stored] == ["hidden", "approved"] + + +def test_withdrawn_responses_drop_off_and_come_back(app, client): + note, _ = crossposted(app) + client.harvest = Harvest([like(1), reply(2)]) + run(app) + + def recheck(harvest): + with app.app_context(): + get_db(app).execute("UPDATE syndications SET backfeed_next_at = datetime('now')") + get_db(app).commit() + client.harvest = harvest + run(app) + return {r["source_url"]: (r["validation_status"], r["validation_error"]) for r in rows(app, note.id)} + + # An incomplete list (cap reached, say) removes nothing + assert all(v == ("valid", None) for v in recheck(Harvest([like(1)], complete=False)).values()) + state = recheck(Harvest([like(1)])) + assert state[reply(2).source_url] == ("invalid", "No longer on GoToSocial") + assert recheck(Harvest([like(1), reply(2)]))[reply(2).source_url] == ("valid", None) + + +def test_people_who_opt_out_are_skipped_and_removed(app, client): + note, _ = crossposted(app) + client.harvest = Harvest([like(1), like(2)]) + run(app) + with app.app_context(): + get_db(app).execute("UPDATE syndications SET backfeed_next_at = datetime('now')") + get_db(app).commit() + client.harvest = Harvest([like(1, bio="

Please #NoBridge thanks

"), like(2), + like(3, opted_out=True)]) + run(app) + assert [r["source_url"] for r in rows(app, note.id)] == [like(2).source_url] + + +def test_admin_delete_is_a_tombstone_the_next_check_keeps(app, client): + from starpunk.webmention_queries import count_webmentions, delete_webmentions, list_webmentions + + note, _ = crossposted(app) + client.harvest = Harvest([like(1)]) + run(app) + with app.test_request_context(): + wm_id = rows(app, note.id)[0]["id"] + assert delete_webmentions([wm_id]) == 1 + get_db(app).execute("UPDATE syndications SET backfeed_next_at = datetime('now')") + get_db(app).commit() + run(app) + assert rows(app, note.id)[0]["moderation_status"] == "deleted" + with app.test_request_context(): + assert list_webmentions()[0] == [] and count_webmentions() == 0 + assert len(list_webmentions(status_filter="deleted")[0]) == 1 + + +def test_old_notes_wait_for_a_backfill(app, client): + old, sid = crossposted(app, age_days=200) + client.harvest = Harvest([reply(1)]) + assert run(app) == 0 + with app.app_context(): + assert backfeed.start_backfill() == 1 + assert backfeed.status()["remaining"] == 1 + assert run(app) == 1 + assert len(rows(app, old.id)) == 1 + with app.app_context(): + status = backfeed.status() + s = get_db(app).execute("SELECT * FROM syndications WHERE id = ?", (sid,)).fetchone() + assert status["remaining"] == 0 and status["finished"] + assert s["backfeed_backfill"] == 0 and s["backfeed_next_at"] is None, "past the cutoff: no more checks" + + +def test_backfill_is_paced(app, client): + crossposted(app, age_days=200) + crossposted(app, age_days=300) + with app.app_context(): + backfeed.start_backfill() + assert run(app) == 1 + assert run(app) == 0, "the next backfill check waits a few seconds" + with app.app_context(): + get_db(app).execute("UPDATE site_state SET value = datetime('now', '-10 seconds') " + "WHERE key = 'backfeed_backfill_last'") + get_db(app).commit() + assert run(app) == 1 + + +def test_routine_checks_skip_likes_reposts_and_unpublished(app, client): + with app.app_context(): + liked = create_note(content="", published=True, like_of="https://other.example/post") + draft = create_note(content="Draft", published=False) + db = get_db(app) + for note in (liked, draft): + db.execute("INSERT INTO syndications (note_id, target, enabled, remote_id, remote_url, state) " + "VALUES (?, 'gotosocial', 1, 'x', 'https://s.example/x', 'synced')", (note.id,)) + db.commit() + assert run(app) == 0 and client.calls == [] + + +def test_refused_credential_stops_that_silo(app, client): + crossposted(app) + error = SyndicationError("GET /api/v1/statuses/01COPY/favourited_by: HTTP 403 forbidden", permanent=True) + error.status = 403 + client.error = error + run(app) + assert "gotosocial" in backfeed.denied and "read:statuses" in backfeed.denied["gotosocial"] + client.error = None + assert run(app) == 0, "not retried until restart" + + +def test_failures_back_off(app, client): + _, sid = crossposted(app) + client.error = SyndicationError("HTTP 502 bad gateway") + run(app) + with app.app_context(): + s = get_db(app).execute("SELECT * FROM syndications WHERE id = ?", (sid,)).fetchone() + assert s["backfeed_attempts"] == 1 and "502" in s["backfeed_error"] + assert run(app) == 0 + + +def test_disabled_does_nothing(app, client): + crossposted(app) + app.config["BACKFEED_ENABLED"] = False + assert run(app) == 0 and client.calls == [] + + +# --- Pages ------------------------------------------------------------------------------ + +def test_note_page_shows_backfed_responses_quotes_and_counts(app, client, web): + note, sid = crossposted(app) + web.files["https://cdn.example/a.png"] = png() + quote = Response(kind="quote", source_url="https://bsky.app/profile/q.example/post/3k", + remote_id="at://did:plc:q/app.bsky.feed.post/3k", author_name="Quoter", + author_url="https://bsky.app/profile/q.example", content_html="

Look at this

") + client.harvest = Harvest([like(1, author_photo="https://cdn.example/a.png"), reply(2), quote]) + run(app) + with app.app_context(): + get_db(app).execute("UPDATE syndications SET likes_count = 12, reposts_count = 1 WHERE id = ?", (sid,)) + get_db(app).commit() + page = app.test_client().get(f"/note/{note.slug}").get_data(as_text=True) + assert "via GoToSocial" in page + assert "1 Quote" in page and "Look at this" in page + assert 'class="silo-counts">On GoToSocial: 12 likes · 1 repost<' in page + assert 'src="/avatars/' in page and 'src="https://cdn.example/a.png"' not in page + + +def test_avatars_are_saved_served_and_cleaned_up(app, web): + url = "https://cdn.example/face.png" + web.files[url] = png(size=(300, 200)) + data = app.config["DATA_PATH"] + assert avatars.cached_name(data, url) is None + with app.app_context(): + assert avatars.fetch(data, url, SITE) + assert not avatars.fetch(data, "https://cdn.example/missing.png", SITE) + name = avatars.cached_name(data, url) + with Image.open(avatars.folder(data) / name) as image: + assert image.size == (96, 96) and image.format == "WEBP" + response = app.test_client().get(f"/avatars/{name}") + assert response.status_code == 200 and response.mimetype == "image/webp" + assert app.test_client().get("/avatars/../starpunk.db").status_code == 404 + # Nothing uses it: the weekly cleanup removes it + assert avatars.remove_unused(app.config["DATABASE_PATH"], data) == 1 + + +def test_webmention_cannot_overwrite_a_backfed_response(app, client): + note, _ = crossposted(app) + client.harvest = Harvest([reply(7)]) + run(app) + response = app.test_client().post("/webmention", data={ + "source": reply(7).source_url, "target": f"{SITE}/note/{note.slug}"}) + assert response.status_code == 202 + assert rows(app, note.id)[0]["validation_status"] == "valid" + + +# --- Silo clients --------------------------------------------------------------------------- + +def mastodon_api(request): + path, query = request.url.path, parse_qs(request.url.query.decode()) + account = lambda n, **kw: {"id": n, "display_name": f"User {n}", "url": f"{GTS}/@u{n}", + "avatar": f"{GTS}/a/{n}.png", "note": "", **kw} + if path == "/api/v1/accounts/verify_credentials": + return httpx.Response(200, json={"id": "me"}) + if path == "/api/v1/statuses/01COPY/favourited_by": + if "max_id" in query: + return httpx.Response(200, json=[account("3")]) + return httpx.Response(200, json=[account("1"), account("me")], + headers={"Link": f'<{GTS}/api/v1/statuses/01COPY/favourited_by?max_id=9>; rel="next"'}) + if path == "/api/v1/statuses/01COPY/reblogged_by": + return httpx.Response(200, json=[account("2")]) + if path == "/api/v1/statuses/01COPY/context": + status = lambda n, parent="01COPY", vis="public", who=None: { + "id": n, "url": f"{GTS}/@u/statuses/{n}", "uri": f"{GTS}/u/{n}", "in_reply_to_id": parent, + "visibility": vis, "content": f"

reply {n}

", "created_at": "2026-11-10T00:00:00Z", + "account": who or account("r" + n)} + return httpx.Response(200, json={"ancestors": [], "descendants": [ + status("10"), status("11", vis="unlisted"), status("12", parent="10"), + status("13", who=account("me"))]}) + return httpx.Response(404) + + +def test_mastodon_client_reads_likes_boosts_and_public_direct_replies(): + gts = MastodonClient(GTS, "t", http=httpx.Client(transport=httpx.MockTransport(mastodon_api))) + harvest = gts.responses("01COPY", COPY) + assert harvest.complete + got = [(r.kind, r.source_url) for r in harvest.responses] + assert got == [("like", f"{COPY}#liked-by-1"), ("like", f"{COPY}#liked-by-3"), + ("repost", f"{COPY}#reposted-by-2"), ("reply", f"{GTS}/@u/statuses/10")] + assert harvest.responses[-1].content_html == "

reply 10

" + assert gts.responses("01COPY", COPY, cap=1).complete is False + + +def test_bluesky_client_reads_likes_reposts_quotes_and_replies(): + me, post = "did:plc:me", "at://did:plc:me/app.bsky.feed.post/abc" + + def api(request): + nsid = request.url.path.removeprefix("/xrpc/") + query = parse_qs(request.url.query.decode()) + profile = lambda d, **kw: {"did": d, "handle": f"{d[8:]}.example", "displayName": d[8:].title(), **kw} + view = lambda d, rkey, text: {"uri": f"at://{d}/app.bsky.feed.post/{rkey}", "author": profile(d), + "record": {"text": text, "createdAt": "2026-11-10T00:00:00Z"}} + if nsid == "com.atproto.server.createSession": + return httpx.Response(200, json={"did": me, "handle": "me.example", "accessJwt": "a", "refreshJwt": "r"}) + if nsid == "app.bsky.feed.getLikes": + if query.get("cursor"): + return httpx.Response(200, json={"likes": [{"actor": profile("did:plc:carol")}]}) + return httpx.Response(200, json={"cursor": "c1", "likes": [ + {"actor": profile("did:plc:alice"), "createdAt": "2026-11-10T01:00:00Z"}, + {"actor": profile("did:plc:shy", labels=[{"val": "!no-unauthenticated"}])}, + {"actor": profile(me)}]}) + if nsid == "app.bsky.feed.getRepostedBy": + return httpx.Response(200, json={"repostedBy": [profile("did:plc:bob")]}) + if nsid == "app.bsky.feed.getQuotes": + return httpx.Response(200, json={"posts": [view("did:plc:dan", "q1", "Quoting this")]}) + if nsid == "app.bsky.feed.getPostThread": + return httpx.Response(200, json={"thread": {"post": {}, "replies": [ + {"post": view("did:plc:eve", "r1", "A reply")}, + {"$type": "app.bsky.feed.defs#blockedPost", "uri": "at://x"}]}}) + if nsid == "app.bsky.actor.getProfiles": + return httpx.Response(200, json={"profiles": [ + profile("did:plc:dan", description="no thanks #nobot"), profile("did:plc:eve")]}) + return httpx.Response(400, json={"error": "Unknown"}) + + bsky = BlueskyClient("https://bsky.example", "me.example", "pw", + http=httpx.Client(transport=httpx.MockTransport(api))) + harvest = bsky.responses(post, "https://bsky.app/profile/me.example/post/abc") + by = {(r.kind, r.author_id): r for r in harvest.responses} + assert set(by) == {("like", "did:plc:alice"), ("like", "did:plc:shy"), ("like", "did:plc:carol"), + ("repost", "did:plc:bob"), ("quote", "did:plc:dan"), ("reply", "did:plc:eve")} + assert by[("like", "did:plc:shy")].opted_out + assert by[("quote", "did:plc:dan")].author_bio == "no thanks #nobot" + assert by[("reply", "did:plc:eve")].source_url == "https://bsky.app/profile/eve.example/post/r1" + assert by[("like", "did:plc:alice")].source_url.endswith("#liked-by-did:plc:alice") + assert harvest.complete + + +def test_threads_client_reads_replies_and_counts(): + def api(request): + path = request.url.path + params = parse_qs(request.url.query.decode()) + if path.endswith("/me"): + return httpx.Response(200, json={"id": "42", "username": "me"}) + if path.endswith("/111/replies"): + assert "profile_picture_url" in params["fields"][0] + return httpx.Response(200, json={"data": [ + {"id": "1", "text": "Hi", "username": "ann", "permalink": "https://www.threads.com/@ann/post/A", + "timestamp": "2026-11-10T00:00:00+0000", "hide_status": "NOT_HUSHED"}, + {"id": "2", "text": "private"}, + {"id": "3", "text": "hidden", "username": "x", "permalink": "p", "hide_status": "HIDDEN"}, + {"id": "4", "text": "mine", "username": "me", "permalink": "p"}]}) + if path.endswith("/111/insights"): + assert params["metric"] == ["likes,reposts,quotes"] + return httpx.Response(200, json={"data": [ + {"name": "likes", "values": [{"value": 12}]}, {"name": "reposts", "values": [{"value": 3}]}, + {"name": "quotes", "values": [{"value": 0}]}]}) + return httpx.Response(404, json={"error": {"code": 100}}) + + threads = ThreadsClient("tok", http=httpx.Client(transport=httpx.MockTransport(api))) + harvest = threads.responses("111", "https://www.threads.com/@me/post/X") + assert [(r.kind, r.author_name, r.content_html) for r in harvest.responses] == [("reply", "ann", "

Hi

")] + assert harvest.counts == {"likes": 12, "reposts": 3, "quotes": 0} + + +# --- Migration --------------------------------------------------------------------------- + +def test_migration_020_keeps_existing_webmentions(tmp_path): + import sqlite3 + from pathlib import Path + + from starpunk.database.schema import INITIAL_SCHEMA_SQL + from starpunk.migrations import execute_in_transaction + + migrations = Path(__file__).parent.parent / "migrations" + conn = sqlite3.connect(tmp_path / "old.db") + conn.execute("PRAGMA foreign_keys=ON") + conn.executescript(INITIAL_SCHEMA_SQL) + for name in ("010_add_webmentions.sql", "012_add_syndications.sql", "014_webmention_source_details.sql"): + matches = list(migrations.glob(name.split("_")[0] + "_*.sql")) + conn.execute("BEGIN") + execute_in_transaction(conn, matches[0].read_text()) + conn.commit() + conn.execute("INSERT INTO notes (slug, file_path, published) VALUES ('n', 'n.md', 1)") + conn.execute("INSERT INTO webmentions (source_url, target_url, target_note_id, mention_type, " + "validation_status, moderation_status, source_published) " + "VALUES ('https://a.example/r', 'https://b.example/note/n', 1, 'reply', 'valid', 'hidden', " + "'2026-01-01')") + conn.commit() + + conn.execute("BEGIN") + execute_in_transaction(conn, (migrations / "020_backfeed.sql").read_text()) + conn.commit() + + row = conn.execute("SELECT mention_type, moderation_status, source_published, origin, target_note_id " + "FROM webmentions").fetchone() + assert row == ("reply", "hidden", "2026-01-01", "webmention", 1) + conn.execute("INSERT INTO webmentions (source_url, target_url, mention_type, origin) " + "VALUES ('https://q.example', 'https://b.example/note/n', 'quote', 'bluesky')") + columns = {r[1] for r in conn.execute("PRAGMA table_info(syndications)")} + assert {"backfeed_next_at", "backfeed_backfill", "likes_count"} <= columns + assert conn.execute("PRAGMA foreign_key_check").fetchall() == [] + + +# --- Admin ---------------------------------------------------------------------------------- + +def test_admin_backfill_and_check_now(app, client): + from starpunk.auth import create_session + + app.config.update(ADMIN_ME="https://example.com", CSRF_ENABLED=False) + note, sid = crossposted(app, age_days=200) + with app.test_request_context(): + token = create_session("https://example.com") + browser = app.test_client() + browser.set_cookie("starpunk_session", token) + + page = browser.get("/admin/webmentions").get_data(as_text=True) + assert "No backfill has been run" in page and "Backfill responses" in page + response = browser.post("/admin/backfeed/backfill", follow_redirects=True) + assert "Backfill started: 1 crosspost(s) to check" in response.get_data(as_text=True) + assert "1 crosspost(s) left to check" in browser.get("/admin/webmentions").get_data(as_text=True) + + edit = browser.get(f"/admin/edit/{note.id}").get_data(as_text=True) + assert "Responses not checked yet" in edit and "Check now" in edit + browser.post(f"/admin/syndication/{sid}/check") + with app.app_context(): + s = get_db(app).execute("SELECT backfeed_next_at FROM syndications WHERE id = ?", (sid,)).fetchone() + assert s["backfeed_next_at"] + + backfeed.denied["gotosocial"] = "Needs a GoToSocial token with read:statuses" + assert "Needs a GoToSocial token" in browser.get("/admin/webmentions?origin=gotosocial").get_data(as_text=True)