From 57b02ad6c7ac974d2cbfade2e742b5e64fb829e8 Mon Sep 17 00:00:00 2001 From: Phil Date: Sat, 31 Oct 2026 12:46:26 -0600 Subject: [PATCH] feat: Crosspost notes to GoToSocial (v2.1.0) Published notes are copied to a GoToSocial (Mastodon-API) account and edits, unpublishing and deletes follow them (ADR-065). On by default for new notes, untickable in the editor and selectable from Micropub via mp-syndicate-to. A background worker reconciles each (note, target) row with atomic claims, an idempotency key and a version so edits during a send are not lost. Synced copies are linked with u-syndication. Also fixes the Micropub Location header and search result links, which pointed at /notes/ (404), and the nested delete form on the edit page. Co-Authored-By: Claude Opus 5.5 --- .env.example | 13 + CHANGELOG.md | 19 + .../ADR-065-direct-syndication-posse.md | 45 ++ migrations/012_add_syndications.sql | 29 ++ starpunk/__init__.py | 14 +- starpunk/config.py | 10 + starpunk/micropub.py | 40 +- starpunk/notes.py | 14 + starpunk/routes/admin.py | 35 +- starpunk/routes/public.py | 5 + starpunk/routes/search.py | 4 +- starpunk/syndication.py | 418 ++++++++++++++++++ templates/admin/edit.html | 51 ++- templates/admin/new.html | 13 + templates/note.html | 3 + tests/test_micropub.py | 4 +- tests/test_search_api.py | 4 +- tests/test_search_integration.py | 4 +- tests/test_syndication.py | 349 +++++++++++++++ 19 files changed, 1049 insertions(+), 25 deletions(-) create mode 100644 docs/decisions/ADR-065-direct-syndication-posse.md create mode 100644 migrations/012_add_syndications.sql create mode 100644 starpunk/syndication.py create mode 100644 tests/test_syndication.py diff --git a/.env.example b/.env.example index 0705511..823c0ec 100644 --- a/.env.example +++ b/.env.example @@ -50,6 +50,19 @@ SESSION_LIFETIME=30 # Lifetime of Micropub access tokens StarPunk issues, in days (default: 90) INDIEAUTH_TOKEN_LIFETIME_DAYS=90 +# ============================================================================= +# CROSSPOSTING (optional, v2.1.0) +# ============================================================================= + +# GoToSocial or Mastodon account to copy new notes to. Create an app token +# with scopes: write:statuses write:media +GOTOSOCIAL_URL= +GOTOSOCIAL_ACCESS_TOKEN= +# Name shown in the editor and Micropub apps (default: GoToSocial) +GOTOSOCIAL_NAME=GoToSocial +# public, unlisted, private (default: public) +GOTOSOCIAL_VISIBILITY=public + # ============================================================================= # DATA STORAGE # ============================================================================= diff --git a/CHANGELOG.md b/CHANGELOG.md index a44208c..72d3235 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [2.1.0] - 2026-10-31 + +### Added + +- **Crossposting to GoToSocial** (or any Mastodon-API server), ADR-065 + - Full note as Markdown with images (alt text from captions), permalink and hashtags + - Edits, unpublishing and deletes follow the note + - On by default for new notes; untick in the editor, or send `mp-syndicate-to` from Micropub apps + - Micropub `q=syndicate-to` and `q=config` list the target + - Editor shows crosspost status, errors and a retry button; note pages link the copy with `u-syndication` + - Configure with `GOTOSOCIAL_URL` and `GOTOSOCIAL_ACCESS_TOKEN` (scopes `write:statuses write:media`) +- Migration 012: `syndications` + +### Fixed + +- Micropub `Location` header pointed at `/notes/` (404); now the real permalink +- "Delete Note" on the edit page submitted the edit form (nested forms) +- Search results linked to `/notes/` (404); now `/note/` + ## [2.0.2] - 2026-10-31 ### Fixed diff --git a/docs/decisions/ADR-065-direct-syndication-posse.md b/docs/decisions/ADR-065-direct-syndication-posse.md new file mode 100644 index 0000000..5ea41a3 --- /dev/null +++ b/docs/decisions/ADR-065-direct-syndication-posse.md @@ -0,0 +1,45 @@ +# ADR-065: Direct Syndication (POSSE) to GoToSocial + +## Status + +Accepted (v2.1.0, 2026-10-31) + +## Context + +StarPunk posts were copied to other networks by Echofeed reading the feed. +That gives no control per post, no record of where a post went, and edits +and deletes never follow. The operator runs GoToSocial locally and wants +StarPunk to post there directly, with Bluesky likely next. + +## Decision + +1. **Publish here, copy out (POSSE).** Each (note, target) has a row in + `syndications` (migration 012). Creating, editing, unpublishing or + deleting a note marks its rows pending; a background worker reconciles + the remote post: create, update (`PUT /api/v1/statuses/:id`) or delete. +2. **Defaults.** New notes go to every configured target unless unticked in + the editor or a Micropub client sends `mp-syndicate-to` (an explicit + list, possibly empty). Existing notes are never backfilled. +3. **Content.** The full note as Markdown (`content_type: text/markdown`), + then the permalink and the note's tags as hashtags. Longer than the + target's limit: shortened at a word boundary with the link kept. + Images are uploaded first (the `large` variant) with their captions as + alt text. +4. **Delivery.** Rows are claimed atomically so the four gunicorn workers' + threads never send twice; an `Idempotency-Key` guards retries; a row + `version` keeps edits made during a send. Network errors and 5xx/429 + retry with exponential backoff (8 attempts); other 4xx fail at once. + A remote post deleted by hand is posted again on the next edit. +5. **Backlinks.** Synced URLs are shown on the note page as `u-syndication` + links, and in the editor with status, last error and a retry button. +6. **Targets** are config-driven. One kind exists, a Mastodon-API client + (`GOTOSOCIAL_URL`, `GOTOSOCIAL_ACCESS_TOKEN`); it also works for + Mastodon. A Bluesky target would add a second client behind the same + reconcile loop. + +## Consequences + +- Echofeed must stop posting to GoToSocial or posts appear twice. +- The token needs only `write:statuses write:media`. +- `syndications.note_id` has no foreign key so a hard-deleted note's + remote post can still be deleted. diff --git a/migrations/012_add_syndications.sql b/migrations/012_add_syndications.sql new file mode 100644 index 0000000..e65cd28 --- /dev/null +++ b/migrations/012_add_syndications.sql @@ -0,0 +1,29 @@ +-- Migration 012: Crossposting (POSSE) to syndication targets +-- Version: 2.1.0 +-- Per ADR-065: Direct syndication to GoToSocial / Mastodon-API servers +-- +-- One row per (note, target). The worker reconciles the remote post with the +-- note: create, update, or delete as needed. note_id has no foreign key on +-- purpose: a row must outlive a hard-deleted note so the remote post can be +-- deleted too. + +CREATE TABLE IF NOT EXISTS syndications ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + note_id INTEGER NOT NULL, + target TEXT NOT NULL, + enabled INTEGER NOT NULL DEFAULT 1, + remote_id TEXT, + remote_url TEXT, + state TEXT NOT NULL DEFAULT 'pending' + CHECK (state IN ('pending', 'synced', 'failed')), + version INTEGER NOT NULL DEFAULT 1, + attempts INTEGER NOT NULL DEFAULT 0, + last_error TEXT, + next_attempt_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + claimed_until TIMESTAMP, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE (note_id, target) +); + +CREATE INDEX IF NOT EXISTS idx_syndications_due ON syndications(state, next_attempt_at); diff --git a/starpunk/__init__.py b/starpunk/__init__.py index 61a8fe1..7a8616b 100644 --- a/starpunk/__init__.py +++ b/starpunk/__init__.py @@ -336,6 +336,16 @@ def create_app(config=None): if hasattr(app, 'webmention_worker') and app.webmention_worker.is_alive(): app.webmention_worker.stop() + # Start syndication worker when a target is configured (v2.1.0) + if (app.config.get("GOTOSOCIAL_URL") and app.config.get("GOTOSOCIAL_ACCESS_TOKEN") + and not app.config.get("TESTING", False)): + from starpunk.syndication import SyndicationWorker + app.syndication_worker = SyndicationWorker( + app, poll_interval=app.config["SYNDICATION_POLL_INTERVAL"] + ) + app.syndication_worker.start() + app.logger.info("Syndication worker started") + # Health check endpoint for containers and monitoring @app.route("/health") def health_check(): @@ -440,5 +450,5 @@ def create_app(config=None): # Package version (Semantic Versioning 2.0.0) # See docs/standards/versioning-strategy.md for details -__version__ = "2.0.2" -__version_info__ = (2, 0, 2) +__version__ = "2.1.0" +__version_info__ = (2, 1, 0) diff --git a/starpunk/config.py b/starpunk/config.py index aa1035d..45d7e7c 100644 --- a/starpunk/config.py +++ b/starpunk/config.py @@ -124,6 +124,16 @@ def load_config(app, config_override=None): app.config["WEBMENTION_RATE_LIMIT_WINDOW"] = int(os.getenv("WEBMENTION_RATE_LIMIT_WINDOW", "60")) app.config["TRUSTED_PROXY"] = os.getenv("TRUSTED_PROXY", "false").lower() == "true" + # Crossposting to GoToSocial / Mastodon-API (v2.1.0, ADR-065) + # Enabled when both URL and token are set + app.config["GOTOSOCIAL_URL"] = os.getenv("GOTOSOCIAL_URL", "") + app.config["GOTOSOCIAL_ACCESS_TOKEN"] = os.getenv("GOTOSOCIAL_ACCESS_TOKEN", "") + app.config["GOTOSOCIAL_NAME"] = os.getenv("GOTOSOCIAL_NAME", "GoToSocial") + app.config["GOTOSOCIAL_VISIBILITY"] = os.getenv("GOTOSOCIAL_VISIBILITY", "public") + app.config["GOTOSOCIAL_MAX_CHARS"] = int(os.getenv("GOTOSOCIAL_MAX_CHARS", "5000")) + app.config["GOTOSOCIAL_MAX_MEDIA"] = int(os.getenv("GOTOSOCIAL_MAX_MEDIA", "4")) + app.config["SYNDICATION_POLL_INTERVAL"] = int(os.getenv("SYNDICATION_POLL_INTERVAL", "10")) + # Apply overrides if provided if config_override: app.config.update(config_override) diff --git a/starpunk/micropub.py b/starpunk/micropub.py index bbd1ec2..d4eeb86 100644 --- a/starpunk/micropub.py +++ b/starpunk/micropub.py @@ -30,6 +30,7 @@ from flask import Request, current_app, jsonify from starpunk.notes import create_note, get_note, InvalidNoteDataError, NoteNotFoundError from starpunk.indieauth import check_scope +from starpunk.utils import absolute_url_for # Custom Exceptions @@ -398,6 +399,17 @@ def handle_create(data: dict, token_info: dict): elif isinstance(slug_value, str): custom_slug = slug_value + # mp-syndicate-to is also a server command, not a property (v2.1.0). + # Absent means "all targets"; present means exactly those. + requested_targets = None + if isinstance(data, dict): + for key in ("mp-syndicate-to", "mp-syndicate-to[]"): + if key in data: + value = data[key] + requested_targets = (requested_targets or []) + ( + value if isinstance(value, list) else [value] + ) + # Normalize and extract properties try: properties = normalize_properties(data) @@ -427,10 +439,14 @@ def handle_create(data: dict, token_info: dict): if photos: _attach_photos_to_note(note.id, photos) - # Build permalink URL - # Note: SITE_URL is normalized to include trailing slash (for IndieAuth spec compliance) - site_url = current_app.config.get("SITE_URL", "http://localhost:5000") - permalink = f"{site_url}notes/{note.slug}" + # Crosspost after photos are attached (v2.1.0) + from starpunk.syndication import get_targets, resolve_targets, select_targets + if requested_targets is None: + select_targets(note.id, list(get_targets())) + else: + select_targets(note.id, resolve_targets(requested_targets)) + + permalink = absolute_url_for("public.note", slug=note.slug) # Return 201 Created with Location header return "", 201, {"Location": permalink} @@ -444,6 +460,13 @@ def handle_create(data: dict, token_info: dict): ) +def _syndicate_to() -> list: + """Configured crosspost targets in Micropub syndicate-to form""" + from starpunk.syndication import get_targets + + return [{"uid": t.uid, "name": t.name} for t in get_targets().values()] + + def handle_query(args: dict, token_info: dict): """ Handle Micropub query endpoints @@ -451,7 +474,7 @@ def handle_query(args: dict, token_info: dict): Supports: - q=config: Return server configuration - q=source: Return post source in Microformats2 JSON - - q=syndicate-to: Return syndication targets (empty for V1) + - q=syndicate-to: Return syndication targets Args: args: Query string arguments @@ -467,7 +490,7 @@ def handle_query(args: dict, token_info: dict): site_url = current_app.config.get("SITE_URL", "http://localhost:5000").rstrip('/') config = { "media-endpoint": f"{site_url}/micropub/media", - "syndicate-to": [], # No syndication targets in V1 + "syndicate-to": _syndicate_to(), "post-types": [ {"type": "note", "name": "Note", "properties": ["content"]}, {"type": "photo", "name": "Photo", "properties": ["photo"]} @@ -483,7 +506,7 @@ def handle_query(args: dict, token_info: dict): # Extract slug from URL try: - # URL format: https://example.com/notes/{slug} + # URL format: https://example.com/note/{slug} slug = url.rstrip("/").split("/")[-1] note = get_note(slug) @@ -520,8 +543,7 @@ def handle_query(args: dict, token_info: dict): return jsonify(mf2), 200 elif q == "syndicate-to": - # Return syndication targets (none for V1) - return jsonify({"syndicate-to": []}), 200 + return jsonify({"syndicate-to": _syndicate_to()}), 200 else: return error_response("invalid_request", f"Unknown query: {q}") diff --git a/starpunk/notes.py b/starpunk/notes.py index 54de270..c2e339d 100644 --- a/starpunk/notes.py +++ b/starpunk/notes.py @@ -851,6 +851,13 @@ def update_note( # Reply context fetch failure should not prevent note update current_app.logger.warning(f"Failed to fetch reply context for {in_reply_to}: {e}") + # Crossposts follow the note (v2.1.0, best-effort) + try: + from starpunk.syndication import note_changed + note_changed(updated_note.id) + except Exception as e: + current_app.logger.warning(f"Failed to queue syndication update: {e}") + # 10. RETURN UPDATED NOTE return updated_note @@ -1018,5 +1025,12 @@ def delete_note( ) # Don't fail - database record already deleted + # Delete crossposts too (v2.1.0, best-effort) + try: + from starpunk.syndication import note_changed + note_changed(existing_note.id) + except Exception as e: + current_app.logger.warning(f"Failed to queue syndication delete: {e}") + # 6. RETURN (no value) return None diff --git a/starpunk/routes/admin.py b/starpunk/routes/admin.py index d51884a..f1f0422 100644 --- a/starpunk/routes/admin.py +++ b/starpunk/routes/admin.py @@ -97,7 +97,9 @@ def new_note_form(): Decorator: @require_auth Template: templates/admin/new.html """ - return render_template("admin/new.html") + from starpunk.syndication import get_targets + + return render_template("admin/new.html", syndication_targets=get_targets().values()) @bp.route("/new", methods=["POST"]) @@ -182,6 +184,11 @@ def create_note_submit(): if errors: flash(f"Note created, but some images failed: {'; '.join(errors)}", "warning") + # Crosspost after media is attached so images go out too (v2.1.0) + if "syndication_form" in request.form: + from starpunk.syndication import select_targets + select_targets(note.id, request.form.getlist("syndicate_to")) + flash(f"Note created: {note.slug}", "success") return redirect(url_for("admin.dashboard")) @@ -223,7 +230,17 @@ def edit_note_form(note_id: int): tags = get_note_tags(note.id) object.__setattr__(note, '_cached_tags', tags) - return render_template("admin/edit.html", note=note) + from starpunk.syndication import get_note_syndications, get_targets + + syndications = get_note_syndications(note.id) + enabled = {s["target"] for s in syndications if s["enabled"]} + return render_template( + "admin/edit.html", + note=note, + syndication_targets=get_targets().values(), + syndications=syndications, + syndication_enabled=enabled, + ) @bp.route("/edit/", methods=["POST"]) @@ -274,6 +291,9 @@ def update_note_submit(note_id: int): published=published, tags=tags if tags else None ) + if "syndication_form" in request.form: + from starpunk.syndication import select_targets + select_targets(note.id, request.form.getlist("syndicate_to")) flash(f"Note updated: {note.slug}", "success") return redirect(url_for("admin.dashboard")) except ValueError as e: @@ -838,3 +858,14 @@ def revoke_token(token_id): revoke_token_by_id(token_id) flash("App access revoked", "success") return redirect(url_for("admin.tokens")) + + +@bp.route("/syndication//retry", methods=["POST"]) +@require_auth +def retry_syndication(syndication_id): + """Retry a crosspost now""" + from starpunk.syndication import retry + + retry(syndication_id) + flash("Crosspost queued", "success") + return redirect(request.referrer or url_for("admin.dashboard")) diff --git a/starpunk/routes/public.py b/starpunk/routes/public.py index 4ead4c4..683039b 100644 --- a/starpunk/routes/public.py +++ b/starpunk/routes/public.py @@ -331,9 +331,14 @@ def note(slug: str): logger = logging.getLogger(__name__) logger.error(f"Error loading webmentions for note {slug}: {e}") + from starpunk.syndication import get_note_syndications + + syndications = [s for s in get_note_syndications(note_obj.id) if s["state"] == "synced"] + return render_template( "note.html", note=note_obj, + syndications=syndications, webmentions=webmentions, reply_context=reply_context, **grouped if webmentions else {} diff --git a/starpunk/routes/search.py b/starpunk/routes/search.py index 35ad9f2..7d9963b 100644 --- a/starpunk/routes/search.py +++ b/starpunk/routes/search.py @@ -108,7 +108,7 @@ def api_search(): "title": r["title"] or f"Note from {r['created_at'][:10]}", "excerpt": r["snippet"], # Already has tags "published_at": r["created_at"], - "url": f"/notes/{r['slug']}", + "url": f"/note/{r['slug']}", } for r in results ], @@ -176,7 +176,7 @@ def search_page(): "title": r["title"] or f"Note from {r['created_at'][:10]}", "excerpt": Markup(safe_snippet), # Mark as safe since we've escaped it ourselves "published_at": r["created_at"], - "url": f"/notes/{r['slug']}", + "url": f"/note/{r['slug']}", }) results = formatted_results except Exception as e: diff --git a/starpunk/syndication.py b/starpunk/syndication.py new file mode 100644 index 0000000..322108c --- /dev/null +++ b/starpunk/syndication.py @@ -0,0 +1,418 @@ +""" +Crossposting (POSSE) for StarPunk (v2.1.0, ADR-065) + +Published notes are copied to syndication targets, and later edits and +deletes follow them. Currently one target kind exists: a GoToSocial (or any +Mastodon-API) account. + +Each (note, target) pair has a row in `syndications`. Changing a note marks +its rows pending; a background worker then reconciles the remote post with +the note: + + note published + target enabled, no remote post -> create + note published + target enabled, remote post -> update + otherwise, remote post exists -> delete + +Rows carry a version so an edit made while a send is in flight is not lost. +""" + +import re +import threading +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, List, Optional + +import httpx +from flask import current_app + +from starpunk.database import get_db + +MAX_ATTEMPTS = 8 +CLAIM_MINUTES = 5 + + +class SyndicationError(Exception): + """A send failed. permanent=True means retrying won't help.""" + + def __init__(self, message: str, permanent: bool = False): + super().__init__(message) + self.permanent = permanent + + +class RemoteGone(SyndicationError): + """The remote post no longer exists""" + + +class MastodonClient: + """Minimal Mastodon-API client (works with GoToSocial)""" + + def __init__(self, base_url: str, token: str, visibility: str = "public", + http: Optional[httpx.Client] = None): + self.base_url = base_url.rstrip("/") + self.visibility = visibility + self.http = http or httpx.Client(timeout=30.0) + self.headers = {"Authorization": f"Bearer {token}"} + + def _call(self, method: str, path: str, **kwargs) -> httpx.Response: + headers = {**self.headers, **kwargs.pop("headers", {})} + try: + response = self.http.request(method, self.base_url + path, headers=headers, **kwargs) + except httpx.HTTPError as e: + raise SyndicationError(f"{method} {path}: {e}") + if response.status_code == 404: + raise RemoteGone(f"{method} {path}: not found", permanent=True) + if response.status_code >= 400: + # 4xx other than rate limiting won't fix itself + permanent = 400 <= response.status_code < 500 and response.status_code != 429 + raise SyndicationError( + f"{method} {path}: HTTP {response.status_code} {response.text[:200]}", + permanent=permanent, + ) + return response + + def upload_media(self, path: Path, mime_type: str, description: str) -> str: + with open(path, "rb") as f: + response = self._call( + "POST", "/api/v2/media", + files={"file": (path.name, f, mime_type)}, + data={"description": description or ""}, + ) + return response.json()["id"] + + def _status_body(self, text: str, media_ids: List[str]) -> dict: + return { + "status": text, + "media_ids": media_ids, + "visibility": self.visibility, + "content_type": "text/markdown", + } + + def create_status(self, text: str, media_ids: List[str], idempotency_key: str) -> dict: + response = self._call( + "POST", "/api/v1/statuses", + json=self._status_body(text, media_ids), + headers={"Idempotency-Key": idempotency_key}, + ) + data = response.json() + return {"id": data["id"], "url": data.get("url") or data.get("uri")} + + def update_status(self, remote_id: str, text: str, media_ids: List[str]) -> dict: + response = self._call("PUT", f"/api/v1/statuses/{remote_id}", + json=self._status_body(text, media_ids)) + data = response.json() + return {"id": data["id"], "url": data.get("url") or data.get("uri")} + + def delete_status(self, remote_id: str) -> None: + try: + self._call("DELETE", f"/api/v1/statuses/{remote_id}") + except RemoteGone: + pass # already gone + + +@dataclass +class Target: + id: str # stored in syndications.target + uid: str # Micropub syndicate-to uid + name: str # shown in the editor and Micropub clients + client: MastodonClient + max_chars: int = 5000 + max_media: int = 4 + + +_client_cache: Dict[str, MastodonClient] = {} + + +def get_targets() -> Dict[str, Target]: + """Configured syndication targets, by id""" + config = current_app.config + targets: Dict[str, Target] = {} + + url = (config.get("GOTOSOCIAL_URL") or "").rstrip("/") + token = config.get("GOTOSOCIAL_ACCESS_TOKEN") or "" + if url and token: + client = _client_cache.get(url) + if client is None: + client = MastodonClient(url, token, config.get("GOTOSOCIAL_VISIBILITY", "public")) + _client_cache[url] = client + targets["gotosocial"] = Target( + id="gotosocial", + uid=url + "/", + name=config.get("GOTOSOCIAL_NAME", "GoToSocial"), + client=client, + max_chars=int(config.get("GOTOSOCIAL_MAX_CHARS", 5000)), + max_media=int(config.get("GOTOSOCIAL_MAX_MEDIA", 4)), + ) + return targets + + +def resolve_targets(requested: List[str]) -> List[str]: + """Map Micropub uids or target ids to configured target ids""" + targets = get_targets() + by_uid = {t.uid.rstrip("/"): t.id for t in targets.values()} + chosen = [] + for value in requested: + value = (value or "").strip() + target_id = value if value in targets else by_uid.get(value.rstrip("/")) + if target_id and target_id not in chosen: + chosen.append(target_id) + return chosen + + +# --- Queue ----------------------------------------------------------------- + +def select_targets(note_id: int, target_ids: List[str]) -> None: + """ + Set which targets a note goes to, and queue the reconcile + + Targets not listed are disabled (their remote post, if any, is deleted). + Nothing is recorded for unlisted targets the note never went to. + """ + db = get_db(current_app) + for target_id in get_targets(): + enabled = 1 if target_id in target_ids else 0 + existing = db.execute( + "SELECT id FROM syndications WHERE note_id = ? AND target = ?", + (note_id, target_id), + ).fetchone() + if existing: + db.execute( + """ + UPDATE syndications + SET enabled = ?, state = 'pending', version = version + 1, attempts = 0, + last_error = NULL, next_attempt_at = datetime('now'), updated_at = datetime('now') + WHERE id = ? + """, + (enabled, existing["id"]), + ) + elif enabled: + db.execute( + "INSERT INTO syndications (note_id, target, enabled) VALUES (?, ?, 1)", + (note_id, target_id), + ) + db.commit() + + +def note_changed(note_id: int) -> None: + """Queue a reconcile for every target the note was sent to""" + db = get_db(current_app) + db.execute( + """ + UPDATE syndications + SET state = 'pending', version = version + 1, attempts = 0, last_error = NULL, + next_attempt_at = datetime('now'), updated_at = datetime('now') + WHERE note_id = ? + """, + (note_id,), + ) + db.commit() + + +def retry(syndication_id: int) -> None: + """Retry a failed or pending row now""" + db = get_db(current_app) + db.execute( + """ + UPDATE syndications + SET state = 'pending', attempts = 0, last_error = NULL, next_attempt_at = datetime('now') + WHERE id = ? + """, + (syndication_id,), + ) + db.commit() + + +def get_note_syndications(note_id: int) -> List[dict]: + """Rows for a note, with the target's display name""" + targets = get_targets() + rows = get_db(current_app).execute( + "SELECT * FROM syndications WHERE note_id = ? ORDER BY target", (note_id,) + ).fetchall() + result = [] + for row in rows: + item = dict(row) + item["name"] = targets[row["target"]].name if row["target"] in targets else row["target"] + result.append(item) + return result + + +# --- Composing -------------------------------------------------------------- + +def _hashtag(display_name: str) -> str: + word = re.sub(r"[^\w]", "", display_name) + return f"#{word}" if word else "" + + +def compose_text(note, permalink: str, tags: List[str], max_chars: int) -> str: + """Full note as Markdown, then the permalink and hashtags; shortened if too long""" + hashtags = " ".join(h for h in (_hashtag(t) for t in tags) if h) + tail = f"\n\n{permalink}" + (f"\n\n{hashtags}" if hashtags else "") + body = note.content.strip() + if len(body) + len(tail) > max_chars: + room = max_chars - len(tail) - 1 + body = body[:room].rsplit(" ", 1)[0].rstrip() + "…" + return body + tail + + +def _media_files(note_id: int, limit: int) -> List[dict]: + """Image files for a note: the 'large' variant when there is one""" + from starpunk.media import get_note_media + + media_dir = Path(current_app.config["DATA_PATH"]) / "media" + files = [] + for media in get_note_media(note_id)[:limit]: + variant = (media.get("variants") or {}).get("large") + rel = variant["path"] if variant else media["path"] + files.append({ + "path": media_dir / rel, + "mime_type": media.get("mime_type") or "image/jpeg", + "description": media.get("caption") or "", + }) + return files + + +# --- Worker ----------------------------------------------------------------- + +def _claim(db, row_id: int) -> bool: + cursor = db.execute( + f""" + UPDATE syndications SET claimed_until = datetime('now', '+{CLAIM_MINUTES} minutes') + WHERE id = ? AND state = 'pending' + AND (claimed_until IS NULL OR claimed_until < datetime('now')) + """, + (row_id,), + ) + db.commit() + return cursor.rowcount == 1 + + +def _finish(db, row, remote_id: Optional[str], remote_url: Optional[str]) -> None: + # Only mark synced if nothing changed while we were sending + db.execute( + """ + UPDATE syndications + SET remote_id = ?, remote_url = ?, claimed_until = NULL, attempts = 0, last_error = NULL, + state = CASE WHEN version = ? THEN 'synced' ELSE 'pending' END, + updated_at = datetime('now') + WHERE id = ? + """, + (remote_id, remote_url, row["version"], row["id"]), + ) + db.commit() + + +def _fail(db, row, error: SyndicationError) -> None: + attempts = row["attempts"] + 1 + failed = error.permanent or attempts >= MAX_ATTEMPTS + delay = min(30 * (2 ** attempts), 6 * 3600) + db.execute( + f""" + UPDATE syndications + SET attempts = ?, last_error = ?, claimed_until = NULL, + state = CASE WHEN version = ? THEN ? ELSE 'pending' END, + next_attempt_at = datetime('now', '+{delay} seconds'), updated_at = datetime('now') + WHERE id = ? + """, + (attempts, str(error)[:500], row["version"], "failed" if failed else "pending", row["id"]), + ) + db.commit() + current_app.logger.warning( + f"Syndication {row['target']} for note {row['note_id']} failed " + f"(attempt {attempts}{', giving up' if failed else ''}): {error}" + ) + + +def _reconcile(row, target: Target) -> tuple: + """Bring the remote post in line with the note. Returns (remote_id, remote_url).""" + from starpunk.notes import get_note + from starpunk.tags import get_note_tags + + note = get_note(id=row["note_id"], load_content=True) + wanted = bool(row["enabled"]) and note is not None and note.published + + if not wanted: + if row["remote_id"]: + target.client.delete_status(row["remote_id"]) + return None, None + + site = current_app.config["SITE_URL"].rstrip("/") + tags = [t["display_name"] for t in get_note_tags(note.id)] + text = compose_text(note, f"{site}/note/{note.slug}", tags, target.max_chars) + media_ids = [ + target.client.upload_media(f["path"], f["mime_type"], f["description"]) + for f in _media_files(note.id, target.max_media) + ] + + if row["remote_id"]: + try: + result = target.client.update_status(row["remote_id"], text, media_ids) + return result["id"], result["url"] + except RemoteGone: + pass # deleted on the remote side; post it again + + result = target.client.create_status( + text, media_ids, idempotency_key=f"starpunk-{row['id']}-{row['version']}" + ) + return result["id"], result["url"] + + +def process_due(limit: int = 5) -> int: + """Reconcile up to `limit` pending rows. Returns how many were handled.""" + targets = get_targets() + if not targets: + return 0 + db = get_db(current_app) + rows = db.execute( + """ + SELECT * FROM syndications + WHERE state = 'pending' AND next_attempt_at <= datetime('now') + AND (claimed_until IS NULL OR claimed_until < datetime('now')) + ORDER BY next_attempt_at LIMIT ? + """, + (limit,), + ).fetchall() + + handled = 0 + for row in rows: + if not _claim(db, row["id"]): + continue # another worker took it + handled += 1 + target = targets.get(row["target"]) + if target is None: + _fail(db, row, SyndicationError(f"target {row['target']} is not configured", permanent=True)) + continue + try: + remote_id, remote_url = _reconcile(row, target) + except SyndicationError as e: + _fail(db, row, e) + continue + except Exception as e: # never let one bad note stop the worker + _fail(db, row, SyndicationError(f"unexpected error: {e}")) + continue + _finish(db, row, remote_id, remote_url) + if remote_id: + current_app.logger.info(f"Syndicated note {row['note_id']} to {row['target']}: {remote_url}") + return handled + + +class SyndicationWorker(threading.Thread): + """Daemon thread that drains the syndication queue""" + + def __init__(self, app, poll_interval: int = 10): + super().__init__(daemon=True, name="syndication-worker") + self.app = app + self.poll_interval = poll_interval + self._stop_event = threading.Event() + + def run(self): + while not self._stop_event.is_set(): + try: + with self.app.app_context(): + handled = process_due() + except Exception as e: + self.app.logger.error(f"Syndication worker error: {e}", exc_info=True) + handled = 0 + if handled == 0: + self._stop_event.wait(self.poll_interval) + + def stop(self): + self._stop_event.set() diff --git a/templates/admin/edit.html b/templates/admin/edit.html index 2e1b02c..e22bd03 100644 --- a/templates/admin/edit.html +++ b/templates/admin/edit.html @@ -52,15 +52,58 @@ + {% if syndication_targets %} + +
+ Crosspost + {% for target in syndication_targets %} +
+ + +
+ {% endfor %} +
+ {% endif %} +
Cancel -
- - -
+
+ + {# Outside the edit form: nested forms are not allowed in HTML #} +
+ +
+ + {% if syndications %} + + {% endif %} {% endblock %} diff --git a/templates/admin/new.html b/templates/admin/new.html index a60f85c..4875a23 100644 --- a/templates/admin/new.html +++ b/templates/admin/new.html @@ -65,6 +65,19 @@ + {% if syndication_targets %} + +
+ Crosspost + {% for target in syndication_targets %} +
+ + +
+ {% endfor %} +
+ {% endif %} +
Cancel diff --git a/templates/note.html b/templates/note.html index 4e1570d..4a7853a 100644 --- a/templates/note.html +++ b/templates/note.html @@ -49,6 +49,9 @@