diff --git a/.env.example b/.env.example index 00d5ea5..01278a4 100644 --- a/.env.example +++ b/.env.example @@ -206,6 +206,22 @@ THREADS_USERNAME= # Stop checking a note's responses after this many days (default: 90) # BACKFEED_MAX_AGE_DAYS=90 +# ============================================================================= +# NOTIFICATIONS (v2.14.0, ADR-074; optional) +# ============================================================================= + +# Apprise URLs to notify, space- or comma-separated +# (https://github.com/caronc/apprise/wiki). Empty: notifications off. +# Examples: gotifys://gotify.example.com/APPTOKEN ntfys://my-topic +# apprises://apprise.example.com/KEY (an apprise-api server) +# NOTIFY_URLS= +# What to notify: reply, mention, quote, like, repost, bookmark, problems +# (a crosspost that failed for good, a silo refusing backfeed) +# NOTIFY_EVENTS=reply,mention,quote,problems +# Priority of every message, added to URLs that don't set one: +# low, moderate, normal, high, emergency (Gotify also takes 0-10) +# NOTIFY_PRIORITY=normal + # ============================================================================= # DEBUGGING # ============================================================================= diff --git a/CHANGELOG.md b/CHANGELOG.md index adc6fc6..efcd090 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [2.14.0] - 2026-11-10 + +Notifications (ADR-074). + +### Added + +- StarPunk can notify you of new responses through [Apprise](https://github.com/caronc/apprise): Gotify, ntfy, Pushover, email, Matrix, Discord and about 100 more, or an apprise-api server. Set `NOTIFY_URLS` to one or more Apprise URLs (e.g. `gotifys://gotify.example.com/APPTOKEN`) +- `NOTIFY_EVENTS` chooses what you hear about: `reply`, `mention`, `quote`, `like`, `repost`, `bookmark`, and `problems` (a crosspost that failed for good, or a silo refusing backfeed). Default `reply,mention,quote,problems`. Replies, mentions and quotes come one at a time with their text; likes, reposts and bookmarks are grouped per note +- `NOTIFY_PRIORITY` (default `normal`) sets the priority of every message +- Received webmentions and backfed responses are notified alike, each once. Responses waiting for moderation say so and link to the admin +- Admin → Webmentions shows the notification settings and the last send error, with **Send test notification** + +### Changed + +- New dependency: `apprise` 2.2. Migration 021 marks every existing response as already notified, and responses found by a backfill (or on a crosspost's first check of an older note) are never notified + ## [2.13.4] - 2026-11-10 ### Added diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index 7560405..ed15200 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -202,9 +202,9 @@ coordinated through the data volume: | Thread | Started when | Coordination | Work | |--------|--------------|--------------|------| -| Webmention worker (`webmention_worker.py`) | `WEBMENTION_WORKER_ENABLED` and not testing | Non-blocking `fcntl.flock` on `webmention-worker.lock` next to the database. Only the holder works the queues; the others try again each poll interval. The OS releases the lock if the holder dies, and the new holder resets rows stuck in `sending`. | Validates received webmentions; discovers endpoints and sends queued outgoing webmentions, with retries | +| Webmention worker (`webmention_worker.py`) | `WEBMENTION_WORKER_ENABLED` and not testing | Non-blocking `fcntl.flock` on `webmention-worker.lock` next to the database. Only the holder works the queues; the others try again each poll interval. The OS releases the lock if the holder dies, and the new holder resets rows stuck in `sending`. | Validates received webmentions; discovers endpoints and sends queued outgoing webmentions, with retries; saves response avatars (v2.13.0); sends notifications of new responses (v2.14.0, `notifications.py`, ADR-074) | | WebSub worker (`websub_worker.py`) | `WEBSUB_ENABLED` and not testing | Same pattern with `websub-worker.lock` | Verifies subscription intents, delivers topic content (signed with `X-Hub-Signature` when the subscriber gave a secret), retries, expires leases; pings an external hub instead if `WEBSUB_HUB_URL` is set | -| Syndication worker (`syndication.py`) | At least one crosspost target configured, and not testing | No file lock. Rows in `syndications` are claimed with a 5-minute `claimed_until` lease by an atomic `UPDATE`, so any process may work a row but only one at a time; a `version` column keeps edits made during a send. Threads token refreshes are serialised with a lock file next to `threads_token.json`. | Creates, updates or deletes the remote post for each (note, target) pair; up to 8 attempts | +| Syndication worker (`syndication.py`) | At least one crosspost target configured, and not testing | No file lock. Rows in `syndications` are claimed with a 5-minute `claimed_until` lease by an atomic `UPDATE`, so any process may work a row but only one at a time; a `version` column keeps edits made during a send. Threads token refreshes are serialised with a lock file next to `threads_token.json`. | Creates, updates or deletes the remote post for each (note, target) pair; up to 8 attempts. Then backfeed: checks copies for responses (v2.13.0, `backfeed.py`, ADR-073) | All workers run inside an app context, are daemon threads (they end with the process) and sleep for their poll interval when there is nothing to do. Under diff --git a/docs/architecture/technology-stack.md b/docs/architecture/technology-stack.md index 15c9262..d9179cf 100644 --- a/docs/architecture/technology-stack.md +++ b/docs/architecture/technology-stack.md @@ -43,6 +43,7 @@ vulnerabilities (`pip-audit`); check again before each release. | python-dateutil | `4.0.*` | Dates in reply contexts | | feedgen | `1.0.*` | RSS 2.0 feed (`starpunk/feeds/rss.py`); Atom and JSON Feed are generated by StarPunk's own code | | httpx | `0.28.*` | All outbound HTTP; requests to user-supplied URLs go through `webmention_ssrf.safe_get`/`safe_post` | +| apprise | `2.2.*` | Notifications to the owner's services (`starpunk/notifications.py`, ADR-074); sends to the URLs the owner configures, outside the SSRF-safe fetcher | | PyJWT[crypto] | `2.15.*` | Verifying OIDC ID tokens (JWKS, RS/ES signatures via `cryptography`) | | python-dotenv | `1.2.*` | Loading `.env` | | Pillow | `13.0.*` | Image validation, resizing and variants | diff --git a/docs/design/INDEX.md b/docs/design/INDEX.md index 499274e..da6de71 100644 --- a/docs/design/INDEX.md +++ b/docs/design/INDEX.md @@ -21,7 +21,7 @@ is in `../architecture/`; decisions are in `../decisions/` (ADRs). ## Contents -[v1.0.0](#v100) · [v1.1.1](#v111) · [v1.1.2](#v112) · [v1.2.0](#v120) · [v1.3.0](#v130) · [v1.3.1](#v131) · [v1.4.0](#v140) · [v1.4.1](#v141) · [v1.4.2](#v142) · [v1.5.0](#v150) · [v1.6.0](#v160) · [v2.0.0](#v200) · [v2.0.1](#v201) · [v2.0.2](#v202) · [v2.1.0](#v210) · [v2.2.0](#v220) · [v2.3.0](#v230) · [v2.3.1](#v231) · [v2.3.2](#v232) · [v2.3.3](#v233) · [v2.4.0](#v240) · [v2.5.0](#v250) · [v2.6.0](#v260) · [v2.6.1](#v261) · [v2.6.2](#v262) · [v2.6.3](#v263) · [v2.6.4](#v264) · [v2.6.5](#v265) · [v2.6.6](#v266) · [v2.6.7](#v267) · [v2.6.8](#v268) · [v2.6.9](#v269) · [v2.7.0](#v270) · [v2.7.1](#v271) · [v2.7.2](#v272) · [v2.8.0](#v280) · [v2.8.1](#v281) · [v2.9.0](#v290) · [v2.10.0](#v2100) · [v2.10.1](#v2101) · [v2.10.2](#v2102) · [v2.11.0](#v2110) · [v2.12.0](#v2120) · [v2.12.1](#v2121) · [v2.13.0](#v2130) · [v2.13.1](#v2131) · [v2.13.2](#v2132) · [v2.13.3](#v2133) · [v2.13.4](#v2134) · [hotfix](#hotfix) +[v1.0.0](#v100) · [v1.1.1](#v111) · [v1.1.2](#v112) · [v1.2.0](#v120) · [v1.3.0](#v130) · [v1.3.1](#v131) · [v1.4.0](#v140) · [v1.4.1](#v141) · [v1.4.2](#v142) · [v1.5.0](#v150) · [v1.6.0](#v160) · [v2.0.0](#v200) · [v2.0.1](#v201) · [v2.0.2](#v202) · [v2.1.0](#v210) · [v2.2.0](#v220) · [v2.3.0](#v230) · [v2.3.1](#v231) · [v2.3.2](#v232) · [v2.3.3](#v233) · [v2.4.0](#v240) · [v2.5.0](#v250) · [v2.6.0](#v260) · [v2.6.1](#v261) · [v2.6.2](#v262) · [v2.6.3](#v263) · [v2.6.4](#v264) · [v2.6.5](#v265) · [v2.6.6](#v266) · [v2.6.7](#v267) · [v2.6.8](#v268) · [v2.6.9](#v269) · [v2.7.0](#v270) · [v2.7.1](#v271) · [v2.7.2](#v272) · [v2.8.0](#v280) · [v2.8.1](#v281) · [v2.9.0](#v290) · [v2.10.0](#v2100) · [v2.10.1](#v2101) · [v2.10.2](#v2102) · [v2.11.0](#v2110) · [v2.12.0](#v2120) · [v2.12.1](#v2121) · [v2.13.0](#v2130) · [v2.13.1](#v2131) · [v2.13.2](#v2132) · [v2.13.3](#v2133) · [v2.13.4](#v2134) · [v2.14.0](#v2140) · [hotfix](#hotfix) ## v1.0.0 @@ -442,6 +442,10 @@ OIDC admin login and built-in IndieAuth server. - [2026-11-10-list-counts.md](v2.13.4/2026-11-10-list-counts.md) - Counts on lists of notes; whole reply card links to the reply; class clash fixed +## v2.14.0 + +- [2026-11-10-notifications.md](v2.14.0/2026-11-10-notifications.md) - Notifications through Apprise (ADR-074) + ## hotfix Not a release folder. diff --git a/docs/design/v2.14.0/2026-11-10-notifications.md b/docs/design/v2.14.0/2026-11-10-notifications.md new file mode 100644 index 0000000..12d519f --- /dev/null +++ b/docs/design/v2.14.0/2026-11-10-notifications.md @@ -0,0 +1,38 @@ +# v2.14.0: Notifications (implementation report) + +Implements ADR-074. + +## What changed + +- `starpunk/notifications.py` (new): settings (`urls`, `events`, + `with_priority`), `send` (Apprise, Markdown, 15 s timeout, never raises, + errors captured with `apprise.LogCapture`), `problem`, and `Sweeper`, + which sends due responses, groups likes/reposts/bookmarks per note, + marks `notified_at`, backs off after a failure (10 s doubling to 15 min), + drops responses older than 24 hours, and records the last error in + `site_state.notify_last_error`. +- `migrations/021_notifications.sql`: `webmentions.notified_at`, set on all + existing rows. +- `webmention_worker.py`: runs the sweep each pass (one process holds the + worker lock). +- `backfeed.py`: `store(..., quiet=True)` inserts new rows as notified; + `check()` is quiet for backfill checks and a copy's first check of a note + older than a day. A refused credential sends a `problems` message. +- `syndication.py`: `_fail` sends a `problems` message when it gives up. +- Admin Webmentions page: status (events, services, priority, last error) + and **Send test notification** (`POST /admin/notifications/test`). +- Settings `NOTIFY_URLS`, `NOTIFY_EVENTS`, `NOTIFY_PRIORITY`; dependency + `apprise==2.2.*`. + +## Verification + +- `tests/test_notifications.py` (11 tests): settings parsing and priority + injection; one message per reply/mention, likes grouped, unselected types + marked, pending moderation linked to the admin; nothing sent twice; + failure backoff and retry, last error set and cleared; 24-hour drop; off + without URLs; quiet backfill and late first checks; problems (selected or + not) and the final crosspost failure; admin status and test button; a + real Apprise send (`json://`) to a local server; migration 021. +- Webmention, syndication, backfeed, admin and migration tests pass. +- Apprise 2.2.0 installs and imports on Python 3.14. +- Production Gotify (3.1.1) accepted a message with the StarPunk app token. diff --git a/docs/projectplan/BACKLOG.md b/docs/projectplan/BACKLOG.md index e371904..f7b3ff9 100644 --- a/docs/projectplan/BACKLOG.md +++ b/docs/projectplan/BACKLOG.md @@ -74,11 +74,6 @@ None open. - **Where**: `templates/base.html` - **Source**: Documentation refresh 2026-11-03 -### Notifications -- **Description**: Notify the owner of new replies, mentions, quotes (and optionally likes, reposts, bookmarks) and of crosspost/backfeed problems, through the embedded Apprise library (`NOTIFY_URLS`, `NOTIFY_EVENTS`). Decided in ADR-074 (Proposed). -- **Where**: new `starpunk/notifications.py`, `starpunk/webmention_worker.py`, `starpunk/backfeed.py`, `starpunk/syndication.py`, migration 021 -- **Source**: Owner request 2026-11-10; ADR-074 - ### Tag-filtered feeds - **Description**: Tag pages are h-feeds and WebSub topics, but there are no RSS, Atom or JSON feeds per tag (for example `/tag/python/feed.atom`). Tag pages also have no pagination, and the editor has no tag autocomplete. - **Where**: `starpunk/routes/public.py`, `starpunk/feeds/`, `templates/admin/` diff --git a/docs/standards/versioning-strategy.md b/docs/standards/versioning-strategy.md index 5516775..2500ad8 100644 --- a/docs/standards/versioning-strategy.md +++ b/docs/standards/versioning-strategy.md @@ -4,7 +4,7 @@ **Last Updated**: 2026-11-03 (v2.10.1) StarPunk uses [Semantic Versioning 2.0.0](https://semver.org/): -`MAJOR.MINOR.PATCH`, currently **2.13.4**. Every release is a git tag +`MAJOR.MINOR.PATCH`, currently **2.14.0**. Every release is a git tag `vX.Y.Z` and a `CHANGELOG.md` entry. Where the number is stored and how to write the changelog entry is in [version-implementation-guide.md](version-implementation-guide.md); the diff --git a/migrations/021_notifications.sql b/migrations/021_notifications.sql new file mode 100644 index 0000000..be14ef9 --- /dev/null +++ b/migrations/021_notifications.sql @@ -0,0 +1,6 @@ +-- Migration 021: Notifications (v2.14.0, ADR-074) +-- When the owner was notified of a response. Everything already stored +-- counts as notified, so turning notifications on doesn't send history. + +ALTER TABLE webmentions ADD COLUMN notified_at TIMESTAMP; +UPDATE webmentions SET notified_at = CURRENT_TIMESTAMP; diff --git a/requirements.txt b/requirements.txt index 5795ea1..01c4747 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,6 +19,9 @@ httpx==0.28.* PyJWT[crypto]==2.15.* python-dotenv==1.2.* +# Notifications (v2.14.0, ADR-074) +apprise==2.2.* + # Images Pillow==13.0.* pillow-heif==1.8.* diff --git a/starpunk/__init__.py b/starpunk/__init__.py index bf87a89..dc8a4cb 100644 --- a/starpunk/__init__.py +++ b/starpunk/__init__.py @@ -465,5 +465,5 @@ def create_app(config=None): # Package version (Semantic Versioning 2.0.0) # See docs/standards/versioning-strategy.md for details -__version__ = "2.13.4" -__version_info__ = (2, 13, 4) +__version__ = "2.14.0" +__version_info__ = (2, 14, 0) diff --git a/starpunk/backfeed.py b/starpunk/backfeed.py index dd2a6cf..83cd924 100644 --- a/starpunk/backfeed.py +++ b/starpunk/backfeed.py @@ -258,12 +258,14 @@ def hidden_counts(harvest: Harvest) -> Dict[str, int]: (("likes", "like"), ("reposts", "repost"), ("quotes", "quote"))} -def store(db, note_id: int, target_url: str, origin: str, harvest: Harvest) -> None: +def store(db, note_id: int, target_url: str, origin: str, harvest: Harvest, + quiet: bool = False) -> None: """ Upsert a check's responses for one note and silo. With a complete harvest, that silo's responses for the note that weren't found are marked invalid. Responses from people who opted out are removed (they - are counted instead, see `hidden_counts`). + are counted instead, see `hidden_counts`). With `quiet`, new responses + are stored as already notified (ADR-074 Decision 5). """ from starpunk.webmention_processor import _snippet @@ -289,8 +291,9 @@ def store(db, note_id: int, target_url: str, origin: str, harvest: Harvest) -> N INSERT INTO webmentions ( source_url, target_url, target_note_id, mention_type, author_name, author_url, author_photo, content_snippet, validation_status, moderation_status, - validated_at, source_published, origin, remote_id) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'valid', ?, CURRENT_TIMESTAMP, ?, ?, ?) + validated_at, source_published, origin, remote_id, notified_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'valid', ?, CURRENT_TIMESTAMP, ?, ?, ?, + CASE WHEN ? THEN CURRENT_TIMESTAMP END) ON CONFLICT(source_url, target_url) DO UPDATE SET target_note_id = excluded.target_note_id, mention_type = excluded.mention_type, @@ -307,7 +310,8 @@ def store(db, note_id: int, target_url: str, origin: str, harvest: Harvest) -> N updated_at = CURRENT_TIMESTAMP """, (r.source_url, target_url, note_id, r.kind, r.author_name or None, r.author_url or None, - r.author_photo or None, snippet, moderation, r.published, origin, r.remote_id), + r.author_photo or None, snippet, moderation, r.published, origin, r.remote_id, + 1 if quiet else 0), ) if dropped: marks = ",".join("?" * len(dropped)) @@ -373,7 +377,11 @@ def check(row, target) -> None: db = get_db(current_app) site = current_app.config["SITE_URL"].rstrip("/") harvest = target.client.responses(row["remote_id"], row["remote_url"] or "") - store(db, row["note_id"], f"{site}/note/{row['note_slug']}", row["target"], harvest) + # No notifications for history: a backfill, or a copy's first check on an + # older note (e.g. one linked after the fact). ADR-074 Decision 5. + quiet = bool(row["backfeed_backfill"]) or ( + row["backfeed_checked_at"] is None and _age_seconds(row["note_created"]) > DAY) + store(db, row["note_id"], f"{site}/note/{row['note_slug']}", row["target"], harvest, quiet=quiet) _store_counts(db, row["id"], hidden_counts(harvest)) @@ -398,6 +406,11 @@ def process_due(limit: int = 3) -> int: if _is_denied(e): denied[row["target"]] = f"Needs {SCOPES.get(row['target'], 'read access')}: {e}" current_app.logger.error(f"Backfeed for {row['target']} stopped: {denied[row['target']]}") + from starpunk import notifications + site = current_app.config["SITE_URL"].rstrip("/") + notifications.problem(current_app.config, current_app.config["DATABASE_PATH"], + f"Backfeed for {SILO_NAMES.get(row['target'], row['target'])} stopped", + denied[row["target"]], f"{site}/admin/webmentions") db.execute("UPDATE syndications SET backfeed_claimed_until = NULL WHERE id = ?", (row["id"],)) db.commit() continue diff --git a/starpunk/config.py b/starpunk/config.py index adbaa74..8439dda 100644 --- a/starpunk/config.py +++ b/starpunk/config.py @@ -149,6 +149,10 @@ def load_config(app, config_override=None): # Backfeed of responses to crossposts (v2.13.0, ADR-073) app.config["BACKFEED_ENABLED"] = os.getenv("BACKFEED_ENABLED", "false").lower() == "true" app.config["BACKFEED_MAX_AGE_DAYS"] = int(os.getenv("BACKFEED_MAX_AGE_DAYS", "90")) + # Notifications through Apprise (v2.14.0, ADR-074) + app.config["NOTIFY_URLS"] = os.getenv("NOTIFY_URLS", "") + app.config["NOTIFY_EVENTS"] = os.getenv("NOTIFY_EVENTS", "reply,mention,quote,problems") + app.config["NOTIFY_PRIORITY"] = os.getenv("NOTIFY_PRIORITY", "normal") # Apply overrides if provided # CSRF protection for admin and consent forms (v2.11.0); on by default. diff --git a/starpunk/notifications.py b/starpunk/notifications.py new file mode 100644 index 0000000..1085dae --- /dev/null +++ b/starpunk/notifications.py @@ -0,0 +1,236 @@ +""" +Notifications through Apprise (v2.14.0, ADR-074) + +The owner hears about new responses (received webmentions and backfeed +alike) and about crossposting problems, on whatever service the Apprise +URLs in NOTIFY_URLS point at (Gotify, ntfy, Pushover, email, an +apprise-api server…). + +Responses are sent by a sweep (`sweep`) in the webmention worker, which +runs in one process at a time: each response with `notified_at` empty is +sent once and marked. Likes, reposts and bookmarks are grouped per note. +Failed sends are retried with backoff; responses still unsent after 24 +hours are dropped. Problems (`problem`) are sent at once, without retry. + +Nothing here may break validation, crossposting or backfeed: every send is +wrapped, and failures are only logged and shown in the admin. +""" + +import html +import logging +import re +import sqlite3 +import time +from typing import Dict, List, Optional, Tuple +from urllib.parse import parse_qsl, urlsplit + +logger = logging.getLogger(__name__) + +RESPONSE_EVENTS = ("reply", "mention", "quote", "like", "repost", "bookmark") +EVENTS = RESPONSE_EVENTS + ("problems",) +DEFAULT_EVENTS = "reply,mention,quote,problems" +GROUPED = ("like", "repost", "bookmark") +DROP_AFTER_HOURS = 24 +MAX_BACKOFF = 15 * 60 +SEND_TIMEOUT = 15 + +HEADINGS = { + "reply": "↩ Reply from {who}", + "mention": "💬 Mention from {who}", + "quote": "❝ Quote from {who}", +} +GROUP_HEADINGS = { + "like": ("♥", "like", "likes"), + "repost": ("🔁", "repost", "reposts"), + "bookmark": ("🔖", "bookmark", "bookmarks"), +} + + +# --- Settings --------------------------------------------------------------------- + +def urls(config) -> List[str]: + """NOTIFY_URLS as a list (space- or comma-separated)""" + return [u for u in re.split(r"[\s,]+", config.get("NOTIFY_URLS") or "") if u] + + +def events(config) -> set: + """NOTIFY_EVENTS as a set of known event names""" + chosen = {e.strip().lower() for e in (config.get("NOTIFY_EVENTS") or DEFAULT_EVENTS).split(",")} + return chosen & set(EVENTS) + + +def enabled(config) -> bool: + return bool(urls(config)) + + +def with_priority(url: str, priority: str) -> str: + """The URL with `priority=` added, unless it sets one itself""" + if not priority or any(k == "priority" for k, _ in parse_qsl(urlsplit(url).query)): + return url + return url + ("&" if "?" in url else "?") + "priority=" + priority + + +# --- Sending ---------------------------------------------------------------------------- + +def _deliver(targets: List[str], title: str, body: str) -> Tuple[bool, str]: + """Send one message to every URL. Returns (all sent, error text).""" + import apprise + + sender = apprise.Apprise() + bad = [u.split("://", 1)[0] + "://…" for u in targets if not sender.add(u)] + if bad: + return False, f"Not a usable Apprise URL: {', '.join(bad)}" + with apprise.LogCapture(level=logging.WARNING, fmt="%(message)s") as log: + result = sender.notify(body, title=title, body_format=apprise.NotifyFormat.MARKDOWN, + timeout=SEND_TIMEOUT) + return bool(result), log.getvalue().strip()[:500] + + +def send(config, title: str, body: str) -> Tuple[bool, str]: + """Send to all NOTIFY_URLS at NOTIFY_PRIORITY. Never raises.""" + targets = [with_priority(u, config.get("NOTIFY_PRIORITY") or "normal") for u in urls(config)] + if not targets: + return False, "No NOTIFY_URLS set" + try: + ok, error = _deliver(targets, title, body) + except Exception as e: # a broken service mustn't break the caller + ok, error = False, f"{type(e).__name__}: {e}" + if not ok: + logger.warning(f"Notification '{title}' not sent: {error or 'unknown error'}") + return ok, error + + +def problem(config, db_path, title: str, detail: str, link: str) -> None: + """A crosspost or backfeed problem, sent at once if 'problems' is selected""" + if not enabled(config) or "problems" not in events(config): + return + ok, error = send(config, f"⚠ {title}", f"{detail}\n\n[Open in StarPunk]({link})") + _record(db_path, ok, error) + + +def _record(db_path, ok: bool, error: str) -> None: + """Remember the last failure for the admin page (cleared by a success)""" + try: + conn = sqlite3.connect(db_path) + try: + conn.execute("INSERT INTO site_state (key, value) VALUES ('notify_last_error', ?) " + "ON CONFLICT(key) DO UPDATE SET value = excluded.value", + ("" if ok else time.strftime("%Y-%m-%d %H:%M:%S UTC ", time.gmtime()) + + (error or "unknown error"),)) + conn.commit() + finally: + conn.close() + except sqlite3.Error as e: + logger.warning(f"Could not record notification result: {e}") + + +def last_error(db) -> str: + row = db.execute("SELECT value FROM site_state WHERE key = 'notify_last_error'").fetchone() + return row[0] if row else "" + + +# --- Messages ----------------------------------------------------------------------------- + +def _plain(snippet: Optional[str], limit: int = 300) -> str: + text = html.unescape(re.sub(r"<[^>]+>", " ", snippet or "")) + text = re.sub(r"\s+", " ", text).strip() + return text if len(text) <= limit else text[:limit - 1].rsplit(" ", 1)[0] + "…" + + +def _note_label(conn, note_id: int) -> str: + from starpunk.notes import get_note + + try: + note = get_note(id=note_id, load_content=True) + label = " ".join((note.title or "").split()) if note else "" + except Exception: + label = "" + return (label[:57] + "…") if len(label) > 60 else (label or "a note") + + +def _messages(rows, site_url: str, conn) -> List[Tuple[str, str, List[int]]]: + """(title, body, webmention ids) for a batch of responses""" + site = site_url.rstrip("/") + names = {"gotosocial": "GoToSocial", "bluesky": "Bluesky", "threads": "Threads"} + messages, groups = [], {} + for row in rows: + via = names.get(row["origin"]) if row["origin"] != "webmention" else None + who = (row["author_name"] or "someone") + (f" via {via}" if via else "") + pending = row["moderation_status"] == "pending" + link = (f"{site}/admin/webmentions/{row['id']}" if pending + else f"{site}/note/{row['slug']}#responses") + if row["mention_type"] in GROUPED: + groups.setdefault((row["target_note_id"], row["mention_type"], row["slug"], pending), []).append( + (row["id"], row["author_name"] or "someone", link)) + continue + title = HEADINGS.get(row["mention_type"], "Response from {who}").format(who=who) + if pending: + title += " (awaiting moderation)" + text = _plain(row["content_snippet"]) + body = (f"{text}\n\n" if text else "") + f"On “{_note_label(conn, row['target_note_id'])}”: [open]({link})" + messages.append((title, body, [row["id"]])) + for (note_id, kind, slug, pending), items in groups.items(): + icon, one, many = GROUP_HEADINGS[kind] + n = len(items) + title = f"{icon} {n} new {one if n == 1 else many} on “{_note_label(conn, note_id)}”" + if pending: + title += " (awaiting moderation)" + people = ", ".join(name for _, name, _ in items[:5]) + (f" and {n - 5} more" if n > 5 else "") + messages.append((title, f"{people}\n\n[open]({items[0][2]})", [i for i, _, _ in items])) + return messages + + +# --- Sweep ---------------------------------------------------------------------------------- + +class Sweeper: + """Sends new responses; one instance per webmention worker""" + + def __init__(self): + self._failures = 0 + self._next_try = 0.0 + + def run(self, config, db_path, site_url: str) -> int: + """Send what's due. Returns how many responses were sent.""" + if not enabled(config) or time.time() < self._next_try: + return 0 + chosen = events(config) + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + try: + # Too old to be news (an outage, say): drop quietly + dropped = conn.execute( + f"""UPDATE webmentions SET notified_at = CURRENT_TIMESTAMP + WHERE notified_at IS NULL + AND received_at < datetime('now', '-{DROP_AFTER_HOURS} hours')""").rowcount + if dropped: + logger.warning(f"Dropped {dropped} notification(s) older than {DROP_AFTER_HOURS} hours") + rows = conn.execute( + """ + SELECT w.*, n.slug FROM webmentions w JOIN notes n ON n.id = w.target_note_id + WHERE w.notified_at IS NULL AND w.validation_status = 'valid' + AND w.moderation_status IN ('approved', 'pending') + ORDER BY w.id LIMIT 100 + """ + ).fetchall() + skip = [r["id"] for r in rows if r["mention_type"] not in chosen] + if skip: + conn.execute(f"UPDATE webmentions SET notified_at = CURRENT_TIMESTAMP " + f"WHERE id IN ({','.join('?' * len(skip))})", skip) + conn.commit() + due = [r for r in rows if r["mention_type"] in chosen] + sent = 0 + for title, body, ids in _messages(due, site_url, conn): + ok, error = send(config, title, body) + _record(db_path, ok, error) + if not ok: + self._failures += 1 + self._next_try = time.time() + min(10 * 2 ** self._failures, MAX_BACKOFF) + break + self._failures = 0 + conn.execute(f"UPDATE webmentions SET notified_at = CURRENT_TIMESTAMP " + f"WHERE id IN ({','.join('?' * len(ids))})", ids) + conn.commit() + sent += len(ids) + return sent + finally: + conn.close() diff --git a/starpunk/routes/admin.py b/starpunk/routes/admin.py index e0997e1..1f48f72 100644 --- a/starpunk/routes/admin.py +++ b/starpunk/routes/admin.py @@ -515,6 +515,7 @@ def webmentions(): status_filter=status_filter, origin_filter=origin_filter, backfeed=backfeed.status(), + notify=_notify_status(), page=page, total_pages=total_pages, user_me=g.me @@ -681,6 +682,34 @@ def revoke_token(token_id): return redirect(url_for("admin.tokens")) +def _notify_status() -> dict: + """Notification settings and last error for the Webmentions page (v2.14.0)""" + from starpunk import notifications + from starpunk.database import get_db + + return { + "enabled": notifications.enabled(current_app.config), + "events": sorted(notifications.events(current_app.config)), + "priority": current_app.config.get("NOTIFY_PRIORITY") or "normal", + "services": [u.split("://", 1)[0] for u in notifications.urls(current_app.config)], + "last_error": notifications.last_error(get_db(current_app)), + } + + +@bp.route("/notifications/test", methods=["POST"]) +@require_auth +def notifications_test(): + """Send a test notification to every NOTIFY_URLS service (v2.14.0)""" + from starpunk import notifications + + ok, error = notifications.send(current_app.config, "StarPunk test notification", + "Notifications from StarPunk are working.") + notifications._record(current_app.config["DATABASE_PATH"], ok, error) + flash("Test notification sent" if ok else f"Test notification failed: {error or 'unknown error'}", + "success" if ok else "error") + return redirect(url_for("admin.webmentions")) + + @bp.route("/backfeed/backfill", methods=["POST"]) @require_auth def backfeed_backfill(): diff --git a/starpunk/syndication.py b/starpunk/syndication.py index 1a2aff8..853e440 100644 --- a/starpunk/syndication.py +++ b/starpunk/syndication.py @@ -1464,6 +1464,13 @@ def _fail(db, row, error: SyndicationError) -> None: f"Syndication {row['target']} for note {row['note_id']} failed " f"(attempt {attempts}{', giving up' if failed else ''}): {error}" ) + if failed: # ADR-074: tell the owner, once, when we give up + from starpunk import notifications + target = get_targets().get(row["target"]) + site = current_app.config["SITE_URL"].rstrip("/") + notifications.problem(current_app.config, current_app.config["DATABASE_PATH"], + f"Crosspost to {target.name if target else row['target']} failed", + str(error)[:300], f"{site}/admin/edit/{row['note_id']}") def _target_title(url: Optional[str]) -> Optional[str]: diff --git a/starpunk/webmention_worker.py b/starpunk/webmention_worker.py index 6c971d0..4eb7639 100644 --- a/starpunk/webmention_worker.py +++ b/starpunk/webmention_worker.py @@ -83,7 +83,9 @@ class WebmentionWorker(threading.Thread): self._last_processed = None self._lock_file = None from starpunk.avatars import Upkeep + from starpunk.notifications import Sweeper self._avatar_upkeep = Upkeep() + self._notifier = Sweeper() def run(self): """ @@ -118,6 +120,11 @@ class WebmentionWorker(threading.Thread): # Local copies of response avatars (v2.13.0) self._avatar_upkeep.run(self.db_path, self.app.config['DATA_PATH'], self.site_url) + # New responses to the owner (v2.14.0, ADR-074) + try: + self._notifier.run(self.app.config, self.db_path, self.site_url) + except Exception as e: + logger.error(f"Notification sweep failed: {e}", exc_info=True) # Update last processed time if processed > 0: diff --git a/templates/admin/webmentions.html b/templates/admin/webmentions.html index 323fb25..51becfb 100644 --- a/templates/admin/webmentions.html +++ b/templates/admin/webmentions.html @@ -12,6 +12,22 @@ + {# Notifications (v2.14.0, ADR-074) #} + {% if notify %} +
+

Notifications

+ {% if notify.enabled %} +

Sending {{ notify.events|join(', ') }} to {{ notify.services|join(', ') }} at {{ notify.priority }} priority.

+ {% if notify.last_error %}

Last send failed: {{ notify.last_error }}

{% endif %} +
+ +
+ {% else %} +

Off. Set NOTIFY_URLS to one or more Apprise URLs to be notified of new responses.

+ {% endif %} +
+ {% endif %} + {# Backfeed (v2.13.0, ADR-073) #} {% if backfeed and backfeed.enabled %}
diff --git a/tests/test_notifications.py b/tests/test_notifications.py new file mode 100644 index 0000000..efbf0db --- /dev/null +++ b/tests/test_notifications.py @@ -0,0 +1,251 @@ +""" +Tests for notifications through Apprise (v2.14.0, ADR-074) + +Most tests replace the Apprise send with a recorder. One sends for real +through Apprise to a local HTTP server (json://), to check the wiring. +""" + +import json +import sqlite3 +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer + +import pytest + +from starpunk import backfeed, create_app, notifications, syndication +from starpunk.backfeed import Harvest, Response +from starpunk.database import get_db +from starpunk.notes import create_note + +SITE = "https://blog.example.com" + + +@pytest.fixture +def sent(monkeypatch): + messages = [] + + def deliver(targets, title, body): + messages.append({"targets": targets, "title": title, "body": body}) + return True, "" + + monkeypatch.setattr(notifications, "_deliver", deliver) + return messages + + +@pytest.fixture +def app(tmp_path): + data = tmp_path / "data" + app = create_app({ + "TESTING": True, "SITE_URL": SITE, "ADMIN_ME": "https://example.com", + "SESSION_SECRET": "x", "DATA_PATH": data, "NOTES_PATH": data / "notes", + "DATABASE_PATH": data / "starpunk.db", + "NOTIFY_URLS": "gotifys://gotify.example.com/TOKEN, ntfys://topic?priority=max", + "NOTIFY_EVENTS": "reply,mention,like,problems", + "NOTIFY_PRIORITY": "high", + "CSRF_ENABLED": False, + }) + yield app + syndication._client_cache.clear() + backfeed.denied.clear() + + +def note(app, content="Hello world", age_days=0): + with app.app_context(): + n = create_note(content=content, published=True) + if age_days: + get_db(app).execute("UPDATE notes SET created_at = datetime('now', ?) WHERE id = ?", + (f"-{age_days} days", n.id)) + get_db(app).commit() + return n + + +def response(app, n, kind, name, source, moderation="approved", origin="webmention", + snippet=None, hours_ago=0): + with app.app_context(): + get_db(app).execute( + "INSERT INTO webmentions (source_url, target_url, target_note_id, mention_type, author_name, " + "validation_status, moderation_status, origin, content_snippet, received_at) " + "VALUES (?, ?, ?, ?, ?, 'valid', ?, ?, ?, datetime('now', ?))", + (source, f"{SITE}/note/{n.slug}", n.id, kind, name, moderation, origin, snippet, + f"-{hours_ago} hours")) + get_db(app).commit() + + +def sweep(app, sweeper=None): + sweeper = sweeper or notifications.Sweeper() + with app.app_context(): + return sweeper.run(app.config, app.config["DATABASE_PATH"], SITE), sweeper + + +def unsent(app): + with app.app_context(): + return [r[0] for r in get_db(app).execute( + "SELECT source_url FROM webmentions WHERE notified_at IS NULL ORDER BY id")] + + +# --- Settings -------------------------------------------------------------------------- + +def test_settings(): + config = {"NOTIFY_URLS": " a://x,b://y c://z ", "NOTIFY_EVENTS": "Reply, likes, problems"} + assert notifications.urls(config) == ["a://x", "b://y", "c://z"] + assert notifications.events(config) == {"reply", "problems"}, "unknown names are ignored" + assert notifications.events({}) == {"reply", "mention", "quote", "problems"} + assert notifications.with_priority("gotifys://h/T", "high") == "gotifys://h/T?priority=high" + assert notifications.with_priority("ntfys://t?tags=x", "5") == "ntfys://t?tags=x&priority=5" + assert notifications.with_priority("ntfys://t?priority=max", "low") == "ntfys://t?priority=max" + + +# --- Sweep ------------------------------------------------------------------------------------- + +def test_new_responses_are_sent_once_and_likes_grouped(app, sent): + n = note(app, "Starpunk now supports backfeeds") + response(app, n, "reply", "Ana", "https://a.example/r", origin="bluesky", + snippet="

Great & useful!

") + response(app, n, "like", "Bob", "https://b.example/l") + response(app, n, "like", "Cy", "https://c.example/l") + response(app, n, "repost", "Dee", "https://d.example/rp") # not selected + response(app, n, "mention", "Eve", "https://e.example/m", moderation="pending") + count, _ = sweep(app) + assert count == 4 + titles = [m["title"] for m in sent] + assert "↩ Reply from Ana via Bluesky" in titles + assert "💬 Mention from Eve (awaiting moderation)" in titles + assert "♥ 2 new likes on “Starpunk now supports backfeeds”" in titles + reply = next(m for m in sent if m["title"].startswith("↩")) + assert reply["body"].startswith("Great & useful!") + assert f"({SITE}/note/{n.slug}#responses)" in reply["body"] + mention = next(m for m in sent if m["title"].startswith("💬")) + assert f"{SITE}/admin/webmentions/" in mention["body"], "pending ones link to moderation" + assert all(t.endswith("priority=high") or t.endswith("priority=max") for t in sent[0]["targets"]) + assert unsent(app) == [] + sent.clear() + assert sweep(app)[0] == 0 and sent == [], "nothing is sent twice" + + +def test_failed_send_is_retried_with_backoff(app, monkeypatch): + calls = [] + monkeypatch.setattr(notifications, "_deliver", lambda t, ti, b: (calls.append(ti), (False, "boom"))[1]) + n = note(app) + response(app, n, "reply", "Ana", "https://a.example/r") + count, sweeper = sweep(app) + assert count == 0 and unsent(app) == ["https://a.example/r"] + assert sweep(app, sweeper)[0] == 0 and len(calls) == 1, "waits before retrying" + with app.app_context(): + assert "boom" in notifications.last_error(get_db(app)) + sweeper._next_try = 0 + monkeypatch.setattr(notifications, "_deliver", lambda t, ti, b: (True, "")) + assert sweep(app, sweeper)[0] == 1 + with app.app_context(): + assert notifications.last_error(get_db(app)) == "" + + +def test_old_unsent_responses_are_dropped(app, sent): + n = note(app) + response(app, n, "reply", "Ana", "https://a.example/r", hours_ago=30) + assert sweep(app)[0] == 0 and sent == [] and unsent(app) == [] + + +def test_off_without_urls(app, sent): + app.config["NOTIFY_URLS"] = "" + n = note(app) + response(app, n, "reply", "Ana", "https://a.example/r") + assert sweep(app)[0] == 0 and unsent(app) == ["https://a.example/r"] + + +def test_backfill_and_late_first_checks_are_quiet(app, sent): + class Client: + config_token = "t" + def responses(self, remote_id, remote_url, cap=1000): + return Harvest([Response(kind="reply", source_url=f"{remote_url}/reply", remote_id="r", + author_name="Ana", content_html="

hi

")]) + def maintain(self): + pass + + app.config.update(GOTOSOCIAL_URL="https://social.example", GOTOSOCIAL_ACCESS_TOKEN="t", + BACKFEED_ENABLED=True) + syndication._client_cache["https://social.example"] = Client() + fresh, old = note(app, "New"), note(app, "Old", age_days=10) + with app.app_context(): + for n, url in ((fresh, "https://social.example/1"), (old, "https://social.example/2")): + get_db(app).execute("INSERT INTO syndications (note_id, target, enabled, remote_id, remote_url, " + "state) VALUES (?, 'gotosocial', 1, 'x', ?, 'synced')", (n.id, url)) + get_db(app).commit() + backfeed.process_due() + assert unsent(app) == ["https://social.example/1/reply"], "only the new note's reply is news" + + +# --- Problems and admin ---------------------------------------------------------------------------- + +def test_problems(app, sent): + with app.app_context(): + notifications.problem(app.config, app.config["DATABASE_PATH"], "Crosspost to Bluesky failed", + "HTTP 400", f"{SITE}/admin/edit/1") + assert sent[0]["title"] == "⚠ Crosspost to Bluesky failed" + assert "HTTP 400" in sent[0]["body"] and "/admin/edit/1" in sent[0]["body"] + app.config["NOTIFY_EVENTS"] = "reply" + with app.app_context(): + notifications.problem(app.config, app.config["DATABASE_PATH"], "x", "y", "z") + assert len(sent) == 1, "not selected: not sent" + + +def test_final_crosspost_failure_notifies(app, sent): + n = note(app) + app.config.update(GOTOSOCIAL_URL="https://social.example", GOTOSOCIAL_ACCESS_TOKEN="t") + with app.app_context(): + cur = get_db(app).execute("INSERT INTO syndications (note_id, target, enabled, state, attempts) " + "VALUES (?, 'gotosocial', 1, 'pending', 0)", (n.id,)) + get_db(app).commit() + row = get_db(app).execute("SELECT * FROM syndications WHERE id = ?", (cur.lastrowid,)).fetchone() + syndication._fail(get_db(app), row, syndication.SyndicationError("temporary", permanent=False)) + assert sent == [], "a retry is coming: no message" + syndication._fail(get_db(app), row, syndication.SyndicationError("HTTP 422 bad", permanent=True)) + assert sent and sent[0]["title"] == "⚠ Crosspost to GoToSocial failed" + + +def test_admin_status_and_test_button(app, sent): + from starpunk.auth import create_session + + with app.test_request_context(): + token = create_session("https://example.com") + browser = app.test_client() + browser.set_cookie("starpunk_session", token) + page = browser.get("/admin/webmentions").get_data(as_text=True) + assert "Sending like, mention, problems, reply to gotifys, ntfys at high priority" in page + page = browser.post("/admin/notifications/test", follow_redirects=True).get_data(as_text=True) + assert "Test notification sent" in page and sent[-1]["title"] == "StarPunk test notification" + + +# --- Real Apprise ------------------------------------------------------------------------------------- + +def test_apprise_delivers_markdown_to_a_service(app): + received = [] + + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + received.append(json.loads(self.rfile.read(int(self.headers["Content-Length"])))) + self.send_response(200) + self.end_headers() + + def log_message(self, *args): + pass + + server = HTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=server.handle_request, daemon=True).start() + app.config["NOTIFY_URLS"] = f"json://127.0.0.1:{server.server_port}/" + ok, error = notifications.send(app.config, "Title", "Body with [a link](https://x.example)") + server.server_close() + assert ok, error + assert received[0]["title"] == "Title" and "a link" in received[0]["message"] + app.config["NOTIFY_URLS"] = "notaservice://nope" + ok, error = notifications.send(app.config, "t", "b") + assert not ok and "Not a usable Apprise URL" in error + + +def test_migration_021_marks_history(tmp_path): + from pathlib import Path + + conn = sqlite3.connect(tmp_path / "db") + conn.execute("CREATE TABLE webmentions (id INTEGER PRIMARY KEY, source_url TEXT)") + conn.execute("INSERT INTO webmentions (source_url) VALUES ('https://old.example')") + conn.executescript((Path(__file__).parent.parent / "migrations" / "021_notifications.sql").read_text()) + assert conn.execute("SELECT notified_at IS NOT NULL FROM webmentions").fetchone()[0] == 1