From 720a1e6d6f0f4b11137ffc650b52a5a4645b90a9 Mon Sep 17 00:00:00 2001 From: Phil Date: Tue, 3 Nov 2026 10:23:00 -0700 Subject: [PATCH 1/2] docs: ADR-071 WebSub publishing with a built-in hub (proposed) Co-Authored-By: Claude Opus 5.5 --- docs/decisions/ADR-071-websub.md | 166 +++++++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) create mode 100644 docs/decisions/ADR-071-websub.md diff --git a/docs/decisions/ADR-071-websub.md b/docs/decisions/ADR-071-websub.md new file mode 100644 index 0000000..0e4743f --- /dev/null +++ b/docs/decisions/ADR-071-websub.md @@ -0,0 +1,166 @@ +# ADR-071: WebSub publishing with a built-in hub + +## Status + +Proposed (v2.10.0, 2026-11-03) + +## Context + +Readers poll StarPunk's feeds. WebSub (W3C Recommendation, +https://www.w3.org/TR/websub/) lets readers that support it (Monocle, +Aperture, Inoreader, NewsBlur, Miniflux and others) subscribe once and +have new content pushed to them within seconds. + +WebSub has three roles. StarPunk is the **publisher**. Someone has to run +the **hub** that keeps subscriptions and delivers content. Public hubs +exist (switchboard.p3k.io, pubsubhubbub.appspot.com), but they are a +third-party dependency and some have shut down. The rest of this site is +self-hosted. + +ADR-014 deferred WebSub for v1 as "complexity, external dependency". + +Two existing problems matter here: + +- Feeds are cached for `FEED_CACHE_SECONDS` (5 minutes) and nothing clears + the cache when a note changes. A hub fetching right after a change would + get stale content. Regular readers are also up to 5 minutes behind. +- The RSS feed is served at `/feed.rss` and `/feed.xml` (legacy alias), and + calls itself `/feed.xml`. + +## Decision + +StarPunk runs its own hub, **for its own topics only**, at `/websub`. +`WEBSUB_HUB_URL` can point at an external hub instead; then StarPunk +advertises that hub, pings it on changes, and its own `/websub` is off. + +### 1. Topics + +Absolute URLs under `SITE_URL`: + +| Topic | Content type | +|---|---| +| `/` (h-feed) | text/html | +| `/feed.atom` | application/atom+xml | +| `/feed.rss` | application/rss+xml | +| `/feed.json` | application/feed+json | +| `/likes` | text/html | +| `/tag/` | text/html | + +`/feed.xml` and the negotiated `/feed` advertise the topic of the format +they serve as `rel=self` (`/feed.rss`, `/feed.atom`, `/feed.json`). The +RSS feed's own self link changes from `/feed.xml` to `/feed.rss`. + +### 2. Discovery (publisher, spec 4) + +Every topic response carries `Link: ; rel="hub"` and exactly one +`Link: ; rel="self"`. Embedded links as well: `` and +`` in the HTML head of topic pages, `` +next to the existing self link in Atom, `` in RSS, and +`"hubs": [{"type": "WebSub", "url": ...}]` in JSON Feed. + +### 3. What changed, and when + +`websub.note_changed(note, old_tags=())` is called wherever +`syndication.note_changed` is, plus on create. It: + +1. clears the feed cache (all formats), and +2. if the note is or was public, queues delivery for `/`, the three + feeds, `/likes` if it is a like, and each `/tag/` for its tags + before and after the change. + +Drafts that stay drafts change nothing. + +### 4. Hub endpoint (spec 5) + +`POST /websub`, `application/x-www-form-urlencoded`: + +- `hub.mode` subscribe or unsubscribe; `hub.topic` must be one of the + topics above (after normalising), else 400 with a plain-text reason; + `hub.callback` must be an absolute http(s) URL that passes the SSRF + check; `hub.secret` under 200 bytes; `hub.lease_seconds` clamped to + 1 day to 30 days (default 10 days). Unknown parameters are ignored. +- Response: 202 Accepted, independent of what verification later finds. +- Limits: the Webmention rate limiter (per IP); at most 25 subscriptions + per callback host and 1,000 in total; beyond that, 429. + +### 5. Verification of intent (spec 5.3) + +A background worker sends `GET ` with the callback's query +string kept and `hub.mode`, `hub.topic`, `hub.challenge` (random, 32 +bytes) and `hub.lease_seconds` appended. Success is a 2xx with the body +equal to the challenge; anything else, including redirects, fails and +leaves the subscription state unchanged. On success a subscribe becomes +active with `expires_at = now + lease`, and an unsubscribe deletes it. +A re-subscribe replaces the secret and lease only once verified. + +### 6. Content distribution (spec 6, 7) + +For each change, one delivery per active, unexpired subscription to the +topic. The worker renders the topic inside the app (no HTTP round trip +through Caddy), so subscribers get exactly what the public URL serves, +then `POST`s it to the callback with: + +- `Content-Type` of the topic, +- `Link` headers with `rel=hub` and `rel=self`, +- `X-Hub-Signature: sha256=` if a secret was + given. + +A 2xx is success. 410 Gone removes the subscription. Anything else is +retried after 1 min, 5 min, 30 min, 2 h and 6 h, then dropped; a +subscription whose deliveries fail for 3 days is removed. Only the latest +content is sent: a queued delivery for a topic is replaced by a newer one. + +### 7. Outbound safety + +Callbacks are arbitrary URLs, so verification and delivery go through +`webmention_ssrf` (`safe_get`/`safe_post`): public addresses only, the +connection pinned to the checked address, no redirects, 10 s time limit, +1 MB response cap. + +### 8. Data (migration 016) + +- `websub_subscriptions(id, topic, callback, secret, lease_seconds, + state ['pending','active'], expires_at, created_at, verified_at, + last_success_at, last_error)`, unique on `(topic, callback)`. +- `websub_intents(id, topic, callback, mode, secret, lease_seconds, + attempts, next_attempt_at, created_at)`: verifications to do. +- `websub_deliveries(id, subscription_id, attempts, next_attempt_at, + last_error, created_at)`, unique on `subscription_id`. + +### 9. Worker + +The same pattern as the Webmention worker: one thread per gunicorn +process, a file lock so only one process works the queues, app context +around each poll. Expired subscriptions are pruned once a day. + +### 10. Admin + +The admin dashboard shows active subscribers per topic and failing +deliveries. + +### 11. Configuration + +`WEBSUB_ENABLED` (default true), `WEBSUB_HUB_URL` (default: built-in), +`WEBSUB_MAX_SUBSCRIPTIONS` (1000), `WEBSUB_MAX_PER_HOST` (25). + +## Consequences + +- Readers that support WebSub get posts and edits within seconds. +- Feeds are fresh right after a change for every reader, because the cache + is cleared. +- StarPunk sends HTTP requests to subscriber-chosen URLs. The SSRF + protections, per-host and total limits, and rate limiting bound that. +- websub.rocks' hub tests use websub.rocks' own topics, which this hub + refuses by design. Testing uses unit tests of each spec section and a + real subscription from a reader after deploy. +- The RSS self link changes to `/feed.rss`. Readers keep whichever URL + they subscribed to; both keep working. + +## Alternatives considered + +- **Public hub only.** Least code, but a third-party dependency for a + self-hosted site. Still available through `WEBSUB_HUB_URL`. +- **A separate hub service** (another container). More to deploy and + keep updated, and few maintained hubs exist. +- **Accepting any topic** (a general-purpose hub). Not needed, and it + would let the hub be used to fetch and push arbitrary URLs. From b520b8b4d34291965bf42da2c49b9959c1707b77 Mon Sep 17 00:00:00 2001 From: Phil Date: Tue, 3 Nov 2026 10:33:41 -0700 Subject: [PATCH 2/2] feat: WebSub with a built-in hub (v2.10.0) ADR-071. Topics (/, /feed.atom, /feed.rss, /feed.json, /likes, /tag/) advertise the hub and themselves in Link headers and embedded links. Note changes bump a content version (fresh feeds in every process) and queue deliveries. /websub accepts subscriptions for these topics only; the worker verifies intent, delivers signed content with retries, and prunes expired leases, all through the SSRF-safe pinned fetcher. WEBSUB_HUB_URL switches to an external hub (publish pings). Also: the memory monitor no longer stops after the first request; RSS self link is /feed.rss. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 16 ++ docs/decisions/ADR-071-websub.md | 2 +- migrations/016_websub.sql | 55 +++++ starpunk/__init__.py | 23 ++- starpunk/config.py | 7 + starpunk/feeds/rss.py | 4 +- starpunk/micropub.py | 3 + starpunk/notes.py | 34 +++- starpunk/routes/__init__.py | 5 +- starpunk/routes/admin.py | 11 + starpunk/routes/public.py | 12 +- starpunk/routes/websub.py | 49 +++++ starpunk/webmention_ssrf.py | 45 ++-- starpunk/websub.py | 340 +++++++++++++++++++++++++++++++ starpunk/websub_worker.py | 287 ++++++++++++++++++++++++++ templates/admin/dashboard.html | 15 ++ templates/base.html | 7 + tests/test_feed.py | 2 +- tests/test_websub.py | 315 ++++++++++++++++++++++++++++ 19 files changed, 1203 insertions(+), 29 deletions(-) create mode 100644 migrations/016_websub.sql create mode 100644 starpunk/routes/websub.py create mode 100644 starpunk/websub.py create mode 100644 starpunk/websub_worker.py create mode 100644 tests/test_websub.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 9dc8927..34a338a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [2.10.0] - 2026-11-03 + +### Added + +- **WebSub** ([ADR-071](docs/decisions/ADR-071-websub.md)): readers that support it get new posts, edits and deletions within seconds instead of polling. StarPunk runs its own hub at `/websub` for its own feeds: the home page, `/feed.atom`, `/feed.rss`, `/feed.json`, `/likes` and each tag page. Each advertises the hub and itself (`Link` headers, and links inside the page or feed). Subscriptions are verified with a challenge, leases expire, deliveries carry the full feed, signed with `X-Hub-Signature` when the subscriber gave a secret, and failures are retried for about nine hours. Requests to subscribers go through the same private-address protections as webmentions, and there are limits per subscriber host and in total. Set `WEBSUB_HUB_URL` to use another hub instead; `WEBSUB_ENABLED=false` turns it off +- The admin dashboard shows WebSub subscribers per feed + +### Fixed + +- Feeds could be up to five minutes out of date after posting, editing or deleting, and each server process cached its own copy. Any change to a public note now refreshes the feeds everywhere at once +- The memory monitor stopped after the first request (the same bug the webmention worker had before v2.6.8) + +### Changed + +- The RSS feed calls itself `/feed.rss` instead of the older `/feed.xml`. Both addresses still work + ## [2.9.0] - 2026-11-03 Micropub review against the [W3C Recommendation](https://www.w3.org/TR/micropub/). diff --git a/docs/decisions/ADR-071-websub.md b/docs/decisions/ADR-071-websub.md index 0e4743f..a6b2597 100644 --- a/docs/decisions/ADR-071-websub.md +++ b/docs/decisions/ADR-071-websub.md @@ -2,7 +2,7 @@ ## Status -Proposed (v2.10.0, 2026-11-03) +Accepted (v2.10.0, 2026-11-03) ## Context diff --git a/migrations/016_websub.sql b/migrations/016_websub.sql new file mode 100644 index 0000000..a4d17bc --- /dev/null +++ b/migrations/016_websub.sql @@ -0,0 +1,55 @@ +-- Migration 016: WebSub hub (v2.10.0, ADR-071) + +-- Active and pending subscriptions to this site's topics +CREATE TABLE IF NOT EXISTS websub_subscriptions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + topic TEXT NOT NULL, + callback TEXT NOT NULL, + secret TEXT, + lease_seconds INTEGER NOT NULL, + expires_at TIMESTAMP NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + verified_at TIMESTAMP, + last_success_at TIMESTAMP, + last_error TEXT, + UNIQUE(topic, callback) +); +CREATE INDEX IF NOT EXISTS idx_websub_subscriptions_topic ON websub_subscriptions(topic); + +-- Subscribe/unsubscribe requests waiting for verification of intent +CREATE TABLE IF NOT EXISTS websub_intents ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + topic TEXT NOT NULL, + callback TEXT NOT NULL, + mode TEXT NOT NULL CHECK (mode IN ('subscribe', 'unsubscribe')), + secret TEXT, + lease_seconds INTEGER, + attempts INTEGER NOT NULL DEFAULT 0, + next_attempt_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE(topic, callback) +); + +-- Content to push: one row per subscription (built-in hub) or per topic +-- (ping to an external hub, subscription_id NULL) +CREATE TABLE IF NOT EXISTS websub_deliveries ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + subscription_id INTEGER REFERENCES websub_subscriptions(id) ON DELETE CASCADE, + topic TEXT NOT NULL, + attempts INTEGER NOT NULL DEFAULT 0, + next_attempt_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + last_error TEXT, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_websub_deliveries_subscription + ON websub_deliveries(subscription_id) WHERE subscription_id IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS idx_websub_deliveries_ping + ON websub_deliveries(topic) WHERE subscription_id IS NULL; + +-- Site-wide values. content_version changes whenever a public note does, so +-- every gunicorn process knows its cached feed note list is stale. +CREATE TABLE IF NOT EXISTS site_state ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); +INSERT OR IGNORE INTO site_state (key, value) VALUES ('content_version', '0'); diff --git a/starpunk/__init__.py b/starpunk/__init__.py index e94c402..d593bb3 100644 --- a/starpunk/__init__.py +++ b/starpunk/__init__.py @@ -309,6 +309,11 @@ def create_app(config=None): response.headers.add('Link', '; rel="webmention"') return response + # WebSub discovery: Link rel=hub and rel=self on topics (v2.10.0) + from starpunk.websub import add_discovery_headers, template_links + app.after_request(add_discovery_headers) + app.add_template_global(template_links, 'websub_links') + # Register centralized error handlers from starpunk.errors import register_error_handlers @@ -322,12 +327,8 @@ def create_app(config=None): memory_monitor.start() app.memory_monitor = memory_monitor app.logger.info(f"Memory monitor started (interval={memory_monitor.interval}s)") - - # Register cleanup handler - @app.teardown_appcontext - def cleanup_memory_monitor(error=None): - if hasattr(app, 'memory_monitor') and app.memory_monitor.is_alive(): - app.memory_monitor.stop() + # No teardown_appcontext stop: that ran after every request and stopped + # the monitor straight away (v2.10.0). It is a daemon thread. # Start webmention worker thread (v1.6.0 Phase 2) # Per background-worker.md: Skip in test mode @@ -350,6 +351,12 @@ def create_app(config=None): f"batch_size={webmention_worker.batch_size})" ) + # Start WebSub worker (v2.10.0) + from starpunk.websub_worker import start_worker + app.websub_worker = start_worker(app) + if app.websub_worker: + app.logger.info("WebSub worker started") + # Start syndication worker when a target is configured (v2.1.0) from starpunk.syndication import SyndicationWorker, targets_configured if targets_configured(app.config) and not app.config.get("TESTING", False): @@ -463,5 +470,5 @@ def create_app(config=None): # Package version (Semantic Versioning 2.0.0) # See docs/standards/versioning-strategy.md for details -__version__ = "2.9.0" -__version_info__ = (2, 9, 0) +__version__ = "2.10.0" +__version_info__ = (2, 10, 0) diff --git a/starpunk/config.py b/starpunk/config.py index 4db2f53..bee665f 100644 --- a/starpunk/config.py +++ b/starpunk/config.py @@ -111,6 +111,13 @@ def load_config(app, config_override=None): # Webmention configuration (v1.6.0) app.config["WEBMENTION_ENABLED"] = os.getenv("WEBMENTION_ENABLED", "true").lower() == "true" + + # WebSub (v2.10.0, ADR-071): built-in hub unless WEBSUB_HUB_URL names another + app.config["WEBSUB_ENABLED"] = os.getenv("WEBSUB_ENABLED", "true").lower() == "true" + app.config["WEBSUB_HUB_URL"] = os.getenv("WEBSUB_HUB_URL", "").strip() + app.config["WEBSUB_MAX_SUBSCRIPTIONS"] = int(os.getenv("WEBSUB_MAX_SUBSCRIPTIONS", "1000")) + app.config["WEBSUB_MAX_PER_HOST"] = int(os.getenv("WEBSUB_MAX_PER_HOST", "25")) + app.config["WEBSUB_POLL_INTERVAL"] = int(os.getenv("WEBSUB_POLL_INTERVAL", "5")) app.config["WEBMENTION_WORKER_ENABLED"] = os.getenv("WEBMENTION_WORKER_ENABLED", "true").lower() == "true" app.config["WEBMENTION_POLL_INTERVAL"] = int(os.getenv("WEBMENTION_POLL_INTERVAL", "10")) app.config["WEBMENTION_ERROR_BACKOFF"] = int(os.getenv("WEBMENTION_ERROR_BACKOFF", "60")) diff --git a/starpunk/feeds/rss.py b/starpunk/feeds/rss.py index 411b772..1778aaa 100644 --- a/starpunk/feeds/rss.py +++ b/starpunk/feeds/rss.py @@ -93,7 +93,7 @@ def generate_rss( fg.language("en") # Add self-link for feed discovery (Atom namespace) - fg.link(href=f"{site_url}/feed.xml", rel="self", type="application/rss+xml") + fg.link(href=f"{site_url}/feed.rss", rel="self", type="application/rss+xml") # Set last build date to now fg.lastBuildDate(datetime.now(timezone.utc)) @@ -246,7 +246,7 @@ def generate_rss_streaming( yield f" {_escape_xml(site_description or site_name)}\n" yield " en\n" yield f" {last_build}\n" - yield f' \n' + yield f' \n' # Yield items (newest first) # Notes from database are already in DESC order (newest first) diff --git a/starpunk/micropub.py b/starpunk/micropub.py index ae024d4..fe225c9 100644 --- a/starpunk/micropub.py +++ b/starpunk/micropub.py @@ -761,6 +761,9 @@ def handle_update(data: dict, token_info: dict): from starpunk.media import attach_media_to_note attach_media_to_note(note.id, media_ids, captions) note_changed(note.id) + if not changes: # update_note already told WebSub otherwise + from starpunk import websub + websub.note_changed(note, was_public=note.published) except InvalidNoteDataError as e: raise MicropubValidationError(str(e)) except MicropubError: diff --git a/starpunk/notes.py b/starpunk/notes.py index 9059432..fb4f27a 100644 --- a/starpunk/notes.py +++ b/starpunk/notes.py @@ -374,9 +374,28 @@ def create_note( # Reply context fetch failure should not prevent note creation current_app.logger.warning(f"Failed to fetch reply context for {in_reply_to}: {e}") + _websub_changed(note) return note +def _tag_names(note_id: int) -> list[str]: + """A note's tag names, for WebSub tag topics (best effort)""" + try: + from starpunk.tags import get_note_tags + return [t["name"] for t in get_note_tags(note_id)] + except Exception: + return [] + + +def _websub_changed(note, was_public: bool = False, old_tags=()) -> None: + """Clear cached feeds and queue WebSub deliveries (v2.10.0, best effort)""" + try: + from starpunk import websub + websub.note_changed(note, was_public=was_public, old_tags=old_tags) + except Exception as e: + current_app.logger.warning(f"Failed to queue WebSub update for note {note.slug}: {e}") + + def get_note( slug: Optional[str] = None, id: Optional[int] = None, load_content: bool = True ) -> Optional[Note]: @@ -705,6 +724,7 @@ def update_note( if existing_note is None: identifier = slug if slug is not None else id raise NoteNotFoundError(identifier) + old_tag_names = _tag_names(existing_note.id) # for WebSub tag topics (v2.10.0) if empty_content and not (existing_note.reacts_to or has_media): raise InvalidNoteDataError( @@ -896,6 +916,8 @@ def update_note( except Exception as e: current_app.logger.warning(f"Failed to queue syndication update: {e}") + _websub_changed(updated_note, was_public=existing_note.published, old_tags=old_tag_names) + # 10. RETURN UPDATED NOTE return updated_note @@ -995,6 +1017,10 @@ def delete_note( # For idempotency, don't raise error - just return return + # What feeds showed before, for WebSub (v2.10.0) + was_public = bool(existing_note.published) and not getattr(existing_note, "deleted_at", None) + deleted_tags = _tag_names(existing_note.id) + # 4. SETUP data_dir = Path(current_app.config["DATA_PATH"]) note_path = data_dir / existing_note.file_path @@ -1081,6 +1107,10 @@ def delete_note( except Exception as e: current_app.logger.warning(f"Failed to queue syndication delete: {e}") + # Feeds and pages no longer show it (v2.10.0) + object.__setattr__(existing_note, "published", False) + _websub_changed(existing_note, was_public=was_public, old_tags=deleted_tags) + # 6. RETURN (no value) return None @@ -1137,4 +1167,6 @@ def undelete_note(slug: str) -> Note: except Exception as e: current_app.logger.warning(f"Failed to queue webmentions after undelete: {e}") - return get_note(slug=slug) + restored = get_note(slug=slug) + _websub_changed(restored) + return restored diff --git a/starpunk/routes/__init__.py b/starpunk/routes/__init__.py index 2bcada9..3054f5e 100644 --- a/starpunk/routes/__init__.py +++ b/starpunk/routes/__init__.py @@ -7,7 +7,7 @@ admin, auth, and (conditionally) dev auth routes. from flask import Flask -from starpunk.routes import admin, auth, indieauth, micropub, public, search, webmention +from starpunk.routes import admin, auth, indieauth, micropub, public, search, webmention, websub def register_routes(app: Flask) -> None: @@ -44,6 +44,9 @@ def register_routes(app: Flask) -> None: if app.config.get("WEBMENTION_ENABLED", True): app.register_blueprint(webmention.bp) + # WebSub hub (v2.10.0); refuses requests when disabled or using another hub + app.register_blueprint(websub.bp) + # Conditionally register dev auth routes if app.config.get("DEV_MODE"): app.logger.warning( diff --git a/starpunk/routes/admin.py b/starpunk/routes/admin.py index 575e51d..34f2be0 100644 --- a/starpunk/routes/admin.py +++ b/starpunk/routes/admin.py @@ -72,12 +72,23 @@ def dashboard(): except Exception: pass + # WebSub subscribers (v2.10.0) + websub_stats = None + if current_app.config.get('WEBSUB_ENABLED', True): + try: + from starpunk import websub + from starpunk.database import get_db + websub_stats = websub.stats(get_db(current_app)) + except Exception: + pass + return render_template( "admin/dashboard.html", notes=notes, page=page, total_pages=total_pages, webmention_pending_count=webmention_pending_count, + websub_stats=websub_stats, user_me=g.me ) diff --git a/starpunk/routes/public.py b/starpunk/routes/public.py index e2589b3..9e3e880 100644 --- a/starpunk/routes/public.py +++ b/starpunk/routes/public.py @@ -11,6 +11,7 @@ from datetime import datetime, timedelta from flask import Blueprint, abort, render_template, Response, current_app, request, send_from_directory from starpunk.notes import list_notes, get_note +from starpunk.websub import embed_hub from starpunk.feed import generate_feed_streaming # Legacy RSS from starpunk.feeds import ( generate_rss, @@ -55,8 +56,14 @@ def _get_cached_notes(): cache_duration = timedelta(seconds=cache_seconds) now = datetime.utcnow() + # The cached list is stale once any public note changed, in any process + # (v2.10.0): every change bumps content_version in the database + from starpunk.database import get_db + from starpunk.websub import get_content_version + version = (str(current_app.config["DATABASE_PATH"]), get_content_version(get_db(current_app))) + # Check if note list cache is valid - if _feed_cache["notes"] and _feed_cache["timestamp"]: + if _feed_cache["notes"] and _feed_cache["timestamp"] and _feed_cache.get("version") == version: cache_age = now - _feed_cache["timestamp"] if cache_age < cache_duration: # Use cached note list @@ -82,6 +89,7 @@ def _get_cached_notes(): _feed_cache["notes"] = notes _feed_cache["timestamp"] = now + _feed_cache["version"] = version return notes @@ -122,6 +130,7 @@ def _generate_feed_with_cache(format_name: str, non_streaming_generator): notes=notes, limit=max_items, ) + content = embed_hub(format_name, content) # WebSub hub link (v2.10.0) # Create response with proper Content-Type including charset mime_type = get_mime_type(format_name) @@ -170,6 +179,7 @@ def _generate_feed_with_cache(format_name: str, non_streaming_generator): notes=notes, limit=max_items, ) + content = embed_hub(format_name, content) # WebSub hub link (v2.10.0) # Store in cache and get ETag etag = feed_cache.set(format_name, content, notes_checksum) diff --git a/starpunk/routes/websub.py b/starpunk/routes/websub.py new file mode 100644 index 0000000..0abbce7 --- /dev/null +++ b/starpunk/routes/websub.py @@ -0,0 +1,49 @@ +""" +WebSub hub endpoint (v2.10.0, ADR-071) + +POST /websub with hub.mode subscribe/unsubscribe, hub.topic, hub.callback, +optional hub.secret and hub.lease_seconds. Answers 202 and verifies later +(spec 5.1); errors are 4xx with a plain-text reason. +""" + +from flask import Blueprint, current_app, request + +from starpunk.database import get_db +from starpunk.rate_limiter import RateLimiter +from starpunk import websub + +bp = Blueprint("websub", __name__) + +# Subscription requests per client IP per minute +_limiter = RateLimiter(limit=30, window_seconds=60) + + +def _text(message: str, status: int): + return message + "\n", status, {"Content-Type": "text/plain; charset=utf-8"} + + +@bp.route("/websub", methods=["POST"]) +def hub(): + if not websub.enabled(): + return _text("WebSub is not enabled on this site", 404) + + from starpunk.routes.webmention import get_client_ip + if not _limiter.is_allowed(get_client_ip()): + return _text("Too many requests", 429) + + if "application/x-www-form-urlencoded" not in request.headers.get("Content-Type", ""): + return _text("Send an application/x-www-form-urlencoded body", 400) + + try: + websub.accept_subscription_request(request.form, get_db(current_app)) + except websub.SubscriptionError as e: + return _text(str(e), e.status) + + return _text("Accepted; the subscription will be verified", 202) + + +@bp.route("/websub", methods=["GET"]) +def hub_info(): + return _text( + "WebSub hub for this site's own feeds. Subscribe with a POST " + "(https://www.w3.org/TR/websub/).", 200) diff --git a/starpunk/webmention_ssrf.py b/starpunk/webmention_ssrf.py index 9aa5b21..4ca5abe 100644 --- a/starpunk/webmention_ssrf.py +++ b/starpunk/webmention_ssrf.py @@ -18,6 +18,7 @@ import ipaddress import socket import time from dataclasses import dataclass +from typing import Optional from urllib.parse import urljoin, urlparse import httpx @@ -157,20 +158,6 @@ def _pinned_request(client: httpx.Client, method: str, url: str, headers: dict, ) -def safe_post(url: str, data: dict, headers: dict, timeout: float = TIMEOUT) -> httpx.Response: - """ - POST form data to a checked, pinned address. Redirects are not followed. - - Raises: - SSRFBlockedError, ValueError: URL is not allowed - httpx.HTTPError: Network errors - """ - with httpx.Client(follow_redirects=False, transport=TRANSPORT, timeout=timeout) as client: - response = client.send(_pinned_request(client, 'POST', url, headers, data=data)) - response.read() - return response - - @dataclass class FetchResult: """A fetched response, after redirects""" @@ -188,6 +175,36 @@ class FetchResult: return self.content.decode('utf-8', errors='replace') +def safe_post(url: str, data: Optional[dict] = None, headers: Optional[dict] = None, + timeout: float = TIMEOUT, content: Optional[bytes] = None, + max_bytes: int = 64 * 1024) -> FetchResult: + """ + POST form data (or a raw body) to a checked, pinned address. + + Redirects are not followed. Only the first max_bytes of the response are + read (callers need the status and maybe an error message). + + Raises: + SSRFBlockedError, ValueError: URL is not allowed + httpx.HTTPError: Network errors + """ + kwargs = {'content': content} if content is not None else {'data': data} + with httpx.Client(follow_redirects=False, transport=TRANSPORT, timeout=timeout) as client: + request = _pinned_request(client, 'POST', url, headers or {}, **kwargs) + response = client.send(request, stream=True) + try: + body = b'' + for chunk in response.iter_bytes(chunk_size=8192): + body += chunk + if len(body) >= max_bytes: + body = body[:max_bytes] + break + return FetchResult(url=url, status_code=response.status_code, headers=response.headers, + content=body, encoding=response.charset_encoding or 'utf-8') + finally: + response.close() + + def safe_get(url: str, headers: dict, max_bytes: int = MAX_BYTES, max_redirects: int = MAX_REDIRECTS, timeout: float = TIMEOUT) -> FetchResult: """ diff --git a/starpunk/websub.py b/starpunk/websub.py new file mode 100644 index 0000000..31c2070 --- /dev/null +++ b/starpunk/websub.py @@ -0,0 +1,340 @@ +""" +WebSub publishing and the built-in hub (v2.10.0, ADR-071) + +Publisher: topic responses advertise the hub and their own URL; a change to a +public note clears cached feeds and queues delivery of every affected topic. + +Hub: subscriptions to this site's own topics only. Requests are accepted +here (202) and verified, delivered and retried by the WebSub worker. + +References: + - W3C WebSub: https://www.w3.org/TR/websub/ + - ADR-071: WebSub publishing with a built-in hub +""" + +import json +import secrets +from typing import Iterable, Optional +from urllib.parse import urlparse + +from flask import current_app, request + + +# Topic endpoints and their fixed paths (tag pages are added per tag) +FEED_TOPICS = { + "public.feed_atom": "/feed.atom", + "public.feed_rss": "/feed.rss", + "public.feed_xml_legacy": "/feed.rss", # legacy alias of the RSS topic + "public.feed_json": "/feed.json", +} +PAGE_TOPICS = {"public.index": "/", "public.likes": "/likes"} +NEGOTIATED = { # /feed: the topic of whichever format was served + "application/atom+xml": "/feed.atom", + "application/rss+xml": "/feed.rss", + "application/feed+json": "/feed.json", + "application/json": "/feed.json", +} + +LEASE_DEFAULT = 10 * 24 * 3600 +LEASE_MIN = 24 * 3600 +LEASE_MAX = 30 * 24 * 3600 +MAX_SECRET_BYTES = 200 + + +class SubscriptionError(Exception): + """A subscription request the hub refuses (4xx with a plain-text reason)""" + + def __init__(self, message: str, status: int = 400): + super().__init__(message) + self.status = status + + +# --- Configuration ----------------------------------------------------------- + +def enabled() -> bool: + return bool(current_app.config.get("WEBSUB_ENABLED", True)) + + +def external_hub() -> Optional[str]: + """URL of an external hub, if one is configured instead of the built-in one""" + return current_app.config.get("WEBSUB_HUB_URL") or None + + +def hub_url() -> str: + # Built from SITE_URL so the worker can use it outside a request + return external_hub() or site_url() + "/websub" + + +def site_url() -> str: + return current_app.config["SITE_URL"].rstrip("/") + + +# --- Topics ------------------------------------------------------------------ + +def topic_url(path: str) -> str: + return site_url() + path + + +def tag_topic(tag_name: str) -> str: + from urllib.parse import quote + return f"{site_url()}/tag/{quote(tag_name, safe='')}" + + +def topic_for_request(response=None) -> Optional[str]: + """The topic URL the current request serves, or None if it isn't a topic""" + endpoint = request.endpoint + if endpoint in PAGE_TOPICS: + return topic_url(PAGE_TOPICS[endpoint]) + if endpoint in FEED_TOPICS: + return topic_url(FEED_TOPICS[endpoint]) + if endpoint == "public.feed" and response is not None: + mime = (response.mimetype or "").lower() + return topic_url(NEGOTIATED[mime]) if mime in NEGOTIATED else None + if endpoint == "public.tag": + from starpunk.tags import normalize_tag + tag = (request.view_args or {}).get("tag", "") + return tag_topic(normalize_tag(tag)[0]) if tag else None + return None + + +def canonical_topic(url: str) -> Optional[str]: + """ + A subscription's hub.topic as one of this site's topics, or None + + Tag topics must name an existing tag. + """ + if not url: + return None + site = urlparse(site_url()) + parsed = urlparse(url.strip()) + if (parsed.scheme, parsed.netloc.lower()) != (site.scheme, site.netloc.lower()): + return None + path = parsed.path or "/" + if parsed.query or parsed.fragment: + return None + if path in set(PAGE_TOPICS.values()) | set(FEED_TOPICS.values()): + return topic_url(path) + if path.startswith("/tag/"): + from starpunk.tags import get_tag_by_name, normalize_tag + name = normalize_tag(path[len("/tag/"):])[0] + if name and get_tag_by_name(name): + return tag_topic(name) + return None + + +def link_header_values(topic: str) -> list[str]: + return [f'<{hub_url()}>; rel="hub"', f'<{topic}>; rel="self"'] + + +def add_discovery_headers(response): + """after_request: Link rel=hub and rel=self on topic responses (spec 4)""" + if not enabled() or response.status_code not in (200, 304): + return response + topic = topic_for_request(response) + if topic: + for value in link_header_values(topic): + response.headers.add("Link", value) + return response + + +def template_links() -> Optional[dict]: + """For base.html: {'hub', 'self'} on topic pages""" + if not enabled() or not request: + return None + topic = topic_for_request() + return {"hub": hub_url(), "self": topic} if topic else None + + +def embed_hub(format_name: str, content: str) -> str: + """Add the hub link inside a generated feed, next to its self link""" + if not enabled(): + return content + hub = hub_url() + if format_name == "atom": + marker = '\n {marker}', 1) + if format_name == "rss": + marker = "\n {marker}', 1) + if format_name == "json": + feed = json.loads(content) + feed["hubs"] = [{"type": "WebSub", "url": hub}] + return json.dumps(feed, ensure_ascii=False, indent=2) + return content + + +def _xml(value: str) -> str: + return (value.replace("&", "&").replace("<", "<") + .replace(">", ">").replace('"', """)) + + +# --- Changes ------------------------------------------------------------------- + +def get_content_version(db) -> str: + row = db.execute("SELECT value FROM site_state WHERE key = 'content_version'").fetchone() + return row[0] if row else "0" + + +def bump_content_version(db) -> None: + db.execute( + "INSERT INTO site_state (key, value) VALUES ('content_version', '1') " + "ON CONFLICT(key) DO UPDATE SET value = CAST(value AS INTEGER) + 1" + ) + + +def note_changed(note, was_public: bool = False, old_tags: Iterable[str] = ()) -> None: + """ + A note was created, edited, deleted, restored or (un)published + + Clears cached feeds in every process and queues delivery of the topics + that show it. Changes that are never public (a draft edited) do nothing. + + Args: + note: The note (after the change; for a delete, before it) + was_public: Whether it was public before the change + old_tags: Its tag names before the change + """ + from starpunk.database import get_db + + is_public = bool(note.published) and not getattr(note, "deleted_at", None) + if not (is_public or was_public): + return + + db = get_db(current_app) + bump_content_version(db) + db.commit() + + if not enabled(): + return + + from starpunk.tags import get_note_tags + try: + tags = {t["name"] for t in get_note_tags(note.id)} + except Exception: + tags = set() + tags |= set(old_tags) + + topics = [topic_url(p) for p in ("/", "/feed.atom", "/feed.rss", "/feed.json")] + if getattr(note, "like_of", None): + topics.append(topic_url("/likes")) + topics += [tag_topic(name) for name in sorted(tags)] + queue_topics(db, topics) + + +def queue_topics(db, topics: Iterable[str]) -> int: + """Queue delivery of topics: to their subscribers, or a ping to an external hub""" + topics = list(dict.fromkeys(topics)) + if not topics: + return 0 + if external_hub(): + for topic in topics: + db.execute("DELETE FROM websub_deliveries WHERE subscription_id IS NULL AND topic = ?", (topic,)) + db.execute("INSERT INTO websub_deliveries (topic) VALUES (?)", (topic,)) + db.commit() + return len(topics) + + placeholders = ",".join("?" * len(topics)) + rows = db.execute( + f"SELECT id, topic FROM websub_subscriptions WHERE topic IN ({placeholders}) " + "AND verified_at IS NOT NULL AND expires_at > datetime('now')", + topics, + ).fetchall() + for sub_id, topic in rows: + # The newest content replaces anything still waiting (ADR-071 6) + db.execute("DELETE FROM websub_deliveries WHERE subscription_id = ?", (sub_id,)) + db.execute("INSERT INTO websub_deliveries (subscription_id, topic) VALUES (?, ?)", (sub_id, topic)) + db.commit() + return len(rows) + + +# --- Hub: subscription requests (spec 5.1) ------------------------------------- + +def accept_subscription_request(form, db) -> None: + """ + Validate a subscription request and queue it for verification + + Raises: + SubscriptionError: with the status and plain-text reason to return + """ + from starpunk.webmention_ssrf import SSRFBlockedError, validate_url_for_fetch + + if external_hub(): + raise SubscriptionError("This site uses another hub: " + external_hub(), 400) + + mode = (form.get("hub.mode") or "").strip() + if mode not in ("subscribe", "unsubscribe"): + raise SubscriptionError("hub.mode must be subscribe or unsubscribe") + + callback = (form.get("hub.callback") or "").strip() + parsed = urlparse(callback) + if parsed.scheme not in ("http", "https") or not parsed.hostname: + raise SubscriptionError("hub.callback must be an absolute http(s) URL") + try: + validate_url_for_fetch(callback) + except (SSRFBlockedError, ValueError) as e: + raise SubscriptionError(f"hub.callback is not reachable: {e}") + + topic = canonical_topic(form.get("hub.topic") or "") + if not topic: + raise SubscriptionError("hub.topic is not a topic of this site; use the rel=self URL") + + secret = form.get("hub.secret") + if mode == "unsubscribe": + secret = None # MUST be ignored for unsubscriptions + if secret is not None and len(secret.encode("utf-8")) >= MAX_SECRET_BYTES: + raise SubscriptionError("hub.secret must be less than 200 bytes") + + lease = LEASE_DEFAULT + if form.get("hub.lease_seconds"): + try: + lease = int(form["hub.lease_seconds"]) + except ValueError: + raise SubscriptionError("hub.lease_seconds must be a number") + lease = max(LEASE_MIN, min(lease, LEASE_MAX)) + + if mode == "subscribe": + existing = db.execute( + "SELECT 1 FROM websub_subscriptions WHERE topic = ? AND callback = ?", (topic, callback) + ).fetchone() + if not existing: + total = db.execute("SELECT COUNT(*) FROM websub_subscriptions").fetchone()[0] + if total >= current_app.config.get("WEBSUB_MAX_SUBSCRIPTIONS", 1000): + raise SubscriptionError("This hub has reached its subscription limit", 429) + host = parsed.hostname.lower() + per_host = sum( + 1 for (cb,) in db.execute("SELECT callback FROM websub_subscriptions") + if (urlparse(cb).hostname or "").lower() == host + ) + if per_host >= current_app.config.get("WEBSUB_MAX_PER_HOST", 25): + raise SubscriptionError(f"Too many subscriptions for {host}", 429) + + # The latest request for a topic and callback replaces any pending one + db.execute( + """ + INSERT INTO websub_intents (topic, callback, mode, secret, lease_seconds) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(topic, callback) DO UPDATE SET + mode = excluded.mode, secret = excluded.secret, + lease_seconds = excluded.lease_seconds, attempts = 0, + next_attempt_at = CURRENT_TIMESTAMP, created_at = CURRENT_TIMESTAMP + """, + (topic, callback, mode, secret, lease), + ) + db.commit() + + +def stats(db) -> dict: + """Active subscribers per topic and deliveries being retried (admin)""" + by_topic = db.execute( + "SELECT topic, COUNT(*) FROM websub_subscriptions WHERE expires_at > datetime('now') " + "GROUP BY topic ORDER BY COUNT(*) DESC, topic" + ).fetchall() + retrying = db.execute( + "SELECT COUNT(*) FROM websub_deliveries WHERE attempts > 0" + ).fetchone()[0] + return {"by_topic": [(t, n) for t, n in by_topic], "retrying": retrying, + "external": external_hub()} + + +def new_challenge() -> str: + return secrets.token_urlsafe(32) diff --git a/starpunk/websub_worker.py b/starpunk/websub_worker.py new file mode 100644 index 0000000..7dec84d --- /dev/null +++ b/starpunk/websub_worker.py @@ -0,0 +1,287 @@ +""" +WebSub worker: verification of intent and content distribution (v2.10.0) + +Runs like the Webmention worker: a daemon thread in every gunicorn process, +but only the process holding the file lock works the queues. + +- Intents (spec 5.3): GET the callback with a challenge; 2xx echoing it + verifies. Anything else (including redirects) leaves the subscription as + it was. Network errors and 5xx are retried a few times. +- Deliveries (spec 6, 7): render the topic in-app and POST it with + Link rel=hub/self and, if there is a secret, X-Hub-Signature (sha256). +- External hub: a delivery without a subscription is a publish ping. + +References: + - https://www.w3.org/TR/websub/ + - ADR-071 +""" + +import fcntl +import hashlib +import hmac +import logging +import threading +from datetime import datetime, timedelta +from pathlib import Path +from typing import Optional +from urllib.parse import urlencode, urlparse, urlsplit, urlunsplit + +import httpx + +logger = logging.getLogger(__name__) + +DELIVERY_BACKOFF = [60, 300, 1800, 7200, 21600] # then give up on this delivery +INTENT_BACKOFF = [60, 300, 900] +DROP_AFTER = timedelta(days=3) # failing this long without a success: unsubscribe +PRUNE_EVERY = timedelta(hours=1) +BATCH = 10 + + +def callback_with_params(callback: str, params: dict) -> str: + """Append parameters to the callback, keeping its query string (spec 5.3)""" + parts = urlsplit(callback) + query = parts.query + ("&" if parts.query else "") + urlencode(params) + return urlunsplit((parts.scheme, parts.netloc, parts.path, query, parts.fragment)) + + +def signature(secret: str, body: bytes) -> str: + return "sha256=" + hmac.new(secret.encode("utf-8"), body, hashlib.sha256).hexdigest() + + +def _now() -> str: + return datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S") + + +def _later(seconds: int) -> str: + return (datetime.utcnow() + timedelta(seconds=seconds)).strftime("%Y-%m-%d %H:%M:%S") + + +class WebSubWorker(threading.Thread): + """Daemon thread that verifies subscriptions and delivers content""" + + def __init__(self, app, poll_interval: int = 5): + super().__init__(daemon=True, name="websub-worker") + self.app = app + self.poll_interval = poll_interval + self._stop_event = threading.Event() + self._lock_file = None + self._last_prune = None + + # --- thread plumbing ---------------------------------------------------- + + def run(self): + logger.info("WebSub worker started") + while not self._stop_event.is_set(): + if not self._acquire_lock(): + self._stop_event.wait(self.poll_interval) + continue + try: + with self.app.app_context(): + done = self.process_once() + except Exception as e: + logger.error(f"WebSub worker error: {e}", exc_info=True) + done = 0 + if not done: + self._stop_event.wait(self.poll_interval) + self._release_lock() + + def stop(self): + self._stop_event.set() + + def _acquire_lock(self) -> bool: + if self._lock_file: + return True + path = Path(self.app.config["DATABASE_PATH"]).parent / "websub-worker.lock" + lock_file = open(path, "w") + try: + fcntl.flock(lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + lock_file.close() + return False + self._lock_file = lock_file + logger.info("WebSub worker holds the queue lock") + return True + + def _release_lock(self): + if self._lock_file: + self._lock_file.close() + self._lock_file = None + + # --- one pass (also called directly by tests) --------------------------- + + def process_once(self) -> int: + from starpunk.database import get_db + + db = get_db(self.app) + done = 0 + for row in db.execute( + "SELECT * FROM websub_intents WHERE next_attempt_at <= ? ORDER BY id LIMIT ?", + (_now(), BATCH), + ).fetchall(): + self.verify(db, row) + done += 1 + for row in db.execute( + "SELECT * FROM websub_deliveries WHERE next_attempt_at <= ? ORDER BY id LIMIT ?", + (_now(), BATCH), + ).fetchall(): + self.deliver(db, row) + done += 1 + if self._last_prune is None or datetime.utcnow() - self._last_prune > PRUNE_EVERY: + self.prune(db) + return done + + # --- verification of intent (spec 5.3) ---------------------------------- + + def verify(self, db, intent) -> None: + from starpunk.websub import new_challenge + from starpunk.webmention_ssrf import SSRFBlockedError, safe_get, user_agent + + challenge = new_challenge() + params = {"hub.mode": intent["mode"], "hub.topic": intent["topic"], "hub.challenge": challenge} + if intent["mode"] == "subscribe": + params["hub.lease_seconds"] = intent["lease_seconds"] + url = callback_with_params(intent["callback"], params) + + try: + result = safe_get(url, headers={"User-Agent": user_agent(self.app.config["SITE_URL"])}, + max_redirects=0, max_bytes=64 * 1024) + verified = 200 <= result.status_code < 300 and result.text.strip() == challenge + retry = result.status_code >= 500 + error = None if verified else f"HTTP {result.status_code}" + ( + "" if result.status_code >= 300 else ": challenge not echoed") + except (SSRFBlockedError, ValueError) as e: + verified, retry, error = False, False, f"Blocked: {e}" + except httpx.TooManyRedirects: + verified, retry, error = False, False, "Redirected (not allowed)" + except httpx.HTTPError as e: + verified, retry, error = False, True, str(e)[:200] + + if verified: + if intent["mode"] == "subscribe": + db.execute( + """ + INSERT INTO websub_subscriptions + (topic, callback, secret, lease_seconds, expires_at, verified_at) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(topic, callback) DO UPDATE SET + secret = excluded.secret, lease_seconds = excluded.lease_seconds, + expires_at = excluded.expires_at, verified_at = excluded.verified_at, + last_error = NULL + """, + (intent["topic"], intent["callback"], intent["secret"], intent["lease_seconds"], + _later(intent["lease_seconds"]), _now()), + ) + else: + db.execute("DELETE FROM websub_subscriptions WHERE topic = ? AND callback = ?", + (intent["topic"], intent["callback"])) + db.execute("DELETE FROM websub_intents WHERE id = ?", (intent["id"],)) + logger.info(f"WebSub {intent['mode']} verified: {intent['callback']} -> {intent['topic']}") + elif retry and intent["attempts"] < len(INTENT_BACKOFF): + db.execute("UPDATE websub_intents SET attempts = attempts + 1, next_attempt_at = ? WHERE id = ?", + (_later(INTENT_BACKOFF[intent["attempts"]]), intent["id"])) + else: + # Verification failed: the subscription state stays as it was + db.execute("DELETE FROM websub_intents WHERE id = ?", (intent["id"],)) + logger.info(f"WebSub {intent['mode']} not verified for {intent['callback']}: {error}") + db.commit() + + # --- content distribution (spec 6, 7) ------------------------------------ + + def render_topic(self, topic: str): + """(status, content_type, body) of a topic, rendered in-app""" + path = urlparse(topic).path or "/" + response = self.app.test_client().get(path, headers={"Accept": "*/*"}) + return response.status_code, response.headers.get("Content-Type", ""), response.get_data() + + def deliver(self, db, delivery) -> None: + from starpunk.websub import hub_url + from starpunk.webmention_ssrf import SSRFBlockedError, safe_post, user_agent + + site = self.app.config["SITE_URL"] + sub = None + try: + if delivery["subscription_id"] is None: + # External hub: ping it (hub.url is the de facto parameter) + result = safe_post(hub_url(), data={"hub.mode": "publish", "hub.url": delivery["topic"], + "hub.topic": delivery["topic"]}, + headers={"User-Agent": user_agent(site)}) + else: + sub = db.execute( + "SELECT * FROM websub_subscriptions WHERE id = ? AND expires_at > ?", + (delivery["subscription_id"], _now()), + ).fetchone() + if sub is None: # unsubscribed or expired meanwhile + db.execute("DELETE FROM websub_deliveries WHERE id = ?", (delivery["id"],)) + db.commit() + return + status, content_type, body = self.render_topic(sub["topic"]) + if status != 200: # e.g. a tag page whose last note went + db.execute("DELETE FROM websub_deliveries WHERE id = ?", (delivery["id"],)) + db.commit() + return + headers = { + "Content-Type": content_type, + "Link": f'<{hub_url()}>; rel="hub", <{sub["topic"]}>; rel="self"', + "User-Agent": user_agent(site), + } + if sub["secret"]: + headers["X-Hub-Signature"] = signature(sub["secret"], body) + result = safe_post(sub["callback"], content=body, headers=headers) + status, error = result.status_code, f"HTTP {result.status_code}" + except (SSRFBlockedError, ValueError) as e: + status, error = None, f"Blocked: {e}" + except httpx.HTTPError as e: + status, error = None, str(e)[:200] or type(e).__name__ + + if status is not None and 200 <= status < 300: + db.execute("DELETE FROM websub_deliveries WHERE id = ?", (delivery["id"],)) + if sub is not None: + db.execute("UPDATE websub_subscriptions SET last_success_at = ?, last_error = NULL WHERE id = ?", + (_now(), sub["id"])) + elif sub is not None and status == 410: + db.execute("DELETE FROM websub_subscriptions WHERE id = ?", (sub["id"],)) + logger.info(f"WebSub subscriber gone (410), removed: {sub['callback']}") + elif delivery["attempts"] < len(DELIVERY_BACKOFF) and not error.startswith("Blocked"): + db.execute( + "UPDATE websub_deliveries SET attempts = attempts + 1, next_attempt_at = ?, last_error = ? WHERE id = ?", + (_later(DELIVERY_BACKOFF[delivery["attempts"]]), error, delivery["id"]), + ) + if sub is not None: + db.execute("UPDATE websub_subscriptions SET last_error = ? WHERE id = ?", (error, sub["id"])) + else: + db.execute("DELETE FROM websub_deliveries WHERE id = ?", (delivery["id"],)) + logger.warning(f"WebSub delivery of {delivery['topic']} gave up: {error}") + if sub is not None: + self._drop_if_failing(db, sub, error) + db.commit() + + def _drop_if_failing(self, db, sub, error: str) -> None: + last_ok = sub["last_success_at"] or sub["verified_at"] or sub["created_at"] + try: + since = datetime.fromisoformat(str(last_ok)) + except ValueError: + return + if datetime.utcnow() - since > DROP_AFTER: + db.execute("DELETE FROM websub_subscriptions WHERE id = ?", (sub["id"],)) + logger.info(f"WebSub subscription failing for {DROP_AFTER.days} days, removed: {sub['callback']}") + else: + db.execute("UPDATE websub_subscriptions SET last_error = ? WHERE id = ?", (error, sub["id"])) + + # --- housekeeping ------------------------------------------------------- + + def prune(self, db) -> int: + """Remove subscriptions whose lease has run out (they weren't renewed)""" + cursor = db.execute("DELETE FROM websub_subscriptions WHERE expires_at <= ?", (_now(),)) + db.commit() + self._last_prune = datetime.utcnow() + if cursor.rowcount: + logger.info(f"WebSub: removed {cursor.rowcount} expired subscriptions") + return cursor.rowcount + + +def start_worker(app) -> Optional[WebSubWorker]: + if not app.config.get("WEBSUB_ENABLED", True) or app.config.get("TESTING", False): + return None + worker = WebSubWorker(app, poll_interval=app.config.get("WEBSUB_POLL_INTERVAL", 5)) + worker.start() + return worker diff --git a/templates/admin/dashboard.html b/templates/admin/dashboard.html index dc2de0b..03fe735 100644 --- a/templates/admin/dashboard.html +++ b/templates/admin/dashboard.html @@ -20,6 +20,21 @@ {% endif %} + {# WebSub subscribers (v2.10.0) #} + {% if websub_stats %} +

+ {% if websub_stats.external %} + WebSub: using {{ websub_stats.external }} + {% elif websub_stats.by_topic %} + WebSub subscribers: + {% for topic, count in websub_stats.by_topic %}{{ topic|replace(config.SITE_URL.rstrip('/'), '') or '/' }} {{ count }}{% if not loop.last %}, {% endif %}{% endfor %} + {% if websub_stats.retrying %} ยท {{ websub_stats.retrying }} deliveries being retried{% endif %} + {% else %} + WebSub: no subscribers yet + {% endif %} +

+ {% endif %} + {% if notes %}
diff --git a/templates/base.html b/templates/base.html index f37a3f4..e9a9aa2 100644 --- a/templates/base.html +++ b/templates/base.html @@ -14,6 +14,13 @@ {% endif %} + {# WebSub hub and topic on feed pages (v2.10.0) #} + {% set websub = websub_links() %} + {% if websub %} + + + {% endif %} + {# Micropub and IndieAuth discovery (independent of Webmention, v2.8.0) #} diff --git a/tests/test_feed.py b/tests/test_feed.py index 8970891..03a122d 100644 --- a/tests/test_feed.py +++ b/tests/test_feed.py @@ -230,7 +230,7 @@ class TestGenerateFeed: # Check for Atom namespace and self-link assert "atom" in feed_xml - assert "feed.xml" in feed_xml + assert "feed.rss" in feed_xml # self link (v2.10.0: /feed.rss is the RSS topic) assert 'rel="self"' in feed_xml def test_generate_feed_item_structure(self, app, sample_notes): diff --git a/tests/test_websub.py b/tests/test_websub.py new file mode 100644 index 0000000..220c14c --- /dev/null +++ b/tests/test_websub.py @@ -0,0 +1,315 @@ +""" +WebSub publisher and built-in hub (v2.10.0, ADR-071) + +Section numbers refer to the W3C WebSub Recommendation. Subscribers are +simulated with an httpx MockTransport behind the SSRF-safe fetcher. +""" + +import hashlib +import hmac +import json +import sqlite3 +from urllib.parse import parse_qs, urlparse + +import httpx +import mf2py +import pytest + +from starpunk.websub_worker import WebSubWorker + +SITE = "http://localhost:5000" +HUB = f"{SITE}/websub" +PUBLIC = [(2, 1, 6, "", ("93.184.216.34", 0))] + + +@pytest.fixture +def web(monkeypatch): + """Subscriber callbacks: handler(request) -> response, by host""" + hosts, seen = {}, [] + + def handler(request): + host = request.headers["host"] + seen.append(request) + return hosts[host](request) + + monkeypatch.setattr("starpunk.webmention_ssrf.socket.getaddrinfo", lambda *a, **k: PUBLIC) + monkeypatch.setattr("starpunk.webmention_ssrf.TRANSPORT", httpx.MockTransport(handler)) + return hosts, seen + + +def echo(request): + """A subscriber that confirms every verification and accepts content""" + if request.method == "GET": + return httpx.Response(200, text=parse_qs(urlparse(str(request.url)).query)["hub.challenge"][0]) + return httpx.Response(202) + + +def db(app): + conn = sqlite3.connect(app.config["DATABASE_PATH"]) + conn.row_factory = sqlite3.Row + return conn + + +def run_worker(app, times=1): + worker = WebSubWorker(app) + with app.app_context(): + for _ in range(times): + worker.process_once() + return worker + + +def subscribe(client, topic, callback="https://sub.example/cb?id=1", **extra): + form = {"hub.mode": "subscribe", "hub.topic": topic, "hub.callback": callback, **extra} + return client.post("/websub", data=form) + + +def make_note(app, content="Hello", **kw): + from starpunk.notes import create_note + with app.test_request_context(): + return create_note(content, published=kw.pop("published", True), **kw) + + +def links(response): + return response.headers.getlist("Link") + + +# 4 Discovery ------------------------------------------------------------------- + +@pytest.mark.parametrize("path,self_url", [ + ("/", f"{SITE}/"), + ("/feed.atom", f"{SITE}/feed.atom"), + ("/feed.rss", f"{SITE}/feed.rss"), + ("/feed.xml", f"{SITE}/feed.rss"), + ("/feed.json", f"{SITE}/feed.json"), + ("/likes", f"{SITE}/likes"), +]) +def test_topics_advertise_hub_and_self(client, path, self_url): + response = client.get(path) + values = links(response) + assert f'<{HUB}>; rel="hub"' in values + assert [v for v in values if 'rel="self"' in v] == [f'<{self_url}>; rel="self"'] + + +def test_negotiated_feed_self_is_the_format_served(client): + response = client.get("/feed", headers={"Accept": "application/atom+xml"}) + assert f'<{SITE}/feed.atom>; rel="self"' in links(response) + + +def test_tag_page_is_a_topic(app, client): + make_note(app, tags=["Python"]) + assert f'<{SITE}/tag/python>; rel="self"' in links(client.get("/tag/Python")) + + +def test_note_page_is_not_a_topic(app, client): + note = make_note(app) + assert not [v for v in links(client.get(f"/note/{note.slug}")) if 'rel="hub"' in v] + + +def test_embedded_links(client): + rels = mf2py.parse(doc=client.get("/").data.decode(), url=SITE + "/")["rels"] + assert rels["hub"] == [HUB] and rels["self"] == [f"{SITE}/"] + assert f'' in client.get("/feed.atom").data.decode() + assert f'' in client.get("/feed.rss").data.decode() + assert json.loads(client.get("/feed.json").data)["hubs"] == [{"type": "WebSub", "url": HUB}] + + +def test_external_hub(app, client, web): + app.config["WEBSUB_HUB_URL"] = "https://hub.example/" + hosts, seen = web + hosts["hub.example"] = lambda r: httpx.Response(204) + assert '; rel="hub"' in links(client.get("/")) + assert subscribe(client, f"{SITE}/").status_code == 400 # our hub is off + + make_note(app) + run_worker(app) + pings = {parse_qs(r.content.decode())["hub.url"][0] for r in seen} + assert pings == {f"{SITE}/", f"{SITE}/feed.atom", f"{SITE}/feed.rss", f"{SITE}/feed.json"} + + +# 5.1 Subscription requests ---------------------------------------------------------- + +def test_subscription_request_is_accepted(app, client, web): + response = subscribe(client, f"{SITE}/feed.atom", **{"hub.secret": "s3cret", "extra": "ignored"}) + assert response.status_code == 202 + row = db(app).execute("SELECT * FROM websub_intents").fetchone() + assert (row["mode"], row["topic"], row["secret"]) == ("subscribe", f"{SITE}/feed.atom", "s3cret") + + +@pytest.mark.parametrize("form,reason", [ + ({"hub.mode": "watch"}, "hub.mode"), + ({"hub.topic": "https://elsewhere.example/feed"}, "hub.topic"), + ({"hub.topic": f"{SITE}/note/x"}, "hub.topic"), + ({"hub.topic": f"{SITE}/tag/missing"}, "hub.topic"), + ({"hub.callback": "ftp://sub.example/cb"}, "hub.callback"), + ({"hub.secret": "x" * 200}, "hub.secret"), +]) +def test_bad_requests(client, web, form, reason): + base = {"hub.mode": "subscribe", "hub.topic": f"{SITE}/", "hub.callback": "https://sub.example/cb"} + response = client.post("/websub", data={**base, **form}) + assert response.status_code == 400 and reason in response.data.decode() + + +def test_private_callback_refused(client, monkeypatch): + monkeypatch.setattr("starpunk.webmention_ssrf.socket.getaddrinfo", + lambda *a, **k: [(2, 1, 6, "", ("10.0.0.2", 0))]) + assert subscribe(client, f"{SITE}/").status_code == 400 + + +def test_per_host_limit(app, client, web): + app.config["WEBSUB_MAX_PER_HOST"] = 1 + hosts, _ = web + hosts["sub.example"] = echo + subscribe(client, f"{SITE}/") + run_worker(app) + assert subscribe(client, f"{SITE}/feed.atom").status_code == 429 + + +# 5.3 Verification of intent -------------------------------------------------------- + +def test_verification_activates_subscription(app, client, web): + hosts, seen = web + hosts["sub.example"] = echo + subscribe(client, f"{SITE}/", **{"hub.lease_seconds": "86400"}) + run_worker(app) + + query = parse_qs(urlparse(str(seen[0].url)).query) + assert query["id"] == ["1"] # the callback's own query string is kept + assert query["hub.mode"] == ["subscribe"] and query["hub.topic"] == [f"{SITE}/"] + assert query["hub.lease_seconds"] == ["86400"] + sub = db(app).execute("SELECT * FROM websub_subscriptions").fetchone() + assert sub["topic"] == f"{SITE}/" and sub["verified_at"] + assert db(app).execute("SELECT COUNT(*) FROM websub_intents").fetchone()[0] == 0 + + +@pytest.mark.parametrize("response", [ + httpx.Response(200, text="wrong"), + httpx.Response(404), + httpx.Response(302, headers={"Location": "https://sub.example/other"}), +]) +def test_failed_verification_changes_nothing(app, client, web, response): + hosts, _ = web + hosts["sub.example"] = lambda r: response + subscribe(client, f"{SITE}/") + run_worker(app) + assert db(app).execute("SELECT COUNT(*) FROM websub_subscriptions").fetchone()[0] == 0 + assert db(app).execute("SELECT COUNT(*) FROM websub_intents").fetchone()[0] == 0 + + +def test_unsubscribe(app, client, web): + hosts, _ = web + hosts["sub.example"] = echo + subscribe(client, f"{SITE}/") + run_worker(app) + client.post("/websub", data={"hub.mode": "unsubscribe", "hub.topic": f"{SITE}/", + "hub.callback": "https://sub.example/cb?id=1"}) + run_worker(app) + assert db(app).execute("SELECT COUNT(*) FROM websub_subscriptions").fetchone()[0] == 0 + + +def test_resubscribe_changes_secret_only_once_verified(app, client, web): + hosts, _ = web + hosts["sub.example"] = echo + subscribe(client, f"{SITE}/", **{"hub.secret": "one"}) + run_worker(app) + hosts["sub.example"] = lambda r: httpx.Response(404) + subscribe(client, f"{SITE}/", **{"hub.secret": "two"}) + run_worker(app) + assert db(app).execute("SELECT secret FROM websub_subscriptions").fetchone()[0] == "one" + + +# 6, 7 Content distribution --------------------------------------------------------- + +def _subscribed(app, client, web, topic=f"{SITE}/feed.atom", secret=None): + hosts, seen = web + hosts["sub.example"] = echo + extra = {"hub.secret": secret} if secret else {} + subscribe(client, topic, **extra) + run_worker(app) + seen.clear() + return hosts, seen + + +def test_new_note_is_delivered_signed(app, client, web): + hosts, seen = _subscribed(app, client, web, secret="s3cret") + make_note(app, "Fresh post") + run_worker(app) + + (post,) = [r for r in seen if r.method == "POST"] + assert post.headers["content-type"].startswith("application/atom+xml") + assert b"Fresh post" in post.content + assert f'<{HUB}>; rel="hub"' in post.headers["link"] and f'<{SITE}/feed.atom>; rel="self"' in post.headers["link"] + expected = "sha256=" + hmac.new(b"s3cret", post.content, hashlib.sha256).hexdigest() + assert post.headers["x-hub-signature"] == expected + assert db(app).execute("SELECT COUNT(*) FROM websub_deliveries").fetchone()[0] == 0 + + +def test_no_signature_without_secret(app, client, web): + hosts, seen = _subscribed(app, client, web) + make_note(app) + run_worker(app) + assert "x-hub-signature" not in [r for r in seen if r.method == "POST"][0].headers + + +def test_drafts_are_not_delivered(app, client, web): + _subscribed(app, client, web) + make_note(app, published=False) + assert db(app).execute("SELECT COUNT(*) FROM websub_deliveries").fetchone()[0] == 0 + + +def test_tag_topics_include_removed_tags(app, client, web): + from starpunk.notes import update_note + note = make_note(app, tags=["old"]) + _subscribed(app, client, web, topic=f"{SITE}/tag/old") + with app.test_request_context(): + update_note(id=note.id, content="Hello", tags=["new"]) + assert db(app).execute("SELECT topic FROM websub_deliveries").fetchone()[0] == f"{SITE}/tag/old" + + +def test_delete_is_delivered(app, client, web): + from starpunk.notes import delete_note + note = make_note(app, "Going away") + hosts, seen = _subscribed(app, client, web) + with app.test_request_context(): + delete_note(id=note.id, soft=False) + run_worker(app) + (post,) = [r for r in seen if r.method == "POST"] + assert b"Going away" not in post.content + + +def test_410_removes_subscription(app, client, web): + hosts, _ = _subscribed(app, client, web) + hosts["sub.example"] = lambda r: httpx.Response(410) + make_note(app) + run_worker(app) + assert db(app).execute("SELECT COUNT(*) FROM websub_subscriptions").fetchone()[0] == 0 + + +def test_failures_are_retried(app, client, web): + hosts, _ = _subscribed(app, client, web) + hosts["sub.example"] = lambda r: httpx.Response(500) + make_note(app) + run_worker(app) + row = db(app).execute("SELECT attempts, next_attempt_at, last_error FROM websub_deliveries").fetchone() + assert row["attempts"] == 1 and row["last_error"] == "HTTP 500" + + +def test_expired_subscriptions_get_nothing(app, client, web): + _subscribed(app, client, web) + conn = db(app) + conn.execute("UPDATE websub_subscriptions SET expires_at = datetime('now', '-1 minute')") + conn.commit() + make_note(app) + assert conn.execute("SELECT COUNT(*) FROM websub_deliveries").fetchone()[0] == 0 + run_worker(app) # prunes + assert conn.execute("SELECT COUNT(*) FROM websub_subscriptions").fetchone()[0] == 0 + + +# Feeds are fresh right after a change (ADR-071 3) ------------------------------------ + +def test_feed_shows_new_note_immediately(app, client): + from starpunk.routes import public + make_note(app, "First") + assert b"First" in client.get("/feed.atom").data + make_note(app, "Second") # cached note list is stale now + assert b"Second" in client.get("/feed.atom").data + assert public._feed_cache["notes"] is not None