diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9dc8927..34a338a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
+## [2.10.0] - 2026-11-03
+
+### Added
+
+- **WebSub** ([ADR-071](docs/decisions/ADR-071-websub.md)): readers that support it get new posts, edits and deletions within seconds instead of polling. StarPunk runs its own hub at `/websub` for its own feeds: the home page, `/feed.atom`, `/feed.rss`, `/feed.json`, `/likes` and each tag page. Each advertises the hub and itself (`Link` headers, and links inside the page or feed). Subscriptions are verified with a challenge, leases expire, deliveries carry the full feed, signed with `X-Hub-Signature` when the subscriber gave a secret, and failures are retried for about nine hours. Requests to subscribers go through the same private-address protections as webmentions, and there are limits per subscriber host and in total. Set `WEBSUB_HUB_URL` to use another hub instead; `WEBSUB_ENABLED=false` turns it off
+- The admin dashboard shows WebSub subscribers per feed
+
+### Fixed
+
+- Feeds could be up to five minutes out of date after posting, editing or deleting, and each server process cached its own copy. Any change to a public note now refreshes the feeds everywhere at once
+- The memory monitor stopped after the first request (the same bug the webmention worker had before v2.6.8)
+
+### Changed
+
+- The RSS feed calls itself `/feed.rss` instead of the older `/feed.xml`. Both addresses still work
+
## [2.9.0] - 2026-11-03
Micropub review against the [W3C Recommendation](https://www.w3.org/TR/micropub/).
diff --git a/docs/decisions/ADR-071-websub.md b/docs/decisions/ADR-071-websub.md
index 0e4743f..a6b2597 100644
--- a/docs/decisions/ADR-071-websub.md
+++ b/docs/decisions/ADR-071-websub.md
@@ -2,7 +2,7 @@
## Status
-Proposed (v2.10.0, 2026-11-03)
+Accepted (v2.10.0, 2026-11-03)
## Context
diff --git a/migrations/016_websub.sql b/migrations/016_websub.sql
new file mode 100644
index 0000000..a4d17bc
--- /dev/null
+++ b/migrations/016_websub.sql
@@ -0,0 +1,55 @@
+-- Migration 016: WebSub hub (v2.10.0, ADR-071)
+
+-- Active and pending subscriptions to this site's topics
+CREATE TABLE IF NOT EXISTS websub_subscriptions (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ topic TEXT NOT NULL,
+ callback TEXT NOT NULL,
+ secret TEXT,
+ lease_seconds INTEGER NOT NULL,
+ expires_at TIMESTAMP NOT NULL,
+ created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ verified_at TIMESTAMP,
+ last_success_at TIMESTAMP,
+ last_error TEXT,
+ UNIQUE(topic, callback)
+);
+CREATE INDEX IF NOT EXISTS idx_websub_subscriptions_topic ON websub_subscriptions(topic);
+
+-- Subscribe/unsubscribe requests waiting for verification of intent
+CREATE TABLE IF NOT EXISTS websub_intents (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ topic TEXT NOT NULL,
+ callback TEXT NOT NULL,
+ mode TEXT NOT NULL CHECK (mode IN ('subscribe', 'unsubscribe')),
+ secret TEXT,
+ lease_seconds INTEGER,
+ attempts INTEGER NOT NULL DEFAULT 0,
+ next_attempt_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ UNIQUE(topic, callback)
+);
+
+-- Content to push: one row per subscription (built-in hub) or per topic
+-- (ping to an external hub, subscription_id NULL)
+CREATE TABLE IF NOT EXISTS websub_deliveries (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ subscription_id INTEGER REFERENCES websub_subscriptions(id) ON DELETE CASCADE,
+ topic TEXT NOT NULL,
+ attempts INTEGER NOT NULL DEFAULT 0,
+ next_attempt_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ last_error TEXT,
+ created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+CREATE UNIQUE INDEX IF NOT EXISTS idx_websub_deliveries_subscription
+ ON websub_deliveries(subscription_id) WHERE subscription_id IS NOT NULL;
+CREATE UNIQUE INDEX IF NOT EXISTS idx_websub_deliveries_ping
+ ON websub_deliveries(topic) WHERE subscription_id IS NULL;
+
+-- Site-wide values. content_version changes whenever a public note does, so
+-- every gunicorn process knows its cached feed note list is stale.
+CREATE TABLE IF NOT EXISTS site_state (
+ key TEXT PRIMARY KEY,
+ value TEXT NOT NULL
+);
+INSERT OR IGNORE INTO site_state (key, value) VALUES ('content_version', '0');
diff --git a/starpunk/__init__.py b/starpunk/__init__.py
index e94c402..d593bb3 100644
--- a/starpunk/__init__.py
+++ b/starpunk/__init__.py
@@ -309,6 +309,11 @@ def create_app(config=None):
response.headers.add('Link', '; rel="webmention"')
return response
+ # WebSub discovery: Link rel=hub and rel=self on topics (v2.10.0)
+ from starpunk.websub import add_discovery_headers, template_links
+ app.after_request(add_discovery_headers)
+ app.add_template_global(template_links, 'websub_links')
+
# Register centralized error handlers
from starpunk.errors import register_error_handlers
@@ -322,12 +327,8 @@ def create_app(config=None):
memory_monitor.start()
app.memory_monitor = memory_monitor
app.logger.info(f"Memory monitor started (interval={memory_monitor.interval}s)")
-
- # Register cleanup handler
- @app.teardown_appcontext
- def cleanup_memory_monitor(error=None):
- if hasattr(app, 'memory_monitor') and app.memory_monitor.is_alive():
- app.memory_monitor.stop()
+ # No teardown_appcontext stop: that ran after every request and stopped
+ # the monitor straight away (v2.10.0). It is a daemon thread.
# Start webmention worker thread (v1.6.0 Phase 2)
# Per background-worker.md: Skip in test mode
@@ -350,6 +351,12 @@ def create_app(config=None):
f"batch_size={webmention_worker.batch_size})"
)
+ # Start WebSub worker (v2.10.0)
+ from starpunk.websub_worker import start_worker
+ app.websub_worker = start_worker(app)
+ if app.websub_worker:
+ app.logger.info("WebSub worker started")
+
# Start syndication worker when a target is configured (v2.1.0)
from starpunk.syndication import SyndicationWorker, targets_configured
if targets_configured(app.config) and not app.config.get("TESTING", False):
@@ -463,5 +470,5 @@ def create_app(config=None):
# Package version (Semantic Versioning 2.0.0)
# See docs/standards/versioning-strategy.md for details
-__version__ = "2.9.0"
-__version_info__ = (2, 9, 0)
+__version__ = "2.10.0"
+__version_info__ = (2, 10, 0)
diff --git a/starpunk/config.py b/starpunk/config.py
index 4db2f53..bee665f 100644
--- a/starpunk/config.py
+++ b/starpunk/config.py
@@ -111,6 +111,13 @@ def load_config(app, config_override=None):
# Webmention configuration (v1.6.0)
app.config["WEBMENTION_ENABLED"] = os.getenv("WEBMENTION_ENABLED", "true").lower() == "true"
+
+ # WebSub (v2.10.0, ADR-071): built-in hub unless WEBSUB_HUB_URL names another
+ app.config["WEBSUB_ENABLED"] = os.getenv("WEBSUB_ENABLED", "true").lower() == "true"
+ app.config["WEBSUB_HUB_URL"] = os.getenv("WEBSUB_HUB_URL", "").strip()
+ app.config["WEBSUB_MAX_SUBSCRIPTIONS"] = int(os.getenv("WEBSUB_MAX_SUBSCRIPTIONS", "1000"))
+ app.config["WEBSUB_MAX_PER_HOST"] = int(os.getenv("WEBSUB_MAX_PER_HOST", "25"))
+ app.config["WEBSUB_POLL_INTERVAL"] = int(os.getenv("WEBSUB_POLL_INTERVAL", "5"))
app.config["WEBMENTION_WORKER_ENABLED"] = os.getenv("WEBMENTION_WORKER_ENABLED", "true").lower() == "true"
app.config["WEBMENTION_POLL_INTERVAL"] = int(os.getenv("WEBMENTION_POLL_INTERVAL", "10"))
app.config["WEBMENTION_ERROR_BACKOFF"] = int(os.getenv("WEBMENTION_ERROR_BACKOFF", "60"))
diff --git a/starpunk/feeds/rss.py b/starpunk/feeds/rss.py
index 411b772..1778aaa 100644
--- a/starpunk/feeds/rss.py
+++ b/starpunk/feeds/rss.py
@@ -93,7 +93,7 @@ def generate_rss(
fg.language("en")
# Add self-link for feed discovery (Atom namespace)
- fg.link(href=f"{site_url}/feed.xml", rel="self", type="application/rss+xml")
+ fg.link(href=f"{site_url}/feed.rss", rel="self", type="application/rss+xml")
# Set last build date to now
fg.lastBuildDate(datetime.now(timezone.utc))
@@ -246,7 +246,7 @@ def generate_rss_streaming(
yield f" {_escape_xml(site_description or site_name)}\n"
yield " en\n"
yield f" {last_build}\n"
- yield f' \n'
+ yield f' \n'
# Yield items (newest first)
# Notes from database are already in DESC order (newest first)
diff --git a/starpunk/micropub.py b/starpunk/micropub.py
index ae024d4..fe225c9 100644
--- a/starpunk/micropub.py
+++ b/starpunk/micropub.py
@@ -761,6 +761,9 @@ def handle_update(data: dict, token_info: dict):
from starpunk.media import attach_media_to_note
attach_media_to_note(note.id, media_ids, captions)
note_changed(note.id)
+ if not changes: # update_note already told WebSub otherwise
+ from starpunk import websub
+ websub.note_changed(note, was_public=note.published)
except InvalidNoteDataError as e:
raise MicropubValidationError(str(e))
except MicropubError:
diff --git a/starpunk/notes.py b/starpunk/notes.py
index 9059432..fb4f27a 100644
--- a/starpunk/notes.py
+++ b/starpunk/notes.py
@@ -374,9 +374,28 @@ def create_note(
# Reply context fetch failure should not prevent note creation
current_app.logger.warning(f"Failed to fetch reply context for {in_reply_to}: {e}")
+ _websub_changed(note)
return note
+def _tag_names(note_id: int) -> list[str]:
+ """A note's tag names, for WebSub tag topics (best effort)"""
+ try:
+ from starpunk.tags import get_note_tags
+ return [t["name"] for t in get_note_tags(note_id)]
+ except Exception:
+ return []
+
+
+def _websub_changed(note, was_public: bool = False, old_tags=()) -> None:
+ """Clear cached feeds and queue WebSub deliveries (v2.10.0, best effort)"""
+ try:
+ from starpunk import websub
+ websub.note_changed(note, was_public=was_public, old_tags=old_tags)
+ except Exception as e:
+ current_app.logger.warning(f"Failed to queue WebSub update for note {note.slug}: {e}")
+
+
def get_note(
slug: Optional[str] = None, id: Optional[int] = None, load_content: bool = True
) -> Optional[Note]:
@@ -705,6 +724,7 @@ def update_note(
if existing_note is None:
identifier = slug if slug is not None else id
raise NoteNotFoundError(identifier)
+ old_tag_names = _tag_names(existing_note.id) # for WebSub tag topics (v2.10.0)
if empty_content and not (existing_note.reacts_to or has_media):
raise InvalidNoteDataError(
@@ -896,6 +916,8 @@ def update_note(
except Exception as e:
current_app.logger.warning(f"Failed to queue syndication update: {e}")
+ _websub_changed(updated_note, was_public=existing_note.published, old_tags=old_tag_names)
+
# 10. RETURN UPDATED NOTE
return updated_note
@@ -995,6 +1017,10 @@ def delete_note(
# For idempotency, don't raise error - just return
return
+ # What feeds showed before, for WebSub (v2.10.0)
+ was_public = bool(existing_note.published) and not getattr(existing_note, "deleted_at", None)
+ deleted_tags = _tag_names(existing_note.id)
+
# 4. SETUP
data_dir = Path(current_app.config["DATA_PATH"])
note_path = data_dir / existing_note.file_path
@@ -1081,6 +1107,10 @@ def delete_note(
except Exception as e:
current_app.logger.warning(f"Failed to queue syndication delete: {e}")
+ # Feeds and pages no longer show it (v2.10.0)
+ object.__setattr__(existing_note, "published", False)
+ _websub_changed(existing_note, was_public=was_public, old_tags=deleted_tags)
+
# 6. RETURN (no value)
return None
@@ -1137,4 +1167,6 @@ def undelete_note(slug: str) -> Note:
except Exception as e:
current_app.logger.warning(f"Failed to queue webmentions after undelete: {e}")
- return get_note(slug=slug)
+ restored = get_note(slug=slug)
+ _websub_changed(restored)
+ return restored
diff --git a/starpunk/routes/__init__.py b/starpunk/routes/__init__.py
index 2bcada9..3054f5e 100644
--- a/starpunk/routes/__init__.py
+++ b/starpunk/routes/__init__.py
@@ -7,7 +7,7 @@ admin, auth, and (conditionally) dev auth routes.
from flask import Flask
-from starpunk.routes import admin, auth, indieauth, micropub, public, search, webmention
+from starpunk.routes import admin, auth, indieauth, micropub, public, search, webmention, websub
def register_routes(app: Flask) -> None:
@@ -44,6 +44,9 @@ def register_routes(app: Flask) -> None:
if app.config.get("WEBMENTION_ENABLED", True):
app.register_blueprint(webmention.bp)
+ # WebSub hub (v2.10.0); refuses requests when disabled or using another hub
+ app.register_blueprint(websub.bp)
+
# Conditionally register dev auth routes
if app.config.get("DEV_MODE"):
app.logger.warning(
diff --git a/starpunk/routes/admin.py b/starpunk/routes/admin.py
index 575e51d..34f2be0 100644
--- a/starpunk/routes/admin.py
+++ b/starpunk/routes/admin.py
@@ -72,12 +72,23 @@ def dashboard():
except Exception:
pass
+ # WebSub subscribers (v2.10.0)
+ websub_stats = None
+ if current_app.config.get('WEBSUB_ENABLED', True):
+ try:
+ from starpunk import websub
+ from starpunk.database import get_db
+ websub_stats = websub.stats(get_db(current_app))
+ except Exception:
+ pass
+
return render_template(
"admin/dashboard.html",
notes=notes,
page=page,
total_pages=total_pages,
webmention_pending_count=webmention_pending_count,
+ websub_stats=websub_stats,
user_me=g.me
)
diff --git a/starpunk/routes/public.py b/starpunk/routes/public.py
index e2589b3..9e3e880 100644
--- a/starpunk/routes/public.py
+++ b/starpunk/routes/public.py
@@ -11,6 +11,7 @@ from datetime import datetime, timedelta
from flask import Blueprint, abort, render_template, Response, current_app, request, send_from_directory
from starpunk.notes import list_notes, get_note
+from starpunk.websub import embed_hub
from starpunk.feed import generate_feed_streaming # Legacy RSS
from starpunk.feeds import (
generate_rss,
@@ -55,8 +56,14 @@ def _get_cached_notes():
cache_duration = timedelta(seconds=cache_seconds)
now = datetime.utcnow()
+ # The cached list is stale once any public note changed, in any process
+ # (v2.10.0): every change bumps content_version in the database
+ from starpunk.database import get_db
+ from starpunk.websub import get_content_version
+ version = (str(current_app.config["DATABASE_PATH"]), get_content_version(get_db(current_app)))
+
# Check if note list cache is valid
- if _feed_cache["notes"] and _feed_cache["timestamp"]:
+ if _feed_cache["notes"] and _feed_cache["timestamp"] and _feed_cache.get("version") == version:
cache_age = now - _feed_cache["timestamp"]
if cache_age < cache_duration:
# Use cached note list
@@ -82,6 +89,7 @@ def _get_cached_notes():
_feed_cache["notes"] = notes
_feed_cache["timestamp"] = now
+ _feed_cache["version"] = version
return notes
@@ -122,6 +130,7 @@ def _generate_feed_with_cache(format_name: str, non_streaming_generator):
notes=notes,
limit=max_items,
)
+ content = embed_hub(format_name, content) # WebSub hub link (v2.10.0)
# Create response with proper Content-Type including charset
mime_type = get_mime_type(format_name)
@@ -170,6 +179,7 @@ def _generate_feed_with_cache(format_name: str, non_streaming_generator):
notes=notes,
limit=max_items,
)
+ content = embed_hub(format_name, content) # WebSub hub link (v2.10.0)
# Store in cache and get ETag
etag = feed_cache.set(format_name, content, notes_checksum)
diff --git a/starpunk/routes/websub.py b/starpunk/routes/websub.py
new file mode 100644
index 0000000..0abbce7
--- /dev/null
+++ b/starpunk/routes/websub.py
@@ -0,0 +1,49 @@
+"""
+WebSub hub endpoint (v2.10.0, ADR-071)
+
+POST /websub with hub.mode subscribe/unsubscribe, hub.topic, hub.callback,
+optional hub.secret and hub.lease_seconds. Answers 202 and verifies later
+(spec 5.1); errors are 4xx with a plain-text reason.
+"""
+
+from flask import Blueprint, current_app, request
+
+from starpunk.database import get_db
+from starpunk.rate_limiter import RateLimiter
+from starpunk import websub
+
+bp = Blueprint("websub", __name__)
+
+# Subscription requests per client IP per minute
+_limiter = RateLimiter(limit=30, window_seconds=60)
+
+
+def _text(message: str, status: int):
+ return message + "\n", status, {"Content-Type": "text/plain; charset=utf-8"}
+
+
+@bp.route("/websub", methods=["POST"])
+def hub():
+ if not websub.enabled():
+ return _text("WebSub is not enabled on this site", 404)
+
+ from starpunk.routes.webmention import get_client_ip
+ if not _limiter.is_allowed(get_client_ip()):
+ return _text("Too many requests", 429)
+
+ if "application/x-www-form-urlencoded" not in request.headers.get("Content-Type", ""):
+ return _text("Send an application/x-www-form-urlencoded body", 400)
+
+ try:
+ websub.accept_subscription_request(request.form, get_db(current_app))
+ except websub.SubscriptionError as e:
+ return _text(str(e), e.status)
+
+ return _text("Accepted; the subscription will be verified", 202)
+
+
+@bp.route("/websub", methods=["GET"])
+def hub_info():
+ return _text(
+ "WebSub hub for this site's own feeds. Subscribe with a POST "
+ "(https://www.w3.org/TR/websub/).", 200)
diff --git a/starpunk/webmention_ssrf.py b/starpunk/webmention_ssrf.py
index 9aa5b21..4ca5abe 100644
--- a/starpunk/webmention_ssrf.py
+++ b/starpunk/webmention_ssrf.py
@@ -18,6 +18,7 @@ import ipaddress
import socket
import time
from dataclasses import dataclass
+from typing import Optional
from urllib.parse import urljoin, urlparse
import httpx
@@ -157,20 +158,6 @@ def _pinned_request(client: httpx.Client, method: str, url: str, headers: dict,
)
-def safe_post(url: str, data: dict, headers: dict, timeout: float = TIMEOUT) -> httpx.Response:
- """
- POST form data to a checked, pinned address. Redirects are not followed.
-
- Raises:
- SSRFBlockedError, ValueError: URL is not allowed
- httpx.HTTPError: Network errors
- """
- with httpx.Client(follow_redirects=False, transport=TRANSPORT, timeout=timeout) as client:
- response = client.send(_pinned_request(client, 'POST', url, headers, data=data))
- response.read()
- return response
-
-
@dataclass
class FetchResult:
"""A fetched response, after redirects"""
@@ -188,6 +175,36 @@ class FetchResult:
return self.content.decode('utf-8', errors='replace')
+def safe_post(url: str, data: Optional[dict] = None, headers: Optional[dict] = None,
+ timeout: float = TIMEOUT, content: Optional[bytes] = None,
+ max_bytes: int = 64 * 1024) -> FetchResult:
+ """
+ POST form data (or a raw body) to a checked, pinned address.
+
+ Redirects are not followed. Only the first max_bytes of the response are
+ read (callers need the status and maybe an error message).
+
+ Raises:
+ SSRFBlockedError, ValueError: URL is not allowed
+ httpx.HTTPError: Network errors
+ """
+ kwargs = {'content': content} if content is not None else {'data': data}
+ with httpx.Client(follow_redirects=False, transport=TRANSPORT, timeout=timeout) as client:
+ request = _pinned_request(client, 'POST', url, headers or {}, **kwargs)
+ response = client.send(request, stream=True)
+ try:
+ body = b''
+ for chunk in response.iter_bytes(chunk_size=8192):
+ body += chunk
+ if len(body) >= max_bytes:
+ body = body[:max_bytes]
+ break
+ return FetchResult(url=url, status_code=response.status_code, headers=response.headers,
+ content=body, encoding=response.charset_encoding or 'utf-8')
+ finally:
+ response.close()
+
+
def safe_get(url: str, headers: dict, max_bytes: int = MAX_BYTES,
max_redirects: int = MAX_REDIRECTS, timeout: float = TIMEOUT) -> FetchResult:
"""
diff --git a/starpunk/websub.py b/starpunk/websub.py
new file mode 100644
index 0000000..31c2070
--- /dev/null
+++ b/starpunk/websub.py
@@ -0,0 +1,340 @@
+"""
+WebSub publishing and the built-in hub (v2.10.0, ADR-071)
+
+Publisher: topic responses advertise the hub and their own URL; a change to a
+public note clears cached feeds and queues delivery of every affected topic.
+
+Hub: subscriptions to this site's own topics only. Requests are accepted
+here (202) and verified, delivered and retried by the WebSub worker.
+
+References:
+ - W3C WebSub: https://www.w3.org/TR/websub/
+ - ADR-071: WebSub publishing with a built-in hub
+"""
+
+import json
+import secrets
+from typing import Iterable, Optional
+from urllib.parse import urlparse
+
+from flask import current_app, request
+
+
+# Topic endpoints and their fixed paths (tag pages are added per tag)
+FEED_TOPICS = {
+ "public.feed_atom": "/feed.atom",
+ "public.feed_rss": "/feed.rss",
+ "public.feed_xml_legacy": "/feed.rss", # legacy alias of the RSS topic
+ "public.feed_json": "/feed.json",
+}
+PAGE_TOPICS = {"public.index": "/", "public.likes": "/likes"}
+NEGOTIATED = { # /feed: the topic of whichever format was served
+ "application/atom+xml": "/feed.atom",
+ "application/rss+xml": "/feed.rss",
+ "application/feed+json": "/feed.json",
+ "application/json": "/feed.json",
+}
+
+LEASE_DEFAULT = 10 * 24 * 3600
+LEASE_MIN = 24 * 3600
+LEASE_MAX = 30 * 24 * 3600
+MAX_SECRET_BYTES = 200
+
+
+class SubscriptionError(Exception):
+ """A subscription request the hub refuses (4xx with a plain-text reason)"""
+
+ def __init__(self, message: str, status: int = 400):
+ super().__init__(message)
+ self.status = status
+
+
+# --- Configuration -----------------------------------------------------------
+
+def enabled() -> bool:
+ return bool(current_app.config.get("WEBSUB_ENABLED", True))
+
+
+def external_hub() -> Optional[str]:
+ """URL of an external hub, if one is configured instead of the built-in one"""
+ return current_app.config.get("WEBSUB_HUB_URL") or None
+
+
+def hub_url() -> str:
+ # Built from SITE_URL so the worker can use it outside a request
+ return external_hub() or site_url() + "/websub"
+
+
+def site_url() -> str:
+ return current_app.config["SITE_URL"].rstrip("/")
+
+
+# --- Topics ------------------------------------------------------------------
+
+def topic_url(path: str) -> str:
+ return site_url() + path
+
+
+def tag_topic(tag_name: str) -> str:
+ from urllib.parse import quote
+ return f"{site_url()}/tag/{quote(tag_name, safe='')}"
+
+
+def topic_for_request(response=None) -> Optional[str]:
+ """The topic URL the current request serves, or None if it isn't a topic"""
+ endpoint = request.endpoint
+ if endpoint in PAGE_TOPICS:
+ return topic_url(PAGE_TOPICS[endpoint])
+ if endpoint in FEED_TOPICS:
+ return topic_url(FEED_TOPICS[endpoint])
+ if endpoint == "public.feed" and response is not None:
+ mime = (response.mimetype or "").lower()
+ return topic_url(NEGOTIATED[mime]) if mime in NEGOTIATED else None
+ if endpoint == "public.tag":
+ from starpunk.tags import normalize_tag
+ tag = (request.view_args or {}).get("tag", "")
+ return tag_topic(normalize_tag(tag)[0]) if tag else None
+ return None
+
+
+def canonical_topic(url: str) -> Optional[str]:
+ """
+ A subscription's hub.topic as one of this site's topics, or None
+
+ Tag topics must name an existing tag.
+ """
+ if not url:
+ return None
+ site = urlparse(site_url())
+ parsed = urlparse(url.strip())
+ if (parsed.scheme, parsed.netloc.lower()) != (site.scheme, site.netloc.lower()):
+ return None
+ path = parsed.path or "/"
+ if parsed.query or parsed.fragment:
+ return None
+ if path in set(PAGE_TOPICS.values()) | set(FEED_TOPICS.values()):
+ return topic_url(path)
+ if path.startswith("/tag/"):
+ from starpunk.tags import get_tag_by_name, normalize_tag
+ name = normalize_tag(path[len("/tag/"):])[0]
+ if name and get_tag_by_name(name):
+ return tag_topic(name)
+ return None
+
+
+def link_header_values(topic: str) -> list[str]:
+ return [f'<{hub_url()}>; rel="hub"', f'<{topic}>; rel="self"']
+
+
+def add_discovery_headers(response):
+ """after_request: Link rel=hub and rel=self on topic responses (spec 4)"""
+ if not enabled() or response.status_code not in (200, 304):
+ return response
+ topic = topic_for_request(response)
+ if topic:
+ for value in link_header_values(topic):
+ response.headers.add("Link", value)
+ return response
+
+
+def template_links() -> Optional[dict]:
+ """For base.html: {'hub', 'self'} on topic pages"""
+ if not enabled() or not request:
+ return None
+ topic = topic_for_request()
+ return {"hub": hub_url(), "self": topic} if topic else None
+
+
+def embed_hub(format_name: str, content: str) -> str:
+ """Add the hub link inside a generated feed, next to its self link"""
+ if not enabled():
+ return content
+ hub = hub_url()
+ if format_name == "atom":
+ marker = '\n {marker}', 1)
+ if format_name == "rss":
+ marker = "\n {marker}', 1)
+ if format_name == "json":
+ feed = json.loads(content)
+ feed["hubs"] = [{"type": "WebSub", "url": hub}]
+ return json.dumps(feed, ensure_ascii=False, indent=2)
+ return content
+
+
+def _xml(value: str) -> str:
+ return (value.replace("&", "&").replace("<", "<")
+ .replace(">", ">").replace('"', """))
+
+
+# --- Changes -------------------------------------------------------------------
+
+def get_content_version(db) -> str:
+ row = db.execute("SELECT value FROM site_state WHERE key = 'content_version'").fetchone()
+ return row[0] if row else "0"
+
+
+def bump_content_version(db) -> None:
+ db.execute(
+ "INSERT INTO site_state (key, value) VALUES ('content_version', '1') "
+ "ON CONFLICT(key) DO UPDATE SET value = CAST(value AS INTEGER) + 1"
+ )
+
+
+def note_changed(note, was_public: bool = False, old_tags: Iterable[str] = ()) -> None:
+ """
+ A note was created, edited, deleted, restored or (un)published
+
+ Clears cached feeds in every process and queues delivery of the topics
+ that show it. Changes that are never public (a draft edited) do nothing.
+
+ Args:
+ note: The note (after the change; for a delete, before it)
+ was_public: Whether it was public before the change
+ old_tags: Its tag names before the change
+ """
+ from starpunk.database import get_db
+
+ is_public = bool(note.published) and not getattr(note, "deleted_at", None)
+ if not (is_public or was_public):
+ return
+
+ db = get_db(current_app)
+ bump_content_version(db)
+ db.commit()
+
+ if not enabled():
+ return
+
+ from starpunk.tags import get_note_tags
+ try:
+ tags = {t["name"] for t in get_note_tags(note.id)}
+ except Exception:
+ tags = set()
+ tags |= set(old_tags)
+
+ topics = [topic_url(p) for p in ("/", "/feed.atom", "/feed.rss", "/feed.json")]
+ if getattr(note, "like_of", None):
+ topics.append(topic_url("/likes"))
+ topics += [tag_topic(name) for name in sorted(tags)]
+ queue_topics(db, topics)
+
+
+def queue_topics(db, topics: Iterable[str]) -> int:
+ """Queue delivery of topics: to their subscribers, or a ping to an external hub"""
+ topics = list(dict.fromkeys(topics))
+ if not topics:
+ return 0
+ if external_hub():
+ for topic in topics:
+ db.execute("DELETE FROM websub_deliveries WHERE subscription_id IS NULL AND topic = ?", (topic,))
+ db.execute("INSERT INTO websub_deliveries (topic) VALUES (?)", (topic,))
+ db.commit()
+ return len(topics)
+
+ placeholders = ",".join("?" * len(topics))
+ rows = db.execute(
+ f"SELECT id, topic FROM websub_subscriptions WHERE topic IN ({placeholders}) "
+ "AND verified_at IS NOT NULL AND expires_at > datetime('now')",
+ topics,
+ ).fetchall()
+ for sub_id, topic in rows:
+ # The newest content replaces anything still waiting (ADR-071 6)
+ db.execute("DELETE FROM websub_deliveries WHERE subscription_id = ?", (sub_id,))
+ db.execute("INSERT INTO websub_deliveries (subscription_id, topic) VALUES (?, ?)", (sub_id, topic))
+ db.commit()
+ return len(rows)
+
+
+# --- Hub: subscription requests (spec 5.1) -------------------------------------
+
+def accept_subscription_request(form, db) -> None:
+ """
+ Validate a subscription request and queue it for verification
+
+ Raises:
+ SubscriptionError: with the status and plain-text reason to return
+ """
+ from starpunk.webmention_ssrf import SSRFBlockedError, validate_url_for_fetch
+
+ if external_hub():
+ raise SubscriptionError("This site uses another hub: " + external_hub(), 400)
+
+ mode = (form.get("hub.mode") or "").strip()
+ if mode not in ("subscribe", "unsubscribe"):
+ raise SubscriptionError("hub.mode must be subscribe or unsubscribe")
+
+ callback = (form.get("hub.callback") or "").strip()
+ parsed = urlparse(callback)
+ if parsed.scheme not in ("http", "https") or not parsed.hostname:
+ raise SubscriptionError("hub.callback must be an absolute http(s) URL")
+ try:
+ validate_url_for_fetch(callback)
+ except (SSRFBlockedError, ValueError) as e:
+ raise SubscriptionError(f"hub.callback is not reachable: {e}")
+
+ topic = canonical_topic(form.get("hub.topic") or "")
+ if not topic:
+ raise SubscriptionError("hub.topic is not a topic of this site; use the rel=self URL")
+
+ secret = form.get("hub.secret")
+ if mode == "unsubscribe":
+ secret = None # MUST be ignored for unsubscriptions
+ if secret is not None and len(secret.encode("utf-8")) >= MAX_SECRET_BYTES:
+ raise SubscriptionError("hub.secret must be less than 200 bytes")
+
+ lease = LEASE_DEFAULT
+ if form.get("hub.lease_seconds"):
+ try:
+ lease = int(form["hub.lease_seconds"])
+ except ValueError:
+ raise SubscriptionError("hub.lease_seconds must be a number")
+ lease = max(LEASE_MIN, min(lease, LEASE_MAX))
+
+ if mode == "subscribe":
+ existing = db.execute(
+ "SELECT 1 FROM websub_subscriptions WHERE topic = ? AND callback = ?", (topic, callback)
+ ).fetchone()
+ if not existing:
+ total = db.execute("SELECT COUNT(*) FROM websub_subscriptions").fetchone()[0]
+ if total >= current_app.config.get("WEBSUB_MAX_SUBSCRIPTIONS", 1000):
+ raise SubscriptionError("This hub has reached its subscription limit", 429)
+ host = parsed.hostname.lower()
+ per_host = sum(
+ 1 for (cb,) in db.execute("SELECT callback FROM websub_subscriptions")
+ if (urlparse(cb).hostname or "").lower() == host
+ )
+ if per_host >= current_app.config.get("WEBSUB_MAX_PER_HOST", 25):
+ raise SubscriptionError(f"Too many subscriptions for {host}", 429)
+
+ # The latest request for a topic and callback replaces any pending one
+ db.execute(
+ """
+ INSERT INTO websub_intents (topic, callback, mode, secret, lease_seconds)
+ VALUES (?, ?, ?, ?, ?)
+ ON CONFLICT(topic, callback) DO UPDATE SET
+ mode = excluded.mode, secret = excluded.secret,
+ lease_seconds = excluded.lease_seconds, attempts = 0,
+ next_attempt_at = CURRENT_TIMESTAMP, created_at = CURRENT_TIMESTAMP
+ """,
+ (topic, callback, mode, secret, lease),
+ )
+ db.commit()
+
+
+def stats(db) -> dict:
+ """Active subscribers per topic and deliveries being retried (admin)"""
+ by_topic = db.execute(
+ "SELECT topic, COUNT(*) FROM websub_subscriptions WHERE expires_at > datetime('now') "
+ "GROUP BY topic ORDER BY COUNT(*) DESC, topic"
+ ).fetchall()
+ retrying = db.execute(
+ "SELECT COUNT(*) FROM websub_deliveries WHERE attempts > 0"
+ ).fetchone()[0]
+ return {"by_topic": [(t, n) for t, n in by_topic], "retrying": retrying,
+ "external": external_hub()}
+
+
+def new_challenge() -> str:
+ return secrets.token_urlsafe(32)
diff --git a/starpunk/websub_worker.py b/starpunk/websub_worker.py
new file mode 100644
index 0000000..7dec84d
--- /dev/null
+++ b/starpunk/websub_worker.py
@@ -0,0 +1,287 @@
+"""
+WebSub worker: verification of intent and content distribution (v2.10.0)
+
+Runs like the Webmention worker: a daemon thread in every gunicorn process,
+but only the process holding the file lock works the queues.
+
+- Intents (spec 5.3): GET the callback with a challenge; 2xx echoing it
+ verifies. Anything else (including redirects) leaves the subscription as
+ it was. Network errors and 5xx are retried a few times.
+- Deliveries (spec 6, 7): render the topic in-app and POST it with
+ Link rel=hub/self and, if there is a secret, X-Hub-Signature (sha256).
+- External hub: a delivery without a subscription is a publish ping.
+
+References:
+ - https://www.w3.org/TR/websub/
+ - ADR-071
+"""
+
+import fcntl
+import hashlib
+import hmac
+import logging
+import threading
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Optional
+from urllib.parse import urlencode, urlparse, urlsplit, urlunsplit
+
+import httpx
+
+logger = logging.getLogger(__name__)
+
+DELIVERY_BACKOFF = [60, 300, 1800, 7200, 21600] # then give up on this delivery
+INTENT_BACKOFF = [60, 300, 900]
+DROP_AFTER = timedelta(days=3) # failing this long without a success: unsubscribe
+PRUNE_EVERY = timedelta(hours=1)
+BATCH = 10
+
+
+def callback_with_params(callback: str, params: dict) -> str:
+ """Append parameters to the callback, keeping its query string (spec 5.3)"""
+ parts = urlsplit(callback)
+ query = parts.query + ("&" if parts.query else "") + urlencode(params)
+ return urlunsplit((parts.scheme, parts.netloc, parts.path, query, parts.fragment))
+
+
+def signature(secret: str, body: bytes) -> str:
+ return "sha256=" + hmac.new(secret.encode("utf-8"), body, hashlib.sha256).hexdigest()
+
+
+def _now() -> str:
+ return datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
+
+
+def _later(seconds: int) -> str:
+ return (datetime.utcnow() + timedelta(seconds=seconds)).strftime("%Y-%m-%d %H:%M:%S")
+
+
+class WebSubWorker(threading.Thread):
+ """Daemon thread that verifies subscriptions and delivers content"""
+
+ def __init__(self, app, poll_interval: int = 5):
+ super().__init__(daemon=True, name="websub-worker")
+ self.app = app
+ self.poll_interval = poll_interval
+ self._stop_event = threading.Event()
+ self._lock_file = None
+ self._last_prune = None
+
+ # --- thread plumbing ----------------------------------------------------
+
+ def run(self):
+ logger.info("WebSub worker started")
+ while not self._stop_event.is_set():
+ if not self._acquire_lock():
+ self._stop_event.wait(self.poll_interval)
+ continue
+ try:
+ with self.app.app_context():
+ done = self.process_once()
+ except Exception as e:
+ logger.error(f"WebSub worker error: {e}", exc_info=True)
+ done = 0
+ if not done:
+ self._stop_event.wait(self.poll_interval)
+ self._release_lock()
+
+ def stop(self):
+ self._stop_event.set()
+
+ def _acquire_lock(self) -> bool:
+ if self._lock_file:
+ return True
+ path = Path(self.app.config["DATABASE_PATH"]).parent / "websub-worker.lock"
+ lock_file = open(path, "w")
+ try:
+ fcntl.flock(lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ except OSError:
+ lock_file.close()
+ return False
+ self._lock_file = lock_file
+ logger.info("WebSub worker holds the queue lock")
+ return True
+
+ def _release_lock(self):
+ if self._lock_file:
+ self._lock_file.close()
+ self._lock_file = None
+
+ # --- one pass (also called directly by tests) ---------------------------
+
+ def process_once(self) -> int:
+ from starpunk.database import get_db
+
+ db = get_db(self.app)
+ done = 0
+ for row in db.execute(
+ "SELECT * FROM websub_intents WHERE next_attempt_at <= ? ORDER BY id LIMIT ?",
+ (_now(), BATCH),
+ ).fetchall():
+ self.verify(db, row)
+ done += 1
+ for row in db.execute(
+ "SELECT * FROM websub_deliveries WHERE next_attempt_at <= ? ORDER BY id LIMIT ?",
+ (_now(), BATCH),
+ ).fetchall():
+ self.deliver(db, row)
+ done += 1
+ if self._last_prune is None or datetime.utcnow() - self._last_prune > PRUNE_EVERY:
+ self.prune(db)
+ return done
+
+ # --- verification of intent (spec 5.3) ----------------------------------
+
+ def verify(self, db, intent) -> None:
+ from starpunk.websub import new_challenge
+ from starpunk.webmention_ssrf import SSRFBlockedError, safe_get, user_agent
+
+ challenge = new_challenge()
+ params = {"hub.mode": intent["mode"], "hub.topic": intent["topic"], "hub.challenge": challenge}
+ if intent["mode"] == "subscribe":
+ params["hub.lease_seconds"] = intent["lease_seconds"]
+ url = callback_with_params(intent["callback"], params)
+
+ try:
+ result = safe_get(url, headers={"User-Agent": user_agent(self.app.config["SITE_URL"])},
+ max_redirects=0, max_bytes=64 * 1024)
+ verified = 200 <= result.status_code < 300 and result.text.strip() == challenge
+ retry = result.status_code >= 500
+ error = None if verified else f"HTTP {result.status_code}" + (
+ "" if result.status_code >= 300 else ": challenge not echoed")
+ except (SSRFBlockedError, ValueError) as e:
+ verified, retry, error = False, False, f"Blocked: {e}"
+ except httpx.TooManyRedirects:
+ verified, retry, error = False, False, "Redirected (not allowed)"
+ except httpx.HTTPError as e:
+ verified, retry, error = False, True, str(e)[:200]
+
+ if verified:
+ if intent["mode"] == "subscribe":
+ db.execute(
+ """
+ INSERT INTO websub_subscriptions
+ (topic, callback, secret, lease_seconds, expires_at, verified_at)
+ VALUES (?, ?, ?, ?, ?, ?)
+ ON CONFLICT(topic, callback) DO UPDATE SET
+ secret = excluded.secret, lease_seconds = excluded.lease_seconds,
+ expires_at = excluded.expires_at, verified_at = excluded.verified_at,
+ last_error = NULL
+ """,
+ (intent["topic"], intent["callback"], intent["secret"], intent["lease_seconds"],
+ _later(intent["lease_seconds"]), _now()),
+ )
+ else:
+ db.execute("DELETE FROM websub_subscriptions WHERE topic = ? AND callback = ?",
+ (intent["topic"], intent["callback"]))
+ db.execute("DELETE FROM websub_intents WHERE id = ?", (intent["id"],))
+ logger.info(f"WebSub {intent['mode']} verified: {intent['callback']} -> {intent['topic']}")
+ elif retry and intent["attempts"] < len(INTENT_BACKOFF):
+ db.execute("UPDATE websub_intents SET attempts = attempts + 1, next_attempt_at = ? WHERE id = ?",
+ (_later(INTENT_BACKOFF[intent["attempts"]]), intent["id"]))
+ else:
+ # Verification failed: the subscription state stays as it was
+ db.execute("DELETE FROM websub_intents WHERE id = ?", (intent["id"],))
+ logger.info(f"WebSub {intent['mode']} not verified for {intent['callback']}: {error}")
+ db.commit()
+
+ # --- content distribution (spec 6, 7) ------------------------------------
+
+ def render_topic(self, topic: str):
+ """(status, content_type, body) of a topic, rendered in-app"""
+ path = urlparse(topic).path or "/"
+ response = self.app.test_client().get(path, headers={"Accept": "*/*"})
+ return response.status_code, response.headers.get("Content-Type", ""), response.get_data()
+
+ def deliver(self, db, delivery) -> None:
+ from starpunk.websub import hub_url
+ from starpunk.webmention_ssrf import SSRFBlockedError, safe_post, user_agent
+
+ site = self.app.config["SITE_URL"]
+ sub = None
+ try:
+ if delivery["subscription_id"] is None:
+ # External hub: ping it (hub.url is the de facto parameter)
+ result = safe_post(hub_url(), data={"hub.mode": "publish", "hub.url": delivery["topic"],
+ "hub.topic": delivery["topic"]},
+ headers={"User-Agent": user_agent(site)})
+ else:
+ sub = db.execute(
+ "SELECT * FROM websub_subscriptions WHERE id = ? AND expires_at > ?",
+ (delivery["subscription_id"], _now()),
+ ).fetchone()
+ if sub is None: # unsubscribed or expired meanwhile
+ db.execute("DELETE FROM websub_deliveries WHERE id = ?", (delivery["id"],))
+ db.commit()
+ return
+ status, content_type, body = self.render_topic(sub["topic"])
+ if status != 200: # e.g. a tag page whose last note went
+ db.execute("DELETE FROM websub_deliveries WHERE id = ?", (delivery["id"],))
+ db.commit()
+ return
+ headers = {
+ "Content-Type": content_type,
+ "Link": f'<{hub_url()}>; rel="hub", <{sub["topic"]}>; rel="self"',
+ "User-Agent": user_agent(site),
+ }
+ if sub["secret"]:
+ headers["X-Hub-Signature"] = signature(sub["secret"], body)
+ result = safe_post(sub["callback"], content=body, headers=headers)
+ status, error = result.status_code, f"HTTP {result.status_code}"
+ except (SSRFBlockedError, ValueError) as e:
+ status, error = None, f"Blocked: {e}"
+ except httpx.HTTPError as e:
+ status, error = None, str(e)[:200] or type(e).__name__
+
+ if status is not None and 200 <= status < 300:
+ db.execute("DELETE FROM websub_deliveries WHERE id = ?", (delivery["id"],))
+ if sub is not None:
+ db.execute("UPDATE websub_subscriptions SET last_success_at = ?, last_error = NULL WHERE id = ?",
+ (_now(), sub["id"]))
+ elif sub is not None and status == 410:
+ db.execute("DELETE FROM websub_subscriptions WHERE id = ?", (sub["id"],))
+ logger.info(f"WebSub subscriber gone (410), removed: {sub['callback']}")
+ elif delivery["attempts"] < len(DELIVERY_BACKOFF) and not error.startswith("Blocked"):
+ db.execute(
+ "UPDATE websub_deliveries SET attempts = attempts + 1, next_attempt_at = ?, last_error = ? WHERE id = ?",
+ (_later(DELIVERY_BACKOFF[delivery["attempts"]]), error, delivery["id"]),
+ )
+ if sub is not None:
+ db.execute("UPDATE websub_subscriptions SET last_error = ? WHERE id = ?", (error, sub["id"]))
+ else:
+ db.execute("DELETE FROM websub_deliveries WHERE id = ?", (delivery["id"],))
+ logger.warning(f"WebSub delivery of {delivery['topic']} gave up: {error}")
+ if sub is not None:
+ self._drop_if_failing(db, sub, error)
+ db.commit()
+
+ def _drop_if_failing(self, db, sub, error: str) -> None:
+ last_ok = sub["last_success_at"] or sub["verified_at"] or sub["created_at"]
+ try:
+ since = datetime.fromisoformat(str(last_ok))
+ except ValueError:
+ return
+ if datetime.utcnow() - since > DROP_AFTER:
+ db.execute("DELETE FROM websub_subscriptions WHERE id = ?", (sub["id"],))
+ logger.info(f"WebSub subscription failing for {DROP_AFTER.days} days, removed: {sub['callback']}")
+ else:
+ db.execute("UPDATE websub_subscriptions SET last_error = ? WHERE id = ?", (error, sub["id"]))
+
+ # --- housekeeping -------------------------------------------------------
+
+ def prune(self, db) -> int:
+ """Remove subscriptions whose lease has run out (they weren't renewed)"""
+ cursor = db.execute("DELETE FROM websub_subscriptions WHERE expires_at <= ?", (_now(),))
+ db.commit()
+ self._last_prune = datetime.utcnow()
+ if cursor.rowcount:
+ logger.info(f"WebSub: removed {cursor.rowcount} expired subscriptions")
+ return cursor.rowcount
+
+
+def start_worker(app) -> Optional[WebSubWorker]:
+ if not app.config.get("WEBSUB_ENABLED", True) or app.config.get("TESTING", False):
+ return None
+ worker = WebSubWorker(app, poll_interval=app.config.get("WEBSUB_POLL_INTERVAL", 5))
+ worker.start()
+ return worker
diff --git a/templates/admin/dashboard.html b/templates/admin/dashboard.html
index dc2de0b..03fe735 100644
--- a/templates/admin/dashboard.html
+++ b/templates/admin/dashboard.html
@@ -20,6 +20,21 @@
{% endif %}
+ {# WebSub subscribers (v2.10.0) #}
+ {% if websub_stats %}
+
+ {% if websub_stats.external %}
+ WebSub: using {{ websub_stats.external }}
+ {% elif websub_stats.by_topic %}
+ WebSub subscribers:
+ {% for topic, count in websub_stats.by_topic %}{{ topic|replace(config.SITE_URL.rstrip('/'), '') or '/' }} {{ count }}{% if not loop.last %}, {% endif %}{% endfor %}
+ {% if websub_stats.retrying %} ยท {{ websub_stats.retrying }} deliveries being retried{% endif %}
+ {% else %}
+ WebSub: no subscribers yet
+ {% endif %}
+
+ {% endif %}
+
{% if notes %}