Files
StarPunk/starpunk/websub_worker.py
T
PhilandClaude Opus 5.5 b520b8b4d3 feat: WebSub with a built-in hub (v2.10.0)
ADR-071. Topics (/, /feed.atom, /feed.rss, /feed.json, /likes, /tag/<name>)
advertise the hub and themselves in Link headers and embedded links. Note
changes bump a content version (fresh feeds in every process) and queue
deliveries. /websub accepts subscriptions for these topics only; the worker
verifies intent, delivers signed content with retries, and prunes expired
leases, all through the SSRF-safe pinned fetcher. WEBSUB_HUB_URL switches
to an external hub (publish pings). Also: the memory monitor no longer
stops after the first request; RSS self link is /feed.rss.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-03 10:33:41 -07:00

288 lines
13 KiB
Python

"""
WebSub worker: verification of intent and content distribution (v2.10.0)
Runs like the Webmention worker: a daemon thread in every gunicorn process,
but only the process holding the file lock works the queues.
- Intents (spec 5.3): GET the callback with a challenge; 2xx echoing it
verifies. Anything else (including redirects) leaves the subscription as
it was. Network errors and 5xx are retried a few times.
- Deliveries (spec 6, 7): render the topic in-app and POST it with
Link rel=hub/self and, if there is a secret, X-Hub-Signature (sha256).
- External hub: a delivery without a subscription is a publish ping.
References:
- https://www.w3.org/TR/websub/
- ADR-071
"""
import fcntl
import hashlib
import hmac
import logging
import threading
from datetime import datetime, timedelta
from pathlib import Path
from typing import Optional
from urllib.parse import urlencode, urlparse, urlsplit, urlunsplit
import httpx
logger = logging.getLogger(__name__)
DELIVERY_BACKOFF = [60, 300, 1800, 7200, 21600] # then give up on this delivery
INTENT_BACKOFF = [60, 300, 900]
DROP_AFTER = timedelta(days=3) # failing this long without a success: unsubscribe
PRUNE_EVERY = timedelta(hours=1)
BATCH = 10
def callback_with_params(callback: str, params: dict) -> str:
"""Append parameters to the callback, keeping its query string (spec 5.3)"""
parts = urlsplit(callback)
query = parts.query + ("&" if parts.query else "") + urlencode(params)
return urlunsplit((parts.scheme, parts.netloc, parts.path, query, parts.fragment))
def signature(secret: str, body: bytes) -> str:
return "sha256=" + hmac.new(secret.encode("utf-8"), body, hashlib.sha256).hexdigest()
def _now() -> str:
return datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
def _later(seconds: int) -> str:
return (datetime.utcnow() + timedelta(seconds=seconds)).strftime("%Y-%m-%d %H:%M:%S")
class WebSubWorker(threading.Thread):
"""Daemon thread that verifies subscriptions and delivers content"""
def __init__(self, app, poll_interval: int = 5):
super().__init__(daemon=True, name="websub-worker")
self.app = app
self.poll_interval = poll_interval
self._stop_event = threading.Event()
self._lock_file = None
self._last_prune = None
# --- thread plumbing ----------------------------------------------------
def run(self):
logger.info("WebSub worker started")
while not self._stop_event.is_set():
if not self._acquire_lock():
self._stop_event.wait(self.poll_interval)
continue
try:
with self.app.app_context():
done = self.process_once()
except Exception as e:
logger.error(f"WebSub worker error: {e}", exc_info=True)
done = 0
if not done:
self._stop_event.wait(self.poll_interval)
self._release_lock()
def stop(self):
self._stop_event.set()
def _acquire_lock(self) -> bool:
if self._lock_file:
return True
path = Path(self.app.config["DATABASE_PATH"]).parent / "websub-worker.lock"
lock_file = open(path, "w")
try:
fcntl.flock(lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB)
except OSError:
lock_file.close()
return False
self._lock_file = lock_file
logger.info("WebSub worker holds the queue lock")
return True
def _release_lock(self):
if self._lock_file:
self._lock_file.close()
self._lock_file = None
# --- one pass (also called directly by tests) ---------------------------
def process_once(self) -> int:
from starpunk.database import get_db
db = get_db(self.app)
done = 0
for row in db.execute(
"SELECT * FROM websub_intents WHERE next_attempt_at <= ? ORDER BY id LIMIT ?",
(_now(), BATCH),
).fetchall():
self.verify(db, row)
done += 1
for row in db.execute(
"SELECT * FROM websub_deliveries WHERE next_attempt_at <= ? ORDER BY id LIMIT ?",
(_now(), BATCH),
).fetchall():
self.deliver(db, row)
done += 1
if self._last_prune is None or datetime.utcnow() - self._last_prune > PRUNE_EVERY:
self.prune(db)
return done
# --- verification of intent (spec 5.3) ----------------------------------
def verify(self, db, intent) -> None:
from starpunk.websub import new_challenge
from starpunk.webmention_ssrf import SSRFBlockedError, safe_get, user_agent
challenge = new_challenge()
params = {"hub.mode": intent["mode"], "hub.topic": intent["topic"], "hub.challenge": challenge}
if intent["mode"] == "subscribe":
params["hub.lease_seconds"] = intent["lease_seconds"]
url = callback_with_params(intent["callback"], params)
try:
result = safe_get(url, headers={"User-Agent": user_agent(self.app.config["SITE_URL"])},
max_redirects=0, max_bytes=64 * 1024)
verified = 200 <= result.status_code < 300 and result.text.strip() == challenge
retry = result.status_code >= 500
error = None if verified else f"HTTP {result.status_code}" + (
"" if result.status_code >= 300 else ": challenge not echoed")
except (SSRFBlockedError, ValueError) as e:
verified, retry, error = False, False, f"Blocked: {e}"
except httpx.TooManyRedirects:
verified, retry, error = False, False, "Redirected (not allowed)"
except httpx.HTTPError as e:
verified, retry, error = False, True, str(e)[:200]
if verified:
if intent["mode"] == "subscribe":
db.execute(
"""
INSERT INTO websub_subscriptions
(topic, callback, secret, lease_seconds, expires_at, verified_at)
VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT(topic, callback) DO UPDATE SET
secret = excluded.secret, lease_seconds = excluded.lease_seconds,
expires_at = excluded.expires_at, verified_at = excluded.verified_at,
last_error = NULL
""",
(intent["topic"], intent["callback"], intent["secret"], intent["lease_seconds"],
_later(intent["lease_seconds"]), _now()),
)
else:
db.execute("DELETE FROM websub_subscriptions WHERE topic = ? AND callback = ?",
(intent["topic"], intent["callback"]))
db.execute("DELETE FROM websub_intents WHERE id = ?", (intent["id"],))
logger.info(f"WebSub {intent['mode']} verified: {intent['callback']} -> {intent['topic']}")
elif retry and intent["attempts"] < len(INTENT_BACKOFF):
db.execute("UPDATE websub_intents SET attempts = attempts + 1, next_attempt_at = ? WHERE id = ?",
(_later(INTENT_BACKOFF[intent["attempts"]]), intent["id"]))
else:
# Verification failed: the subscription state stays as it was
db.execute("DELETE FROM websub_intents WHERE id = ?", (intent["id"],))
logger.info(f"WebSub {intent['mode']} not verified for {intent['callback']}: {error}")
db.commit()
# --- content distribution (spec 6, 7) ------------------------------------
def render_topic(self, topic: str):
"""(status, content_type, body) of a topic, rendered in-app"""
path = urlparse(topic).path or "/"
response = self.app.test_client().get(path, headers={"Accept": "*/*"})
return response.status_code, response.headers.get("Content-Type", ""), response.get_data()
def deliver(self, db, delivery) -> None:
from starpunk.websub import hub_url
from starpunk.webmention_ssrf import SSRFBlockedError, safe_post, user_agent
site = self.app.config["SITE_URL"]
sub = None
try:
if delivery["subscription_id"] is None:
# External hub: ping it (hub.url is the de facto parameter)
result = safe_post(hub_url(), data={"hub.mode": "publish", "hub.url": delivery["topic"],
"hub.topic": delivery["topic"]},
headers={"User-Agent": user_agent(site)})
else:
sub = db.execute(
"SELECT * FROM websub_subscriptions WHERE id = ? AND expires_at > ?",
(delivery["subscription_id"], _now()),
).fetchone()
if sub is None: # unsubscribed or expired meanwhile
db.execute("DELETE FROM websub_deliveries WHERE id = ?", (delivery["id"],))
db.commit()
return
status, content_type, body = self.render_topic(sub["topic"])
if status != 200: # e.g. a tag page whose last note went
db.execute("DELETE FROM websub_deliveries WHERE id = ?", (delivery["id"],))
db.commit()
return
headers = {
"Content-Type": content_type,
"Link": f'<{hub_url()}>; rel="hub", <{sub["topic"]}>; rel="self"',
"User-Agent": user_agent(site),
}
if sub["secret"]:
headers["X-Hub-Signature"] = signature(sub["secret"], body)
result = safe_post(sub["callback"], content=body, headers=headers)
status, error = result.status_code, f"HTTP {result.status_code}"
except (SSRFBlockedError, ValueError) as e:
status, error = None, f"Blocked: {e}"
except httpx.HTTPError as e:
status, error = None, str(e)[:200] or type(e).__name__
if status is not None and 200 <= status < 300:
db.execute("DELETE FROM websub_deliveries WHERE id = ?", (delivery["id"],))
if sub is not None:
db.execute("UPDATE websub_subscriptions SET last_success_at = ?, last_error = NULL WHERE id = ?",
(_now(), sub["id"]))
elif sub is not None and status == 410:
db.execute("DELETE FROM websub_subscriptions WHERE id = ?", (sub["id"],))
logger.info(f"WebSub subscriber gone (410), removed: {sub['callback']}")
elif delivery["attempts"] < len(DELIVERY_BACKOFF) and not error.startswith("Blocked"):
db.execute(
"UPDATE websub_deliveries SET attempts = attempts + 1, next_attempt_at = ?, last_error = ? WHERE id = ?",
(_later(DELIVERY_BACKOFF[delivery["attempts"]]), error, delivery["id"]),
)
if sub is not None:
db.execute("UPDATE websub_subscriptions SET last_error = ? WHERE id = ?", (error, sub["id"]))
else:
db.execute("DELETE FROM websub_deliveries WHERE id = ?", (delivery["id"],))
logger.warning(f"WebSub delivery of {delivery['topic']} gave up: {error}")
if sub is not None:
self._drop_if_failing(db, sub, error)
db.commit()
def _drop_if_failing(self, db, sub, error: str) -> None:
last_ok = sub["last_success_at"] or sub["verified_at"] or sub["created_at"]
try:
since = datetime.fromisoformat(str(last_ok))
except ValueError:
return
if datetime.utcnow() - since > DROP_AFTER:
db.execute("DELETE FROM websub_subscriptions WHERE id = ?", (sub["id"],))
logger.info(f"WebSub subscription failing for {DROP_AFTER.days} days, removed: {sub['callback']}")
else:
db.execute("UPDATE websub_subscriptions SET last_error = ? WHERE id = ?", (error, sub["id"]))
# --- housekeeping -------------------------------------------------------
def prune(self, db) -> int:
"""Remove subscriptions whose lease has run out (they weren't renewed)"""
cursor = db.execute("DELETE FROM websub_subscriptions WHERE expires_at <= ?", (_now(),))
db.commit()
self._last_prune = datetime.utcnow()
if cursor.rowcount:
logger.info(f"WebSub: removed {cursor.rowcount} expired subscriptions")
return cursor.rowcount
def start_worker(app) -> Optional[WebSubWorker]:
if not app.config.get("WEBSUB_ENABLED", True) or app.config.get("TESTING", False):
return None
worker = WebSubWorker(app, poll_interval=app.config.get("WEBSUB_POLL_INTERVAL", 5))
worker.start()
return worker