ADR-071. Topics (/, /feed.atom, /feed.rss, /feed.json, /likes, /tag/<name>) advertise the hub and themselves in Link headers and embedded links. Note changes bump a content version (fresh feeds in every process) and queue deliveries. /websub accepts subscriptions for these topics only; the worker verifies intent, delivers signed content with retries, and prunes expired leases, all through the SSRF-safe pinned fetcher. WEBSUB_HUB_URL switches to an external hub (publish pings). Also: the memory monitor no longer stops after the first request; RSS self link is /feed.rss. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
288 lines
13 KiB
Python
288 lines
13 KiB
Python
"""
|
|
WebSub worker: verification of intent and content distribution (v2.10.0)
|
|
|
|
Runs like the Webmention worker: a daemon thread in every gunicorn process,
|
|
but only the process holding the file lock works the queues.
|
|
|
|
- Intents (spec 5.3): GET the callback with a challenge; 2xx echoing it
|
|
verifies. Anything else (including redirects) leaves the subscription as
|
|
it was. Network errors and 5xx are retried a few times.
|
|
- Deliveries (spec 6, 7): render the topic in-app and POST it with
|
|
Link rel=hub/self and, if there is a secret, X-Hub-Signature (sha256).
|
|
- External hub: a delivery without a subscription is a publish ping.
|
|
|
|
References:
|
|
- https://www.w3.org/TR/websub/
|
|
- ADR-071
|
|
"""
|
|
|
|
import fcntl
|
|
import hashlib
|
|
import hmac
|
|
import logging
|
|
import threading
|
|
from datetime import datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
from urllib.parse import urlencode, urlparse, urlsplit, urlunsplit
|
|
|
|
import httpx
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
DELIVERY_BACKOFF = [60, 300, 1800, 7200, 21600] # then give up on this delivery
|
|
INTENT_BACKOFF = [60, 300, 900]
|
|
DROP_AFTER = timedelta(days=3) # failing this long without a success: unsubscribe
|
|
PRUNE_EVERY = timedelta(hours=1)
|
|
BATCH = 10
|
|
|
|
|
|
def callback_with_params(callback: str, params: dict) -> str:
|
|
"""Append parameters to the callback, keeping its query string (spec 5.3)"""
|
|
parts = urlsplit(callback)
|
|
query = parts.query + ("&" if parts.query else "") + urlencode(params)
|
|
return urlunsplit((parts.scheme, parts.netloc, parts.path, query, parts.fragment))
|
|
|
|
|
|
def signature(secret: str, body: bytes) -> str:
|
|
return "sha256=" + hmac.new(secret.encode("utf-8"), body, hashlib.sha256).hexdigest()
|
|
|
|
|
|
def _now() -> str:
|
|
return datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
|
|
|
|
|
|
def _later(seconds: int) -> str:
|
|
return (datetime.utcnow() + timedelta(seconds=seconds)).strftime("%Y-%m-%d %H:%M:%S")
|
|
|
|
|
|
class WebSubWorker(threading.Thread):
|
|
"""Daemon thread that verifies subscriptions and delivers content"""
|
|
|
|
def __init__(self, app, poll_interval: int = 5):
|
|
super().__init__(daemon=True, name="websub-worker")
|
|
self.app = app
|
|
self.poll_interval = poll_interval
|
|
self._stop_event = threading.Event()
|
|
self._lock_file = None
|
|
self._last_prune = None
|
|
|
|
# --- thread plumbing ----------------------------------------------------
|
|
|
|
def run(self):
|
|
logger.info("WebSub worker started")
|
|
while not self._stop_event.is_set():
|
|
if not self._acquire_lock():
|
|
self._stop_event.wait(self.poll_interval)
|
|
continue
|
|
try:
|
|
with self.app.app_context():
|
|
done = self.process_once()
|
|
except Exception as e:
|
|
logger.error(f"WebSub worker error: {e}", exc_info=True)
|
|
done = 0
|
|
if not done:
|
|
self._stop_event.wait(self.poll_interval)
|
|
self._release_lock()
|
|
|
|
def stop(self):
|
|
self._stop_event.set()
|
|
|
|
def _acquire_lock(self) -> bool:
|
|
if self._lock_file:
|
|
return True
|
|
path = Path(self.app.config["DATABASE_PATH"]).parent / "websub-worker.lock"
|
|
lock_file = open(path, "w")
|
|
try:
|
|
fcntl.flock(lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except OSError:
|
|
lock_file.close()
|
|
return False
|
|
self._lock_file = lock_file
|
|
logger.info("WebSub worker holds the queue lock")
|
|
return True
|
|
|
|
def _release_lock(self):
|
|
if self._lock_file:
|
|
self._lock_file.close()
|
|
self._lock_file = None
|
|
|
|
# --- one pass (also called directly by tests) ---------------------------
|
|
|
|
def process_once(self) -> int:
|
|
from starpunk.database import get_db
|
|
|
|
db = get_db(self.app)
|
|
done = 0
|
|
for row in db.execute(
|
|
"SELECT * FROM websub_intents WHERE next_attempt_at <= ? ORDER BY id LIMIT ?",
|
|
(_now(), BATCH),
|
|
).fetchall():
|
|
self.verify(db, row)
|
|
done += 1
|
|
for row in db.execute(
|
|
"SELECT * FROM websub_deliveries WHERE next_attempt_at <= ? ORDER BY id LIMIT ?",
|
|
(_now(), BATCH),
|
|
).fetchall():
|
|
self.deliver(db, row)
|
|
done += 1
|
|
if self._last_prune is None or datetime.utcnow() - self._last_prune > PRUNE_EVERY:
|
|
self.prune(db)
|
|
return done
|
|
|
|
# --- verification of intent (spec 5.3) ----------------------------------
|
|
|
|
def verify(self, db, intent) -> None:
|
|
from starpunk.websub import new_challenge
|
|
from starpunk.webmention_ssrf import SSRFBlockedError, safe_get, user_agent
|
|
|
|
challenge = new_challenge()
|
|
params = {"hub.mode": intent["mode"], "hub.topic": intent["topic"], "hub.challenge": challenge}
|
|
if intent["mode"] == "subscribe":
|
|
params["hub.lease_seconds"] = intent["lease_seconds"]
|
|
url = callback_with_params(intent["callback"], params)
|
|
|
|
try:
|
|
result = safe_get(url, headers={"User-Agent": user_agent(self.app.config["SITE_URL"])},
|
|
max_redirects=0, max_bytes=64 * 1024)
|
|
verified = 200 <= result.status_code < 300 and result.text.strip() == challenge
|
|
retry = result.status_code >= 500
|
|
error = None if verified else f"HTTP {result.status_code}" + (
|
|
"" if result.status_code >= 300 else ": challenge not echoed")
|
|
except (SSRFBlockedError, ValueError) as e:
|
|
verified, retry, error = False, False, f"Blocked: {e}"
|
|
except httpx.TooManyRedirects:
|
|
verified, retry, error = False, False, "Redirected (not allowed)"
|
|
except httpx.HTTPError as e:
|
|
verified, retry, error = False, True, str(e)[:200]
|
|
|
|
if verified:
|
|
if intent["mode"] == "subscribe":
|
|
db.execute(
|
|
"""
|
|
INSERT INTO websub_subscriptions
|
|
(topic, callback, secret, lease_seconds, expires_at, verified_at)
|
|
VALUES (?, ?, ?, ?, ?, ?)
|
|
ON CONFLICT(topic, callback) DO UPDATE SET
|
|
secret = excluded.secret, lease_seconds = excluded.lease_seconds,
|
|
expires_at = excluded.expires_at, verified_at = excluded.verified_at,
|
|
last_error = NULL
|
|
""",
|
|
(intent["topic"], intent["callback"], intent["secret"], intent["lease_seconds"],
|
|
_later(intent["lease_seconds"]), _now()),
|
|
)
|
|
else:
|
|
db.execute("DELETE FROM websub_subscriptions WHERE topic = ? AND callback = ?",
|
|
(intent["topic"], intent["callback"]))
|
|
db.execute("DELETE FROM websub_intents WHERE id = ?", (intent["id"],))
|
|
logger.info(f"WebSub {intent['mode']} verified: {intent['callback']} -> {intent['topic']}")
|
|
elif retry and intent["attempts"] < len(INTENT_BACKOFF):
|
|
db.execute("UPDATE websub_intents SET attempts = attempts + 1, next_attempt_at = ? WHERE id = ?",
|
|
(_later(INTENT_BACKOFF[intent["attempts"]]), intent["id"]))
|
|
else:
|
|
# Verification failed: the subscription state stays as it was
|
|
db.execute("DELETE FROM websub_intents WHERE id = ?", (intent["id"],))
|
|
logger.info(f"WebSub {intent['mode']} not verified for {intent['callback']}: {error}")
|
|
db.commit()
|
|
|
|
# --- content distribution (spec 6, 7) ------------------------------------
|
|
|
|
def render_topic(self, topic: str):
|
|
"""(status, content_type, body) of a topic, rendered in-app"""
|
|
path = urlparse(topic).path or "/"
|
|
response = self.app.test_client().get(path, headers={"Accept": "*/*"})
|
|
return response.status_code, response.headers.get("Content-Type", ""), response.get_data()
|
|
|
|
def deliver(self, db, delivery) -> None:
|
|
from starpunk.websub import hub_url
|
|
from starpunk.webmention_ssrf import SSRFBlockedError, safe_post, user_agent
|
|
|
|
site = self.app.config["SITE_URL"]
|
|
sub = None
|
|
try:
|
|
if delivery["subscription_id"] is None:
|
|
# External hub: ping it (hub.url is the de facto parameter)
|
|
result = safe_post(hub_url(), data={"hub.mode": "publish", "hub.url": delivery["topic"],
|
|
"hub.topic": delivery["topic"]},
|
|
headers={"User-Agent": user_agent(site)})
|
|
else:
|
|
sub = db.execute(
|
|
"SELECT * FROM websub_subscriptions WHERE id = ? AND expires_at > ?",
|
|
(delivery["subscription_id"], _now()),
|
|
).fetchone()
|
|
if sub is None: # unsubscribed or expired meanwhile
|
|
db.execute("DELETE FROM websub_deliveries WHERE id = ?", (delivery["id"],))
|
|
db.commit()
|
|
return
|
|
status, content_type, body = self.render_topic(sub["topic"])
|
|
if status != 200: # e.g. a tag page whose last note went
|
|
db.execute("DELETE FROM websub_deliveries WHERE id = ?", (delivery["id"],))
|
|
db.commit()
|
|
return
|
|
headers = {
|
|
"Content-Type": content_type,
|
|
"Link": f'<{hub_url()}>; rel="hub", <{sub["topic"]}>; rel="self"',
|
|
"User-Agent": user_agent(site),
|
|
}
|
|
if sub["secret"]:
|
|
headers["X-Hub-Signature"] = signature(sub["secret"], body)
|
|
result = safe_post(sub["callback"], content=body, headers=headers)
|
|
status, error = result.status_code, f"HTTP {result.status_code}"
|
|
except (SSRFBlockedError, ValueError) as e:
|
|
status, error = None, f"Blocked: {e}"
|
|
except httpx.HTTPError as e:
|
|
status, error = None, str(e)[:200] or type(e).__name__
|
|
|
|
if status is not None and 200 <= status < 300:
|
|
db.execute("DELETE FROM websub_deliveries WHERE id = ?", (delivery["id"],))
|
|
if sub is not None:
|
|
db.execute("UPDATE websub_subscriptions SET last_success_at = ?, last_error = NULL WHERE id = ?",
|
|
(_now(), sub["id"]))
|
|
elif sub is not None and status == 410:
|
|
db.execute("DELETE FROM websub_subscriptions WHERE id = ?", (sub["id"],))
|
|
logger.info(f"WebSub subscriber gone (410), removed: {sub['callback']}")
|
|
elif delivery["attempts"] < len(DELIVERY_BACKOFF) and not error.startswith("Blocked"):
|
|
db.execute(
|
|
"UPDATE websub_deliveries SET attempts = attempts + 1, next_attempt_at = ?, last_error = ? WHERE id = ?",
|
|
(_later(DELIVERY_BACKOFF[delivery["attempts"]]), error, delivery["id"]),
|
|
)
|
|
if sub is not None:
|
|
db.execute("UPDATE websub_subscriptions SET last_error = ? WHERE id = ?", (error, sub["id"]))
|
|
else:
|
|
db.execute("DELETE FROM websub_deliveries WHERE id = ?", (delivery["id"],))
|
|
logger.warning(f"WebSub delivery of {delivery['topic']} gave up: {error}")
|
|
if sub is not None:
|
|
self._drop_if_failing(db, sub, error)
|
|
db.commit()
|
|
|
|
def _drop_if_failing(self, db, sub, error: str) -> None:
|
|
last_ok = sub["last_success_at"] or sub["verified_at"] or sub["created_at"]
|
|
try:
|
|
since = datetime.fromisoformat(str(last_ok))
|
|
except ValueError:
|
|
return
|
|
if datetime.utcnow() - since > DROP_AFTER:
|
|
db.execute("DELETE FROM websub_subscriptions WHERE id = ?", (sub["id"],))
|
|
logger.info(f"WebSub subscription failing for {DROP_AFTER.days} days, removed: {sub['callback']}")
|
|
else:
|
|
db.execute("UPDATE websub_subscriptions SET last_error = ? WHERE id = ?", (error, sub["id"]))
|
|
|
|
# --- housekeeping -------------------------------------------------------
|
|
|
|
def prune(self, db) -> int:
|
|
"""Remove subscriptions whose lease has run out (they weren't renewed)"""
|
|
cursor = db.execute("DELETE FROM websub_subscriptions WHERE expires_at <= ?", (_now(),))
|
|
db.commit()
|
|
self._last_prune = datetime.utcnow()
|
|
if cursor.rowcount:
|
|
logger.info(f"WebSub: removed {cursor.rowcount} expired subscriptions")
|
|
return cursor.rowcount
|
|
|
|
|
|
def start_worker(app) -> Optional[WebSubWorker]:
|
|
if not app.config.get("WEBSUB_ENABLED", True) or app.config.get("TESTING", False):
|
|
return None
|
|
worker = WebSubWorker(app, poll_interval=app.config.get("WEBSUB_POLL_INTERVAL", 5))
|
|
worker.start()
|
|
return worker
|