- Per-item error handling in the worker; hard time limits on requests - Notes with photos announced after attach; feed cache/ETag follow content_version - Keep failing subscriptions until lease end; retry DNS and render errors; hub.mode=denied for gone topics - Spec 5.1: decode unreserved chars, echo requested topic, ignore lease and secret on unsubscribe, case-insensitive Content-Type, refuse empty secret - Limits count pending requests per callback address; callback validation; request_id guards the verification race (migration 017) - Render once per topic per pass; likes only push /likes; Vary: Accept Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
363 lines
16 KiB
Python
363 lines
16 KiB
Python
"""
|
|
WebSub worker: verification of intent and content distribution (v2.10.0)
|
|
|
|
Runs like the Webmention worker: a daemon thread in every gunicorn process,
|
|
but only the process holding the file lock works the queues.
|
|
|
|
- Intents (spec 5.3): GET the callback with a challenge; a 2xx echoing it
|
|
verifies. Anything else (including redirects) leaves the subscription as
|
|
it was. Network errors, DNS failures and 5xx are retried a few times.
|
|
- Deliveries (spec 7): render the topic in-app and POST it with
|
|
Link rel=hub/self and, if there is a secret, X-Hub-Signature (sha256).
|
|
Failures are retried; the subscription stays until its lease ends.
|
|
- External hub: a delivery without a subscription is a publish ping.
|
|
|
|
v2.10.1: every queue item is handled on its own, so one bad item can't stop
|
|
the worker; all outbound requests have a hard time limit.
|
|
|
|
References:
|
|
- https://www.w3.org/TR/websub/
|
|
- ADR-071
|
|
"""
|
|
|
|
import fcntl
|
|
import hashlib
|
|
import hmac
|
|
import logging
|
|
import threading
|
|
from datetime import datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
from urllib.parse import urlencode, urlparse, urlsplit, urlunsplit
|
|
|
|
import httpx
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
DELIVERY_BACKOFF = [60, 300, 1800, 7200, 21600] # then give up on this delivery
|
|
INTENT_BACKOFF = [60, 300, 900]
|
|
PRUNE_EVERY = timedelta(hours=1)
|
|
BATCH = 10
|
|
VERIFY_MAX_BYTES = 4096 # a challenge reply is short; larger means not verified
|
|
|
|
|
|
def callback_with_params(callback: str, params: dict) -> str:
|
|
"""Append parameters to the callback, keeping its query string (spec 5.3)"""
|
|
parts = urlsplit(callback)
|
|
query = parts.query + ("&" if parts.query else "") + urlencode(params)
|
|
return urlunsplit((parts.scheme, parts.netloc, parts.path, query, parts.fragment))
|
|
|
|
|
|
def signature(secret: str, body: bytes) -> str:
|
|
return "sha256=" + hmac.new(secret.encode("utf-8"), body, hashlib.sha256).hexdigest()
|
|
|
|
|
|
def _now() -> str:
|
|
return datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
|
|
|
|
|
|
def _later(seconds: int) -> str:
|
|
return (datetime.utcnow() + timedelta(seconds=seconds)).strftime("%Y-%m-%d %H:%M:%S")
|
|
|
|
|
|
class WebSubWorker(threading.Thread):
|
|
"""Daemon thread that verifies subscriptions and delivers content"""
|
|
|
|
def __init__(self, app, poll_interval: int = 5):
|
|
super().__init__(daemon=True, name="websub-worker")
|
|
self.app = app
|
|
self.poll_interval = poll_interval
|
|
self._stop_event = threading.Event()
|
|
self._lock_file = None
|
|
self._last_prune = None
|
|
|
|
# --- thread plumbing ----------------------------------------------------
|
|
|
|
def run(self):
|
|
logger.info("WebSub worker started")
|
|
while not self._stop_event.is_set():
|
|
if not self._acquire_lock():
|
|
self._stop_event.wait(self.poll_interval)
|
|
continue
|
|
try:
|
|
with self.app.app_context():
|
|
done = self.process_once()
|
|
except Exception as e:
|
|
logger.error(f"WebSub worker error: {e}", exc_info=True)
|
|
done = 0
|
|
if not done:
|
|
self._stop_event.wait(self.poll_interval)
|
|
self._release_lock()
|
|
|
|
def stop(self):
|
|
self._stop_event.set()
|
|
|
|
def _acquire_lock(self) -> bool:
|
|
if self._lock_file:
|
|
return True
|
|
path = Path(self.app.config["DATABASE_PATH"]).parent / "websub-worker.lock"
|
|
lock_file = open(path, "w")
|
|
try:
|
|
fcntl.flock(lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except OSError:
|
|
lock_file.close()
|
|
return False
|
|
self._lock_file = lock_file
|
|
logger.info("WebSub worker holds the queue lock")
|
|
return True
|
|
|
|
def _release_lock(self):
|
|
if self._lock_file:
|
|
self._lock_file.close()
|
|
self._lock_file = None
|
|
|
|
# --- one pass (also called directly by tests) ---------------------------
|
|
|
|
def process_once(self) -> int:
|
|
from starpunk.database import get_db
|
|
|
|
db = get_db(self.app)
|
|
done = 0
|
|
rendered = {} # topic -> (status, content type, body), once per pass
|
|
|
|
for row in db.execute(
|
|
"SELECT * FROM websub_intents WHERE next_attempt_at <= ? ORDER BY next_attempt_at, id LIMIT ?",
|
|
(_now(), BATCH),
|
|
).fetchall():
|
|
self._guarded(db, row, "verification", self.verify, self._intent_crashed)
|
|
done += 1
|
|
for row in db.execute(
|
|
"SELECT * FROM websub_deliveries WHERE next_attempt_at <= ? ORDER BY next_attempt_at, id LIMIT ?",
|
|
(_now(), BATCH),
|
|
).fetchall():
|
|
self._guarded(db, row, "delivery", lambda d, r: self.deliver(d, r, rendered), self._retry_or_drop)
|
|
done += 1
|
|
if self._last_prune is None or datetime.utcnow() - self._last_prune > PRUNE_EVERY:
|
|
self.prune(db)
|
|
return done
|
|
|
|
def _guarded(self, db, row, what, handler, on_crash) -> None:
|
|
"""Handle one queue item; an unexpected error affects only that item"""
|
|
try:
|
|
handler(db, row)
|
|
except Exception as e:
|
|
logger.error(f"WebSub {what} {row['id']} failed unexpectedly: {e}", exc_info=True)
|
|
try:
|
|
db.rollback()
|
|
on_crash(db, row, f"{type(e).__name__}: {e}"[:200])
|
|
db.commit()
|
|
except Exception as cleanup:
|
|
logger.error(f"WebSub could not record the failure: {cleanup}")
|
|
|
|
def _intent_crashed(self, db, intent, error) -> None:
|
|
db.execute("DELETE FROM websub_intents WHERE id = ? AND request_id IS ?",
|
|
(intent["id"], intent["request_id"]))
|
|
|
|
# --- verification of intent (spec 5.3) ----------------------------------
|
|
|
|
def verify(self, db, intent) -> None:
|
|
from starpunk.websub import new_challenge
|
|
from starpunk.webmention_ssrf import (
|
|
FetchTooLargeError, ResolutionError, SSRFBlockedError, safe_get, user_agent,
|
|
)
|
|
|
|
challenge = new_challenge()
|
|
# The topic exactly as the subscriber sent it (spec 5.3)
|
|
params = {"hub.mode": intent["mode"],
|
|
"hub.topic": intent["requested_topic"] or intent["topic"],
|
|
"hub.challenge": challenge}
|
|
if intent["mode"] == "subscribe":
|
|
params["hub.lease_seconds"] = intent["lease_seconds"]
|
|
url = callback_with_params(intent["callback"], params)
|
|
|
|
retry = False
|
|
try:
|
|
result = safe_get(url, headers={"User-Agent": user_agent(self.app.config["SITE_URL"])},
|
|
max_redirects=0, max_bytes=VERIFY_MAX_BYTES)
|
|
# Exact match; a trailing newline from a framework is tolerated
|
|
verified = 200 <= result.status_code < 300 and result.text.rstrip("\r\n") == challenge
|
|
retry = result.status_code >= 500
|
|
error = None if verified else f"HTTP {result.status_code}" + (
|
|
"" if result.status_code >= 300 else ": challenge not echoed")
|
|
except ResolutionError as e:
|
|
verified, retry, error = False, True, str(e)
|
|
except (SSRFBlockedError, ValueError) as e:
|
|
verified, error = False, f"Blocked: {e}"
|
|
except FetchTooLargeError:
|
|
verified, error = False, "Reply too large to be a challenge"
|
|
except httpx.TooManyRedirects:
|
|
verified, error = False, "Redirected (not allowed)"
|
|
except httpx.HTTPError as e:
|
|
verified, retry, error = False, True, (str(e) or type(e).__name__)[:200]
|
|
|
|
# Only act if no newer request replaced this one meanwhile
|
|
current = db.execute("SELECT request_id FROM websub_intents WHERE id = ?", (intent["id"],)).fetchone()
|
|
if current is None or current[0] != intent["request_id"]:
|
|
return
|
|
|
|
if verified:
|
|
if intent["mode"] == "subscribe":
|
|
db.execute(
|
|
"""
|
|
INSERT INTO websub_subscriptions
|
|
(topic, callback, secret, lease_seconds, expires_at, verified_at, callback_ip)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
ON CONFLICT(topic, callback) DO UPDATE SET
|
|
secret = excluded.secret, lease_seconds = excluded.lease_seconds,
|
|
expires_at = excluded.expires_at, verified_at = excluded.verified_at,
|
|
callback_ip = excluded.callback_ip, last_error = NULL
|
|
""",
|
|
(intent["topic"], intent["callback"], intent["secret"], intent["lease_seconds"],
|
|
_later(intent["lease_seconds"]), _now(), intent["callback_ip"]),
|
|
)
|
|
else:
|
|
db.execute("DELETE FROM websub_subscriptions WHERE topic = ? AND callback = ?",
|
|
(intent["topic"], intent["callback"]))
|
|
db.execute("DELETE FROM websub_intents WHERE id = ? AND request_id IS ?",
|
|
(intent["id"], intent["request_id"]))
|
|
logger.info(f"WebSub {intent['mode']} verified: {intent['callback']} -> {intent['topic']}")
|
|
elif retry and intent["attempts"] < len(INTENT_BACKOFF):
|
|
db.execute(
|
|
"UPDATE websub_intents SET attempts = attempts + 1, next_attempt_at = ? "
|
|
"WHERE id = ? AND request_id IS ?",
|
|
(_later(INTENT_BACKOFF[intent["attempts"]]), intent["id"], intent["request_id"]),
|
|
)
|
|
else:
|
|
# Verification failed: the subscription state stays as it was
|
|
db.execute("DELETE FROM websub_intents WHERE id = ? AND request_id IS ?",
|
|
(intent["id"], intent["request_id"]))
|
|
logger.info(f"WebSub {intent['mode']} not verified for {intent['callback']}: {error}")
|
|
db.commit()
|
|
|
|
# --- content distribution (spec 7) ---------------------------------------
|
|
|
|
def render_topic(self, topic: str):
|
|
"""(status, content_type, body) of a topic, rendered in-app"""
|
|
path = urlparse(topic).path or "/"
|
|
response = self.app.test_client().get(path, headers={"Accept": "*/*"})
|
|
return response.status_code, response.headers.get("Content-Type", ""), response.get_data()
|
|
|
|
def deliver(self, db, delivery, rendered: Optional[dict] = None) -> None:
|
|
from starpunk.websub import hub_url
|
|
from starpunk.webmention_ssrf import ResolutionError, SSRFBlockedError, safe_post, user_agent
|
|
|
|
rendered = {} if rendered is None else rendered
|
|
site = self.app.config["SITE_URL"]
|
|
sub = None
|
|
if delivery["subscription_id"] is None:
|
|
# External hub: ping it (hub.url is the de facto parameter)
|
|
target, kwargs = hub_url(), {
|
|
"data": {"hub.mode": "publish", "hub.url": delivery["topic"], "hub.topic": delivery["topic"]},
|
|
"headers": {"User-Agent": user_agent(site)},
|
|
}
|
|
else:
|
|
sub = db.execute(
|
|
"SELECT * FROM websub_subscriptions WHERE id = ? AND expires_at > ?",
|
|
(delivery["subscription_id"], _now()),
|
|
).fetchone()
|
|
if sub is None: # unsubscribed or expired meanwhile
|
|
db.execute("DELETE FROM websub_deliveries WHERE id = ?", (delivery["id"],))
|
|
db.commit()
|
|
return
|
|
if sub["topic"] not in rendered:
|
|
rendered[sub["topic"]] = self.render_topic(sub["topic"])
|
|
status, content_type, body = rendered[sub["topic"]]
|
|
if status in (404, 410):
|
|
# The topic is gone: end the subscription and say so (spec 5.2)
|
|
self._deny(sub, "The topic no longer exists")
|
|
db.execute("DELETE FROM websub_subscriptions WHERE id = ?", (sub["id"],))
|
|
db.execute("DELETE FROM websub_deliveries WHERE id = ?", (delivery["id"],))
|
|
db.commit()
|
|
return
|
|
if status != 200:
|
|
self._retry_or_drop(db, delivery, f"Topic rendered HTTP {status}")
|
|
db.commit()
|
|
return
|
|
headers = {
|
|
"Content-Type": content_type,
|
|
"Link": f'<{hub_url()}>; rel="hub", <{sub["topic"]}>; rel="self"',
|
|
"User-Agent": user_agent(site),
|
|
}
|
|
if sub["secret"] is not None:
|
|
headers["X-Hub-Signature"] = signature(sub["secret"], body)
|
|
target, kwargs = sub["callback"], {"content": body, "headers": headers}
|
|
|
|
permanent = False
|
|
try:
|
|
result = safe_post(target, **kwargs)
|
|
status, error = result.status_code, f"HTTP {result.status_code}"
|
|
except ResolutionError as e:
|
|
status, error = None, str(e) # DNS can recover: retry
|
|
except (SSRFBlockedError, ValueError) as e:
|
|
status, error, permanent = None, f"Blocked: {e}", True
|
|
except httpx.HTTPError as e:
|
|
status, error = None, (str(e) or type(e).__name__)[:200]
|
|
|
|
succeeded = status is not None and 200 <= status < 300
|
|
if succeeded:
|
|
db.execute("DELETE FROM websub_deliveries WHERE id = ?", (delivery["id"],))
|
|
if sub is not None:
|
|
db.execute("UPDATE websub_subscriptions SET last_success_at = ?, last_error = NULL WHERE id = ?",
|
|
(_now(), sub["id"]))
|
|
elif sub is not None and status == 410:
|
|
db.execute("DELETE FROM websub_subscriptions WHERE id = ?", (sub["id"],))
|
|
db.execute("DELETE FROM websub_deliveries WHERE id = ?", (delivery["id"],))
|
|
logger.info(f"WebSub subscriber gone (410), removed: {sub['callback']}")
|
|
else:
|
|
if permanent:
|
|
db.execute("DELETE FROM websub_deliveries WHERE id = ?", (delivery["id"],))
|
|
logger.warning(f"WebSub delivery of {delivery['topic']} refused: {error}")
|
|
else:
|
|
self._retry_or_drop(db, delivery, error)
|
|
if sub is not None:
|
|
db.execute("UPDATE websub_subscriptions SET last_error = ? WHERE id = ?", (error, sub["id"]))
|
|
db.commit()
|
|
|
|
def _retry_or_drop(self, db, delivery, error: str) -> None:
|
|
"""
|
|
Retry a failed delivery with backoff, then give up on it
|
|
|
|
The subscription itself stays active until its lease ends (spec 7);
|
|
the next change is delivered again.
|
|
"""
|
|
if delivery["attempts"] < len(DELIVERY_BACKOFF):
|
|
db.execute(
|
|
"UPDATE websub_deliveries SET attempts = attempts + 1, next_attempt_at = ?, last_error = ? WHERE id = ?",
|
|
(_later(DELIVERY_BACKOFF[delivery["attempts"]]), error, delivery["id"]),
|
|
)
|
|
else:
|
|
db.execute("DELETE FROM websub_deliveries WHERE id = ?", (delivery["id"],))
|
|
logger.warning(f"WebSub delivery of {delivery['topic']} gave up: {error}")
|
|
|
|
def _deny(self, sub, reason: str) -> None:
|
|
"""Tell a subscriber its subscription has ended (hub.mode=denied), best effort"""
|
|
from starpunk.webmention_ssrf import safe_get, user_agent
|
|
url = callback_with_params(sub["callback"], {"hub.mode": "denied", "hub.topic": sub["topic"],
|
|
"hub.reason": reason})
|
|
try:
|
|
safe_get(url, headers={"User-Agent": user_agent(self.app.config["SITE_URL"])},
|
|
max_redirects=0, max_bytes=VERIFY_MAX_BYTES)
|
|
except Exception as e:
|
|
logger.info(f"WebSub denial to {sub['callback']} not delivered: {e}")
|
|
|
|
# --- housekeeping -------------------------------------------------------
|
|
|
|
def prune(self, db) -> int:
|
|
"""Remove subscriptions whose lease has run out, and their deliveries"""
|
|
cursor = db.execute("DELETE FROM websub_subscriptions WHERE expires_at <= ?", (_now(),))
|
|
db.execute("DELETE FROM websub_deliveries WHERE subscription_id IS NOT NULL AND "
|
|
"subscription_id NOT IN (SELECT id FROM websub_subscriptions)")
|
|
db.commit()
|
|
self._last_prune = datetime.utcnow()
|
|
if cursor.rowcount:
|
|
logger.info(f"WebSub: removed {cursor.rowcount} expired subscriptions")
|
|
return cursor.rowcount
|
|
|
|
|
|
def start_worker(app) -> Optional[WebSubWorker]:
|
|
if not app.config.get("WEBSUB_ENABLED", True) or app.config.get("TESTING", False):
|
|
return None
|
|
worker = WebSubWorker(app, poll_interval=app.config.get("WEBSUB_POLL_INTERVAL", 5))
|
|
worker.start()
|
|
return worker
|