Files
StarPunk/starpunk/websub.py
T
PhilandClaude Opus 5.5 b2043286fa fix: WebSub hardening from the adversarial review (v2.10.1)
- Per-item error handling in the worker; hard time limits on requests
- Notes with photos announced after attach; feed cache/ETag follow
  content_version
- Keep failing subscriptions until lease end; retry DNS and render errors;
  hub.mode=denied for gone topics
- Spec 5.1: decode unreserved chars, echo requested topic, ignore lease and
  secret on unsubscribe, case-insensitive Content-Type, refuse empty secret
- Limits count pending requests per callback address; callback validation;
  request_id guards the verification race (migration 017)
- Render once per topic per pass; likes only push /likes; Vary: Accept

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-03 11:23:36 -07:00

444 lines
16 KiB
Python

"""
WebSub publishing and the built-in hub (v2.10.0, ADR-071)
Publisher: topic responses advertise the hub and their own URL; a change to a
public note clears cached feeds and queues delivery of every affected topic.
Hub: subscriptions to this site's own topics only. Requests are accepted
here (202) and verified, delivered and retried by the WebSub worker.
References:
- W3C WebSub: https://www.w3.org/TR/websub/
- ADR-071: WebSub publishing with a built-in hub
"""
import json
import secrets
from typing import Iterable, Optional
from urllib.parse import urlparse
from flask import current_app, request
# Topic endpoints and their fixed paths (tag pages are added per tag)
FEED_TOPICS = {
"public.feed_atom": "/feed.atom",
"public.feed_rss": "/feed.rss",
"public.feed_xml_legacy": "/feed.rss", # legacy alias of the RSS topic
"public.feed_json": "/feed.json",
}
PAGE_TOPICS = {"public.index": "/", "public.likes": "/likes"}
NEGOTIATED = { # /feed: the topic of whichever format was served
"application/atom+xml": "/feed.atom",
"application/rss+xml": "/feed.rss",
"application/feed+json": "/feed.json",
"application/json": "/feed.json",
}
LEASE_DEFAULT = 10 * 24 * 3600
LEASE_MIN = 24 * 3600
LEASE_MAX = 30 * 24 * 3600
MAX_SECRET_BYTES = 200
class SubscriptionError(Exception):
"""A subscription request the hub refuses (4xx with a plain-text reason)"""
def __init__(self, message: str, status: int = 400):
super().__init__(message)
self.status = status
# --- Configuration -----------------------------------------------------------
def enabled() -> bool:
return bool(current_app.config.get("WEBSUB_ENABLED", True))
def external_hub() -> Optional[str]:
"""URL of an external hub, if one is configured instead of the built-in one"""
return current_app.config.get("WEBSUB_HUB_URL") or None
def hub_url() -> str:
# Built from SITE_URL so the worker can use it outside a request
return external_hub() or site_url() + "/websub"
def site_url() -> str:
return current_app.config["SITE_URL"].rstrip("/")
# --- Topics ------------------------------------------------------------------
def topic_url(path: str) -> str:
return site_url() + path
def tag_topic(tag_name: str) -> str:
from urllib.parse import quote
return f"{site_url()}/tag/{quote(tag_name, safe='')}"
def topic_for_request(response=None) -> Optional[str]:
"""The topic URL the current request serves, or None if it isn't a topic"""
endpoint = request.endpoint
if endpoint in PAGE_TOPICS:
return topic_url(PAGE_TOPICS[endpoint])
if endpoint in FEED_TOPICS:
return topic_url(FEED_TOPICS[endpoint])
if endpoint == "public.feed" and response is not None:
mime = (response.mimetype or "").lower()
return topic_url(NEGOTIATED[mime]) if mime in NEGOTIATED else None
if endpoint == "public.tag":
from starpunk.tags import normalize_tag
tag = (request.view_args or {}).get("tag", "")
return tag_topic(normalize_tag(tag)[0]) if tag else None
return None
_UNRESERVED = set("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~")
def decode_unreserved(value: str) -> str:
"""
Decode percent-encoded unreserved characters (%66 -> f), leaving the rest
Spec 5.1.1: hubs MUST always decode non-reserved characters in URL
parameters, so /%66eed.atom and /feed.atom are the same topic.
"""
out, i = [], 0
while i < len(value):
if value[i] == "%" and i + 3 <= len(value):
try:
char = chr(int(value[i + 1:i + 3], 16))
except ValueError:
char = None
if char is not None and char in _UNRESERVED:
out.append(char)
i += 3
continue
out.append(value[i])
i += 1
return "".join(out)
def canonical_topic(url: str) -> Optional[str]:
"""
A subscription's hub.topic as one of this site's topics, or None
Unreserved percent-encodings are decoded and the scheme and host are
compared case-insensitively. Tag topics must name an existing tag.
"""
if not url:
return None
site = urlparse(site_url())
parsed = urlparse(decode_unreserved(url.strip()))
if (parsed.scheme.lower(), parsed.netloc.lower()) != (site.scheme, site.netloc.lower()):
return None
path = parsed.path or "/"
if parsed.query or parsed.fragment:
return None
if path in set(PAGE_TOPICS.values()) | set(FEED_TOPICS.values()):
return topic_url(path)
if path.startswith("/tag/"):
from urllib.parse import unquote
from starpunk.tags import get_tag_by_name, normalize_tag
name = normalize_tag(unquote(path[len("/tag/"):]))[0]
if name and get_tag_by_name(name):
return tag_topic(name)
return None
def link_header_values(topic: str) -> list[str]:
return [f'<{hub_url()}>; rel="hub"', f'<{topic}>; rel="self"']
def add_discovery_headers(response):
"""after_request: Link rel=hub and rel=self on topic responses (spec 4)"""
if not enabled() or response.status_code not in (200, 304):
return response
topic = topic_for_request(response)
if topic:
for value in link_header_values(topic):
response.headers.add("Link", value)
return response
def template_links() -> Optional[dict]:
"""For base.html: {'hub', 'self'} on topic pages"""
if not enabled() or not request:
return None
topic = topic_for_request()
return {"hub": hub_url(), "self": topic} if topic else None
def embed_hub(format_name: str, content: str) -> str:
"""Add the hub link inside a generated feed, next to its self link"""
if not enabled():
return content
hub = hub_url()
if format_name == "atom":
marker = '<link rel="self"'
return content.replace(marker, f'<link rel="hub" href="{_xml(hub)}"/>\n {marker}', 1)
if format_name == "rss":
marker = "<atom:link "
return content.replace(marker, f'<atom:link href="{_xml(hub)}" rel="hub"/>\n {marker}', 1)
if format_name == "json":
feed = json.loads(content)
feed["hubs"] = [{"type": "WebSub", "url": hub}]
return json.dumps(feed, ensure_ascii=False, indent=2)
return content
def _xml(value: str) -> str:
return (value.replace("&", "&amp;").replace("<", "&lt;")
.replace(">", "&gt;").replace('"', "&quot;"))
# --- Changes -------------------------------------------------------------------
def get_content_version(db) -> str:
row = db.execute("SELECT value FROM site_state WHERE key = 'content_version'").fetchone()
return row[0] if row else "0"
def bump_content_version(db) -> None:
db.execute(
"INSERT INTO site_state (key, value) VALUES ('content_version', '1') "
"ON CONFLICT(key) DO UPDATE SET value = CAST(value AS INTEGER) + 1"
)
def note_changed(note, was_public: bool = False, old_tags: Iterable[str] = ()) -> None:
"""
A note was created, edited, deleted, restored or (un)published
Clears cached feeds in every process and queues delivery of the topics
that show it. Changes that are never public (a draft edited) do nothing.
Args:
note: The note (after the change; for a delete, before it)
was_public: Whether it was public before the change
old_tags: Its tag names before the change
"""
from starpunk.database import get_db
is_public = bool(note.published) and not getattr(note, "deleted_at", None)
if not (is_public or was_public):
return
db = get_db(current_app)
bump_content_version(db)
db.commit()
if not enabled():
return
from starpunk.tags import get_note_tags
try:
tags = {t["name"] for t in get_note_tags(note.id)}
except Exception:
tags = set()
tags |= set(old_tags)
# Likes appear on /likes, not on the home page or in the feeds
if getattr(note, "like_of", None):
topics = [topic_url("/likes")]
else:
topics = [topic_url(p) for p in ("/", "/feed.atom", "/feed.rss", "/feed.json")]
topics += [tag_topic(name) for name in sorted(tags)]
queue_topics(db, topics)
def queue_topics(db, topics: Iterable[str]) -> int:
"""Queue delivery of topics: to their subscribers, or a ping to an external hub"""
topics = list(dict.fromkeys(topics))
if not topics:
return 0
if external_hub():
for topic in topics:
db.execute("DELETE FROM websub_deliveries WHERE subscription_id IS NULL AND topic = ?", (topic,))
db.execute("INSERT INTO websub_deliveries (topic) VALUES (?)", (topic,))
db.commit()
return len(topics)
placeholders = ",".join("?" * len(topics))
rows = db.execute(
f"SELECT id, topic FROM websub_subscriptions WHERE topic IN ({placeholders}) "
"AND verified_at IS NOT NULL AND expires_at > datetime('now')",
topics,
).fetchall()
for sub_id, topic in rows:
# The newest content replaces anything still waiting (ADR-071 6)
db.execute("DELETE FROM websub_deliveries WHERE subscription_id = ?", (sub_id,))
db.execute("INSERT INTO websub_deliveries (subscription_id, topic) VALUES (?, ?)", (sub_id, topic))
db.commit()
return len(rows)
# --- Hub: subscription requests (spec 5.1) -------------------------------------
MAX_CALLBACK_LENGTH = 2048
MAX_PENDING_INTENTS = 200
ALLOWED_PORTS = {80, 443} # plus any port from 1024 up
def _clean_callback(callback: str) -> str:
"""
Check a hub.callback and return it with an ASCII (IDNA) host
Raises:
SubscriptionError: if it is not an acceptable callback URL
"""
import httpx
if not callback or len(callback) > MAX_CALLBACK_LENGTH:
raise SubscriptionError(f"hub.callback must be a URL of at most {MAX_CALLBACK_LENGTH} characters")
if any(ord(c) < 0x20 or ord(c) == 0x7f for c in callback) or " " in callback:
raise SubscriptionError("hub.callback contains spaces or control characters")
parsed = urlparse(callback)
if parsed.scheme.lower() not in ("http", "https") or not parsed.hostname:
raise SubscriptionError("hub.callback must be an absolute http(s) URL")
try:
port = parsed.port
except ValueError:
raise SubscriptionError("hub.callback has an invalid port")
if port is not None and port not in ALLOWED_PORTS and port < 1024:
raise SubscriptionError("hub.callback may not use that port")
try:
host = parsed.hostname.encode("idna").decode("ascii")
except UnicodeError:
raise SubscriptionError("hub.callback has an invalid host name")
netloc = host + (f":{port}" if port is not None else "")
callback = parsed._replace(scheme=parsed.scheme.lower(), netloc=netloc).geturl()
if host.lower() == (urlparse(site_url()).hostname or "").lower():
raise SubscriptionError("hub.callback may not be on this site")
try:
httpx.URL(callback)
except Exception:
raise SubscriptionError("hub.callback is not a valid URL")
return callback
def accept_subscription_request(form, db) -> None:
"""
Validate a subscription request and queue it for verification
Raises:
SubscriptionError: with the status and plain-text reason to return
"""
from starpunk.webmention_ssrf import SSRFBlockedError, validate_url_for_fetch
mode = (form.get("hub.mode") or "").strip()
if mode not in ("subscribe", "unsubscribe"):
raise SubscriptionError("hub.mode must be subscribe or unsubscribe")
if mode == "subscribe" and external_hub():
# Unsubscribes are still taken, so old subscribers can leave
raise SubscriptionError("This site uses another hub: " + external_hub(), 400)
callback = _clean_callback((form.get("hub.callback") or "").strip())
try:
callback_ip = validate_url_for_fetch(callback)
except (SSRFBlockedError, ValueError) as e:
raise SubscriptionError(f"hub.callback is not reachable: {e}")
requested_topic = (form.get("hub.topic") or "").strip()
topic = canonical_topic(requested_topic)
if not topic:
raise SubscriptionError("hub.topic is not a topic of this site; use the rel=self URL")
secret, lease = None, None
if mode == "subscribe":
# hub.secret and hub.lease_seconds MUST be ignored for unsubscriptions
secret = form.get("hub.secret")
if secret is not None:
if secret == "":
raise SubscriptionError("hub.secret may not be empty")
if len(secret.encode("utf-8")) >= MAX_SECRET_BYTES:
raise SubscriptionError("hub.secret must be less than 200 bytes")
lease = LEASE_DEFAULT
if form.get("hub.lease_seconds"):
try:
lease = int(form["hub.lease_seconds"])
except ValueError:
raise SubscriptionError("hub.lease_seconds must be a number")
lease = max(LEASE_MIN, min(lease, LEASE_MAX))
_check_limits(db, topic, callback, callback_ip)
# Each request gets its own id; the latest replaces any pending one
db.execute(
"""
INSERT INTO websub_intents
(topic, callback, mode, secret, lease_seconds, request_id, requested_topic, callback_ip)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(topic, callback) DO UPDATE SET
mode = excluded.mode, secret = excluded.secret,
lease_seconds = excluded.lease_seconds, request_id = excluded.request_id,
requested_topic = excluded.requested_topic, callback_ip = excluded.callback_ip,
attempts = 0, next_attempt_at = CURRENT_TIMESTAMP, created_at = CURRENT_TIMESTAMP
""",
(topic, callback, mode, secret, lease, secrets.token_hex(16), requested_topic, callback_ip),
)
db.commit()
def _check_limits(db, topic: str, callback: str, callback_ip: str) -> None:
"""
Subscription limits, counting pending requests too (ADR-071 4)
Per callback address rather than host name, so many subdomains on one
server share a limit. Renewing an existing subscription is always fine.
"""
if db.execute("SELECT 1 FROM websub_subscriptions WHERE topic = ? AND callback = ?",
(topic, callback)).fetchone():
return
pending = db.execute("SELECT COUNT(*) FROM websub_intents").fetchone()[0]
if pending >= MAX_PENDING_INTENTS:
raise SubscriptionError("Too many subscription requests waiting; try again later", 429)
total = db.execute(
"""
SELECT COUNT(*) FROM (
SELECT topic, callback FROM websub_subscriptions
UNION
SELECT topic, callback FROM websub_intents WHERE mode = 'subscribe'
)
"""
).fetchone()[0]
if total >= current_app.config.get("WEBSUB_MAX_SUBSCRIPTIONS", 1000):
raise SubscriptionError("This hub has reached its subscription limit", 429)
per_address = db.execute(
"""
SELECT COUNT(*) FROM (
SELECT topic, callback FROM websub_subscriptions WHERE callback_ip = ?
UNION
SELECT topic, callback FROM websub_intents WHERE mode = 'subscribe' AND callback_ip = ?
)
""",
(callback_ip, callback_ip),
).fetchone()[0]
if per_address >= current_app.config.get("WEBSUB_MAX_PER_HOST", 25):
raise SubscriptionError("Too many subscriptions for that callback's server", 429)
def stats(db) -> dict:
"""Active subscribers per topic and deliveries being retried (admin)"""
by_topic = db.execute(
"SELECT topic, COUNT(*) FROM websub_subscriptions WHERE expires_at > datetime('now') "
"GROUP BY topic ORDER BY COUNT(*) DESC, topic"
).fetchall()
retrying = db.execute(
"SELECT COUNT(*) FROM websub_deliveries WHERE attempts > 0"
).fetchone()[0]
return {"by_topic": [(t, n) for t, n in by_topic], "retrying": retrying,
"external": external_hub()}
def new_challenge() -> str:
return secrets.token_urlsafe(32)