ADR-071. Topics (/, /feed.atom, /feed.rss, /feed.json, /likes, /tag/<name>) advertise the hub and themselves in Link headers and embedded links. Note changes bump a content version (fresh feeds in every process) and queue deliveries. /websub accepts subscriptions for these topics only; the worker verifies intent, delivers signed content with retries, and prunes expired leases, all through the SSRF-safe pinned fetcher. WEBSUB_HUB_URL switches to an external hub (publish pings). Also: the memory monitor no longer stops after the first request; RSS self link is /feed.rss. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
341 lines
12 KiB
Python
341 lines
12 KiB
Python
"""
|
|
WebSub publishing and the built-in hub (v2.10.0, ADR-071)
|
|
|
|
Publisher: topic responses advertise the hub and their own URL; a change to a
|
|
public note clears cached feeds and queues delivery of every affected topic.
|
|
|
|
Hub: subscriptions to this site's own topics only. Requests are accepted
|
|
here (202) and verified, delivered and retried by the WebSub worker.
|
|
|
|
References:
|
|
- W3C WebSub: https://www.w3.org/TR/websub/
|
|
- ADR-071: WebSub publishing with a built-in hub
|
|
"""
|
|
|
|
import json
|
|
import secrets
|
|
from typing import Iterable, Optional
|
|
from urllib.parse import urlparse
|
|
|
|
from flask import current_app, request
|
|
|
|
|
|
# Topic endpoints and their fixed paths (tag pages are added per tag)
|
|
FEED_TOPICS = {
|
|
"public.feed_atom": "/feed.atom",
|
|
"public.feed_rss": "/feed.rss",
|
|
"public.feed_xml_legacy": "/feed.rss", # legacy alias of the RSS topic
|
|
"public.feed_json": "/feed.json",
|
|
}
|
|
PAGE_TOPICS = {"public.index": "/", "public.likes": "/likes"}
|
|
NEGOTIATED = { # /feed: the topic of whichever format was served
|
|
"application/atom+xml": "/feed.atom",
|
|
"application/rss+xml": "/feed.rss",
|
|
"application/feed+json": "/feed.json",
|
|
"application/json": "/feed.json",
|
|
}
|
|
|
|
LEASE_DEFAULT = 10 * 24 * 3600
|
|
LEASE_MIN = 24 * 3600
|
|
LEASE_MAX = 30 * 24 * 3600
|
|
MAX_SECRET_BYTES = 200
|
|
|
|
|
|
class SubscriptionError(Exception):
|
|
"""A subscription request the hub refuses (4xx with a plain-text reason)"""
|
|
|
|
def __init__(self, message: str, status: int = 400):
|
|
super().__init__(message)
|
|
self.status = status
|
|
|
|
|
|
# --- Configuration -----------------------------------------------------------
|
|
|
|
def enabled() -> bool:
|
|
return bool(current_app.config.get("WEBSUB_ENABLED", True))
|
|
|
|
|
|
def external_hub() -> Optional[str]:
|
|
"""URL of an external hub, if one is configured instead of the built-in one"""
|
|
return current_app.config.get("WEBSUB_HUB_URL") or None
|
|
|
|
|
|
def hub_url() -> str:
|
|
# Built from SITE_URL so the worker can use it outside a request
|
|
return external_hub() or site_url() + "/websub"
|
|
|
|
|
|
def site_url() -> str:
|
|
return current_app.config["SITE_URL"].rstrip("/")
|
|
|
|
|
|
# --- Topics ------------------------------------------------------------------
|
|
|
|
def topic_url(path: str) -> str:
|
|
return site_url() + path
|
|
|
|
|
|
def tag_topic(tag_name: str) -> str:
|
|
from urllib.parse import quote
|
|
return f"{site_url()}/tag/{quote(tag_name, safe='')}"
|
|
|
|
|
|
def topic_for_request(response=None) -> Optional[str]:
|
|
"""The topic URL the current request serves, or None if it isn't a topic"""
|
|
endpoint = request.endpoint
|
|
if endpoint in PAGE_TOPICS:
|
|
return topic_url(PAGE_TOPICS[endpoint])
|
|
if endpoint in FEED_TOPICS:
|
|
return topic_url(FEED_TOPICS[endpoint])
|
|
if endpoint == "public.feed" and response is not None:
|
|
mime = (response.mimetype or "").lower()
|
|
return topic_url(NEGOTIATED[mime]) if mime in NEGOTIATED else None
|
|
if endpoint == "public.tag":
|
|
from starpunk.tags import normalize_tag
|
|
tag = (request.view_args or {}).get("tag", "")
|
|
return tag_topic(normalize_tag(tag)[0]) if tag else None
|
|
return None
|
|
|
|
|
|
def canonical_topic(url: str) -> Optional[str]:
|
|
"""
|
|
A subscription's hub.topic as one of this site's topics, or None
|
|
|
|
Tag topics must name an existing tag.
|
|
"""
|
|
if not url:
|
|
return None
|
|
site = urlparse(site_url())
|
|
parsed = urlparse(url.strip())
|
|
if (parsed.scheme, parsed.netloc.lower()) != (site.scheme, site.netloc.lower()):
|
|
return None
|
|
path = parsed.path or "/"
|
|
if parsed.query or parsed.fragment:
|
|
return None
|
|
if path in set(PAGE_TOPICS.values()) | set(FEED_TOPICS.values()):
|
|
return topic_url(path)
|
|
if path.startswith("/tag/"):
|
|
from starpunk.tags import get_tag_by_name, normalize_tag
|
|
name = normalize_tag(path[len("/tag/"):])[0]
|
|
if name and get_tag_by_name(name):
|
|
return tag_topic(name)
|
|
return None
|
|
|
|
|
|
def link_header_values(topic: str) -> list[str]:
|
|
return [f'<{hub_url()}>; rel="hub"', f'<{topic}>; rel="self"']
|
|
|
|
|
|
def add_discovery_headers(response):
|
|
"""after_request: Link rel=hub and rel=self on topic responses (spec 4)"""
|
|
if not enabled() or response.status_code not in (200, 304):
|
|
return response
|
|
topic = topic_for_request(response)
|
|
if topic:
|
|
for value in link_header_values(topic):
|
|
response.headers.add("Link", value)
|
|
return response
|
|
|
|
|
|
def template_links() -> Optional[dict]:
|
|
"""For base.html: {'hub', 'self'} on topic pages"""
|
|
if not enabled() or not request:
|
|
return None
|
|
topic = topic_for_request()
|
|
return {"hub": hub_url(), "self": topic} if topic else None
|
|
|
|
|
|
def embed_hub(format_name: str, content: str) -> str:
|
|
"""Add the hub link inside a generated feed, next to its self link"""
|
|
if not enabled():
|
|
return content
|
|
hub = hub_url()
|
|
if format_name == "atom":
|
|
marker = '<link rel="self"'
|
|
return content.replace(marker, f'<link rel="hub" href="{_xml(hub)}"/>\n {marker}', 1)
|
|
if format_name == "rss":
|
|
marker = "<atom:link "
|
|
return content.replace(marker, f'<atom:link href="{_xml(hub)}" rel="hub"/>\n {marker}', 1)
|
|
if format_name == "json":
|
|
feed = json.loads(content)
|
|
feed["hubs"] = [{"type": "WebSub", "url": hub}]
|
|
return json.dumps(feed, ensure_ascii=False, indent=2)
|
|
return content
|
|
|
|
|
|
def _xml(value: str) -> str:
|
|
return (value.replace("&", "&").replace("<", "<")
|
|
.replace(">", ">").replace('"', """))
|
|
|
|
|
|
# --- Changes -------------------------------------------------------------------
|
|
|
|
def get_content_version(db) -> str:
|
|
row = db.execute("SELECT value FROM site_state WHERE key = 'content_version'").fetchone()
|
|
return row[0] if row else "0"
|
|
|
|
|
|
def bump_content_version(db) -> None:
|
|
db.execute(
|
|
"INSERT INTO site_state (key, value) VALUES ('content_version', '1') "
|
|
"ON CONFLICT(key) DO UPDATE SET value = CAST(value AS INTEGER) + 1"
|
|
)
|
|
|
|
|
|
def note_changed(note, was_public: bool = False, old_tags: Iterable[str] = ()) -> None:
|
|
"""
|
|
A note was created, edited, deleted, restored or (un)published
|
|
|
|
Clears cached feeds in every process and queues delivery of the topics
|
|
that show it. Changes that are never public (a draft edited) do nothing.
|
|
|
|
Args:
|
|
note: The note (after the change; for a delete, before it)
|
|
was_public: Whether it was public before the change
|
|
old_tags: Its tag names before the change
|
|
"""
|
|
from starpunk.database import get_db
|
|
|
|
is_public = bool(note.published) and not getattr(note, "deleted_at", None)
|
|
if not (is_public or was_public):
|
|
return
|
|
|
|
db = get_db(current_app)
|
|
bump_content_version(db)
|
|
db.commit()
|
|
|
|
if not enabled():
|
|
return
|
|
|
|
from starpunk.tags import get_note_tags
|
|
try:
|
|
tags = {t["name"] for t in get_note_tags(note.id)}
|
|
except Exception:
|
|
tags = set()
|
|
tags |= set(old_tags)
|
|
|
|
topics = [topic_url(p) for p in ("/", "/feed.atom", "/feed.rss", "/feed.json")]
|
|
if getattr(note, "like_of", None):
|
|
topics.append(topic_url("/likes"))
|
|
topics += [tag_topic(name) for name in sorted(tags)]
|
|
queue_topics(db, topics)
|
|
|
|
|
|
def queue_topics(db, topics: Iterable[str]) -> int:
|
|
"""Queue delivery of topics: to their subscribers, or a ping to an external hub"""
|
|
topics = list(dict.fromkeys(topics))
|
|
if not topics:
|
|
return 0
|
|
if external_hub():
|
|
for topic in topics:
|
|
db.execute("DELETE FROM websub_deliveries WHERE subscription_id IS NULL AND topic = ?", (topic,))
|
|
db.execute("INSERT INTO websub_deliveries (topic) VALUES (?)", (topic,))
|
|
db.commit()
|
|
return len(topics)
|
|
|
|
placeholders = ",".join("?" * len(topics))
|
|
rows = db.execute(
|
|
f"SELECT id, topic FROM websub_subscriptions WHERE topic IN ({placeholders}) "
|
|
"AND verified_at IS NOT NULL AND expires_at > datetime('now')",
|
|
topics,
|
|
).fetchall()
|
|
for sub_id, topic in rows:
|
|
# The newest content replaces anything still waiting (ADR-071 6)
|
|
db.execute("DELETE FROM websub_deliveries WHERE subscription_id = ?", (sub_id,))
|
|
db.execute("INSERT INTO websub_deliveries (subscription_id, topic) VALUES (?, ?)", (sub_id, topic))
|
|
db.commit()
|
|
return len(rows)
|
|
|
|
|
|
# --- Hub: subscription requests (spec 5.1) -------------------------------------
|
|
|
|
def accept_subscription_request(form, db) -> None:
|
|
"""
|
|
Validate a subscription request and queue it for verification
|
|
|
|
Raises:
|
|
SubscriptionError: with the status and plain-text reason to return
|
|
"""
|
|
from starpunk.webmention_ssrf import SSRFBlockedError, validate_url_for_fetch
|
|
|
|
if external_hub():
|
|
raise SubscriptionError("This site uses another hub: " + external_hub(), 400)
|
|
|
|
mode = (form.get("hub.mode") or "").strip()
|
|
if mode not in ("subscribe", "unsubscribe"):
|
|
raise SubscriptionError("hub.mode must be subscribe or unsubscribe")
|
|
|
|
callback = (form.get("hub.callback") or "").strip()
|
|
parsed = urlparse(callback)
|
|
if parsed.scheme not in ("http", "https") or not parsed.hostname:
|
|
raise SubscriptionError("hub.callback must be an absolute http(s) URL")
|
|
try:
|
|
validate_url_for_fetch(callback)
|
|
except (SSRFBlockedError, ValueError) as e:
|
|
raise SubscriptionError(f"hub.callback is not reachable: {e}")
|
|
|
|
topic = canonical_topic(form.get("hub.topic") or "")
|
|
if not topic:
|
|
raise SubscriptionError("hub.topic is not a topic of this site; use the rel=self URL")
|
|
|
|
secret = form.get("hub.secret")
|
|
if mode == "unsubscribe":
|
|
secret = None # MUST be ignored for unsubscriptions
|
|
if secret is not None and len(secret.encode("utf-8")) >= MAX_SECRET_BYTES:
|
|
raise SubscriptionError("hub.secret must be less than 200 bytes")
|
|
|
|
lease = LEASE_DEFAULT
|
|
if form.get("hub.lease_seconds"):
|
|
try:
|
|
lease = int(form["hub.lease_seconds"])
|
|
except ValueError:
|
|
raise SubscriptionError("hub.lease_seconds must be a number")
|
|
lease = max(LEASE_MIN, min(lease, LEASE_MAX))
|
|
|
|
if mode == "subscribe":
|
|
existing = db.execute(
|
|
"SELECT 1 FROM websub_subscriptions WHERE topic = ? AND callback = ?", (topic, callback)
|
|
).fetchone()
|
|
if not existing:
|
|
total = db.execute("SELECT COUNT(*) FROM websub_subscriptions").fetchone()[0]
|
|
if total >= current_app.config.get("WEBSUB_MAX_SUBSCRIPTIONS", 1000):
|
|
raise SubscriptionError("This hub has reached its subscription limit", 429)
|
|
host = parsed.hostname.lower()
|
|
per_host = sum(
|
|
1 for (cb,) in db.execute("SELECT callback FROM websub_subscriptions")
|
|
if (urlparse(cb).hostname or "").lower() == host
|
|
)
|
|
if per_host >= current_app.config.get("WEBSUB_MAX_PER_HOST", 25):
|
|
raise SubscriptionError(f"Too many subscriptions for {host}", 429)
|
|
|
|
# The latest request for a topic and callback replaces any pending one
|
|
db.execute(
|
|
"""
|
|
INSERT INTO websub_intents (topic, callback, mode, secret, lease_seconds)
|
|
VALUES (?, ?, ?, ?, ?)
|
|
ON CONFLICT(topic, callback) DO UPDATE SET
|
|
mode = excluded.mode, secret = excluded.secret,
|
|
lease_seconds = excluded.lease_seconds, attempts = 0,
|
|
next_attempt_at = CURRENT_TIMESTAMP, created_at = CURRENT_TIMESTAMP
|
|
""",
|
|
(topic, callback, mode, secret, lease),
|
|
)
|
|
db.commit()
|
|
|
|
|
|
def stats(db) -> dict:
|
|
"""Active subscribers per topic and deliveries being retried (admin)"""
|
|
by_topic = db.execute(
|
|
"SELECT topic, COUNT(*) FROM websub_subscriptions WHERE expires_at > datetime('now') "
|
|
"GROUP BY topic ORDER BY COUNT(*) DESC, topic"
|
|
).fetchall()
|
|
retrying = db.execute(
|
|
"SELECT COUNT(*) FROM websub_deliveries WHERE attempts > 0"
|
|
).fetchone()[0]
|
|
return {"by_topic": [(t, n) for t, n in by_topic], "retrying": retrying,
|
|
"external": external_hub()}
|
|
|
|
|
|
def new_challenge() -> str:
|
|
return secrets.token_urlsafe(32)
|