- Per-item error handling in the worker; hard time limits on requests - Notes with photos announced after attach; feed cache/ETag follow content_version - Keep failing subscriptions until lease end; retry DNS and render errors; hub.mode=denied for gone topics - Spec 5.1: decode unreserved chars, echo requested topic, ignore lease and secret on unsubscribe, case-insensitive Content-Type, refuse empty secret - Limits count pending requests per callback address; callback validation; request_id guards the verification race (migration 017) - Render once per topic per pass; likes only push /likes; Vary: Accept Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
444 lines
16 KiB
Python
444 lines
16 KiB
Python
"""
|
|
WebSub publishing and the built-in hub (v2.10.0, ADR-071)
|
|
|
|
Publisher: topic responses advertise the hub and their own URL; a change to a
|
|
public note clears cached feeds and queues delivery of every affected topic.
|
|
|
|
Hub: subscriptions to this site's own topics only. Requests are accepted
|
|
here (202) and verified, delivered and retried by the WebSub worker.
|
|
|
|
References:
|
|
- W3C WebSub: https://www.w3.org/TR/websub/
|
|
- ADR-071: WebSub publishing with a built-in hub
|
|
"""
|
|
|
|
import json
|
|
import secrets
|
|
from typing import Iterable, Optional
|
|
from urllib.parse import urlparse
|
|
|
|
from flask import current_app, request
|
|
|
|
|
|
# Topic endpoints and their fixed paths (tag pages are added per tag)
|
|
FEED_TOPICS = {
|
|
"public.feed_atom": "/feed.atom",
|
|
"public.feed_rss": "/feed.rss",
|
|
"public.feed_xml_legacy": "/feed.rss", # legacy alias of the RSS topic
|
|
"public.feed_json": "/feed.json",
|
|
}
|
|
PAGE_TOPICS = {"public.index": "/", "public.likes": "/likes"}
|
|
NEGOTIATED = { # /feed: the topic of whichever format was served
|
|
"application/atom+xml": "/feed.atom",
|
|
"application/rss+xml": "/feed.rss",
|
|
"application/feed+json": "/feed.json",
|
|
"application/json": "/feed.json",
|
|
}
|
|
|
|
LEASE_DEFAULT = 10 * 24 * 3600
|
|
LEASE_MIN = 24 * 3600
|
|
LEASE_MAX = 30 * 24 * 3600
|
|
MAX_SECRET_BYTES = 200
|
|
|
|
|
|
class SubscriptionError(Exception):
|
|
"""A subscription request the hub refuses (4xx with a plain-text reason)"""
|
|
|
|
def __init__(self, message: str, status: int = 400):
|
|
super().__init__(message)
|
|
self.status = status
|
|
|
|
|
|
# --- Configuration -----------------------------------------------------------
|
|
|
|
def enabled() -> bool:
|
|
return bool(current_app.config.get("WEBSUB_ENABLED", True))
|
|
|
|
|
|
def external_hub() -> Optional[str]:
|
|
"""URL of an external hub, if one is configured instead of the built-in one"""
|
|
return current_app.config.get("WEBSUB_HUB_URL") or None
|
|
|
|
|
|
def hub_url() -> str:
|
|
# Built from SITE_URL so the worker can use it outside a request
|
|
return external_hub() or site_url() + "/websub"
|
|
|
|
|
|
def site_url() -> str:
|
|
return current_app.config["SITE_URL"].rstrip("/")
|
|
|
|
|
|
# --- Topics ------------------------------------------------------------------
|
|
|
|
def topic_url(path: str) -> str:
|
|
return site_url() + path
|
|
|
|
|
|
def tag_topic(tag_name: str) -> str:
|
|
from urllib.parse import quote
|
|
return f"{site_url()}/tag/{quote(tag_name, safe='')}"
|
|
|
|
|
|
def topic_for_request(response=None) -> Optional[str]:
|
|
"""The topic URL the current request serves, or None if it isn't a topic"""
|
|
endpoint = request.endpoint
|
|
if endpoint in PAGE_TOPICS:
|
|
return topic_url(PAGE_TOPICS[endpoint])
|
|
if endpoint in FEED_TOPICS:
|
|
return topic_url(FEED_TOPICS[endpoint])
|
|
if endpoint == "public.feed" and response is not None:
|
|
mime = (response.mimetype or "").lower()
|
|
return topic_url(NEGOTIATED[mime]) if mime in NEGOTIATED else None
|
|
if endpoint == "public.tag":
|
|
from starpunk.tags import normalize_tag
|
|
tag = (request.view_args or {}).get("tag", "")
|
|
return tag_topic(normalize_tag(tag)[0]) if tag else None
|
|
return None
|
|
|
|
|
|
_UNRESERVED = set("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~")
|
|
|
|
|
|
def decode_unreserved(value: str) -> str:
|
|
"""
|
|
Decode percent-encoded unreserved characters (%66 -> f), leaving the rest
|
|
|
|
Spec 5.1.1: hubs MUST always decode non-reserved characters in URL
|
|
parameters, so /%66eed.atom and /feed.atom are the same topic.
|
|
"""
|
|
out, i = [], 0
|
|
while i < len(value):
|
|
if value[i] == "%" and i + 3 <= len(value):
|
|
try:
|
|
char = chr(int(value[i + 1:i + 3], 16))
|
|
except ValueError:
|
|
char = None
|
|
if char is not None and char in _UNRESERVED:
|
|
out.append(char)
|
|
i += 3
|
|
continue
|
|
out.append(value[i])
|
|
i += 1
|
|
return "".join(out)
|
|
|
|
|
|
def canonical_topic(url: str) -> Optional[str]:
|
|
"""
|
|
A subscription's hub.topic as one of this site's topics, or None
|
|
|
|
Unreserved percent-encodings are decoded and the scheme and host are
|
|
compared case-insensitively. Tag topics must name an existing tag.
|
|
"""
|
|
if not url:
|
|
return None
|
|
site = urlparse(site_url())
|
|
parsed = urlparse(decode_unreserved(url.strip()))
|
|
if (parsed.scheme.lower(), parsed.netloc.lower()) != (site.scheme, site.netloc.lower()):
|
|
return None
|
|
path = parsed.path or "/"
|
|
if parsed.query or parsed.fragment:
|
|
return None
|
|
if path in set(PAGE_TOPICS.values()) | set(FEED_TOPICS.values()):
|
|
return topic_url(path)
|
|
if path.startswith("/tag/"):
|
|
from urllib.parse import unquote
|
|
from starpunk.tags import get_tag_by_name, normalize_tag
|
|
name = normalize_tag(unquote(path[len("/tag/"):]))[0]
|
|
if name and get_tag_by_name(name):
|
|
return tag_topic(name)
|
|
return None
|
|
|
|
|
|
def link_header_values(topic: str) -> list[str]:
|
|
return [f'<{hub_url()}>; rel="hub"', f'<{topic}>; rel="self"']
|
|
|
|
|
|
def add_discovery_headers(response):
|
|
"""after_request: Link rel=hub and rel=self on topic responses (spec 4)"""
|
|
if not enabled() or response.status_code not in (200, 304):
|
|
return response
|
|
topic = topic_for_request(response)
|
|
if topic:
|
|
for value in link_header_values(topic):
|
|
response.headers.add("Link", value)
|
|
return response
|
|
|
|
|
|
def template_links() -> Optional[dict]:
|
|
"""For base.html: {'hub', 'self'} on topic pages"""
|
|
if not enabled() or not request:
|
|
return None
|
|
topic = topic_for_request()
|
|
return {"hub": hub_url(), "self": topic} if topic else None
|
|
|
|
|
|
def embed_hub(format_name: str, content: str) -> str:
|
|
"""Add the hub link inside a generated feed, next to its self link"""
|
|
if not enabled():
|
|
return content
|
|
hub = hub_url()
|
|
if format_name == "atom":
|
|
marker = '<link rel="self"'
|
|
return content.replace(marker, f'<link rel="hub" href="{_xml(hub)}"/>\n {marker}', 1)
|
|
if format_name == "rss":
|
|
marker = "<atom:link "
|
|
return content.replace(marker, f'<atom:link href="{_xml(hub)}" rel="hub"/>\n {marker}', 1)
|
|
if format_name == "json":
|
|
feed = json.loads(content)
|
|
feed["hubs"] = [{"type": "WebSub", "url": hub}]
|
|
return json.dumps(feed, ensure_ascii=False, indent=2)
|
|
return content
|
|
|
|
|
|
def _xml(value: str) -> str:
|
|
return (value.replace("&", "&").replace("<", "<")
|
|
.replace(">", ">").replace('"', """))
|
|
|
|
|
|
# --- Changes -------------------------------------------------------------------
|
|
|
|
def get_content_version(db) -> str:
|
|
row = db.execute("SELECT value FROM site_state WHERE key = 'content_version'").fetchone()
|
|
return row[0] if row else "0"
|
|
|
|
|
|
def bump_content_version(db) -> None:
|
|
db.execute(
|
|
"INSERT INTO site_state (key, value) VALUES ('content_version', '1') "
|
|
"ON CONFLICT(key) DO UPDATE SET value = CAST(value AS INTEGER) + 1"
|
|
)
|
|
|
|
|
|
def note_changed(note, was_public: bool = False, old_tags: Iterable[str] = ()) -> None:
|
|
"""
|
|
A note was created, edited, deleted, restored or (un)published
|
|
|
|
Clears cached feeds in every process and queues delivery of the topics
|
|
that show it. Changes that are never public (a draft edited) do nothing.
|
|
|
|
Args:
|
|
note: The note (after the change; for a delete, before it)
|
|
was_public: Whether it was public before the change
|
|
old_tags: Its tag names before the change
|
|
"""
|
|
from starpunk.database import get_db
|
|
|
|
is_public = bool(note.published) and not getattr(note, "deleted_at", None)
|
|
if not (is_public or was_public):
|
|
return
|
|
|
|
db = get_db(current_app)
|
|
bump_content_version(db)
|
|
db.commit()
|
|
|
|
if not enabled():
|
|
return
|
|
|
|
from starpunk.tags import get_note_tags
|
|
try:
|
|
tags = {t["name"] for t in get_note_tags(note.id)}
|
|
except Exception:
|
|
tags = set()
|
|
tags |= set(old_tags)
|
|
|
|
# Likes appear on /likes, not on the home page or in the feeds
|
|
if getattr(note, "like_of", None):
|
|
topics = [topic_url("/likes")]
|
|
else:
|
|
topics = [topic_url(p) for p in ("/", "/feed.atom", "/feed.rss", "/feed.json")]
|
|
topics += [tag_topic(name) for name in sorted(tags)]
|
|
queue_topics(db, topics)
|
|
|
|
|
|
def queue_topics(db, topics: Iterable[str]) -> int:
|
|
"""Queue delivery of topics: to their subscribers, or a ping to an external hub"""
|
|
topics = list(dict.fromkeys(topics))
|
|
if not topics:
|
|
return 0
|
|
if external_hub():
|
|
for topic in topics:
|
|
db.execute("DELETE FROM websub_deliveries WHERE subscription_id IS NULL AND topic = ?", (topic,))
|
|
db.execute("INSERT INTO websub_deliveries (topic) VALUES (?)", (topic,))
|
|
db.commit()
|
|
return len(topics)
|
|
|
|
placeholders = ",".join("?" * len(topics))
|
|
rows = db.execute(
|
|
f"SELECT id, topic FROM websub_subscriptions WHERE topic IN ({placeholders}) "
|
|
"AND verified_at IS NOT NULL AND expires_at > datetime('now')",
|
|
topics,
|
|
).fetchall()
|
|
for sub_id, topic in rows:
|
|
# The newest content replaces anything still waiting (ADR-071 6)
|
|
db.execute("DELETE FROM websub_deliveries WHERE subscription_id = ?", (sub_id,))
|
|
db.execute("INSERT INTO websub_deliveries (subscription_id, topic) VALUES (?, ?)", (sub_id, topic))
|
|
db.commit()
|
|
return len(rows)
|
|
|
|
|
|
# --- Hub: subscription requests (spec 5.1) -------------------------------------
|
|
|
|
MAX_CALLBACK_LENGTH = 2048
|
|
MAX_PENDING_INTENTS = 200
|
|
ALLOWED_PORTS = {80, 443} # plus any port from 1024 up
|
|
|
|
|
|
def _clean_callback(callback: str) -> str:
|
|
"""
|
|
Check a hub.callback and return it with an ASCII (IDNA) host
|
|
|
|
Raises:
|
|
SubscriptionError: if it is not an acceptable callback URL
|
|
"""
|
|
import httpx
|
|
|
|
if not callback or len(callback) > MAX_CALLBACK_LENGTH:
|
|
raise SubscriptionError(f"hub.callback must be a URL of at most {MAX_CALLBACK_LENGTH} characters")
|
|
if any(ord(c) < 0x20 or ord(c) == 0x7f for c in callback) or " " in callback:
|
|
raise SubscriptionError("hub.callback contains spaces or control characters")
|
|
parsed = urlparse(callback)
|
|
if parsed.scheme.lower() not in ("http", "https") or not parsed.hostname:
|
|
raise SubscriptionError("hub.callback must be an absolute http(s) URL")
|
|
try:
|
|
port = parsed.port
|
|
except ValueError:
|
|
raise SubscriptionError("hub.callback has an invalid port")
|
|
if port is not None and port not in ALLOWED_PORTS and port < 1024:
|
|
raise SubscriptionError("hub.callback may not use that port")
|
|
|
|
try:
|
|
host = parsed.hostname.encode("idna").decode("ascii")
|
|
except UnicodeError:
|
|
raise SubscriptionError("hub.callback has an invalid host name")
|
|
netloc = host + (f":{port}" if port is not None else "")
|
|
callback = parsed._replace(scheme=parsed.scheme.lower(), netloc=netloc).geturl()
|
|
|
|
if host.lower() == (urlparse(site_url()).hostname or "").lower():
|
|
raise SubscriptionError("hub.callback may not be on this site")
|
|
try:
|
|
httpx.URL(callback)
|
|
except Exception:
|
|
raise SubscriptionError("hub.callback is not a valid URL")
|
|
return callback
|
|
|
|
|
|
def accept_subscription_request(form, db) -> None:
|
|
"""
|
|
Validate a subscription request and queue it for verification
|
|
|
|
Raises:
|
|
SubscriptionError: with the status and plain-text reason to return
|
|
"""
|
|
from starpunk.webmention_ssrf import SSRFBlockedError, validate_url_for_fetch
|
|
|
|
mode = (form.get("hub.mode") or "").strip()
|
|
if mode not in ("subscribe", "unsubscribe"):
|
|
raise SubscriptionError("hub.mode must be subscribe or unsubscribe")
|
|
if mode == "subscribe" and external_hub():
|
|
# Unsubscribes are still taken, so old subscribers can leave
|
|
raise SubscriptionError("This site uses another hub: " + external_hub(), 400)
|
|
|
|
callback = _clean_callback((form.get("hub.callback") or "").strip())
|
|
try:
|
|
callback_ip = validate_url_for_fetch(callback)
|
|
except (SSRFBlockedError, ValueError) as e:
|
|
raise SubscriptionError(f"hub.callback is not reachable: {e}")
|
|
|
|
requested_topic = (form.get("hub.topic") or "").strip()
|
|
topic = canonical_topic(requested_topic)
|
|
if not topic:
|
|
raise SubscriptionError("hub.topic is not a topic of this site; use the rel=self URL")
|
|
|
|
secret, lease = None, None
|
|
if mode == "subscribe":
|
|
# hub.secret and hub.lease_seconds MUST be ignored for unsubscriptions
|
|
secret = form.get("hub.secret")
|
|
if secret is not None:
|
|
if secret == "":
|
|
raise SubscriptionError("hub.secret may not be empty")
|
|
if len(secret.encode("utf-8")) >= MAX_SECRET_BYTES:
|
|
raise SubscriptionError("hub.secret must be less than 200 bytes")
|
|
lease = LEASE_DEFAULT
|
|
if form.get("hub.lease_seconds"):
|
|
try:
|
|
lease = int(form["hub.lease_seconds"])
|
|
except ValueError:
|
|
raise SubscriptionError("hub.lease_seconds must be a number")
|
|
lease = max(LEASE_MIN, min(lease, LEASE_MAX))
|
|
_check_limits(db, topic, callback, callback_ip)
|
|
|
|
# Each request gets its own id; the latest replaces any pending one
|
|
db.execute(
|
|
"""
|
|
INSERT INTO websub_intents
|
|
(topic, callback, mode, secret, lease_seconds, request_id, requested_topic, callback_ip)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
ON CONFLICT(topic, callback) DO UPDATE SET
|
|
mode = excluded.mode, secret = excluded.secret,
|
|
lease_seconds = excluded.lease_seconds, request_id = excluded.request_id,
|
|
requested_topic = excluded.requested_topic, callback_ip = excluded.callback_ip,
|
|
attempts = 0, next_attempt_at = CURRENT_TIMESTAMP, created_at = CURRENT_TIMESTAMP
|
|
""",
|
|
(topic, callback, mode, secret, lease, secrets.token_hex(16), requested_topic, callback_ip),
|
|
)
|
|
db.commit()
|
|
|
|
|
|
def _check_limits(db, topic: str, callback: str, callback_ip: str) -> None:
|
|
"""
|
|
Subscription limits, counting pending requests too (ADR-071 4)
|
|
|
|
Per callback address rather than host name, so many subdomains on one
|
|
server share a limit. Renewing an existing subscription is always fine.
|
|
"""
|
|
if db.execute("SELECT 1 FROM websub_subscriptions WHERE topic = ? AND callback = ?",
|
|
(topic, callback)).fetchone():
|
|
return
|
|
|
|
pending = db.execute("SELECT COUNT(*) FROM websub_intents").fetchone()[0]
|
|
if pending >= MAX_PENDING_INTENTS:
|
|
raise SubscriptionError("Too many subscription requests waiting; try again later", 429)
|
|
|
|
total = db.execute(
|
|
"""
|
|
SELECT COUNT(*) FROM (
|
|
SELECT topic, callback FROM websub_subscriptions
|
|
UNION
|
|
SELECT topic, callback FROM websub_intents WHERE mode = 'subscribe'
|
|
)
|
|
"""
|
|
).fetchone()[0]
|
|
if total >= current_app.config.get("WEBSUB_MAX_SUBSCRIPTIONS", 1000):
|
|
raise SubscriptionError("This hub has reached its subscription limit", 429)
|
|
|
|
per_address = db.execute(
|
|
"""
|
|
SELECT COUNT(*) FROM (
|
|
SELECT topic, callback FROM websub_subscriptions WHERE callback_ip = ?
|
|
UNION
|
|
SELECT topic, callback FROM websub_intents WHERE mode = 'subscribe' AND callback_ip = ?
|
|
)
|
|
""",
|
|
(callback_ip, callback_ip),
|
|
).fetchone()[0]
|
|
if per_address >= current_app.config.get("WEBSUB_MAX_PER_HOST", 25):
|
|
raise SubscriptionError("Too many subscriptions for that callback's server", 429)
|
|
|
|
|
|
def stats(db) -> dict:
|
|
"""Active subscribers per topic and deliveries being retried (admin)"""
|
|
by_topic = db.execute(
|
|
"SELECT topic, COUNT(*) FROM websub_subscriptions WHERE expires_at > datetime('now') "
|
|
"GROUP BY topic ORDER BY COUNT(*) DESC, topic"
|
|
).fetchall()
|
|
retrying = db.execute(
|
|
"SELECT COUNT(*) FROM websub_deliveries WHERE attempts > 0"
|
|
).fetchone()[0]
|
|
return {"by_topic": [(t, n) for t, n in by_topic], "retrying": retrying,
|
|
"external": external_hub()}
|
|
|
|
|
|
def new_challenge() -> str:
|
|
return secrets.token_urlsafe(32)
|