- Per-item error handling in the worker; hard time limits on requests - Notes with photos announced after attach; feed cache/ETag follow content_version - Keep failing subscriptions until lease end; retry DNS and render errors; hub.mode=denied for gone topics - Spec 5.1: decode unreserved chars, echo requested topic, ignore lease and secret on unsubscribe, case-insensitive Content-Type, refuse empty secret - Limits count pending requests per callback address; callback validation; request_id guards the verification race (migration 017) - Render once per topic per pass; likes only push /likes; Vary: Accept Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
319 lines
12 KiB
Python
319 lines
12 KiB
Python
"""
|
|
WebSub publisher and built-in hub (v2.10.0, ADR-071)
|
|
|
|
Section numbers refer to the W3C WebSub Recommendation. Subscribers are
|
|
simulated with an httpx MockTransport behind the SSRF-safe fetcher.
|
|
"""
|
|
|
|
import hashlib
|
|
import hmac
|
|
import json
|
|
import sqlite3
|
|
from urllib.parse import parse_qs, urlparse
|
|
|
|
import httpx
|
|
import mf2py
|
|
import pytest
|
|
|
|
from starpunk.websub_worker import WebSubWorker
|
|
|
|
SITE = "http://localhost:5000"
|
|
HUB = f"{SITE}/websub"
|
|
PUBLIC = [(2, 1, 6, "", ("93.184.216.34", 0))]
|
|
|
|
|
|
@pytest.fixture
|
|
def web(monkeypatch):
|
|
"""Subscriber callbacks: handler(request) -> response, by host"""
|
|
hosts, seen = {}, []
|
|
# Every test client shares one IP; don't let earlier tests rate-limit this one
|
|
from starpunk.rate_limiter import RateLimiter
|
|
monkeypatch.setattr("starpunk.routes.websub._limiter", RateLimiter(limit=1000, window_seconds=60))
|
|
|
|
def handler(request):
|
|
host = request.headers["host"]
|
|
seen.append(request)
|
|
return hosts[host](request)
|
|
|
|
monkeypatch.setattr("starpunk.webmention_ssrf.socket.getaddrinfo", lambda *a, **k: PUBLIC)
|
|
monkeypatch.setattr("starpunk.webmention_ssrf.TRANSPORT", httpx.MockTransport(handler))
|
|
return hosts, seen
|
|
|
|
|
|
def echo(request):
|
|
"""A subscriber that confirms every verification and accepts content"""
|
|
if request.method == "GET":
|
|
return httpx.Response(200, text=parse_qs(urlparse(str(request.url)).query)["hub.challenge"][0])
|
|
return httpx.Response(202)
|
|
|
|
|
|
def db(app):
|
|
conn = sqlite3.connect(app.config["DATABASE_PATH"])
|
|
conn.row_factory = sqlite3.Row
|
|
return conn
|
|
|
|
|
|
def run_worker(app, times=1):
|
|
worker = WebSubWorker(app)
|
|
with app.app_context():
|
|
for _ in range(times):
|
|
worker.process_once()
|
|
return worker
|
|
|
|
|
|
def subscribe(client, topic, callback="https://sub.example/cb?id=1", **extra):
|
|
form = {"hub.mode": "subscribe", "hub.topic": topic, "hub.callback": callback, **extra}
|
|
return client.post("/websub", data=form)
|
|
|
|
|
|
def make_note(app, content="Hello", **kw):
|
|
from starpunk.notes import create_note
|
|
with app.test_request_context():
|
|
return create_note(content, published=kw.pop("published", True), **kw)
|
|
|
|
|
|
def links(response):
|
|
return response.headers.getlist("Link")
|
|
|
|
|
|
# 4 Discovery -------------------------------------------------------------------
|
|
|
|
@pytest.mark.parametrize("path,self_url", [
|
|
("/", f"{SITE}/"),
|
|
("/feed.atom", f"{SITE}/feed.atom"),
|
|
("/feed.rss", f"{SITE}/feed.rss"),
|
|
("/feed.xml", f"{SITE}/feed.rss"),
|
|
("/feed.json", f"{SITE}/feed.json"),
|
|
("/likes", f"{SITE}/likes"),
|
|
])
|
|
def test_topics_advertise_hub_and_self(client, path, self_url):
|
|
response = client.get(path)
|
|
values = links(response)
|
|
assert f'<{HUB}>; rel="hub"' in values
|
|
assert [v for v in values if 'rel="self"' in v] == [f'<{self_url}>; rel="self"']
|
|
|
|
|
|
def test_negotiated_feed_self_is_the_format_served(client):
|
|
response = client.get("/feed", headers={"Accept": "application/atom+xml"})
|
|
assert f'<{SITE}/feed.atom>; rel="self"' in links(response)
|
|
|
|
|
|
def test_tag_page_is_a_topic(app, client):
|
|
make_note(app, tags=["Python"])
|
|
assert f'<{SITE}/tag/python>; rel="self"' in links(client.get("/tag/Python"))
|
|
|
|
|
|
def test_note_page_is_not_a_topic(app, client):
|
|
note = make_note(app)
|
|
assert not [v for v in links(client.get(f"/note/{note.slug}")) if 'rel="hub"' in v]
|
|
|
|
|
|
def test_embedded_links(client):
|
|
rels = mf2py.parse(doc=client.get("/").data.decode(), url=SITE + "/")["rels"]
|
|
assert rels["hub"] == [HUB] and rels["self"] == [f"{SITE}/"]
|
|
assert f'<link rel="hub" href="{HUB}"/>' in client.get("/feed.atom").data.decode()
|
|
assert f'<atom:link href="{HUB}" rel="hub"/>' in client.get("/feed.rss").data.decode()
|
|
assert json.loads(client.get("/feed.json").data)["hubs"] == [{"type": "WebSub", "url": HUB}]
|
|
|
|
|
|
def test_external_hub(app, client, web):
|
|
app.config["WEBSUB_HUB_URL"] = "https://hub.example/"
|
|
hosts, seen = web
|
|
hosts["hub.example"] = lambda r: httpx.Response(204)
|
|
assert '<https://hub.example/>; rel="hub"' in links(client.get("/"))
|
|
assert subscribe(client, f"{SITE}/").status_code == 400 # our hub is off
|
|
|
|
make_note(app)
|
|
run_worker(app)
|
|
pings = {parse_qs(r.content.decode())["hub.url"][0] for r in seen}
|
|
assert pings == {f"{SITE}/", f"{SITE}/feed.atom", f"{SITE}/feed.rss", f"{SITE}/feed.json"}
|
|
|
|
|
|
# 5.1 Subscription requests ----------------------------------------------------------
|
|
|
|
def test_subscription_request_is_accepted(app, client, web):
|
|
response = subscribe(client, f"{SITE}/feed.atom", **{"hub.secret": "s3cret", "extra": "ignored"})
|
|
assert response.status_code == 202
|
|
row = db(app).execute("SELECT * FROM websub_intents").fetchone()
|
|
assert (row["mode"], row["topic"], row["secret"]) == ("subscribe", f"{SITE}/feed.atom", "s3cret")
|
|
|
|
|
|
@pytest.mark.parametrize("form,reason", [
|
|
({"hub.mode": "watch"}, "hub.mode"),
|
|
({"hub.topic": "https://elsewhere.example/feed"}, "hub.topic"),
|
|
({"hub.topic": f"{SITE}/note/x"}, "hub.topic"),
|
|
({"hub.topic": f"{SITE}/tag/missing"}, "hub.topic"),
|
|
({"hub.callback": "ftp://sub.example/cb"}, "hub.callback"),
|
|
({"hub.secret": "x" * 200}, "hub.secret"),
|
|
])
|
|
def test_bad_requests(client, web, form, reason):
|
|
base = {"hub.mode": "subscribe", "hub.topic": f"{SITE}/", "hub.callback": "https://sub.example/cb"}
|
|
response = client.post("/websub", data={**base, **form})
|
|
assert response.status_code == 400 and reason in response.data.decode()
|
|
|
|
|
|
def test_private_callback_refused(client, monkeypatch):
|
|
monkeypatch.setattr("starpunk.webmention_ssrf.socket.getaddrinfo",
|
|
lambda *a, **k: [(2, 1, 6, "", ("10.0.0.2", 0))])
|
|
assert subscribe(client, f"{SITE}/").status_code == 400
|
|
|
|
|
|
def test_per_host_limit(app, client, web):
|
|
app.config["WEBSUB_MAX_PER_HOST"] = 1
|
|
hosts, _ = web
|
|
hosts["sub.example"] = echo
|
|
subscribe(client, f"{SITE}/")
|
|
run_worker(app)
|
|
assert subscribe(client, f"{SITE}/feed.atom").status_code == 429
|
|
|
|
|
|
# 5.3 Verification of intent --------------------------------------------------------
|
|
|
|
def test_verification_activates_subscription(app, client, web):
|
|
hosts, seen = web
|
|
hosts["sub.example"] = echo
|
|
subscribe(client, f"{SITE}/", **{"hub.lease_seconds": "86400"})
|
|
run_worker(app)
|
|
|
|
query = parse_qs(urlparse(str(seen[0].url)).query)
|
|
assert query["id"] == ["1"] # the callback's own query string is kept
|
|
assert query["hub.mode"] == ["subscribe"] and query["hub.topic"] == [f"{SITE}/"]
|
|
assert query["hub.lease_seconds"] == ["86400"]
|
|
sub = db(app).execute("SELECT * FROM websub_subscriptions").fetchone()
|
|
assert sub["topic"] == f"{SITE}/" and sub["verified_at"]
|
|
assert db(app).execute("SELECT COUNT(*) FROM websub_intents").fetchone()[0] == 0
|
|
|
|
|
|
@pytest.mark.parametrize("response", [
|
|
httpx.Response(200, text="wrong"),
|
|
httpx.Response(404),
|
|
httpx.Response(302, headers={"Location": "https://sub.example/other"}),
|
|
])
|
|
def test_failed_verification_changes_nothing(app, client, web, response):
|
|
hosts, _ = web
|
|
hosts["sub.example"] = lambda r: response
|
|
subscribe(client, f"{SITE}/")
|
|
run_worker(app)
|
|
assert db(app).execute("SELECT COUNT(*) FROM websub_subscriptions").fetchone()[0] == 0
|
|
assert db(app).execute("SELECT COUNT(*) FROM websub_intents").fetchone()[0] == 0
|
|
|
|
|
|
def test_unsubscribe(app, client, web):
|
|
hosts, _ = web
|
|
hosts["sub.example"] = echo
|
|
subscribe(client, f"{SITE}/")
|
|
run_worker(app)
|
|
client.post("/websub", data={"hub.mode": "unsubscribe", "hub.topic": f"{SITE}/",
|
|
"hub.callback": "https://sub.example/cb?id=1"})
|
|
run_worker(app)
|
|
assert db(app).execute("SELECT COUNT(*) FROM websub_subscriptions").fetchone()[0] == 0
|
|
|
|
|
|
def test_resubscribe_changes_secret_only_once_verified(app, client, web):
|
|
hosts, _ = web
|
|
hosts["sub.example"] = echo
|
|
subscribe(client, f"{SITE}/", **{"hub.secret": "one"})
|
|
run_worker(app)
|
|
hosts["sub.example"] = lambda r: httpx.Response(404)
|
|
subscribe(client, f"{SITE}/", **{"hub.secret": "two"})
|
|
run_worker(app)
|
|
assert db(app).execute("SELECT secret FROM websub_subscriptions").fetchone()[0] == "one"
|
|
|
|
|
|
# 6, 7 Content distribution ---------------------------------------------------------
|
|
|
|
def _subscribed(app, client, web, topic=f"{SITE}/feed.atom", secret=None):
|
|
hosts, seen = web
|
|
hosts["sub.example"] = echo
|
|
extra = {"hub.secret": secret} if secret else {}
|
|
subscribe(client, topic, **extra)
|
|
run_worker(app)
|
|
seen.clear()
|
|
return hosts, seen
|
|
|
|
|
|
def test_new_note_is_delivered_signed(app, client, web):
|
|
hosts, seen = _subscribed(app, client, web, secret="s3cret")
|
|
make_note(app, "Fresh post")
|
|
run_worker(app)
|
|
|
|
(post,) = [r for r in seen if r.method == "POST"]
|
|
assert post.headers["content-type"].startswith("application/atom+xml")
|
|
assert b"Fresh post" in post.content
|
|
assert f'<{HUB}>; rel="hub"' in post.headers["link"] and f'<{SITE}/feed.atom>; rel="self"' in post.headers["link"]
|
|
expected = "sha256=" + hmac.new(b"s3cret", post.content, hashlib.sha256).hexdigest()
|
|
assert post.headers["x-hub-signature"] == expected
|
|
assert db(app).execute("SELECT COUNT(*) FROM websub_deliveries").fetchone()[0] == 0
|
|
|
|
|
|
def test_no_signature_without_secret(app, client, web):
|
|
hosts, seen = _subscribed(app, client, web)
|
|
make_note(app)
|
|
run_worker(app)
|
|
assert "x-hub-signature" not in [r for r in seen if r.method == "POST"][0].headers
|
|
|
|
|
|
def test_drafts_are_not_delivered(app, client, web):
|
|
_subscribed(app, client, web)
|
|
make_note(app, published=False)
|
|
assert db(app).execute("SELECT COUNT(*) FROM websub_deliveries").fetchone()[0] == 0
|
|
|
|
|
|
def test_tag_topics_include_removed_tags(app, client, web):
|
|
from starpunk.notes import update_note
|
|
note = make_note(app, tags=["old"])
|
|
_subscribed(app, client, web, topic=f"{SITE}/tag/old")
|
|
with app.test_request_context():
|
|
update_note(id=note.id, content="Hello", tags=["new"])
|
|
assert db(app).execute("SELECT topic FROM websub_deliveries").fetchone()[0] == f"{SITE}/tag/old"
|
|
|
|
|
|
def test_delete_is_delivered(app, client, web):
|
|
from starpunk.notes import delete_note
|
|
note = make_note(app, "Going away")
|
|
hosts, seen = _subscribed(app, client, web)
|
|
with app.test_request_context():
|
|
delete_note(id=note.id, soft=False)
|
|
run_worker(app)
|
|
(post,) = [r for r in seen if r.method == "POST"]
|
|
assert b"Going away" not in post.content
|
|
|
|
|
|
def test_410_removes_subscription(app, client, web):
|
|
hosts, _ = _subscribed(app, client, web)
|
|
hosts["sub.example"] = lambda r: httpx.Response(410)
|
|
make_note(app)
|
|
run_worker(app)
|
|
assert db(app).execute("SELECT COUNT(*) FROM websub_subscriptions").fetchone()[0] == 0
|
|
|
|
|
|
def test_failures_are_retried(app, client, web):
|
|
hosts, _ = _subscribed(app, client, web)
|
|
hosts["sub.example"] = lambda r: httpx.Response(500)
|
|
make_note(app)
|
|
run_worker(app)
|
|
row = db(app).execute("SELECT attempts, next_attempt_at, last_error FROM websub_deliveries").fetchone()
|
|
assert row["attempts"] == 1 and row["last_error"] == "HTTP 500"
|
|
|
|
|
|
def test_expired_subscriptions_get_nothing(app, client, web):
|
|
_subscribed(app, client, web)
|
|
conn = db(app)
|
|
conn.execute("UPDATE websub_subscriptions SET expires_at = datetime('now', '-1 minute')")
|
|
conn.commit()
|
|
make_note(app)
|
|
assert conn.execute("SELECT COUNT(*) FROM websub_deliveries").fetchone()[0] == 0
|
|
run_worker(app) # prunes
|
|
assert conn.execute("SELECT COUNT(*) FROM websub_subscriptions").fetchone()[0] == 0
|
|
|
|
|
|
# Feeds are fresh right after a change (ADR-071 3) ------------------------------------
|
|
|
|
def test_feed_shows_new_note_immediately(app, client):
|
|
from starpunk.routes import public
|
|
make_note(app, "First")
|
|
assert b"First" in client.get("/feed.atom").data
|
|
make_note(app, "Second") # cached note list is stale now
|
|
assert b"Second" in client.get("/feed.atom").data
|
|
assert public._feed_cache["notes"] is not None
|