- Per-item error handling in the worker; hard time limits on requests - Notes with photos announced after attach; feed cache/ETag follow content_version - Keep failing subscriptions until lease end; retry DNS and render errors; hub.mode=denied for gone topics - Spec 5.1: decode unreserved chars, echo requested topic, ignore lease and secret on unsubscribe, case-insensitive Content-Type, refuse empty secret - Limits count pending requests per callback address; callback validation; request_id guards the verification race (migration 017) - Render once per topic per pass; likes only push /likes; Vary: Accept Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
317 lines
11 KiB
Python
317 lines
11 KiB
Python
"""
|
|
WebSub hardening (v2.10.1): one test per finding of the v2.10.0 review
|
|
"""
|
|
|
|
import io
|
|
import socket
|
|
import sqlite3
|
|
import threading
|
|
import time
|
|
from unittest.mock import patch
|
|
from urllib.parse import parse_qs, urlparse
|
|
|
|
import httpx
|
|
import pytest
|
|
from PIL import Image
|
|
|
|
from tests.test_websub import ( # noqa: F401
|
|
HUB, PUBLIC, SITE, db, echo, make_note, run_worker, subscribe, web,
|
|
)
|
|
from starpunk.websub_worker import WebSubWorker
|
|
|
|
|
|
def jpeg(color="red"):
|
|
buf = io.BytesIO()
|
|
Image.new("RGB", (40, 30), color).save(buf, format="JPEG")
|
|
return buf.getvalue()
|
|
|
|
|
|
def intents(app):
|
|
return db(app).execute("SELECT * FROM websub_intents ORDER BY id").fetchall()
|
|
|
|
|
|
def subs(app):
|
|
return db(app).execute("SELECT * FROM websub_subscriptions ORDER BY id").fetchall()
|
|
|
|
|
|
def subscribed(app, client, web, topic=f"{SITE}/feed.atom", callback="https://sub.example/cb"):
|
|
hosts, seen = web
|
|
hosts.setdefault(urlparse(callback).hostname, echo)
|
|
subscribe(client, topic, callback=callback)
|
|
run_worker(app)
|
|
seen.clear()
|
|
return hosts, seen
|
|
|
|
|
|
# H1, H2: one bad item can't stop the worker -----------------------------------
|
|
|
|
def test_oversized_challenge_reply_does_not_wedge_worker(app, client, web):
|
|
hosts, _ = web
|
|
hosts["big.example"] = lambda r: httpx.Response(200, content=b"x" * 100_000)
|
|
hosts["good.example"] = echo
|
|
subscribe(client, f"{SITE}/", callback="https://big.example/cb")
|
|
subscribe(client, f"{SITE}/", callback="https://good.example/cb")
|
|
run_worker(app)
|
|
assert [s["callback"] for s in subs(app)] == ["https://good.example/cb"]
|
|
assert intents(app) == []
|
|
|
|
|
|
@pytest.mark.parametrize("callback", [
|
|
"https://example.com/\x00", "https://example.com/a b", "https://example.com:25/cb",
|
|
f"{SITE}/cb", "https://" + "a" * 2100 + ".example/",
|
|
])
|
|
def test_bad_callbacks_refused(client, web, callback):
|
|
assert subscribe(client, f"{SITE}/", callback=callback).status_code == 400
|
|
|
|
|
|
def test_unexpected_error_affects_only_that_item(app, client, web):
|
|
hosts, _ = web
|
|
hosts["good.example"] = echo
|
|
|
|
def crash(request):
|
|
raise RuntimeError("something nobody expected")
|
|
|
|
hosts["bad.example"] = crash
|
|
subscribe(client, f"{SITE}/", callback="https://bad.example/cb") # first in the queue
|
|
subscribe(client, f"{SITE}/", callback="https://good.example/cb")
|
|
run_worker(app)
|
|
assert [s["callback"] for s in subs(app)] == ["https://good.example/cb"]
|
|
assert intents(app) == []
|
|
|
|
|
|
def test_idn_callback_is_punycoded(app, client, web):
|
|
assert subscribe(client, f"{SITE}/", callback="https://bücher.example/cb").status_code == 202
|
|
assert intents(app)[0]["callback"] == "https://xn--bcher-kva.example/cb"
|
|
|
|
|
|
# H3, H4: photos are in what's delivered and in the feeds -----------------------
|
|
|
|
def test_new_photo_note_is_delivered_with_its_photo(app, client, web):
|
|
from starpunk.media import attach_media_to_note, save_media
|
|
hosts, seen = subscribed(app, client, web)
|
|
with app.test_request_context():
|
|
note = make_note(app, "With a photo", has_media=True)
|
|
assert db(app).execute("SELECT COUNT(*) FROM websub_deliveries").fetchone()[0] == 0
|
|
media = save_media(jpeg(), "cat.jpg")
|
|
attach_media_to_note(note.id, [media["id"]], ["A cat"])
|
|
run_worker(app)
|
|
(post,) = [r for r in seen if r.method == "POST"]
|
|
assert media["path"].encode() in post.content
|
|
|
|
|
|
def test_photo_change_refreshes_feed_and_is_delivered(app, client, web):
|
|
from starpunk.media import attach_media_to_note, save_media
|
|
with app.test_request_context():
|
|
note = make_note(app, "Photo later")
|
|
first = save_media(jpeg("red"), "a.jpg")
|
|
attach_media_to_note(note.id, [first["id"]], [""])
|
|
assert first["path"].encode() in client.get("/feed.atom").data # cached now
|
|
hosts, seen = subscribed(app, client, web)
|
|
with app.test_request_context():
|
|
second = save_media(jpeg("blue"), "b.jpg")
|
|
attach_media_to_note(note.id, [second["id"]], [""])
|
|
feed = client.get("/feed.atom").data
|
|
assert second["path"].encode() in feed and first["path"].encode() not in feed
|
|
run_worker(app)
|
|
assert second["path"].encode() in [r for r in seen if r.method == "POST"][0].content
|
|
|
|
|
|
# H5: a slow subscriber can't hold the worker ------------------------------------
|
|
|
|
def test_slow_callback_is_cut_off():
|
|
from starpunk import webmention_ssrf as w
|
|
|
|
server = socket.socket()
|
|
server.bind(("127.0.0.1", 0))
|
|
server.listen(1)
|
|
port = server.getsockname()[1]
|
|
|
|
def trickle():
|
|
conn, _ = server.accept()
|
|
conn.recv(65536)
|
|
try:
|
|
for b in b"HTTP/1.1 200 OK\r\nX-Slow: " + b"y" * 100:
|
|
conn.send(bytes([b]))
|
|
time.sleep(0.3)
|
|
except OSError:
|
|
pass
|
|
finally:
|
|
conn.close()
|
|
|
|
threading.Thread(target=trickle, daemon=True).start()
|
|
start = time.monotonic()
|
|
with patch.object(w, "safe_resolve", lambda host: "127.0.0.1"), pytest.raises(httpx.TimeoutException):
|
|
w.safe_post(f"http://slow.example:{port}/", data={"a": "b"}, timeout=1)
|
|
assert time.monotonic() - start < 3
|
|
|
|
|
|
# Spec 7: subscriptions stay until their lease ends --------------------------------
|
|
|
|
def test_failing_subscriber_is_kept_until_lease_end(app, client, web):
|
|
hosts, _ = subscribed(app, client, web)
|
|
hosts["sub.example"] = lambda r: httpx.Response(500)
|
|
conn = db(app)
|
|
conn.execute("UPDATE websub_subscriptions SET verified_at = datetime('now', '-10 days')")
|
|
conn.commit()
|
|
make_note(app)
|
|
conn.execute("UPDATE websub_deliveries SET attempts = 5") # last retry
|
|
conn.commit()
|
|
run_worker(app)
|
|
assert conn.execute("SELECT COUNT(*) FROM websub_deliveries").fetchone()[0] == 0
|
|
assert len(subs(app)) == 1
|
|
|
|
|
|
def test_dns_failure_is_retried(app, client, web, monkeypatch):
|
|
subscribed(app, client, web)
|
|
make_note(app)
|
|
|
|
def fail(*a, **k):
|
|
raise socket.gaierror("temporary failure")
|
|
|
|
monkeypatch.setattr("starpunk.webmention_ssrf.socket.getaddrinfo", fail)
|
|
run_worker(app)
|
|
row = db(app).execute("SELECT attempts, last_error FROM websub_deliveries").fetchone()
|
|
assert row["attempts"] == 1 and "resolve" in row["last_error"]
|
|
|
|
|
|
def test_render_error_is_retried(app, client, web):
|
|
subscribed(app, client, web)
|
|
make_note(app)
|
|
with patch.object(WebSubWorker, "render_topic", lambda self, t: (500, "text/plain", b"")):
|
|
run_worker(app)
|
|
assert db(app).execute("SELECT attempts FROM websub_deliveries").fetchone()[0] == 1
|
|
|
|
|
|
def test_gone_topic_ends_subscription_with_denial(app, client, web):
|
|
hosts, seen = subscribed(app, client, web)
|
|
make_note(app)
|
|
with patch.object(WebSubWorker, "render_topic", lambda self, t: (404, "text/html", b"")):
|
|
run_worker(app)
|
|
assert subs(app) == []
|
|
denial = [r for r in seen if r.method == "GET"][0]
|
|
assert parse_qs(urlparse(str(denial.url)).query)["hub.mode"] == ["denied"]
|
|
|
|
|
|
# M1: limits count pending requests and the callback's address ----------------------
|
|
|
|
def test_limits_count_pending_requests(app, client, web):
|
|
app.config["WEBSUB_MAX_PER_HOST"] = 3
|
|
codes = [subscribe(client, f"{SITE}/", callback=f"https://s{i}.example/cb").status_code for i in range(5)]
|
|
assert codes == [202, 202, 202, 429, 429] # all resolve to one address
|
|
|
|
|
|
def test_renewal_is_allowed_at_the_limit(app, client, web):
|
|
app.config["WEBSUB_MAX_PER_HOST"] = 1
|
|
subscribed(app, client, web)
|
|
assert subscribe(client, f"{SITE}/feed.atom", callback="https://sub.example/cb").status_code == 202
|
|
|
|
|
|
# Spec 5.1 details -----------------------------------------------------------------
|
|
|
|
def test_unsubscribe_ignores_lease_and_secret(app, client, web):
|
|
form = {"hub.mode": "unsubscribe", "hub.topic": f"{SITE}/", "hub.callback": "https://sub.example/cb",
|
|
"hub.lease_seconds": "abc", "hub.secret": ""}
|
|
assert client.post("/websub", data=form).status_code == 202
|
|
|
|
|
|
def test_unreserved_characters_are_decoded(app, client, web):
|
|
assert subscribe(client, f"{SITE}/%66eed%2Eatom").status_code == 202
|
|
assert intents(app)[0]["topic"] == f"{SITE}/feed.atom"
|
|
|
|
|
|
def test_verification_echoes_the_requested_topic(app, client, web):
|
|
hosts, seen = web
|
|
hosts["sub.example"] = echo
|
|
make_note(app, tags=["Python"])
|
|
subscribe(client, f"{SITE}/tag/Python")
|
|
run_worker(app)
|
|
assert parse_qs(urlparse(str(seen[0].url)).query)["hub.topic"] == [f"{SITE}/tag/Python"]
|
|
assert subs(app)[0]["topic"] == f"{SITE}/tag/python"
|
|
|
|
|
|
def test_content_type_is_case_insensitive(client, web):
|
|
response = client.post("/websub", data="hub.mode=subscribe&hub.topic=" + SITE + "/&hub.callback=https://sub.example/cb",
|
|
content_type="Application/X-WWW-Form-Urlencoded")
|
|
assert response.status_code == 202
|
|
|
|
|
|
def test_empty_secret_is_refused(client, web):
|
|
assert subscribe(client, f"{SITE}/", **{"hub.secret": ""}).status_code == 400
|
|
|
|
|
|
def test_challenge_must_match(app, client, web):
|
|
hosts, _ = web
|
|
|
|
def padded(request):
|
|
challenge = parse_qs(urlparse(str(request.url)).query)["hub.challenge"][0]
|
|
return httpx.Response(200, text=f" {challenge} ")
|
|
|
|
hosts["sub.example"] = padded
|
|
subscribe(client, f"{SITE}/")
|
|
run_worker(app)
|
|
assert subs(app) == []
|
|
|
|
|
|
# L1: a newer request isn't lost while an older one is verified ------------------------
|
|
|
|
def test_newer_request_survives_older_verification(app, client, web):
|
|
hosts, _ = web
|
|
|
|
def slow_echo(request):
|
|
# A newer request (new secret) arrives while this one is being verified
|
|
subscribe(client, f"{SITE}/", **{"hub.secret": "new"})
|
|
return echo(request)
|
|
|
|
hosts["sub.example"] = slow_echo
|
|
subscribe(client, f"{SITE}/", **{"hub.secret": "old"})
|
|
run_worker(app)
|
|
pending = intents(app)
|
|
assert len(pending) == 1 and pending[0]["secret"] == "new"
|
|
|
|
|
|
# L2: fewer renders and pushes ------------------------------------------------------
|
|
|
|
def test_topic_rendered_once_per_pass(app, client, web):
|
|
hosts, _ = web
|
|
hosts["a.example"] = hosts["b.example"] = echo
|
|
for cb in ("https://a.example/cb", "https://b.example/cb"):
|
|
subscribe(client, f"{SITE}/feed.atom", callback=cb)
|
|
run_worker(app)
|
|
make_note(app)
|
|
calls = []
|
|
original = WebSubWorker.render_topic
|
|
|
|
def counting(self, topic):
|
|
calls.append(topic)
|
|
return original(self, topic)
|
|
|
|
with patch.object(WebSubWorker, "render_topic", counting):
|
|
run_worker(app)
|
|
assert calls == [f"{SITE}/feed.atom"]
|
|
|
|
|
|
def test_like_only_updates_likes_page(app, client, web):
|
|
hosts, _ = web
|
|
hosts["sub.example"] = echo
|
|
for topic in (f"{SITE}/", f"{SITE}/likes"):
|
|
subscribe(client, topic)
|
|
run_worker(app)
|
|
make_note(app, "", like_of="https://elsewhere.example/post")
|
|
topics = [r[0] for r in db(app).execute("SELECT topic FROM websub_deliveries")]
|
|
assert topics == [f"{SITE}/likes"]
|
|
|
|
|
|
# L5: feeds ----------------------------------------------------------------------
|
|
|
|
def test_etag_is_stable_across_processes(app, client):
|
|
from starpunk.feeds import get_cache
|
|
make_note(app)
|
|
first = client.get("/feed.rss").headers["ETag"]
|
|
get_cache().invalidate() # as if another process rendered it
|
|
time.sleep(1.1) # a different build time
|
|
assert client.get("/feed.rss").headers["ETag"] == first
|
|
|
|
|
|
def test_negotiated_feed_varies_on_accept(client):
|
|
assert "Accept" in client.get("/feed", headers={"Accept": "application/atom+xml"}).headers.get("Vary", "")
|