Files
StarPunk/tests/test_webmention_spec.py
PhilandClaude Opus 5.5 0640730416 feat: Pin webmention connections; re-send when responses change (v2.7.2)
- Connect to the address that passed the SSRF check (Host header and SNI
  carry the hostname, so TLS still verifies it), closing the DNS rebinding
  gap (spec 4.5). The webmention POST uses the same path.
- Treat a change in the responses a note shows as an update and re-send its
  webmentions (spec 3.1.4), from validation and from admin moderation.
  Unchanged re-validations send nothing, so there is no ping-pong.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-03 09:01:38 -07:00

365 lines
16 KiB
Python

"""
W3C Webmention conformance tests (v2.7.0)
https://www.w3.org/TR/webmention/ — section numbers in test names.
"""
import sqlite3
from unittest.mock import patch
import httpx
import pytest
from starpunk import webmention_ssrf
from starpunk.webmention_processor import (
TransientError, ValidationError, fetch_source, verify_link_presence,
)
from starpunk.webmention_worker import WebmentionWorker
PUBLIC = [(2, 1, 6, '', ('93.184.216.34', 0))]
def original_url(request):
"""The URL a pinned request was for (connections go to the checked IP)"""
return f"{request.url.scheme}://{request.headers['host']}{request.url.raw_path.decode()}"
BASE = 'https://t.example/test/1?x=1'
@pytest.fixture
def worker(app):
return WebmentionWorker(app, db_path=app.config["DATABASE_PATH"],
site_url=app.config["SITE_URL"])
@pytest.fixture
def mock_web():
"""Route safe_get through a MockTransport; all hosts resolve public"""
routes = {}
def handler(request):
return routes[original_url(request)](request)
with patch("starpunk.webmention_ssrf.socket.getaddrinfo", return_value=PUBLIC), \
patch("starpunk.webmention_ssrf.TRANSPORT", httpx.MockTransport(handler)):
yield routes
# 3.1.2 Sender discovers receiver Webmention endpoint
@pytest.mark.parametrize("header,expected", [
('<https://e.example/wm>; rel="webmention"', 'https://e.example/wm'),
('</wm>; rel=webmention', 'https://t.example/wm'),
('<wm>; rel="other webmention"', 'https://t.example/test/wm'),
('<wm>; title="a;b,c"; rel="webmention"', 'https://t.example/test/wm'),
('<wm>; REL="WebMention"', 'https://t.example/test/wm'),
('<a>; rel="other", </wm?q=1>; rel="webmention"', 'https://t.example/wm?q=1'),
('<x>; rel="webmention-not"', None),
('<x>; rel="not-webmention"', None),
])
def test_3_1_2_link_header(worker, header, expected):
assert worker._parse_link_header(header, BASE) == expected
@pytest.mark.parametrize("html,expected", [
# <a> before <link>: first in document order wins
('<a rel="webmention" href="/a">x</a><link rel="webmention" href="/l">', 'https://t.example/a'),
('<link rel="webmention somethingelse" href="/l">', 'https://t.example/l'),
('<link rel="Webmention" href="/l">', 'https://t.example/l'),
# empty href is the target itself
('<link rel="webmention" href="">', BASE),
# no href: skipped
('<link rel="webmention"><a rel="webmention" href="/a">', 'https://t.example/a'),
# comments, escaped markup and other rels are ignored
('<!-- <link rel="webmention" href="/bad"> -->'
'<a rel="not-webmention" href="/bad"></a>'
'<code>&lt;a rel="webmention" href="/bad"&gt;</code>'
'<link rel="webmention" href="/ok">', 'https://t.example/ok'),
('<p>nothing</p>', None),
])
def test_3_1_2_html(worker, html, expected):
assert worker._parse_html_for_endpoint(html, BASE) == expected
def test_3_1_2_header_before_html_and_relative_to_final_url(app, worker, mock_web):
mock_web['https://t.example/old'] = lambda r: httpx.Response(
301, headers={'Location': 'https://t.example/new/page'})
mock_web['https://t.example/new/page'] = lambda r: httpx.Response(
200, headers={'Link': '<wm>; rel="webmention"', 'Content-Type': 'text/html'},
text='<link rel="webmention" href="/html">')
with app.app_context():
assert worker._discover_endpoint('https://t.example/old') == 'https://t.example/new/wm'
def test_3_1_2_multiple_link_headers(app, worker, mock_web):
mock_web['https://t.example/p'] = lambda r: httpx.Response(
200, headers=[('Link', '</other>; rel="other"'), ('Link', '</wm>; rel="webmention"')])
with app.app_context():
assert worker._discover_endpoint('https://t.example/p') == 'https://t.example/wm'
def test_3_1_2_server_error_is_retried_not_final(app, worker, mock_web):
mock_web['https://t.example/p'] = lambda r: httpx.Response(503)
with app.app_context(), pytest.raises(httpx.HTTPError):
worker._discover_endpoint('https://t.example/p')
# 4.5 SSRF: every redirect hop is checked
def test_4_5_redirect_to_private_address_blocked(app):
def resolve(host, *a, **k):
ip = '169.254.169.254' if host == 'metadata.internal' else '93.184.216.34'
return [(2, 1, 6, '', (ip, 0))]
def handler(request):
assert request.url.host == '93.184.216.34' # never connects to the private one
return httpx.Response(302, headers={'Location': 'http://metadata.internal/'})
with app.app_context(), \
patch("starpunk.webmention_ssrf.socket.getaddrinfo", side_effect=resolve), \
patch("starpunk.webmention_ssrf.TRANSPORT", httpx.MockTransport(handler)):
with pytest.raises(ValidationError, match="Blocked"):
fetch_source('https://evil.example/', 'http://localhost:5000')
@pytest.mark.parametrize("ip", [
'127.0.0.1', '10.1.2.3', '169.254.169.254', '0.0.0.0', '100.64.0.1',
'::1', '::ffff:127.0.0.1', 'fe80::1', '224.0.0.1',
])
def test_4_5_blocked_addresses(ip):
assert webmention_ssrf.is_private_ip(ip)
def test_4_5_any_blocked_address_blocks(app):
mixed = PUBLIC + [(10, 1, 6, '', ('::1', 0, 0, 0))]
with app.app_context(), patch("starpunk.webmention_ssrf.socket.getaddrinfo", return_value=mixed):
with pytest.raises(webmention_ssrf.SSRFBlockedError):
webmention_ssrf.validate_url_for_fetch('https://x.example/')
# 3.2.2 Webmention verification
TARGET = 'https://me.example/note/abc'
@pytest.mark.parametrize("content,content_type,expected", [
('<a href="/note/abc#c">x</a>', 'text/html', True),
('<img src="https://me.example/note/abc">', 'text/html; charset=utf-8', True),
('<a href="https://me.example/note/abcdef">x</a>', 'text/html', False),
('<p>https://me.example/note/abc</p>', 'text/html', False),
('<!-- <a href="https://me.example/note/abc"> -->', 'text/html', False),
('{"a": ["https://me.example/note/abc"]}', 'application/json', True),
('{"a": "https://me.example/note/abcdef"}', 'application/json', False),
('see https://me.example/note/abc', 'text/plain', True),
])
def test_3_2_2_exact_match(content, content_type, expected):
assert verify_link_presence(content, TARGET, content_type, 'https://me.example/x') is expected
def test_3_2_2_accept_header_and_status(app, mock_web):
seen = {}
def page(request):
seen['accept'] = request.headers.get('Accept')
return httpx.Response(410)
mock_web['https://s.example/p'] = page
with app.app_context():
with pytest.raises(ValidationError, match="410"):
fetch_source('https://s.example/p', 'http://localhost:5000')
assert 'text/html' in seen['accept']
mock_web['https://s.example/p'] = lambda r: httpx.Response(500)
with app.app_context(), pytest.raises(TransientError):
fetch_source('https://s.example/p', 'http://localhost:5000')
# 3.2.1 Request verification
def _post(client, source, target):
return client.post('/webmention', data={'source': source, 'target': target})
def test_3_2_1_target_must_be_a_note(app, client, sample_note):
note_url = f"http://localhost:5000/note/{sample_note.slug}"
assert _post(client, 'https://s.example/a', note_url).status_code == 202
assert _post(client, 'https://s.example/a', 'http://localhost:5000/nope').status_code == 400
assert _post(client, 'https://s.example/a', 'http://localhost:5000/').status_code == 400
assert _post(client, 'https://s.example/a', 'https://other.example/note/x').status_code == 400
assert _post(client, note_url, note_url).status_code == 400
assert _post(client, 'ftp://s.example/a', note_url).status_code == 400
def test_3_2_1_host_case_and_fragment(app, client, sample_note):
note_url = f"http://LOCALHOST:5000/note/{sample_note.slug}"
assert _post(client, 'https://s.example/a', note_url + '#one').status_code == 202
assert _post(client, 'https://s.example/a', note_url + '#two').status_code == 202
db = sqlite3.connect(app.config['DATABASE_PATH'])
rows = db.execute("SELECT target_url FROM webmentions").fetchall()
assert rows == [(note_url,)]
# 3.2.4 Updating existing Webmentions
def _insert_mention(app, note_id, **cols):
db = sqlite3.connect(app.config['DATABASE_PATH'])
fields = {'source_url': 'https://s.example/a', 'target_url': 'http://localhost:5000/n',
'target_note_id': note_id, **cols}
db.execute(f"INSERT INTO webmentions ({', '.join(fields)}) VALUES ({', '.join('?' * len(fields))})",
tuple(fields.values()))
db.commit()
return db
VALID = {'valid': True, 'mention_type': 'reply', 'author_name': 'A', 'author_url': 'https://a.example/',
'author_photo': None, 'content_snippet': 'hi'}
def test_3_2_4_update_keeps_moderation(app, worker, sample_note):
db = _insert_mention(app, sample_note.id, validation_status='valid', moderation_status='hidden',
validated_at='2026-01-01')
with patch("starpunk.webmention_worker.process_webmention", return_value=VALID):
worker._validate_webmention(1, 'https://s.example/a', 'http://localhost:5000/n', sample_note.id)
assert db.execute("SELECT validation_status, moderation_status FROM webmentions").fetchone() == ('valid', 'hidden')
def test_3_2_4_new_mention_gets_default_moderation(app, worker, sample_note):
db = _insert_mention(app, sample_note.id)
with patch("starpunk.webmention_worker.process_webmention", return_value=VALID):
worker._validate_webmention(1, 'https://s.example/a', 'http://localhost:5000/n', sample_note.id)
assert db.execute("SELECT validation_status, moderation_status FROM webmentions").fetchone() == ('valid', 'approved')
def test_3_2_4_transient_error_keeps_valid_mention(app, worker, sample_note):
db = _insert_mention(app, sample_note.id, validation_status='pending', moderation_status='approved',
validated_at='2026-01-01', content_snippet='kept')
with patch("starpunk.webmention_worker.process_webmention", side_effect=TransientError("timeout")):
worker._validate_webmention(1, 'https://s.example/a', 'http://localhost:5000/n', sample_note.id)
assert db.execute("SELECT validation_status, content_snippet FROM webmentions").fetchone() == ('valid', 'kept')
def test_3_2_4_gone_source_removes_mention(app, worker, sample_note):
db = _insert_mention(app, sample_note.id, validation_status='pending', moderation_status='approved',
validated_at='2026-01-01')
with patch("starpunk.webmention_worker.process_webmention",
side_effect=ValidationError("Source returned 410")):
worker._validate_webmention(1, 'https://s.example/a', 'http://localhost:5000/n', sample_note.id)
assert db.execute("SELECT validation_status FROM webmentions").fetchone() == ('invalid',)
# 3.1.4 Updated posts / 3.1.5 Deleted posts
def _queue(app):
db = sqlite3.connect(app.config['DATABASE_PATH'])
return db.execute(
"SELECT target_url, status, source_note_id FROM webmention_queue ORDER BY target_url"
).fetchall()
def _sent(app):
db = sqlite3.connect(app.config['DATABASE_PATH'])
db.execute("UPDATE webmention_queue SET status = 'sent'")
db.commit()
def test_3_1_4_update_resends_previous_targets(app):
from starpunk.notes import create_note, update_note
with app.test_request_context():
note = create_note("Reply", published=True, in_reply_to="https://a.example/post")
_sent(app)
update_note(id=note.id, content="Now about [b](https://b.example/x)", in_reply_to="")
targets = {t: s for t, s, _ in _queue(app)}
assert targets == {'https://a.example/post': 'pending', 'https://b.example/x': 'pending'}
@pytest.mark.parametrize("soft", [True, False])
def test_3_1_5_delete_resends_and_returns_410(app, client, soft):
from starpunk.notes import create_note, delete_note
with app.test_request_context():
note = create_note("See [a](https://a.example/x)", published=True)
_sent(app)
delete_note(id=note.id, soft=soft)
rows = _queue(app)
assert [(t, s) for t, s, _ in rows] == [('https://a.example/x', 'pending')]
if not soft:
assert rows[0][2] is None # detached, so it survives the delete
assert client.get(f"/note/{note.slug}").status_code == 410
assert client.get("/note/never-existed").status_code == 404
# 4.5 Connections go to the address that was checked (no DNS rebinding)
def test_4_5_connection_pinned_to_checked_address(app):
seen = []
def handler(request):
seen.append((str(request.url), request.headers['host'],
request.extensions.get('sni_hostname')))
return httpx.Response(200, text='ok')
with app.app_context(), \
patch("starpunk.webmention_ssrf.socket.getaddrinfo", return_value=PUBLIC) as resolve, \
patch("starpunk.webmention_ssrf.TRANSPORT", httpx.MockTransport(handler)):
webmention_ssrf.safe_get('https://site.example:8443/p?q=1', {})
webmention_ssrf.safe_post('http://site.example/wm', {'source': 's'}, {})
assert seen == [
('https://93.184.216.34:8443/p?q=1', 'site.example:8443', 'site.example'),
('http://93.184.216.34/wm', 'site.example', None),
]
assert resolve.call_count == 2 # one lookup per request, then pinned
def test_4_5_ipv4_preferred(app):
with app.app_context(), patch("starpunk.webmention_ssrf.socket.getaddrinfo",
return_value=[(10, 1, 6, '', ('2606:2800::1', 0, 0, 0))] + PUBLIC):
assert webmention_ssrf.safe_resolve('x.example') == '93.184.216.34'
# 3.1.4 A change in the responses a note shows is an update of the note
def _note_with_sent_mention(app):
from starpunk.notes import create_note
with app.test_request_context():
note = create_note("See [a](https://a.example/x)", published=True)
_sent(app)
db = sqlite3.connect(app.config['DATABASE_PATH'])
db.execute("INSERT INTO webmentions (source_url, target_url, target_note_id) VALUES (?, ?, ?)",
('https://s.example/reply', f'http://localhost:5000/note/{note.slug}', note.id))
db.commit()
return note, db
def test_3_1_4_new_response_resends(app, worker):
note, db = _note_with_sent_mention(app)
with patch("starpunk.webmention_worker.process_webmention", return_value=VALID):
worker._validate_webmention(1, 'https://s.example/reply', 'x', note.id)
assert [(t, s) for t, s, _ in _queue(app)] == [('https://a.example/x', 'pending')]
def test_3_1_4_unchanged_response_does_not_resend(app, worker):
note, db = _note_with_sent_mention(app)
with patch("starpunk.webmention_worker.process_webmention", return_value=VALID):
worker._validate_webmention(1, 'https://s.example/reply', 'x', note.id)
_sent(app)
db.execute("UPDATE webmentions SET validation_status = 'pending'") # re-sent by source
db.commit()
worker._validate_webmention(1, 'https://s.example/reply', 'x', note.id)
assert [(t, s) for t, s, _ in _queue(app)] == [('https://a.example/x', 'sent')]
def test_3_1_4_moderation_change_resends(app, worker):
from starpunk.webmention_queries import update_webmention_status
note, db = _note_with_sent_mention(app)
with patch("starpunk.webmention_worker.process_webmention", return_value=VALID):
worker._validate_webmention(1, 'https://s.example/reply', 'x', note.id)
_sent(app)
with app.app_context():
update_webmention_status([1], 'approved') # already approved: no change
assert _queue(app)[0][1] == 'sent'
with app.app_context():
update_webmention_status([1], 'hidden')
assert _queue(app)[0][1] == 'pending'