Merge branch 'feat/v2.7.2-webmention-shoulds' for v2.7.2 release
Build Container / build (push) Successful in 9m28s

This commit is contained in:
Phil committed 2026-11-03 09:01:42 -07:00
commit c8a7ef309f
10 files changed
+315 -26

No files matched your search

+10
View File
@@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased] ## [Unreleased]
## [2.7.2] - 2026-11-03
### Added
- When the replies, likes and other responses shown on a note change (a webmention arrives, is updated or removed, or you approve, hide or delete one), the note's webmentions are sent again, as the Webmention spec recommends, so the sites it links to can pick up the conversation. A response that is checked again without changing sends nothing
### Security
- Webmention fetches and sends connect to the exact address that passed the private-address check. Before, the hostname was looked up a second time when connecting, so a DNS server that changed its answer could still point StarPunk at a private address. TLS certificates are still checked against the hostname. IPv4 addresses are preferred when a host has both
## [2.7.1] - 2026-11-03 ## [2.7.1] - 2026-11-03
### Changed ### Changed
@@ -40,9 +40,10 @@ v2.6.8 and v2.6.9 fixed the bugs that stopped webmentions working at all.
## Known limits ## Known limits
- DNS is checked before connecting, not pinned for the connection, so a
rebinding DNS server could still answer differently on connect.
- Homepage mentions are refused rather than stored. - Homepage mentions are refused rather than stored.
- Discovery does not apply HTTP cache headers (5.3); targets are only
fetched when a note is created, edited or deleted.
- The 410 page is a generic tombstone without microformats.
- `error` rows (unexpected failures) are not retried automatically. - `error` rows (unexpected failures) are not retried automatically.
## Tests ## Tests
@@ -0,0 +1,36 @@
# v2.7.2: Remaining Webmention SHOULDs
Follow-up to the v2.7.0 compliance review. Every SHOULD in the
Recommendation was checked; two were not met.
## 4.5 DNS pinning
`validate_url_for_fetch()` now returns the checked address.
`_pinned_request()` builds each request against that IP, with the original
host in the `Host` header and, for https, in the `sni_hostname` extension.
httpcore uses `sni_hostname` as `server_hostname` with `check_hostname`
on, so the certificate is verified against the real hostname. Used by
`safe_get()` (every redirect hop) and the new `safe_post()` (the webmention
POST). IPv4 is preferred when a host has both families.
## 3.1.4 Displayed responses are an update
"If the source post displays responses... then sender SHOULD treat that as
an update of the source URL and re-send any previously sent Webmentions."
`note_responses_changed()` re-queues a published note's previous webmentions.
It is called when:
- the worker validates a webmention and what the note shows for it changed
(`_display_state()` before and after: shown or not, type, author, snippet).
A re-send from the source resets the row to `pending`; a previously valid
row still counts as shown, so an unchanged re-validation sends nothing.
This keeps two sites that both implement this from ping-ponging.
- an admin approves, hides or deletes webmentions and a valid one changes
between shown and not shown.
## Tests
`tests/test_webmention_spec.py`: pinning (URL, Host, SNI, one lookup per
request), IPv4 preference, re-send on a new response, none on an unchanged
one, re-send on moderation changes.
+2 -2
View File
@@ -459,5 +459,5 @@ def create_app(config=None):
# Package version (Semantic Versioning 2.0.0) # Package version (Semantic Versioning 2.0.0)
# See docs/standards/versioning-strategy.md for details # See docs/standards/versioning-strategy.md for details
__version__ = "2.7.1" __version__ = "2.7.2"
__version_info__ = (2, 7, 1) __version_info__ = (2, 7, 2)
+39
View File
@@ -17,7 +17,10 @@ Functions:
import logging import logging
import math import math
from datetime import datetime from datetime import datetime
from pathlib import Path
from typing import List, Dict, Any, Optional from typing import List, Dict, Any, Optional
from flask import current_app
from starpunk.database import get_db from starpunk.database import get_db
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -233,10 +236,19 @@ def update_webmention_status(webmention_ids: List[int], status: str) -> int:
WHERE id IN ({placeholders}) WHERE id IN ({placeholders})
""" """
# Notes whose displayed mentions change (valid ones switching between
# approved and not)
changed_notes = _visibility_changes(
db, webmention_ids,
"validation_status = 'valid' AND (moderation_status = 'approved') != (? = 'approved')",
[status],
)
params = [status] + webmention_ids params = [status] + webmention_ids
cursor = db.execute(query, params) cursor = db.execute(query, params)
db.commit() db.commit()
_notify_responses_changed(changed_notes)
return cursor.rowcount return cursor.rowcount
@@ -255,15 +267,42 @@ def delete_webmentions(webmention_ids: List[int]) -> int:
db = get_db() db = get_db()
changed_notes = _visibility_changes(
db, webmention_ids, "validation_status = 'valid' AND moderation_status = 'approved'", [],
)
placeholders = ','.join('?' * len(webmention_ids)) placeholders = ','.join('?' * len(webmention_ids))
query = f"DELETE FROM webmentions WHERE id IN ({placeholders})" query = f"DELETE FROM webmentions WHERE id IN ({placeholders})"
cursor = db.execute(query, webmention_ids) cursor = db.execute(query, webmention_ids)
db.commit() db.commit()
_notify_responses_changed(changed_notes)
return cursor.rowcount return cursor.rowcount
def _visibility_changes(db, webmention_ids: List[int], condition: str, params: list) -> List[int]:
"""Target note IDs of the given webmentions that match condition"""
placeholders = ','.join('?' * len(webmention_ids))
rows = db.execute(
f"SELECT DISTINCT target_note_id FROM webmentions "
f"WHERE id IN ({placeholders}) AND {condition}",
list(webmention_ids) + params,
).fetchall()
return [row[0] for row in rows]
def _notify_responses_changed(note_ids: List[int]) -> None:
"""Re-send webmentions for notes whose shown responses changed (best effort)"""
if not note_ids:
return
try:
from starpunk.webmention_sender import note_responses_changed
note_responses_changed(Path(current_app.config['DATABASE_PATH']), note_ids)
except Exception as e:
current_app.logger.warning(f"Failed to re-send webmentions after moderation: {e}")
def get_reply_context(url: str) -> Optional[Dict[str, Any]]: def get_reply_context(url: str) -> Optional[Dict[str, Any]]:
""" """
Get cached reply context for a URL. Get cached reply context for a URL.
+31
View File
@@ -129,6 +129,37 @@ def requeue_previous_webmentions(db_path: Path, note_id: int, detach: bool = Fal
return queued return queued
def note_responses_changed(db_path: Path, note_ids) -> int:
"""
Re-send a note's webmentions because the responses it shows changed
Spec 3.1.4: if the source post displays responses (e.g. comments), a
change to them is an update of the post. Only published, undeleted notes.
Args:
db_path: Path to SQLite database
note_ids: IDs of notes whose displayed responses changed
Returns:
Number of webmentions queued
"""
ids = sorted({i for i in note_ids if i is not None})
if not ids:
return 0
conn = sqlite3.connect(db_path)
try:
placeholders = ','.join('?' * len(ids))
live = [row[0] for row in conn.execute(
f"SELECT id FROM notes WHERE id IN ({placeholders}) "
"AND published = 1 AND deleted_at IS NULL", ids,
)]
finally:
conn.close()
return sum(requeue_previous_webmentions(db_path, note_id) for note_id in live)
def queue_webmentions_for_note( def queue_webmentions_for_note(
db_path: Path, db_path: Path,
note_id: int, note_id: int,
+52 -6
View File
@@ -4,6 +4,8 @@ SSRF protection for webmention fetches
Per ADR-063 Decision 9 and Webmention sections 4.2, 4.3 and 4.5: Per ADR-063 Decision 9 and Webmention sections 4.2, 4.3 and 4.5:
- Every address a hostname resolves to must be public - Every address a hostname resolves to must be public
- Redirects are followed by hand so each hop is checked - Redirects are followed by hand so each hop is checked
- Each request connects to the address that was checked (no second DNS
lookup that a rebinding DNS server could answer differently)
- Redirect count, response size and total time are limited - Redirect count, response size and total time are limited
- DEV_MODE exception for localhost testing - DEV_MODE exception for localhost testing
@@ -89,7 +91,7 @@ def safe_resolve(hostname: str) -> str:
Per background-worker.md: DEV_MODE exception for localhost testing. Per background-worker.md: DEV_MODE exception for localhost testing.
Returns: Returns:
First resolved IP address Address to connect to (IPv4 preferred)
Raises: Raises:
SSRFBlockedError: If any address is blocked SSRFBlockedError: If any address is blocked
@@ -98,7 +100,8 @@ def safe_resolve(hostname: str) -> str:
if current_app.config.get('DEV_MODE') and hostname in ('localhost', '127.0.0.1', '::1'): if current_app.config.get('DEV_MODE') and hostname in ('localhost', '127.0.0.1', '::1'):
return '127.0.0.1' return '127.0.0.1'
addresses = sorted({info[4][0] for info in socket.getaddrinfo(hostname, None)}) addresses = sorted({info[4][0] for info in socket.getaddrinfo(hostname, None)},
key=lambda ip: (':' in ip, ip))
for ip in addresses: for ip in addresses:
if is_private_ip(ip): if is_private_ip(ip):
raise SSRFBlockedError(f"Blocked request to private IP: {ip}") raise SSRFBlockedError(f"Blocked request to private IP: {ip}")
@@ -106,10 +109,13 @@ def safe_resolve(hostname: str) -> str:
return addresses[0] return addresses[0]
def validate_url_for_fetch(url: str) -> None: def validate_url_for_fetch(url: str) -> str:
""" """
Validate URL is safe to fetch (http(s), public addresses only). Validate URL is safe to fetch (http(s), public addresses only).
Returns:
The checked IP address to connect to
Raises: Raises:
SSRFBlockedError: If URL points to a blocked address SSRFBlockedError: If URL points to a blocked address
ValueError: If URL is malformed or its host does not resolve ValueError: If URL is malformed or its host does not resolve
@@ -123,11 +129,48 @@ def validate_url_for_fetch(url: str) -> None:
raise ValueError("URL must include a hostname") raise ValueError("URL must include a hostname")
try: try:
safe_resolve(parsed.hostname) return safe_resolve(parsed.hostname)
except socket.gaierror as e: except socket.gaierror as e:
raise ValueError(f"Could not resolve {parsed.hostname}: {e}") raise ValueError(f"Could not resolve {parsed.hostname}: {e}")
def _pinned_request(client: httpx.Client, method: str, url: str, headers: dict,
**kwargs) -> httpx.Request:
"""
Build a request that connects to the checked address for url's host.
The URL's host is replaced by the IP that passed the SSRF check; the
real host goes in the Host header and, for https, in SNI, so virtual
hosting and certificate verification still use the hostname.
"""
ip = validate_url_for_fetch(url)
parsed = urlparse(url)
ip_host = f"[{ip}]" if ':' in ip else ip
port = f":{parsed.port}" if parsed.port else ''
pinned_url = parsed._replace(netloc=f"{ip_host}{port}").geturl()
host_header = f"{parsed.hostname}{port}"
extensions = {'sni_hostname': parsed.hostname} if parsed.scheme == 'https' else {}
return client.build_request(
method, pinned_url, headers={**headers, 'Host': host_header},
extensions=extensions, **kwargs,
)
def safe_post(url: str, data: dict, headers: dict, timeout: float = TIMEOUT) -> httpx.Response:
"""
POST form data to a checked, pinned address. Redirects are not followed.
Raises:
SSRFBlockedError, ValueError: URL is not allowed
httpx.HTTPError: Network errors
"""
with httpx.Client(follow_redirects=False, transport=TRANSPORT, timeout=timeout) as client:
response = client.send(_pinned_request(client, 'POST', url, headers, data=data))
response.read()
return response
@dataclass @dataclass
class FetchResult: class FetchResult:
"""A fetched response, after redirects""" """A fetched response, after redirects"""
@@ -169,12 +212,13 @@ def safe_get(url: str, headers: dict, max_bytes: int = MAX_BYTES,
with httpx.Client(follow_redirects=False, transport=TRANSPORT) as client: with httpx.Client(follow_redirects=False, transport=TRANSPORT) as client:
for _ in range(max_redirects + 1): for _ in range(max_redirects + 1):
validate_url_for_fetch(url)
remaining = deadline - time.monotonic() remaining = deadline - time.monotonic()
if remaining <= 0: if remaining <= 0:
raise httpx.TimeoutException("Fetch exceeded time limit") raise httpx.TimeoutException("Fetch exceeded time limit")
with client.stream('GET', url, headers=headers, timeout=remaining) as response: request = _pinned_request(client, 'GET', url, headers, timeout=remaining)
response = client.send(request, stream=True)
try:
location = response.headers.get('Location') location = response.headers.get('Location')
if response.is_redirect and location: if response.is_redirect and location:
url = urljoin(url, location) url = urljoin(url, location)
@@ -199,5 +243,7 @@ def safe_get(url: str, headers: dict, max_bytes: int = MAX_BYTES,
content=content, content=content,
encoding=response.charset_encoding or 'utf-8', encoding=response.charset_encoding or 'utf-8',
) )
finally:
response.close()
raise httpx.TooManyRedirects(f"More than {max_redirects} redirects") raise httpx.TooManyRedirects(f"More than {max_redirects} redirects")
+51 -8
View File
@@ -26,8 +26,9 @@ from urllib.parse import urljoin
from lxml import html as lxml_html from lxml import html as lxml_html
from starpunk.webmention_processor import process_webmention, TransientError, ValidationError from starpunk.webmention_processor import process_webmention, TransientError, ValidationError
from starpunk.webmention_sender import note_responses_changed
from starpunk.webmention_ssrf import ( from starpunk.webmention_ssrf import (
FetchTooLargeError, SSRFBlockedError, safe_get, user_agent, validate_url_for_fetch, FetchTooLargeError, SSRFBlockedError, safe_get, safe_post, user_agent,
) )
@@ -244,6 +245,52 @@ class WebmentionWorker(threading.Thread):
source_url: str, source_url: str,
target_url: str, target_url: str,
target_note_id: Optional[int] target_note_id: Optional[int]
):
"""
Validate a webmention, then re-send the target note's webmentions if
what the note shows changed (spec 3.1.4: displayed responses are an
update of the post). Unchanged re-validations send nothing, so two
sites doing this don't ping-pong.
"""
before = self._display_state(webmention_id)
try:
self._validate_and_store(webmention_id, source_url, target_url, target_note_id)
finally:
after = self._display_state(webmention_id)
if before != after and target_note_id is not None:
try:
note_responses_changed(self.db_path, [target_note_id])
except Exception as e:
logger.warning(f"Failed to re-send webmentions for note {target_note_id}: {e}")
def _display_state(self, webmention_id: int) -> Optional[tuple]:
"""
What the note page shows for this webmention, or None if hidden
A row reset to 'pending' by a re-send still counts as shown if it was
valid before (validated with no error), as it was until now.
"""
conn = sqlite3.connect(self.db_path)
try:
row = conn.execute("""
SELECT mention_type, author_name, author_url, author_photo, content_snippet
FROM webmentions
WHERE id = ?
AND moderation_status = 'approved'
AND (validation_status = 'valid'
OR (validation_status = 'pending' AND validated_at IS NOT NULL
AND validation_error IS NULL))
""", (webmention_id,)).fetchone()
finally:
conn.close()
return tuple(row) if row else None
def _validate_and_store(
self,
webmention_id: int,
source_url: str,
target_url: str,
target_note_id: Optional[int]
): ):
""" """
Validate a single webmention Validate a single webmention
@@ -495,20 +542,16 @@ class WebmentionWorker(threading.Thread):
# Send webmention POST (spec 3.1.3) # Send webmention POST (spec 3.1.3)
try: try:
# SSRF protection for endpoint URL (spec 4.3) # SSRF-checked and pinned to the checked address (spec 4.3).
validate_url_for_fetch(endpoint) # Redirects are not followed: a 301/302 would turn the POST
# Redirects are not followed: httpx would turn a 301/302 POST
# into a GET without the body # into a GET without the body
response = httpx.post( response = safe_post(
endpoint, endpoint,
data={ data={
'source': source_url, 'source': source_url,
'target': target_url 'target': target_url
}, },
timeout=10,
headers={'User-Agent': user_agent(self.site_url)}, headers={'User-Agent': user_agent(self.site_url)},
follow_redirects=False
) )
# Any 2xx is success (spec 3.1.3) # Any 2xx is success (spec 3.1.3)
+82 -1
View File
@@ -17,6 +17,11 @@ from starpunk.webmention_processor import (
from starpunk.webmention_worker import WebmentionWorker from starpunk.webmention_worker import WebmentionWorker
PUBLIC = [(2, 1, 6, '', ('93.184.216.34', 0))] PUBLIC = [(2, 1, 6, '', ('93.184.216.34', 0))]
def original_url(request):
"""The URL a pinned request was for (connections go to the checked IP)"""
return f"{request.url.scheme}://{request.headers['host']}{request.url.raw_path.decode()}"
BASE = 'https://t.example/test/1?x=1' BASE = 'https://t.example/test/1?x=1'
@@ -32,7 +37,7 @@ def mock_web():
routes = {} routes = {}
def handler(request): def handler(request):
return routes[str(request.url)](request) return routes[original_url(request)](request)
with patch("starpunk.webmention_ssrf.socket.getaddrinfo", return_value=PUBLIC), \ with patch("starpunk.webmention_ssrf.socket.getaddrinfo", return_value=PUBLIC), \
patch("starpunk.webmention_ssrf.TRANSPORT", httpx.MockTransport(handler)): patch("starpunk.webmention_ssrf.TRANSPORT", httpx.MockTransport(handler)):
@@ -106,6 +111,7 @@ def test_4_5_redirect_to_private_address_blocked(app):
return [(2, 1, 6, '', (ip, 0))] return [(2, 1, 6, '', (ip, 0))]
def handler(request): def handler(request):
assert request.url.host == '93.184.216.34' # never connects to the private one
return httpx.Response(302, headers={'Location': 'http://metadata.internal/'}) return httpx.Response(302, headers={'Location': 'http://metadata.internal/'})
with app.app_context(), \ with app.app_context(), \
@@ -281,3 +287,78 @@ def test_3_1_5_delete_resends_and_returns_410(app, client, soft):
assert rows[0][2] is None # detached, so it survives the delete assert rows[0][2] is None # detached, so it survives the delete
assert client.get(f"/note/{note.slug}").status_code == 410 assert client.get(f"/note/{note.slug}").status_code == 410
assert client.get("/note/never-existed").status_code == 404 assert client.get("/note/never-existed").status_code == 404
# 4.5 Connections go to the address that was checked (no DNS rebinding)
def test_4_5_connection_pinned_to_checked_address(app):
seen = []
def handler(request):
seen.append((str(request.url), request.headers['host'],
request.extensions.get('sni_hostname')))
return httpx.Response(200, text='ok')
with app.app_context(), \
patch("starpunk.webmention_ssrf.socket.getaddrinfo", return_value=PUBLIC) as resolve, \
patch("starpunk.webmention_ssrf.TRANSPORT", httpx.MockTransport(handler)):
webmention_ssrf.safe_get('https://site.example:8443/p?q=1', {})
webmention_ssrf.safe_post('http://site.example/wm', {'source': 's'}, {})
assert seen == [
('https://93.184.216.34:8443/p?q=1', 'site.example:8443', 'site.example'),
('http://93.184.216.34/wm', 'site.example', None),
]
assert resolve.call_count == 2 # one lookup per request, then pinned
def test_4_5_ipv4_preferred(app):
with app.app_context(), patch("starpunk.webmention_ssrf.socket.getaddrinfo",
return_value=[(10, 1, 6, '', ('2606:2800::1', 0, 0, 0))] + PUBLIC):
assert webmention_ssrf.safe_resolve('x.example') == '93.184.216.34'
# 3.1.4 A change in the responses a note shows is an update of the note
def _note_with_sent_mention(app):
from starpunk.notes import create_note
with app.test_request_context():
note = create_note("See [a](https://a.example/x)", published=True)
_sent(app)
db = sqlite3.connect(app.config['DATABASE_PATH'])
db.execute("INSERT INTO webmentions (source_url, target_url, target_note_id) VALUES (?, ?, ?)",
('https://s.example/reply', f'http://localhost:5000/note/{note.slug}', note.id))
db.commit()
return note, db
def test_3_1_4_new_response_resends(app, worker):
note, db = _note_with_sent_mention(app)
with patch("starpunk.webmention_worker.process_webmention", return_value=VALID):
worker._validate_webmention(1, 'https://s.example/reply', 'x', note.id)
assert [(t, s) for t, s, _ in _queue(app)] == [('https://a.example/x', 'pending')]
def test_3_1_4_unchanged_response_does_not_resend(app, worker):
note, db = _note_with_sent_mention(app)
with patch("starpunk.webmention_worker.process_webmention", return_value=VALID):
worker._validate_webmention(1, 'https://s.example/reply', 'x', note.id)
_sent(app)
db.execute("UPDATE webmentions SET validation_status = 'pending'") # re-sent by source
db.commit()
worker._validate_webmention(1, 'https://s.example/reply', 'x', note.id)
assert [(t, s) for t, s, _ in _queue(app)] == [('https://a.example/x', 'sent')]
def test_3_1_4_moderation_change_resends(app, worker):
from starpunk.webmention_queries import update_webmention_status
note, db = _note_with_sent_mention(app)
with patch("starpunk.webmention_worker.process_webmention", return_value=VALID):
worker._validate_webmention(1, 'https://s.example/reply', 'x', note.id)
_sent(app)
with app.app_context():
update_webmention_status([1], 'approved') # already approved: no change
assert _queue(app)[0][1] == 'sent'
with app.app_context():
update_webmention_status([1], 'hidden')
assert _queue(app)[0][1] == 'pending'
+9 -7
View File
@@ -37,16 +37,18 @@ def test_worker_sends_queued_webmention(app):
conn.commit() conn.commit()
conn.close() conn.close()
def target_site(request): posts = []
return httpx.Response(200, headers={"Link": '</wm>; rel="webmention"'})
accepted = MagicMock(status_code=202, text="") def target_site(request):
if request.method == "POST":
posts.append((request.headers["host"], request.url.path, request.content))
return httpx.Response(202)
return httpx.Response(200, headers={"Link": '</wm>; rel="webmention"'})
worker = WebmentionWorker(app, db_path=db_path, site_url=app.config["SITE_URL"], worker = WebmentionWorker(app, db_path=db_path, site_url=app.config["SITE_URL"],
poll_interval=0.05) poll_interval=0.05)
with patch("starpunk.webmention_ssrf.socket.getaddrinfo", return_value=PUBLIC), \ with patch("starpunk.webmention_ssrf.socket.getaddrinfo", return_value=PUBLIC), \
patch("starpunk.webmention_ssrf.TRANSPORT", httpx.MockTransport(target_site)), \ patch("starpunk.webmention_ssrf.TRANSPORT", httpx.MockTransport(target_site)):
patch("starpunk.webmention_worker.httpx.post", return_value=accepted) as post:
worker.start() worker.start()
try: try:
def status(): def status():
@@ -56,8 +58,8 @@ def test_worker_sends_queued_webmention(app):
return row return row
assert _wait_for(lambda: status()[0] != "pending"), status() assert _wait_for(lambda: status()[0] != "pending"), status()
assert status() == ("sent", None) assert status() == ("sent", None)
post.assert_called_once() assert [(h, p) for h, p, _ in posts] == [("example.org", "/wm")]
assert post.call_args.args[0] == "https://example.org/wm" assert b"source=" in posts[0][2] and b"target=" in posts[0][2]
finally: finally:
worker.stop() worker.stop()
worker.join(timeout=2) worker.join(timeout=2)