Files
StarPunk/starpunk/webmention_ssrf.py
T
PhilandClaude Opus 5.5 0640730416 feat: Pin webmention connections; re-send when responses change (v2.7.2)
- Connect to the address that passed the SSRF check (Host header and SNI
  carry the hostname, so TLS still verifies it), closing the DNS rebinding
  gap (spec 4.5). The webmention POST uses the same path.
- Treat a change in the responses a note shows as an update and re-send its
  webmentions (spec 3.1.4), from validation and from admin moderation.
  Unchanged re-validations send nothing, so there is no ping-pong.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-03 09:01:38 -07:00

250 lines
8.0 KiB
Python

"""
SSRF protection for webmention fetches
Per ADR-063 Decision 9 and Webmention sections 4.2, 4.3 and 4.5:
- Every address a hostname resolves to must be public
- Redirects are followed by hand so each hop is checked
- Each request connects to the address that was checked (no second DNS
lookup that a rebinding DNS server could answer differently)
- Redirect count, response size and total time are limited
- DEV_MODE exception for localhost testing
References:
- ADR-063: Webmention Implementation
- W3C Webmention: https://www.w3.org/TR/webmention/#security-considerations
"""
import ipaddress
import socket
import time
from dataclasses import dataclass
from urllib.parse import urljoin, urlparse
import httpx
from flask import current_app
MAX_REDIRECTS = 10
MAX_BYTES = 1024 * 1024
TIMEOUT = 10
# Tests can set an httpx transport here (e.g. httpx.MockTransport)
TRANSPORT = None
CGNAT = ipaddress.ip_network('100.64.0.0/10')
THIS_NETWORK = ipaddress.ip_network('0.0.0.0/8')
class SSRFBlockedError(Exception):
"""Raised when SSRF protection blocks a request"""
pass
class FetchTooLargeError(Exception):
"""Raised when a response is larger than the limit"""
pass
def user_agent(site_url: str) -> str:
"""User-Agent for webmention requests (spec 3.1.2 suggests including 'Webmention')"""
from starpunk import __version__
return f"StarPunk/{__version__} Webmention (+{site_url})"
def is_private_ip(ip: str) -> bool:
"""
Check if an IP address must not be fetched.
Blocks private, loopback, link-local, multicast, reserved, unspecified,
carrier-grade NAT and IPv4-mapped forms of these.
Args:
ip: IP address string (IPv4 or IPv6)
Returns:
True if IP is blocked, False otherwise
"""
try:
addr = ipaddress.ip_address(ip.split('%')[0])
except ValueError:
return True
if isinstance(addr, ipaddress.IPv6Address) and addr.ipv4_mapped:
addr = addr.ipv4_mapped
if isinstance(addr, ipaddress.IPv4Address) and (addr in CGNAT or addr in THIS_NETWORK):
return True
return (
addr.is_private
or addr.is_loopback
or addr.is_link_local
or addr.is_multicast
or addr.is_reserved
or addr.is_unspecified
)
def safe_resolve(hostname: str) -> str:
"""
Resolve hostname and check every address is public.
Per background-worker.md: DEV_MODE exception for localhost testing.
Returns:
Address to connect to (IPv4 preferred)
Raises:
SSRFBlockedError: If any address is blocked
socket.gaierror: If hostname cannot be resolved
"""
if current_app.config.get('DEV_MODE') and hostname in ('localhost', '127.0.0.1', '::1'):
return '127.0.0.1'
addresses = sorted({info[4][0] for info in socket.getaddrinfo(hostname, None)},
key=lambda ip: (':' in ip, ip))
for ip in addresses:
if is_private_ip(ip):
raise SSRFBlockedError(f"Blocked request to private IP: {ip}")
return addresses[0]
def validate_url_for_fetch(url: str) -> str:
"""
Validate URL is safe to fetch (http(s), public addresses only).
Returns:
The checked IP address to connect to
Raises:
SSRFBlockedError: If URL points to a blocked address
ValueError: If URL is malformed or its host does not resolve
"""
parsed = urlparse(url)
if parsed.scheme not in ('http', 'https'):
raise ValueError(f"Unsupported URL scheme: {parsed.scheme}")
if not parsed.hostname:
raise ValueError("URL must include a hostname")
try:
return safe_resolve(parsed.hostname)
except socket.gaierror as e:
raise ValueError(f"Could not resolve {parsed.hostname}: {e}")
def _pinned_request(client: httpx.Client, method: str, url: str, headers: dict,
**kwargs) -> httpx.Request:
"""
Build a request that connects to the checked address for url's host.
The URL's host is replaced by the IP that passed the SSRF check; the
real host goes in the Host header and, for https, in SNI, so virtual
hosting and certificate verification still use the hostname.
"""
ip = validate_url_for_fetch(url)
parsed = urlparse(url)
ip_host = f"[{ip}]" if ':' in ip else ip
port = f":{parsed.port}" if parsed.port else ''
pinned_url = parsed._replace(netloc=f"{ip_host}{port}").geturl()
host_header = f"{parsed.hostname}{port}"
extensions = {'sni_hostname': parsed.hostname} if parsed.scheme == 'https' else {}
return client.build_request(
method, pinned_url, headers={**headers, 'Host': host_header},
extensions=extensions, **kwargs,
)
def safe_post(url: str, data: dict, headers: dict, timeout: float = TIMEOUT) -> httpx.Response:
"""
POST form data to a checked, pinned address. Redirects are not followed.
Raises:
SSRFBlockedError, ValueError: URL is not allowed
httpx.HTTPError: Network errors
"""
with httpx.Client(follow_redirects=False, transport=TRANSPORT, timeout=timeout) as client:
response = client.send(_pinned_request(client, 'POST', url, headers, data=data))
response.read()
return response
@dataclass
class FetchResult:
"""A fetched response, after redirects"""
url: str
status_code: int
headers: httpx.Headers
content: bytes
encoding: str
@property
def text(self) -> str:
try:
return self.content.decode(self.encoding, errors='replace')
except LookupError:
return self.content.decode('utf-8', errors='replace')
def safe_get(url: str, headers: dict, max_bytes: int = MAX_BYTES,
max_redirects: int = MAX_REDIRECTS, timeout: float = TIMEOUT) -> FetchResult:
"""
GET a URL, following redirects by hand and checking each hop.
Args:
url: URL to fetch
headers: Request headers
max_bytes: Response body limit
max_redirects: Redirect limit
timeout: Total time limit in seconds
Returns:
FetchResult for the final response (any status code)
Raises:
SSRFBlockedError, ValueError: URL or a redirect target is not allowed
FetchTooLargeError: Body exceeds max_bytes
httpx.HTTPError: Network errors, too many redirects, time limit
"""
deadline = time.monotonic() + timeout
with httpx.Client(follow_redirects=False, transport=TRANSPORT) as client:
for _ in range(max_redirects + 1):
remaining = deadline - time.monotonic()
if remaining <= 0:
raise httpx.TimeoutException("Fetch exceeded time limit")
request = _pinned_request(client, 'GET', url, headers, timeout=remaining)
response = client.send(request, stream=True)
try:
location = response.headers.get('Location')
if response.is_redirect and location:
url = urljoin(url, location)
continue
content_length = response.headers.get('Content-Length', '')
if content_length.isdigit() and int(content_length) > max_bytes:
raise FetchTooLargeError(f"Response exceeds size limit ({max_bytes} bytes)")
content = b''
for chunk in response.iter_bytes(chunk_size=8192):
content += chunk
if len(content) > max_bytes:
raise FetchTooLargeError(f"Response exceeds size limit ({max_bytes} bytes)")
if time.monotonic() > deadline:
raise httpx.TimeoutException("Fetch exceeded time limit")
return FetchResult(
url=url,
status_code=response.status_code,
headers=response.headers,
content=content,
encoding=response.charset_encoding or 'utf-8',
)
finally:
response.close()
raise httpx.TooManyRedirects(f"More than {max_redirects} redirects")