Files
StarPunk/starpunk/webmention_ssrf.py
T
PhilandClaude Opus 5.5 b520b8b4d3 feat: WebSub with a built-in hub (v2.10.0)
ADR-071. Topics (/, /feed.atom, /feed.rss, /feed.json, /likes, /tag/<name>)
advertise the hub and themselves in Link headers and embedded links. Note
changes bump a content version (fresh feeds in every process) and queue
deliveries. /websub accepts subscriptions for these topics only; the worker
verifies intent, delivers signed content with retries, and prunes expired
leases, all through the SSRF-safe pinned fetcher. WEBSUB_HUB_URL switches
to an external hub (publish pings). Also: the memory monitor no longer
stops after the first request; RSS self link is /feed.rss.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-03 10:33:41 -07:00

267 lines
8.8 KiB
Python

"""
SSRF protection for webmention fetches
Per ADR-063 Decision 9 and Webmention sections 4.2, 4.3 and 4.5:
- Every address a hostname resolves to must be public
- Redirects are followed by hand so each hop is checked
- Each request connects to the address that was checked (no second DNS
lookup that a rebinding DNS server could answer differently)
- Redirect count, response size and total time are limited
- DEV_MODE exception for localhost testing
References:
- ADR-063: Webmention Implementation
- W3C Webmention: https://www.w3.org/TR/webmention/#security-considerations
"""
import ipaddress
import socket
import time
from dataclasses import dataclass
from typing import Optional
from urllib.parse import urljoin, urlparse
import httpx
from flask import current_app
MAX_REDIRECTS = 10
MAX_BYTES = 1024 * 1024
TIMEOUT = 10
# Tests can set an httpx transport here (e.g. httpx.MockTransport)
TRANSPORT = None
CGNAT = ipaddress.ip_network('100.64.0.0/10')
THIS_NETWORK = ipaddress.ip_network('0.0.0.0/8')
class SSRFBlockedError(Exception):
"""Raised when SSRF protection blocks a request"""
pass
class FetchTooLargeError(Exception):
"""Raised when a response is larger than the limit"""
pass
def user_agent(site_url: str) -> str:
"""User-Agent for webmention requests (spec 3.1.2 suggests including 'Webmention')"""
from starpunk import __version__
return f"StarPunk/{__version__} Webmention (+{site_url})"
def is_private_ip(ip: str) -> bool:
"""
Check if an IP address must not be fetched.
Blocks private, loopback, link-local, multicast, reserved, unspecified,
carrier-grade NAT and IPv4-mapped forms of these.
Args:
ip: IP address string (IPv4 or IPv6)
Returns:
True if IP is blocked, False otherwise
"""
try:
addr = ipaddress.ip_address(ip.split('%')[0])
except ValueError:
return True
if isinstance(addr, ipaddress.IPv6Address) and addr.ipv4_mapped:
addr = addr.ipv4_mapped
if isinstance(addr, ipaddress.IPv4Address) and (addr in CGNAT or addr in THIS_NETWORK):
return True
return (
addr.is_private
or addr.is_loopback
or addr.is_link_local
or addr.is_multicast
or addr.is_reserved
or addr.is_unspecified
)
def safe_resolve(hostname: str) -> str:
"""
Resolve hostname and check every address is public.
Per background-worker.md: DEV_MODE exception for localhost testing.
Returns:
Address to connect to (IPv4 preferred)
Raises:
SSRFBlockedError: If any address is blocked
socket.gaierror: If hostname cannot be resolved
"""
if current_app.config.get('DEV_MODE') and hostname in ('localhost', '127.0.0.1', '::1'):
return '127.0.0.1'
addresses = sorted({info[4][0] for info in socket.getaddrinfo(hostname, None)},
key=lambda ip: (':' in ip, ip))
for ip in addresses:
if is_private_ip(ip):
raise SSRFBlockedError(f"Blocked request to private IP: {ip}")
return addresses[0]
def validate_url_for_fetch(url: str) -> str:
"""
Validate URL is safe to fetch (http(s), public addresses only).
Returns:
The checked IP address to connect to
Raises:
SSRFBlockedError: If URL points to a blocked address
ValueError: If URL is malformed or its host does not resolve
"""
parsed = urlparse(url)
if parsed.scheme not in ('http', 'https'):
raise ValueError(f"Unsupported URL scheme: {parsed.scheme}")
if not parsed.hostname:
raise ValueError("URL must include a hostname")
try:
return safe_resolve(parsed.hostname)
except socket.gaierror as e:
raise ValueError(f"Could not resolve {parsed.hostname}: {e}")
def _pinned_request(client: httpx.Client, method: str, url: str, headers: dict,
**kwargs) -> httpx.Request:
"""
Build a request that connects to the checked address for url's host.
The URL's host is replaced by the IP that passed the SSRF check; the
real host goes in the Host header and, for https, in SNI, so virtual
hosting and certificate verification still use the hostname.
"""
ip = validate_url_for_fetch(url)
parsed = urlparse(url)
ip_host = f"[{ip}]" if ':' in ip else ip
port = f":{parsed.port}" if parsed.port else ''
pinned_url = parsed._replace(netloc=f"{ip_host}{port}").geturl()
host_header = f"{parsed.hostname}{port}"
extensions = {'sni_hostname': parsed.hostname} if parsed.scheme == 'https' else {}
return client.build_request(
method, pinned_url, headers={**headers, 'Host': host_header},
extensions=extensions, **kwargs,
)
@dataclass
class FetchResult:
"""A fetched response, after redirects"""
url: str
status_code: int
headers: httpx.Headers
content: bytes
encoding: str
@property
def text(self) -> str:
try:
return self.content.decode(self.encoding, errors='replace')
except LookupError:
return self.content.decode('utf-8', errors='replace')
def safe_post(url: str, data: Optional[dict] = None, headers: Optional[dict] = None,
timeout: float = TIMEOUT, content: Optional[bytes] = None,
max_bytes: int = 64 * 1024) -> FetchResult:
"""
POST form data (or a raw body) to a checked, pinned address.
Redirects are not followed. Only the first max_bytes of the response are
read (callers need the status and maybe an error message).
Raises:
SSRFBlockedError, ValueError: URL is not allowed
httpx.HTTPError: Network errors
"""
kwargs = {'content': content} if content is not None else {'data': data}
with httpx.Client(follow_redirects=False, transport=TRANSPORT, timeout=timeout) as client:
request = _pinned_request(client, 'POST', url, headers or {}, **kwargs)
response = client.send(request, stream=True)
try:
body = b''
for chunk in response.iter_bytes(chunk_size=8192):
body += chunk
if len(body) >= max_bytes:
body = body[:max_bytes]
break
return FetchResult(url=url, status_code=response.status_code, headers=response.headers,
content=body, encoding=response.charset_encoding or 'utf-8')
finally:
response.close()
def safe_get(url: str, headers: dict, max_bytes: int = MAX_BYTES,
max_redirects: int = MAX_REDIRECTS, timeout: float = TIMEOUT) -> FetchResult:
"""
GET a URL, following redirects by hand and checking each hop.
Args:
url: URL to fetch
headers: Request headers
max_bytes: Response body limit
max_redirects: Redirect limit
timeout: Total time limit in seconds
Returns:
FetchResult for the final response (any status code)
Raises:
SSRFBlockedError, ValueError: URL or a redirect target is not allowed
FetchTooLargeError: Body exceeds max_bytes
httpx.HTTPError: Network errors, too many redirects, time limit
"""
deadline = time.monotonic() + timeout
with httpx.Client(follow_redirects=False, transport=TRANSPORT) as client:
for _ in range(max_redirects + 1):
remaining = deadline - time.monotonic()
if remaining <= 0:
raise httpx.TimeoutException("Fetch exceeded time limit")
request = _pinned_request(client, 'GET', url, headers, timeout=remaining)
response = client.send(request, stream=True)
try:
location = response.headers.get('Location')
if response.is_redirect and location:
url = urljoin(url, location)
continue
content_length = response.headers.get('Content-Length', '')
if content_length.isdigit() and int(content_length) > max_bytes:
raise FetchTooLargeError(f"Response exceeds size limit ({max_bytes} bytes)")
content = b''
for chunk in response.iter_bytes(chunk_size=8192):
content += chunk
if len(content) > max_bytes:
raise FetchTooLargeError(f"Response exceeds size limit ({max_bytes} bytes)")
if time.monotonic() > deadline:
raise httpx.TimeoutException("Fetch exceeded time limit")
return FetchResult(
url=url,
status_code=response.status_code,
headers=response.headers,
content=content,
encoding=response.charset_encoding or 'utf-8',
)
finally:
response.close()
raise httpx.TooManyRedirects(f"More than {max_redirects} redirects")