- Connect to the address that passed the SSRF check (Host header and SNI carry the hostname, so TLS still verifies it), closing the DNS rebinding gap (spec 4.5). The webmention POST uses the same path. - Treat a change in the responses a note shows as an update and re-send its webmentions (spec 3.1.4), from validation and from admin moderation. Unchanged re-validations send nothing, so there is no ping-pong. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
250 lines
8.0 KiB
Python
250 lines
8.0 KiB
Python
"""
|
|
SSRF protection for webmention fetches
|
|
|
|
Per ADR-063 Decision 9 and Webmention sections 4.2, 4.3 and 4.5:
|
|
- Every address a hostname resolves to must be public
|
|
- Redirects are followed by hand so each hop is checked
|
|
- Each request connects to the address that was checked (no second DNS
|
|
lookup that a rebinding DNS server could answer differently)
|
|
- Redirect count, response size and total time are limited
|
|
- DEV_MODE exception for localhost testing
|
|
|
|
References:
|
|
- ADR-063: Webmention Implementation
|
|
- W3C Webmention: https://www.w3.org/TR/webmention/#security-considerations
|
|
"""
|
|
|
|
import ipaddress
|
|
import socket
|
|
import time
|
|
from dataclasses import dataclass
|
|
from urllib.parse import urljoin, urlparse
|
|
|
|
import httpx
|
|
from flask import current_app
|
|
|
|
MAX_REDIRECTS = 10
|
|
MAX_BYTES = 1024 * 1024
|
|
TIMEOUT = 10
|
|
|
|
# Tests can set an httpx transport here (e.g. httpx.MockTransport)
|
|
TRANSPORT = None
|
|
|
|
CGNAT = ipaddress.ip_network('100.64.0.0/10')
|
|
THIS_NETWORK = ipaddress.ip_network('0.0.0.0/8')
|
|
|
|
|
|
class SSRFBlockedError(Exception):
|
|
"""Raised when SSRF protection blocks a request"""
|
|
pass
|
|
|
|
|
|
class FetchTooLargeError(Exception):
|
|
"""Raised when a response is larger than the limit"""
|
|
pass
|
|
|
|
|
|
def user_agent(site_url: str) -> str:
|
|
"""User-Agent for webmention requests (spec 3.1.2 suggests including 'Webmention')"""
|
|
from starpunk import __version__
|
|
return f"StarPunk/{__version__} Webmention (+{site_url})"
|
|
|
|
|
|
def is_private_ip(ip: str) -> bool:
|
|
"""
|
|
Check if an IP address must not be fetched.
|
|
|
|
Blocks private, loopback, link-local, multicast, reserved, unspecified,
|
|
carrier-grade NAT and IPv4-mapped forms of these.
|
|
|
|
Args:
|
|
ip: IP address string (IPv4 or IPv6)
|
|
|
|
Returns:
|
|
True if IP is blocked, False otherwise
|
|
"""
|
|
try:
|
|
addr = ipaddress.ip_address(ip.split('%')[0])
|
|
except ValueError:
|
|
return True
|
|
|
|
if isinstance(addr, ipaddress.IPv6Address) and addr.ipv4_mapped:
|
|
addr = addr.ipv4_mapped
|
|
|
|
if isinstance(addr, ipaddress.IPv4Address) and (addr in CGNAT or addr in THIS_NETWORK):
|
|
return True
|
|
|
|
return (
|
|
addr.is_private
|
|
or addr.is_loopback
|
|
or addr.is_link_local
|
|
or addr.is_multicast
|
|
or addr.is_reserved
|
|
or addr.is_unspecified
|
|
)
|
|
|
|
|
|
def safe_resolve(hostname: str) -> str:
|
|
"""
|
|
Resolve hostname and check every address is public.
|
|
|
|
Per background-worker.md: DEV_MODE exception for localhost testing.
|
|
|
|
Returns:
|
|
Address to connect to (IPv4 preferred)
|
|
|
|
Raises:
|
|
SSRFBlockedError: If any address is blocked
|
|
socket.gaierror: If hostname cannot be resolved
|
|
"""
|
|
if current_app.config.get('DEV_MODE') and hostname in ('localhost', '127.0.0.1', '::1'):
|
|
return '127.0.0.1'
|
|
|
|
addresses = sorted({info[4][0] for info in socket.getaddrinfo(hostname, None)},
|
|
key=lambda ip: (':' in ip, ip))
|
|
for ip in addresses:
|
|
if is_private_ip(ip):
|
|
raise SSRFBlockedError(f"Blocked request to private IP: {ip}")
|
|
|
|
return addresses[0]
|
|
|
|
|
|
def validate_url_for_fetch(url: str) -> str:
|
|
"""
|
|
Validate URL is safe to fetch (http(s), public addresses only).
|
|
|
|
Returns:
|
|
The checked IP address to connect to
|
|
|
|
Raises:
|
|
SSRFBlockedError: If URL points to a blocked address
|
|
ValueError: If URL is malformed or its host does not resolve
|
|
"""
|
|
parsed = urlparse(url)
|
|
|
|
if parsed.scheme not in ('http', 'https'):
|
|
raise ValueError(f"Unsupported URL scheme: {parsed.scheme}")
|
|
|
|
if not parsed.hostname:
|
|
raise ValueError("URL must include a hostname")
|
|
|
|
try:
|
|
return safe_resolve(parsed.hostname)
|
|
except socket.gaierror as e:
|
|
raise ValueError(f"Could not resolve {parsed.hostname}: {e}")
|
|
|
|
|
|
def _pinned_request(client: httpx.Client, method: str, url: str, headers: dict,
|
|
**kwargs) -> httpx.Request:
|
|
"""
|
|
Build a request that connects to the checked address for url's host.
|
|
|
|
The URL's host is replaced by the IP that passed the SSRF check; the
|
|
real host goes in the Host header and, for https, in SNI, so virtual
|
|
hosting and certificate verification still use the hostname.
|
|
"""
|
|
ip = validate_url_for_fetch(url)
|
|
parsed = urlparse(url)
|
|
ip_host = f"[{ip}]" if ':' in ip else ip
|
|
port = f":{parsed.port}" if parsed.port else ''
|
|
pinned_url = parsed._replace(netloc=f"{ip_host}{port}").geturl()
|
|
host_header = f"{parsed.hostname}{port}"
|
|
|
|
extensions = {'sni_hostname': parsed.hostname} if parsed.scheme == 'https' else {}
|
|
return client.build_request(
|
|
method, pinned_url, headers={**headers, 'Host': host_header},
|
|
extensions=extensions, **kwargs,
|
|
)
|
|
|
|
|
|
def safe_post(url: str, data: dict, headers: dict, timeout: float = TIMEOUT) -> httpx.Response:
|
|
"""
|
|
POST form data to a checked, pinned address. Redirects are not followed.
|
|
|
|
Raises:
|
|
SSRFBlockedError, ValueError: URL is not allowed
|
|
httpx.HTTPError: Network errors
|
|
"""
|
|
with httpx.Client(follow_redirects=False, transport=TRANSPORT, timeout=timeout) as client:
|
|
response = client.send(_pinned_request(client, 'POST', url, headers, data=data))
|
|
response.read()
|
|
return response
|
|
|
|
|
|
@dataclass
|
|
class FetchResult:
|
|
"""A fetched response, after redirects"""
|
|
url: str
|
|
status_code: int
|
|
headers: httpx.Headers
|
|
content: bytes
|
|
encoding: str
|
|
|
|
@property
|
|
def text(self) -> str:
|
|
try:
|
|
return self.content.decode(self.encoding, errors='replace')
|
|
except LookupError:
|
|
return self.content.decode('utf-8', errors='replace')
|
|
|
|
|
|
def safe_get(url: str, headers: dict, max_bytes: int = MAX_BYTES,
|
|
max_redirects: int = MAX_REDIRECTS, timeout: float = TIMEOUT) -> FetchResult:
|
|
"""
|
|
GET a URL, following redirects by hand and checking each hop.
|
|
|
|
Args:
|
|
url: URL to fetch
|
|
headers: Request headers
|
|
max_bytes: Response body limit
|
|
max_redirects: Redirect limit
|
|
timeout: Total time limit in seconds
|
|
|
|
Returns:
|
|
FetchResult for the final response (any status code)
|
|
|
|
Raises:
|
|
SSRFBlockedError, ValueError: URL or a redirect target is not allowed
|
|
FetchTooLargeError: Body exceeds max_bytes
|
|
httpx.HTTPError: Network errors, too many redirects, time limit
|
|
"""
|
|
deadline = time.monotonic() + timeout
|
|
|
|
with httpx.Client(follow_redirects=False, transport=TRANSPORT) as client:
|
|
for _ in range(max_redirects + 1):
|
|
remaining = deadline - time.monotonic()
|
|
if remaining <= 0:
|
|
raise httpx.TimeoutException("Fetch exceeded time limit")
|
|
|
|
request = _pinned_request(client, 'GET', url, headers, timeout=remaining)
|
|
response = client.send(request, stream=True)
|
|
try:
|
|
location = response.headers.get('Location')
|
|
if response.is_redirect and location:
|
|
url = urljoin(url, location)
|
|
continue
|
|
|
|
content_length = response.headers.get('Content-Length', '')
|
|
if content_length.isdigit() and int(content_length) > max_bytes:
|
|
raise FetchTooLargeError(f"Response exceeds size limit ({max_bytes} bytes)")
|
|
|
|
content = b''
|
|
for chunk in response.iter_bytes(chunk_size=8192):
|
|
content += chunk
|
|
if len(content) > max_bytes:
|
|
raise FetchTooLargeError(f"Response exceeds size limit ({max_bytes} bytes)")
|
|
if time.monotonic() > deadline:
|
|
raise httpx.TimeoutException("Fetch exceeded time limit")
|
|
|
|
return FetchResult(
|
|
url=url,
|
|
status_code=response.status_code,
|
|
headers=response.headers,
|
|
content=content,
|
|
encoding=response.charset_encoding or 'utf-8',
|
|
)
|
|
finally:
|
|
response.close()
|
|
|
|
raise httpx.TooManyRedirects(f"More than {max_redirects} redirects")
|