- Per-item error handling in the worker; hard time limits on requests - Notes with photos announced after attach; feed cache/ETag follow content_version - Keep failing subscriptions until lease end; retry DNS and render errors; hub.mode=denied for gone topics - Spec 5.1: decode unreserved chars, echo requested topic, ignore lease and secret on unsubscribe, case-insensitive Content-Type, refuse empty secret - Limits count pending requests per callback address; callback validation; request_id guards the verification race (migration 017) - Render once per topic per pass; likes only push /likes; Vary: Accept Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
312 lines
10 KiB
Python
312 lines
10 KiB
Python
"""
|
|
SSRF protection for webmention fetches
|
|
|
|
Per ADR-063 Decision 9 and Webmention sections 4.2, 4.3 and 4.5:
|
|
- Every address a hostname resolves to must be public
|
|
- Redirects are followed by hand so each hop is checked
|
|
- Each request connects to the address that was checked (no second DNS
|
|
lookup that a rebinding DNS server could answer differently)
|
|
- Redirect count, response size and total time are limited
|
|
- DEV_MODE exception for localhost testing
|
|
|
|
References:
|
|
- ADR-063: Webmention Implementation
|
|
- W3C Webmention: https://www.w3.org/TR/webmention/#security-considerations
|
|
"""
|
|
|
|
import ipaddress
|
|
import socket
|
|
import threading
|
|
import time
|
|
from dataclasses import dataclass
|
|
from typing import Optional
|
|
from urllib.parse import urljoin, urlparse
|
|
|
|
import httpx
|
|
from flask import current_app
|
|
|
|
MAX_REDIRECTS = 10
|
|
MAX_BYTES = 1024 * 1024
|
|
TIMEOUT = 10
|
|
|
|
# Tests can set an httpx transport here (e.g. httpx.MockTransport)
|
|
TRANSPORT = None
|
|
|
|
CGNAT = ipaddress.ip_network('100.64.0.0/10')
|
|
THIS_NETWORK = ipaddress.ip_network('0.0.0.0/8')
|
|
|
|
|
|
class SSRFBlockedError(Exception):
|
|
"""Raised when SSRF protection blocks a request"""
|
|
pass
|
|
|
|
|
|
class ResolutionError(ValueError):
|
|
"""The host name could not be resolved (may be temporary; v2.10.1)"""
|
|
pass
|
|
|
|
|
|
class FetchTooLargeError(Exception):
|
|
"""Raised when a response is larger than the limit"""
|
|
pass
|
|
|
|
|
|
def user_agent(site_url: str) -> str:
|
|
"""User-Agent for webmention requests (spec 3.1.2 suggests including 'Webmention')"""
|
|
from starpunk import __version__
|
|
return f"StarPunk/{__version__} Webmention (+{site_url})"
|
|
|
|
|
|
def is_private_ip(ip: str) -> bool:
|
|
"""
|
|
Check if an IP address must not be fetched.
|
|
|
|
Blocks private, loopback, link-local, multicast, reserved, unspecified,
|
|
carrier-grade NAT and IPv4-mapped forms of these.
|
|
|
|
Args:
|
|
ip: IP address string (IPv4 or IPv6)
|
|
|
|
Returns:
|
|
True if IP is blocked, False otherwise
|
|
"""
|
|
try:
|
|
addr = ipaddress.ip_address(ip.split('%')[0])
|
|
except ValueError:
|
|
return True
|
|
|
|
if isinstance(addr, ipaddress.IPv6Address) and addr.ipv4_mapped:
|
|
addr = addr.ipv4_mapped
|
|
|
|
if isinstance(addr, ipaddress.IPv4Address) and (addr in CGNAT or addr in THIS_NETWORK):
|
|
return True
|
|
|
|
return (
|
|
addr.is_private
|
|
or addr.is_loopback
|
|
or addr.is_link_local
|
|
or addr.is_multicast
|
|
or addr.is_reserved
|
|
or addr.is_unspecified
|
|
)
|
|
|
|
|
|
def safe_resolve(hostname: str) -> str:
|
|
"""
|
|
Resolve hostname and check every address is public.
|
|
|
|
Per background-worker.md: DEV_MODE exception for localhost testing.
|
|
|
|
Returns:
|
|
Address to connect to (IPv4 preferred)
|
|
|
|
Raises:
|
|
SSRFBlockedError: If any address is blocked
|
|
socket.gaierror: If hostname cannot be resolved
|
|
"""
|
|
if current_app.config.get('DEV_MODE') and hostname in ('localhost', '127.0.0.1', '::1'):
|
|
return '127.0.0.1'
|
|
|
|
addresses = sorted({info[4][0] for info in socket.getaddrinfo(hostname, None)},
|
|
key=lambda ip: (':' in ip, ip))
|
|
for ip in addresses:
|
|
if is_private_ip(ip):
|
|
raise SSRFBlockedError(f"Blocked request to private IP: {ip}")
|
|
|
|
return addresses[0]
|
|
|
|
|
|
def validate_url_for_fetch(url: str) -> str:
|
|
"""
|
|
Validate URL is safe to fetch (http(s), public addresses only).
|
|
|
|
Returns:
|
|
The checked IP address to connect to
|
|
|
|
Raises:
|
|
SSRFBlockedError: If URL points to a blocked address
|
|
ValueError: If URL is malformed or its host does not resolve
|
|
"""
|
|
parsed = urlparse(url)
|
|
|
|
if parsed.scheme not in ('http', 'https'):
|
|
raise ValueError(f"Unsupported URL scheme: {parsed.scheme}")
|
|
|
|
if not parsed.hostname:
|
|
raise ValueError("URL must include a hostname")
|
|
|
|
try:
|
|
return safe_resolve(parsed.hostname)
|
|
except socket.gaierror as e:
|
|
raise ResolutionError(f"Could not resolve {parsed.hostname}: {e}")
|
|
|
|
|
|
def _pinned_request(client: httpx.Client, method: str, url: str, headers: dict,
|
|
**kwargs) -> httpx.Request:
|
|
"""
|
|
Build a request that connects to the checked address for url's host.
|
|
|
|
The URL's host is replaced by the IP that passed the SSRF check; the
|
|
real host goes in the Host header and, for https, in SNI, so virtual
|
|
hosting and certificate verification still use the hostname.
|
|
"""
|
|
ip = validate_url_for_fetch(url)
|
|
parsed = urlparse(url)
|
|
ip_host = f"[{ip}]" if ':' in ip else ip
|
|
port = f":{parsed.port}" if parsed.port else ''
|
|
pinned_url = parsed._replace(netloc=f"{ip_host}{port}").geturl()
|
|
host_header = f"{parsed.hostname}{port}"
|
|
|
|
extensions = {'sni_hostname': parsed.hostname} if parsed.scheme == 'https' else {}
|
|
return client.build_request(
|
|
method, pinned_url, headers={**headers, 'Host': host_header},
|
|
extensions=extensions, **kwargs,
|
|
)
|
|
|
|
|
|
class _Watchdog:
|
|
"""
|
|
Close a client when its deadline passes (v2.10.1)
|
|
|
|
httpx timeouts apply per read, so a server sending one byte at a time
|
|
could hold a request for hours; closing the client ends it.
|
|
"""
|
|
|
|
def __init__(self, client: httpx.Client, seconds: float):
|
|
self.fired = False
|
|
|
|
def fire():
|
|
self.fired = True
|
|
try:
|
|
client.close()
|
|
except Exception:
|
|
pass
|
|
|
|
self._timer = threading.Timer(max(seconds, 0.01), fire)
|
|
self._timer.daemon = True
|
|
|
|
def __enter__(self):
|
|
self._timer.start()
|
|
return self
|
|
|
|
def __exit__(self, exc_type, exc, tb):
|
|
self._timer.cancel()
|
|
if self.fired and exc_type is not None and not isinstance(exc, httpx.TimeoutException):
|
|
raise httpx.TimeoutException("Exceeded time limit") from exc
|
|
return False
|
|
|
|
|
|
@dataclass
|
|
class FetchResult:
|
|
"""A fetched response, after redirects"""
|
|
url: str
|
|
status_code: int
|
|
headers: httpx.Headers
|
|
content: bytes
|
|
encoding: str
|
|
|
|
@property
|
|
def text(self) -> str:
|
|
try:
|
|
return self.content.decode(self.encoding, errors='replace')
|
|
except LookupError:
|
|
return self.content.decode('utf-8', errors='replace')
|
|
|
|
|
|
def safe_post(url: str, data: Optional[dict] = None, headers: Optional[dict] = None,
|
|
timeout: float = TIMEOUT, content: Optional[bytes] = None,
|
|
max_bytes: int = 4096) -> FetchResult:
|
|
"""
|
|
POST form data (or a raw body) to a checked, pinned address.
|
|
|
|
Redirects are not followed. `timeout` bounds the whole exchange, so a
|
|
server trickling its response can't hold the caller (v2.10.1). Only the
|
|
first max_bytes of the response are read (callers need the status and
|
|
maybe an error message).
|
|
|
|
Raises:
|
|
SSRFBlockedError, ValueError: URL is not allowed
|
|
httpx.HTTPError: Network errors
|
|
"""
|
|
kwargs = {'content': content} if content is not None else {'data': data}
|
|
deadline = time.monotonic() + timeout
|
|
with httpx.Client(follow_redirects=False, transport=TRANSPORT, timeout=timeout) as client, \
|
|
_Watchdog(client, timeout):
|
|
request = _pinned_request(client, 'POST', url, headers or {}, **kwargs)
|
|
response = client.send(request, stream=True)
|
|
try:
|
|
body = b''
|
|
for chunk in response.iter_bytes(chunk_size=1024):
|
|
body += chunk
|
|
if len(body) >= max_bytes:
|
|
body = body[:max_bytes]
|
|
break
|
|
if time.monotonic() > deadline:
|
|
raise httpx.TimeoutException("Response exceeded time limit")
|
|
return FetchResult(url=url, status_code=response.status_code, headers=response.headers,
|
|
content=body, encoding=response.charset_encoding or 'utf-8')
|
|
finally:
|
|
response.close()
|
|
|
|
|
|
def safe_get(url: str, headers: dict, max_bytes: int = MAX_BYTES,
|
|
max_redirects: int = MAX_REDIRECTS, timeout: float = TIMEOUT) -> FetchResult:
|
|
"""
|
|
GET a URL, following redirects by hand and checking each hop.
|
|
|
|
Args:
|
|
url: URL to fetch
|
|
headers: Request headers
|
|
max_bytes: Response body limit
|
|
max_redirects: Redirect limit
|
|
timeout: Total time limit in seconds
|
|
|
|
Returns:
|
|
FetchResult for the final response (any status code)
|
|
|
|
Raises:
|
|
SSRFBlockedError, ValueError: URL or a redirect target is not allowed
|
|
FetchTooLargeError: Body exceeds max_bytes
|
|
httpx.HTTPError: Network errors, too many redirects, time limit
|
|
"""
|
|
deadline = time.monotonic() + timeout
|
|
|
|
with httpx.Client(follow_redirects=False, transport=TRANSPORT) as client, \
|
|
_Watchdog(client, timeout):
|
|
for _ in range(max_redirects + 1):
|
|
remaining = deadline - time.monotonic()
|
|
if remaining <= 0:
|
|
raise httpx.TimeoutException("Fetch exceeded time limit")
|
|
|
|
request = _pinned_request(client, 'GET', url, headers, timeout=remaining)
|
|
response = client.send(request, stream=True)
|
|
try:
|
|
location = response.headers.get('Location')
|
|
if response.is_redirect and location:
|
|
url = urljoin(url, location)
|
|
continue
|
|
|
|
content_length = response.headers.get('Content-Length', '')
|
|
if content_length.isdigit() and int(content_length) > max_bytes:
|
|
raise FetchTooLargeError(f"Response exceeds size limit ({max_bytes} bytes)")
|
|
|
|
content = b''
|
|
for chunk in response.iter_bytes(chunk_size=8192):
|
|
content += chunk
|
|
if len(content) > max_bytes:
|
|
raise FetchTooLargeError(f"Response exceeds size limit ({max_bytes} bytes)")
|
|
if time.monotonic() > deadline:
|
|
raise httpx.TimeoutException("Fetch exceeded time limit")
|
|
|
|
return FetchResult(
|
|
url=url,
|
|
status_code=response.status_code,
|
|
headers=response.headers,
|
|
content=content,
|
|
encoding=response.charset_encoding or 'utf-8',
|
|
)
|
|
finally:
|
|
response.close()
|
|
|
|
raise httpx.TooManyRedirects(f"More than {max_redirects} redirects")
|