""" Webmention validation and processing utilities Per background-worker.md and api-contract.md: - Fetch and validate source URLs - Parse microformats with mf2py - Extract author information from h-cards - Detect mention types (reply, like, repost, etc.) - Sanitize content with nh3 References: - Background Worker Design: /docs/design/v1.6.0/background-worker.md - API Contract: /docs/design/v1.6.0/api-contract.md - W3C Webmention: https://www.w3.org/TR/webmention/ """ import json import logging import mf2py import nh3 import httpx from typing import Dict, Any, Optional from urllib.parse import urldefrag, urlparse from starpunk import mf2_utils from starpunk.utils import iso_utc from starpunk.webmention_ssrf import ( FetchTooLargeError, SSRFBlockedError, safe_get, user_agent, ) logger = logging.getLogger(__name__) class ValidationError(Exception): """Raised when webmention validation fails""" pass ACCEPT = ('text/html, application/xhtml+xml;q=0.9, application/json;q=0.8, ' 'text/plain;q=0.7, */*;q=0.1') class TransientError(ValidationError): """Source could not be checked right now (network error, 5xx)""" pass def fetch_source(url: str, site_url: str) -> tuple[str, str, str]: """ Fetch source URL content with SSRF protection. Per spec 3.2.2: GET following redirects (limited), with an Accept header. Args: url: Source URL to fetch site_url: Site URL for User-Agent header Returns: Tuple of (content, content_type, final_url) Raises: ValidationError: Source is blocked, too large, or returned 4xx TransientError: Network error, timeout or 5xx """ try: result = safe_get(url, headers={'User-Agent': user_agent(site_url), 'Accept': ACCEPT}) except (SSRFBlockedError, ValueError) as e: logger.warning(f"SSRF protection blocked URL {url}: {e}") raise ValidationError(f"Blocked request: {e}") except FetchTooLargeError as e: raise ValidationError(f"Source {e}") except httpx.TimeoutException: raise TransientError("Source fetch timed out") except httpx.HTTPError as e: raise TransientError(f"Failed to fetch source: {e}") if 400 <= result.status_code < 500: raise ValidationError(f"Source returned {result.status_code}") if result.status_code >= 300: raise TransientError(f"Source server error: {result.status_code}") return result.text, result.headers.get('Content-Type', ''), result.url def _strip_fragment(url: str) -> str: return urldefrag(url.strip())[0] def _json_strings(value): """Yield every string in a parsed JSON document""" if isinstance(value, str): yield value elif isinstance(value, dict): for item in value.values(): yield from _json_strings(item) elif isinstance(value, list): for item in value: yield from _json_strings(item) def verify_link_presence(content: str, target_url: str, content_type: str, source_url: str = '') -> bool: """ Verify the source links to the target (spec 3.2.2). The source MUST have an exact match of the target URL, checked per media type: in HTML, any link-bearing attribute (a/link href, img/video/audio src, ...) resolved against the source URL; in JSON, any string value; in other text, the URL anywhere. Fragments are ignored on both sides. Args: content: Source content target_url: Target URL to find content_type: Content type of source source_url: Final source URL, for resolving relative links Returns: True if the target is linked, False otherwise """ target = _strip_fragment(target_url) media_type = content_type.split(';')[0].strip().lower() if 'html' in media_type: from lxml import html as lxml_html try: doc = lxml_html.document_fromstring(content) except Exception as e: logger.debug(f"HTML parsing failed: {e}") return False if source_url: doc.make_links_absolute(source_url, resolve_base_href=True, handle_failures='ignore') return any(_strip_fragment(link) == target for _, _, link, _ in doc.iterlinks()) if media_type == 'application/json' or media_type.endswith('+json'): try: data = json.loads(content) except ValueError: return False return any(_strip_fragment(s) == target for s in _json_strings(data)) if media_type.startswith('text/'): return target in content return False def detect_mention_type(parsed_mf2: Dict, target_url: str) -> str: """ Detect mention type from the h-entry that responds to the target Returns: 'reply', 'like', 'repost', 'bookmark', or 'mention' """ entry = mf2_utils.responding_entry(parsed_mf2, target_url) return mf2_utils.response_type(entry, target_url) if entry else 'mention' def _is_http_url(value: Optional[str]) -> bool: """True if value is an absolute http or https URL""" if not value: return False parsed = urlparse(value.strip()) return parsed.scheme.lower() in ('http', 'https') and bool(parsed.netloc) def extract_author(parsed_mf2: Dict, source_url: str, entry: Optional[Dict] = None) -> Dict[str, Optional[str]]: """ Author of the source's h-entry (authorship algorithm, without fetching) Args: parsed_mf2: Parsed microformats data from mf2py source_url: Source URL (fallback for author URL and name) entry: The h-entry to attribute (default: first top-level h-entry) Returns: Dict with author_name, author_url, author_photo; only http(s) URLs are kept, so they are safe to render as links (XSS) """ if entry is None: entry = mf2_utils.responding_entry(parsed_mf2, '') author = mf2_utils.entry_author(parsed_mf2, entry, source_url) author_url = author['url'] if _is_http_url(author['url']) else source_url author_photo = author['photo'] if _is_http_url(author['photo']) else None author_name = author['name'] or urlparse(author_url).netloc return { 'author_name': author_name, 'author_url': author_url, 'author_photo': author_photo } def extract_content(parsed_mf2: Dict, entry: Optional[Dict] = None) -> Optional[str]: """ Content snippet of an h-entry (e-content, else p-summary) Args: parsed_mf2: Parsed microformats data from mf2py entry: The h-entry (default: first top-level h-entry) Returns: Sanitized content snippet (max 500 chars) or None """ if entry is None: entry = mf2_utils.responding_entry(parsed_mf2, '') if not entry: return None props = entry.get('properties', {}) for content in props.get('content', []): html = (content.get('html') or content.get('value')) if isinstance(content, dict) else content if html: return _snippet(html) summary = mf2_utils.first(entry, 'summary') return _snippet(summary) if summary else None def extract_source_details(entry: Optional[Dict]) -> Dict[str, Optional[str]]: """ The source post's own URL, uid and published time Returns: source_post_url, source_uid (http(s) only) and source_published (RFC 3339 UTC, or None if missing or unparseable) """ if not entry: return {'source_post_url': None, 'source_uid': None, 'source_published': None} url = mf2_utils.first(entry, 'url') uid = mf2_utils.first(entry, 'uid') published = mf2_utils.first(entry, 'published') try: published = iso_utc(published) or None except (ValueError, TypeError): published = None return { 'source_post_url': url if _is_http_url(url) else None, 'source_uid': uid, 'source_published': published, } def _snippet(html: str, limit: int = 500) -> str: """Sanitized HTML of at most ~limit characters, with tags closed""" sanitized = sanitize_content(html) if len(sanitized) > limit: # Cutting can leave a tag or attribute open; sanitizing again closes it sanitized = sanitize_content(sanitized[:limit - 3] + '...') return sanitized def sanitize_content(html: str) -> str: """ Sanitize HTML content using nh3. Per api-contract.md: - Allowed tags: p, br, strong, em, b, i, a, code, pre, blockquote - Allowed attributes: a[href, rel] Args: html: HTML content to sanitize Returns: Sanitized HTML """ allowed_tags = {'p', 'br', 'strong', 'em', 'b', 'i', 'a', 'code', 'pre', 'blockquote'} allowed_attributes = {'a': {'href'}} return nh3.clean( html, tags=allowed_tags, attributes=allowed_attributes, link_rel='nofollow noopener' ) def process_webmention(source_url: str, target_url: str, site_url: str) -> Dict[str, Any]: """ Process and validate a webmention. Args: source_url: Source URL of the webmention target_url: Target URL being mentioned site_url: Site URL for User-Agent header Returns: Dict with validation results: - valid: bool - error: str (if valid=False) - mention_type: str - author_name: str - author_url: str - author_photo: str or None - content_snippet: str or None Raises: ValidationError: If validation fails with specific error """ # Fetch source content content, content_type, final_url = fetch_source(source_url, site_url) # Verify target link presence if not verify_link_presence(content, target_url, content_type, final_url): raise ValidationError("Source does not link to target") # Parse microformats (only for HTML) parsed_mf2 = None if 'html' in content_type.lower(): try: parsed_mf2 = mf2py.parse(doc=content, url=final_url) except Exception as e: logger.debug(f"Microformat parsing failed for {source_url}: {e}") # Extract data mention_type = 'mention' author_info = { 'author_name': urlparse(source_url).netloc, 'author_url': source_url, 'author_photo': None } content_snippet = None source_details = extract_source_details(None) if parsed_mf2: entry = mf2_utils.responding_entry(parsed_mf2, target_url) mention_type = mf2_utils.response_type(entry, target_url) if entry else 'mention' author_info = extract_author(parsed_mf2, final_url, entry) content_snippet = extract_content(parsed_mf2, entry) source_details = extract_source_details(entry) return { 'valid': True, 'mention_type': mention_type, **author_info, 'content_snippet': content_snippet, **source_details, }