Files
StarPunk/starpunk/webmention_processor.py
T
PhilandClaude Opus 5.5 4aaa7a8122 feat: W3C Webmention compliance (v2.7.0)
Sending: RFC 8288 Link header parsing, rel token matching and document
order for <link>/<a>, endpoint re-discovery on every send, re-send to
previous targets on update/delete/undelete, 410 Gone for deleted notes,
no redirect following on the POST, 429 retried.

Receiving: exact target match per media type, Accept header, final URL
for relative links, only published notes accepted, fragments ignored,
moderation kept on updates, transient errors keep valid mentions.

Security: every redirect hop SSRF-checked, all resolved addresses
checked, broader blocklist, total time and size limits.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-03 08:07:04 -07:00

413 lines
13 KiB
Python

"""
Webmention validation and processing utilities
Per background-worker.md and api-contract.md:
- Fetch and validate source URLs
- Parse microformats with mf2py
- Extract author information from h-cards
- Detect mention types (reply, like, repost, etc.)
- Sanitize content with nh3
References:
- Background Worker Design: /docs/design/v1.6.0/background-worker.md
- API Contract: /docs/design/v1.6.0/api-contract.md
- W3C Webmention: https://www.w3.org/TR/webmention/
"""
import json
import logging
import mf2py
import nh3
import httpx
from datetime import datetime
from typing import Dict, Any, Optional
from urllib.parse import urldefrag, urlparse
from starpunk.webmention_ssrf import (
FetchTooLargeError, SSRFBlockedError, safe_get, user_agent,
)
logger = logging.getLogger(__name__)
class ValidationError(Exception):
"""Raised when webmention validation fails"""
pass
ACCEPT = ('text/html, application/xhtml+xml;q=0.9, application/json;q=0.8, '
'text/plain;q=0.7, */*;q=0.1')
class TransientError(ValidationError):
"""Source could not be checked right now (network error, 5xx)"""
pass
def fetch_source(url: str, site_url: str) -> tuple[str, str, str]:
"""
Fetch source URL content with SSRF protection.
Per spec 3.2.2: GET following redirects (limited), with an Accept header.
Args:
url: Source URL to fetch
site_url: Site URL for User-Agent header
Returns:
Tuple of (content, content_type, final_url)
Raises:
ValidationError: Source is blocked, too large, or returned 4xx
TransientError: Network error, timeout or 5xx
"""
try:
result = safe_get(url, headers={'User-Agent': user_agent(site_url), 'Accept': ACCEPT})
except (SSRFBlockedError, ValueError) as e:
logger.warning(f"SSRF protection blocked URL {url}: {e}")
raise ValidationError(f"Blocked request: {e}")
except FetchTooLargeError as e:
raise ValidationError(f"Source {e}")
except httpx.TimeoutException:
raise TransientError("Source fetch timed out")
except httpx.HTTPError as e:
raise TransientError(f"Failed to fetch source: {e}")
if 400 <= result.status_code < 500:
raise ValidationError(f"Source returned {result.status_code}")
if result.status_code >= 300:
raise TransientError(f"Source server error: {result.status_code}")
return result.text, result.headers.get('Content-Type', ''), result.url
def _strip_fragment(url: str) -> str:
return urldefrag(url.strip())[0]
def _json_strings(value):
"""Yield every string in a parsed JSON document"""
if isinstance(value, str):
yield value
elif isinstance(value, dict):
for item in value.values():
yield from _json_strings(item)
elif isinstance(value, list):
for item in value:
yield from _json_strings(item)
def verify_link_presence(content: str, target_url: str, content_type: str,
source_url: str = '') -> bool:
"""
Verify the source links to the target (spec 3.2.2).
The source MUST have an exact match of the target URL, checked per media
type: in HTML, any link-bearing attribute (a/link href, img/video/audio
src, ...) resolved against the source URL; in JSON, any string value; in
other text, the URL anywhere. Fragments are ignored on both sides.
Args:
content: Source content
target_url: Target URL to find
content_type: Content type of source
source_url: Final source URL, for resolving relative links
Returns:
True if the target is linked, False otherwise
"""
target = _strip_fragment(target_url)
media_type = content_type.split(';')[0].strip().lower()
if 'html' in media_type:
from lxml import html as lxml_html
try:
doc = lxml_html.document_fromstring(content)
except Exception as e:
logger.debug(f"HTML parsing failed: {e}")
return False
if source_url:
doc.make_links_absolute(source_url, resolve_base_href=True,
handle_failures='ignore')
return any(_strip_fragment(link) == target for _, _, link, _ in doc.iterlinks())
if media_type == 'application/json' or media_type.endswith('+json'):
try:
data = json.loads(content)
except ValueError:
return False
return any(_strip_fragment(s) == target for s in _json_strings(data))
if media_type.startswith('text/'):
return target in content
return False
def detect_mention_type(parsed_mf2: Dict, target_url: str) -> str:
"""
Detect mention type from microformats.
Priority: reply > like > repost > bookmark > mention
Args:
parsed_mf2: Parsed microformats data from mf2py
target_url: Target URL being mentioned
Returns:
Mention type: 'reply', 'like', 'repost', 'bookmark', or 'mention'
"""
target_normalized = urlparse(target_url)._replace(fragment='').geturl().rstrip('/')
# Find h-entry that references target
for item in parsed_mf2.get('items', []):
if 'h-entry' not in item.get('type', []):
continue
props = item.get('properties', {})
# Check in-reply-to (highest priority)
for reply_to in props.get('in-reply-to', []):
url = reply_to if isinstance(reply_to, str) else reply_to.get('value', '')
if urlparse(url)._replace(fragment='').geturl().rstrip('/') == target_normalized:
return 'reply'
# Check like-of
for like in props.get('like-of', []):
url = like if isinstance(like, str) else like.get('value', '')
if urlparse(url)._replace(fragment='').geturl().rstrip('/') == target_normalized:
return 'like'
# Check repost-of
for repost in props.get('repost-of', []):
url = repost if isinstance(repost, str) else repost.get('value', '')
if urlparse(url)._replace(fragment='').geturl().rstrip('/') == target_normalized:
return 'repost'
# Check bookmark-of
for bookmark in props.get('bookmark-of', []):
url = bookmark if isinstance(bookmark, str) else bookmark.get('value', '')
if urlparse(url)._replace(fragment='').geturl().rstrip('/') == target_normalized:
return 'bookmark'
# Default fallback
return 'mention'
def _first_str(props: Dict, name: str) -> Optional[str]:
"""
First value of an mf2 property as a string
mf2py gives u-photo with alt text as {'value': url, 'alt': ...} and nested
microformats as dicts with a 'value'; reduce those to the plain value.
"""
values = props.get(name) or [None]
value = values[0]
if isinstance(value, dict):
value = value.get('value')
return value if isinstance(value, str) and value else None
def _is_http_url(value: Optional[str]) -> bool:
"""True if value is an absolute http or https URL"""
if not value:
return False
parsed = urlparse(value.strip())
return parsed.scheme.lower() in ('http', 'https') and bool(parsed.netloc)
def extract_author(parsed_mf2: Dict, source_url: str) -> Dict[str, Optional[str]]:
"""
Extract author information from h-card.
Args:
parsed_mf2: Parsed microformats data from mf2py
source_url: Source URL (fallback for author URL)
Returns:
Dict with author_name, author_url, author_photo
"""
author_name = None
author_url = None
author_photo = None
# Find h-entry first
entry = None
for item in parsed_mf2.get('items', []):
if 'h-entry' in item.get('type', []):
entry = item
break
# Check for author property in h-entry
if entry:
author_prop = entry.get('properties', {}).get('author', [])
if author_prop:
author_data = author_prop[0]
if isinstance(author_data, dict):
# h-card object
props = author_data.get('properties', {})
author_name = _first_str(props, 'name')
author_url = _first_str(props, 'url')
author_photo = _first_str(props, 'photo')
# If no author in h-entry, find representative h-card
if not author_name:
for item in parsed_mf2.get('items', []):
if 'h-card' in item.get('type', []):
props = item.get('properties', {})
author_name = _first_str(props, 'name')
author_url = _first_str(props, 'url')
author_photo = _first_str(props, 'photo')
break
# Fallback to hostname if no author found
if not author_name:
author_name = urlparse(source_url).netloc
# Only http(s) URLs are rendered as links (blocks javascript: and data:)
if not _is_http_url(author_url):
author_url = source_url
if not _is_http_url(author_photo):
author_photo = None
return {
'author_name': author_name,
'author_url': author_url,
'author_photo': author_photo
}
def extract_content(parsed_mf2: Dict) -> Optional[str]:
"""
Extract content snippet from h-entry.
Args:
parsed_mf2: Parsed microformats data from mf2py
Returns:
Sanitized content snippet (max 500 chars) or None
"""
# Find h-entry
for item in parsed_mf2.get('items', []):
if 'h-entry' not in item.get('type', []):
continue
props = item.get('properties', {})
# Try e-content first (HTML)
content_prop = props.get('content', [])
if content_prop:
content_data = content_prop[0]
if isinstance(content_data, dict):
content = content_data.get('html', '') or content_data.get('value', '')
else:
content = content_data
if content:
return _snippet(content)
# Try p-summary as fallback
summary = props.get('summary', [])
if summary:
content = summary[0]
if isinstance(content, dict):
content = content.get('value', '')
if content:
return _snippet(content)
return None
def _snippet(html: str, limit: int = 500) -> str:
"""Sanitized HTML of at most ~limit characters, with tags closed"""
sanitized = sanitize_content(html)
if len(sanitized) > limit:
# Cutting can leave a tag or attribute open; sanitizing again closes it
sanitized = sanitize_content(sanitized[:limit - 3] + '...')
return sanitized
def sanitize_content(html: str) -> str:
"""
Sanitize HTML content using nh3.
Per api-contract.md:
- Allowed tags: p, br, strong, em, b, i, a, code, pre, blockquote
- Allowed attributes: a[href, rel]
Args:
html: HTML content to sanitize
Returns:
Sanitized HTML
"""
allowed_tags = {'p', 'br', 'strong', 'em', 'b', 'i', 'a', 'code', 'pre', 'blockquote'}
allowed_attributes = {'a': {'href'}}
return nh3.clean(
html,
tags=allowed_tags,
attributes=allowed_attributes,
link_rel='nofollow noopener'
)
def process_webmention(source_url: str, target_url: str, site_url: str) -> Dict[str, Any]:
"""
Process and validate a webmention.
Args:
source_url: Source URL of the webmention
target_url: Target URL being mentioned
site_url: Site URL for User-Agent header
Returns:
Dict with validation results:
- valid: bool
- error: str (if valid=False)
- mention_type: str
- author_name: str
- author_url: str
- author_photo: str or None
- content_snippet: str or None
Raises:
ValidationError: If validation fails with specific error
"""
# Fetch source content
content, content_type, final_url = fetch_source(source_url, site_url)
# Verify target link presence
if not verify_link_presence(content, target_url, content_type, final_url):
raise ValidationError("Source does not link to target")
# Parse microformats (only for HTML)
parsed_mf2 = None
if 'html' in content_type.lower():
try:
parsed_mf2 = mf2py.parse(doc=content, url=final_url)
except Exception as e:
logger.debug(f"Microformat parsing failed for {source_url}: {e}")
# Extract data
mention_type = 'mention'
author_info = {
'author_name': urlparse(source_url).netloc,
'author_url': source_url,
'author_photo': None
}
content_snippet = None
if parsed_mf2:
mention_type = detect_mention_type(parsed_mf2, target_url)
author_info = extract_author(parsed_mf2, source_url)
content_snippet = extract_content(parsed_mf2)
return {
'valid': True,
'mention_type': mention_type,
**author_info,
'content_snippet': content_snippet
}