Sending: RFC 8288 Link header parsing, rel token matching and document order for <link>/<a>, endpoint re-discovery on every send, re-send to previous targets on update/delete/undelete, 410 Gone for deleted notes, no redirect following on the POST, 429 retried. Receiving: exact target match per media type, Accept header, final URL for relative links, only published notes accepted, fragments ignored, moderation kept on updates, transient errors keep valid mentions. Security: every redirect hop SSRF-checked, all resolved addresses checked, broader blocklist, total time and size limits. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
413 lines
13 KiB
Python
413 lines
13 KiB
Python
"""
|
|
Webmention validation and processing utilities
|
|
|
|
Per background-worker.md and api-contract.md:
|
|
- Fetch and validate source URLs
|
|
- Parse microformats with mf2py
|
|
- Extract author information from h-cards
|
|
- Detect mention types (reply, like, repost, etc.)
|
|
- Sanitize content with nh3
|
|
|
|
References:
|
|
- Background Worker Design: /docs/design/v1.6.0/background-worker.md
|
|
- API Contract: /docs/design/v1.6.0/api-contract.md
|
|
- W3C Webmention: https://www.w3.org/TR/webmention/
|
|
"""
|
|
|
|
import json
|
|
import logging
|
|
import mf2py
|
|
import nh3
|
|
import httpx
|
|
from datetime import datetime
|
|
from typing import Dict, Any, Optional
|
|
from urllib.parse import urldefrag, urlparse
|
|
|
|
from starpunk.webmention_ssrf import (
|
|
FetchTooLargeError, SSRFBlockedError, safe_get, user_agent,
|
|
)
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class ValidationError(Exception):
|
|
"""Raised when webmention validation fails"""
|
|
pass
|
|
|
|
|
|
ACCEPT = ('text/html, application/xhtml+xml;q=0.9, application/json;q=0.8, '
|
|
'text/plain;q=0.7, */*;q=0.1')
|
|
|
|
|
|
class TransientError(ValidationError):
|
|
"""Source could not be checked right now (network error, 5xx)"""
|
|
pass
|
|
|
|
|
|
def fetch_source(url: str, site_url: str) -> tuple[str, str, str]:
|
|
"""
|
|
Fetch source URL content with SSRF protection.
|
|
|
|
Per spec 3.2.2: GET following redirects (limited), with an Accept header.
|
|
|
|
Args:
|
|
url: Source URL to fetch
|
|
site_url: Site URL for User-Agent header
|
|
|
|
Returns:
|
|
Tuple of (content, content_type, final_url)
|
|
|
|
Raises:
|
|
ValidationError: Source is blocked, too large, or returned 4xx
|
|
TransientError: Network error, timeout or 5xx
|
|
"""
|
|
try:
|
|
result = safe_get(url, headers={'User-Agent': user_agent(site_url), 'Accept': ACCEPT})
|
|
except (SSRFBlockedError, ValueError) as e:
|
|
logger.warning(f"SSRF protection blocked URL {url}: {e}")
|
|
raise ValidationError(f"Blocked request: {e}")
|
|
except FetchTooLargeError as e:
|
|
raise ValidationError(f"Source {e}")
|
|
except httpx.TimeoutException:
|
|
raise TransientError("Source fetch timed out")
|
|
except httpx.HTTPError as e:
|
|
raise TransientError(f"Failed to fetch source: {e}")
|
|
|
|
if 400 <= result.status_code < 500:
|
|
raise ValidationError(f"Source returned {result.status_code}")
|
|
if result.status_code >= 300:
|
|
raise TransientError(f"Source server error: {result.status_code}")
|
|
|
|
return result.text, result.headers.get('Content-Type', ''), result.url
|
|
|
|
|
|
def _strip_fragment(url: str) -> str:
|
|
return urldefrag(url.strip())[0]
|
|
|
|
|
|
def _json_strings(value):
|
|
"""Yield every string in a parsed JSON document"""
|
|
if isinstance(value, str):
|
|
yield value
|
|
elif isinstance(value, dict):
|
|
for item in value.values():
|
|
yield from _json_strings(item)
|
|
elif isinstance(value, list):
|
|
for item in value:
|
|
yield from _json_strings(item)
|
|
|
|
|
|
def verify_link_presence(content: str, target_url: str, content_type: str,
|
|
source_url: str = '') -> bool:
|
|
"""
|
|
Verify the source links to the target (spec 3.2.2).
|
|
|
|
The source MUST have an exact match of the target URL, checked per media
|
|
type: in HTML, any link-bearing attribute (a/link href, img/video/audio
|
|
src, ...) resolved against the source URL; in JSON, any string value; in
|
|
other text, the URL anywhere. Fragments are ignored on both sides.
|
|
|
|
Args:
|
|
content: Source content
|
|
target_url: Target URL to find
|
|
content_type: Content type of source
|
|
source_url: Final source URL, for resolving relative links
|
|
|
|
Returns:
|
|
True if the target is linked, False otherwise
|
|
"""
|
|
target = _strip_fragment(target_url)
|
|
media_type = content_type.split(';')[0].strip().lower()
|
|
|
|
if 'html' in media_type:
|
|
from lxml import html as lxml_html
|
|
try:
|
|
doc = lxml_html.document_fromstring(content)
|
|
except Exception as e:
|
|
logger.debug(f"HTML parsing failed: {e}")
|
|
return False
|
|
if source_url:
|
|
doc.make_links_absolute(source_url, resolve_base_href=True,
|
|
handle_failures='ignore')
|
|
return any(_strip_fragment(link) == target for _, _, link, _ in doc.iterlinks())
|
|
|
|
if media_type == 'application/json' or media_type.endswith('+json'):
|
|
try:
|
|
data = json.loads(content)
|
|
except ValueError:
|
|
return False
|
|
return any(_strip_fragment(s) == target for s in _json_strings(data))
|
|
|
|
if media_type.startswith('text/'):
|
|
return target in content
|
|
|
|
return False
|
|
|
|
|
|
def detect_mention_type(parsed_mf2: Dict, target_url: str) -> str:
|
|
"""
|
|
Detect mention type from microformats.
|
|
|
|
Priority: reply > like > repost > bookmark > mention
|
|
|
|
Args:
|
|
parsed_mf2: Parsed microformats data from mf2py
|
|
target_url: Target URL being mentioned
|
|
|
|
Returns:
|
|
Mention type: 'reply', 'like', 'repost', 'bookmark', or 'mention'
|
|
"""
|
|
target_normalized = urlparse(target_url)._replace(fragment='').geturl().rstrip('/')
|
|
|
|
# Find h-entry that references target
|
|
for item in parsed_mf2.get('items', []):
|
|
if 'h-entry' not in item.get('type', []):
|
|
continue
|
|
|
|
props = item.get('properties', {})
|
|
|
|
# Check in-reply-to (highest priority)
|
|
for reply_to in props.get('in-reply-to', []):
|
|
url = reply_to if isinstance(reply_to, str) else reply_to.get('value', '')
|
|
if urlparse(url)._replace(fragment='').geturl().rstrip('/') == target_normalized:
|
|
return 'reply'
|
|
|
|
# Check like-of
|
|
for like in props.get('like-of', []):
|
|
url = like if isinstance(like, str) else like.get('value', '')
|
|
if urlparse(url)._replace(fragment='').geturl().rstrip('/') == target_normalized:
|
|
return 'like'
|
|
|
|
# Check repost-of
|
|
for repost in props.get('repost-of', []):
|
|
url = repost if isinstance(repost, str) else repost.get('value', '')
|
|
if urlparse(url)._replace(fragment='').geturl().rstrip('/') == target_normalized:
|
|
return 'repost'
|
|
|
|
# Check bookmark-of
|
|
for bookmark in props.get('bookmark-of', []):
|
|
url = bookmark if isinstance(bookmark, str) else bookmark.get('value', '')
|
|
if urlparse(url)._replace(fragment='').geturl().rstrip('/') == target_normalized:
|
|
return 'bookmark'
|
|
|
|
# Default fallback
|
|
return 'mention'
|
|
|
|
|
|
def _first_str(props: Dict, name: str) -> Optional[str]:
|
|
"""
|
|
First value of an mf2 property as a string
|
|
|
|
mf2py gives u-photo with alt text as {'value': url, 'alt': ...} and nested
|
|
microformats as dicts with a 'value'; reduce those to the plain value.
|
|
"""
|
|
values = props.get(name) or [None]
|
|
value = values[0]
|
|
if isinstance(value, dict):
|
|
value = value.get('value')
|
|
return value if isinstance(value, str) and value else None
|
|
|
|
|
|
def _is_http_url(value: Optional[str]) -> bool:
|
|
"""True if value is an absolute http or https URL"""
|
|
if not value:
|
|
return False
|
|
parsed = urlparse(value.strip())
|
|
return parsed.scheme.lower() in ('http', 'https') and bool(parsed.netloc)
|
|
|
|
|
|
def extract_author(parsed_mf2: Dict, source_url: str) -> Dict[str, Optional[str]]:
|
|
"""
|
|
Extract author information from h-card.
|
|
|
|
Args:
|
|
parsed_mf2: Parsed microformats data from mf2py
|
|
source_url: Source URL (fallback for author URL)
|
|
|
|
Returns:
|
|
Dict with author_name, author_url, author_photo
|
|
"""
|
|
author_name = None
|
|
author_url = None
|
|
author_photo = None
|
|
|
|
# Find h-entry first
|
|
entry = None
|
|
for item in parsed_mf2.get('items', []):
|
|
if 'h-entry' in item.get('type', []):
|
|
entry = item
|
|
break
|
|
|
|
# Check for author property in h-entry
|
|
if entry:
|
|
author_prop = entry.get('properties', {}).get('author', [])
|
|
if author_prop:
|
|
author_data = author_prop[0]
|
|
if isinstance(author_data, dict):
|
|
# h-card object
|
|
props = author_data.get('properties', {})
|
|
author_name = _first_str(props, 'name')
|
|
author_url = _first_str(props, 'url')
|
|
author_photo = _first_str(props, 'photo')
|
|
|
|
# If no author in h-entry, find representative h-card
|
|
if not author_name:
|
|
for item in parsed_mf2.get('items', []):
|
|
if 'h-card' in item.get('type', []):
|
|
props = item.get('properties', {})
|
|
author_name = _first_str(props, 'name')
|
|
author_url = _first_str(props, 'url')
|
|
author_photo = _first_str(props, 'photo')
|
|
break
|
|
|
|
# Fallback to hostname if no author found
|
|
if not author_name:
|
|
author_name = urlparse(source_url).netloc
|
|
|
|
# Only http(s) URLs are rendered as links (blocks javascript: and data:)
|
|
if not _is_http_url(author_url):
|
|
author_url = source_url
|
|
if not _is_http_url(author_photo):
|
|
author_photo = None
|
|
|
|
return {
|
|
'author_name': author_name,
|
|
'author_url': author_url,
|
|
'author_photo': author_photo
|
|
}
|
|
|
|
|
|
def extract_content(parsed_mf2: Dict) -> Optional[str]:
|
|
"""
|
|
Extract content snippet from h-entry.
|
|
|
|
Args:
|
|
parsed_mf2: Parsed microformats data from mf2py
|
|
|
|
Returns:
|
|
Sanitized content snippet (max 500 chars) or None
|
|
"""
|
|
# Find h-entry
|
|
for item in parsed_mf2.get('items', []):
|
|
if 'h-entry' not in item.get('type', []):
|
|
continue
|
|
|
|
props = item.get('properties', {})
|
|
|
|
# Try e-content first (HTML)
|
|
content_prop = props.get('content', [])
|
|
if content_prop:
|
|
content_data = content_prop[0]
|
|
if isinstance(content_data, dict):
|
|
content = content_data.get('html', '') or content_data.get('value', '')
|
|
else:
|
|
content = content_data
|
|
|
|
if content:
|
|
return _snippet(content)
|
|
|
|
# Try p-summary as fallback
|
|
summary = props.get('summary', [])
|
|
if summary:
|
|
content = summary[0]
|
|
if isinstance(content, dict):
|
|
content = content.get('value', '')
|
|
if content:
|
|
return _snippet(content)
|
|
|
|
return None
|
|
|
|
|
|
def _snippet(html: str, limit: int = 500) -> str:
|
|
"""Sanitized HTML of at most ~limit characters, with tags closed"""
|
|
sanitized = sanitize_content(html)
|
|
if len(sanitized) > limit:
|
|
# Cutting can leave a tag or attribute open; sanitizing again closes it
|
|
sanitized = sanitize_content(sanitized[:limit - 3] + '...')
|
|
return sanitized
|
|
|
|
|
|
def sanitize_content(html: str) -> str:
|
|
"""
|
|
Sanitize HTML content using nh3.
|
|
|
|
Per api-contract.md:
|
|
- Allowed tags: p, br, strong, em, b, i, a, code, pre, blockquote
|
|
- Allowed attributes: a[href, rel]
|
|
|
|
Args:
|
|
html: HTML content to sanitize
|
|
|
|
Returns:
|
|
Sanitized HTML
|
|
"""
|
|
allowed_tags = {'p', 'br', 'strong', 'em', 'b', 'i', 'a', 'code', 'pre', 'blockquote'}
|
|
allowed_attributes = {'a': {'href'}}
|
|
|
|
return nh3.clean(
|
|
html,
|
|
tags=allowed_tags,
|
|
attributes=allowed_attributes,
|
|
link_rel='nofollow noopener'
|
|
)
|
|
|
|
|
|
def process_webmention(source_url: str, target_url: str, site_url: str) -> Dict[str, Any]:
|
|
"""
|
|
Process and validate a webmention.
|
|
|
|
Args:
|
|
source_url: Source URL of the webmention
|
|
target_url: Target URL being mentioned
|
|
site_url: Site URL for User-Agent header
|
|
|
|
Returns:
|
|
Dict with validation results:
|
|
- valid: bool
|
|
- error: str (if valid=False)
|
|
- mention_type: str
|
|
- author_name: str
|
|
- author_url: str
|
|
- author_photo: str or None
|
|
- content_snippet: str or None
|
|
|
|
Raises:
|
|
ValidationError: If validation fails with specific error
|
|
"""
|
|
# Fetch source content
|
|
content, content_type, final_url = fetch_source(source_url, site_url)
|
|
|
|
# Verify target link presence
|
|
if not verify_link_presence(content, target_url, content_type, final_url):
|
|
raise ValidationError("Source does not link to target")
|
|
|
|
# Parse microformats (only for HTML)
|
|
parsed_mf2 = None
|
|
if 'html' in content_type.lower():
|
|
try:
|
|
parsed_mf2 = mf2py.parse(doc=content, url=final_url)
|
|
except Exception as e:
|
|
logger.debug(f"Microformat parsing failed for {source_url}: {e}")
|
|
|
|
# Extract data
|
|
mention_type = 'mention'
|
|
author_info = {
|
|
'author_name': urlparse(source_url).netloc,
|
|
'author_url': source_url,
|
|
'author_photo': None
|
|
}
|
|
content_snippet = None
|
|
|
|
if parsed_mf2:
|
|
mention_type = detect_mention_type(parsed_mf2, target_url)
|
|
author_info = extract_author(parsed_mf2, source_url)
|
|
content_snippet = extract_content(parsed_mf2)
|
|
|
|
return {
|
|
'valid': True,
|
|
'mention_type': mention_type,
|
|
**author_info,
|
|
'content_snippet': content_snippet
|
|
}
|