- Responses as u-comment/u-like/u-repost/u-bookmark/u-mention h-cites with author h-cards; source's own url and published (migration 014) - dt-* and Micropub published in RFC 3339 UTC - Title only for a real Markdown heading (hashtag notes stay notes) - Ignore webmentions from a note's own syndicated copies - Micropub/IndieAuth links independent of Webmention; rel=me/rel=author - Representative h-card and authorship algorithm (starpunk/mf2_utils.py) - Tag/likes h-feeds get url and author; previews get u-uid Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
345 lines
11 KiB
Python
345 lines
11 KiB
Python
"""
|
|
Webmention validation and processing utilities
|
|
|
|
Per background-worker.md and api-contract.md:
|
|
- Fetch and validate source URLs
|
|
- Parse microformats with mf2py
|
|
- Extract author information from h-cards
|
|
- Detect mention types (reply, like, repost, etc.)
|
|
- Sanitize content with nh3
|
|
|
|
References:
|
|
- Background Worker Design: /docs/design/v1.6.0/background-worker.md
|
|
- API Contract: /docs/design/v1.6.0/api-contract.md
|
|
- W3C Webmention: https://www.w3.org/TR/webmention/
|
|
"""
|
|
|
|
import json
|
|
import logging
|
|
import mf2py
|
|
import nh3
|
|
import httpx
|
|
from typing import Dict, Any, Optional
|
|
from urllib.parse import urldefrag, urlparse
|
|
|
|
from starpunk import mf2_utils
|
|
from starpunk.utils import iso_utc
|
|
from starpunk.webmention_ssrf import (
|
|
FetchTooLargeError, SSRFBlockedError, safe_get, user_agent,
|
|
)
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class ValidationError(Exception):
|
|
"""Raised when webmention validation fails"""
|
|
pass
|
|
|
|
|
|
ACCEPT = ('text/html, application/xhtml+xml;q=0.9, application/json;q=0.8, '
|
|
'text/plain;q=0.7, */*;q=0.1')
|
|
|
|
|
|
class TransientError(ValidationError):
|
|
"""Source could not be checked right now (network error, 5xx)"""
|
|
pass
|
|
|
|
|
|
def fetch_source(url: str, site_url: str) -> tuple[str, str, str]:
|
|
"""
|
|
Fetch source URL content with SSRF protection.
|
|
|
|
Per spec 3.2.2: GET following redirects (limited), with an Accept header.
|
|
|
|
Args:
|
|
url: Source URL to fetch
|
|
site_url: Site URL for User-Agent header
|
|
|
|
Returns:
|
|
Tuple of (content, content_type, final_url)
|
|
|
|
Raises:
|
|
ValidationError: Source is blocked, too large, or returned 4xx
|
|
TransientError: Network error, timeout or 5xx
|
|
"""
|
|
try:
|
|
result = safe_get(url, headers={'User-Agent': user_agent(site_url), 'Accept': ACCEPT})
|
|
except (SSRFBlockedError, ValueError) as e:
|
|
logger.warning(f"SSRF protection blocked URL {url}: {e}")
|
|
raise ValidationError(f"Blocked request: {e}")
|
|
except FetchTooLargeError as e:
|
|
raise ValidationError(f"Source {e}")
|
|
except httpx.TimeoutException:
|
|
raise TransientError("Source fetch timed out")
|
|
except httpx.HTTPError as e:
|
|
raise TransientError(f"Failed to fetch source: {e}")
|
|
|
|
if 400 <= result.status_code < 500:
|
|
raise ValidationError(f"Source returned {result.status_code}")
|
|
if result.status_code >= 300:
|
|
raise TransientError(f"Source server error: {result.status_code}")
|
|
|
|
return result.text, result.headers.get('Content-Type', ''), result.url
|
|
|
|
|
|
def _strip_fragment(url: str) -> str:
|
|
return urldefrag(url.strip())[0]
|
|
|
|
|
|
def _json_strings(value):
|
|
"""Yield every string in a parsed JSON document"""
|
|
if isinstance(value, str):
|
|
yield value
|
|
elif isinstance(value, dict):
|
|
for item in value.values():
|
|
yield from _json_strings(item)
|
|
elif isinstance(value, list):
|
|
for item in value:
|
|
yield from _json_strings(item)
|
|
|
|
|
|
def verify_link_presence(content: str, target_url: str, content_type: str,
|
|
source_url: str = '') -> bool:
|
|
"""
|
|
Verify the source links to the target (spec 3.2.2).
|
|
|
|
The source MUST have an exact match of the target URL, checked per media
|
|
type: in HTML, any link-bearing attribute (a/link href, img/video/audio
|
|
src, ...) resolved against the source URL; in JSON, any string value; in
|
|
other text, the URL anywhere. Fragments are ignored on both sides.
|
|
|
|
Args:
|
|
content: Source content
|
|
target_url: Target URL to find
|
|
content_type: Content type of source
|
|
source_url: Final source URL, for resolving relative links
|
|
|
|
Returns:
|
|
True if the target is linked, False otherwise
|
|
"""
|
|
target = _strip_fragment(target_url)
|
|
media_type = content_type.split(';')[0].strip().lower()
|
|
|
|
if 'html' in media_type:
|
|
from lxml import html as lxml_html
|
|
try:
|
|
doc = lxml_html.document_fromstring(content)
|
|
except Exception as e:
|
|
logger.debug(f"HTML parsing failed: {e}")
|
|
return False
|
|
if source_url:
|
|
doc.make_links_absolute(source_url, resolve_base_href=True,
|
|
handle_failures='ignore')
|
|
return any(_strip_fragment(link) == target for _, _, link, _ in doc.iterlinks())
|
|
|
|
if media_type == 'application/json' or media_type.endswith('+json'):
|
|
try:
|
|
data = json.loads(content)
|
|
except ValueError:
|
|
return False
|
|
return any(_strip_fragment(s) == target for s in _json_strings(data))
|
|
|
|
if media_type.startswith('text/'):
|
|
return target in content
|
|
|
|
return False
|
|
|
|
|
|
def detect_mention_type(parsed_mf2: Dict, target_url: str) -> str:
|
|
"""
|
|
Detect mention type from the h-entry that responds to the target
|
|
|
|
Returns:
|
|
'reply', 'like', 'repost', 'bookmark', or 'mention'
|
|
"""
|
|
entry = mf2_utils.responding_entry(parsed_mf2, target_url)
|
|
return mf2_utils.response_type(entry, target_url) if entry else 'mention'
|
|
|
|
|
|
def _is_http_url(value: Optional[str]) -> bool:
|
|
"""True if value is an absolute http or https URL"""
|
|
if not value:
|
|
return False
|
|
parsed = urlparse(value.strip())
|
|
return parsed.scheme.lower() in ('http', 'https') and bool(parsed.netloc)
|
|
|
|
|
|
def extract_author(parsed_mf2: Dict, source_url: str,
|
|
entry: Optional[Dict] = None) -> Dict[str, Optional[str]]:
|
|
"""
|
|
Author of the source's h-entry (authorship algorithm, without fetching)
|
|
|
|
Args:
|
|
parsed_mf2: Parsed microformats data from mf2py
|
|
source_url: Source URL (fallback for author URL and name)
|
|
entry: The h-entry to attribute (default: first top-level h-entry)
|
|
|
|
Returns:
|
|
Dict with author_name, author_url, author_photo; only http(s) URLs
|
|
are kept, so they are safe to render as links (XSS)
|
|
"""
|
|
if entry is None:
|
|
entry = mf2_utils.responding_entry(parsed_mf2, '')
|
|
author = mf2_utils.entry_author(parsed_mf2, entry, source_url)
|
|
|
|
author_url = author['url'] if _is_http_url(author['url']) else source_url
|
|
author_photo = author['photo'] if _is_http_url(author['photo']) else None
|
|
author_name = author['name'] or urlparse(author_url).netloc
|
|
|
|
return {
|
|
'author_name': author_name,
|
|
'author_url': author_url,
|
|
'author_photo': author_photo
|
|
}
|
|
|
|
|
|
def extract_content(parsed_mf2: Dict, entry: Optional[Dict] = None) -> Optional[str]:
|
|
"""
|
|
Content snippet of an h-entry (e-content, else p-summary)
|
|
|
|
Args:
|
|
parsed_mf2: Parsed microformats data from mf2py
|
|
entry: The h-entry (default: first top-level h-entry)
|
|
|
|
Returns:
|
|
Sanitized content snippet (max 500 chars) or None
|
|
"""
|
|
if entry is None:
|
|
entry = mf2_utils.responding_entry(parsed_mf2, '')
|
|
if not entry:
|
|
return None
|
|
|
|
props = entry.get('properties', {})
|
|
for content in props.get('content', []):
|
|
html = (content.get('html') or content.get('value')) if isinstance(content, dict) else content
|
|
if html:
|
|
return _snippet(html)
|
|
|
|
summary = mf2_utils.first(entry, 'summary')
|
|
return _snippet(summary) if summary else None
|
|
|
|
|
|
def extract_source_details(entry: Optional[Dict]) -> Dict[str, Optional[str]]:
|
|
"""
|
|
The source post's own URL, uid and published time
|
|
|
|
Returns:
|
|
source_post_url, source_uid (http(s) only) and source_published
|
|
(RFC 3339 UTC, or None if missing or unparseable)
|
|
"""
|
|
if not entry:
|
|
return {'source_post_url': None, 'source_uid': None, 'source_published': None}
|
|
|
|
url = mf2_utils.first(entry, 'url')
|
|
uid = mf2_utils.first(entry, 'uid')
|
|
published = mf2_utils.first(entry, 'published')
|
|
try:
|
|
published = iso_utc(published) or None
|
|
except (ValueError, TypeError):
|
|
published = None
|
|
|
|
return {
|
|
'source_post_url': url if _is_http_url(url) else None,
|
|
'source_uid': uid,
|
|
'source_published': published,
|
|
}
|
|
|
|
|
|
def _snippet(html: str, limit: int = 500) -> str:
|
|
"""Sanitized HTML of at most ~limit characters, with tags closed"""
|
|
sanitized = sanitize_content(html)
|
|
if len(sanitized) > limit:
|
|
# Cutting can leave a tag or attribute open; sanitizing again closes it
|
|
sanitized = sanitize_content(sanitized[:limit - 3] + '...')
|
|
return sanitized
|
|
|
|
|
|
def sanitize_content(html: str) -> str:
|
|
"""
|
|
Sanitize HTML content using nh3.
|
|
|
|
Per api-contract.md:
|
|
- Allowed tags: p, br, strong, em, b, i, a, code, pre, blockquote
|
|
- Allowed attributes: a[href, rel]
|
|
|
|
Args:
|
|
html: HTML content to sanitize
|
|
|
|
Returns:
|
|
Sanitized HTML
|
|
"""
|
|
allowed_tags = {'p', 'br', 'strong', 'em', 'b', 'i', 'a', 'code', 'pre', 'blockquote'}
|
|
allowed_attributes = {'a': {'href'}}
|
|
|
|
return nh3.clean(
|
|
html,
|
|
tags=allowed_tags,
|
|
attributes=allowed_attributes,
|
|
link_rel='nofollow noopener'
|
|
)
|
|
|
|
|
|
def process_webmention(source_url: str, target_url: str, site_url: str) -> Dict[str, Any]:
|
|
"""
|
|
Process and validate a webmention.
|
|
|
|
Args:
|
|
source_url: Source URL of the webmention
|
|
target_url: Target URL being mentioned
|
|
site_url: Site URL for User-Agent header
|
|
|
|
Returns:
|
|
Dict with validation results:
|
|
- valid: bool
|
|
- error: str (if valid=False)
|
|
- mention_type: str
|
|
- author_name: str
|
|
- author_url: str
|
|
- author_photo: str or None
|
|
- content_snippet: str or None
|
|
|
|
Raises:
|
|
ValidationError: If validation fails with specific error
|
|
"""
|
|
# Fetch source content
|
|
content, content_type, final_url = fetch_source(source_url, site_url)
|
|
|
|
# Verify target link presence
|
|
if not verify_link_presence(content, target_url, content_type, final_url):
|
|
raise ValidationError("Source does not link to target")
|
|
|
|
# Parse microformats (only for HTML)
|
|
parsed_mf2 = None
|
|
if 'html' in content_type.lower():
|
|
try:
|
|
parsed_mf2 = mf2py.parse(doc=content, url=final_url)
|
|
except Exception as e:
|
|
logger.debug(f"Microformat parsing failed for {source_url}: {e}")
|
|
|
|
# Extract data
|
|
mention_type = 'mention'
|
|
author_info = {
|
|
'author_name': urlparse(source_url).netloc,
|
|
'author_url': source_url,
|
|
'author_photo': None
|
|
}
|
|
content_snippet = None
|
|
|
|
source_details = extract_source_details(None)
|
|
|
|
if parsed_mf2:
|
|
entry = mf2_utils.responding_entry(parsed_mf2, target_url)
|
|
mention_type = mf2_utils.response_type(entry, target_url) if entry else 'mention'
|
|
author_info = extract_author(parsed_mf2, final_url, entry)
|
|
content_snippet = extract_content(parsed_mf2, entry)
|
|
source_details = extract_source_details(entry)
|
|
|
|
return {
|
|
'valid': True,
|
|
'mention_type': mention_type,
|
|
**author_info,
|
|
'content_snippet': content_snippet,
|
|
**source_details,
|
|
}
|