The v1.6.0 work (webmentions, reply contexts, rate limiter, migration 010) was built and deployed as git.thesatelliteoflove.com/phil/starpunk:v1.6.0-rc.2 but never committed. This restores the application code from /app in the running container so later work builds on what production runs. Tests and docs for v1.6.0 were not in the image and are not recovered. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
374 lines
12 KiB
Python
374 lines
12 KiB
Python
"""
|
|
Webmention validation and processing utilities
|
|
|
|
Per background-worker.md and api-contract.md:
|
|
- Fetch and validate source URLs
|
|
- Parse microformats with mf2py
|
|
- Extract author information from h-cards
|
|
- Detect mention types (reply, like, repost, etc.)
|
|
- Sanitize content with nh3
|
|
|
|
References:
|
|
- Background Worker Design: /docs/design/v1.6.0/background-worker.md
|
|
- API Contract: /docs/design/v1.6.0/api-contract.md
|
|
- W3C Webmention: https://www.w3.org/TR/webmention/
|
|
"""
|
|
|
|
import logging
|
|
import mf2py
|
|
import nh3
|
|
import httpx
|
|
from datetime import datetime
|
|
from typing import Dict, Any, Optional
|
|
from urllib.parse import urlparse
|
|
|
|
from starpunk.webmention_ssrf import validate_url_for_fetch, SSRFBlockedError
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class ValidationError(Exception):
|
|
"""Raised when webmention validation fails"""
|
|
pass
|
|
|
|
|
|
def fetch_source(url: str, site_url: str, timeout: int = 10, max_size: int = 1048576) -> tuple[str, str]:
|
|
"""
|
|
Fetch source URL content with SSRF protection.
|
|
|
|
Args:
|
|
url: Source URL to fetch
|
|
site_url: Site URL for User-Agent header
|
|
timeout: Request timeout in seconds (default: 10)
|
|
max_size: Maximum content size in bytes (default: 1MB)
|
|
|
|
Returns:
|
|
Tuple of (content, content_type)
|
|
|
|
Raises:
|
|
SSRFBlockedError: If URL points to private IP
|
|
ValidationError: If fetch fails or content too large
|
|
"""
|
|
# SSRF protection
|
|
try:
|
|
validate_url_for_fetch(url)
|
|
except (SSRFBlockedError, ValueError) as e:
|
|
logger.warning(f"SSRF protection blocked URL {url}: {e}")
|
|
raise ValidationError(f"Blocked request: {e}")
|
|
|
|
# Fetch URL
|
|
try:
|
|
# Use streaming to check size before downloading entire content
|
|
with httpx.stream(
|
|
'GET',
|
|
url,
|
|
timeout=timeout,
|
|
headers={'User-Agent': f'StarPunk/1.6.0 (+{site_url})'},
|
|
follow_redirects=True
|
|
) as response:
|
|
response.raise_for_status()
|
|
|
|
# Check content length if provided
|
|
content_length = response.headers.get('Content-Length')
|
|
if content_length and int(content_length) > max_size:
|
|
raise ValidationError(f"Source exceeds size limit ({max_size} bytes)")
|
|
|
|
# Read content with size limit
|
|
content = b''
|
|
for chunk in response.iter_bytes(chunk_size=8192):
|
|
content += chunk
|
|
if len(content) > max_size:
|
|
raise ValidationError(f"Source exceeds size limit ({max_size} bytes)")
|
|
|
|
content_type = response.headers.get('Content-Type', '')
|
|
|
|
return content.decode('utf-8', errors='replace'), content_type
|
|
|
|
except httpx.TimeoutException:
|
|
raise ValidationError("Source fetch timed out")
|
|
except httpx.HTTPStatusError as e:
|
|
if 400 <= e.response.status_code < 500:
|
|
raise ValidationError(f"Source returned {e.response.status_code}")
|
|
else:
|
|
raise ValidationError(f"Source server error: {e.response.status_code}")
|
|
except httpx.HTTPError as e:
|
|
raise ValidationError(f"Failed to fetch source: {e}")
|
|
except UnicodeDecodeError:
|
|
raise ValidationError("Source content encoding is not supported")
|
|
|
|
|
|
def verify_link_presence(content: str, target_url: str, content_type: str) -> bool:
|
|
"""
|
|
Verify that source content contains a link to target URL.
|
|
|
|
Args:
|
|
content: Source content (HTML or text)
|
|
target_url: Target URL to find
|
|
content_type: Content type of source
|
|
|
|
Returns:
|
|
True if target link found, False otherwise
|
|
"""
|
|
# Normalize URLs for comparison (remove fragments, trailing slashes)
|
|
target_normalized = urlparse(target_url)._replace(fragment='').geturl().rstrip('/')
|
|
|
|
# For HTML, parse and check all links
|
|
if 'html' in content_type.lower():
|
|
try:
|
|
from lxml import html as lxml_html
|
|
doc = lxml_html.fromstring(content)
|
|
links = doc.xpath('//a/@href')
|
|
|
|
for link in links:
|
|
# Make absolute if relative
|
|
link_normalized = urlparse(link)._replace(fragment='').geturl().rstrip('/')
|
|
if link_normalized == target_normalized:
|
|
return True
|
|
except Exception as e:
|
|
logger.debug(f"HTML parsing failed, falling back to string search: {e}")
|
|
|
|
# Fallback: simple string search
|
|
return target_normalized in content
|
|
|
|
|
|
def detect_mention_type(parsed_mf2: Dict, target_url: str) -> str:
|
|
"""
|
|
Detect mention type from microformats.
|
|
|
|
Priority: reply > like > repost > bookmark > mention
|
|
|
|
Args:
|
|
parsed_mf2: Parsed microformats data from mf2py
|
|
target_url: Target URL being mentioned
|
|
|
|
Returns:
|
|
Mention type: 'reply', 'like', 'repost', 'bookmark', or 'mention'
|
|
"""
|
|
target_normalized = urlparse(target_url)._replace(fragment='').geturl().rstrip('/')
|
|
|
|
# Find h-entry that references target
|
|
for item in parsed_mf2.get('items', []):
|
|
if 'h-entry' not in item.get('type', []):
|
|
continue
|
|
|
|
props = item.get('properties', {})
|
|
|
|
# Check in-reply-to (highest priority)
|
|
for reply_to in props.get('in-reply-to', []):
|
|
url = reply_to if isinstance(reply_to, str) else reply_to.get('value', '')
|
|
if urlparse(url)._replace(fragment='').geturl().rstrip('/') == target_normalized:
|
|
return 'reply'
|
|
|
|
# Check like-of
|
|
for like in props.get('like-of', []):
|
|
url = like if isinstance(like, str) else like.get('value', '')
|
|
if urlparse(url)._replace(fragment='').geturl().rstrip('/') == target_normalized:
|
|
return 'like'
|
|
|
|
# Check repost-of
|
|
for repost in props.get('repost-of', []):
|
|
url = repost if isinstance(repost, str) else repost.get('value', '')
|
|
if urlparse(url)._replace(fragment='').geturl().rstrip('/') == target_normalized:
|
|
return 'repost'
|
|
|
|
# Check bookmark-of
|
|
for bookmark in props.get('bookmark-of', []):
|
|
url = bookmark if isinstance(bookmark, str) else bookmark.get('value', '')
|
|
if urlparse(url)._replace(fragment='').geturl().rstrip('/') == target_normalized:
|
|
return 'bookmark'
|
|
|
|
# Default fallback
|
|
return 'mention'
|
|
|
|
|
|
def extract_author(parsed_mf2: Dict, source_url: str) -> Dict[str, Optional[str]]:
|
|
"""
|
|
Extract author information from h-card.
|
|
|
|
Args:
|
|
parsed_mf2: Parsed microformats data from mf2py
|
|
source_url: Source URL (fallback for author URL)
|
|
|
|
Returns:
|
|
Dict with author_name, author_url, author_photo
|
|
"""
|
|
author_name = None
|
|
author_url = None
|
|
author_photo = None
|
|
|
|
# Find h-entry first
|
|
entry = None
|
|
for item in parsed_mf2.get('items', []):
|
|
if 'h-entry' in item.get('type', []):
|
|
entry = item
|
|
break
|
|
|
|
# Check for author property in h-entry
|
|
if entry:
|
|
author_prop = entry.get('properties', {}).get('author', [])
|
|
if author_prop:
|
|
author_data = author_prop[0]
|
|
if isinstance(author_data, dict):
|
|
# h-card object
|
|
props = author_data.get('properties', {})
|
|
author_name = props.get('name', [None])[0]
|
|
author_url = props.get('url', [None])[0]
|
|
author_photo = props.get('photo', [None])[0]
|
|
|
|
# If no author in h-entry, find representative h-card
|
|
if not author_name:
|
|
for item in parsed_mf2.get('items', []):
|
|
if 'h-card' in item.get('type', []):
|
|
props = item.get('properties', {})
|
|
author_name = props.get('name', [None])[0]
|
|
author_url = props.get('url', [None])[0]
|
|
author_photo = props.get('photo', [None])[0]
|
|
break
|
|
|
|
# Fallback to hostname if no author found
|
|
if not author_name:
|
|
author_name = urlparse(source_url).netloc
|
|
|
|
if not author_url:
|
|
author_url = source_url
|
|
|
|
return {
|
|
'author_name': author_name,
|
|
'author_url': author_url,
|
|
'author_photo': author_photo
|
|
}
|
|
|
|
|
|
def extract_content(parsed_mf2: Dict) -> Optional[str]:
|
|
"""
|
|
Extract content snippet from h-entry.
|
|
|
|
Args:
|
|
parsed_mf2: Parsed microformats data from mf2py
|
|
|
|
Returns:
|
|
Sanitized content snippet (max 500 chars) or None
|
|
"""
|
|
# Find h-entry
|
|
for item in parsed_mf2.get('items', []):
|
|
if 'h-entry' not in item.get('type', []):
|
|
continue
|
|
|
|
props = item.get('properties', {})
|
|
|
|
# Try e-content first (HTML)
|
|
content_prop = props.get('content', [])
|
|
if content_prop:
|
|
content_data = content_prop[0]
|
|
if isinstance(content_data, dict):
|
|
content = content_data.get('html', '') or content_data.get('value', '')
|
|
else:
|
|
content = content_data
|
|
|
|
if content:
|
|
# Sanitize HTML
|
|
sanitized = sanitize_content(content)
|
|
# Truncate to 500 chars
|
|
if len(sanitized) > 500:
|
|
sanitized = sanitized[:497] + '...'
|
|
return sanitized
|
|
|
|
# Try p-summary as fallback
|
|
summary = props.get('summary', [])
|
|
if summary:
|
|
content = summary[0]
|
|
if isinstance(content, dict):
|
|
content = content.get('value', '')
|
|
if content:
|
|
sanitized = sanitize_content(content)
|
|
if len(sanitized) > 500:
|
|
sanitized = sanitized[:497] + '...'
|
|
return sanitized
|
|
|
|
return None
|
|
|
|
|
|
def sanitize_content(html: str) -> str:
|
|
"""
|
|
Sanitize HTML content using nh3.
|
|
|
|
Per api-contract.md:
|
|
- Allowed tags: p, br, strong, em, b, i, a, code, pre, blockquote
|
|
- Allowed attributes: a[href, rel]
|
|
|
|
Args:
|
|
html: HTML content to sanitize
|
|
|
|
Returns:
|
|
Sanitized HTML
|
|
"""
|
|
allowed_tags = {'p', 'br', 'strong', 'em', 'b', 'i', 'a', 'code', 'pre', 'blockquote'}
|
|
allowed_attributes = {'a': {'href'}}
|
|
|
|
return nh3.clean(
|
|
html,
|
|
tags=allowed_tags,
|
|
attributes=allowed_attributes,
|
|
link_rel='nofollow noopener'
|
|
)
|
|
|
|
|
|
def process_webmention(source_url: str, target_url: str, site_url: str) -> Dict[str, Any]:
|
|
"""
|
|
Process and validate a webmention.
|
|
|
|
Args:
|
|
source_url: Source URL of the webmention
|
|
target_url: Target URL being mentioned
|
|
site_url: Site URL for User-Agent header
|
|
|
|
Returns:
|
|
Dict with validation results:
|
|
- valid: bool
|
|
- error: str (if valid=False)
|
|
- mention_type: str
|
|
- author_name: str
|
|
- author_url: str
|
|
- author_photo: str or None
|
|
- content_snippet: str or None
|
|
|
|
Raises:
|
|
ValidationError: If validation fails with specific error
|
|
"""
|
|
# Fetch source content
|
|
content, content_type = fetch_source(source_url, site_url)
|
|
|
|
# Verify target link presence
|
|
if not verify_link_presence(content, target_url, content_type):
|
|
raise ValidationError("Source does not link to target")
|
|
|
|
# Parse microformats (only for HTML)
|
|
parsed_mf2 = None
|
|
if 'html' in content_type.lower():
|
|
try:
|
|
parsed_mf2 = mf2py.parse(doc=content, url=source_url)
|
|
except Exception as e:
|
|
logger.debug(f"Microformat parsing failed for {source_url}: {e}")
|
|
|
|
# Extract data
|
|
mention_type = 'mention'
|
|
author_info = {
|
|
'author_name': urlparse(source_url).netloc,
|
|
'author_url': source_url,
|
|
'author_photo': None
|
|
}
|
|
content_snippet = None
|
|
|
|
if parsed_mf2:
|
|
mention_type = detect_mention_type(parsed_mf2, target_url)
|
|
author_info = extract_author(parsed_mf2, source_url)
|
|
content_snippet = extract_content(parsed_mf2)
|
|
|
|
return {
|
|
'valid': True,
|
|
'mention_type': mention_type,
|
|
**author_info,
|
|
'content_snippet': content_snippet
|
|
}
|