The v1.6.0 work (webmentions, reply contexts, rate limiter, migration 010) was built and deployed as git.thesatelliteoflove.com/phil/starpunk:v1.6.0-rc.2 but never committed. This restores the application code from /app in the running container so later work builds on what production runs. Tests and docs for v1.6.0 were not in the image and are not recovered. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
330 lines
8.9 KiB
Python
330 lines
8.9 KiB
Python
"""
|
|
Reply context fetching for in-reply-to URLs
|
|
|
|
Per database-schema.md Story 2.4:
|
|
- Fetch context from in_reply_to URL when note is reply
|
|
- Parse microformats for author, content
|
|
- Store in reply_contexts table
|
|
- Graceful degradation on fetch failure
|
|
|
|
References:
|
|
- Database Schema: /docs/design/v1.6.0/database-schema.md
|
|
- Background Worker: /docs/design/v1.6.0/background-worker.md
|
|
"""
|
|
|
|
import logging
|
|
import sqlite3
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from typing import Optional, Dict, Any
|
|
|
|
from starpunk.webmention_processor import (
|
|
fetch_source,
|
|
extract_author,
|
|
extract_content,
|
|
ValidationError
|
|
)
|
|
import mf2py
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def fetch_reply_context(
|
|
db_path: Path,
|
|
reply_url: str,
|
|
site_url: str
|
|
) -> Dict[str, Any]:
|
|
"""
|
|
Fetch and parse reply context from a URL.
|
|
|
|
Per database-schema.md:
|
|
- Fetch URL content with SSRF protection
|
|
- Parse h-entry for title, content, published date
|
|
- Parse h-card for author information
|
|
- Store in reply_contexts table
|
|
- Graceful degradation: errors don't prevent reply creation
|
|
|
|
Args:
|
|
db_path: Path to SQLite database
|
|
reply_url: URL being replied to
|
|
site_url: Site URL for User-Agent header
|
|
|
|
Returns:
|
|
Dict with fetch status and context data
|
|
"""
|
|
# Check if context already exists
|
|
try:
|
|
conn = sqlite3.connect(db_path)
|
|
conn.row_factory = sqlite3.Row
|
|
cursor = conn.cursor()
|
|
|
|
cursor.execute("""
|
|
SELECT * FROM reply_contexts WHERE url = ?
|
|
""", (reply_url,))
|
|
|
|
existing = cursor.fetchone()
|
|
conn.close()
|
|
|
|
if existing:
|
|
logger.debug(f"Reply context already cached for {reply_url}")
|
|
return dict(existing)
|
|
|
|
except Exception as e:
|
|
logger.warning(f"Failed to check existing context: {e}")
|
|
|
|
# Fetch and parse context
|
|
try:
|
|
# Fetch URL content
|
|
content, content_type = fetch_source(reply_url, site_url)
|
|
|
|
# Parse microformats (only for HTML)
|
|
if 'html' not in content_type.lower():
|
|
# Not HTML - store minimal context
|
|
return _store_minimal_context(db_path, reply_url, "Not HTML content")
|
|
|
|
parsed_mf2 = mf2py.parse(doc=content, url=reply_url)
|
|
|
|
# Extract h-entry data
|
|
entry_data = _extract_entry_data(parsed_mf2)
|
|
|
|
# Extract author data
|
|
author_data = extract_author(parsed_mf2, reply_url)
|
|
|
|
# Store in database
|
|
context = _store_context(
|
|
db_path,
|
|
reply_url,
|
|
entry_data.get('title'),
|
|
entry_data.get('content_snippet'),
|
|
entry_data.get('published_at'),
|
|
author_data.get('author_name'),
|
|
author_data.get('author_url'),
|
|
author_data.get('author_photo'),
|
|
entry_data.get('site_name')
|
|
)
|
|
|
|
logger.info(f"Fetched reply context for {reply_url}")
|
|
return context
|
|
|
|
except ValidationError as e:
|
|
# Fetch/validation error - store error status
|
|
logger.warning(f"Failed to fetch reply context for {reply_url}: {e}")
|
|
return _store_minimal_context(db_path, reply_url, str(e))
|
|
|
|
except Exception as e:
|
|
# Unexpected error - store error status
|
|
logger.error(f"Error fetching reply context for {reply_url}: {e}", exc_info=True)
|
|
return _store_minimal_context(db_path, reply_url, f"Unexpected error: {e}")
|
|
|
|
|
|
def _extract_entry_data(parsed_mf2: Dict) -> Dict[str, Any]:
|
|
"""
|
|
Extract entry data from parsed microformats.
|
|
|
|
Args:
|
|
parsed_mf2: Parsed microformats data from mf2py
|
|
|
|
Returns:
|
|
Dict with title, content_snippet, published_at, site_name
|
|
"""
|
|
title = None
|
|
content_snippet = None
|
|
published_at = None
|
|
site_name = None
|
|
|
|
# Find h-entry
|
|
for item in parsed_mf2.get('items', []):
|
|
if 'h-entry' not in item.get('type', []):
|
|
continue
|
|
|
|
props = item.get('properties', {})
|
|
|
|
# Extract title (p-name)
|
|
name = props.get('name', [])
|
|
if name:
|
|
title = name[0]
|
|
if isinstance(title, dict):
|
|
title = title.get('value', '')
|
|
# Truncate if too long
|
|
if title and len(title) > 200:
|
|
title = title[:197] + '...'
|
|
|
|
# Extract content (reuse processor function)
|
|
content_snippet = extract_content(parsed_mf2)
|
|
|
|
# Extract published date
|
|
published = props.get('published', [])
|
|
if published:
|
|
pub_str = published[0]
|
|
if isinstance(pub_str, dict):
|
|
pub_str = pub_str.get('value', '')
|
|
# Try to parse ISO 8601 datetime
|
|
try:
|
|
from dateutil import parser as dateutil_parser
|
|
published_at = dateutil_parser.parse(pub_str)
|
|
except Exception:
|
|
# Parsing failed, leave as None
|
|
pass
|
|
|
|
break # Use first h-entry
|
|
|
|
# Try to extract site name from h-card
|
|
for item in parsed_mf2.get('items', []):
|
|
if 'h-card' in item.get('type', []):
|
|
props = item.get('properties', {})
|
|
name = props.get('name', [])
|
|
if name:
|
|
site_name = name[0]
|
|
if isinstance(site_name, dict):
|
|
site_name = site_name.get('value', '')
|
|
break
|
|
|
|
return {
|
|
'title': title,
|
|
'content_snippet': content_snippet,
|
|
'published_at': published_at,
|
|
'site_name': site_name
|
|
}
|
|
|
|
|
|
def _store_context(
|
|
db_path: Path,
|
|
url: str,
|
|
title: Optional[str],
|
|
content_snippet: Optional[str],
|
|
published_at: Optional[datetime],
|
|
author_name: Optional[str],
|
|
author_url: Optional[str],
|
|
author_photo: Optional[str],
|
|
site_name: Optional[str]
|
|
) -> Dict[str, Any]:
|
|
"""
|
|
Store reply context in database.
|
|
|
|
Args:
|
|
db_path: Database path
|
|
url: Reply-to URL
|
|
title: Entry title
|
|
content_snippet: Content snippet
|
|
published_at: Published datetime
|
|
author_name: Author name
|
|
author_url: Author URL
|
|
author_photo: Author photo URL
|
|
site_name: Site name
|
|
|
|
Returns:
|
|
Stored context dict
|
|
"""
|
|
try:
|
|
conn = sqlite3.connect(db_path)
|
|
cursor = conn.cursor()
|
|
|
|
cursor.execute("""
|
|
INSERT OR REPLACE INTO reply_contexts
|
|
(url, title, content_snippet, published_at,
|
|
author_name, author_url, author_photo, site_name,
|
|
fetch_status, fetched_at)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'fetched', CURRENT_TIMESTAMP)
|
|
""", (
|
|
url, title, content_snippet, published_at,
|
|
author_name, author_url, author_photo, site_name
|
|
))
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
return {
|
|
'url': url,
|
|
'title': title,
|
|
'content_snippet': content_snippet,
|
|
'published_at': published_at,
|
|
'author_name': author_name,
|
|
'author_url': author_url,
|
|
'author_photo': author_photo,
|
|
'site_name': site_name,
|
|
'fetch_status': 'fetched'
|
|
}
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to store reply context: {e}")
|
|
raise
|
|
|
|
|
|
def _store_minimal_context(
|
|
db_path: Path,
|
|
url: str,
|
|
error: str
|
|
) -> Dict[str, Any]:
|
|
"""
|
|
Store minimal context with error status.
|
|
|
|
Args:
|
|
db_path: Database path
|
|
url: Reply-to URL
|
|
error: Error message
|
|
|
|
Returns:
|
|
Minimal context dict with error
|
|
"""
|
|
try:
|
|
conn = sqlite3.connect(db_path)
|
|
cursor = conn.cursor()
|
|
|
|
cursor.execute("""
|
|
INSERT OR REPLACE INTO reply_contexts
|
|
(url, fetch_status, fetch_error, created_at)
|
|
VALUES (?, 'error', ?, CURRENT_TIMESTAMP)
|
|
""", (url, error))
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
return {
|
|
'url': url,
|
|
'fetch_status': 'error',
|
|
'fetch_error': error
|
|
}
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to store minimal context: {e}")
|
|
# Return in-memory dict even if storage fails
|
|
return {
|
|
'url': url,
|
|
'fetch_status': 'error',
|
|
'fetch_error': error
|
|
}
|
|
|
|
|
|
def get_reply_context(db_path: Path, reply_url: str) -> Optional[Dict[str, Any]]:
|
|
"""
|
|
Get cached reply context from database.
|
|
|
|
Args:
|
|
db_path: Database path
|
|
reply_url: Reply-to URL
|
|
|
|
Returns:
|
|
Context dict if exists, None otherwise
|
|
"""
|
|
try:
|
|
conn = sqlite3.connect(db_path)
|
|
conn.row_factory = sqlite3.Row
|
|
cursor = conn.cursor()
|
|
|
|
cursor.execute("""
|
|
SELECT * FROM reply_contexts WHERE url = ?
|
|
""", (reply_url,))
|
|
|
|
row = cursor.fetchone()
|
|
conn.close()
|
|
|
|
if row:
|
|
return dict(row)
|
|
|
|
return None
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to get reply context: {e}")
|
|
return None
|