""" Reply context fetching for in-reply-to URLs Per database-schema.md Story 2.4: - Fetch context from in_reply_to URL when note is reply - Parse microformats for author, content - Store in reply_contexts table - Graceful degradation on fetch failure References: - Database Schema: /docs/design/v1.6.0/database-schema.md - Background Worker: /docs/design/v1.6.0/background-worker.md """ import logging import sqlite3 from datetime import datetime from pathlib import Path from typing import Optional, Dict, Any from starpunk.webmention_processor import ( fetch_source, extract_author, extract_content, ValidationError ) import mf2py logger = logging.getLogger(__name__) def fetch_reply_context( db_path: Path, reply_url: str, site_url: str ) -> Dict[str, Any]: """ Fetch and parse reply context from a URL. Per database-schema.md: - Fetch URL content with SSRF protection - Parse h-entry for title, content, published date - Parse h-card for author information - Store in reply_contexts table - Graceful degradation: errors don't prevent reply creation Args: db_path: Path to SQLite database reply_url: URL being replied to site_url: Site URL for User-Agent header Returns: Dict with fetch status and context data """ # Check if context already exists try: conn = sqlite3.connect(db_path) conn.row_factory = sqlite3.Row cursor = conn.cursor() cursor.execute(""" SELECT * FROM reply_contexts WHERE url = ? """, (reply_url,)) existing = cursor.fetchone() conn.close() if existing: logger.debug(f"Reply context already cached for {reply_url}") return dict(existing) except Exception as e: logger.warning(f"Failed to check existing context: {e}") # Fetch and parse context try: # Fetch URL content content, content_type = fetch_source(reply_url, site_url) # Parse microformats (only for HTML) if 'html' not in content_type.lower(): # Not HTML - store minimal context return _store_minimal_context(db_path, reply_url, "Not HTML content") parsed_mf2 = mf2py.parse(doc=content, url=reply_url) # Extract h-entry data entry_data = _extract_entry_data(parsed_mf2) # Extract author data author_data = extract_author(parsed_mf2, reply_url) # Store in database context = _store_context( db_path, reply_url, entry_data.get('title'), entry_data.get('content_snippet'), entry_data.get('published_at'), author_data.get('author_name'), author_data.get('author_url'), author_data.get('author_photo'), entry_data.get('site_name') ) logger.info(f"Fetched reply context for {reply_url}") return context except ValidationError as e: # Fetch/validation error - store error status logger.warning(f"Failed to fetch reply context for {reply_url}: {e}") return _store_minimal_context(db_path, reply_url, str(e)) except Exception as e: # Unexpected error - store error status logger.error(f"Error fetching reply context for {reply_url}: {e}", exc_info=True) return _store_minimal_context(db_path, reply_url, f"Unexpected error: {e}") def _extract_entry_data(parsed_mf2: Dict) -> Dict[str, Any]: """ Extract entry data from parsed microformats. Args: parsed_mf2: Parsed microformats data from mf2py Returns: Dict with title, content_snippet, published_at, site_name """ title = None content_snippet = None published_at = None site_name = None # Find h-entry for item in parsed_mf2.get('items', []): if 'h-entry' not in item.get('type', []): continue props = item.get('properties', {}) # Extract title (p-name) name = props.get('name', []) if name: title = name[0] if isinstance(title, dict): title = title.get('value', '') # Truncate if too long if title and len(title) > 200: title = title[:197] + '...' # Extract content (reuse processor function) content_snippet = extract_content(parsed_mf2) # Extract published date published = props.get('published', []) if published: pub_str = published[0] if isinstance(pub_str, dict): pub_str = pub_str.get('value', '') # Try to parse ISO 8601 datetime try: from dateutil import parser as dateutil_parser published_at = dateutil_parser.parse(pub_str) except Exception: # Parsing failed, leave as None pass break # Use first h-entry # Try to extract site name from h-card for item in parsed_mf2.get('items', []): if 'h-card' in item.get('type', []): props = item.get('properties', {}) name = props.get('name', []) if name: site_name = name[0] if isinstance(site_name, dict): site_name = site_name.get('value', '') break return { 'title': title, 'content_snippet': content_snippet, 'published_at': published_at, 'site_name': site_name } def _store_context( db_path: Path, url: str, title: Optional[str], content_snippet: Optional[str], published_at: Optional[datetime], author_name: Optional[str], author_url: Optional[str], author_photo: Optional[str], site_name: Optional[str] ) -> Dict[str, Any]: """ Store reply context in database. Args: db_path: Database path url: Reply-to URL title: Entry title content_snippet: Content snippet published_at: Published datetime author_name: Author name author_url: Author URL author_photo: Author photo URL site_name: Site name Returns: Stored context dict """ try: conn = sqlite3.connect(db_path) cursor = conn.cursor() cursor.execute(""" INSERT OR REPLACE INTO reply_contexts (url, title, content_snippet, published_at, author_name, author_url, author_photo, site_name, fetch_status, fetched_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'fetched', CURRENT_TIMESTAMP) """, ( url, title, content_snippet, published_at, author_name, author_url, author_photo, site_name )) conn.commit() conn.close() return { 'url': url, 'title': title, 'content_snippet': content_snippet, 'published_at': published_at, 'author_name': author_name, 'author_url': author_url, 'author_photo': author_photo, 'site_name': site_name, 'fetch_status': 'fetched' } except Exception as e: logger.error(f"Failed to store reply context: {e}") raise def _store_minimal_context( db_path: Path, url: str, error: str ) -> Dict[str, Any]: """ Store minimal context with error status. Args: db_path: Database path url: Reply-to URL error: Error message Returns: Minimal context dict with error """ try: conn = sqlite3.connect(db_path) cursor = conn.cursor() cursor.execute(""" INSERT OR REPLACE INTO reply_contexts (url, fetch_status, fetch_error, created_at) VALUES (?, 'error', ?, CURRENT_TIMESTAMP) """, (url, error)) conn.commit() conn.close() return { 'url': url, 'fetch_status': 'error', 'fetch_error': error } except Exception as e: logger.error(f"Failed to store minimal context: {e}") # Return in-memory dict even if storage fails return { 'url': url, 'fetch_status': 'error', 'fetch_error': error } def get_reply_context(db_path: Path, reply_url: str) -> Optional[Dict[str, Any]]: """ Get cached reply context from database. Args: db_path: Database path reply_url: Reply-to URL Returns: Context dict if exists, None otherwise """ try: conn = sqlite3.connect(db_path) conn.row_factory = sqlite3.Row cursor = conn.cursor() cursor.execute(""" SELECT * FROM reply_contexts WHERE url = ? """, (reply_url,)) row = cursor.fetchone() conn.close() if row: return dict(row) return None except Exception as e: logger.error(f"Failed to get reply context: {e}") return None