diff --git a/migrations/010_add_webmentions.sql b/migrations/010_add_webmentions.sql new file mode 100644 index 0000000..2b1b6d2 --- /dev/null +++ b/migrations/010_add_webmentions.sql @@ -0,0 +1,79 @@ +-- Migration 010: Add Webmention support +-- Version: 1.6.0 +-- Per ADR-063: Webmention Implementation +-- Per W3C Webmention Recommendation + +-- Add reply-to support to notes table +ALTER TABLE notes ADD COLUMN in_reply_to TEXT; + +-- Incoming webmentions storage +CREATE TABLE IF NOT EXISTS webmentions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_url TEXT NOT NULL, + target_url TEXT NOT NULL, + target_note_id INTEGER, + mention_type TEXT NOT NULL DEFAULT 'mention' + CHECK (mention_type IN ('reply', 'like', 'repost', 'bookmark', 'mention')), + author_name TEXT, + author_url TEXT, + author_photo TEXT, + content_snippet TEXT, + validation_status TEXT NOT NULL DEFAULT 'pending' + CHECK (validation_status IN ('pending', 'valid', 'invalid', 'error')), + moderation_status TEXT NOT NULL DEFAULT 'pending' + CHECK (moderation_status IN ('pending', 'approved', 'hidden', 'deleted')), + validation_error TEXT, + validated_at TIMESTAMP, + received_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (target_note_id) REFERENCES notes(id) ON DELETE SET NULL, + UNIQUE(source_url, target_url) +); + +CREATE INDEX IF NOT EXISTS idx_webmentions_target_note ON webmentions(target_note_id); +CREATE INDEX IF NOT EXISTS idx_webmentions_moderation ON webmentions(moderation_status); +CREATE INDEX IF NOT EXISTS idx_webmentions_validation ON webmentions(validation_status); +CREATE INDEX IF NOT EXISTS idx_webmentions_received ON webmentions(received_at DESC); + +-- Outgoing webmention queue +CREATE TABLE IF NOT EXISTS webmention_queue ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_url TEXT NOT NULL, + target_url TEXT NOT NULL, + source_note_id INTEGER, + endpoint_url TEXT, + status TEXT NOT NULL DEFAULT 'pending' + CHECK (status IN ('pending', 'sending', 'sent', 'failed', 'no_endpoint')), + retry_count INTEGER NOT NULL DEFAULT 0, + next_retry_at TIMESTAMP, + last_error TEXT, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + started_at TIMESTAMP, + sent_at TIMESTAMP, + FOREIGN KEY (source_note_id) REFERENCES notes(id) ON DELETE CASCADE, + UNIQUE(source_url, target_url) +); + +CREATE INDEX IF NOT EXISTS idx_webmention_queue_status ON webmention_queue(status); +CREATE INDEX IF NOT EXISTS idx_webmention_queue_retry ON webmention_queue(next_retry_at); +CREATE INDEX IF NOT EXISTS idx_webmention_queue_note ON webmention_queue(source_note_id); + +-- Cached reply contexts +CREATE TABLE IF NOT EXISTS reply_contexts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + url TEXT UNIQUE NOT NULL, + title TEXT, + content_snippet TEXT, + published_at TIMESTAMP, + author_name TEXT, + author_url TEXT, + author_photo TEXT, + site_name TEXT, + fetch_status TEXT NOT NULL DEFAULT 'pending' + CHECK (fetch_status IN ('pending', 'fetched', 'error')), + fetch_error TEXT, + fetched_at TIMESTAMP, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_reply_contexts_url ON reply_contexts(url); diff --git a/requirements.txt b/requirements.txt index 04aa93e..4aeed68 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,3 +36,6 @@ Pillow==10.1.* # HEIC/HEIF Support (v1.4.2 - iPhone photos) pillow-heif==0.18.* + +# HTML Sanitization (v1.6.0 - Webmention content) +nh3==0.2.* diff --git a/starpunk/__init__.py b/starpunk/__init__.py index 8abe228..a0cadeb 100644 --- a/starpunk/__init__.py +++ b/starpunk/__init__.py @@ -156,6 +156,10 @@ def create_app(config=None): configure_cache(max_size=max_size, ttl=ttl) app.logger.info(f"Feed cache enabled (max_size={max_size}, ttl={ttl}s)") + # Initialize rate limiters (v1.6.0 Phase 2) + from starpunk.rate_limiter import init_rate_limiters + init_rate_limiters(app) + # Initialize FTS index if needed from pathlib import Path from starpunk.search import has_fts_table, rebuild_fts_index @@ -211,12 +215,77 @@ def create_app(config=None): return {'author': author} + # Template filter - URL to color for facepile (v1.6.0) + @app.template_filter('url_to_color') + def url_to_color_filter(url): + """ + Generate consistent color from URL hash for facepile initials + + Per display-moderation.md design: Hash URL to generate deterministic + color for facepile fallback initials. + + Args: + url: Author URL or any string + + Returns: + CSS color string (hex format) + """ + import hashlib + + if not url: + return '#6B7280' # Gray fallback + + # Hash the URL to get consistent color + hash_value = int(hashlib.md5(url.encode()).hexdigest()[:6], 16) + + # Generate pleasant, readable colors + # Use HSL-style generation for better color distribution + hue = hash_value % 360 + saturation = 65 # Medium saturation for readability + lightness = 50 # Medium lightness + + # Convert HSL to RGB (simplified) + def hsl_to_rgb(h, s, l): + s = s / 100.0 + l = l / 100.0 + c = (1 - abs(2 * l - 1)) * s + x = c * (1 - abs((h / 60) % 2 - 1)) + m = l - c / 2 + + if h < 60: + r, g, b = c, x, 0 + elif h < 120: + r, g, b = x, c, 0 + elif h < 180: + r, g, b = 0, c, x + elif h < 240: + r, g, b = 0, x, c + elif h < 300: + r, g, b = x, 0, c + else: + r, g, b = c, 0, x + + return int((r + m) * 255), int((g + m) * 255), int((b + m) * 255) + + r, g, b = hsl_to_rgb(hue, saturation, lightness) + return f'#{r:02x}{g:02x}{b:02x}' + # Request middleware - Add correlation ID to each request @app.before_request def before_request(): """Add correlation ID to request context for tracing""" add_correlation_id() + # Response middleware - Add webmention Link header (v1.6.0) + @app.after_request + def add_webmention_headers(response): + """Add webmention Link header to HTML responses""" + if (app.config.get('WEBMENTION_ENABLED', True) and + response.content_type and + 'text/html' in response.content_type): + response.headers['Link'] = '; rel="webmention"' + return response + # Register centralized error handlers from starpunk.errors import register_error_handlers @@ -237,6 +306,32 @@ def create_app(config=None): if hasattr(app, 'memory_monitor') and app.memory_monitor.is_alive(): app.memory_monitor.stop() + # Start webmention worker thread (v1.6.0 Phase 2) + # Per background-worker.md: Skip in test mode + if app.config.get('WEBMENTION_WORKER_ENABLED', True) and not app.config.get('TESTING', False): + from starpunk.webmention_worker import WebmentionWorker + webmention_worker = WebmentionWorker( + db_path=app.config['DATABASE_PATH'], + site_url=app.config['SITE_URL'], + poll_interval=app.config.get('WEBMENTION_POLL_INTERVAL', 10), + error_backoff=app.config.get('WEBMENTION_ERROR_BACKOFF', 60), + batch_size=app.config.get('WEBMENTION_BATCH_SIZE', 5), + auto_approve=app.config.get('WEBMENTION_AUTO_APPROVE', True) + ) + webmention_worker.start() + app.webmention_worker = webmention_worker + app.logger.info( + f"Webmention worker started " + f"(poll_interval={webmention_worker.poll_interval}s, " + f"batch_size={webmention_worker.batch_size})" + ) + + # Register cleanup handler + @app.teardown_appcontext + def cleanup_webmention_worker(error=None): + if hasattr(app, 'webmention_worker') and app.webmention_worker.is_alive(): + app.webmention_worker.stop() + # Health check endpoint for containers and monitoring @app.route("/health") def health_check(): @@ -322,15 +417,24 @@ def create_app(config=None): else: # Basic mode - just return 200 OK (for load balancers) # No authentication required, minimal checks - return jsonify({ + response = { "status": "ok", "version": app.config.get("VERSION", __version__) - }), 200 + } + + # Add webmention worker stats if enabled (v1.6.0 Phase 2) + if hasattr(app, 'webmention_worker'): + try: + response['webmention_worker'] = app.webmention_worker.get_stats() + except Exception as e: + app.logger.debug(f"Failed to get webmention worker stats: {e}") + + return jsonify(response), 200 return app # Package version (Semantic Versioning 2.0.0) # See docs/standards/versioning-strategy.md for details -__version__ = "1.5.0" -__version_info__ = (1, 5, 0) +__version__ = "1.6.0" +__version_info__ = (1, 6, 0) diff --git a/starpunk/config.py b/starpunk/config.py index 62ac68a..c38be42 100644 --- a/starpunk/config.py +++ b/starpunk/config.py @@ -110,6 +110,21 @@ def load_config(app, config_override=None): app.config["DEBUG_FILE_MAX_AGE_DAYS"] = int(os.getenv("DEBUG_FILE_MAX_AGE_DAYS", "7")) app.config["DEBUG_FILE_MAX_SIZE_MB"] = int(os.getenv("DEBUG_FILE_MAX_SIZE_MB", "100")) + # Webmention configuration (v1.6.0) + app.config["WEBMENTION_ENABLED"] = os.getenv("WEBMENTION_ENABLED", "true").lower() == "true" + app.config["WEBMENTION_WORKER_ENABLED"] = os.getenv("WEBMENTION_WORKER_ENABLED", "true").lower() == "true" + app.config["WEBMENTION_POLL_INTERVAL"] = int(os.getenv("WEBMENTION_POLL_INTERVAL", "10")) + app.config["WEBMENTION_ERROR_BACKOFF"] = int(os.getenv("WEBMENTION_ERROR_BACKOFF", "60")) + app.config["WEBMENTION_BATCH_SIZE"] = int(os.getenv("WEBMENTION_BATCH_SIZE", "5")) + app.config["WEBMENTION_AUTO_APPROVE"] = os.getenv("WEBMENTION_AUTO_APPROVE", "true").lower() == "true" + + # Webmention rate limiting (v1.6.0 Phase 2) + app.config["WEBMENTION_RATE_LIMIT_ENABLED"] = os.getenv("WEBMENTION_RATE_LIMIT_ENABLED", "true").lower() == "true" + app.config["WEBMENTION_RATE_LIMIT_IP"] = int(os.getenv("WEBMENTION_RATE_LIMIT_IP", "60")) + app.config["WEBMENTION_RATE_LIMIT_DOMAIN"] = int(os.getenv("WEBMENTION_RATE_LIMIT_DOMAIN", "10")) + app.config["WEBMENTION_RATE_LIMIT_WINDOW"] = int(os.getenv("WEBMENTION_RATE_LIMIT_WINDOW", "60")) + app.config["TRUSTED_PROXY"] = os.getenv("TRUSTED_PROXY", "false").lower() == "true" + # Apply overrides if provided if config_override: app.config.update(config_override) diff --git a/starpunk/media.py b/starpunk/media.py index c152f78..373c19c 100644 --- a/starpunk/media.py +++ b/starpunk/media.py @@ -14,15 +14,21 @@ Debug file management (v1.5.0 Phase 2): - Debug file saving disabled by default - Automatic cleanup of old debug files - Size limit enforcement + +Memory-safe image processing (v1.5.1): +- Early downscale for large images (>4096px) to prevent OOM +- PIL Image passed through pipeline (not bytes) to avoid memory copies +- Explicit memory cleanup after processing steps """ from PIL import Image, ImageOps from pathlib import Path from datetime import datetime, timedelta +import time import uuid import io import shutil -from typing import Optional, List, Dict, Tuple +from typing import Optional, List, Dict, Tuple, Union from flask import current_app # HEIC/HEIF support - import registers with Pillow automatically @@ -50,6 +56,11 @@ MIN_QUALITY = 70 # Minimum JPEG quality before rejection (v1.4. MIN_DIMENSION = 640 # Minimum dimension before rejection (v1.4.0) MAX_IMAGES_PER_NOTE = 4 +# Memory safety limits (v1.5.1 - fix for OOM on large phone images) +# Early downscale very large images to prevent memory exhaustion +# A 4096x4096 image = 50MB uncompressed, manageable per worker +MEMORY_SAFE_DIMENSION = 4096 # Downscale to this before heavy processing + # Variant specifications (v1.4.0 Phase 2) VARIANT_SPECS = { 'thumb': {'size': (150, 150), 'crop': True}, @@ -82,21 +93,22 @@ def get_optimization_params(file_size: int) -> Tuple[int, int]: return (1280, 85) -def validate_image(file_data: bytes, filename: str) -> Tuple[bytes, str, int, int]: +def validate_image(file_data: bytes, filename: str) -> Tuple[Image.Image, str, int, int]: """ - Validate image file + Validate image file and return PIL Image for memory-efficient processing Per Q11: Validate MIME type using Pillow - Per Q6: Reject if >50MB or >4096px (updated v1.4.0) + Per Q6: Reject if >50MB or >12000px Per v1.4.2: Convert HEIC to JPEG (browsers cannot display HEIC) + Per v1.5.1: Return PIL Image (not bytes) to avoid memory copies Args: file_data: Raw file bytes filename: Original filename Returns: - Tuple of (file_data, mime_type, width, height) - Note: file_data may be converted (e.g., HEIC to JPEG) + Tuple of (pil_image, mime_type, original_width, original_height) + Note: Image may be downscaled for memory safety but original dimensions returned Raises: ValueError: If file is invalid @@ -150,40 +162,34 @@ def validate_image(file_data: bytes, filename: str) -> Tuple[bytes, str, int, in else: raise ValueError(f"Invalid or corrupted image: {e}") + # Store original format before any conversions + original_format = img.format + # HEIC/HEIF conversion (v1.4.2) # HEIC cannot be displayed in browsers, convert to JPEG - if img.format in ('HEIF', 'HEIC'): + if original_format in ('HEIF', 'HEIC'): if not HEIC_SUPPORTED: raise ValueError( "HEIC/HEIF images require pillow-heif library. " "Please convert to JPEG before uploading." ) - # Convert HEIC to JPEG in memory - output = io.BytesIO() # Convert to RGB if needed (HEIC may have alpha channel) if img.mode in ('RGBA', 'P'): img = img.convert('RGB') - img.save(output, format='JPEG', quality=95) - output.seek(0) - # Re-open as JPEG for further processing - file_data = output.getvalue() - img = Image.open(io.BytesIO(file_data)) + # Mark format as JPEG for later processing + original_format = 'JPEG' # MPO (Multi-Picture Object) conversion (v1.4.2) - # MPO is used by iPhones for depth/portrait photos - extract primary image as JPEG - if img.format == 'MPO': - output = io.BytesIO() + # MPO is used by iPhones for depth/portrait photos - extract primary image + if original_format == 'MPO': # Convert to RGB if needed if img.mode in ('RGBA', 'P'): img = img.convert('RGB') - img.save(output, format='JPEG', quality=95) - output.seek(0) - file_data = output.getvalue() - img = Image.open(io.BytesIO(file_data)) + original_format = 'JPEG' # Check format is allowed - if img.format: - format_lower = img.format.lower() + if original_format: + format_lower = original_format.lower() mime_type = f'image/{format_lower}' # Special case: JPEG format can be reported as 'jpeg' @@ -195,26 +201,29 @@ def validate_image(file_data: bytes, filename: str) -> Tuple[bytes, str, int, in try: current_app.logger.warning( f'Media upload rejected format: filename="{filename}", ' - f'detected_format="{img.format}", mime_type="{mime_type}"' + f'detected_format="{original_format}", mime_type="{mime_type}"' ) except RuntimeError: pass # Outside app context - raise ValueError(f"Invalid image format '{img.format}'. Accepted: JPEG, PNG, GIF, WebP") + raise ValueError(f"Invalid image format '{original_format}'. Accepted: JPEG, PNG, GIF, WebP") else: raise ValueError("Could not determine image format") + # Store original dimensions before any resizing + orig_width, orig_height = img.size + # Check dimensions (v1.4.2: increased to 12000px to support modern phone cameras) - # Images will be resized by optimize_image() anyway - width, height = img.size - if max(width, height) > MAX_DIMENSION: + if max(orig_width, orig_height) > MAX_DIMENSION: raise ValueError(f"Image dimensions too large. Maximum is {MAX_DIMENSION}x{MAX_DIMENSION} pixels") # Check for animated GIF (v1.4.0) # Animated GIFs cannot be resized, so reject if >10MB - if img.format == 'GIF': + is_animated_gif = False + if original_format == 'GIF': try: img.seek(1) # Try to seek to second frame # If successful, it's animated + is_animated_gif = True if file_size > MAX_OUTPUT_SIZE: raise ValueError( "Animated GIF too large. Maximum size for animated GIFs is 10MB. " @@ -225,10 +234,38 @@ def validate_image(file_data: bytes, filename: str) -> Tuple[bytes, str, int, in # Not animated, continue normally pass - return file_data, mime_type, width, height + # v1.5.1: Early downscale for memory safety + # Large phone images (48MP+) can use 500MB+ per copy in memory + # Downscale early to MEMORY_SAFE_DIMENSION to reduce memory footprint + # This happens BEFORE optimize_image to prevent OOM during processing + if not is_animated_gif and max(orig_width, orig_height) > MEMORY_SAFE_DIMENSION: + try: + current_app.logger.info( + f'Memory safety downscale: {orig_width}x{orig_height} -> ' + f'{MEMORY_SAFE_DIMENSION}x{MEMORY_SAFE_DIMENSION} max' + ) + except RuntimeError: + pass # Outside app context + + # Apply EXIF orientation before resize + img = ImageOps.exif_transpose(img) + + # Use thumbnail() for memory-efficient in-place resize + img.thumbnail((MEMORY_SAFE_DIMENSION, MEMORY_SAFE_DIMENSION), Image.Resampling.LANCZOS) + + # Store the format on the image object for later use + img.format = original_format + + # Clear file_data reference to allow garbage collection + del file_data + + return img, mime_type, orig_width, orig_height -def optimize_image(image_data: bytes, original_size: int = None) -> Tuple[Image.Image, int, int, bytes]: +def optimize_image( + image_input: Union[Image.Image, bytes], + original_size: int = None +) -> Tuple[Image.Image, int, int, bytes]: """ Optimize image for web display with size-aware strategy @@ -237,8 +274,12 @@ def optimize_image(image_data: bytes, original_size: int = None) -> Tuple[Image. - Iterative quality reduction if needed - Target output <=10MB + Per v1.5.1: + - Accept PIL Image directly to avoid memory copies + - Image may already be downscaled for memory safety + Args: - image_data: Raw image bytes + image_input: PIL Image object or raw image bytes original_size: Original file size (for tiered optimization) Returns: @@ -247,24 +288,35 @@ def optimize_image(image_data: bytes, original_size: int = None) -> Tuple[Image. Raises: ValueError: If image cannot be optimized to target size """ - if original_size is None: - original_size = len(image_data) + # Handle both PIL Image and bytes input (v1.5.1) + if isinstance(image_input, Image.Image): + img = image_input + # For PIL input, use provided size or estimate from current dimensions + if original_size is None: + # Estimate: assume worst case of uncompressed RGB + original_size = img.width * img.height * 3 + else: + # Legacy bytes input path + if original_size is None: + original_size = len(image_input) + img = Image.open(io.BytesIO(image_input)) # Get initial optimization parameters based on input size max_dim, quality = get_optimization_params(original_size) - img = Image.open(io.BytesIO(image_data)) - # Save original format before any processing (copy() loses this) - original_format = img.format + original_format = img.format or 'JPEG' # Correct EXIF orientation (per ADR-058), except for GIFs - img = ImageOps.exif_transpose(img) if img.format != 'GIF' else img + # Note: For v1.5.1, this may already be done in validate_image for large images + if original_format != 'GIF': + img = ImageOps.exif_transpose(img) - # For animated GIFs, return as-is (already validated in validate_image) - if img.format == 'GIF' and getattr(img, 'is_animated', False): - # Already checked size in validate_image, just return original - return img, img.size[0], img.size[1], image_data + # For animated GIFs, serialize and return (already validated in validate_image) + if original_format == 'GIF' and getattr(img, 'is_animated', False): + output = io.BytesIO() + img.save(output, format='GIF') + return img, img.size[0], img.size[1], output.getvalue() # Iterative optimization loop while True: @@ -277,8 +329,7 @@ def optimize_image(image_data: bytes, original_size: int = None) -> Tuple[Image. # Save to bytes to check size output = io.BytesIO() - # Use original format (copy() loses the format attribute) - save_format = original_format or 'JPEG' + save_format = original_format save_kwargs = {'optimize': True} if save_format in ['JPEG', 'JPG']: @@ -528,6 +579,7 @@ def save_media(file_data: bytes, filename: str) -> Dict: Per Q2: Date-organized path: /media/YYYY/MM/uuid.ext Per Q6: Validate, optimize, then save Per v1.4.0: Size-aware optimization with iterative quality reduction + Per v1.6.0 Phase 0: Timing logs for performance monitoring Args: file_data: Raw file bytes @@ -544,24 +596,42 @@ def save_media(file_data: bytes, filename: str) -> Dict: # Capture file size for logging file_size = len(file_data) + # Timing instrumentation (v1.6.0 Phase 0) + timing_start = time.monotonic() + timing_validation = 0.0 + timing_optimization = 0.0 + timing_variants = 0.0 + timing_save = 0.0 + try: - # Validate image (returns 4-tuple with potentially converted bytes) + # Validate image (v1.5.1: returns PIL Image to avoid memory copies) try: - file_data, mime_type, orig_width, orig_height = validate_image(file_data, filename) + t_start = time.monotonic() + validated_img, mime_type, orig_width, orig_height = validate_image(file_data, filename) + timing_validation = time.monotonic() - t_start + # Clear file_data early to reduce memory pressure + del file_data except ValueError as e: + timing_validation = time.monotonic() - timing_start current_app.logger.warning( f'Media upload validation failed: filename="{filename}", ' - f'size={file_size}b, error="{e}"' + f'size={file_size}b, error="{e}", timing: validation={timing_validation:.1f}s' ) raise - # Optimize image with size-aware strategy (now returns 4-tuple with bytes) + # Optimize image with size-aware strategy (v1.5.1: accepts PIL Image) try: - optimized_img, width, height, optimized_bytes = optimize_image(file_data, file_size) + t_start = time.monotonic() + optimized_img, width, height, optimized_bytes = optimize_image(validated_img, file_size) + timing_optimization = time.monotonic() - t_start + # Clear validated_img reference (optimized_img is the result now) + del validated_img except ValueError as e: + timing_optimization = time.monotonic() - t_start current_app.logger.warning( f'Media upload optimization failed: filename="{filename}", ' - f'size={file_size}b, error="{e}"' + f'size={file_size}b, error="{e}", ' + f'timing: validation={timing_validation:.1f}s, optimization={timing_optimization:.1f}s' ) raise @@ -601,7 +671,8 @@ def save_media(file_data: bytes, filename: str) -> Dict: temp_subdir = None try: - # Step 1: Save original to temp directory + # Step 1: Generate variants to temp directory + t_start = time.monotonic() media_dir = Path(current_app.config.get('DATA_PATH', 'data')) / 'media' temp_dir = media_dir / '.tmp' temp_dir.mkdir(parents=True, exist_ok=True) @@ -623,8 +694,10 @@ def save_media(file_data: bytes, filename: str) -> Dict: optimized_bytes, db ) + timing_variants = time.monotonic() - t_start - # Step 3: Begin transaction + # Step 3: Begin transaction and save to database + t_start = time.monotonic() db.execute("BEGIN TRANSACTION") # Step 4: Insert media record @@ -667,6 +740,7 @@ def save_media(file_data: bytes, filename: str) -> Dict: # Step 7: Commit transaction (after files are moved successfully) db.commit() + timing_save = time.monotonic() - t_start # Step 8: Clean up temp directory try: @@ -679,6 +753,9 @@ def save_media(file_data: bytes, filename: str) -> Dict: variants = [v for v in variant_metadata if v['variant_type'] != 'original'] except Exception as e: + # Capture timing for failed save operation + timing_save = time.monotonic() - t_start if 't_start' in locals() else 0.0 + # Rollback database on any failure (best effort) try: db.rollback() @@ -722,19 +799,27 @@ def save_media(file_data: bytes, filename: str) -> Dict: except OSError: pass - # Log and re-raise + # Calculate total timing + timing_total = time.monotonic() - timing_start + + # Log and re-raise (with timing data) current_app.logger.warning( f'Media upload atomic operation failed: filename="{filename}", ' - f'error="{e}"' + f'error="{e}", timing: validation={timing_validation:.1f}s, ' + f'optimization={timing_optimization:.1f}s, variants={timing_variants:.1f}s, ' + f'save={timing_save:.1f}s, total={timing_total:.1f}s' ) raise - # Log success + # Log success with timing data (v1.6.0 Phase 0) + timing_total = time.monotonic() - timing_start was_optimized = len(optimized_bytes) < file_size current_app.logger.info( f'Media upload successful: filename="{filename}", ' f'stored="{stored_filename}", size={len(optimized_bytes)}b, ' - f'optimized={was_optimized}, variants={len(variants)}' + f'optimized={was_optimized}, variants={len(variants)}, ' + f'timing: validation={timing_validation:.1f}s, optimization={timing_optimization:.1f}s, ' + f'variants={timing_variants:.1f}s, save={timing_save:.1f}s, total={timing_total:.1f}s' ) return { diff --git a/starpunk/models.py b/starpunk/models.py index 70c95a8..58cc04d 100644 --- a/starpunk/models.py +++ b/starpunk/models.py @@ -114,6 +114,7 @@ class Note: # Optional fields deleted_at: Optional[datetime] = None content_hash: Optional[str] = None + in_reply_to: Optional[str] = None # v1.6.0: URL being replied to _cached_content: Optional[str] = field( default=None, repr=False, compare=False, init=False ) @@ -171,6 +172,7 @@ class Note: deleted_at=deleted_at, _data_dir=data_dir, content_hash=data.get("content_hash"), + in_reply_to=data.get("in_reply_to"), ) @property diff --git a/starpunk/notes.py b/starpunk/notes.py index a72de69..54de270 100644 --- a/starpunk/notes.py +++ b/starpunk/notes.py @@ -138,7 +138,8 @@ def create_note( published: bool = False, created_at: Optional[datetime] = None, custom_slug: Optional[str] = None, - tags: Optional[list[str]] = None + tags: Optional[list[str]] = None, + in_reply_to: Optional[str] = None ) -> Note: """ Create a new note @@ -153,6 +154,7 @@ def create_note( created_at: Creation timestamp (default: current UTC time) custom_slug: Optional custom slug (from Micropub mp-slug property) tags: Optional list of tag display names (v1.3.0) + in_reply_to: Optional URL being replied to (v1.6.0) Returns: Note object with all metadata and content loaded @@ -262,10 +264,10 @@ def create_note( try: db.execute( """ - INSERT INTO notes (slug, file_path, published, created_at, updated_at, content_hash) - VALUES (?, ?, ?, ?, ?, ?) + INSERT INTO notes (slug, file_path, published, created_at, updated_at, content_hash, in_reply_to) + VALUES (?, ?, ?, ?, ?, ?, ?) """, - (slug, file_path_rel, published, created_at, updated_at, content_hash), + (slug, file_path_rel, published, created_at, updated_at, content_hash, in_reply_to), ) db.commit() except Exception as e: @@ -313,6 +315,44 @@ def create_note( # FTS update failure should not prevent note creation current_app.logger.warning(f"Failed to update FTS index for note {slug}: {e}") + # 11. QUEUE WEBMENTIONS (v1.6.0 Phase 3) + # Only queue if note is published (not draft) + if published: + try: + from starpunk.webmention_sender import queue_webmentions_for_note + db_path = Path(current_app.config["DATABASE_PATH"]) + site_url = current_app.config["SITE_URL"] + note_url = f"{site_url}/note/{slug}" + + # Get rendered HTML from note + html_content = note.html + + # Queue webmentions (best-effort, failures logged but don't fail note creation) + queue_webmentions_for_note( + db_path=db_path, + note_id=note_id, + note_url=note_url, + content_html=html_content, + in_reply_to=in_reply_to + ) + except Exception as e: + # Webmention queueing failure should not prevent note creation + current_app.logger.warning(f"Failed to queue webmentions for note {slug}: {e}") + + # 12. FETCH REPLY CONTEXT (v1.6.0 Phase 3) + # Fetch context if this is a reply + if in_reply_to: + try: + from starpunk.reply_context import fetch_reply_context + db_path = Path(current_app.config["DATABASE_PATH"]) + site_url = current_app.config["SITE_URL"] + + # Fetch context (best-effort, failures logged but don't fail note creation) + fetch_reply_context(db_path, in_reply_to, site_url) + except Exception as e: + # Reply context fetch failure should not prevent note creation + current_app.logger.warning(f"Failed to fetch reply context for {in_reply_to}: {e}") + return note @@ -548,7 +588,8 @@ def update_note( id: Optional[int] = None, content: Optional[str] = None, published: Optional[bool] = None, - tags: Optional[list[str]] = None + tags: Optional[list[str]] = None, + in_reply_to: Optional[str] = None ) -> Note: """ Update a note's content and/or published status @@ -563,6 +604,7 @@ def update_note( content: New markdown content (None = no change) published: New published status (None = no change) tags: New tags list (None = no change, [] = remove all tags) (v1.3.0) + in_reply_to: New in-reply-to URL (None = no change) (v1.6.0) Returns: Updated Note object with new content and metadata @@ -618,8 +660,8 @@ def update_note( if slug is not None and id is not None: raise ValueError("Cannot provide both slug and id") - if content is None and published is None and tags is None: - raise ValueError("Must provide at least one of content, published, or tags to update") + if content is None and published is None and tags is None and in_reply_to is None: + raise ValueError("Must provide at least one of content, published, tags, or in_reply_to to update") # Validate content if provided if content is not None: @@ -629,7 +671,8 @@ def update_note( ) # 2. GET EXISTING NOTE - existing_note = get_note(slug=slug, id=id, load_content=False) + # Phase 6: Load content to get old HTML for webmention URL diffing + existing_note = get_note(slug=slug, id=id, load_content=True) if existing_note is None: identifier = slug if slug is not None else id @@ -648,7 +691,18 @@ def update_note( "Path validation failed", ) - # 4. UPDATE FILE (if content changed) + # 4. CAPTURE OLD HTML (Phase 6: for webmention URL diffing) + # Capture old HTML before writing new content + # Only if note is/was published (unpublished notes don't send webmentions) + old_html = None + if existing_note.published: + try: + old_html = existing_note.html + except Exception as e: + # HTML rendering failed - continue with update, skip old_html + current_app.logger.warning(f"Failed to render old HTML for note {existing_note.slug}: {e}") + + # 5. UPDATE FILE (if content changed) new_content_hash = existing_note.content_hash if content is not None: try: @@ -664,7 +718,7 @@ def update_note( f"Could not update note file: {note_path}", ) - # 5. UPDATE DATABASE + # 6. UPDATE DATABASE db = get_db(current_app) # Build update query based on what changed @@ -679,6 +733,10 @@ def update_note( update_fields.append("published = ?") params.append(published) + if in_reply_to is not None: + update_fields.append("in_reply_to = ?") + params.append(in_reply_to) + # Add WHERE clause parameter if slug is not None: where_clause = "slug = ?" @@ -726,9 +784,74 @@ def update_note( # FTS update failure should not prevent note update current_app.logger.warning(f"Failed to update FTS index for note {existing_note.slug}: {e}") - # 8. RETURN UPDATED NOTE + # 8. QUEUE WEBMENTIONS (v1.6.0 Phase 6) + # Queue webmentions if: + # - Note is being unpublished (notify removed URLs) + # - Note is being published (content changed OR publish status changed) + # - Note was published and content/in_reply_to changed updated_note = get_note(slug=existing_note.slug, load_content=True) + # Detect unpublishing (published -> unpublished) + is_unpublishing = existing_note.published and published is False + + # Check if content actually changed (Phase 6 optimization) + content_changed = (new_content_hash != existing_note.content_hash) + publish_status_changed = (published is not None and published != existing_note.published) + + # Determine if we should queue webmentions + should_queue = False + if is_unpublishing: + # Unpublishing - queue for ALL previously-linked URLs + should_queue = True + elif published is not None and published: + # Being published now - queue if content changed OR just being published + should_queue = content_changed or publish_status_changed + elif existing_note.published and (content_changed or in_reply_to is not None): + # Was already published and content/reply-to changed + should_queue = True + + if should_queue: + try: + from starpunk.webmention_sender import queue_webmentions_for_note + db_path = Path(current_app.config["DATABASE_PATH"]) + site_url = current_app.config["SITE_URL"] + note_url = f"{site_url}/note/{updated_note.slug}" + + # Get rendered HTML from updated note (if published) + html_content = updated_note.html if updated_note.published else None + + # Determine in_reply_to (use new value if provided, otherwise existing) + reply_to = in_reply_to if in_reply_to is not None else updated_note.in_reply_to + + # Queue webmentions (best-effort) + # Pass old_html for URL diffing (Phase 6) + queue_webmentions_for_note( + db_path=db_path, + note_id=updated_note.id, + note_url=note_url, + content_html=html_content, + in_reply_to=reply_to, + old_content_html=old_html # Phase 6: for URL diffing + ) + except Exception as e: + # Webmention queueing failure should not prevent note update + current_app.logger.warning(f"Failed to queue webmentions for note {updated_note.slug}: {e}") + + # 9. FETCH REPLY CONTEXT (v1.6.0 Phase 3) + # Fetch context if in_reply_to changed and is not None + if in_reply_to is not None: + try: + from starpunk.reply_context import fetch_reply_context + db_path = Path(current_app.config["DATABASE_PATH"]) + site_url = current_app.config["SITE_URL"] + + # Fetch context (best-effort) + fetch_reply_context(db_path, in_reply_to, site_url) + except Exception as e: + # Reply context fetch failure should not prevent note update + current_app.logger.warning(f"Failed to fetch reply context for {in_reply_to}: {e}") + + # 10. RETURN UPDATED NOTE return updated_note diff --git a/starpunk/rate_limiter.py b/starpunk/rate_limiter.py new file mode 100644 index 0000000..adf7f2d --- /dev/null +++ b/starpunk/rate_limiter.py @@ -0,0 +1,210 @@ +""" +Rate limiting for webmention endpoint + +Implements sliding window log algorithm with thread-safe in-memory storage. +Per ADR-064, this uses in-memory counters consistent with the existing +_discovery_cache pattern in webmention_worker.py. + +References: + - ADR-064: Webmention Rate Limiting + - Design: /docs/design/v1.6.0/rate-limiting.md +""" + +import threading +import time +from datetime import datetime, timedelta +from typing import Dict, List, Tuple + + +class RateLimiter: + """ + Thread-safe sliding window rate limiter. + + Uses sliding window log algorithm: + - Stores timestamp of each request in window + - Removes expired entries on each check + - Rejects if count >= limit + """ + + def __init__(self, limit: int, window_seconds: int): + """ + Initialize rate limiter. + + Args: + limit: Maximum requests allowed in window + window_seconds: Time window in seconds + """ + self.limit = limit + self.window_seconds = window_seconds + self._requests: Dict[str, List[float]] = {} + self._lock = threading.Lock() + + def is_allowed(self, key: str) -> bool: + """ + Check if request is allowed and record it if so. + + Args: + key: Identifier (e.g., IP address or domain) + + Returns: + True if request is allowed, False if rate limited + """ + with self._lock: + now = time.time() + cutoff = now - self.window_seconds + + # Get or create request log for this key + if key not in self._requests: + self._requests[key] = [] + + # Remove expired entries + self._requests[key] = [ts for ts in self._requests[key] if ts > cutoff] + + # Check if at or over limit + if len(self._requests[key]) >= self.limit: + return False + + # Allow request and record timestamp + self._requests[key].append(now) + return True + + def get_retry_after(self, key: str) -> int: + """ + Get seconds until oldest request expires (for Retry-After header). + + Args: + key: Identifier (e.g., IP address or domain) + + Returns: + Seconds until rate limit resets, or window_seconds if no requests + """ + with self._lock: + if key not in self._requests or not self._requests[key]: + return self.window_seconds + + now = time.time() + oldest_request = self._requests[key][0] + expires_at = oldest_request + self.window_seconds + retry_after = int(expires_at - now) + + # Return at least 1 second, max window_seconds + return max(1, min(retry_after, self.window_seconds)) + + def cleanup(self, max_idle_seconds: int = 300): + """ + Remove keys with no recent requests to prevent memory growth. + + Args: + max_idle_seconds: Remove keys idle for this many seconds + """ + with self._lock: + now = time.time() + cutoff = now - max_idle_seconds + keys_to_remove = [] + + for key, timestamps in self._requests.items(): + # Remove expired timestamps first + self._requests[key] = [ts for ts in timestamps if ts > cutoff] + # Mark empty keys for removal + if not self._requests[key]: + keys_to_remove.append(key) + + for key in keys_to_remove: + del self._requests[key] + + def get_request_count(self, key: str) -> int: + """ + Get current request count for a key (for testing/logging). + + Args: + key: Identifier (e.g., IP address or domain) + + Returns: + Number of requests in current window + """ + with self._lock: + if key not in self._requests: + return 0 + + now = time.time() + cutoff = now - self.window_seconds + # Count non-expired requests + return len([ts for ts in self._requests[key] if ts > cutoff]) + + +# Module-level rate limiters (initialized in init_rate_limiters) +_ip_limiter: RateLimiter | None = None +_domain_limiter: RateLimiter | None = None + + +def init_rate_limiters(app) -> None: + """ + Initialize rate limiters with Flask app configuration. + + Args: + app: Flask application instance + """ + global _ip_limiter, _domain_limiter + + if not app.config.get('WEBMENTION_RATE_LIMIT_ENABLED', True): + app.logger.info("Rate limiting disabled") + _ip_limiter = None + _domain_limiter = None + return + + ip_limit = app.config.get('WEBMENTION_RATE_LIMIT_IP', 60) + domain_limit = app.config.get('WEBMENTION_RATE_LIMIT_DOMAIN', 10) + window = app.config.get('WEBMENTION_RATE_LIMIT_WINDOW', 60) + + _ip_limiter = RateLimiter(ip_limit, window) + _domain_limiter = RateLimiter(domain_limit, window) + + app.logger.info( + f"Webmention rate limiter initialized: {ip_limit}/min IP, " + f"{domain_limit}/min domain" + ) + + +def check_rate_limit(ip: str, source_domain: str | None) -> Tuple[bool, str | None]: + """ + Check rate limits for a request. + + Args: + ip: Client IP address + source_domain: Source URL domain (optional) + + Returns: + (allowed, error_message) tuple + - allowed: True if request allowed, False if rate limited + - error_message: Error description if rate limited, None otherwise + """ + # If rate limiting disabled, allow all requests + if _ip_limiter is None: + return True, None + + # Check IP limit first + if not _ip_limiter.is_allowed(ip): + retry_after = _ip_limiter.get_retry_after(ip) + count = _ip_limiter.get_request_count(ip) + return False, f"Too many requests from your IP. Please retry after {retry_after} seconds." + + # Check domain limit if source domain provided + if source_domain and _domain_limiter is not None: + if not _domain_limiter.is_allowed(source_domain): + retry_after = _domain_limiter.get_retry_after(source_domain) + count = _domain_limiter.get_request_count(source_domain) + return False, f"Too many requests from source domain. Please retry after {retry_after} seconds." + + return True, None + + +def cleanup_rate_limiters() -> None: + """ + Cleanup expired entries from rate limiters. + + Called periodically to prevent memory growth. + """ + if _ip_limiter: + _ip_limiter.cleanup() + if _domain_limiter: + _domain_limiter.cleanup() diff --git a/starpunk/reply_context.py b/starpunk/reply_context.py new file mode 100644 index 0000000..8a61e2b --- /dev/null +++ b/starpunk/reply_context.py @@ -0,0 +1,329 @@ +""" +Reply context fetching for in-reply-to URLs + +Per database-schema.md Story 2.4: +- Fetch context from in_reply_to URL when note is reply +- Parse microformats for author, content +- Store in reply_contexts table +- Graceful degradation on fetch failure + +References: + - Database Schema: /docs/design/v1.6.0/database-schema.md + - Background Worker: /docs/design/v1.6.0/background-worker.md +""" + +import logging +import sqlite3 +from datetime import datetime +from pathlib import Path +from typing import Optional, Dict, Any + +from starpunk.webmention_processor import ( + fetch_source, + extract_author, + extract_content, + ValidationError +) +import mf2py + + +logger = logging.getLogger(__name__) + + +def fetch_reply_context( + db_path: Path, + reply_url: str, + site_url: str +) -> Dict[str, Any]: + """ + Fetch and parse reply context from a URL. + + Per database-schema.md: + - Fetch URL content with SSRF protection + - Parse h-entry for title, content, published date + - Parse h-card for author information + - Store in reply_contexts table + - Graceful degradation: errors don't prevent reply creation + + Args: + db_path: Path to SQLite database + reply_url: URL being replied to + site_url: Site URL for User-Agent header + + Returns: + Dict with fetch status and context data + """ + # Check if context already exists + try: + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + + cursor.execute(""" + SELECT * FROM reply_contexts WHERE url = ? + """, (reply_url,)) + + existing = cursor.fetchone() + conn.close() + + if existing: + logger.debug(f"Reply context already cached for {reply_url}") + return dict(existing) + + except Exception as e: + logger.warning(f"Failed to check existing context: {e}") + + # Fetch and parse context + try: + # Fetch URL content + content, content_type = fetch_source(reply_url, site_url) + + # Parse microformats (only for HTML) + if 'html' not in content_type.lower(): + # Not HTML - store minimal context + return _store_minimal_context(db_path, reply_url, "Not HTML content") + + parsed_mf2 = mf2py.parse(doc=content, url=reply_url) + + # Extract h-entry data + entry_data = _extract_entry_data(parsed_mf2) + + # Extract author data + author_data = extract_author(parsed_mf2, reply_url) + + # Store in database + context = _store_context( + db_path, + reply_url, + entry_data.get('title'), + entry_data.get('content_snippet'), + entry_data.get('published_at'), + author_data.get('author_name'), + author_data.get('author_url'), + author_data.get('author_photo'), + entry_data.get('site_name') + ) + + logger.info(f"Fetched reply context for {reply_url}") + return context + + except ValidationError as e: + # Fetch/validation error - store error status + logger.warning(f"Failed to fetch reply context for {reply_url}: {e}") + return _store_minimal_context(db_path, reply_url, str(e)) + + except Exception as e: + # Unexpected error - store error status + logger.error(f"Error fetching reply context for {reply_url}: {e}", exc_info=True) + return _store_minimal_context(db_path, reply_url, f"Unexpected error: {e}") + + +def _extract_entry_data(parsed_mf2: Dict) -> Dict[str, Any]: + """ + Extract entry data from parsed microformats. + + Args: + parsed_mf2: Parsed microformats data from mf2py + + Returns: + Dict with title, content_snippet, published_at, site_name + """ + title = None + content_snippet = None + published_at = None + site_name = None + + # Find h-entry + for item in parsed_mf2.get('items', []): + if 'h-entry' not in item.get('type', []): + continue + + props = item.get('properties', {}) + + # Extract title (p-name) + name = props.get('name', []) + if name: + title = name[0] + if isinstance(title, dict): + title = title.get('value', '') + # Truncate if too long + if title and len(title) > 200: + title = title[:197] + '...' + + # Extract content (reuse processor function) + content_snippet = extract_content(parsed_mf2) + + # Extract published date + published = props.get('published', []) + if published: + pub_str = published[0] + if isinstance(pub_str, dict): + pub_str = pub_str.get('value', '') + # Try to parse ISO 8601 datetime + try: + from dateutil import parser as dateutil_parser + published_at = dateutil_parser.parse(pub_str) + except Exception: + # Parsing failed, leave as None + pass + + break # Use first h-entry + + # Try to extract site name from h-card + for item in parsed_mf2.get('items', []): + if 'h-card' in item.get('type', []): + props = item.get('properties', {}) + name = props.get('name', []) + if name: + site_name = name[0] + if isinstance(site_name, dict): + site_name = site_name.get('value', '') + break + + return { + 'title': title, + 'content_snippet': content_snippet, + 'published_at': published_at, + 'site_name': site_name + } + + +def _store_context( + db_path: Path, + url: str, + title: Optional[str], + content_snippet: Optional[str], + published_at: Optional[datetime], + author_name: Optional[str], + author_url: Optional[str], + author_photo: Optional[str], + site_name: Optional[str] +) -> Dict[str, Any]: + """ + Store reply context in database. + + Args: + db_path: Database path + url: Reply-to URL + title: Entry title + content_snippet: Content snippet + published_at: Published datetime + author_name: Author name + author_url: Author URL + author_photo: Author photo URL + site_name: Site name + + Returns: + Stored context dict + """ + try: + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + + cursor.execute(""" + INSERT OR REPLACE INTO reply_contexts + (url, title, content_snippet, published_at, + author_name, author_url, author_photo, site_name, + fetch_status, fetched_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'fetched', CURRENT_TIMESTAMP) + """, ( + url, title, content_snippet, published_at, + author_name, author_url, author_photo, site_name + )) + + conn.commit() + conn.close() + + return { + 'url': url, + 'title': title, + 'content_snippet': content_snippet, + 'published_at': published_at, + 'author_name': author_name, + 'author_url': author_url, + 'author_photo': author_photo, + 'site_name': site_name, + 'fetch_status': 'fetched' + } + + except Exception as e: + logger.error(f"Failed to store reply context: {e}") + raise + + +def _store_minimal_context( + db_path: Path, + url: str, + error: str +) -> Dict[str, Any]: + """ + Store minimal context with error status. + + Args: + db_path: Database path + url: Reply-to URL + error: Error message + + Returns: + Minimal context dict with error + """ + try: + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + + cursor.execute(""" + INSERT OR REPLACE INTO reply_contexts + (url, fetch_status, fetch_error, created_at) + VALUES (?, 'error', ?, CURRENT_TIMESTAMP) + """, (url, error)) + + conn.commit() + conn.close() + + return { + 'url': url, + 'fetch_status': 'error', + 'fetch_error': error + } + + except Exception as e: + logger.error(f"Failed to store minimal context: {e}") + # Return in-memory dict even if storage fails + return { + 'url': url, + 'fetch_status': 'error', + 'fetch_error': error + } + + +def get_reply_context(db_path: Path, reply_url: str) -> Optional[Dict[str, Any]]: + """ + Get cached reply context from database. + + Args: + db_path: Database path + reply_url: Reply-to URL + + Returns: + Context dict if exists, None otherwise + """ + try: + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + cursor = conn.cursor() + + cursor.execute(""" + SELECT * FROM reply_contexts WHERE url = ? + """, (reply_url,)) + + row = cursor.fetchone() + conn.close() + + if row: + return dict(row) + + return None + + except Exception as e: + logger.error(f"Failed to get reply context: {e}") + return None diff --git a/starpunk/routes/__init__.py b/starpunk/routes/__init__.py index 1a9c690..0ac464a 100644 --- a/starpunk/routes/__init__.py +++ b/starpunk/routes/__init__.py @@ -7,7 +7,7 @@ admin, auth, and (conditionally) dev auth routes. from flask import Flask -from starpunk.routes import admin, auth, micropub, public, search +from starpunk.routes import admin, auth, micropub, public, search, webmention def register_routes(app: Flask) -> None: @@ -39,6 +39,10 @@ def register_routes(app: Flask) -> None: # Register search routes app.register_blueprint(search.bp) + # Register webmention routes (v1.6.0) + if app.config.get("WEBMENTION_ENABLED", True): + app.register_blueprint(webmention.bp) + # Conditionally register dev auth routes if app.config.get("DEV_MODE"): app.logger.warning( diff --git a/starpunk/routes/admin.py b/starpunk/routes/admin.py index 69fc38f..057b0ac 100644 --- a/starpunk/routes/admin.py +++ b/starpunk/routes/admin.py @@ -6,10 +6,13 @@ editing, and deletion. All routes require authentication. """ from flask import Blueprint, flash, g, jsonify, redirect, render_template, request, url_for +import logging import os import shutil from datetime import datetime +logger = logging.getLogger(__name__) + from starpunk.auth import require_auth from starpunk.notes import ( create_note, @@ -32,17 +35,51 @@ def dashboard(): Displays all notes (published and drafts) with management controls. Requires authentication. + Per v1.6.0: Added pagination (25 items per page) + Returns: - Rendered dashboard template with complete note list + Rendered dashboard template with paginated note list Decorator: @require_auth Template: templates/admin/dashboard.html Access: g.user_me (set by require_auth decorator) """ - # Get all notes (published and drafts) - notes = list_notes() + from flask import current_app + import math - return render_template("admin/dashboard.html", notes=notes, user_me=g.me) + # Get pagination params (v1.6.0) + page = int(request.args.get('page', 1)) + per_page = 25 + + # Get all notes (published and drafts) + all_notes = list_notes() + + # Calculate pagination + total_notes = len(all_notes) + total_pages = math.ceil(total_notes / per_page) if total_notes > 0 else 1 + + # Slice for current page + start_idx = (page - 1) * per_page + end_idx = start_idx + per_page + notes = all_notes[start_idx:end_idx] + + # Get pending webmention count (v1.6.0) + webmention_pending_count = 0 + if current_app.config.get('WEBMENTION_ENABLED', False): + try: + from starpunk.webmention_queries import count_webmentions + webmention_pending_count = count_webmentions(status_filter='pending') + except Exception: + pass + + return render_template( + "admin/dashboard.html", + notes=notes, + page=page, + total_pages=total_pages, + webmention_pending_count=webmention_pending_count, + user_me=g.me + ) @bp.route("/new", methods=["GET"]) @@ -600,3 +637,182 @@ def health_diagnostics(): diagnostics["status"] = "healthy" if overall_healthy else "unhealthy" return jsonify(diagnostics), 200 if overall_healthy else 500 + + +# Webmention Admin Routes (v1.6.0) + +@bp.route("/webmentions") +@require_auth +def webmentions(): + """ + Webmention list with filtering and pagination (v1.6.0 Phase 5) + + Lists all webmentions with status filters and pagination. + + Query params: + status: Filter by moderation_status ('pending', 'approved', 'hidden') + page: Page number (default: 1) + + Returns: + Rendered webmentions list template + + Decorator: @require_auth + Template: templates/admin/webmentions.html + """ + from starpunk.webmention_queries import list_webmentions + + status_filter = request.args.get('status') + page = int(request.args.get('page', 1)) + + webmentions, total_pages = list_webmentions( + status_filter=status_filter, + page=page, + per_page=25 + ) + + return render_template( + "admin/webmentions.html", + webmentions=webmentions, + status_filter=status_filter, + page=page, + total_pages=total_pages, + user_me=g.me + ) + + +@bp.route("/webmentions/") +@require_auth +def webmention_detail(id: int): + """ + Webmention detail view (v1.6.0 Phase 5) + + Shows full details of a single webmention with moderation actions. + + Args: + id: Webmention database ID + + Returns: + Rendered webmention detail template or 404 if not found + + Decorator: @require_auth + Template: templates/admin/webmention_detail.html + """ + from starpunk.webmention_queries import get_webmention + + webmention = get_webmention(id) + + if not webmention: + flash("Webmention not found", "error") + return redirect(url_for("admin.webmentions")), 404 + + return render_template( + "admin/webmention_detail.html", + webmention=webmention, + user_me=g.me + ) + + +@bp.route("/webmentions//action", methods=["POST"]) +@require_auth +def webmention_action(id: int): + """ + Individual webmention moderation action (v1.6.0 Phase 5) + + Approve, hide, or delete a single webmention. + + Args: + id: Webmention database ID + + Form data: + action: 'approve', 'hide', or 'delete' + + Returns: + Redirect to webmention detail or list + + Decorator: @require_auth + """ + from starpunk.webmention_queries import update_webmention_status, delete_webmentions + + action = request.form.get('action') + + if action not in ('approve', 'hide', 'delete'): + flash("Invalid action", "error") + return redirect(url_for("admin.webmention_detail", id=id)) + + try: + if action == 'delete': + deleted = delete_webmentions([id]) + if deleted > 0: + flash("Webmention deleted", "success") + else: + flash("Webmention not found", "error") + return redirect(url_for("admin.webmentions")) + else: + # Approve or hide + status = 'approved' if action == 'approve' else 'hidden' + updated = update_webmention_status([id], status) + if updated > 0: + flash(f"Webmention {status}", "success") + else: + flash("Webmention not found", "error") + return redirect(url_for("admin.webmention_detail", id=id)) + + except Exception as e: + logger.error(f"Error performing webmention action: {e}") + flash(f"Error: {e}", "error") + return redirect(url_for("admin.webmention_detail", id=id)) + + +@bp.route("/webmentions/bulk-action", methods=["POST"]) +@require_auth +def webmention_bulk_action(): + """ + Bulk webmention moderation actions (v1.6.0 Phase 5) + + Approve, hide, or delete multiple webmentions at once. + + Form data: + action: 'approve', 'hide', or 'delete' + ids[]: List of webmention IDs + + Returns: + Redirect to webmentions list + + Decorator: @require_auth + """ + from starpunk.webmention_queries import update_webmention_status, delete_webmentions + + action = request.form.get('action') + ids = request.form.getlist('ids') + + if not ids: + flash("No webmentions selected", "warning") + return redirect(url_for("admin.webmentions")) + + # Convert IDs to integers + try: + webmention_ids = [int(id) for id in ids] + except ValueError: + flash("Invalid webmention IDs", "error") + return redirect(url_for("admin.webmentions")) + + if action not in ('approve', 'hide', 'delete'): + flash("Invalid action", "error") + return redirect(url_for("admin.webmentions")) + + try: + if action == 'delete': + deleted = delete_webmentions(webmention_ids) + flash(f"Deleted {deleted} webmention(s)", "success") + else: + # Approve or hide + status = 'approved' if action == 'approve' else 'hidden' + updated = update_webmention_status(webmention_ids, status) + flash(f"Updated {updated} webmention(s) to {status}", "success") + + return redirect(url_for("admin.webmentions")) + + except Exception as e: + logger.error(f"Error performing bulk action: {e}") + flash(f"Error: {e}", "error") + return redirect(url_for("admin.webmentions")) diff --git a/starpunk/routes/public.py b/starpunk/routes/public.py index 7167f00..4ead4c4 100644 --- a/starpunk/routes/public.py +++ b/starpunk/routes/public.py @@ -281,6 +281,8 @@ def note(slug: str): Template: templates/note.html Microformats: h-entry + + Per v1.6.0: Added webmentions and reply context display """ from starpunk.media import get_note_media from starpunk.tags import get_note_tags @@ -303,7 +305,39 @@ def note(slug: str): tags = get_note_tags(note_obj.id) object.__setattr__(note_obj, '_cached_tags', tags) - return render_template("note.html", note=note_obj) + # Get webmentions for note (v1.6.0 Phase 5) + webmentions = None + reply_context = None + + if current_app.config.get('WEBMENTION_ENABLED', False): + try: + from starpunk.webmention_queries import get_webmentions_for_note, get_reply_context + + # Get approved webmentions grouped by type + grouped = get_webmentions_for_note(note_obj.id) + + # Check if there are any webmentions + has_webmentions = any(grouped.values()) + if has_webmentions: + webmentions = grouped + + # Get reply context if this is a reply post + if note_obj.in_reply_to: + reply_context = get_reply_context(note_obj.in_reply_to) + + except Exception as e: + # Log error but don't fail page load + import logging + logger = logging.getLogger(__name__) + logger.error(f"Error loading webmentions for note {slug}: {e}") + + return render_template( + "note.html", + note=note_obj, + webmentions=webmentions, + reply_context=reply_context, + **grouped if webmentions else {} + ) @bp.route("/tag/") diff --git a/starpunk/routes/webmention.py b/starpunk/routes/webmention.py new file mode 100644 index 0000000..be8267c --- /dev/null +++ b/starpunk/routes/webmention.py @@ -0,0 +1,272 @@ +""" +Webmention routes for StarPunk + +Per W3C Webmention Recommendation and ADR-063: +- POST /webmention - Receive webmention notifications +- GET /webmention - Optional endpoint information + +References: + - W3C Webmention Specification: https://www.w3.org/TR/webmention/ + - ADR-063: Webmention Implementation +""" + +from flask import Blueprint, current_app, request, jsonify +from urllib.parse import urlparse +from starpunk.database import get_db +from starpunk.notes import get_note, NoteNotFoundError +from starpunk.rate_limiter import check_rate_limit + +bp = Blueprint("webmention", __name__) + + +def get_client_ip() -> str: + """ + Get client IP address, accounting for reverse proxy. + + If TRUSTED_PROXY is enabled, uses X-Forwarded-For header. + Otherwise uses request.remote_addr directly. + """ + if current_app.config.get('TRUSTED_PROXY', False): + forwarded = request.headers.get('X-Forwarded-For') + if forwarded: + # First IP in chain is the client + return forwarded.split(',')[0].strip() + return request.remote_addr or '0.0.0.0' + + +@bp.before_request +def rate_limit_webmention(): + """ + Check rate limits before processing webmention requests. + + Per ADR-064, checks: + 1. IP rate limit (60/min by default) + 2. Source domain rate limit (10/min by default) + + Returns 429 with Retry-After header if rate limited. + """ + # Only rate limit POST requests (the actual webmention submission) + if request.method != 'POST': + return None + + # Get client IP + ip = get_client_ip() + + # Get source domain from request (if available) + source_domain = None + content_type = request.headers.get("Content-Type", "") + + if "application/json" in content_type: + data = request.get_json(silent=True) or {} + source = data.get('source') + else: + source = request.form.get('source') + + if source: + try: + source_domain = urlparse(source).netloc + except Exception: + # Invalid URL, domain check will be skipped + # This is acceptable - IP limit still applies + current_app.logger.debug(f"Could not parse source URL for domain extraction: {source}") + + # Check rate limits + allowed, error = check_rate_limit(ip, source_domain) + if not allowed: + current_app.logger.warning( + f"Rate limit exceeded - IP: {ip}, Domain: {source_domain or 'N/A'}" + ) + return jsonify({ + "error": "rate_limit_exceeded", + "error_description": error + }), 429, {'Retry-After': str(current_app.config.get('WEBMENTION_RATE_LIMIT_WINDOW', 60))} + + return None + + +def validate_url(url: str) -> tuple[bool, str | None]: + """ + Validate a URL is well-formed and uses http/https scheme. + + Returns: + (is_valid, error_message) + """ + if not url: + return False, "URL is required" + + try: + parsed = urlparse(url) + if parsed.scheme not in ('http', 'https'): + return False, "Only http and https URLs are supported" + if not parsed.netloc: + return False, "URL must include a hostname" + if len(url) > 2048: + return False, "URL exceeds maximum length of 2048 characters" + return True, None + except Exception: + return False, "URL is malformed" + + +def is_target_on_site(target_url: str) -> bool: + """ + Check if target URL is on this site. + + Compares hostname of target_url with SITE_URL configuration. + """ + site_url = current_app.config.get('SITE_URL', '') + site_parsed = urlparse(site_url) + target_parsed = urlparse(target_url) + + return site_parsed.netloc == target_parsed.netloc + + +def resolve_target_note(target_url: str) -> int | None: + """ + Resolve target URL to a note ID if it points to a note. + + Returns: + Note ID if target is a valid note, None otherwise + """ + target_parsed = urlparse(target_url) + + # Strip query params and fragments for matching + target_path = target_parsed.path.rstrip('/') + + # Check if it's a note URL pattern: /note/{slug} + if target_path.startswith('/note/'): + slug = target_path[6:] # Remove '/note/' prefix + if slug: + try: + note = get_note(slug=slug) + # get_note() returns None if not found, or a Note object + if note and note.published: + return note.id + except NoteNotFoundError: + pass + + # Homepage or other valid page (not a note) + return None + + +@bp.route("/webmention", methods=["GET"]) +def webmention_info(): + """ + Optional endpoint information. + + Returns basic information about the webmention endpoint. + """ + return jsonify({ + "name": "Webmention endpoint", + "spec": "https://www.w3.org/TR/webmention/" + }) + + +@bp.route("/webmention", methods=["POST"]) +def receive_webmention(): + """ + Receive webmention notification per W3C spec. + + Accepts application/x-www-form-urlencoded (required) and + application/json (optional extension). + + Returns: + 202 Accepted with JSON body on success + 400 Bad Request with error details on failure + """ + # Parse request based on content type + content_type = request.headers.get("Content-Type", "") + + if "application/json" in content_type: + data = request.get_json() or {} + source = data.get("source") + target = data.get("target") + else: + # Default to form-encoded (required by W3C spec) + source = request.form.get("source") + target = request.form.get("target") + + # Validate required parameters + if not source: + return jsonify({ + "error": "invalid_request", + "error_description": "Missing required parameter: source" + }), 400 + + if not target: + return jsonify({ + "error": "invalid_request", + "error_description": "Missing required parameter: target" + }), 400 + + # Validate source URL + source_valid, source_error = validate_url(source) + if not source_valid: + return jsonify({ + "error": "invalid_request", + "error_description": f"Source must be a valid URL" + }), 400 + + # Validate target URL + target_valid, target_error = validate_url(target) + if not target_valid: + return jsonify({ + "error": "invalid_request", + "error_description": f"Target must be a valid URL" + }), 400 + + # Validate source and target are different + if source == target: + return jsonify({ + "error": "invalid_request", + "error_description": "Source and target must be different" + }), 400 + + # Validate target is on this site + if not is_target_on_site(target): + return jsonify({ + "error": "invalid_target", + "error_description": "Target URL is not on this site" + }), 400 + + # Resolve target to note (may be None for homepage/other pages) + target_note_id = resolve_target_note(target) + + # If target looks like a note URL but doesn't resolve, it's not found + target_parsed = urlparse(target) + target_path = target_parsed.path.rstrip('/') + if target_path.startswith('/note/') and target_note_id is None: + return jsonify({ + "error": "invalid_target", + "error_description": "Target URL not found" + }), 400 + + # Insert webmention into database (queued for validation) + # Use INSERT ... ON CONFLICT to handle duplicate webmentions + db = get_db() + try: + db.execute(""" + INSERT INTO webmentions (source_url, target_url, target_note_id) + VALUES (?, ?, ?) + ON CONFLICT(source_url, target_url) DO UPDATE SET + validation_status = 'pending', + updated_at = CURRENT_TIMESTAMP + """, (source, target, target_note_id)) + db.commit() + + current_app.logger.info( + f"Webmention received: {source} -> {target}" + f"{' (note_id=' + str(target_note_id) + ')' if target_note_id else ''}" + ) + + return jsonify({ + "status": "accepted", + "message": "Webmention queued for verification" + }), 202 + + except Exception as e: + db.rollback() + current_app.logger.error(f"Failed to queue webmention: {e}") + return jsonify({ + "error": "server_error", + "error_description": "Failed to queue webmention" + }), 500 diff --git a/starpunk/webmention_processor.py b/starpunk/webmention_processor.py new file mode 100644 index 0000000..291df24 --- /dev/null +++ b/starpunk/webmention_processor.py @@ -0,0 +1,373 @@ +""" +Webmention validation and processing utilities + +Per background-worker.md and api-contract.md: +- Fetch and validate source URLs +- Parse microformats with mf2py +- Extract author information from h-cards +- Detect mention types (reply, like, repost, etc.) +- Sanitize content with nh3 + +References: + - Background Worker Design: /docs/design/v1.6.0/background-worker.md + - API Contract: /docs/design/v1.6.0/api-contract.md + - W3C Webmention: https://www.w3.org/TR/webmention/ +""" + +import logging +import mf2py +import nh3 +import httpx +from datetime import datetime +from typing import Dict, Any, Optional +from urllib.parse import urlparse + +from starpunk.webmention_ssrf import validate_url_for_fetch, SSRFBlockedError + + +logger = logging.getLogger(__name__) + + +class ValidationError(Exception): + """Raised when webmention validation fails""" + pass + + +def fetch_source(url: str, site_url: str, timeout: int = 10, max_size: int = 1048576) -> tuple[str, str]: + """ + Fetch source URL content with SSRF protection. + + Args: + url: Source URL to fetch + site_url: Site URL for User-Agent header + timeout: Request timeout in seconds (default: 10) + max_size: Maximum content size in bytes (default: 1MB) + + Returns: + Tuple of (content, content_type) + + Raises: + SSRFBlockedError: If URL points to private IP + ValidationError: If fetch fails or content too large + """ + # SSRF protection + try: + validate_url_for_fetch(url) + except (SSRFBlockedError, ValueError) as e: + logger.warning(f"SSRF protection blocked URL {url}: {e}") + raise ValidationError(f"Blocked request: {e}") + + # Fetch URL + try: + # Use streaming to check size before downloading entire content + with httpx.stream( + 'GET', + url, + timeout=timeout, + headers={'User-Agent': f'StarPunk/1.6.0 (+{site_url})'}, + follow_redirects=True + ) as response: + response.raise_for_status() + + # Check content length if provided + content_length = response.headers.get('Content-Length') + if content_length and int(content_length) > max_size: + raise ValidationError(f"Source exceeds size limit ({max_size} bytes)") + + # Read content with size limit + content = b'' + for chunk in response.iter_bytes(chunk_size=8192): + content += chunk + if len(content) > max_size: + raise ValidationError(f"Source exceeds size limit ({max_size} bytes)") + + content_type = response.headers.get('Content-Type', '') + + return content.decode('utf-8', errors='replace'), content_type + + except httpx.TimeoutException: + raise ValidationError("Source fetch timed out") + except httpx.HTTPStatusError as e: + if 400 <= e.response.status_code < 500: + raise ValidationError(f"Source returned {e.response.status_code}") + else: + raise ValidationError(f"Source server error: {e.response.status_code}") + except httpx.HTTPError as e: + raise ValidationError(f"Failed to fetch source: {e}") + except UnicodeDecodeError: + raise ValidationError("Source content encoding is not supported") + + +def verify_link_presence(content: str, target_url: str, content_type: str) -> bool: + """ + Verify that source content contains a link to target URL. + + Args: + content: Source content (HTML or text) + target_url: Target URL to find + content_type: Content type of source + + Returns: + True if target link found, False otherwise + """ + # Normalize URLs for comparison (remove fragments, trailing slashes) + target_normalized = urlparse(target_url)._replace(fragment='').geturl().rstrip('/') + + # For HTML, parse and check all links + if 'html' in content_type.lower(): + try: + from lxml import html as lxml_html + doc = lxml_html.fromstring(content) + links = doc.xpath('//a/@href') + + for link in links: + # Make absolute if relative + link_normalized = urlparse(link)._replace(fragment='').geturl().rstrip('/') + if link_normalized == target_normalized: + return True + except Exception as e: + logger.debug(f"HTML parsing failed, falling back to string search: {e}") + + # Fallback: simple string search + return target_normalized in content + + +def detect_mention_type(parsed_mf2: Dict, target_url: str) -> str: + """ + Detect mention type from microformats. + + Priority: reply > like > repost > bookmark > mention + + Args: + parsed_mf2: Parsed microformats data from mf2py + target_url: Target URL being mentioned + + Returns: + Mention type: 'reply', 'like', 'repost', 'bookmark', or 'mention' + """ + target_normalized = urlparse(target_url)._replace(fragment='').geturl().rstrip('/') + + # Find h-entry that references target + for item in parsed_mf2.get('items', []): + if 'h-entry' not in item.get('type', []): + continue + + props = item.get('properties', {}) + + # Check in-reply-to (highest priority) + for reply_to in props.get('in-reply-to', []): + url = reply_to if isinstance(reply_to, str) else reply_to.get('value', '') + if urlparse(url)._replace(fragment='').geturl().rstrip('/') == target_normalized: + return 'reply' + + # Check like-of + for like in props.get('like-of', []): + url = like if isinstance(like, str) else like.get('value', '') + if urlparse(url)._replace(fragment='').geturl().rstrip('/') == target_normalized: + return 'like' + + # Check repost-of + for repost in props.get('repost-of', []): + url = repost if isinstance(repost, str) else repost.get('value', '') + if urlparse(url)._replace(fragment='').geturl().rstrip('/') == target_normalized: + return 'repost' + + # Check bookmark-of + for bookmark in props.get('bookmark-of', []): + url = bookmark if isinstance(bookmark, str) else bookmark.get('value', '') + if urlparse(url)._replace(fragment='').geturl().rstrip('/') == target_normalized: + return 'bookmark' + + # Default fallback + return 'mention' + + +def extract_author(parsed_mf2: Dict, source_url: str) -> Dict[str, Optional[str]]: + """ + Extract author information from h-card. + + Args: + parsed_mf2: Parsed microformats data from mf2py + source_url: Source URL (fallback for author URL) + + Returns: + Dict with author_name, author_url, author_photo + """ + author_name = None + author_url = None + author_photo = None + + # Find h-entry first + entry = None + for item in parsed_mf2.get('items', []): + if 'h-entry' in item.get('type', []): + entry = item + break + + # Check for author property in h-entry + if entry: + author_prop = entry.get('properties', {}).get('author', []) + if author_prop: + author_data = author_prop[0] + if isinstance(author_data, dict): + # h-card object + props = author_data.get('properties', {}) + author_name = props.get('name', [None])[0] + author_url = props.get('url', [None])[0] + author_photo = props.get('photo', [None])[0] + + # If no author in h-entry, find representative h-card + if not author_name: + for item in parsed_mf2.get('items', []): + if 'h-card' in item.get('type', []): + props = item.get('properties', {}) + author_name = props.get('name', [None])[0] + author_url = props.get('url', [None])[0] + author_photo = props.get('photo', [None])[0] + break + + # Fallback to hostname if no author found + if not author_name: + author_name = urlparse(source_url).netloc + + if not author_url: + author_url = source_url + + return { + 'author_name': author_name, + 'author_url': author_url, + 'author_photo': author_photo + } + + +def extract_content(parsed_mf2: Dict) -> Optional[str]: + """ + Extract content snippet from h-entry. + + Args: + parsed_mf2: Parsed microformats data from mf2py + + Returns: + Sanitized content snippet (max 500 chars) or None + """ + # Find h-entry + for item in parsed_mf2.get('items', []): + if 'h-entry' not in item.get('type', []): + continue + + props = item.get('properties', {}) + + # Try e-content first (HTML) + content_prop = props.get('content', []) + if content_prop: + content_data = content_prop[0] + if isinstance(content_data, dict): + content = content_data.get('html', '') or content_data.get('value', '') + else: + content = content_data + + if content: + # Sanitize HTML + sanitized = sanitize_content(content) + # Truncate to 500 chars + if len(sanitized) > 500: + sanitized = sanitized[:497] + '...' + return sanitized + + # Try p-summary as fallback + summary = props.get('summary', []) + if summary: + content = summary[0] + if isinstance(content, dict): + content = content.get('value', '') + if content: + sanitized = sanitize_content(content) + if len(sanitized) > 500: + sanitized = sanitized[:497] + '...' + return sanitized + + return None + + +def sanitize_content(html: str) -> str: + """ + Sanitize HTML content using nh3. + + Per api-contract.md: + - Allowed tags: p, br, strong, em, b, i, a, code, pre, blockquote + - Allowed attributes: a[href, rel] + + Args: + html: HTML content to sanitize + + Returns: + Sanitized HTML + """ + allowed_tags = {'p', 'br', 'strong', 'em', 'b', 'i', 'a', 'code', 'pre', 'blockquote'} + allowed_attributes = {'a': {'href'}} + + return nh3.clean( + html, + tags=allowed_tags, + attributes=allowed_attributes, + link_rel='nofollow noopener' + ) + + +def process_webmention(source_url: str, target_url: str, site_url: str) -> Dict[str, Any]: + """ + Process and validate a webmention. + + Args: + source_url: Source URL of the webmention + target_url: Target URL being mentioned + site_url: Site URL for User-Agent header + + Returns: + Dict with validation results: + - valid: bool + - error: str (if valid=False) + - mention_type: str + - author_name: str + - author_url: str + - author_photo: str or None + - content_snippet: str or None + + Raises: + ValidationError: If validation fails with specific error + """ + # Fetch source content + content, content_type = fetch_source(source_url, site_url) + + # Verify target link presence + if not verify_link_presence(content, target_url, content_type): + raise ValidationError("Source does not link to target") + + # Parse microformats (only for HTML) + parsed_mf2 = None + if 'html' in content_type.lower(): + try: + parsed_mf2 = mf2py.parse(doc=content, url=source_url) + except Exception as e: + logger.debug(f"Microformat parsing failed for {source_url}: {e}") + + # Extract data + mention_type = 'mention' + author_info = { + 'author_name': urlparse(source_url).netloc, + 'author_url': source_url, + 'author_photo': None + } + content_snippet = None + + if parsed_mf2: + mention_type = detect_mention_type(parsed_mf2, target_url) + author_info = extract_author(parsed_mf2, source_url) + content_snippet = extract_content(parsed_mf2) + + return { + 'valid': True, + 'mention_type': mention_type, + **author_info, + 'content_snippet': content_snippet + } diff --git a/starpunk/webmention_queries.py b/starpunk/webmention_queries.py new file mode 100644 index 0000000..f767511 --- /dev/null +++ b/starpunk/webmention_queries.py @@ -0,0 +1,295 @@ +""" +Webmention database query functions + +Provides database query functions for webmention management. +Per display-moderation.md design specification. + +Functions: + - get_webmentions_for_note: Get approved webmentions for public display + - list_webmentions: List webmentions for admin with filters and pagination + - get_webmention: Get single webmention by ID + - count_webmentions: Count webmentions (optionally filtered) + - update_webmention_status: Update moderation status + - delete_webmentions: Delete webmentions by IDs + - get_reply_context: Get cached reply context for a URL +""" + +import logging +import math +from datetime import datetime +from typing import List, Dict, Any, Optional +from starpunk.database import get_db + +logger = logging.getLogger(__name__) + + +def get_webmentions_for_note(note_id: int) -> Dict[str, List[Dict[str, Any]]]: + """ + Get approved webmentions for a note, grouped by type for public display. + + Per display-moderation.md: + - Only returns validation_status='valid' AND moderation_status='approved' + - Groups by mention_type (reply, like, repost, bookmark, mention) + - Sorts within groups by received_at ASC (oldest first) + + Args: + note_id: Note database ID + + Returns: + Dict with keys: 'replies', 'likes', 'reposts', 'bookmarks', 'mentions' + Each value is a list of webmention dicts + """ + db = get_db() + + # Query for approved webmentions only + query = """ + SELECT * FROM webmentions + WHERE target_note_id = ? + AND validation_status = 'valid' + AND moderation_status = 'approved' + ORDER BY + CASE mention_type + WHEN 'reply' THEN 1 + WHEN 'like' THEN 2 + WHEN 'repost' THEN 3 + WHEN 'bookmark' THEN 4 + WHEN 'mention' THEN 5 + END, + received_at ASC + """ + + rows = db.execute(query, (note_id,)).fetchall() + + # Group by type + grouped = { + 'replies': [], + 'likes': [], + 'reposts': [], + 'bookmarks': [], + 'mentions': [] + } + + for row in rows: + wm = dict(row) + # Parse received_at timestamp + if isinstance(wm.get('received_at'), str): + wm['received_at'] = datetime.fromisoformat(wm['received_at'].replace('Z', '+00:00')) + + mention_type = wm.get('mention_type', 'mention') + if mention_type == 'reply': + grouped['replies'].append(wm) + elif mention_type == 'like': + grouped['likes'].append(wm) + elif mention_type == 'repost': + grouped['reposts'].append(wm) + elif mention_type == 'bookmark': + grouped['bookmarks'].append(wm) + else: + grouped['mentions'].append(wm) + + return grouped + + +def list_webmentions( + status_filter: Optional[str] = None, + page: int = 1, + per_page: int = 25 +) -> tuple[List[Dict[str, Any]], int]: + """ + List webmentions for admin with pagination and filtering. + + Args: + status_filter: Filter by moderation_status ('pending', 'approved', 'hidden') + page: Page number (1-indexed) + per_page: Items per page (default: 25) + + Returns: + Tuple of (webmentions list, total_pages) + """ + db = get_db() + + # Build query + base_query = """ + SELECT w.*, n.slug as target_slug + FROM webmentions w + LEFT JOIN notes n ON w.target_note_id = n.id + """ + count_query = "SELECT COUNT(*) FROM webmentions" + + params = [] + where_clause = "" + + if status_filter: + where_clause = " WHERE w.moderation_status = ?" + count_query += " WHERE moderation_status = ?" + params.append(status_filter) + + # Count total + total_count = db.execute(count_query, params).fetchone()[0] + total_pages = math.ceil(total_count / per_page) if total_count > 0 else 1 + + # Get paginated results + offset = (page - 1) * per_page + query = base_query + where_clause + " ORDER BY w.received_at DESC LIMIT ? OFFSET ?" + params.extend([per_page, offset]) + + rows = db.execute(query, params).fetchall() + + # Convert to dicts + webmentions = [] + for row in rows: + wm = dict(row) + # Parse timestamps + if isinstance(wm.get('received_at'), str): + wm['received_at'] = datetime.fromisoformat(wm['received_at'].replace('Z', '+00:00')) + if wm.get('validated_at') and isinstance(wm['validated_at'], str): + wm['validated_at'] = datetime.fromisoformat(wm['validated_at'].replace('Z', '+00:00')) + webmentions.append(wm) + + return webmentions, total_pages + + +def get_webmention(webmention_id: int) -> Optional[Dict[str, Any]]: + """ + Get single webmention by ID. + + Args: + webmention_id: Webmention database ID + + Returns: + Webmention dict or None if not found + """ + db = get_db() + + query = """ + SELECT w.*, n.slug as target_slug + FROM webmentions w + LEFT JOIN notes n ON w.target_note_id = n.id + WHERE w.id = ? + """ + + row = db.execute(query, (webmention_id,)).fetchone() + + if not row: + return None + + wm = dict(row) + # Parse timestamps + if isinstance(wm.get('received_at'), str): + wm['received_at'] = datetime.fromisoformat(wm['received_at'].replace('Z', '+00:00')) + if wm.get('validated_at') and isinstance(wm['validated_at'], str): + wm['validated_at'] = datetime.fromisoformat(wm['validated_at'].replace('Z', '+00:00')) + + return wm + + +def count_webmentions(status_filter: Optional[str] = None) -> int: + """ + Count webmentions, optionally filtered by status. + + Args: + status_filter: Filter by moderation_status + + Returns: + Count of webmentions + """ + db = get_db() + + if status_filter: + query = "SELECT COUNT(*) FROM webmentions WHERE moderation_status = ?" + result = db.execute(query, (status_filter,)).fetchone() + else: + query = "SELECT COUNT(*) FROM webmentions" + result = db.execute(query).fetchone() + + return result[0] if result else 0 + + +def update_webmention_status(webmention_ids: List[int], status: str) -> int: + """ + Update moderation status for multiple webmentions. + + Args: + webmention_ids: List of webmention IDs + status: New moderation_status ('approved', 'hidden', 'pending') + + Returns: + Number of webmentions updated + """ + if not webmention_ids: + return 0 + + if status not in ('approved', 'hidden', 'pending'): + raise ValueError(f"Invalid status: {status}") + + db = get_db() + + # Use parameterized query with placeholders + placeholders = ','.join('?' * len(webmention_ids)) + query = f""" + UPDATE webmentions + SET moderation_status = ?, + updated_at = CURRENT_TIMESTAMP + WHERE id IN ({placeholders}) + """ + + params = [status] + webmention_ids + cursor = db.execute(query, params) + db.commit() + + return cursor.rowcount + + +def delete_webmentions(webmention_ids: List[int]) -> int: + """ + Delete webmentions by IDs. + + Args: + webmention_ids: List of webmention IDs to delete + + Returns: + Number of webmentions deleted + """ + if not webmention_ids: + return 0 + + db = get_db() + + placeholders = ','.join('?' * len(webmention_ids)) + query = f"DELETE FROM webmentions WHERE id IN ({placeholders})" + + cursor = db.execute(query, webmention_ids) + db.commit() + + return cursor.rowcount + + +def get_reply_context(url: str) -> Optional[Dict[str, Any]]: + """ + Get cached reply context for a URL. + + Args: + url: The in_reply_to URL + + Returns: + Reply context dict or None if not found + """ + db = get_db() + + query = "SELECT * FROM reply_contexts WHERE url = ?" + row = db.execute(query, (url,)).fetchone() + + if not row: + return None + + context = dict(row) + + # Parse timestamps + if context.get('fetched_at') and isinstance(context['fetched_at'], str): + context['fetched_at'] = datetime.fromisoformat(context['fetched_at'].replace('Z', '+00:00')) + if context.get('created_at') and isinstance(context['created_at'], str): + context['created_at'] = datetime.fromisoformat(context['created_at'].replace('Z', '+00:00')) + if context.get('published_at') and isinstance(context['published_at'], str): + context['published_at'] = datetime.fromisoformat(context['published_at'].replace('Z', '+00:00')) + + return context diff --git a/starpunk/webmention_sender.py b/starpunk/webmention_sender.py new file mode 100644 index 0000000..f583374 --- /dev/null +++ b/starpunk/webmention_sender.py @@ -0,0 +1,198 @@ +""" +Outgoing webmention link extraction and queueing + +Per background-worker.md and architect's answers: +- Extract links from rendered HTML (not markdown) +- Extract only elements, exclude
 and  blocks
+- Queue max 10 links per note per operation
+- Priority: in_reply_to first, then document order
+
+References:
+    - Background Worker Design: /docs/design/v1.6.0/background-worker.md
+    - API Contract: /docs/design/v1.6.0/api-contract.md
+"""
+
+import logging
+import sqlite3
+from datetime import datetime
+from pathlib import Path
+from typing import Optional, Set
+from lxml import html as lxml_html
+from urllib.parse import urlparse
+
+
+logger = logging.getLogger(__name__)
+
+
+def extract_links_from_html(html_content: str, exclude_code_blocks: bool = True) -> list[str]:
+    """
+    Extract links from HTML content.
+
+    Per architect's answer to Q1:
+    - Extract from rendered HTML (not markdown)
+    - Extract  elements only
+    - Exclude links in 
 and  blocks
+    - Return in document order
+
+    Args:
+        html_content: Rendered HTML content
+        exclude_code_blocks: If True, skip links in 
 and  (default: True)
+
+    Returns:
+        List of URLs in document order
+    """
+    if not html_content or not html_content.strip():
+        return []
+
+    try:
+        # Parse HTML
+        doc = lxml_html.fromstring(html_content)
+
+        if exclude_code_blocks:
+            # Remove 
 and  elements before extracting links
+            for element in doc.xpath('//pre | //code'):
+                element.getparent().remove(element)
+
+        # Extract all  attributes in document order
+        links = doc.xpath('//a/@href')
+
+        # Filter to valid absolute URLs (http/https)
+        absolute_links = []
+        for link in links:
+            parsed = urlparse(link)
+            if parsed.scheme in ('http', 'https') and parsed.netloc:
+                absolute_links.append(link)
+
+        return absolute_links
+
+    except Exception as e:
+        logger.warning(f"Failed to parse HTML for link extraction: {e}")
+        return []
+
+
+def queue_webmentions_for_note(
+    db_path: Path,
+    note_id: int,
+    note_url: str,
+    content_html: str,
+    in_reply_to: Optional[str] = None,
+    old_content_html: Optional[str] = None  # Phase 6: update semantics
+) -> int:
+    """
+    Extract links and queue webmentions for a note.
+
+    Per architect's answers (Phase 6):
+    - Called from notes.py AFTER successful database commit
+    - For updates: diff old vs new URLs, queue for new/existing/removed
+    - For creates: queue for all URLs in content
+    - Max 10 webmentions per operation
+    - Priority: in_reply_to URL first, then document order
+    - Webmention sending is best-effort; failures do not affect note creation
+
+    Args:
+        db_path: Path to SQLite database
+        note_id: Database ID of the note
+        note_url: Full URL of the published note
+        content_html: Rendered HTML content (None if unpublishing)
+        in_reply_to: Reply-to URL (if reply post)
+        old_content_html: Previous HTML (for update diff, Phase 6)
+
+    Returns:
+        Number of webmentions queued
+    """
+    # Phase 6: Implement URL diffing for updates
+    if old_content_html is not None:
+        # This is an update - compute URL diff
+        old_links = set(extract_links_from_html(old_content_html))
+
+        # Handle unpublishing case (content_html is None)
+        if content_html is None:
+            # Unpublishing - queue for all old URLs (they're all removed)
+            new_links = set()
+        else:
+            new_links = set(extract_links_from_html(content_html))
+
+        # Compute sets for URL diffing
+        new_urls = new_links - old_links  # In new, not in old
+        existing_urls = new_links & old_links  # In both (re-validate)
+        removed_urls = old_links - new_links  # In old, not in new
+
+        # Build target list: new + existing + removed
+        # (All need webmentions per W3C spec)
+        all_update_urls = list(new_urls) + list(existing_urls) + list(removed_urls)
+
+        # Add in_reply_to if changed
+        target_urls = []
+        if in_reply_to and in_reply_to not in all_update_urls:
+            target_urls.append(in_reply_to)
+
+        # Add the diffed URLs
+        target_urls.extend(all_update_urls)
+
+        # Limit to 10
+        target_urls = target_urls[:10]
+
+    else:
+        # This is a create (no old content) - use original logic
+        links = extract_links_from_html(content_html) if content_html else []
+
+        # Build priority-ordered list of target URLs
+        target_urls = []
+
+        # 1. in_reply_to gets highest priority
+        if in_reply_to:
+            target_urls.append(in_reply_to)
+
+        # 2. Add extracted links in document order (deduplicated)
+        seen = {in_reply_to} if in_reply_to else set()
+        for link in links:
+            if link not in seen:
+                target_urls.append(link)
+                seen.add(link)
+
+        # 3. Limit to 10 webmentions per operation
+        target_urls = target_urls[:10]
+
+    if not target_urls:
+        logger.debug(f"No links to send webmentions for note {note_id}")
+        return 0
+
+    # Queue webmentions
+    queued = 0
+    try:
+        conn = sqlite3.connect(db_path)
+        cursor = conn.cursor()
+
+        for target_url in target_urls:
+            try:
+                # Use UPSERT to handle duplicate sends
+                # Phase 6: Force endpoint rediscovery per W3C MUST requirement
+                cursor.execute("""
+                    INSERT INTO webmention_queue
+                        (source_url, target_url, source_note_id, status, created_at)
+                    VALUES (?, ?, ?, 'pending', CURRENT_TIMESTAMP)
+                    ON CONFLICT(source_url, target_url) DO UPDATE SET
+                        status = 'pending',
+                        retry_count = 0,
+                        next_retry_at = NULL,
+                        last_error = NULL,
+                        endpoint_url = NULL,
+                        created_at = CURRENT_TIMESTAMP
+                """, (note_url, target_url, note_id))
+
+                queued += 1
+
+            except Exception as e:
+                logger.warning(
+                    f"Failed to queue webmention {note_url} -> {target_url}: {e}"
+                )
+
+        conn.commit()
+        conn.close()
+
+        logger.info(f"Queued {queued} webmentions for note {note_id}")
+
+    except Exception as e:
+        logger.error(f"Failed to queue webmentions for note {note_id}: {e}")
+
+    return queued
diff --git a/starpunk/webmention_ssrf.py b/starpunk/webmention_ssrf.py
new file mode 100644
index 0000000..5f0c794
--- /dev/null
+++ b/starpunk/webmention_ssrf.py
@@ -0,0 +1,113 @@
+"""
+SSRF protection utilities for webmention processing
+
+Per ADR-063 Decision 9 and background-worker.md:
+- Block private IP address ranges
+- Resolve hostname before making requests
+- DEV_MODE exception for localhost testing
+- Applied to all outgoing HTTP requests
+
+References:
+    - ADR-063: Webmention Implementation
+    - Background Worker Design: /docs/design/v1.6.0/background-worker.md
+"""
+
+import ipaddress
+import socket
+from flask import current_app
+
+
+class SSRFBlockedError(Exception):
+    """Raised when SSRF protection blocks a request"""
+    pass
+
+
+# Private IP ranges per RFC 1918 and link-local ranges
+BLOCKED_NETWORKS = [
+    ipaddress.ip_network('10.0.0.0/8'),         # RFC 1918 private
+    ipaddress.ip_network('172.16.0.0/12'),      # RFC 1918 private
+    ipaddress.ip_network('192.168.0.0/16'),     # RFC 1918 private
+    ipaddress.ip_network('127.0.0.0/8'),        # Localhost
+    ipaddress.ip_network('169.254.0.0/16'),     # Link-local
+    ipaddress.ip_network('::1/128'),            # IPv6 localhost
+    ipaddress.ip_network('fc00::/7'),           # IPv6 unique local
+    ipaddress.ip_network('fe80::/10'),          # IPv6 link-local
+]
+
+
+def is_private_ip(ip: str) -> bool:
+    """
+    Check if IP address is in a private/blocked range.
+
+    Args:
+        ip: IP address string (IPv4 or IPv6)
+
+    Returns:
+        True if IP is private/blocked, False otherwise
+    """
+    try:
+        addr = ipaddress.ip_address(ip)
+        return any(addr in network for network in BLOCKED_NETWORKS)
+    except ValueError:
+        # Invalid IP format, treat as blocked
+        return True
+
+
+def safe_resolve(hostname: str) -> str:
+    """
+    Resolve hostname and validate IP is not private.
+
+    Per background-worker.md: DEV_MODE exception for localhost testing.
+
+    Args:
+        hostname: Hostname to resolve
+
+    Returns:
+        Resolved IP address if safe
+
+    Raises:
+        SSRFBlockedError: If hostname resolves to private IP
+        socket.gaierror: If hostname cannot be resolved
+    """
+    # DEV_MODE exception for localhost
+    if current_app.config.get('DEV_MODE') and hostname in ('localhost', '127.0.0.1', '::1'):
+        return '127.0.0.1'
+
+    # Resolve hostname
+    ip = socket.gethostbyname(hostname)
+
+    # Check if resolved IP is private
+    if is_private_ip(ip):
+        raise SSRFBlockedError(f"Blocked request to private IP: {ip}")
+
+    return ip
+
+
+def validate_url_for_fetch(url: str) -> None:
+    """
+    Validate URL is safe to fetch (not private IP).
+
+    Args:
+        url: URL to validate
+
+    Raises:
+        SSRFBlockedError: If URL points to private IP
+        ValueError: If URL is malformed
+    """
+    from urllib.parse import urlparse
+
+    parsed = urlparse(url)
+
+    if not parsed.scheme in ('http', 'https'):
+        raise ValueError(f"Unsupported URL scheme: {parsed.scheme}")
+
+    if not parsed.netloc:
+        raise ValueError("URL must include a hostname")
+
+    # Extract hostname (remove port if present)
+    hostname = parsed.hostname
+    if not hostname:
+        raise ValueError("Could not extract hostname from URL")
+
+    # Validate hostname resolves to public IP
+    safe_resolve(hostname)
diff --git a/starpunk/webmention_worker.py b/starpunk/webmention_worker.py
new file mode 100644
index 0000000..93b3586
--- /dev/null
+++ b/starpunk/webmention_worker.py
@@ -0,0 +1,787 @@
+"""
+Background worker thread for webmention processing
+
+Per background-worker.md:
+- Daemon thread that processes webmention queues
+- Handles both incoming validation and outgoing sending
+- Graceful shutdown on app termination
+- Orphan detection and cleanup on startup
+
+References:
+    - Background Worker Design: /docs/design/v1.6.0/background-worker.md
+    - ADR-063: Webmention Implementation
+"""
+
+import logging
+import threading
+import time
+import sqlite3
+import httpx
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Dict, Any, Optional, TypedDict
+from urllib.parse import urlparse
+from lxml import html as lxml_html
+
+from starpunk.webmention_processor import process_webmention, ValidationError
+from starpunk.webmention_ssrf import validate_url_for_fetch, SSRFBlockedError
+from flask import has_app_context
+
+
+logger = logging.getLogger(__name__)
+
+
+# Discovery cache entry type
+class DiscoveryCacheEntry(TypedDict):
+    """Cache entry for endpoint discovery results"""
+    endpoint: Optional[str]  # None = negative cache (no endpoint found)
+    expires_at: datetime
+
+
+class WebmentionWorker(threading.Thread):
+    """
+    Background thread for webmention queue processing
+
+    Follows MemoryMonitor pattern: daemon thread with stop event
+    """
+
+    def __init__(
+        self,
+        db_path: Path,
+        site_url: str,
+        poll_interval: int = 10,
+        error_backoff: int = 60,
+        batch_size: int = 5,
+        auto_approve: bool = True
+    ):
+        """
+        Initialize webmention worker thread
+
+        Args:
+            db_path: Path to SQLite database
+            site_url: Site URL for User-Agent header
+            poll_interval: Seconds between queue checks (default: 10)
+            error_backoff: Seconds to wait after error (default: 60)
+            batch_size: Items to process per iteration (default: 5)
+            auto_approve: Auto-approve valid webmentions (default: True)
+        """
+        super().__init__(daemon=True, name="webmention-worker")
+        self.db_path = db_path
+        self.site_url = site_url
+        self.poll_interval = poll_interval
+        self.error_backoff = error_backoff
+        self.batch_size = batch_size
+        self.auto_approve = auto_approve
+        self._stop_event = threading.Event()
+        self._last_processed = None
+
+        # Phase 3: Discovery cache (worker instance variable)
+        # Per architect's answer to Q4
+        self._discovery_cache: Dict[str, DiscoveryCacheEntry] = {}
+
+    def run(self):
+        """
+        Main processing loop
+
+        Per background-worker.md:
+        1. Clean up orphaned items on startup
+        2. Process pending validations (incoming)
+        3. Process pending sends (outgoing) - Phase 3
+        4. Sleep if nothing to process
+        """
+        try:
+            logger.info("Webmention worker started")
+
+            # Cleanup orphaned items from previous crashes
+            self._cleanup_orphaned_items()
+
+            # Main processing loop
+            while not self._stop_event.is_set():
+                try:
+                    processed = 0
+
+                    # Process incoming webmentions
+                    processed += self._process_pending_validations()
+
+                    # Phase 3: Process outgoing webmentions
+                    processed += self._process_pending_sends()
+
+                    # Update last processed time
+                    if processed > 0:
+                        self._last_processed = datetime.utcnow()
+
+                    # Sleep if nothing processed
+                    if processed == 0:
+                        self._stop_event.wait(self.poll_interval)
+
+                except Exception as e:
+                    logger.error(f"Worker error: {e}", exc_info=True)
+                    self._stop_event.wait(self.error_backoff)
+
+        except Exception as e:
+            logger.error(f"Webmention worker thread failed: {e}", exc_info=True)
+        finally:
+            logger.info("Webmention worker stopped")
+
+    def _cleanup_orphaned_items(self):
+        """
+        Clean up orphaned 'sending' status items on startup
+
+        Per background-worker.md:
+        Reset items stuck in 'sending' status from previous crashes
+        """
+        try:
+            conn = sqlite3.connect(self.db_path)
+            cursor = conn.cursor()
+
+            # Reset orphaned outgoing webmentions
+            cursor.execute("""
+                UPDATE webmention_queue
+                SET status = 'pending',
+                    started_at = NULL
+                WHERE status = 'sending'
+                  AND started_at < datetime('now', '-5 minutes')
+            """)
+
+            orphaned = cursor.rowcount
+            conn.commit()
+            conn.close()
+
+            if orphaned > 0:
+                logger.info(f"Reset {orphaned} orphaned webmention queue items")
+
+        except Exception as e:
+            logger.error(f"Failed to cleanup orphaned items: {e}")
+
+    def _process_pending_validations(self) -> int:
+        """
+        Process pending incoming webmentions
+
+        Returns:
+            Number of webmentions processed
+        """
+        processed = 0
+
+        try:
+            conn = sqlite3.connect(self.db_path)
+            conn.row_factory = sqlite3.Row
+            cursor = conn.cursor()
+
+            # Get pending webmentions
+            cursor.execute("""
+                SELECT id, source_url, target_url, target_note_id
+                FROM webmentions
+                WHERE validation_status = 'pending'
+                ORDER BY received_at ASC
+                LIMIT ?
+            """, (self.batch_size,))
+
+            pending = cursor.fetchall()
+            conn.close()
+
+            # Process each webmention
+            for row in pending:
+                try:
+                    self._validate_webmention(
+                        row['id'],
+                        row['source_url'],
+                        row['target_url'],
+                        row['target_note_id']
+                    )
+                    processed += 1
+                except Exception as e:
+                    logger.error(
+                        f"Failed to validate webmention {row['id']}: {e}",
+                        exc_info=True
+                    )
+
+        except Exception as e:
+            logger.error(f"Error fetching pending validations: {e}")
+
+        return processed
+
+    def _validate_webmention(
+        self,
+        webmention_id: int,
+        source_url: str,
+        target_url: str,
+        target_note_id: Optional[int]
+    ):
+        """
+        Validate a single webmention
+
+        Args:
+            webmention_id: Database ID of webmention
+            source_url: Source URL
+            target_url: Target URL
+            target_note_id: Target note ID (if applicable)
+        """
+        try:
+            # Process webmention (fetch, validate, extract)
+            result = process_webmention(source_url, target_url, self.site_url)
+
+            # Update database with successful validation
+            conn = sqlite3.connect(self.db_path)
+            cursor = conn.cursor()
+
+            moderation_status = 'approved' if self.auto_approve else 'pending'
+
+            cursor.execute("""
+                UPDATE webmentions
+                SET validation_status = 'valid',
+                    validated_at = CURRENT_TIMESTAMP,
+                    mention_type = ?,
+                    author_name = ?,
+                    author_url = ?,
+                    author_photo = ?,
+                    content_snippet = ?,
+                    moderation_status = ?,
+                    updated_at = CURRENT_TIMESTAMP
+                WHERE id = ?
+            """, (
+                result['mention_type'],
+                result['author_name'],
+                result['author_url'],
+                result['author_photo'],
+                result['content_snippet'],
+                moderation_status,
+                webmention_id
+            ))
+
+            conn.commit()
+            conn.close()
+
+            logger.info(
+                f"Validated webmention: {source_url} -> {target_url} "
+                f"(type={result['mention_type']}, status={moderation_status})"
+            )
+
+        except ValidationError as e:
+            # Validation failed - update with error
+            self._mark_validation_failed(webmention_id, source_url, target_url, str(e))
+
+        except Exception as e:
+            # Unexpected error - mark as error
+            self._mark_validation_failed(
+                webmention_id,
+                source_url,
+                target_url,
+                f"Unexpected error: {e}"
+            )
+            raise
+
+    def _mark_validation_failed(
+        self,
+        webmention_id: int,
+        source_url: str,
+        target_url: str,
+        error: str
+    ):
+        """
+        Mark webmention validation as failed/invalid
+
+        Args:
+            webmention_id: Database ID
+            source_url: Source URL
+            target_url: Target URL
+            error: Error message
+        """
+        # Determine if error or invalid
+        # Per background-worker.md: HTTP 4xx and "no link" are invalid
+        # All other errors are 'error' status
+        if any(x in error.lower() for x in ['does not link', 'returned 4']):
+            status = 'invalid'
+        else:
+            status = 'error'
+
+        try:
+            conn = sqlite3.connect(self.db_path)
+            cursor = conn.cursor()
+
+            cursor.execute("""
+                UPDATE webmentions
+                SET validation_status = ?,
+                    validation_error = ?,
+                    validated_at = CURRENT_TIMESTAMP,
+                    updated_at = CURRENT_TIMESTAMP
+                WHERE id = ?
+            """, (status, error, webmention_id))
+
+            conn.commit()
+            conn.close()
+
+            logger.warning(
+                f"Invalid webmention: {source_url} -> {target_url}: {error}"
+            )
+
+        except Exception as e:
+            logger.error(f"Failed to mark webmention as failed: {e}")
+
+    def _process_pending_sends(self) -> int:
+        """
+        Process pending outgoing webmentions (Phase 3)
+
+        Returns:
+            Number of webmentions processed
+        """
+        processed = 0
+
+        try:
+            conn = sqlite3.connect(self.db_path)
+            conn.row_factory = sqlite3.Row
+            cursor = conn.cursor()
+
+            # Get pending webmentions ready to send
+            cursor.execute("""
+                SELECT id, source_url, target_url, endpoint_url, retry_count
+                FROM webmention_queue
+                WHERE status = 'pending'
+                  AND (next_retry_at IS NULL OR next_retry_at <= datetime('now'))
+                ORDER BY created_at ASC
+                LIMIT ?
+            """, (self.batch_size,))
+
+            pending = cursor.fetchall()
+            conn.close()
+
+            # Process each webmention
+            for row in pending:
+                try:
+                    self._send_webmention(
+                        row['id'],
+                        row['source_url'],
+                        row['target_url'],
+                        row['endpoint_url'],
+                        row['retry_count']
+                    )
+                    processed += 1
+                except Exception as e:
+                    logger.error(
+                        f"Failed to send webmention {row['id']}: {e}",
+                        exc_info=True
+                    )
+
+        except Exception as e:
+            logger.error(f"Error fetching pending sends: {e}")
+
+        return processed
+
+    def _send_webmention(
+        self,
+        queue_id: int,
+        source_url: str,
+        target_url: str,
+        cached_endpoint: Optional[str],
+        retry_count: int
+    ):
+        """
+        Send a single outgoing webmention
+
+        Args:
+            queue_id: Database ID of queue item
+            source_url: Source URL (our note)
+            target_url: Target URL to notify
+            cached_endpoint: Previously discovered endpoint (if any)
+            retry_count: Current retry count
+        """
+        # Mark as sending
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()
+        cursor.execute("""
+            UPDATE webmention_queue
+            SET status = 'sending',
+                started_at = CURRENT_TIMESTAMP
+            WHERE id = ?
+        """, (queue_id,))
+        conn.commit()
+        conn.close()
+
+        try:
+            # Discover endpoint if not cached
+            endpoint = cached_endpoint
+            if not endpoint:
+                endpoint = self._discover_endpoint(target_url)
+
+                if not endpoint:
+                    # No endpoint found - permanent failure
+                    self._mark_send_no_endpoint(queue_id, source_url, target_url)
+                    return
+
+                # Cache the discovered endpoint in database
+                conn = sqlite3.connect(self.db_path)
+                cursor = conn.cursor()
+                cursor.execute("""
+                    UPDATE webmention_queue
+                    SET endpoint_url = ?
+                    WHERE id = ?
+                """, (endpoint, queue_id))
+                conn.commit()
+                conn.close()
+
+            # Send webmention POST
+            try:
+                # SSRF protection for endpoint URL
+                validate_url_for_fetch(endpoint)
+
+                response = httpx.post(
+                    endpoint,
+                    data={
+                        'source': source_url,
+                        'target': target_url
+                    },
+                    timeout=10,
+                    headers={'User-Agent': f'StarPunk/1.6.0 (+{self.site_url})'},
+                    follow_redirects=True
+                )
+
+                # Check response
+                if 200 <= response.status_code < 300:
+                    # Success
+                    self._mark_send_success(queue_id, source_url, target_url)
+
+                elif 400 <= response.status_code < 500:
+                    # Permanent failure (4xx)
+                    error = f"HTTP {response.status_code}: {response.text[:200]}"
+                    self._mark_send_failed(queue_id, source_url, target_url, error)
+
+                else:
+                    # Temporary failure (5xx) - retry
+                    error = f"HTTP {response.status_code}: {response.text[:200]}"
+                    self._mark_send_retry(queue_id, source_url, target_url, retry_count, error)
+
+            except (httpx.TimeoutException, httpx.HTTPError) as e:
+                # Connection error - retry
+                self._mark_send_retry(queue_id, source_url, target_url, retry_count, str(e))
+
+            except (SSRFBlockedError, ValueError) as e:
+                # SSRF blocked - permanent failure
+                logger.warning(f"SSRF blocked endpoint {endpoint}: {e}")
+                self._mark_send_failed(queue_id, source_url, target_url, f"Blocked: {e}")
+
+        except Exception as e:
+            # Discovery failed - retry
+            self._mark_send_retry(queue_id, source_url, target_url, retry_count, str(e))
+
+    def _discover_endpoint(self, target_url: str) -> Optional[str]:
+        """
+        Discover webmention endpoint for target URL with caching
+
+        Per architect's answers to Q3, Q4, Q10:
+        - GET only (no HEAD)
+        - Check Link header →  → 
+        - First match wins
+        - Cache results: 1h for found, 24h for not found
+        - SSRF protection on target URL
+
+        Args:
+            target_url: Target URL to discover endpoint for
+
+        Returns:
+            Endpoint URL if found, None otherwise
+        """
+        # Check cache first
+        cached = self._discovery_cache.get(target_url)
+        if cached and cached['expires_at'] > datetime.utcnow():
+            logger.debug(f"Using cached endpoint for {target_url}: {cached['endpoint']}")
+            return cached['endpoint']
+
+        # SSRF protection
+        try:
+            validate_url_for_fetch(target_url)
+        except (SSRFBlockedError, ValueError) as e:
+            logger.warning(f"SSRF protection blocked target {target_url}: {e}")
+            # Cache negative result
+            self._cache_endpoint(target_url, None, hours=24)
+            return None
+
+        # Fetch target URL
+        try:
+            response = httpx.get(
+                target_url,
+                timeout=10,
+                headers={'User-Agent': f'StarPunk/1.6.0 (+{self.site_url})'},
+                follow_redirects=True
+            )
+            response.raise_for_status()
+
+            # 1. Check Link header
+            link_header = response.headers.get('Link', '')
+            if link_header:
+                endpoint = self._parse_link_header(link_header, response.url)
+                if endpoint:
+                    self._cache_endpoint(target_url, endpoint, hours=1)
+                    return endpoint
+
+            # 2. Parse HTML for  and  elements
+            content_type = response.headers.get('Content-Type', '')
+            if 'html' in content_type.lower():
+                endpoint = self._parse_html_for_endpoint(response.text, response.url)
+                if endpoint:
+                    self._cache_endpoint(target_url, endpoint, hours=1)
+                    return endpoint
+
+            # No endpoint found - cache negative result
+            logger.info(f"No webmention endpoint found for {target_url}")
+            self._cache_endpoint(target_url, None, hours=24)
+            return None
+
+        except Exception as e:
+            logger.warning(f"Failed to discover endpoint for {target_url}: {e}")
+            # Don't cache on error - allow retry
+            return None
+
+    def _parse_link_header(self, link_header: str, base_url: str) -> Optional[str]:
+        """
+        Parse Link header for webmention endpoint
+
+        Args:
+            link_header: Raw Link header value
+            base_url: Base URL for resolving relative URLs
+
+        Returns:
+            Absolute endpoint URL if found, None otherwise
+        """
+        # Simple parsing - look for rel="webmention"
+        # Format: ; rel="webmention"
+        import re
+        pattern = r'<([^>]+)>\s*;\s*rel\s*=\s*["\']?webmention["\']?'
+        match = re.search(pattern, link_header, re.IGNORECASE)
+
+        if match:
+            endpoint = match.group(1)
+            # Make absolute if relative
+            from urllib.parse import urljoin
+            return urljoin(base_url, endpoint)
+
+        return None
+
+    def _parse_html_for_endpoint(self, html_content: str, base_url: str) -> Optional[str]:
+        """
+        Parse HTML for webmention endpoint in  or  elements
+
+        Args:
+            html_content: HTML content
+            base_url: Base URL for resolving relative URLs
+
+        Returns:
+            Absolute endpoint URL if found, None otherwise
+        """
+        try:
+            doc = lxml_html.fromstring(html_content)
+
+            # Check  first
+            links = doc.xpath('//link[@rel="webmention"]/@href')
+            if links:
+                from urllib.parse import urljoin
+                return urljoin(base_url, links[0])
+
+            # Check 
+            anchors = doc.xpath('//a[@rel="webmention"]/@href')
+            if anchors:
+                from urllib.parse import urljoin
+                return urljoin(base_url, anchors[0])
+
+        except Exception as e:
+            logger.debug(f"HTML parsing failed: {e}")
+
+        return None
+
+    def _cache_endpoint(self, target_url: str, endpoint: Optional[str], hours: int):
+        """
+        Cache endpoint discovery result
+
+        Args:
+            target_url: Target URL
+            endpoint: Discovered endpoint (None = negative cache)
+            hours: TTL in hours
+        """
+        self._discovery_cache[target_url] = {
+            'endpoint': endpoint,
+            'expires_at': datetime.utcnow() + timedelta(hours=hours)
+        }
+
+        # Optional: Evict old entries (keep cache bounded)
+        if len(self._discovery_cache) > 1000:
+            # Remove expired entries
+            now = datetime.utcnow()
+            self._discovery_cache = {
+                k: v for k, v in self._discovery_cache.items()
+                if v['expires_at'] > now
+            }
+
+    def _mark_send_success(self, queue_id: int, source_url: str, target_url: str):
+        """Mark webmention send as successful"""
+        try:
+            conn = sqlite3.connect(self.db_path)
+            cursor = conn.cursor()
+            cursor.execute("""
+                UPDATE webmention_queue
+                SET status = 'sent',
+                    sent_at = CURRENT_TIMESTAMP
+                WHERE id = ?
+            """, (queue_id,))
+            conn.commit()
+            conn.close()
+
+            logger.info(f"Sent webmention: {source_url} -> {target_url}")
+
+        except Exception as e:
+            logger.error(f"Failed to mark send success: {e}")
+
+    def _mark_send_failed(self, queue_id: int, source_url: str, target_url: str, error: str):
+        """Mark webmention send as permanently failed"""
+        try:
+            conn = sqlite3.connect(self.db_path)
+            cursor = conn.cursor()
+            cursor.execute("""
+                UPDATE webmention_queue
+                SET status = 'failed',
+                    last_error = ?
+                WHERE id = ?
+            """, (error, queue_id))
+            conn.commit()
+            conn.close()
+
+            logger.warning(f"Webmention send failed permanently: {source_url} -> {target_url}: {error}")
+
+        except Exception as e:
+            logger.error(f"Failed to mark send failed: {e}")
+
+    def _mark_send_no_endpoint(self, queue_id: int, source_url: str, target_url: str):
+        """Mark webmention as no endpoint found"""
+        try:
+            conn = sqlite3.connect(self.db_path)
+            cursor = conn.cursor()
+            cursor.execute("""
+                UPDATE webmention_queue
+                SET status = 'no_endpoint'
+                WHERE id = ?
+            """, (queue_id,))
+            conn.commit()
+            conn.close()
+
+            logger.info(f"No webmention endpoint for {target_url}")
+
+        except Exception as e:
+            logger.error(f"Failed to mark no endpoint: {e}")
+
+    def _mark_send_retry(
+        self,
+        queue_id: int,
+        source_url: str,
+        target_url: str,
+        current_retry_count: int,
+        error: str
+    ):
+        """
+        Mark webmention for retry with exponential backoff
+
+        Per architect's answer to Q5 and background-worker.md:
+        - Increment retry_count AFTER failed attempt
+        - Check retry_count >= 4 AFTER incrementing
+        - Retry schedule: 1h, 6h, 24h
+        """
+        new_retry_count = current_retry_count + 1
+
+        # Calculate next retry time
+        next_retry_at = None
+        if new_retry_count < 4:
+            delays = {
+                1: timedelta(hours=1),
+                2: timedelta(hours=6),
+                3: timedelta(hours=24),
+            }
+            delay = delays.get(new_retry_count)
+            if delay:
+                next_retry_at = datetime.utcnow() + delay
+
+        try:
+            conn = sqlite3.connect(self.db_path)
+            cursor = conn.cursor()
+
+            if new_retry_count >= 4:
+                # Max retries reached - mark as failed
+                cursor.execute("""
+                    UPDATE webmention_queue
+                    SET status = 'failed',
+                        retry_count = ?,
+                        last_error = ?
+                    WHERE id = ?
+                """, (new_retry_count, error, queue_id))
+
+                logger.warning(
+                    f"Webmention send failed permanently after {new_retry_count} attempts: "
+                    f"{source_url} -> {target_url}: {error}"
+                )
+
+            else:
+                # Schedule retry
+                cursor.execute("""
+                    UPDATE webmention_queue
+                    SET status = 'pending',
+                        retry_count = ?,
+                        next_retry_at = ?,
+                        last_error = ?
+                    WHERE id = ?
+                """, (new_retry_count, next_retry_at, error, queue_id))
+
+                logger.warning(
+                    f"Webmention send failed, retry {new_retry_count}: "
+                    f"{source_url} -> {target_url}: {error}"
+                )
+
+            conn.commit()
+            conn.close()
+
+        except Exception as e:
+            logger.error(f"Failed to mark send retry: {e}")
+
+    def stop(self):
+        """
+        Stop the worker thread gracefully
+
+        Sets the stop event to signal the thread to exit
+        """
+        logger.info("Stopping webmention worker")
+        self._stop_event.set()
+
+    def get_stats(self) -> Dict[str, Any]:
+        """
+        Get current worker statistics
+
+        Returns:
+            Dict with worker stats for health check
+        """
+        try:
+            conn = sqlite3.connect(self.db_path)
+            cursor = conn.cursor()
+
+            # Count pending validations
+            cursor.execute("""
+                SELECT COUNT(*) FROM webmentions
+                WHERE validation_status = 'pending'
+            """)
+            pending_validations = cursor.fetchone()[0]
+
+            # Count pending sends (Phase 3)
+            cursor.execute("""
+                SELECT COUNT(*) FROM webmention_queue
+                WHERE status = 'pending'
+            """)
+            pending_sends = cursor.fetchone()[0]
+
+            conn.close()
+
+            return {
+                'running': self.is_alive(),
+                'last_processed': self._last_processed.isoformat() if self._last_processed else None,
+                'pending_validations': pending_validations,
+                'pending_sends': pending_sends
+            }
+
+        except Exception as e:
+            logger.error(f"Failed to get worker stats: {e}")
+            return {
+                'running': self.is_alive(),
+                'error': str(e)
+            }
diff --git a/static/css/style.css b/static/css/style.css
index 72203da..12660cb 100644
--- a/static/css/style.css
+++ b/static/css/style.css
@@ -191,3 +191,265 @@ small { display: block; margin-top: var(--spacing-xs); color: var(--color-text-l
     aspect-ratio: 16 / 9; /* Wider aspect on mobile */
   }
 }
+
+/* Webmention Styles (v1.6.0 Phase 5) */
+
+/* Reply Context */
+.reply-context {
+  margin-bottom: var(--spacing-md);
+  padding: var(--spacing-sm);
+  border-left: 3px solid var(--color-link);
+  background: var(--color-bg-alt);
+}
+
+.reply-context .reply-label {
+  font-size: 0.875rem;
+  color: var(--color-text-light);
+  display: block;
+  margin-bottom: var(--spacing-xs);
+}
+
+.reply-context blockquote {
+  margin: 0;
+  padding: 0;
+}
+
+.reply-context .p-content {
+  font-style: italic;
+  margin-bottom: var(--spacing-xs);
+}
+
+/* Webmentions Section */
+.webmentions {
+  margin-top: var(--spacing-lg);
+  padding-top: var(--spacing-lg);
+  border-top: 1px solid var(--color-border);
+}
+
+.webmentions h2 {
+  margin-bottom: var(--spacing-md);
+}
+
+.webmention-group {
+  margin-bottom: var(--spacing-lg);
+}
+
+.webmention-group h3 {
+  font-size: 1rem;
+  margin-bottom: var(--spacing-sm);
+  color: var(--color-text-light);
+}
+
+/* Reply Webmentions */
+.webmention-reply {
+  margin-bottom: var(--spacing-md);
+  padding: var(--spacing-sm);
+  background: var(--color-bg-alt);
+  border-radius: 4px;
+}
+
+.webmention-author {
+  display: flex;
+  align-items: center;
+  gap: var(--spacing-sm);
+  margin-bottom: var(--spacing-xs);
+}
+
+.webmention-author img {
+  border-radius: 50%;
+}
+
+.webmention-author a {
+  font-weight: bold;
+}
+
+.webmention-date {
+  font-size: 0.875rem;
+  color: var(--color-text-light);
+}
+
+/* Facepile */
+.facepile {
+  display: flex;
+  align-items: center;
+  flex-wrap: wrap;
+}
+
+.facepile .face {
+  display: block;
+  width: 32px;
+  height: 32px;
+  border-radius: 50%;
+  overflow: hidden;
+  border: 2px solid var(--color-bg);
+  margin-left: -8px;
+}
+
+.facepile .face:first-child {
+  margin-left: 0;
+}
+
+.facepile .face img {
+  width: 100%;
+  height: 100%;
+  object-fit: cover;
+}
+
+.facepile .face-initial {
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  width: 100%;
+  height: 100%;
+  font-size: 14px;
+  font-weight: bold;
+  color: white;
+}
+
+.facepile .face-overflow {
+  margin-left: var(--spacing-xs);
+  font-size: 0.875rem;
+  color: var(--color-text-light);
+}
+
+/* Generic Mentions */
+.webmention-mention {
+  margin-bottom: var(--spacing-sm);
+  padding: var(--spacing-xs);
+  background: var(--color-bg-alt);
+  border-radius: 4px;
+}
+
+.webmention-mention p {
+  margin: var(--spacing-xs) 0 0 0;
+  font-size: 0.875rem;
+  color: var(--color-text-light);
+}
+
+/* Admin Webmention Styles */
+
+.filter-tabs {
+  display: flex;
+  gap: var(--spacing-sm);
+  margin-bottom: var(--spacing-md);
+  border-bottom: 1px solid var(--color-border);
+}
+
+.filter-tabs a {
+  padding: var(--spacing-sm) var(--spacing-md);
+  text-decoration: none;
+  color: var(--color-text-light);
+  border-bottom: 2px solid transparent;
+}
+
+.filter-tabs a.active {
+  color: var(--color-link);
+  border-bottom-color: var(--color-link);
+}
+
+.bulk-actions {
+  display: flex;
+  gap: var(--spacing-sm);
+  margin-bottom: var(--spacing-md);
+}
+
+.webmention-table {
+  width: 100%;
+  border-collapse: collapse;
+}
+
+.webmention-table th,
+.webmention-table td {
+  padding: var(--spacing-sm);
+  text-align: left;
+  border-bottom: 1px solid var(--color-border);
+}
+
+.webmention-table th {
+  background: var(--color-bg-alt);
+  font-weight: bold;
+}
+
+.badge {
+  display: inline-block;
+  padding: 0.25rem 0.5rem;
+  font-size: 0.75rem;
+  border-radius: 4px;
+  background: var(--color-bg-alt);
+}
+
+.badge-reply { background: #DBEAFE; color: #1E40AF; }
+.badge-like { background: #FECACA; color: #991B1B; }
+.badge-repost { background: #D1FAE5; color: #065F46; }
+.badge-bookmark { background: #FEF3C7; color: #92400E; }
+.badge-mention { background: #E5E7EB; color: #374151; }
+
+.status-badge {
+  display: inline-block;
+  padding: 0.25rem 0.5rem;
+  font-size: 0.75rem;
+  border-radius: 4px;
+  font-weight: 600;
+}
+
+.status-pending { background: #FEF3C7; color: #92400E; }
+.status-approved { background: #D1FAE5; color: #065F46; }
+.status-hidden { background: #E5E7EB; color: #6B7280; }
+.status-valid { background: #D1FAE5; color: #065F46; }
+.status-invalid { background: #FECACA; color: #991B1B; }
+.status-error { background: #FECACA; color: #991B1B; }
+
+.pagination {
+  display: flex;
+  align-items: center;
+  gap: var(--spacing-md);
+  justify-content: center;
+  margin-top: var(--spacing-lg);
+}
+
+.detail-card {
+  background: var(--color-bg-alt);
+  padding: var(--spacing-md);
+  border-radius: 4px;
+  margin-bottom: var(--spacing-lg);
+}
+
+.detail-row {
+  margin-bottom: var(--spacing-md);
+}
+
+.detail-row strong {
+  display: block;
+  margin-bottom: var(--spacing-xs);
+  color: var(--color-text-light);
+}
+
+.content-preview {
+  padding: var(--spacing-sm);
+  background: white;
+  border: 1px solid var(--color-border);
+  border-radius: 4px;
+  margin-top: var(--spacing-xs);
+}
+
+.moderation-actions {
+  margin-top: var(--spacing-lg);
+}
+
+.moderation-actions h3 {
+  margin-bottom: var(--spacing-sm);
+}
+
+.error-text {
+  color: #991B1B;
+}
+
+.empty-state {
+  text-align: center;
+  padding: var(--spacing-lg);
+  color: var(--color-text-light);
+}
+
+.empty-state .text-muted {
+  font-size: 0.875rem;
+}
diff --git a/templates/admin/dashboard.html b/templates/admin/dashboard.html
index 02b52f8..e3f2abe 100644
--- a/templates/admin/dashboard.html
+++ b/templates/admin/dashboard.html
@@ -10,6 +10,14 @@
     
       + New Note
     
+    {% if config.WEBMENTION_ENABLED %}
+    
+      Webmentions
+      {% if webmention_pending_count > 0 %}
+      {{ webmention_pending_count }}
+      {% endif %}
+    
+    {% endif %}
   
 
   {% if notes %}
@@ -59,6 +67,20 @@
       {% endfor %}
     
   
+
+  {# Pagination (v1.6.0) #}
+  {% if total_pages > 1 %}
+  
+  {% endif %}
+
   {% else %}
   

No notes yet. Create your first note!

diff --git a/templates/admin/webmention_detail.html b/templates/admin/webmention_detail.html new file mode 100644 index 0000000..752fb09 --- /dev/null +++ b/templates/admin/webmention_detail.html @@ -0,0 +1,108 @@ +{% extends "admin/base.html" %} + +{% block title %}Webmention Detail - StarPunk Admin{% endblock %} + +{% block admin_content %} +
+

Webmention Detail

+ + + +
+ + +
+ Target: + {% if webmention.target_note_id %} + {{ webmention.target_url }} + {% else %} + {{ webmention.target_url }} (Homepage) + {% endif %} +
+ +
+ Type: + {{ webmention.mention_type }} +
+ +
+ Author: + {{ webmention.author_name or 'Anonymous' }} + {% if webmention.author_url %} + ({{ webmention.author_url }}) + {% endif %} +
+ + {% if webmention.author_photo %} +
+ Author Photo: + {{ webmention.author_name or 'Author' }} +
+ {% endif %} + + {% if webmention.content_snippet %} +
+ Content: +
{{ webmention.content_snippet|safe }}
+
+ {% endif %} + +
+ Validation Status: + {{ webmention.validation_status }} + {% if webmention.validation_error %} +
{{ webmention.validation_error }} + {% endif %} +
+ +
+ Moderation Status: + {{ webmention.moderation_status }} +
+ +
+ Received: + {{ webmention.received_at.strftime('%B %d, %Y at %I:%M %p') }} +
+ + {% if webmention.validated_at %} +
+ Validated: + {{ webmention.validated_at.strftime('%B %d, %Y at %I:%M %p') }} +
+ {% endif %} +
+ + {# Moderation Actions #} +
+

Actions

+
+ + +
+ +
+ + +
+ +
+ + +
+
+
+{% endblock %} diff --git a/templates/admin/webmentions.html b/templates/admin/webmentions.html new file mode 100644 index 0000000..0dc101b --- /dev/null +++ b/templates/admin/webmentions.html @@ -0,0 +1,125 @@ +{% extends "admin/base.html" %} + +{% block title %}Webmentions - StarPunk Admin{% endblock %} + +{% block admin_content %} +
+

Webmentions

+ + + + {# Filter tabs #} + + + {% if webmentions %} +
+
+ + + +
+ + + + + + + + + + + + + + + + {% for wm in webmentions %} + + + + + + + + + + + {% endfor %} + +
SourceTargetTypeAuthorStatusReceivedActions
{{ wm.source_url|truncate(40) }} + {% if wm.target_note_id %} + {{ wm.target_slug }} + {% else %} + Homepage + {% endif %} + {{ wm.mention_type }}{{ wm.author_name or 'Anonymous' }} + + {{ wm.moderation_status }} + + {{ wm.received_at.strftime('%b %d, %Y %H:%M') }} + + View + +
+
+ + {# Pagination #} + {% if total_pages > 1 %} + + {% endif %} + + {% else %} + {# Empty states #} + {% if status_filter %} +
+

No webmentions match the current filters.

+ Clear Filters +
+ {% else %} +
+

No webmentions received yet.

+

They will appear here when other sites mention your content.

+
+ {% endif %} + {% endif %} +
+ + +{% endblock %} diff --git a/templates/base.html b/templates/base.html index 23f81a2..b73cb41 100644 --- a/templates/base.html +++ b/templates/base.html @@ -8,6 +8,11 @@ + {# Webmention endpoint (v1.6.0) #} + {% if config.WEBMENTION_ENABLED %} + + {% endif %} + {# rel-me links from discovered author profile (v1.2.0 Phase 2) #} {% if author and author.rel_me_links %} {% for profile_url in author.rel_me_links %} @@ -30,7 +35,9 @@ Home RSS {% if g.me %} - Admin + Dashboard + {% else %} + Login {% endif %}
{{ note.title }} {% endif %} + {# Reply context (v1.6.0 Phase 4) - displayed if this is a reply post #} + {% if note.in_reply_to %} +
+ In reply to: +
+ {% if reply_context and reply_context.fetch_status == 'fetched' %} + {% if reply_context.content_snippet %} +

{{ reply_context.content_snippet }}

+ {% endif %} + + + {% if reply_context.author_name %}{{ reply_context.author_name }}{% endif %} + {% if reply_context.site_name %} on {{ reply_context.site_name }}{% endif %} + {% if not reply_context.author_name and not reply_context.site_name %}{{ note.in_reply_to }}{% endif %} + + + {% else %} + + {{ note.in_reply_to }} + + {% endif %} +
+
+ {% endif %} + {# u-photo placement: Per draft spec, u-photo must be direct child of h-entry, #} {# NOT inside e-content. Media is rendered ABOVE e-content to meet this requirement. #} {{ display_media(note.media) }} @@ -68,5 +93,118 @@ + + {# Webmentions section (v1.6.0 Phase 5) - only shown if approved webmentions exist #} + {% if webmentions %} +
+

Responses

+ + {% if replies %} +
+

{{ replies|length }} {{ 'Reply' if replies|length == 1 else 'Replies' }}

+ {% for reply in replies %} + + {% endfor %} +
+ {% endif %} + + {% if likes %} +
+

{{ likes|length }} {{ 'Like' if likes|length == 1 else 'Likes' }}

+
+ {% for wm in likes[:8] %} + + {% if wm.author_photo %} + {{ wm.author_name or '' }} + {% else %} + + {{ (wm.author_name or '?')[0]|upper }} + + {% endif %} + + {% endfor %} + {% if likes|length > 8 %} + +{{ likes|length - 8 }} others + {% endif %} +
+
+ {% endif %} + + {% if reposts %} +
+

{{ reposts|length }} {{ 'Repost' if reposts|length == 1 else 'Reposts' }}

+
+ {% for wm in reposts[:8] %} + + {% if wm.author_photo %} + {{ wm.author_name or '' }} + {% else %} + + {{ (wm.author_name or '?')[0]|upper }} + + {% endif %} + + {% endfor %} + {% if reposts|length > 8 %} + +{{ reposts|length - 8 }} others + {% endif %} +
+
+ {% endif %} + + {% if bookmarks %} +
+

{{ bookmarks|length }} {{ 'Bookmark' if bookmarks|length == 1 else 'Bookmarks' }}

+
+ {% for wm in bookmarks[:8] %} + + {% if wm.author_photo %} + {{ wm.author_name or '' }} + {% else %} + + {{ (wm.author_name or '?')[0]|upper }} + + {% endif %} + + {% endfor %} + {% if bookmarks|length > 8 %} + +{{ bookmarks|length - 8 }} others + {% endif %} +
+
+ {% endif %} + + {% if mentions %} +
+

{{ mentions|length }} {{ 'Mention' if mentions|length == 1 else 'Mentions' }}

+ {% for mention in mentions %} +
+ {{ mention.author_name or mention.source_url }} + {% if mention.content_snippet %} +

{{ mention.content_snippet|truncate(200) }}

+ {% endif %} +
+ {% endfor %} +
+ {% endif %} +
+ {% endif %} {% endblock %}