chore: Recover v1.6.0-rc.2 source from production image
The v1.6.0 work (webmentions, reply contexts, rate limiter, migration 010) was built and deployed as git.thesatelliteoflove.com/phil/starpunk:v1.6.0-rc.2 but never committed. This restores the application code from /app in the running container so later work builds on what production runs. Tests and docs for v1.6.0 were not in the image and are not recovered. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
06a8aabc01
commit
d3fa5b7a2a
24 files changed
+3982
-78
No files matched your search
@@ -0,0 +1,79 @@
|
||||
-- Migration 010: Add Webmention support
|
||||
-- Version: 1.6.0
|
||||
-- Per ADR-063: Webmention Implementation
|
||||
-- Per W3C Webmention Recommendation
|
||||
|
||||
-- Add reply-to support to notes table
|
||||
ALTER TABLE notes ADD COLUMN in_reply_to TEXT;
|
||||
|
||||
-- Incoming webmentions storage
|
||||
CREATE TABLE IF NOT EXISTS webmentions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
source_url TEXT NOT NULL,
|
||||
target_url TEXT NOT NULL,
|
||||
target_note_id INTEGER,
|
||||
mention_type TEXT NOT NULL DEFAULT 'mention'
|
||||
CHECK (mention_type IN ('reply', 'like', 'repost', 'bookmark', 'mention')),
|
||||
author_name TEXT,
|
||||
author_url TEXT,
|
||||
author_photo TEXT,
|
||||
content_snippet TEXT,
|
||||
validation_status TEXT NOT NULL DEFAULT 'pending'
|
||||
CHECK (validation_status IN ('pending', 'valid', 'invalid', 'error')),
|
||||
moderation_status TEXT NOT NULL DEFAULT 'pending'
|
||||
CHECK (moderation_status IN ('pending', 'approved', 'hidden', 'deleted')),
|
||||
validation_error TEXT,
|
||||
validated_at TIMESTAMP,
|
||||
received_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (target_note_id) REFERENCES notes(id) ON DELETE SET NULL,
|
||||
UNIQUE(source_url, target_url)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_webmentions_target_note ON webmentions(target_note_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_webmentions_moderation ON webmentions(moderation_status);
|
||||
CREATE INDEX IF NOT EXISTS idx_webmentions_validation ON webmentions(validation_status);
|
||||
CREATE INDEX IF NOT EXISTS idx_webmentions_received ON webmentions(received_at DESC);
|
||||
|
||||
-- Outgoing webmention queue
|
||||
CREATE TABLE IF NOT EXISTS webmention_queue (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
source_url TEXT NOT NULL,
|
||||
target_url TEXT NOT NULL,
|
||||
source_note_id INTEGER,
|
||||
endpoint_url TEXT,
|
||||
status TEXT NOT NULL DEFAULT 'pending'
|
||||
CHECK (status IN ('pending', 'sending', 'sent', 'failed', 'no_endpoint')),
|
||||
retry_count INTEGER NOT NULL DEFAULT 0,
|
||||
next_retry_at TIMESTAMP,
|
||||
last_error TEXT,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
started_at TIMESTAMP,
|
||||
sent_at TIMESTAMP,
|
||||
FOREIGN KEY (source_note_id) REFERENCES notes(id) ON DELETE CASCADE,
|
||||
UNIQUE(source_url, target_url)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_webmention_queue_status ON webmention_queue(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_webmention_queue_retry ON webmention_queue(next_retry_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_webmention_queue_note ON webmention_queue(source_note_id);
|
||||
|
||||
-- Cached reply contexts
|
||||
CREATE TABLE IF NOT EXISTS reply_contexts (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
url TEXT UNIQUE NOT NULL,
|
||||
title TEXT,
|
||||
content_snippet TEXT,
|
||||
published_at TIMESTAMP,
|
||||
author_name TEXT,
|
||||
author_url TEXT,
|
||||
author_photo TEXT,
|
||||
site_name TEXT,
|
||||
fetch_status TEXT NOT NULL DEFAULT 'pending'
|
||||
CHECK (fetch_status IN ('pending', 'fetched', 'error')),
|
||||
fetch_error TEXT,
|
||||
fetched_at TIMESTAMP,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_reply_contexts_url ON reply_contexts(url);
|
||||
@@ -36,3 +36,6 @@ Pillow==10.1.*
|
||||
|
||||
# HEIC/HEIF Support (v1.4.2 - iPhone photos)
|
||||
pillow-heif==0.18.*
|
||||
|
||||
# HTML Sanitization (v1.6.0 - Webmention content)
|
||||
nh3==0.2.*
|
||||
+108
-4
@@ -156,6 +156,10 @@ def create_app(config=None):
|
||||
configure_cache(max_size=max_size, ttl=ttl)
|
||||
app.logger.info(f"Feed cache enabled (max_size={max_size}, ttl={ttl}s)")
|
||||
|
||||
# Initialize rate limiters (v1.6.0 Phase 2)
|
||||
from starpunk.rate_limiter import init_rate_limiters
|
||||
init_rate_limiters(app)
|
||||
|
||||
# Initialize FTS index if needed
|
||||
from pathlib import Path
|
||||
from starpunk.search import has_fts_table, rebuild_fts_index
|
||||
@@ -211,12 +215,77 @@ def create_app(config=None):
|
||||
|
||||
return {'author': author}
|
||||
|
||||
# Template filter - URL to color for facepile (v1.6.0)
|
||||
@app.template_filter('url_to_color')
|
||||
def url_to_color_filter(url):
|
||||
"""
|
||||
Generate consistent color from URL hash for facepile initials
|
||||
|
||||
Per display-moderation.md design: Hash URL to generate deterministic
|
||||
color for facepile fallback initials.
|
||||
|
||||
Args:
|
||||
url: Author URL or any string
|
||||
|
||||
Returns:
|
||||
CSS color string (hex format)
|
||||
"""
|
||||
import hashlib
|
||||
|
||||
if not url:
|
||||
return '#6B7280' # Gray fallback
|
||||
|
||||
# Hash the URL to get consistent color
|
||||
hash_value = int(hashlib.md5(url.encode()).hexdigest()[:6], 16)
|
||||
|
||||
# Generate pleasant, readable colors
|
||||
# Use HSL-style generation for better color distribution
|
||||
hue = hash_value % 360
|
||||
saturation = 65 # Medium saturation for readability
|
||||
lightness = 50 # Medium lightness
|
||||
|
||||
# Convert HSL to RGB (simplified)
|
||||
def hsl_to_rgb(h, s, l):
|
||||
s = s / 100.0
|
||||
l = l / 100.0
|
||||
c = (1 - abs(2 * l - 1)) * s
|
||||
x = c * (1 - abs((h / 60) % 2 - 1))
|
||||
m = l - c / 2
|
||||
|
||||
if h < 60:
|
||||
r, g, b = c, x, 0
|
||||
elif h < 120:
|
||||
r, g, b = x, c, 0
|
||||
elif h < 180:
|
||||
r, g, b = 0, c, x
|
||||
elif h < 240:
|
||||
r, g, b = 0, x, c
|
||||
elif h < 300:
|
||||
r, g, b = x, 0, c
|
||||
else:
|
||||
r, g, b = c, 0, x
|
||||
|
||||
return int((r + m) * 255), int((g + m) * 255), int((b + m) * 255)
|
||||
|
||||
r, g, b = hsl_to_rgb(hue, saturation, lightness)
|
||||
return f'#{r:02x}{g:02x}{b:02x}'
|
||||
|
||||
# Request middleware - Add correlation ID to each request
|
||||
@app.before_request
|
||||
def before_request():
|
||||
"""Add correlation ID to request context for tracing"""
|
||||
add_correlation_id()
|
||||
|
||||
# Response middleware - Add webmention Link header (v1.6.0)
|
||||
@app.after_request
|
||||
def add_webmention_headers(response):
|
||||
"""Add webmention Link header to HTML responses"""
|
||||
if (app.config.get('WEBMENTION_ENABLED', True) and
|
||||
response.content_type and
|
||||
'text/html' in response.content_type):
|
||||
response.headers['Link'] = '</webmention>; rel="webmention"'
|
||||
return response
|
||||
|
||||
# Register centralized error handlers
|
||||
from starpunk.errors import register_error_handlers
|
||||
|
||||
@@ -237,6 +306,32 @@ def create_app(config=None):
|
||||
if hasattr(app, 'memory_monitor') and app.memory_monitor.is_alive():
|
||||
app.memory_monitor.stop()
|
||||
|
||||
# Start webmention worker thread (v1.6.0 Phase 2)
|
||||
# Per background-worker.md: Skip in test mode
|
||||
if app.config.get('WEBMENTION_WORKER_ENABLED', True) and not app.config.get('TESTING', False):
|
||||
from starpunk.webmention_worker import WebmentionWorker
|
||||
webmention_worker = WebmentionWorker(
|
||||
db_path=app.config['DATABASE_PATH'],
|
||||
site_url=app.config['SITE_URL'],
|
||||
poll_interval=app.config.get('WEBMENTION_POLL_INTERVAL', 10),
|
||||
error_backoff=app.config.get('WEBMENTION_ERROR_BACKOFF', 60),
|
||||
batch_size=app.config.get('WEBMENTION_BATCH_SIZE', 5),
|
||||
auto_approve=app.config.get('WEBMENTION_AUTO_APPROVE', True)
|
||||
)
|
||||
webmention_worker.start()
|
||||
app.webmention_worker = webmention_worker
|
||||
app.logger.info(
|
||||
f"Webmention worker started "
|
||||
f"(poll_interval={webmention_worker.poll_interval}s, "
|
||||
f"batch_size={webmention_worker.batch_size})"
|
||||
)
|
||||
|
||||
# Register cleanup handler
|
||||
@app.teardown_appcontext
|
||||
def cleanup_webmention_worker(error=None):
|
||||
if hasattr(app, 'webmention_worker') and app.webmention_worker.is_alive():
|
||||
app.webmention_worker.stop()
|
||||
|
||||
# Health check endpoint for containers and monitoring
|
||||
@app.route("/health")
|
||||
def health_check():
|
||||
@@ -322,15 +417,24 @@ def create_app(config=None):
|
||||
else:
|
||||
# Basic mode - just return 200 OK (for load balancers)
|
||||
# No authentication required, minimal checks
|
||||
return jsonify({
|
||||
response = {
|
||||
"status": "ok",
|
||||
"version": app.config.get("VERSION", __version__)
|
||||
}), 200
|
||||
}
|
||||
|
||||
# Add webmention worker stats if enabled (v1.6.0 Phase 2)
|
||||
if hasattr(app, 'webmention_worker'):
|
||||
try:
|
||||
response['webmention_worker'] = app.webmention_worker.get_stats()
|
||||
except Exception as e:
|
||||
app.logger.debug(f"Failed to get webmention worker stats: {e}")
|
||||
|
||||
return jsonify(response), 200
|
||||
|
||||
return app
|
||||
|
||||
|
||||
# Package version (Semantic Versioning 2.0.0)
|
||||
# See docs/standards/versioning-strategy.md for details
|
||||
__version__ = "1.5.0"
|
||||
__version_info__ = (1, 5, 0)
|
||||
__version__ = "1.6.0"
|
||||
__version_info__ = (1, 6, 0)
|
||||
@@ -110,6 +110,21 @@ def load_config(app, config_override=None):
|
||||
app.config["DEBUG_FILE_MAX_AGE_DAYS"] = int(os.getenv("DEBUG_FILE_MAX_AGE_DAYS", "7"))
|
||||
app.config["DEBUG_FILE_MAX_SIZE_MB"] = int(os.getenv("DEBUG_FILE_MAX_SIZE_MB", "100"))
|
||||
|
||||
# Webmention configuration (v1.6.0)
|
||||
app.config["WEBMENTION_ENABLED"] = os.getenv("WEBMENTION_ENABLED", "true").lower() == "true"
|
||||
app.config["WEBMENTION_WORKER_ENABLED"] = os.getenv("WEBMENTION_WORKER_ENABLED", "true").lower() == "true"
|
||||
app.config["WEBMENTION_POLL_INTERVAL"] = int(os.getenv("WEBMENTION_POLL_INTERVAL", "10"))
|
||||
app.config["WEBMENTION_ERROR_BACKOFF"] = int(os.getenv("WEBMENTION_ERROR_BACKOFF", "60"))
|
||||
app.config["WEBMENTION_BATCH_SIZE"] = int(os.getenv("WEBMENTION_BATCH_SIZE", "5"))
|
||||
app.config["WEBMENTION_AUTO_APPROVE"] = os.getenv("WEBMENTION_AUTO_APPROVE", "true").lower() == "true"
|
||||
|
||||
# Webmention rate limiting (v1.6.0 Phase 2)
|
||||
app.config["WEBMENTION_RATE_LIMIT_ENABLED"] = os.getenv("WEBMENTION_RATE_LIMIT_ENABLED", "true").lower() == "true"
|
||||
app.config["WEBMENTION_RATE_LIMIT_IP"] = int(os.getenv("WEBMENTION_RATE_LIMIT_IP", "60"))
|
||||
app.config["WEBMENTION_RATE_LIMIT_DOMAIN"] = int(os.getenv("WEBMENTION_RATE_LIMIT_DOMAIN", "10"))
|
||||
app.config["WEBMENTION_RATE_LIMIT_WINDOW"] = int(os.getenv("WEBMENTION_RATE_LIMIT_WINDOW", "60"))
|
||||
app.config["TRUSTED_PROXY"] = os.getenv("TRUSTED_PROXY", "false").lower() == "true"
|
||||
|
||||
# Apply overrides if provided
|
||||
if config_override:
|
||||
app.config.update(config_override)
|
||||
|
||||
+141
-56
@@ -14,15 +14,21 @@ Debug file management (v1.5.0 Phase 2):
|
||||
- Debug file saving disabled by default
|
||||
- Automatic cleanup of old debug files
|
||||
- Size limit enforcement
|
||||
|
||||
Memory-safe image processing (v1.5.1):
|
||||
- Early downscale for large images (>4096px) to prevent OOM
|
||||
- PIL Image passed through pipeline (not bytes) to avoid memory copies
|
||||
- Explicit memory cleanup after processing steps
|
||||
"""
|
||||
|
||||
from PIL import Image, ImageOps
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timedelta
|
||||
import time
|
||||
import uuid
|
||||
import io
|
||||
import shutil
|
||||
from typing import Optional, List, Dict, Tuple
|
||||
from typing import Optional, List, Dict, Tuple, Union
|
||||
from flask import current_app
|
||||
|
||||
# HEIC/HEIF support - import registers with Pillow automatically
|
||||
@@ -50,6 +56,11 @@ MIN_QUALITY = 70 # Minimum JPEG quality before rejection (v1.4.
|
||||
MIN_DIMENSION = 640 # Minimum dimension before rejection (v1.4.0)
|
||||
MAX_IMAGES_PER_NOTE = 4
|
||||
|
||||
# Memory safety limits (v1.5.1 - fix for OOM on large phone images)
|
||||
# Early downscale very large images to prevent memory exhaustion
|
||||
# A 4096x4096 image = 50MB uncompressed, manageable per worker
|
||||
MEMORY_SAFE_DIMENSION = 4096 # Downscale to this before heavy processing
|
||||
|
||||
# Variant specifications (v1.4.0 Phase 2)
|
||||
VARIANT_SPECS = {
|
||||
'thumb': {'size': (150, 150), 'crop': True},
|
||||
@@ -82,21 +93,22 @@ def get_optimization_params(file_size: int) -> Tuple[int, int]:
|
||||
return (1280, 85)
|
||||
|
||||
|
||||
def validate_image(file_data: bytes, filename: str) -> Tuple[bytes, str, int, int]:
|
||||
def validate_image(file_data: bytes, filename: str) -> Tuple[Image.Image, str, int, int]:
|
||||
"""
|
||||
Validate image file
|
||||
Validate image file and return PIL Image for memory-efficient processing
|
||||
|
||||
Per Q11: Validate MIME type using Pillow
|
||||
Per Q6: Reject if >50MB or >4096px (updated v1.4.0)
|
||||
Per Q6: Reject if >50MB or >12000px
|
||||
Per v1.4.2: Convert HEIC to JPEG (browsers cannot display HEIC)
|
||||
Per v1.5.1: Return PIL Image (not bytes) to avoid memory copies
|
||||
|
||||
Args:
|
||||
file_data: Raw file bytes
|
||||
filename: Original filename
|
||||
|
||||
Returns:
|
||||
Tuple of (file_data, mime_type, width, height)
|
||||
Note: file_data may be converted (e.g., HEIC to JPEG)
|
||||
Tuple of (pil_image, mime_type, original_width, original_height)
|
||||
Note: Image may be downscaled for memory safety but original dimensions returned
|
||||
|
||||
Raises:
|
||||
ValueError: If file is invalid
|
||||
@@ -150,40 +162,34 @@ def validate_image(file_data: bytes, filename: str) -> Tuple[bytes, str, int, in
|
||||
else:
|
||||
raise ValueError(f"Invalid or corrupted image: {e}")
|
||||
|
||||
# Store original format before any conversions
|
||||
original_format = img.format
|
||||
|
||||
# HEIC/HEIF conversion (v1.4.2)
|
||||
# HEIC cannot be displayed in browsers, convert to JPEG
|
||||
if img.format in ('HEIF', 'HEIC'):
|
||||
if original_format in ('HEIF', 'HEIC'):
|
||||
if not HEIC_SUPPORTED:
|
||||
raise ValueError(
|
||||
"HEIC/HEIF images require pillow-heif library. "
|
||||
"Please convert to JPEG before uploading."
|
||||
)
|
||||
# Convert HEIC to JPEG in memory
|
||||
output = io.BytesIO()
|
||||
# Convert to RGB if needed (HEIC may have alpha channel)
|
||||
if img.mode in ('RGBA', 'P'):
|
||||
img = img.convert('RGB')
|
||||
img.save(output, format='JPEG', quality=95)
|
||||
output.seek(0)
|
||||
# Re-open as JPEG for further processing
|
||||
file_data = output.getvalue()
|
||||
img = Image.open(io.BytesIO(file_data))
|
||||
# Mark format as JPEG for later processing
|
||||
original_format = 'JPEG'
|
||||
|
||||
# MPO (Multi-Picture Object) conversion (v1.4.2)
|
||||
# MPO is used by iPhones for depth/portrait photos - extract primary image as JPEG
|
||||
if img.format == 'MPO':
|
||||
output = io.BytesIO()
|
||||
# MPO is used by iPhones for depth/portrait photos - extract primary image
|
||||
if original_format == 'MPO':
|
||||
# Convert to RGB if needed
|
||||
if img.mode in ('RGBA', 'P'):
|
||||
img = img.convert('RGB')
|
||||
img.save(output, format='JPEG', quality=95)
|
||||
output.seek(0)
|
||||
file_data = output.getvalue()
|
||||
img = Image.open(io.BytesIO(file_data))
|
||||
original_format = 'JPEG'
|
||||
|
||||
# Check format is allowed
|
||||
if img.format:
|
||||
format_lower = img.format.lower()
|
||||
if original_format:
|
||||
format_lower = original_format.lower()
|
||||
mime_type = f'image/{format_lower}'
|
||||
|
||||
# Special case: JPEG format can be reported as 'jpeg'
|
||||
@@ -195,26 +201,29 @@ def validate_image(file_data: bytes, filename: str) -> Tuple[bytes, str, int, in
|
||||
try:
|
||||
current_app.logger.warning(
|
||||
f'Media upload rejected format: filename="{filename}", '
|
||||
f'detected_format="{img.format}", mime_type="{mime_type}"'
|
||||
f'detected_format="{original_format}", mime_type="{mime_type}"'
|
||||
)
|
||||
except RuntimeError:
|
||||
pass # Outside app context
|
||||
raise ValueError(f"Invalid image format '{img.format}'. Accepted: JPEG, PNG, GIF, WebP")
|
||||
raise ValueError(f"Invalid image format '{original_format}'. Accepted: JPEG, PNG, GIF, WebP")
|
||||
else:
|
||||
raise ValueError("Could not determine image format")
|
||||
|
||||
# Store original dimensions before any resizing
|
||||
orig_width, orig_height = img.size
|
||||
|
||||
# Check dimensions (v1.4.2: increased to 12000px to support modern phone cameras)
|
||||
# Images will be resized by optimize_image() anyway
|
||||
width, height = img.size
|
||||
if max(width, height) > MAX_DIMENSION:
|
||||
if max(orig_width, orig_height) > MAX_DIMENSION:
|
||||
raise ValueError(f"Image dimensions too large. Maximum is {MAX_DIMENSION}x{MAX_DIMENSION} pixels")
|
||||
|
||||
# Check for animated GIF (v1.4.0)
|
||||
# Animated GIFs cannot be resized, so reject if >10MB
|
||||
if img.format == 'GIF':
|
||||
is_animated_gif = False
|
||||
if original_format == 'GIF':
|
||||
try:
|
||||
img.seek(1) # Try to seek to second frame
|
||||
# If successful, it's animated
|
||||
is_animated_gif = True
|
||||
if file_size > MAX_OUTPUT_SIZE:
|
||||
raise ValueError(
|
||||
"Animated GIF too large. Maximum size for animated GIFs is 10MB. "
|
||||
@@ -225,10 +234,38 @@ def validate_image(file_data: bytes, filename: str) -> Tuple[bytes, str, int, in
|
||||
# Not animated, continue normally
|
||||
pass
|
||||
|
||||
return file_data, mime_type, width, height
|
||||
# v1.5.1: Early downscale for memory safety
|
||||
# Large phone images (48MP+) can use 500MB+ per copy in memory
|
||||
# Downscale early to MEMORY_SAFE_DIMENSION to reduce memory footprint
|
||||
# This happens BEFORE optimize_image to prevent OOM during processing
|
||||
if not is_animated_gif and max(orig_width, orig_height) > MEMORY_SAFE_DIMENSION:
|
||||
try:
|
||||
current_app.logger.info(
|
||||
f'Memory safety downscale: {orig_width}x{orig_height} -> '
|
||||
f'{MEMORY_SAFE_DIMENSION}x{MEMORY_SAFE_DIMENSION} max'
|
||||
)
|
||||
except RuntimeError:
|
||||
pass # Outside app context
|
||||
|
||||
# Apply EXIF orientation before resize
|
||||
img = ImageOps.exif_transpose(img)
|
||||
|
||||
# Use thumbnail() for memory-efficient in-place resize
|
||||
img.thumbnail((MEMORY_SAFE_DIMENSION, MEMORY_SAFE_DIMENSION), Image.Resampling.LANCZOS)
|
||||
|
||||
# Store the format on the image object for later use
|
||||
img.format = original_format
|
||||
|
||||
# Clear file_data reference to allow garbage collection
|
||||
del file_data
|
||||
|
||||
return img, mime_type, orig_width, orig_height
|
||||
|
||||
|
||||
def optimize_image(image_data: bytes, original_size: int = None) -> Tuple[Image.Image, int, int, bytes]:
|
||||
def optimize_image(
|
||||
image_input: Union[Image.Image, bytes],
|
||||
original_size: int = None
|
||||
) -> Tuple[Image.Image, int, int, bytes]:
|
||||
"""
|
||||
Optimize image for web display with size-aware strategy
|
||||
|
||||
@@ -237,8 +274,12 @@ def optimize_image(image_data: bytes, original_size: int = None) -> Tuple[Image.
|
||||
- Iterative quality reduction if needed
|
||||
- Target output <=10MB
|
||||
|
||||
Per v1.5.1:
|
||||
- Accept PIL Image directly to avoid memory copies
|
||||
- Image may already be downscaled for memory safety
|
||||
|
||||
Args:
|
||||
image_data: Raw image bytes
|
||||
image_input: PIL Image object or raw image bytes
|
||||
original_size: Original file size (for tiered optimization)
|
||||
|
||||
Returns:
|
||||
@@ -247,24 +288,35 @@ def optimize_image(image_data: bytes, original_size: int = None) -> Tuple[Image.
|
||||
Raises:
|
||||
ValueError: If image cannot be optimized to target size
|
||||
"""
|
||||
if original_size is None:
|
||||
original_size = len(image_data)
|
||||
# Handle both PIL Image and bytes input (v1.5.1)
|
||||
if isinstance(image_input, Image.Image):
|
||||
img = image_input
|
||||
# For PIL input, use provided size or estimate from current dimensions
|
||||
if original_size is None:
|
||||
# Estimate: assume worst case of uncompressed RGB
|
||||
original_size = img.width * img.height * 3
|
||||
else:
|
||||
# Legacy bytes input path
|
||||
if original_size is None:
|
||||
original_size = len(image_input)
|
||||
img = Image.open(io.BytesIO(image_input))
|
||||
|
||||
# Get initial optimization parameters based on input size
|
||||
max_dim, quality = get_optimization_params(original_size)
|
||||
|
||||
img = Image.open(io.BytesIO(image_data))
|
||||
|
||||
# Save original format before any processing (copy() loses this)
|
||||
original_format = img.format
|
||||
original_format = img.format or 'JPEG'
|
||||
|
||||
# Correct EXIF orientation (per ADR-058), except for GIFs
|
||||
img = ImageOps.exif_transpose(img) if img.format != 'GIF' else img
|
||||
# Note: For v1.5.1, this may already be done in validate_image for large images
|
||||
if original_format != 'GIF':
|
||||
img = ImageOps.exif_transpose(img)
|
||||
|
||||
# For animated GIFs, return as-is (already validated in validate_image)
|
||||
if img.format == 'GIF' and getattr(img, 'is_animated', False):
|
||||
# Already checked size in validate_image, just return original
|
||||
return img, img.size[0], img.size[1], image_data
|
||||
# For animated GIFs, serialize and return (already validated in validate_image)
|
||||
if original_format == 'GIF' and getattr(img, 'is_animated', False):
|
||||
output = io.BytesIO()
|
||||
img.save(output, format='GIF')
|
||||
return img, img.size[0], img.size[1], output.getvalue()
|
||||
|
||||
# Iterative optimization loop
|
||||
while True:
|
||||
@@ -277,8 +329,7 @@ def optimize_image(image_data: bytes, original_size: int = None) -> Tuple[Image.
|
||||
|
||||
# Save to bytes to check size
|
||||
output = io.BytesIO()
|
||||
# Use original format (copy() loses the format attribute)
|
||||
save_format = original_format or 'JPEG'
|
||||
save_format = original_format
|
||||
save_kwargs = {'optimize': True}
|
||||
|
||||
if save_format in ['JPEG', 'JPG']:
|
||||
@@ -528,6 +579,7 @@ def save_media(file_data: bytes, filename: str) -> Dict:
|
||||
Per Q2: Date-organized path: /media/YYYY/MM/uuid.ext
|
||||
Per Q6: Validate, optimize, then save
|
||||
Per v1.4.0: Size-aware optimization with iterative quality reduction
|
||||
Per v1.6.0 Phase 0: Timing logs for performance monitoring
|
||||
|
||||
Args:
|
||||
file_data: Raw file bytes
|
||||
@@ -544,24 +596,42 @@ def save_media(file_data: bytes, filename: str) -> Dict:
|
||||
# Capture file size for logging
|
||||
file_size = len(file_data)
|
||||
|
||||
# Timing instrumentation (v1.6.0 Phase 0)
|
||||
timing_start = time.monotonic()
|
||||
timing_validation = 0.0
|
||||
timing_optimization = 0.0
|
||||
timing_variants = 0.0
|
||||
timing_save = 0.0
|
||||
|
||||
try:
|
||||
# Validate image (returns 4-tuple with potentially converted bytes)
|
||||
# Validate image (v1.5.1: returns PIL Image to avoid memory copies)
|
||||
try:
|
||||
file_data, mime_type, orig_width, orig_height = validate_image(file_data, filename)
|
||||
t_start = time.monotonic()
|
||||
validated_img, mime_type, orig_width, orig_height = validate_image(file_data, filename)
|
||||
timing_validation = time.monotonic() - t_start
|
||||
# Clear file_data early to reduce memory pressure
|
||||
del file_data
|
||||
except ValueError as e:
|
||||
timing_validation = time.monotonic() - timing_start
|
||||
current_app.logger.warning(
|
||||
f'Media upload validation failed: filename="{filename}", '
|
||||
f'size={file_size}b, error="{e}"'
|
||||
f'size={file_size}b, error="{e}", timing: validation={timing_validation:.1f}s'
|
||||
)
|
||||
raise
|
||||
|
||||
# Optimize image with size-aware strategy (now returns 4-tuple with bytes)
|
||||
# Optimize image with size-aware strategy (v1.5.1: accepts PIL Image)
|
||||
try:
|
||||
optimized_img, width, height, optimized_bytes = optimize_image(file_data, file_size)
|
||||
t_start = time.monotonic()
|
||||
optimized_img, width, height, optimized_bytes = optimize_image(validated_img, file_size)
|
||||
timing_optimization = time.monotonic() - t_start
|
||||
# Clear validated_img reference (optimized_img is the result now)
|
||||
del validated_img
|
||||
except ValueError as e:
|
||||
timing_optimization = time.monotonic() - t_start
|
||||
current_app.logger.warning(
|
||||
f'Media upload optimization failed: filename="{filename}", '
|
||||
f'size={file_size}b, error="{e}"'
|
||||
f'size={file_size}b, error="{e}", '
|
||||
f'timing: validation={timing_validation:.1f}s, optimization={timing_optimization:.1f}s'
|
||||
)
|
||||
raise
|
||||
|
||||
@@ -601,7 +671,8 @@ def save_media(file_data: bytes, filename: str) -> Dict:
|
||||
temp_subdir = None
|
||||
|
||||
try:
|
||||
# Step 1: Save original to temp directory
|
||||
# Step 1: Generate variants to temp directory
|
||||
t_start = time.monotonic()
|
||||
media_dir = Path(current_app.config.get('DATA_PATH', 'data')) / 'media'
|
||||
temp_dir = media_dir / '.tmp'
|
||||
temp_dir.mkdir(parents=True, exist_ok=True)
|
||||
@@ -623,8 +694,10 @@ def save_media(file_data: bytes, filename: str) -> Dict:
|
||||
optimized_bytes,
|
||||
db
|
||||
)
|
||||
timing_variants = time.monotonic() - t_start
|
||||
|
||||
# Step 3: Begin transaction
|
||||
# Step 3: Begin transaction and save to database
|
||||
t_start = time.monotonic()
|
||||
db.execute("BEGIN TRANSACTION")
|
||||
|
||||
# Step 4: Insert media record
|
||||
@@ -667,6 +740,7 @@ def save_media(file_data: bytes, filename: str) -> Dict:
|
||||
|
||||
# Step 7: Commit transaction (after files are moved successfully)
|
||||
db.commit()
|
||||
timing_save = time.monotonic() - t_start
|
||||
|
||||
# Step 8: Clean up temp directory
|
||||
try:
|
||||
@@ -679,6 +753,9 @@ def save_media(file_data: bytes, filename: str) -> Dict:
|
||||
variants = [v for v in variant_metadata if v['variant_type'] != 'original']
|
||||
|
||||
except Exception as e:
|
||||
# Capture timing for failed save operation
|
||||
timing_save = time.monotonic() - t_start if 't_start' in locals() else 0.0
|
||||
|
||||
# Rollback database on any failure (best effort)
|
||||
try:
|
||||
db.rollback()
|
||||
@@ -722,19 +799,27 @@ def save_media(file_data: bytes, filename: str) -> Dict:
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
# Log and re-raise
|
||||
# Calculate total timing
|
||||
timing_total = time.monotonic() - timing_start
|
||||
|
||||
# Log and re-raise (with timing data)
|
||||
current_app.logger.warning(
|
||||
f'Media upload atomic operation failed: filename="{filename}", '
|
||||
f'error="{e}"'
|
||||
f'error="{e}", timing: validation={timing_validation:.1f}s, '
|
||||
f'optimization={timing_optimization:.1f}s, variants={timing_variants:.1f}s, '
|
||||
f'save={timing_save:.1f}s, total={timing_total:.1f}s'
|
||||
)
|
||||
raise
|
||||
|
||||
# Log success
|
||||
# Log success with timing data (v1.6.0 Phase 0)
|
||||
timing_total = time.monotonic() - timing_start
|
||||
was_optimized = len(optimized_bytes) < file_size
|
||||
current_app.logger.info(
|
||||
f'Media upload successful: filename="{filename}", '
|
||||
f'stored="{stored_filename}", size={len(optimized_bytes)}b, '
|
||||
f'optimized={was_optimized}, variants={len(variants)}'
|
||||
f'optimized={was_optimized}, variants={len(variants)}, '
|
||||
f'timing: validation={timing_validation:.1f}s, optimization={timing_optimization:.1f}s, '
|
||||
f'variants={timing_variants:.1f}s, save={timing_save:.1f}s, total={timing_total:.1f}s'
|
||||
)
|
||||
|
||||
return {
|
||||
|
||||
@@ -114,6 +114,7 @@ class Note:
|
||||
# Optional fields
|
||||
deleted_at: Optional[datetime] = None
|
||||
content_hash: Optional[str] = None
|
||||
in_reply_to: Optional[str] = None # v1.6.0: URL being replied to
|
||||
_cached_content: Optional[str] = field(
|
||||
default=None, repr=False, compare=False, init=False
|
||||
)
|
||||
@@ -171,6 +172,7 @@ class Note:
|
||||
deleted_at=deleted_at,
|
||||
_data_dir=data_dir,
|
||||
content_hash=data.get("content_hash"),
|
||||
in_reply_to=data.get("in_reply_to"),
|
||||
)
|
||||
|
||||
@property
|
||||
|
||||
+134
-11
@@ -138,7 +138,8 @@ def create_note(
|
||||
published: bool = False,
|
||||
created_at: Optional[datetime] = None,
|
||||
custom_slug: Optional[str] = None,
|
||||
tags: Optional[list[str]] = None
|
||||
tags: Optional[list[str]] = None,
|
||||
in_reply_to: Optional[str] = None
|
||||
) -> Note:
|
||||
"""
|
||||
Create a new note
|
||||
@@ -153,6 +154,7 @@ def create_note(
|
||||
created_at: Creation timestamp (default: current UTC time)
|
||||
custom_slug: Optional custom slug (from Micropub mp-slug property)
|
||||
tags: Optional list of tag display names (v1.3.0)
|
||||
in_reply_to: Optional URL being replied to (v1.6.0)
|
||||
|
||||
Returns:
|
||||
Note object with all metadata and content loaded
|
||||
@@ -262,10 +264,10 @@ def create_note(
|
||||
try:
|
||||
db.execute(
|
||||
"""
|
||||
INSERT INTO notes (slug, file_path, published, created_at, updated_at, content_hash)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
INSERT INTO notes (slug, file_path, published, created_at, updated_at, content_hash, in_reply_to)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(slug, file_path_rel, published, created_at, updated_at, content_hash),
|
||||
(slug, file_path_rel, published, created_at, updated_at, content_hash, in_reply_to),
|
||||
)
|
||||
db.commit()
|
||||
except Exception as e:
|
||||
@@ -313,6 +315,44 @@ def create_note(
|
||||
# FTS update failure should not prevent note creation
|
||||
current_app.logger.warning(f"Failed to update FTS index for note {slug}: {e}")
|
||||
|
||||
# 11. QUEUE WEBMENTIONS (v1.6.0 Phase 3)
|
||||
# Only queue if note is published (not draft)
|
||||
if published:
|
||||
try:
|
||||
from starpunk.webmention_sender import queue_webmentions_for_note
|
||||
db_path = Path(current_app.config["DATABASE_PATH"])
|
||||
site_url = current_app.config["SITE_URL"]
|
||||
note_url = f"{site_url}/note/{slug}"
|
||||
|
||||
# Get rendered HTML from note
|
||||
html_content = note.html
|
||||
|
||||
# Queue webmentions (best-effort, failures logged but don't fail note creation)
|
||||
queue_webmentions_for_note(
|
||||
db_path=db_path,
|
||||
note_id=note_id,
|
||||
note_url=note_url,
|
||||
content_html=html_content,
|
||||
in_reply_to=in_reply_to
|
||||
)
|
||||
except Exception as e:
|
||||
# Webmention queueing failure should not prevent note creation
|
||||
current_app.logger.warning(f"Failed to queue webmentions for note {slug}: {e}")
|
||||
|
||||
# 12. FETCH REPLY CONTEXT (v1.6.0 Phase 3)
|
||||
# Fetch context if this is a reply
|
||||
if in_reply_to:
|
||||
try:
|
||||
from starpunk.reply_context import fetch_reply_context
|
||||
db_path = Path(current_app.config["DATABASE_PATH"])
|
||||
site_url = current_app.config["SITE_URL"]
|
||||
|
||||
# Fetch context (best-effort, failures logged but don't fail note creation)
|
||||
fetch_reply_context(db_path, in_reply_to, site_url)
|
||||
except Exception as e:
|
||||
# Reply context fetch failure should not prevent note creation
|
||||
current_app.logger.warning(f"Failed to fetch reply context for {in_reply_to}: {e}")
|
||||
|
||||
return note
|
||||
|
||||
|
||||
@@ -548,7 +588,8 @@ def update_note(
|
||||
id: Optional[int] = None,
|
||||
content: Optional[str] = None,
|
||||
published: Optional[bool] = None,
|
||||
tags: Optional[list[str]] = None
|
||||
tags: Optional[list[str]] = None,
|
||||
in_reply_to: Optional[str] = None
|
||||
) -> Note:
|
||||
"""
|
||||
Update a note's content and/or published status
|
||||
@@ -563,6 +604,7 @@ def update_note(
|
||||
content: New markdown content (None = no change)
|
||||
published: New published status (None = no change)
|
||||
tags: New tags list (None = no change, [] = remove all tags) (v1.3.0)
|
||||
in_reply_to: New in-reply-to URL (None = no change) (v1.6.0)
|
||||
|
||||
Returns:
|
||||
Updated Note object with new content and metadata
|
||||
@@ -618,8 +660,8 @@ def update_note(
|
||||
if slug is not None and id is not None:
|
||||
raise ValueError("Cannot provide both slug and id")
|
||||
|
||||
if content is None and published is None and tags is None:
|
||||
raise ValueError("Must provide at least one of content, published, or tags to update")
|
||||
if content is None and published is None and tags is None and in_reply_to is None:
|
||||
raise ValueError("Must provide at least one of content, published, tags, or in_reply_to to update")
|
||||
|
||||
# Validate content if provided
|
||||
if content is not None:
|
||||
@@ -629,7 +671,8 @@ def update_note(
|
||||
)
|
||||
|
||||
# 2. GET EXISTING NOTE
|
||||
existing_note = get_note(slug=slug, id=id, load_content=False)
|
||||
# Phase 6: Load content to get old HTML for webmention URL diffing
|
||||
existing_note = get_note(slug=slug, id=id, load_content=True)
|
||||
|
||||
if existing_note is None:
|
||||
identifier = slug if slug is not None else id
|
||||
@@ -648,7 +691,18 @@ def update_note(
|
||||
"Path validation failed",
|
||||
)
|
||||
|
||||
# 4. UPDATE FILE (if content changed)
|
||||
# 4. CAPTURE OLD HTML (Phase 6: for webmention URL diffing)
|
||||
# Capture old HTML before writing new content
|
||||
# Only if note is/was published (unpublished notes don't send webmentions)
|
||||
old_html = None
|
||||
if existing_note.published:
|
||||
try:
|
||||
old_html = existing_note.html
|
||||
except Exception as e:
|
||||
# HTML rendering failed - continue with update, skip old_html
|
||||
current_app.logger.warning(f"Failed to render old HTML for note {existing_note.slug}: {e}")
|
||||
|
||||
# 5. UPDATE FILE (if content changed)
|
||||
new_content_hash = existing_note.content_hash
|
||||
if content is not None:
|
||||
try:
|
||||
@@ -664,7 +718,7 @@ def update_note(
|
||||
f"Could not update note file: {note_path}",
|
||||
)
|
||||
|
||||
# 5. UPDATE DATABASE
|
||||
# 6. UPDATE DATABASE
|
||||
db = get_db(current_app)
|
||||
|
||||
# Build update query based on what changed
|
||||
@@ -679,6 +733,10 @@ def update_note(
|
||||
update_fields.append("published = ?")
|
||||
params.append(published)
|
||||
|
||||
if in_reply_to is not None:
|
||||
update_fields.append("in_reply_to = ?")
|
||||
params.append(in_reply_to)
|
||||
|
||||
# Add WHERE clause parameter
|
||||
if slug is not None:
|
||||
where_clause = "slug = ?"
|
||||
@@ -726,9 +784,74 @@ def update_note(
|
||||
# FTS update failure should not prevent note update
|
||||
current_app.logger.warning(f"Failed to update FTS index for note {existing_note.slug}: {e}")
|
||||
|
||||
# 8. RETURN UPDATED NOTE
|
||||
# 8. QUEUE WEBMENTIONS (v1.6.0 Phase 6)
|
||||
# Queue webmentions if:
|
||||
# - Note is being unpublished (notify removed URLs)
|
||||
# - Note is being published (content changed OR publish status changed)
|
||||
# - Note was published and content/in_reply_to changed
|
||||
updated_note = get_note(slug=existing_note.slug, load_content=True)
|
||||
|
||||
# Detect unpublishing (published -> unpublished)
|
||||
is_unpublishing = existing_note.published and published is False
|
||||
|
||||
# Check if content actually changed (Phase 6 optimization)
|
||||
content_changed = (new_content_hash != existing_note.content_hash)
|
||||
publish_status_changed = (published is not None and published != existing_note.published)
|
||||
|
||||
# Determine if we should queue webmentions
|
||||
should_queue = False
|
||||
if is_unpublishing:
|
||||
# Unpublishing - queue for ALL previously-linked URLs
|
||||
should_queue = True
|
||||
elif published is not None and published:
|
||||
# Being published now - queue if content changed OR just being published
|
||||
should_queue = content_changed or publish_status_changed
|
||||
elif existing_note.published and (content_changed or in_reply_to is not None):
|
||||
# Was already published and content/reply-to changed
|
||||
should_queue = True
|
||||
|
||||
if should_queue:
|
||||
try:
|
||||
from starpunk.webmention_sender import queue_webmentions_for_note
|
||||
db_path = Path(current_app.config["DATABASE_PATH"])
|
||||
site_url = current_app.config["SITE_URL"]
|
||||
note_url = f"{site_url}/note/{updated_note.slug}"
|
||||
|
||||
# Get rendered HTML from updated note (if published)
|
||||
html_content = updated_note.html if updated_note.published else None
|
||||
|
||||
# Determine in_reply_to (use new value if provided, otherwise existing)
|
||||
reply_to = in_reply_to if in_reply_to is not None else updated_note.in_reply_to
|
||||
|
||||
# Queue webmentions (best-effort)
|
||||
# Pass old_html for URL diffing (Phase 6)
|
||||
queue_webmentions_for_note(
|
||||
db_path=db_path,
|
||||
note_id=updated_note.id,
|
||||
note_url=note_url,
|
||||
content_html=html_content,
|
||||
in_reply_to=reply_to,
|
||||
old_content_html=old_html # Phase 6: for URL diffing
|
||||
)
|
||||
except Exception as e:
|
||||
# Webmention queueing failure should not prevent note update
|
||||
current_app.logger.warning(f"Failed to queue webmentions for note {updated_note.slug}: {e}")
|
||||
|
||||
# 9. FETCH REPLY CONTEXT (v1.6.0 Phase 3)
|
||||
# Fetch context if in_reply_to changed and is not None
|
||||
if in_reply_to is not None:
|
||||
try:
|
||||
from starpunk.reply_context import fetch_reply_context
|
||||
db_path = Path(current_app.config["DATABASE_PATH"])
|
||||
site_url = current_app.config["SITE_URL"]
|
||||
|
||||
# Fetch context (best-effort)
|
||||
fetch_reply_context(db_path, in_reply_to, site_url)
|
||||
except Exception as e:
|
||||
# Reply context fetch failure should not prevent note update
|
||||
current_app.logger.warning(f"Failed to fetch reply context for {in_reply_to}: {e}")
|
||||
|
||||
# 10. RETURN UPDATED NOTE
|
||||
return updated_note
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
"""
|
||||
Rate limiting for webmention endpoint
|
||||
|
||||
Implements sliding window log algorithm with thread-safe in-memory storage.
|
||||
Per ADR-064, this uses in-memory counters consistent with the existing
|
||||
_discovery_cache pattern in webmention_worker.py.
|
||||
|
||||
References:
|
||||
- ADR-064: Webmention Rate Limiting
|
||||
- Design: /docs/design/v1.6.0/rate-limiting.md
|
||||
"""
|
||||
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, List, Tuple
|
||||
|
||||
|
||||
class RateLimiter:
|
||||
"""
|
||||
Thread-safe sliding window rate limiter.
|
||||
|
||||
Uses sliding window log algorithm:
|
||||
- Stores timestamp of each request in window
|
||||
- Removes expired entries on each check
|
||||
- Rejects if count >= limit
|
||||
"""
|
||||
|
||||
def __init__(self, limit: int, window_seconds: int):
|
||||
"""
|
||||
Initialize rate limiter.
|
||||
|
||||
Args:
|
||||
limit: Maximum requests allowed in window
|
||||
window_seconds: Time window in seconds
|
||||
"""
|
||||
self.limit = limit
|
||||
self.window_seconds = window_seconds
|
||||
self._requests: Dict[str, List[float]] = {}
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def is_allowed(self, key: str) -> bool:
|
||||
"""
|
||||
Check if request is allowed and record it if so.
|
||||
|
||||
Args:
|
||||
key: Identifier (e.g., IP address or domain)
|
||||
|
||||
Returns:
|
||||
True if request is allowed, False if rate limited
|
||||
"""
|
||||
with self._lock:
|
||||
now = time.time()
|
||||
cutoff = now - self.window_seconds
|
||||
|
||||
# Get or create request log for this key
|
||||
if key not in self._requests:
|
||||
self._requests[key] = []
|
||||
|
||||
# Remove expired entries
|
||||
self._requests[key] = [ts for ts in self._requests[key] if ts > cutoff]
|
||||
|
||||
# Check if at or over limit
|
||||
if len(self._requests[key]) >= self.limit:
|
||||
return False
|
||||
|
||||
# Allow request and record timestamp
|
||||
self._requests[key].append(now)
|
||||
return True
|
||||
|
||||
def get_retry_after(self, key: str) -> int:
|
||||
"""
|
||||
Get seconds until oldest request expires (for Retry-After header).
|
||||
|
||||
Args:
|
||||
key: Identifier (e.g., IP address or domain)
|
||||
|
||||
Returns:
|
||||
Seconds until rate limit resets, or window_seconds if no requests
|
||||
"""
|
||||
with self._lock:
|
||||
if key not in self._requests or not self._requests[key]:
|
||||
return self.window_seconds
|
||||
|
||||
now = time.time()
|
||||
oldest_request = self._requests[key][0]
|
||||
expires_at = oldest_request + self.window_seconds
|
||||
retry_after = int(expires_at - now)
|
||||
|
||||
# Return at least 1 second, max window_seconds
|
||||
return max(1, min(retry_after, self.window_seconds))
|
||||
|
||||
def cleanup(self, max_idle_seconds: int = 300):
|
||||
"""
|
||||
Remove keys with no recent requests to prevent memory growth.
|
||||
|
||||
Args:
|
||||
max_idle_seconds: Remove keys idle for this many seconds
|
||||
"""
|
||||
with self._lock:
|
||||
now = time.time()
|
||||
cutoff = now - max_idle_seconds
|
||||
keys_to_remove = []
|
||||
|
||||
for key, timestamps in self._requests.items():
|
||||
# Remove expired timestamps first
|
||||
self._requests[key] = [ts for ts in timestamps if ts > cutoff]
|
||||
# Mark empty keys for removal
|
||||
if not self._requests[key]:
|
||||
keys_to_remove.append(key)
|
||||
|
||||
for key in keys_to_remove:
|
||||
del self._requests[key]
|
||||
|
||||
def get_request_count(self, key: str) -> int:
|
||||
"""
|
||||
Get current request count for a key (for testing/logging).
|
||||
|
||||
Args:
|
||||
key: Identifier (e.g., IP address or domain)
|
||||
|
||||
Returns:
|
||||
Number of requests in current window
|
||||
"""
|
||||
with self._lock:
|
||||
if key not in self._requests:
|
||||
return 0
|
||||
|
||||
now = time.time()
|
||||
cutoff = now - self.window_seconds
|
||||
# Count non-expired requests
|
||||
return len([ts for ts in self._requests[key] if ts > cutoff])
|
||||
|
||||
|
||||
# Module-level rate limiters (initialized in init_rate_limiters)
|
||||
_ip_limiter: RateLimiter | None = None
|
||||
_domain_limiter: RateLimiter | None = None
|
||||
|
||||
|
||||
def init_rate_limiters(app) -> None:
|
||||
"""
|
||||
Initialize rate limiters with Flask app configuration.
|
||||
|
||||
Args:
|
||||
app: Flask application instance
|
||||
"""
|
||||
global _ip_limiter, _domain_limiter
|
||||
|
||||
if not app.config.get('WEBMENTION_RATE_LIMIT_ENABLED', True):
|
||||
app.logger.info("Rate limiting disabled")
|
||||
_ip_limiter = None
|
||||
_domain_limiter = None
|
||||
return
|
||||
|
||||
ip_limit = app.config.get('WEBMENTION_RATE_LIMIT_IP', 60)
|
||||
domain_limit = app.config.get('WEBMENTION_RATE_LIMIT_DOMAIN', 10)
|
||||
window = app.config.get('WEBMENTION_RATE_LIMIT_WINDOW', 60)
|
||||
|
||||
_ip_limiter = RateLimiter(ip_limit, window)
|
||||
_domain_limiter = RateLimiter(domain_limit, window)
|
||||
|
||||
app.logger.info(
|
||||
f"Webmention rate limiter initialized: {ip_limit}/min IP, "
|
||||
f"{domain_limit}/min domain"
|
||||
)
|
||||
|
||||
|
||||
def check_rate_limit(ip: str, source_domain: str | None) -> Tuple[bool, str | None]:
|
||||
"""
|
||||
Check rate limits for a request.
|
||||
|
||||
Args:
|
||||
ip: Client IP address
|
||||
source_domain: Source URL domain (optional)
|
||||
|
||||
Returns:
|
||||
(allowed, error_message) tuple
|
||||
- allowed: True if request allowed, False if rate limited
|
||||
- error_message: Error description if rate limited, None otherwise
|
||||
"""
|
||||
# If rate limiting disabled, allow all requests
|
||||
if _ip_limiter is None:
|
||||
return True, None
|
||||
|
||||
# Check IP limit first
|
||||
if not _ip_limiter.is_allowed(ip):
|
||||
retry_after = _ip_limiter.get_retry_after(ip)
|
||||
count = _ip_limiter.get_request_count(ip)
|
||||
return False, f"Too many requests from your IP. Please retry after {retry_after} seconds."
|
||||
|
||||
# Check domain limit if source domain provided
|
||||
if source_domain and _domain_limiter is not None:
|
||||
if not _domain_limiter.is_allowed(source_domain):
|
||||
retry_after = _domain_limiter.get_retry_after(source_domain)
|
||||
count = _domain_limiter.get_request_count(source_domain)
|
||||
return False, f"Too many requests from source domain. Please retry after {retry_after} seconds."
|
||||
|
||||
return True, None
|
||||
|
||||
|
||||
def cleanup_rate_limiters() -> None:
|
||||
"""
|
||||
Cleanup expired entries from rate limiters.
|
||||
|
||||
Called periodically to prevent memory growth.
|
||||
"""
|
||||
if _ip_limiter:
|
||||
_ip_limiter.cleanup()
|
||||
if _domain_limiter:
|
||||
_domain_limiter.cleanup()
|
||||
@@ -0,0 +1,329 @@
|
||||
"""
|
||||
Reply context fetching for in-reply-to URLs
|
||||
|
||||
Per database-schema.md Story 2.4:
|
||||
- Fetch context from in_reply_to URL when note is reply
|
||||
- Parse microformats for author, content
|
||||
- Store in reply_contexts table
|
||||
- Graceful degradation on fetch failure
|
||||
|
||||
References:
|
||||
- Database Schema: /docs/design/v1.6.0/database-schema.md
|
||||
- Background Worker: /docs/design/v1.6.0/background-worker.md
|
||||
"""
|
||||
|
||||
import logging
|
||||
import sqlite3
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Optional, Dict, Any
|
||||
|
||||
from starpunk.webmention_processor import (
|
||||
fetch_source,
|
||||
extract_author,
|
||||
extract_content,
|
||||
ValidationError
|
||||
)
|
||||
import mf2py
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def fetch_reply_context(
|
||||
db_path: Path,
|
||||
reply_url: str,
|
||||
site_url: str
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Fetch and parse reply context from a URL.
|
||||
|
||||
Per database-schema.md:
|
||||
- Fetch URL content with SSRF protection
|
||||
- Parse h-entry for title, content, published date
|
||||
- Parse h-card for author information
|
||||
- Store in reply_contexts table
|
||||
- Graceful degradation: errors don't prevent reply creation
|
||||
|
||||
Args:
|
||||
db_path: Path to SQLite database
|
||||
reply_url: URL being replied to
|
||||
site_url: Site URL for User-Agent header
|
||||
|
||||
Returns:
|
||||
Dict with fetch status and context data
|
||||
"""
|
||||
# Check if context already exists
|
||||
try:
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
cursor = conn.cursor()
|
||||
|
||||
cursor.execute("""
|
||||
SELECT * FROM reply_contexts WHERE url = ?
|
||||
""", (reply_url,))
|
||||
|
||||
existing = cursor.fetchone()
|
||||
conn.close()
|
||||
|
||||
if existing:
|
||||
logger.debug(f"Reply context already cached for {reply_url}")
|
||||
return dict(existing)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to check existing context: {e}")
|
||||
|
||||
# Fetch and parse context
|
||||
try:
|
||||
# Fetch URL content
|
||||
content, content_type = fetch_source(reply_url, site_url)
|
||||
|
||||
# Parse microformats (only for HTML)
|
||||
if 'html' not in content_type.lower():
|
||||
# Not HTML - store minimal context
|
||||
return _store_minimal_context(db_path, reply_url, "Not HTML content")
|
||||
|
||||
parsed_mf2 = mf2py.parse(doc=content, url=reply_url)
|
||||
|
||||
# Extract h-entry data
|
||||
entry_data = _extract_entry_data(parsed_mf2)
|
||||
|
||||
# Extract author data
|
||||
author_data = extract_author(parsed_mf2, reply_url)
|
||||
|
||||
# Store in database
|
||||
context = _store_context(
|
||||
db_path,
|
||||
reply_url,
|
||||
entry_data.get('title'),
|
||||
entry_data.get('content_snippet'),
|
||||
entry_data.get('published_at'),
|
||||
author_data.get('author_name'),
|
||||
author_data.get('author_url'),
|
||||
author_data.get('author_photo'),
|
||||
entry_data.get('site_name')
|
||||
)
|
||||
|
||||
logger.info(f"Fetched reply context for {reply_url}")
|
||||
return context
|
||||
|
||||
except ValidationError as e:
|
||||
# Fetch/validation error - store error status
|
||||
logger.warning(f"Failed to fetch reply context for {reply_url}: {e}")
|
||||
return _store_minimal_context(db_path, reply_url, str(e))
|
||||
|
||||
except Exception as e:
|
||||
# Unexpected error - store error status
|
||||
logger.error(f"Error fetching reply context for {reply_url}: {e}", exc_info=True)
|
||||
return _store_minimal_context(db_path, reply_url, f"Unexpected error: {e}")
|
||||
|
||||
|
||||
def _extract_entry_data(parsed_mf2: Dict) -> Dict[str, Any]:
|
||||
"""
|
||||
Extract entry data from parsed microformats.
|
||||
|
||||
Args:
|
||||
parsed_mf2: Parsed microformats data from mf2py
|
||||
|
||||
Returns:
|
||||
Dict with title, content_snippet, published_at, site_name
|
||||
"""
|
||||
title = None
|
||||
content_snippet = None
|
||||
published_at = None
|
||||
site_name = None
|
||||
|
||||
# Find h-entry
|
||||
for item in parsed_mf2.get('items', []):
|
||||
if 'h-entry' not in item.get('type', []):
|
||||
continue
|
||||
|
||||
props = item.get('properties', {})
|
||||
|
||||
# Extract title (p-name)
|
||||
name = props.get('name', [])
|
||||
if name:
|
||||
title = name[0]
|
||||
if isinstance(title, dict):
|
||||
title = title.get('value', '')
|
||||
# Truncate if too long
|
||||
if title and len(title) > 200:
|
||||
title = title[:197] + '...'
|
||||
|
||||
# Extract content (reuse processor function)
|
||||
content_snippet = extract_content(parsed_mf2)
|
||||
|
||||
# Extract published date
|
||||
published = props.get('published', [])
|
||||
if published:
|
||||
pub_str = published[0]
|
||||
if isinstance(pub_str, dict):
|
||||
pub_str = pub_str.get('value', '')
|
||||
# Try to parse ISO 8601 datetime
|
||||
try:
|
||||
from dateutil import parser as dateutil_parser
|
||||
published_at = dateutil_parser.parse(pub_str)
|
||||
except Exception:
|
||||
# Parsing failed, leave as None
|
||||
pass
|
||||
|
||||
break # Use first h-entry
|
||||
|
||||
# Try to extract site name from h-card
|
||||
for item in parsed_mf2.get('items', []):
|
||||
if 'h-card' in item.get('type', []):
|
||||
props = item.get('properties', {})
|
||||
name = props.get('name', [])
|
||||
if name:
|
||||
site_name = name[0]
|
||||
if isinstance(site_name, dict):
|
||||
site_name = site_name.get('value', '')
|
||||
break
|
||||
|
||||
return {
|
||||
'title': title,
|
||||
'content_snippet': content_snippet,
|
||||
'published_at': published_at,
|
||||
'site_name': site_name
|
||||
}
|
||||
|
||||
|
||||
def _store_context(
|
||||
db_path: Path,
|
||||
url: str,
|
||||
title: Optional[str],
|
||||
content_snippet: Optional[str],
|
||||
published_at: Optional[datetime],
|
||||
author_name: Optional[str],
|
||||
author_url: Optional[str],
|
||||
author_photo: Optional[str],
|
||||
site_name: Optional[str]
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Store reply context in database.
|
||||
|
||||
Args:
|
||||
db_path: Database path
|
||||
url: Reply-to URL
|
||||
title: Entry title
|
||||
content_snippet: Content snippet
|
||||
published_at: Published datetime
|
||||
author_name: Author name
|
||||
author_url: Author URL
|
||||
author_photo: Author photo URL
|
||||
site_name: Site name
|
||||
|
||||
Returns:
|
||||
Stored context dict
|
||||
"""
|
||||
try:
|
||||
conn = sqlite3.connect(db_path)
|
||||
cursor = conn.cursor()
|
||||
|
||||
cursor.execute("""
|
||||
INSERT OR REPLACE INTO reply_contexts
|
||||
(url, title, content_snippet, published_at,
|
||||
author_name, author_url, author_photo, site_name,
|
||||
fetch_status, fetched_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'fetched', CURRENT_TIMESTAMP)
|
||||
""", (
|
||||
url, title, content_snippet, published_at,
|
||||
author_name, author_url, author_photo, site_name
|
||||
))
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
return {
|
||||
'url': url,
|
||||
'title': title,
|
||||
'content_snippet': content_snippet,
|
||||
'published_at': published_at,
|
||||
'author_name': author_name,
|
||||
'author_url': author_url,
|
||||
'author_photo': author_photo,
|
||||
'site_name': site_name,
|
||||
'fetch_status': 'fetched'
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to store reply context: {e}")
|
||||
raise
|
||||
|
||||
|
||||
def _store_minimal_context(
|
||||
db_path: Path,
|
||||
url: str,
|
||||
error: str
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Store minimal context with error status.
|
||||
|
||||
Args:
|
||||
db_path: Database path
|
||||
url: Reply-to URL
|
||||
error: Error message
|
||||
|
||||
Returns:
|
||||
Minimal context dict with error
|
||||
"""
|
||||
try:
|
||||
conn = sqlite3.connect(db_path)
|
||||
cursor = conn.cursor()
|
||||
|
||||
cursor.execute("""
|
||||
INSERT OR REPLACE INTO reply_contexts
|
||||
(url, fetch_status, fetch_error, created_at)
|
||||
VALUES (?, 'error', ?, CURRENT_TIMESTAMP)
|
||||
""", (url, error))
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
return {
|
||||
'url': url,
|
||||
'fetch_status': 'error',
|
||||
'fetch_error': error
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to store minimal context: {e}")
|
||||
# Return in-memory dict even if storage fails
|
||||
return {
|
||||
'url': url,
|
||||
'fetch_status': 'error',
|
||||
'fetch_error': error
|
||||
}
|
||||
|
||||
|
||||
def get_reply_context(db_path: Path, reply_url: str) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Get cached reply context from database.
|
||||
|
||||
Args:
|
||||
db_path: Database path
|
||||
reply_url: Reply-to URL
|
||||
|
||||
Returns:
|
||||
Context dict if exists, None otherwise
|
||||
"""
|
||||
try:
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
cursor = conn.cursor()
|
||||
|
||||
cursor.execute("""
|
||||
SELECT * FROM reply_contexts WHERE url = ?
|
||||
""", (reply_url,))
|
||||
|
||||
row = cursor.fetchone()
|
||||
conn.close()
|
||||
|
||||
if row:
|
||||
return dict(row)
|
||||
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get reply context: {e}")
|
||||
return None
|
||||
@@ -7,7 +7,7 @@ admin, auth, and (conditionally) dev auth routes.
|
||||
|
||||
from flask import Flask
|
||||
|
||||
from starpunk.routes import admin, auth, micropub, public, search
|
||||
from starpunk.routes import admin, auth, micropub, public, search, webmention
|
||||
|
||||
|
||||
def register_routes(app: Flask) -> None:
|
||||
@@ -39,6 +39,10 @@ def register_routes(app: Flask) -> None:
|
||||
# Register search routes
|
||||
app.register_blueprint(search.bp)
|
||||
|
||||
# Register webmention routes (v1.6.0)
|
||||
if app.config.get("WEBMENTION_ENABLED", True):
|
||||
app.register_blueprint(webmention.bp)
|
||||
|
||||
# Conditionally register dev auth routes
|
||||
if app.config.get("DEV_MODE"):
|
||||
app.logger.warning(
|
||||
|
||||
+220
-4
@@ -6,10 +6,13 @@ editing, and deletion. All routes require authentication.
|
||||
"""
|
||||
|
||||
from flask import Blueprint, flash, g, jsonify, redirect, render_template, request, url_for
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
from datetime import datetime
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
from starpunk.auth import require_auth
|
||||
from starpunk.notes import (
|
||||
create_note,
|
||||
@@ -32,17 +35,51 @@ def dashboard():
|
||||
Displays all notes (published and drafts) with management controls.
|
||||
Requires authentication.
|
||||
|
||||
Per v1.6.0: Added pagination (25 items per page)
|
||||
|
||||
Returns:
|
||||
Rendered dashboard template with complete note list
|
||||
Rendered dashboard template with paginated note list
|
||||
|
||||
Decorator: @require_auth
|
||||
Template: templates/admin/dashboard.html
|
||||
Access: g.user_me (set by require_auth decorator)
|
||||
"""
|
||||
# Get all notes (published and drafts)
|
||||
notes = list_notes()
|
||||
from flask import current_app
|
||||
import math
|
||||
|
||||
return render_template("admin/dashboard.html", notes=notes, user_me=g.me)
|
||||
# Get pagination params (v1.6.0)
|
||||
page = int(request.args.get('page', 1))
|
||||
per_page = 25
|
||||
|
||||
# Get all notes (published and drafts)
|
||||
all_notes = list_notes()
|
||||
|
||||
# Calculate pagination
|
||||
total_notes = len(all_notes)
|
||||
total_pages = math.ceil(total_notes / per_page) if total_notes > 0 else 1
|
||||
|
||||
# Slice for current page
|
||||
start_idx = (page - 1) * per_page
|
||||
end_idx = start_idx + per_page
|
||||
notes = all_notes[start_idx:end_idx]
|
||||
|
||||
# Get pending webmention count (v1.6.0)
|
||||
webmention_pending_count = 0
|
||||
if current_app.config.get('WEBMENTION_ENABLED', False):
|
||||
try:
|
||||
from starpunk.webmention_queries import count_webmentions
|
||||
webmention_pending_count = count_webmentions(status_filter='pending')
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return render_template(
|
||||
"admin/dashboard.html",
|
||||
notes=notes,
|
||||
page=page,
|
||||
total_pages=total_pages,
|
||||
webmention_pending_count=webmention_pending_count,
|
||||
user_me=g.me
|
||||
)
|
||||
|
||||
|
||||
@bp.route("/new", methods=["GET"])
|
||||
@@ -600,3 +637,182 @@ def health_diagnostics():
|
||||
diagnostics["status"] = "healthy" if overall_healthy else "unhealthy"
|
||||
|
||||
return jsonify(diagnostics), 200 if overall_healthy else 500
|
||||
|
||||
|
||||
# Webmention Admin Routes (v1.6.0)
|
||||
|
||||
@bp.route("/webmentions")
|
||||
@require_auth
|
||||
def webmentions():
|
||||
"""
|
||||
Webmention list with filtering and pagination (v1.6.0 Phase 5)
|
||||
|
||||
Lists all webmentions with status filters and pagination.
|
||||
|
||||
Query params:
|
||||
status: Filter by moderation_status ('pending', 'approved', 'hidden')
|
||||
page: Page number (default: 1)
|
||||
|
||||
Returns:
|
||||
Rendered webmentions list template
|
||||
|
||||
Decorator: @require_auth
|
||||
Template: templates/admin/webmentions.html
|
||||
"""
|
||||
from starpunk.webmention_queries import list_webmentions
|
||||
|
||||
status_filter = request.args.get('status')
|
||||
page = int(request.args.get('page', 1))
|
||||
|
||||
webmentions, total_pages = list_webmentions(
|
||||
status_filter=status_filter,
|
||||
page=page,
|
||||
per_page=25
|
||||
)
|
||||
|
||||
return render_template(
|
||||
"admin/webmentions.html",
|
||||
webmentions=webmentions,
|
||||
status_filter=status_filter,
|
||||
page=page,
|
||||
total_pages=total_pages,
|
||||
user_me=g.me
|
||||
)
|
||||
|
||||
|
||||
@bp.route("/webmentions/<int:id>")
|
||||
@require_auth
|
||||
def webmention_detail(id: int):
|
||||
"""
|
||||
Webmention detail view (v1.6.0 Phase 5)
|
||||
|
||||
Shows full details of a single webmention with moderation actions.
|
||||
|
||||
Args:
|
||||
id: Webmention database ID
|
||||
|
||||
Returns:
|
||||
Rendered webmention detail template or 404 if not found
|
||||
|
||||
Decorator: @require_auth
|
||||
Template: templates/admin/webmention_detail.html
|
||||
"""
|
||||
from starpunk.webmention_queries import get_webmention
|
||||
|
||||
webmention = get_webmention(id)
|
||||
|
||||
if not webmention:
|
||||
flash("Webmention not found", "error")
|
||||
return redirect(url_for("admin.webmentions")), 404
|
||||
|
||||
return render_template(
|
||||
"admin/webmention_detail.html",
|
||||
webmention=webmention,
|
||||
user_me=g.me
|
||||
)
|
||||
|
||||
|
||||
@bp.route("/webmentions/<int:id>/action", methods=["POST"])
|
||||
@require_auth
|
||||
def webmention_action(id: int):
|
||||
"""
|
||||
Individual webmention moderation action (v1.6.0 Phase 5)
|
||||
|
||||
Approve, hide, or delete a single webmention.
|
||||
|
||||
Args:
|
||||
id: Webmention database ID
|
||||
|
||||
Form data:
|
||||
action: 'approve', 'hide', or 'delete'
|
||||
|
||||
Returns:
|
||||
Redirect to webmention detail or list
|
||||
|
||||
Decorator: @require_auth
|
||||
"""
|
||||
from starpunk.webmention_queries import update_webmention_status, delete_webmentions
|
||||
|
||||
action = request.form.get('action')
|
||||
|
||||
if action not in ('approve', 'hide', 'delete'):
|
||||
flash("Invalid action", "error")
|
||||
return redirect(url_for("admin.webmention_detail", id=id))
|
||||
|
||||
try:
|
||||
if action == 'delete':
|
||||
deleted = delete_webmentions([id])
|
||||
if deleted > 0:
|
||||
flash("Webmention deleted", "success")
|
||||
else:
|
||||
flash("Webmention not found", "error")
|
||||
return redirect(url_for("admin.webmentions"))
|
||||
else:
|
||||
# Approve or hide
|
||||
status = 'approved' if action == 'approve' else 'hidden'
|
||||
updated = update_webmention_status([id], status)
|
||||
if updated > 0:
|
||||
flash(f"Webmention {status}", "success")
|
||||
else:
|
||||
flash("Webmention not found", "error")
|
||||
return redirect(url_for("admin.webmention_detail", id=id))
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error performing webmention action: {e}")
|
||||
flash(f"Error: {e}", "error")
|
||||
return redirect(url_for("admin.webmention_detail", id=id))
|
||||
|
||||
|
||||
@bp.route("/webmentions/bulk-action", methods=["POST"])
|
||||
@require_auth
|
||||
def webmention_bulk_action():
|
||||
"""
|
||||
Bulk webmention moderation actions (v1.6.0 Phase 5)
|
||||
|
||||
Approve, hide, or delete multiple webmentions at once.
|
||||
|
||||
Form data:
|
||||
action: 'approve', 'hide', or 'delete'
|
||||
ids[]: List of webmention IDs
|
||||
|
||||
Returns:
|
||||
Redirect to webmentions list
|
||||
|
||||
Decorator: @require_auth
|
||||
"""
|
||||
from starpunk.webmention_queries import update_webmention_status, delete_webmentions
|
||||
|
||||
action = request.form.get('action')
|
||||
ids = request.form.getlist('ids')
|
||||
|
||||
if not ids:
|
||||
flash("No webmentions selected", "warning")
|
||||
return redirect(url_for("admin.webmentions"))
|
||||
|
||||
# Convert IDs to integers
|
||||
try:
|
||||
webmention_ids = [int(id) for id in ids]
|
||||
except ValueError:
|
||||
flash("Invalid webmention IDs", "error")
|
||||
return redirect(url_for("admin.webmentions"))
|
||||
|
||||
if action not in ('approve', 'hide', 'delete'):
|
||||
flash("Invalid action", "error")
|
||||
return redirect(url_for("admin.webmentions"))
|
||||
|
||||
try:
|
||||
if action == 'delete':
|
||||
deleted = delete_webmentions(webmention_ids)
|
||||
flash(f"Deleted {deleted} webmention(s)", "success")
|
||||
else:
|
||||
# Approve or hide
|
||||
status = 'approved' if action == 'approve' else 'hidden'
|
||||
updated = update_webmention_status(webmention_ids, status)
|
||||
flash(f"Updated {updated} webmention(s) to {status}", "success")
|
||||
|
||||
return redirect(url_for("admin.webmentions"))
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error performing bulk action: {e}")
|
||||
flash(f"Error: {e}", "error")
|
||||
return redirect(url_for("admin.webmentions"))
|
||||
@@ -281,6 +281,8 @@ def note(slug: str):
|
||||
|
||||
Template: templates/note.html
|
||||
Microformats: h-entry
|
||||
|
||||
Per v1.6.0: Added webmentions and reply context display
|
||||
"""
|
||||
from starpunk.media import get_note_media
|
||||
from starpunk.tags import get_note_tags
|
||||
@@ -303,7 +305,39 @@ def note(slug: str):
|
||||
tags = get_note_tags(note_obj.id)
|
||||
object.__setattr__(note_obj, '_cached_tags', tags)
|
||||
|
||||
return render_template("note.html", note=note_obj)
|
||||
# Get webmentions for note (v1.6.0 Phase 5)
|
||||
webmentions = None
|
||||
reply_context = None
|
||||
|
||||
if current_app.config.get('WEBMENTION_ENABLED', False):
|
||||
try:
|
||||
from starpunk.webmention_queries import get_webmentions_for_note, get_reply_context
|
||||
|
||||
# Get approved webmentions grouped by type
|
||||
grouped = get_webmentions_for_note(note_obj.id)
|
||||
|
||||
# Check if there are any webmentions
|
||||
has_webmentions = any(grouped.values())
|
||||
if has_webmentions:
|
||||
webmentions = grouped
|
||||
|
||||
# Get reply context if this is a reply post
|
||||
if note_obj.in_reply_to:
|
||||
reply_context = get_reply_context(note_obj.in_reply_to)
|
||||
|
||||
except Exception as e:
|
||||
# Log error but don't fail page load
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.error(f"Error loading webmentions for note {slug}: {e}")
|
||||
|
||||
return render_template(
|
||||
"note.html",
|
||||
note=note_obj,
|
||||
webmentions=webmentions,
|
||||
reply_context=reply_context,
|
||||
**grouped if webmentions else {}
|
||||
)
|
||||
|
||||
|
||||
@bp.route("/tag/<tag>")
|
||||
|
||||
@@ -0,0 +1,272 @@
|
||||
"""
|
||||
Webmention routes for StarPunk
|
||||
|
||||
Per W3C Webmention Recommendation and ADR-063:
|
||||
- POST /webmention - Receive webmention notifications
|
||||
- GET /webmention - Optional endpoint information
|
||||
|
||||
References:
|
||||
- W3C Webmention Specification: https://www.w3.org/TR/webmention/
|
||||
- ADR-063: Webmention Implementation
|
||||
"""
|
||||
|
||||
from flask import Blueprint, current_app, request, jsonify
|
||||
from urllib.parse import urlparse
|
||||
from starpunk.database import get_db
|
||||
from starpunk.notes import get_note, NoteNotFoundError
|
||||
from starpunk.rate_limiter import check_rate_limit
|
||||
|
||||
bp = Blueprint("webmention", __name__)
|
||||
|
||||
|
||||
def get_client_ip() -> str:
|
||||
"""
|
||||
Get client IP address, accounting for reverse proxy.
|
||||
|
||||
If TRUSTED_PROXY is enabled, uses X-Forwarded-For header.
|
||||
Otherwise uses request.remote_addr directly.
|
||||
"""
|
||||
if current_app.config.get('TRUSTED_PROXY', False):
|
||||
forwarded = request.headers.get('X-Forwarded-For')
|
||||
if forwarded:
|
||||
# First IP in chain is the client
|
||||
return forwarded.split(',')[0].strip()
|
||||
return request.remote_addr or '0.0.0.0'
|
||||
|
||||
|
||||
@bp.before_request
|
||||
def rate_limit_webmention():
|
||||
"""
|
||||
Check rate limits before processing webmention requests.
|
||||
|
||||
Per ADR-064, checks:
|
||||
1. IP rate limit (60/min by default)
|
||||
2. Source domain rate limit (10/min by default)
|
||||
|
||||
Returns 429 with Retry-After header if rate limited.
|
||||
"""
|
||||
# Only rate limit POST requests (the actual webmention submission)
|
||||
if request.method != 'POST':
|
||||
return None
|
||||
|
||||
# Get client IP
|
||||
ip = get_client_ip()
|
||||
|
||||
# Get source domain from request (if available)
|
||||
source_domain = None
|
||||
content_type = request.headers.get("Content-Type", "")
|
||||
|
||||
if "application/json" in content_type:
|
||||
data = request.get_json(silent=True) or {}
|
||||
source = data.get('source')
|
||||
else:
|
||||
source = request.form.get('source')
|
||||
|
||||
if source:
|
||||
try:
|
||||
source_domain = urlparse(source).netloc
|
||||
except Exception:
|
||||
# Invalid URL, domain check will be skipped
|
||||
# This is acceptable - IP limit still applies
|
||||
current_app.logger.debug(f"Could not parse source URL for domain extraction: {source}")
|
||||
|
||||
# Check rate limits
|
||||
allowed, error = check_rate_limit(ip, source_domain)
|
||||
if not allowed:
|
||||
current_app.logger.warning(
|
||||
f"Rate limit exceeded - IP: {ip}, Domain: {source_domain or 'N/A'}"
|
||||
)
|
||||
return jsonify({
|
||||
"error": "rate_limit_exceeded",
|
||||
"error_description": error
|
||||
}), 429, {'Retry-After': str(current_app.config.get('WEBMENTION_RATE_LIMIT_WINDOW', 60))}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def validate_url(url: str) -> tuple[bool, str | None]:
|
||||
"""
|
||||
Validate a URL is well-formed and uses http/https scheme.
|
||||
|
||||
Returns:
|
||||
(is_valid, error_message)
|
||||
"""
|
||||
if not url:
|
||||
return False, "URL is required"
|
||||
|
||||
try:
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme not in ('http', 'https'):
|
||||
return False, "Only http and https URLs are supported"
|
||||
if not parsed.netloc:
|
||||
return False, "URL must include a hostname"
|
||||
if len(url) > 2048:
|
||||
return False, "URL exceeds maximum length of 2048 characters"
|
||||
return True, None
|
||||
except Exception:
|
||||
return False, "URL is malformed"
|
||||
|
||||
|
||||
def is_target_on_site(target_url: str) -> bool:
|
||||
"""
|
||||
Check if target URL is on this site.
|
||||
|
||||
Compares hostname of target_url with SITE_URL configuration.
|
||||
"""
|
||||
site_url = current_app.config.get('SITE_URL', '')
|
||||
site_parsed = urlparse(site_url)
|
||||
target_parsed = urlparse(target_url)
|
||||
|
||||
return site_parsed.netloc == target_parsed.netloc
|
||||
|
||||
|
||||
def resolve_target_note(target_url: str) -> int | None:
|
||||
"""
|
||||
Resolve target URL to a note ID if it points to a note.
|
||||
|
||||
Returns:
|
||||
Note ID if target is a valid note, None otherwise
|
||||
"""
|
||||
target_parsed = urlparse(target_url)
|
||||
|
||||
# Strip query params and fragments for matching
|
||||
target_path = target_parsed.path.rstrip('/')
|
||||
|
||||
# Check if it's a note URL pattern: /note/{slug}
|
||||
if target_path.startswith('/note/'):
|
||||
slug = target_path[6:] # Remove '/note/' prefix
|
||||
if slug:
|
||||
try:
|
||||
note = get_note(slug=slug)
|
||||
# get_note() returns None if not found, or a Note object
|
||||
if note and note.published:
|
||||
return note.id
|
||||
except NoteNotFoundError:
|
||||
pass
|
||||
|
||||
# Homepage or other valid page (not a note)
|
||||
return None
|
||||
|
||||
|
||||
@bp.route("/webmention", methods=["GET"])
|
||||
def webmention_info():
|
||||
"""
|
||||
Optional endpoint information.
|
||||
|
||||
Returns basic information about the webmention endpoint.
|
||||
"""
|
||||
return jsonify({
|
||||
"name": "Webmention endpoint",
|
||||
"spec": "https://www.w3.org/TR/webmention/"
|
||||
})
|
||||
|
||||
|
||||
@bp.route("/webmention", methods=["POST"])
|
||||
def receive_webmention():
|
||||
"""
|
||||
Receive webmention notification per W3C spec.
|
||||
|
||||
Accepts application/x-www-form-urlencoded (required) and
|
||||
application/json (optional extension).
|
||||
|
||||
Returns:
|
||||
202 Accepted with JSON body on success
|
||||
400 Bad Request with error details on failure
|
||||
"""
|
||||
# Parse request based on content type
|
||||
content_type = request.headers.get("Content-Type", "")
|
||||
|
||||
if "application/json" in content_type:
|
||||
data = request.get_json() or {}
|
||||
source = data.get("source")
|
||||
target = data.get("target")
|
||||
else:
|
||||
# Default to form-encoded (required by W3C spec)
|
||||
source = request.form.get("source")
|
||||
target = request.form.get("target")
|
||||
|
||||
# Validate required parameters
|
||||
if not source:
|
||||
return jsonify({
|
||||
"error": "invalid_request",
|
||||
"error_description": "Missing required parameter: source"
|
||||
}), 400
|
||||
|
||||
if not target:
|
||||
return jsonify({
|
||||
"error": "invalid_request",
|
||||
"error_description": "Missing required parameter: target"
|
||||
}), 400
|
||||
|
||||
# Validate source URL
|
||||
source_valid, source_error = validate_url(source)
|
||||
if not source_valid:
|
||||
return jsonify({
|
||||
"error": "invalid_request",
|
||||
"error_description": f"Source must be a valid URL"
|
||||
}), 400
|
||||
|
||||
# Validate target URL
|
||||
target_valid, target_error = validate_url(target)
|
||||
if not target_valid:
|
||||
return jsonify({
|
||||
"error": "invalid_request",
|
||||
"error_description": f"Target must be a valid URL"
|
||||
}), 400
|
||||
|
||||
# Validate source and target are different
|
||||
if source == target:
|
||||
return jsonify({
|
||||
"error": "invalid_request",
|
||||
"error_description": "Source and target must be different"
|
||||
}), 400
|
||||
|
||||
# Validate target is on this site
|
||||
if not is_target_on_site(target):
|
||||
return jsonify({
|
||||
"error": "invalid_target",
|
||||
"error_description": "Target URL is not on this site"
|
||||
}), 400
|
||||
|
||||
# Resolve target to note (may be None for homepage/other pages)
|
||||
target_note_id = resolve_target_note(target)
|
||||
|
||||
# If target looks like a note URL but doesn't resolve, it's not found
|
||||
target_parsed = urlparse(target)
|
||||
target_path = target_parsed.path.rstrip('/')
|
||||
if target_path.startswith('/note/') and target_note_id is None:
|
||||
return jsonify({
|
||||
"error": "invalid_target",
|
||||
"error_description": "Target URL not found"
|
||||
}), 400
|
||||
|
||||
# Insert webmention into database (queued for validation)
|
||||
# Use INSERT ... ON CONFLICT to handle duplicate webmentions
|
||||
db = get_db()
|
||||
try:
|
||||
db.execute("""
|
||||
INSERT INTO webmentions (source_url, target_url, target_note_id)
|
||||
VALUES (?, ?, ?)
|
||||
ON CONFLICT(source_url, target_url) DO UPDATE SET
|
||||
validation_status = 'pending',
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""", (source, target, target_note_id))
|
||||
db.commit()
|
||||
|
||||
current_app.logger.info(
|
||||
f"Webmention received: {source} -> {target}"
|
||||
f"{' (note_id=' + str(target_note_id) + ')' if target_note_id else ''}"
|
||||
)
|
||||
|
||||
return jsonify({
|
||||
"status": "accepted",
|
||||
"message": "Webmention queued for verification"
|
||||
}), 202
|
||||
|
||||
except Exception as e:
|
||||
db.rollback()
|
||||
current_app.logger.error(f"Failed to queue webmention: {e}")
|
||||
return jsonify({
|
||||
"error": "server_error",
|
||||
"error_description": "Failed to queue webmention"
|
||||
}), 500
|
||||
@@ -0,0 +1,373 @@
|
||||
"""
|
||||
Webmention validation and processing utilities
|
||||
|
||||
Per background-worker.md and api-contract.md:
|
||||
- Fetch and validate source URLs
|
||||
- Parse microformats with mf2py
|
||||
- Extract author information from h-cards
|
||||
- Detect mention types (reply, like, repost, etc.)
|
||||
- Sanitize content with nh3
|
||||
|
||||
References:
|
||||
- Background Worker Design: /docs/design/v1.6.0/background-worker.md
|
||||
- API Contract: /docs/design/v1.6.0/api-contract.md
|
||||
- W3C Webmention: https://www.w3.org/TR/webmention/
|
||||
"""
|
||||
|
||||
import logging
|
||||
import mf2py
|
||||
import nh3
|
||||
import httpx
|
||||
from datetime import datetime
|
||||
from typing import Dict, Any, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from starpunk.webmention_ssrf import validate_url_for_fetch, SSRFBlockedError
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ValidationError(Exception):
|
||||
"""Raised when webmention validation fails"""
|
||||
pass
|
||||
|
||||
|
||||
def fetch_source(url: str, site_url: str, timeout: int = 10, max_size: int = 1048576) -> tuple[str, str]:
|
||||
"""
|
||||
Fetch source URL content with SSRF protection.
|
||||
|
||||
Args:
|
||||
url: Source URL to fetch
|
||||
site_url: Site URL for User-Agent header
|
||||
timeout: Request timeout in seconds (default: 10)
|
||||
max_size: Maximum content size in bytes (default: 1MB)
|
||||
|
||||
Returns:
|
||||
Tuple of (content, content_type)
|
||||
|
||||
Raises:
|
||||
SSRFBlockedError: If URL points to private IP
|
||||
ValidationError: If fetch fails or content too large
|
||||
"""
|
||||
# SSRF protection
|
||||
try:
|
||||
validate_url_for_fetch(url)
|
||||
except (SSRFBlockedError, ValueError) as e:
|
||||
logger.warning(f"SSRF protection blocked URL {url}: {e}")
|
||||
raise ValidationError(f"Blocked request: {e}")
|
||||
|
||||
# Fetch URL
|
||||
try:
|
||||
# Use streaming to check size before downloading entire content
|
||||
with httpx.stream(
|
||||
'GET',
|
||||
url,
|
||||
timeout=timeout,
|
||||
headers={'User-Agent': f'StarPunk/1.6.0 (+{site_url})'},
|
||||
follow_redirects=True
|
||||
) as response:
|
||||
response.raise_for_status()
|
||||
|
||||
# Check content length if provided
|
||||
content_length = response.headers.get('Content-Length')
|
||||
if content_length and int(content_length) > max_size:
|
||||
raise ValidationError(f"Source exceeds size limit ({max_size} bytes)")
|
||||
|
||||
# Read content with size limit
|
||||
content = b''
|
||||
for chunk in response.iter_bytes(chunk_size=8192):
|
||||
content += chunk
|
||||
if len(content) > max_size:
|
||||
raise ValidationError(f"Source exceeds size limit ({max_size} bytes)")
|
||||
|
||||
content_type = response.headers.get('Content-Type', '')
|
||||
|
||||
return content.decode('utf-8', errors='replace'), content_type
|
||||
|
||||
except httpx.TimeoutException:
|
||||
raise ValidationError("Source fetch timed out")
|
||||
except httpx.HTTPStatusError as e:
|
||||
if 400 <= e.response.status_code < 500:
|
||||
raise ValidationError(f"Source returned {e.response.status_code}")
|
||||
else:
|
||||
raise ValidationError(f"Source server error: {e.response.status_code}")
|
||||
except httpx.HTTPError as e:
|
||||
raise ValidationError(f"Failed to fetch source: {e}")
|
||||
except UnicodeDecodeError:
|
||||
raise ValidationError("Source content encoding is not supported")
|
||||
|
||||
|
||||
def verify_link_presence(content: str, target_url: str, content_type: str) -> bool:
|
||||
"""
|
||||
Verify that source content contains a link to target URL.
|
||||
|
||||
Args:
|
||||
content: Source content (HTML or text)
|
||||
target_url: Target URL to find
|
||||
content_type: Content type of source
|
||||
|
||||
Returns:
|
||||
True if target link found, False otherwise
|
||||
"""
|
||||
# Normalize URLs for comparison (remove fragments, trailing slashes)
|
||||
target_normalized = urlparse(target_url)._replace(fragment='').geturl().rstrip('/')
|
||||
|
||||
# For HTML, parse and check all links
|
||||
if 'html' in content_type.lower():
|
||||
try:
|
||||
from lxml import html as lxml_html
|
||||
doc = lxml_html.fromstring(content)
|
||||
links = doc.xpath('//a/@href')
|
||||
|
||||
for link in links:
|
||||
# Make absolute if relative
|
||||
link_normalized = urlparse(link)._replace(fragment='').geturl().rstrip('/')
|
||||
if link_normalized == target_normalized:
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.debug(f"HTML parsing failed, falling back to string search: {e}")
|
||||
|
||||
# Fallback: simple string search
|
||||
return target_normalized in content
|
||||
|
||||
|
||||
def detect_mention_type(parsed_mf2: Dict, target_url: str) -> str:
|
||||
"""
|
||||
Detect mention type from microformats.
|
||||
|
||||
Priority: reply > like > repost > bookmark > mention
|
||||
|
||||
Args:
|
||||
parsed_mf2: Parsed microformats data from mf2py
|
||||
target_url: Target URL being mentioned
|
||||
|
||||
Returns:
|
||||
Mention type: 'reply', 'like', 'repost', 'bookmark', or 'mention'
|
||||
"""
|
||||
target_normalized = urlparse(target_url)._replace(fragment='').geturl().rstrip('/')
|
||||
|
||||
# Find h-entry that references target
|
||||
for item in parsed_mf2.get('items', []):
|
||||
if 'h-entry' not in item.get('type', []):
|
||||
continue
|
||||
|
||||
props = item.get('properties', {})
|
||||
|
||||
# Check in-reply-to (highest priority)
|
||||
for reply_to in props.get('in-reply-to', []):
|
||||
url = reply_to if isinstance(reply_to, str) else reply_to.get('value', '')
|
||||
if urlparse(url)._replace(fragment='').geturl().rstrip('/') == target_normalized:
|
||||
return 'reply'
|
||||
|
||||
# Check like-of
|
||||
for like in props.get('like-of', []):
|
||||
url = like if isinstance(like, str) else like.get('value', '')
|
||||
if urlparse(url)._replace(fragment='').geturl().rstrip('/') == target_normalized:
|
||||
return 'like'
|
||||
|
||||
# Check repost-of
|
||||
for repost in props.get('repost-of', []):
|
||||
url = repost if isinstance(repost, str) else repost.get('value', '')
|
||||
if urlparse(url)._replace(fragment='').geturl().rstrip('/') == target_normalized:
|
||||
return 'repost'
|
||||
|
||||
# Check bookmark-of
|
||||
for bookmark in props.get('bookmark-of', []):
|
||||
url = bookmark if isinstance(bookmark, str) else bookmark.get('value', '')
|
||||
if urlparse(url)._replace(fragment='').geturl().rstrip('/') == target_normalized:
|
||||
return 'bookmark'
|
||||
|
||||
# Default fallback
|
||||
return 'mention'
|
||||
|
||||
|
||||
def extract_author(parsed_mf2: Dict, source_url: str) -> Dict[str, Optional[str]]:
|
||||
"""
|
||||
Extract author information from h-card.
|
||||
|
||||
Args:
|
||||
parsed_mf2: Parsed microformats data from mf2py
|
||||
source_url: Source URL (fallback for author URL)
|
||||
|
||||
Returns:
|
||||
Dict with author_name, author_url, author_photo
|
||||
"""
|
||||
author_name = None
|
||||
author_url = None
|
||||
author_photo = None
|
||||
|
||||
# Find h-entry first
|
||||
entry = None
|
||||
for item in parsed_mf2.get('items', []):
|
||||
if 'h-entry' in item.get('type', []):
|
||||
entry = item
|
||||
break
|
||||
|
||||
# Check for author property in h-entry
|
||||
if entry:
|
||||
author_prop = entry.get('properties', {}).get('author', [])
|
||||
if author_prop:
|
||||
author_data = author_prop[0]
|
||||
if isinstance(author_data, dict):
|
||||
# h-card object
|
||||
props = author_data.get('properties', {})
|
||||
author_name = props.get('name', [None])[0]
|
||||
author_url = props.get('url', [None])[0]
|
||||
author_photo = props.get('photo', [None])[0]
|
||||
|
||||
# If no author in h-entry, find representative h-card
|
||||
if not author_name:
|
||||
for item in parsed_mf2.get('items', []):
|
||||
if 'h-card' in item.get('type', []):
|
||||
props = item.get('properties', {})
|
||||
author_name = props.get('name', [None])[0]
|
||||
author_url = props.get('url', [None])[0]
|
||||
author_photo = props.get('photo', [None])[0]
|
||||
break
|
||||
|
||||
# Fallback to hostname if no author found
|
||||
if not author_name:
|
||||
author_name = urlparse(source_url).netloc
|
||||
|
||||
if not author_url:
|
||||
author_url = source_url
|
||||
|
||||
return {
|
||||
'author_name': author_name,
|
||||
'author_url': author_url,
|
||||
'author_photo': author_photo
|
||||
}
|
||||
|
||||
|
||||
def extract_content(parsed_mf2: Dict) -> Optional[str]:
|
||||
"""
|
||||
Extract content snippet from h-entry.
|
||||
|
||||
Args:
|
||||
parsed_mf2: Parsed microformats data from mf2py
|
||||
|
||||
Returns:
|
||||
Sanitized content snippet (max 500 chars) or None
|
||||
"""
|
||||
# Find h-entry
|
||||
for item in parsed_mf2.get('items', []):
|
||||
if 'h-entry' not in item.get('type', []):
|
||||
continue
|
||||
|
||||
props = item.get('properties', {})
|
||||
|
||||
# Try e-content first (HTML)
|
||||
content_prop = props.get('content', [])
|
||||
if content_prop:
|
||||
content_data = content_prop[0]
|
||||
if isinstance(content_data, dict):
|
||||
content = content_data.get('html', '') or content_data.get('value', '')
|
||||
else:
|
||||
content = content_data
|
||||
|
||||
if content:
|
||||
# Sanitize HTML
|
||||
sanitized = sanitize_content(content)
|
||||
# Truncate to 500 chars
|
||||
if len(sanitized) > 500:
|
||||
sanitized = sanitized[:497] + '...'
|
||||
return sanitized
|
||||
|
||||
# Try p-summary as fallback
|
||||
summary = props.get('summary', [])
|
||||
if summary:
|
||||
content = summary[0]
|
||||
if isinstance(content, dict):
|
||||
content = content.get('value', '')
|
||||
if content:
|
||||
sanitized = sanitize_content(content)
|
||||
if len(sanitized) > 500:
|
||||
sanitized = sanitized[:497] + '...'
|
||||
return sanitized
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def sanitize_content(html: str) -> str:
|
||||
"""
|
||||
Sanitize HTML content using nh3.
|
||||
|
||||
Per api-contract.md:
|
||||
- Allowed tags: p, br, strong, em, b, i, a, code, pre, blockquote
|
||||
- Allowed attributes: a[href, rel]
|
||||
|
||||
Args:
|
||||
html: HTML content to sanitize
|
||||
|
||||
Returns:
|
||||
Sanitized HTML
|
||||
"""
|
||||
allowed_tags = {'p', 'br', 'strong', 'em', 'b', 'i', 'a', 'code', 'pre', 'blockquote'}
|
||||
allowed_attributes = {'a': {'href'}}
|
||||
|
||||
return nh3.clean(
|
||||
html,
|
||||
tags=allowed_tags,
|
||||
attributes=allowed_attributes,
|
||||
link_rel='nofollow noopener'
|
||||
)
|
||||
|
||||
|
||||
def process_webmention(source_url: str, target_url: str, site_url: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Process and validate a webmention.
|
||||
|
||||
Args:
|
||||
source_url: Source URL of the webmention
|
||||
target_url: Target URL being mentioned
|
||||
site_url: Site URL for User-Agent header
|
||||
|
||||
Returns:
|
||||
Dict with validation results:
|
||||
- valid: bool
|
||||
- error: str (if valid=False)
|
||||
- mention_type: str
|
||||
- author_name: str
|
||||
- author_url: str
|
||||
- author_photo: str or None
|
||||
- content_snippet: str or None
|
||||
|
||||
Raises:
|
||||
ValidationError: If validation fails with specific error
|
||||
"""
|
||||
# Fetch source content
|
||||
content, content_type = fetch_source(source_url, site_url)
|
||||
|
||||
# Verify target link presence
|
||||
if not verify_link_presence(content, target_url, content_type):
|
||||
raise ValidationError("Source does not link to target")
|
||||
|
||||
# Parse microformats (only for HTML)
|
||||
parsed_mf2 = None
|
||||
if 'html' in content_type.lower():
|
||||
try:
|
||||
parsed_mf2 = mf2py.parse(doc=content, url=source_url)
|
||||
except Exception as e:
|
||||
logger.debug(f"Microformat parsing failed for {source_url}: {e}")
|
||||
|
||||
# Extract data
|
||||
mention_type = 'mention'
|
||||
author_info = {
|
||||
'author_name': urlparse(source_url).netloc,
|
||||
'author_url': source_url,
|
||||
'author_photo': None
|
||||
}
|
||||
content_snippet = None
|
||||
|
||||
if parsed_mf2:
|
||||
mention_type = detect_mention_type(parsed_mf2, target_url)
|
||||
author_info = extract_author(parsed_mf2, source_url)
|
||||
content_snippet = extract_content(parsed_mf2)
|
||||
|
||||
return {
|
||||
'valid': True,
|
||||
'mention_type': mention_type,
|
||||
**author_info,
|
||||
'content_snippet': content_snippet
|
||||
}
|
||||
@@ -0,0 +1,295 @@
|
||||
"""
|
||||
Webmention database query functions
|
||||
|
||||
Provides database query functions for webmention management.
|
||||
Per display-moderation.md design specification.
|
||||
|
||||
Functions:
|
||||
- get_webmentions_for_note: Get approved webmentions for public display
|
||||
- list_webmentions: List webmentions for admin with filters and pagination
|
||||
- get_webmention: Get single webmention by ID
|
||||
- count_webmentions: Count webmentions (optionally filtered)
|
||||
- update_webmention_status: Update moderation status
|
||||
- delete_webmentions: Delete webmentions by IDs
|
||||
- get_reply_context: Get cached reply context for a URL
|
||||
"""
|
||||
|
||||
import logging
|
||||
import math
|
||||
from datetime import datetime
|
||||
from typing import List, Dict, Any, Optional
|
||||
from starpunk.database import get_db
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_webmentions_for_note(note_id: int) -> Dict[str, List[Dict[str, Any]]]:
|
||||
"""
|
||||
Get approved webmentions for a note, grouped by type for public display.
|
||||
|
||||
Per display-moderation.md:
|
||||
- Only returns validation_status='valid' AND moderation_status='approved'
|
||||
- Groups by mention_type (reply, like, repost, bookmark, mention)
|
||||
- Sorts within groups by received_at ASC (oldest first)
|
||||
|
||||
Args:
|
||||
note_id: Note database ID
|
||||
|
||||
Returns:
|
||||
Dict with keys: 'replies', 'likes', 'reposts', 'bookmarks', 'mentions'
|
||||
Each value is a list of webmention dicts
|
||||
"""
|
||||
db = get_db()
|
||||
|
||||
# Query for approved webmentions only
|
||||
query = """
|
||||
SELECT * FROM webmentions
|
||||
WHERE target_note_id = ?
|
||||
AND validation_status = 'valid'
|
||||
AND moderation_status = 'approved'
|
||||
ORDER BY
|
||||
CASE mention_type
|
||||
WHEN 'reply' THEN 1
|
||||
WHEN 'like' THEN 2
|
||||
WHEN 'repost' THEN 3
|
||||
WHEN 'bookmark' THEN 4
|
||||
WHEN 'mention' THEN 5
|
||||
END,
|
||||
received_at ASC
|
||||
"""
|
||||
|
||||
rows = db.execute(query, (note_id,)).fetchall()
|
||||
|
||||
# Group by type
|
||||
grouped = {
|
||||
'replies': [],
|
||||
'likes': [],
|
||||
'reposts': [],
|
||||
'bookmarks': [],
|
||||
'mentions': []
|
||||
}
|
||||
|
||||
for row in rows:
|
||||
wm = dict(row)
|
||||
# Parse received_at timestamp
|
||||
if isinstance(wm.get('received_at'), str):
|
||||
wm['received_at'] = datetime.fromisoformat(wm['received_at'].replace('Z', '+00:00'))
|
||||
|
||||
mention_type = wm.get('mention_type', 'mention')
|
||||
if mention_type == 'reply':
|
||||
grouped['replies'].append(wm)
|
||||
elif mention_type == 'like':
|
||||
grouped['likes'].append(wm)
|
||||
elif mention_type == 'repost':
|
||||
grouped['reposts'].append(wm)
|
||||
elif mention_type == 'bookmark':
|
||||
grouped['bookmarks'].append(wm)
|
||||
else:
|
||||
grouped['mentions'].append(wm)
|
||||
|
||||
return grouped
|
||||
|
||||
|
||||
def list_webmentions(
|
||||
status_filter: Optional[str] = None,
|
||||
page: int = 1,
|
||||
per_page: int = 25
|
||||
) -> tuple[List[Dict[str, Any]], int]:
|
||||
"""
|
||||
List webmentions for admin with pagination and filtering.
|
||||
|
||||
Args:
|
||||
status_filter: Filter by moderation_status ('pending', 'approved', 'hidden')
|
||||
page: Page number (1-indexed)
|
||||
per_page: Items per page (default: 25)
|
||||
|
||||
Returns:
|
||||
Tuple of (webmentions list, total_pages)
|
||||
"""
|
||||
db = get_db()
|
||||
|
||||
# Build query
|
||||
base_query = """
|
||||
SELECT w.*, n.slug as target_slug
|
||||
FROM webmentions w
|
||||
LEFT JOIN notes n ON w.target_note_id = n.id
|
||||
"""
|
||||
count_query = "SELECT COUNT(*) FROM webmentions"
|
||||
|
||||
params = []
|
||||
where_clause = ""
|
||||
|
||||
if status_filter:
|
||||
where_clause = " WHERE w.moderation_status = ?"
|
||||
count_query += " WHERE moderation_status = ?"
|
||||
params.append(status_filter)
|
||||
|
||||
# Count total
|
||||
total_count = db.execute(count_query, params).fetchone()[0]
|
||||
total_pages = math.ceil(total_count / per_page) if total_count > 0 else 1
|
||||
|
||||
# Get paginated results
|
||||
offset = (page - 1) * per_page
|
||||
query = base_query + where_clause + " ORDER BY w.received_at DESC LIMIT ? OFFSET ?"
|
||||
params.extend([per_page, offset])
|
||||
|
||||
rows = db.execute(query, params).fetchall()
|
||||
|
||||
# Convert to dicts
|
||||
webmentions = []
|
||||
for row in rows:
|
||||
wm = dict(row)
|
||||
# Parse timestamps
|
||||
if isinstance(wm.get('received_at'), str):
|
||||
wm['received_at'] = datetime.fromisoformat(wm['received_at'].replace('Z', '+00:00'))
|
||||
if wm.get('validated_at') and isinstance(wm['validated_at'], str):
|
||||
wm['validated_at'] = datetime.fromisoformat(wm['validated_at'].replace('Z', '+00:00'))
|
||||
webmentions.append(wm)
|
||||
|
||||
return webmentions, total_pages
|
||||
|
||||
|
||||
def get_webmention(webmention_id: int) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Get single webmention by ID.
|
||||
|
||||
Args:
|
||||
webmention_id: Webmention database ID
|
||||
|
||||
Returns:
|
||||
Webmention dict or None if not found
|
||||
"""
|
||||
db = get_db()
|
||||
|
||||
query = """
|
||||
SELECT w.*, n.slug as target_slug
|
||||
FROM webmentions w
|
||||
LEFT JOIN notes n ON w.target_note_id = n.id
|
||||
WHERE w.id = ?
|
||||
"""
|
||||
|
||||
row = db.execute(query, (webmention_id,)).fetchone()
|
||||
|
||||
if not row:
|
||||
return None
|
||||
|
||||
wm = dict(row)
|
||||
# Parse timestamps
|
||||
if isinstance(wm.get('received_at'), str):
|
||||
wm['received_at'] = datetime.fromisoformat(wm['received_at'].replace('Z', '+00:00'))
|
||||
if wm.get('validated_at') and isinstance(wm['validated_at'], str):
|
||||
wm['validated_at'] = datetime.fromisoformat(wm['validated_at'].replace('Z', '+00:00'))
|
||||
|
||||
return wm
|
||||
|
||||
|
||||
def count_webmentions(status_filter: Optional[str] = None) -> int:
|
||||
"""
|
||||
Count webmentions, optionally filtered by status.
|
||||
|
||||
Args:
|
||||
status_filter: Filter by moderation_status
|
||||
|
||||
Returns:
|
||||
Count of webmentions
|
||||
"""
|
||||
db = get_db()
|
||||
|
||||
if status_filter:
|
||||
query = "SELECT COUNT(*) FROM webmentions WHERE moderation_status = ?"
|
||||
result = db.execute(query, (status_filter,)).fetchone()
|
||||
else:
|
||||
query = "SELECT COUNT(*) FROM webmentions"
|
||||
result = db.execute(query).fetchone()
|
||||
|
||||
return result[0] if result else 0
|
||||
|
||||
|
||||
def update_webmention_status(webmention_ids: List[int], status: str) -> int:
|
||||
"""
|
||||
Update moderation status for multiple webmentions.
|
||||
|
||||
Args:
|
||||
webmention_ids: List of webmention IDs
|
||||
status: New moderation_status ('approved', 'hidden', 'pending')
|
||||
|
||||
Returns:
|
||||
Number of webmentions updated
|
||||
"""
|
||||
if not webmention_ids:
|
||||
return 0
|
||||
|
||||
if status not in ('approved', 'hidden', 'pending'):
|
||||
raise ValueError(f"Invalid status: {status}")
|
||||
|
||||
db = get_db()
|
||||
|
||||
# Use parameterized query with placeholders
|
||||
placeholders = ','.join('?' * len(webmention_ids))
|
||||
query = f"""
|
||||
UPDATE webmentions
|
||||
SET moderation_status = ?,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id IN ({placeholders})
|
||||
"""
|
||||
|
||||
params = [status] + webmention_ids
|
||||
cursor = db.execute(query, params)
|
||||
db.commit()
|
||||
|
||||
return cursor.rowcount
|
||||
|
||||
|
||||
def delete_webmentions(webmention_ids: List[int]) -> int:
|
||||
"""
|
||||
Delete webmentions by IDs.
|
||||
|
||||
Args:
|
||||
webmention_ids: List of webmention IDs to delete
|
||||
|
||||
Returns:
|
||||
Number of webmentions deleted
|
||||
"""
|
||||
if not webmention_ids:
|
||||
return 0
|
||||
|
||||
db = get_db()
|
||||
|
||||
placeholders = ','.join('?' * len(webmention_ids))
|
||||
query = f"DELETE FROM webmentions WHERE id IN ({placeholders})"
|
||||
|
||||
cursor = db.execute(query, webmention_ids)
|
||||
db.commit()
|
||||
|
||||
return cursor.rowcount
|
||||
|
||||
|
||||
def get_reply_context(url: str) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Get cached reply context for a URL.
|
||||
|
||||
Args:
|
||||
url: The in_reply_to URL
|
||||
|
||||
Returns:
|
||||
Reply context dict or None if not found
|
||||
"""
|
||||
db = get_db()
|
||||
|
||||
query = "SELECT * FROM reply_contexts WHERE url = ?"
|
||||
row = db.execute(query, (url,)).fetchone()
|
||||
|
||||
if not row:
|
||||
return None
|
||||
|
||||
context = dict(row)
|
||||
|
||||
# Parse timestamps
|
||||
if context.get('fetched_at') and isinstance(context['fetched_at'], str):
|
||||
context['fetched_at'] = datetime.fromisoformat(context['fetched_at'].replace('Z', '+00:00'))
|
||||
if context.get('created_at') and isinstance(context['created_at'], str):
|
||||
context['created_at'] = datetime.fromisoformat(context['created_at'].replace('Z', '+00:00'))
|
||||
if context.get('published_at') and isinstance(context['published_at'], str):
|
||||
context['published_at'] = datetime.fromisoformat(context['published_at'].replace('Z', '+00:00'))
|
||||
|
||||
return context
|
||||
@@ -0,0 +1,198 @@
|
||||
"""
|
||||
Outgoing webmention link extraction and queueing
|
||||
|
||||
Per background-worker.md and architect's answers:
|
||||
- Extract links from rendered HTML (not markdown)
|
||||
- Extract only <a href> elements, exclude <pre> and <code> blocks
|
||||
- Queue max 10 links per note per operation
|
||||
- Priority: in_reply_to first, then document order
|
||||
|
||||
References:
|
||||
- Background Worker Design: /docs/design/v1.6.0/background-worker.md
|
||||
- API Contract: /docs/design/v1.6.0/api-contract.md
|
||||
"""
|
||||
|
||||
import logging
|
||||
import sqlite3
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Optional, Set
|
||||
from lxml import html as lxml_html
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def extract_links_from_html(html_content: str, exclude_code_blocks: bool = True) -> list[str]:
|
||||
"""
|
||||
Extract links from HTML content.
|
||||
|
||||
Per architect's answer to Q1:
|
||||
- Extract from rendered HTML (not markdown)
|
||||
- Extract <a href> elements only
|
||||
- Exclude links in <pre> and <code> blocks
|
||||
- Return in document order
|
||||
|
||||
Args:
|
||||
html_content: Rendered HTML content
|
||||
exclude_code_blocks: If True, skip links in <pre> and <code> (default: True)
|
||||
|
||||
Returns:
|
||||
List of URLs in document order
|
||||
"""
|
||||
if not html_content or not html_content.strip():
|
||||
return []
|
||||
|
||||
try:
|
||||
# Parse HTML
|
||||
doc = lxml_html.fromstring(html_content)
|
||||
|
||||
if exclude_code_blocks:
|
||||
# Remove <pre> and <code> elements before extracting links
|
||||
for element in doc.xpath('//pre | //code'):
|
||||
element.getparent().remove(element)
|
||||
|
||||
# Extract all <a href> attributes in document order
|
||||
links = doc.xpath('//a/@href')
|
||||
|
||||
# Filter to valid absolute URLs (http/https)
|
||||
absolute_links = []
|
||||
for link in links:
|
||||
parsed = urlparse(link)
|
||||
if parsed.scheme in ('http', 'https') and parsed.netloc:
|
||||
absolute_links.append(link)
|
||||
|
||||
return absolute_links
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to parse HTML for link extraction: {e}")
|
||||
return []
|
||||
|
||||
|
||||
def queue_webmentions_for_note(
|
||||
db_path: Path,
|
||||
note_id: int,
|
||||
note_url: str,
|
||||
content_html: str,
|
||||
in_reply_to: Optional[str] = None,
|
||||
old_content_html: Optional[str] = None # Phase 6: update semantics
|
||||
) -> int:
|
||||
"""
|
||||
Extract links and queue webmentions for a note.
|
||||
|
||||
Per architect's answers (Phase 6):
|
||||
- Called from notes.py AFTER successful database commit
|
||||
- For updates: diff old vs new URLs, queue for new/existing/removed
|
||||
- For creates: queue for all URLs in content
|
||||
- Max 10 webmentions per operation
|
||||
- Priority: in_reply_to URL first, then document order
|
||||
- Webmention sending is best-effort; failures do not affect note creation
|
||||
|
||||
Args:
|
||||
db_path: Path to SQLite database
|
||||
note_id: Database ID of the note
|
||||
note_url: Full URL of the published note
|
||||
content_html: Rendered HTML content (None if unpublishing)
|
||||
in_reply_to: Reply-to URL (if reply post)
|
||||
old_content_html: Previous HTML (for update diff, Phase 6)
|
||||
|
||||
Returns:
|
||||
Number of webmentions queued
|
||||
"""
|
||||
# Phase 6: Implement URL diffing for updates
|
||||
if old_content_html is not None:
|
||||
# This is an update - compute URL diff
|
||||
old_links = set(extract_links_from_html(old_content_html))
|
||||
|
||||
# Handle unpublishing case (content_html is None)
|
||||
if content_html is None:
|
||||
# Unpublishing - queue for all old URLs (they're all removed)
|
||||
new_links = set()
|
||||
else:
|
||||
new_links = set(extract_links_from_html(content_html))
|
||||
|
||||
# Compute sets for URL diffing
|
||||
new_urls = new_links - old_links # In new, not in old
|
||||
existing_urls = new_links & old_links # In both (re-validate)
|
||||
removed_urls = old_links - new_links # In old, not in new
|
||||
|
||||
# Build target list: new + existing + removed
|
||||
# (All need webmentions per W3C spec)
|
||||
all_update_urls = list(new_urls) + list(existing_urls) + list(removed_urls)
|
||||
|
||||
# Add in_reply_to if changed
|
||||
target_urls = []
|
||||
if in_reply_to and in_reply_to not in all_update_urls:
|
||||
target_urls.append(in_reply_to)
|
||||
|
||||
# Add the diffed URLs
|
||||
target_urls.extend(all_update_urls)
|
||||
|
||||
# Limit to 10
|
||||
target_urls = target_urls[:10]
|
||||
|
||||
else:
|
||||
# This is a create (no old content) - use original logic
|
||||
links = extract_links_from_html(content_html) if content_html else []
|
||||
|
||||
# Build priority-ordered list of target URLs
|
||||
target_urls = []
|
||||
|
||||
# 1. in_reply_to gets highest priority
|
||||
if in_reply_to:
|
||||
target_urls.append(in_reply_to)
|
||||
|
||||
# 2. Add extracted links in document order (deduplicated)
|
||||
seen = {in_reply_to} if in_reply_to else set()
|
||||
for link in links:
|
||||
if link not in seen:
|
||||
target_urls.append(link)
|
||||
seen.add(link)
|
||||
|
||||
# 3. Limit to 10 webmentions per operation
|
||||
target_urls = target_urls[:10]
|
||||
|
||||
if not target_urls:
|
||||
logger.debug(f"No links to send webmentions for note {note_id}")
|
||||
return 0
|
||||
|
||||
# Queue webmentions
|
||||
queued = 0
|
||||
try:
|
||||
conn = sqlite3.connect(db_path)
|
||||
cursor = conn.cursor()
|
||||
|
||||
for target_url in target_urls:
|
||||
try:
|
||||
# Use UPSERT to handle duplicate sends
|
||||
# Phase 6: Force endpoint rediscovery per W3C MUST requirement
|
||||
cursor.execute("""
|
||||
INSERT INTO webmention_queue
|
||||
(source_url, target_url, source_note_id, status, created_at)
|
||||
VALUES (?, ?, ?, 'pending', CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(source_url, target_url) DO UPDATE SET
|
||||
status = 'pending',
|
||||
retry_count = 0,
|
||||
next_retry_at = NULL,
|
||||
last_error = NULL,
|
||||
endpoint_url = NULL,
|
||||
created_at = CURRENT_TIMESTAMP
|
||||
""", (note_url, target_url, note_id))
|
||||
|
||||
queued += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Failed to queue webmention {note_url} -> {target_url}: {e}"
|
||||
)
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
logger.info(f"Queued {queued} webmentions for note {note_id}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to queue webmentions for note {note_id}: {e}")
|
||||
|
||||
return queued
|
||||
@@ -0,0 +1,113 @@
|
||||
"""
|
||||
SSRF protection utilities for webmention processing
|
||||
|
||||
Per ADR-063 Decision 9 and background-worker.md:
|
||||
- Block private IP address ranges
|
||||
- Resolve hostname before making requests
|
||||
- DEV_MODE exception for localhost testing
|
||||
- Applied to all outgoing HTTP requests
|
||||
|
||||
References:
|
||||
- ADR-063: Webmention Implementation
|
||||
- Background Worker Design: /docs/design/v1.6.0/background-worker.md
|
||||
"""
|
||||
|
||||
import ipaddress
|
||||
import socket
|
||||
from flask import current_app
|
||||
|
||||
|
||||
class SSRFBlockedError(Exception):
|
||||
"""Raised when SSRF protection blocks a request"""
|
||||
pass
|
||||
|
||||
|
||||
# Private IP ranges per RFC 1918 and link-local ranges
|
||||
BLOCKED_NETWORKS = [
|
||||
ipaddress.ip_network('10.0.0.0/8'), # RFC 1918 private
|
||||
ipaddress.ip_network('172.16.0.0/12'), # RFC 1918 private
|
||||
ipaddress.ip_network('192.168.0.0/16'), # RFC 1918 private
|
||||
ipaddress.ip_network('127.0.0.0/8'), # Localhost
|
||||
ipaddress.ip_network('169.254.0.0/16'), # Link-local
|
||||
ipaddress.ip_network('::1/128'), # IPv6 localhost
|
||||
ipaddress.ip_network('fc00::/7'), # IPv6 unique local
|
||||
ipaddress.ip_network('fe80::/10'), # IPv6 link-local
|
||||
]
|
||||
|
||||
|
||||
def is_private_ip(ip: str) -> bool:
|
||||
"""
|
||||
Check if IP address is in a private/blocked range.
|
||||
|
||||
Args:
|
||||
ip: IP address string (IPv4 or IPv6)
|
||||
|
||||
Returns:
|
||||
True if IP is private/blocked, False otherwise
|
||||
"""
|
||||
try:
|
||||
addr = ipaddress.ip_address(ip)
|
||||
return any(addr in network for network in BLOCKED_NETWORKS)
|
||||
except ValueError:
|
||||
# Invalid IP format, treat as blocked
|
||||
return True
|
||||
|
||||
|
||||
def safe_resolve(hostname: str) -> str:
|
||||
"""
|
||||
Resolve hostname and validate IP is not private.
|
||||
|
||||
Per background-worker.md: DEV_MODE exception for localhost testing.
|
||||
|
||||
Args:
|
||||
hostname: Hostname to resolve
|
||||
|
||||
Returns:
|
||||
Resolved IP address if safe
|
||||
|
||||
Raises:
|
||||
SSRFBlockedError: If hostname resolves to private IP
|
||||
socket.gaierror: If hostname cannot be resolved
|
||||
"""
|
||||
# DEV_MODE exception for localhost
|
||||
if current_app.config.get('DEV_MODE') and hostname in ('localhost', '127.0.0.1', '::1'):
|
||||
return '127.0.0.1'
|
||||
|
||||
# Resolve hostname
|
||||
ip = socket.gethostbyname(hostname)
|
||||
|
||||
# Check if resolved IP is private
|
||||
if is_private_ip(ip):
|
||||
raise SSRFBlockedError(f"Blocked request to private IP: {ip}")
|
||||
|
||||
return ip
|
||||
|
||||
|
||||
def validate_url_for_fetch(url: str) -> None:
|
||||
"""
|
||||
Validate URL is safe to fetch (not private IP).
|
||||
|
||||
Args:
|
||||
url: URL to validate
|
||||
|
||||
Raises:
|
||||
SSRFBlockedError: If URL points to private IP
|
||||
ValueError: If URL is malformed
|
||||
"""
|
||||
from urllib.parse import urlparse
|
||||
|
||||
parsed = urlparse(url)
|
||||
|
||||
if not parsed.scheme in ('http', 'https'):
|
||||
raise ValueError(f"Unsupported URL scheme: {parsed.scheme}")
|
||||
|
||||
if not parsed.netloc:
|
||||
raise ValueError("URL must include a hostname")
|
||||
|
||||
# Extract hostname (remove port if present)
|
||||
hostname = parsed.hostname
|
||||
if not hostname:
|
||||
raise ValueError("Could not extract hostname from URL")
|
||||
|
||||
# Validate hostname resolves to public IP
|
||||
safe_resolve(hostname)
|
||||
@@ -0,0 +1,787 @@
|
||||
"""
|
||||
Background worker thread for webmention processing
|
||||
|
||||
Per background-worker.md:
|
||||
- Daemon thread that processes webmention queues
|
||||
- Handles both incoming validation and outgoing sending
|
||||
- Graceful shutdown on app termination
|
||||
- Orphan detection and cleanup on startup
|
||||
|
||||
References:
|
||||
- Background Worker Design: /docs/design/v1.6.0/background-worker.md
|
||||
- ADR-063: Webmention Implementation
|
||||
"""
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import sqlite3
|
||||
import httpx
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, Optional, TypedDict
|
||||
from urllib.parse import urlparse
|
||||
from lxml import html as lxml_html
|
||||
|
||||
from starpunk.webmention_processor import process_webmention, ValidationError
|
||||
from starpunk.webmention_ssrf import validate_url_for_fetch, SSRFBlockedError
|
||||
from flask import has_app_context
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Discovery cache entry type
|
||||
class DiscoveryCacheEntry(TypedDict):
|
||||
"""Cache entry for endpoint discovery results"""
|
||||
endpoint: Optional[str] # None = negative cache (no endpoint found)
|
||||
expires_at: datetime
|
||||
|
||||
|
||||
class WebmentionWorker(threading.Thread):
|
||||
"""
|
||||
Background thread for webmention queue processing
|
||||
|
||||
Follows MemoryMonitor pattern: daemon thread with stop event
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
db_path: Path,
|
||||
site_url: str,
|
||||
poll_interval: int = 10,
|
||||
error_backoff: int = 60,
|
||||
batch_size: int = 5,
|
||||
auto_approve: bool = True
|
||||
):
|
||||
"""
|
||||
Initialize webmention worker thread
|
||||
|
||||
Args:
|
||||
db_path: Path to SQLite database
|
||||
site_url: Site URL for User-Agent header
|
||||
poll_interval: Seconds between queue checks (default: 10)
|
||||
error_backoff: Seconds to wait after error (default: 60)
|
||||
batch_size: Items to process per iteration (default: 5)
|
||||
auto_approve: Auto-approve valid webmentions (default: True)
|
||||
"""
|
||||
super().__init__(daemon=True, name="webmention-worker")
|
||||
self.db_path = db_path
|
||||
self.site_url = site_url
|
||||
self.poll_interval = poll_interval
|
||||
self.error_backoff = error_backoff
|
||||
self.batch_size = batch_size
|
||||
self.auto_approve = auto_approve
|
||||
self._stop_event = threading.Event()
|
||||
self._last_processed = None
|
||||
|
||||
# Phase 3: Discovery cache (worker instance variable)
|
||||
# Per architect's answer to Q4
|
||||
self._discovery_cache: Dict[str, DiscoveryCacheEntry] = {}
|
||||
|
||||
def run(self):
|
||||
"""
|
||||
Main processing loop
|
||||
|
||||
Per background-worker.md:
|
||||
1. Clean up orphaned items on startup
|
||||
2. Process pending validations (incoming)
|
||||
3. Process pending sends (outgoing) - Phase 3
|
||||
4. Sleep if nothing to process
|
||||
"""
|
||||
try:
|
||||
logger.info("Webmention worker started")
|
||||
|
||||
# Cleanup orphaned items from previous crashes
|
||||
self._cleanup_orphaned_items()
|
||||
|
||||
# Main processing loop
|
||||
while not self._stop_event.is_set():
|
||||
try:
|
||||
processed = 0
|
||||
|
||||
# Process incoming webmentions
|
||||
processed += self._process_pending_validations()
|
||||
|
||||
# Phase 3: Process outgoing webmentions
|
||||
processed += self._process_pending_sends()
|
||||
|
||||
# Update last processed time
|
||||
if processed > 0:
|
||||
self._last_processed = datetime.utcnow()
|
||||
|
||||
# Sleep if nothing processed
|
||||
if processed == 0:
|
||||
self._stop_event.wait(self.poll_interval)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Worker error: {e}", exc_info=True)
|
||||
self._stop_event.wait(self.error_backoff)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Webmention worker thread failed: {e}", exc_info=True)
|
||||
finally:
|
||||
logger.info("Webmention worker stopped")
|
||||
|
||||
def _cleanup_orphaned_items(self):
|
||||
"""
|
||||
Clean up orphaned 'sending' status items on startup
|
||||
|
||||
Per background-worker.md:
|
||||
Reset items stuck in 'sending' status from previous crashes
|
||||
"""
|
||||
try:
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Reset orphaned outgoing webmentions
|
||||
cursor.execute("""
|
||||
UPDATE webmention_queue
|
||||
SET status = 'pending',
|
||||
started_at = NULL
|
||||
WHERE status = 'sending'
|
||||
AND started_at < datetime('now', '-5 minutes')
|
||||
""")
|
||||
|
||||
orphaned = cursor.rowcount
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
if orphaned > 0:
|
||||
logger.info(f"Reset {orphaned} orphaned webmention queue items")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to cleanup orphaned items: {e}")
|
||||
|
||||
def _process_pending_validations(self) -> int:
|
||||
"""
|
||||
Process pending incoming webmentions
|
||||
|
||||
Returns:
|
||||
Number of webmentions processed
|
||||
"""
|
||||
processed = 0
|
||||
|
||||
try:
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Get pending webmentions
|
||||
cursor.execute("""
|
||||
SELECT id, source_url, target_url, target_note_id
|
||||
FROM webmentions
|
||||
WHERE validation_status = 'pending'
|
||||
ORDER BY received_at ASC
|
||||
LIMIT ?
|
||||
""", (self.batch_size,))
|
||||
|
||||
pending = cursor.fetchall()
|
||||
conn.close()
|
||||
|
||||
# Process each webmention
|
||||
for row in pending:
|
||||
try:
|
||||
self._validate_webmention(
|
||||
row['id'],
|
||||
row['source_url'],
|
||||
row['target_url'],
|
||||
row['target_note_id']
|
||||
)
|
||||
processed += 1
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Failed to validate webmention {row['id']}: {e}",
|
||||
exc_info=True
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching pending validations: {e}")
|
||||
|
||||
return processed
|
||||
|
||||
def _validate_webmention(
|
||||
self,
|
||||
webmention_id: int,
|
||||
source_url: str,
|
||||
target_url: str,
|
||||
target_note_id: Optional[int]
|
||||
):
|
||||
"""
|
||||
Validate a single webmention
|
||||
|
||||
Args:
|
||||
webmention_id: Database ID of webmention
|
||||
source_url: Source URL
|
||||
target_url: Target URL
|
||||
target_note_id: Target note ID (if applicable)
|
||||
"""
|
||||
try:
|
||||
# Process webmention (fetch, validate, extract)
|
||||
result = process_webmention(source_url, target_url, self.site_url)
|
||||
|
||||
# Update database with successful validation
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
cursor = conn.cursor()
|
||||
|
||||
moderation_status = 'approved' if self.auto_approve else 'pending'
|
||||
|
||||
cursor.execute("""
|
||||
UPDATE webmentions
|
||||
SET validation_status = 'valid',
|
||||
validated_at = CURRENT_TIMESTAMP,
|
||||
mention_type = ?,
|
||||
author_name = ?,
|
||||
author_url = ?,
|
||||
author_photo = ?,
|
||||
content_snippet = ?,
|
||||
moderation_status = ?,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?
|
||||
""", (
|
||||
result['mention_type'],
|
||||
result['author_name'],
|
||||
result['author_url'],
|
||||
result['author_photo'],
|
||||
result['content_snippet'],
|
||||
moderation_status,
|
||||
webmention_id
|
||||
))
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
logger.info(
|
||||
f"Validated webmention: {source_url} -> {target_url} "
|
||||
f"(type={result['mention_type']}, status={moderation_status})"
|
||||
)
|
||||
|
||||
except ValidationError as e:
|
||||
# Validation failed - update with error
|
||||
self._mark_validation_failed(webmention_id, source_url, target_url, str(e))
|
||||
|
||||
except Exception as e:
|
||||
# Unexpected error - mark as error
|
||||
self._mark_validation_failed(
|
||||
webmention_id,
|
||||
source_url,
|
||||
target_url,
|
||||
f"Unexpected error: {e}"
|
||||
)
|
||||
raise
|
||||
|
||||
def _mark_validation_failed(
|
||||
self,
|
||||
webmention_id: int,
|
||||
source_url: str,
|
||||
target_url: str,
|
||||
error: str
|
||||
):
|
||||
"""
|
||||
Mark webmention validation as failed/invalid
|
||||
|
||||
Args:
|
||||
webmention_id: Database ID
|
||||
source_url: Source URL
|
||||
target_url: Target URL
|
||||
error: Error message
|
||||
"""
|
||||
# Determine if error or invalid
|
||||
# Per background-worker.md: HTTP 4xx and "no link" are invalid
|
||||
# All other errors are 'error' status
|
||||
if any(x in error.lower() for x in ['does not link', 'returned 4']):
|
||||
status = 'invalid'
|
||||
else:
|
||||
status = 'error'
|
||||
|
||||
try:
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
cursor = conn.cursor()
|
||||
|
||||
cursor.execute("""
|
||||
UPDATE webmentions
|
||||
SET validation_status = ?,
|
||||
validation_error = ?,
|
||||
validated_at = CURRENT_TIMESTAMP,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?
|
||||
""", (status, error, webmention_id))
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
logger.warning(
|
||||
f"Invalid webmention: {source_url} -> {target_url}: {error}"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to mark webmention as failed: {e}")
|
||||
|
||||
def _process_pending_sends(self) -> int:
|
||||
"""
|
||||
Process pending outgoing webmentions (Phase 3)
|
||||
|
||||
Returns:
|
||||
Number of webmentions processed
|
||||
"""
|
||||
processed = 0
|
||||
|
||||
try:
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Get pending webmentions ready to send
|
||||
cursor.execute("""
|
||||
SELECT id, source_url, target_url, endpoint_url, retry_count
|
||||
FROM webmention_queue
|
||||
WHERE status = 'pending'
|
||||
AND (next_retry_at IS NULL OR next_retry_at <= datetime('now'))
|
||||
ORDER BY created_at ASC
|
||||
LIMIT ?
|
||||
""", (self.batch_size,))
|
||||
|
||||
pending = cursor.fetchall()
|
||||
conn.close()
|
||||
|
||||
# Process each webmention
|
||||
for row in pending:
|
||||
try:
|
||||
self._send_webmention(
|
||||
row['id'],
|
||||
row['source_url'],
|
||||
row['target_url'],
|
||||
row['endpoint_url'],
|
||||
row['retry_count']
|
||||
)
|
||||
processed += 1
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Failed to send webmention {row['id']}: {e}",
|
||||
exc_info=True
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching pending sends: {e}")
|
||||
|
||||
return processed
|
||||
|
||||
def _send_webmention(
|
||||
self,
|
||||
queue_id: int,
|
||||
source_url: str,
|
||||
target_url: str,
|
||||
cached_endpoint: Optional[str],
|
||||
retry_count: int
|
||||
):
|
||||
"""
|
||||
Send a single outgoing webmention
|
||||
|
||||
Args:
|
||||
queue_id: Database ID of queue item
|
||||
source_url: Source URL (our note)
|
||||
target_url: Target URL to notify
|
||||
cached_endpoint: Previously discovered endpoint (if any)
|
||||
retry_count: Current retry count
|
||||
"""
|
||||
# Mark as sending
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
UPDATE webmention_queue
|
||||
SET status = 'sending',
|
||||
started_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?
|
||||
""", (queue_id,))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
try:
|
||||
# Discover endpoint if not cached
|
||||
endpoint = cached_endpoint
|
||||
if not endpoint:
|
||||
endpoint = self._discover_endpoint(target_url)
|
||||
|
||||
if not endpoint:
|
||||
# No endpoint found - permanent failure
|
||||
self._mark_send_no_endpoint(queue_id, source_url, target_url)
|
||||
return
|
||||
|
||||
# Cache the discovered endpoint in database
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
UPDATE webmention_queue
|
||||
SET endpoint_url = ?
|
||||
WHERE id = ?
|
||||
""", (endpoint, queue_id))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
# Send webmention POST
|
||||
try:
|
||||
# SSRF protection for endpoint URL
|
||||
validate_url_for_fetch(endpoint)
|
||||
|
||||
response = httpx.post(
|
||||
endpoint,
|
||||
data={
|
||||
'source': source_url,
|
||||
'target': target_url
|
||||
},
|
||||
timeout=10,
|
||||
headers={'User-Agent': f'StarPunk/1.6.0 (+{self.site_url})'},
|
||||
follow_redirects=True
|
||||
)
|
||||
|
||||
# Check response
|
||||
if 200 <= response.status_code < 300:
|
||||
# Success
|
||||
self._mark_send_success(queue_id, source_url, target_url)
|
||||
|
||||
elif 400 <= response.status_code < 500:
|
||||
# Permanent failure (4xx)
|
||||
error = f"HTTP {response.status_code}: {response.text[:200]}"
|
||||
self._mark_send_failed(queue_id, source_url, target_url, error)
|
||||
|
||||
else:
|
||||
# Temporary failure (5xx) - retry
|
||||
error = f"HTTP {response.status_code}: {response.text[:200]}"
|
||||
self._mark_send_retry(queue_id, source_url, target_url, retry_count, error)
|
||||
|
||||
except (httpx.TimeoutException, httpx.HTTPError) as e:
|
||||
# Connection error - retry
|
||||
self._mark_send_retry(queue_id, source_url, target_url, retry_count, str(e))
|
||||
|
||||
except (SSRFBlockedError, ValueError) as e:
|
||||
# SSRF blocked - permanent failure
|
||||
logger.warning(f"SSRF blocked endpoint {endpoint}: {e}")
|
||||
self._mark_send_failed(queue_id, source_url, target_url, f"Blocked: {e}")
|
||||
|
||||
except Exception as e:
|
||||
# Discovery failed - retry
|
||||
self._mark_send_retry(queue_id, source_url, target_url, retry_count, str(e))
|
||||
|
||||
def _discover_endpoint(self, target_url: str) -> Optional[str]:
|
||||
"""
|
||||
Discover webmention endpoint for target URL with caching
|
||||
|
||||
Per architect's answers to Q3, Q4, Q10:
|
||||
- GET only (no HEAD)
|
||||
- Check Link header → <link rel="webmention"> → <a rel="webmention">
|
||||
- First match wins
|
||||
- Cache results: 1h for found, 24h for not found
|
||||
- SSRF protection on target URL
|
||||
|
||||
Args:
|
||||
target_url: Target URL to discover endpoint for
|
||||
|
||||
Returns:
|
||||
Endpoint URL if found, None otherwise
|
||||
"""
|
||||
# Check cache first
|
||||
cached = self._discovery_cache.get(target_url)
|
||||
if cached and cached['expires_at'] > datetime.utcnow():
|
||||
logger.debug(f"Using cached endpoint for {target_url}: {cached['endpoint']}")
|
||||
return cached['endpoint']
|
||||
|
||||
# SSRF protection
|
||||
try:
|
||||
validate_url_for_fetch(target_url)
|
||||
except (SSRFBlockedError, ValueError) as e:
|
||||
logger.warning(f"SSRF protection blocked target {target_url}: {e}")
|
||||
# Cache negative result
|
||||
self._cache_endpoint(target_url, None, hours=24)
|
||||
return None
|
||||
|
||||
# Fetch target URL
|
||||
try:
|
||||
response = httpx.get(
|
||||
target_url,
|
||||
timeout=10,
|
||||
headers={'User-Agent': f'StarPunk/1.6.0 (+{self.site_url})'},
|
||||
follow_redirects=True
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
# 1. Check Link header
|
||||
link_header = response.headers.get('Link', '')
|
||||
if link_header:
|
||||
endpoint = self._parse_link_header(link_header, response.url)
|
||||
if endpoint:
|
||||
self._cache_endpoint(target_url, endpoint, hours=1)
|
||||
return endpoint
|
||||
|
||||
# 2. Parse HTML for <link> and <a> elements
|
||||
content_type = response.headers.get('Content-Type', '')
|
||||
if 'html' in content_type.lower():
|
||||
endpoint = self._parse_html_for_endpoint(response.text, response.url)
|
||||
if endpoint:
|
||||
self._cache_endpoint(target_url, endpoint, hours=1)
|
||||
return endpoint
|
||||
|
||||
# No endpoint found - cache negative result
|
||||
logger.info(f"No webmention endpoint found for {target_url}")
|
||||
self._cache_endpoint(target_url, None, hours=24)
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to discover endpoint for {target_url}: {e}")
|
||||
# Don't cache on error - allow retry
|
||||
return None
|
||||
|
||||
def _parse_link_header(self, link_header: str, base_url: str) -> Optional[str]:
|
||||
"""
|
||||
Parse Link header for webmention endpoint
|
||||
|
||||
Args:
|
||||
link_header: Raw Link header value
|
||||
base_url: Base URL for resolving relative URLs
|
||||
|
||||
Returns:
|
||||
Absolute endpoint URL if found, None otherwise
|
||||
"""
|
||||
# Simple parsing - look for rel="webmention"
|
||||
# Format: <URL>; rel="webmention"
|
||||
import re
|
||||
pattern = r'<([^>]+)>\s*;\s*rel\s*=\s*["\']?webmention["\']?'
|
||||
match = re.search(pattern, link_header, re.IGNORECASE)
|
||||
|
||||
if match:
|
||||
endpoint = match.group(1)
|
||||
# Make absolute if relative
|
||||
from urllib.parse import urljoin
|
||||
return urljoin(base_url, endpoint)
|
||||
|
||||
return None
|
||||
|
||||
def _parse_html_for_endpoint(self, html_content: str, base_url: str) -> Optional[str]:
|
||||
"""
|
||||
Parse HTML for webmention endpoint in <link> or <a> elements
|
||||
|
||||
Args:
|
||||
html_content: HTML content
|
||||
base_url: Base URL for resolving relative URLs
|
||||
|
||||
Returns:
|
||||
Absolute endpoint URL if found, None otherwise
|
||||
"""
|
||||
try:
|
||||
doc = lxml_html.fromstring(html_content)
|
||||
|
||||
# Check <link rel="webmention"> first
|
||||
links = doc.xpath('//link[@rel="webmention"]/@href')
|
||||
if links:
|
||||
from urllib.parse import urljoin
|
||||
return urljoin(base_url, links[0])
|
||||
|
||||
# Check <a rel="webmention">
|
||||
anchors = doc.xpath('//a[@rel="webmention"]/@href')
|
||||
if anchors:
|
||||
from urllib.parse import urljoin
|
||||
return urljoin(base_url, anchors[0])
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"HTML parsing failed: {e}")
|
||||
|
||||
return None
|
||||
|
||||
def _cache_endpoint(self, target_url: str, endpoint: Optional[str], hours: int):
|
||||
"""
|
||||
Cache endpoint discovery result
|
||||
|
||||
Args:
|
||||
target_url: Target URL
|
||||
endpoint: Discovered endpoint (None = negative cache)
|
||||
hours: TTL in hours
|
||||
"""
|
||||
self._discovery_cache[target_url] = {
|
||||
'endpoint': endpoint,
|
||||
'expires_at': datetime.utcnow() + timedelta(hours=hours)
|
||||
}
|
||||
|
||||
# Optional: Evict old entries (keep cache bounded)
|
||||
if len(self._discovery_cache) > 1000:
|
||||
# Remove expired entries
|
||||
now = datetime.utcnow()
|
||||
self._discovery_cache = {
|
||||
k: v for k, v in self._discovery_cache.items()
|
||||
if v['expires_at'] > now
|
||||
}
|
||||
|
||||
def _mark_send_success(self, queue_id: int, source_url: str, target_url: str):
|
||||
"""Mark webmention send as successful"""
|
||||
try:
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
UPDATE webmention_queue
|
||||
SET status = 'sent',
|
||||
sent_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?
|
||||
""", (queue_id,))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
logger.info(f"Sent webmention: {source_url} -> {target_url}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to mark send success: {e}")
|
||||
|
||||
def _mark_send_failed(self, queue_id: int, source_url: str, target_url: str, error: str):
|
||||
"""Mark webmention send as permanently failed"""
|
||||
try:
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
UPDATE webmention_queue
|
||||
SET status = 'failed',
|
||||
last_error = ?
|
||||
WHERE id = ?
|
||||
""", (error, queue_id))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
logger.warning(f"Webmention send failed permanently: {source_url} -> {target_url}: {error}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to mark send failed: {e}")
|
||||
|
||||
def _mark_send_no_endpoint(self, queue_id: int, source_url: str, target_url: str):
|
||||
"""Mark webmention as no endpoint found"""
|
||||
try:
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
UPDATE webmention_queue
|
||||
SET status = 'no_endpoint'
|
||||
WHERE id = ?
|
||||
""", (queue_id,))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
logger.info(f"No webmention endpoint for {target_url}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to mark no endpoint: {e}")
|
||||
|
||||
def _mark_send_retry(
|
||||
self,
|
||||
queue_id: int,
|
||||
source_url: str,
|
||||
target_url: str,
|
||||
current_retry_count: int,
|
||||
error: str
|
||||
):
|
||||
"""
|
||||
Mark webmention for retry with exponential backoff
|
||||
|
||||
Per architect's answer to Q5 and background-worker.md:
|
||||
- Increment retry_count AFTER failed attempt
|
||||
- Check retry_count >= 4 AFTER incrementing
|
||||
- Retry schedule: 1h, 6h, 24h
|
||||
"""
|
||||
new_retry_count = current_retry_count + 1
|
||||
|
||||
# Calculate next retry time
|
||||
next_retry_at = None
|
||||
if new_retry_count < 4:
|
||||
delays = {
|
||||
1: timedelta(hours=1),
|
||||
2: timedelta(hours=6),
|
||||
3: timedelta(hours=24),
|
||||
}
|
||||
delay = delays.get(new_retry_count)
|
||||
if delay:
|
||||
next_retry_at = datetime.utcnow() + delay
|
||||
|
||||
try:
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
cursor = conn.cursor()
|
||||
|
||||
if new_retry_count >= 4:
|
||||
# Max retries reached - mark as failed
|
||||
cursor.execute("""
|
||||
UPDATE webmention_queue
|
||||
SET status = 'failed',
|
||||
retry_count = ?,
|
||||
last_error = ?
|
||||
WHERE id = ?
|
||||
""", (new_retry_count, error, queue_id))
|
||||
|
||||
logger.warning(
|
||||
f"Webmention send failed permanently after {new_retry_count} attempts: "
|
||||
f"{source_url} -> {target_url}: {error}"
|
||||
)
|
||||
|
||||
else:
|
||||
# Schedule retry
|
||||
cursor.execute("""
|
||||
UPDATE webmention_queue
|
||||
SET status = 'pending',
|
||||
retry_count = ?,
|
||||
next_retry_at = ?,
|
||||
last_error = ?
|
||||
WHERE id = ?
|
||||
""", (new_retry_count, next_retry_at, error, queue_id))
|
||||
|
||||
logger.warning(
|
||||
f"Webmention send failed, retry {new_retry_count}: "
|
||||
f"{source_url} -> {target_url}: {error}"
|
||||
)
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to mark send retry: {e}")
|
||||
|
||||
def stop(self):
|
||||
"""
|
||||
Stop the worker thread gracefully
|
||||
|
||||
Sets the stop event to signal the thread to exit
|
||||
"""
|
||||
logger.info("Stopping webmention worker")
|
||||
self._stop_event.set()
|
||||
|
||||
def get_stats(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Get current worker statistics
|
||||
|
||||
Returns:
|
||||
Dict with worker stats for health check
|
||||
"""
|
||||
try:
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Count pending validations
|
||||
cursor.execute("""
|
||||
SELECT COUNT(*) FROM webmentions
|
||||
WHERE validation_status = 'pending'
|
||||
""")
|
||||
pending_validations = cursor.fetchone()[0]
|
||||
|
||||
# Count pending sends (Phase 3)
|
||||
cursor.execute("""
|
||||
SELECT COUNT(*) FROM webmention_queue
|
||||
WHERE status = 'pending'
|
||||
""")
|
||||
pending_sends = cursor.fetchone()[0]
|
||||
|
||||
conn.close()
|
||||
|
||||
return {
|
||||
'running': self.is_alive(),
|
||||
'last_processed': self._last_processed.isoformat() if self._last_processed else None,
|
||||
'pending_validations': pending_validations,
|
||||
'pending_sends': pending_sends
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get worker stats: {e}")
|
||||
return {
|
||||
'running': self.is_alive(),
|
||||
'error': str(e)
|
||||
}
|
||||
@@ -191,3 +191,265 @@ small { display: block; margin-top: var(--spacing-xs); color: var(--color-text-l
|
||||
aspect-ratio: 16 / 9; /* Wider aspect on mobile */
|
||||
}
|
||||
}
|
||||
|
||||
/* Webmention Styles (v1.6.0 Phase 5) */
|
||||
|
||||
/* Reply Context */
|
||||
.reply-context {
|
||||
margin-bottom: var(--spacing-md);
|
||||
padding: var(--spacing-sm);
|
||||
border-left: 3px solid var(--color-link);
|
||||
background: var(--color-bg-alt);
|
||||
}
|
||||
|
||||
.reply-context .reply-label {
|
||||
font-size: 0.875rem;
|
||||
color: var(--color-text-light);
|
||||
display: block;
|
||||
margin-bottom: var(--spacing-xs);
|
||||
}
|
||||
|
||||
.reply-context blockquote {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.reply-context .p-content {
|
||||
font-style: italic;
|
||||
margin-bottom: var(--spacing-xs);
|
||||
}
|
||||
|
||||
/* Webmentions Section */
|
||||
.webmentions {
|
||||
margin-top: var(--spacing-lg);
|
||||
padding-top: var(--spacing-lg);
|
||||
border-top: 1px solid var(--color-border);
|
||||
}
|
||||
|
||||
.webmentions h2 {
|
||||
margin-bottom: var(--spacing-md);
|
||||
}
|
||||
|
||||
.webmention-group {
|
||||
margin-bottom: var(--spacing-lg);
|
||||
}
|
||||
|
||||
.webmention-group h3 {
|
||||
font-size: 1rem;
|
||||
margin-bottom: var(--spacing-sm);
|
||||
color: var(--color-text-light);
|
||||
}
|
||||
|
||||
/* Reply Webmentions */
|
||||
.webmention-reply {
|
||||
margin-bottom: var(--spacing-md);
|
||||
padding: var(--spacing-sm);
|
||||
background: var(--color-bg-alt);
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
.webmention-author {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: var(--spacing-sm);
|
||||
margin-bottom: var(--spacing-xs);
|
||||
}
|
||||
|
||||
.webmention-author img {
|
||||
border-radius: 50%;
|
||||
}
|
||||
|
||||
.webmention-author a {
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
.webmention-date {
|
||||
font-size: 0.875rem;
|
||||
color: var(--color-text-light);
|
||||
}
|
||||
|
||||
/* Facepile */
|
||||
.facepile {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.facepile .face {
|
||||
display: block;
|
||||
width: 32px;
|
||||
height: 32px;
|
||||
border-radius: 50%;
|
||||
overflow: hidden;
|
||||
border: 2px solid var(--color-bg);
|
||||
margin-left: -8px;
|
||||
}
|
||||
|
||||
.facepile .face:first-child {
|
||||
margin-left: 0;
|
||||
}
|
||||
|
||||
.facepile .face img {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
object-fit: cover;
|
||||
}
|
||||
|
||||
.facepile .face-initial {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
font-size: 14px;
|
||||
font-weight: bold;
|
||||
color: white;
|
||||
}
|
||||
|
||||
.facepile .face-overflow {
|
||||
margin-left: var(--spacing-xs);
|
||||
font-size: 0.875rem;
|
||||
color: var(--color-text-light);
|
||||
}
|
||||
|
||||
/* Generic Mentions */
|
||||
.webmention-mention {
|
||||
margin-bottom: var(--spacing-sm);
|
||||
padding: var(--spacing-xs);
|
||||
background: var(--color-bg-alt);
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
.webmention-mention p {
|
||||
margin: var(--spacing-xs) 0 0 0;
|
||||
font-size: 0.875rem;
|
||||
color: var(--color-text-light);
|
||||
}
|
||||
|
||||
/* Admin Webmention Styles */
|
||||
|
||||
.filter-tabs {
|
||||
display: flex;
|
||||
gap: var(--spacing-sm);
|
||||
margin-bottom: var(--spacing-md);
|
||||
border-bottom: 1px solid var(--color-border);
|
||||
}
|
||||
|
||||
.filter-tabs a {
|
||||
padding: var(--spacing-sm) var(--spacing-md);
|
||||
text-decoration: none;
|
||||
color: var(--color-text-light);
|
||||
border-bottom: 2px solid transparent;
|
||||
}
|
||||
|
||||
.filter-tabs a.active {
|
||||
color: var(--color-link);
|
||||
border-bottom-color: var(--color-link);
|
||||
}
|
||||
|
||||
.bulk-actions {
|
||||
display: flex;
|
||||
gap: var(--spacing-sm);
|
||||
margin-bottom: var(--spacing-md);
|
||||
}
|
||||
|
||||
.webmention-table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
}
|
||||
|
||||
.webmention-table th,
|
||||
.webmention-table td {
|
||||
padding: var(--spacing-sm);
|
||||
text-align: left;
|
||||
border-bottom: 1px solid var(--color-border);
|
||||
}
|
||||
|
||||
.webmention-table th {
|
||||
background: var(--color-bg-alt);
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
.badge {
|
||||
display: inline-block;
|
||||
padding: 0.25rem 0.5rem;
|
||||
font-size: 0.75rem;
|
||||
border-radius: 4px;
|
||||
background: var(--color-bg-alt);
|
||||
}
|
||||
|
||||
.badge-reply { background: #DBEAFE; color: #1E40AF; }
|
||||
.badge-like { background: #FECACA; color: #991B1B; }
|
||||
.badge-repost { background: #D1FAE5; color: #065F46; }
|
||||
.badge-bookmark { background: #FEF3C7; color: #92400E; }
|
||||
.badge-mention { background: #E5E7EB; color: #374151; }
|
||||
|
||||
.status-badge {
|
||||
display: inline-block;
|
||||
padding: 0.25rem 0.5rem;
|
||||
font-size: 0.75rem;
|
||||
border-radius: 4px;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.status-pending { background: #FEF3C7; color: #92400E; }
|
||||
.status-approved { background: #D1FAE5; color: #065F46; }
|
||||
.status-hidden { background: #E5E7EB; color: #6B7280; }
|
||||
.status-valid { background: #D1FAE5; color: #065F46; }
|
||||
.status-invalid { background: #FECACA; color: #991B1B; }
|
||||
.status-error { background: #FECACA; color: #991B1B; }
|
||||
|
||||
.pagination {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: var(--spacing-md);
|
||||
justify-content: center;
|
||||
margin-top: var(--spacing-lg);
|
||||
}
|
||||
|
||||
.detail-card {
|
||||
background: var(--color-bg-alt);
|
||||
padding: var(--spacing-md);
|
||||
border-radius: 4px;
|
||||
margin-bottom: var(--spacing-lg);
|
||||
}
|
||||
|
||||
.detail-row {
|
||||
margin-bottom: var(--spacing-md);
|
||||
}
|
||||
|
||||
.detail-row strong {
|
||||
display: block;
|
||||
margin-bottom: var(--spacing-xs);
|
||||
color: var(--color-text-light);
|
||||
}
|
||||
|
||||
.content-preview {
|
||||
padding: var(--spacing-sm);
|
||||
background: white;
|
||||
border: 1px solid var(--color-border);
|
||||
border-radius: 4px;
|
||||
margin-top: var(--spacing-xs);
|
||||
}
|
||||
|
||||
.moderation-actions {
|
||||
margin-top: var(--spacing-lg);
|
||||
}
|
||||
|
||||
.moderation-actions h3 {
|
||||
margin-bottom: var(--spacing-sm);
|
||||
}
|
||||
|
||||
.error-text {
|
||||
color: #991B1B;
|
||||
}
|
||||
|
||||
.empty-state {
|
||||
text-align: center;
|
||||
padding: var(--spacing-lg);
|
||||
color: var(--color-text-light);
|
||||
}
|
||||
|
||||
.empty-state .text-muted {
|
||||
font-size: 0.875rem;
|
||||
}
|
||||
@@ -10,6 +10,14 @@
|
||||
<a href="{{ url_for('admin.new_note_form') }}" class="button button-primary">
|
||||
+ New Note
|
||||
</a>
|
||||
{% if config.WEBMENTION_ENABLED %}
|
||||
<a href="{{ url_for('admin.webmentions') }}" class="button">
|
||||
Webmentions
|
||||
{% if webmention_pending_count > 0 %}
|
||||
<span class="badge">{{ webmention_pending_count }}</span>
|
||||
{% endif %}
|
||||
</a>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
{% if notes %}
|
||||
@@ -59,6 +67,20 @@
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
{# Pagination (v1.6.0) #}
|
||||
{% if total_pages > 1 %}
|
||||
<nav class="pagination">
|
||||
{% if page > 1 %}
|
||||
<a href="?page={{ page - 1 }}" class="button">Previous</a>
|
||||
{% endif %}
|
||||
<span>Page {{ page }} of {{ total_pages }}</span>
|
||||
{% if page < total_pages %}
|
||||
<a href="?page={{ page + 1 }}" class="button">Next</a>
|
||||
{% endif %}
|
||||
</nav>
|
||||
{% endif %}
|
||||
|
||||
{% else %}
|
||||
<div class="empty-state">
|
||||
<p>No notes yet. Create your first note!</p>
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
{% extends "admin/base.html" %}
|
||||
|
||||
{% block title %}Webmention Detail - StarPunk Admin{% endblock %}
|
||||
|
||||
{% block admin_content %}
|
||||
<div class="webmention-detail">
|
||||
<h2>Webmention Detail</h2>
|
||||
|
||||
<div class="dashboard-actions">
|
||||
<a href="{{ url_for('admin.webmentions') }}" class="button">
|
||||
Back to Webmentions
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<div class="detail-card">
|
||||
<div class="detail-row">
|
||||
<strong>Source:</strong>
|
||||
<a href="{{ webmention.source_url }}" target="_blank">{{ webmention.source_url }}</a>
|
||||
</div>
|
||||
|
||||
<div class="detail-row">
|
||||
<strong>Target:</strong>
|
||||
{% if webmention.target_note_id %}
|
||||
<a href="{{ url_for('public.note', slug=webmention.target_slug) }}">{{ webmention.target_url }}</a>
|
||||
{% else %}
|
||||
{{ webmention.target_url }} (Homepage)
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
<div class="detail-row">
|
||||
<strong>Type:</strong>
|
||||
<span class="badge badge-{{ webmention.mention_type }}">{{ webmention.mention_type }}</span>
|
||||
</div>
|
||||
|
||||
<div class="detail-row">
|
||||
<strong>Author:</strong>
|
||||
{{ webmention.author_name or 'Anonymous' }}
|
||||
{% if webmention.author_url %}
|
||||
(<a href="{{ webmention.author_url }}" target="_blank">{{ webmention.author_url }}</a>)
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
{% if webmention.author_photo %}
|
||||
<div class="detail-row">
|
||||
<strong>Author Photo:</strong>
|
||||
<img src="{{ webmention.author_photo }}" alt="{{ webmention.author_name or 'Author' }}" width="48" height="48" loading="lazy">
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{% if webmention.content_snippet %}
|
||||
<div class="detail-row">
|
||||
<strong>Content:</strong>
|
||||
<div class="content-preview">{{ webmention.content_snippet|safe }}</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<div class="detail-row">
|
||||
<strong>Validation Status:</strong>
|
||||
<span class="status-badge status-{{ webmention.validation_status }}">{{ webmention.validation_status }}</span>
|
||||
{% if webmention.validation_error %}
|
||||
<br><small class="error-text">{{ webmention.validation_error }}</small>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
<div class="detail-row">
|
||||
<strong>Moderation Status:</strong>
|
||||
<span class="status-badge status-{{ webmention.moderation_status }}">{{ webmention.moderation_status }}</span>
|
||||
</div>
|
||||
|
||||
<div class="detail-row">
|
||||
<strong>Received:</strong>
|
||||
{{ webmention.received_at.strftime('%B %d, %Y at %I:%M %p') }}
|
||||
</div>
|
||||
|
||||
{% if webmention.validated_at %}
|
||||
<div class="detail-row">
|
||||
<strong>Validated:</strong>
|
||||
{{ webmention.validated_at.strftime('%B %d, %Y at %I:%M %p') }}
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
{# Moderation Actions #}
|
||||
<div class="moderation-actions">
|
||||
<h3>Actions</h3>
|
||||
<form method="POST" action="{{ url_for('admin.webmention_action', id=webmention.id) }}" style="display: inline;">
|
||||
<input type="hidden" name="action" value="approve">
|
||||
<button type="submit" class="button button-primary" {% if webmention.moderation_status == 'approved' %}disabled{% endif %}>
|
||||
Approve
|
||||
</button>
|
||||
</form>
|
||||
|
||||
<form method="POST" action="{{ url_for('admin.webmention_action', id=webmention.id) }}" style="display: inline;">
|
||||
<input type="hidden" name="action" value="hide">
|
||||
<button type="submit" class="button" {% if webmention.moderation_status == 'hidden' %}disabled{% endif %}>
|
||||
Hide
|
||||
</button>
|
||||
</form>
|
||||
|
||||
<form method="POST" action="{{ url_for('admin.webmention_action', id=webmention.id) }}" style="display: inline;">
|
||||
<input type="hidden" name="action" value="delete">
|
||||
<button type="submit" class="button button-danger" onclick="return confirm('Delete this webmention permanently?')">
|
||||
Delete
|
||||
</button>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,125 @@
|
||||
{% extends "admin/base.html" %}
|
||||
|
||||
{% block title %}Webmentions - StarPunk Admin{% endblock %}
|
||||
|
||||
{% block admin_content %}
|
||||
<div class="webmentions-admin">
|
||||
<h2>Webmentions</h2>
|
||||
|
||||
<div class="dashboard-actions">
|
||||
<a href="{{ url_for('admin.dashboard') }}" class="button">
|
||||
Back to Dashboard
|
||||
</a>
|
||||
</div>
|
||||
|
||||
{# Filter tabs #}
|
||||
<div class="filter-tabs">
|
||||
<a href="{{ url_for('admin.webmentions') }}" class="{% if not status_filter %}active{% endif %}">
|
||||
All
|
||||
</a>
|
||||
<a href="{{ url_for('admin.webmentions', status='pending') }}" class="{% if status_filter == 'pending' %}active{% endif %}">
|
||||
Pending
|
||||
</a>
|
||||
<a href="{{ url_for('admin.webmentions', status='approved') }}" class="{% if status_filter == 'approved' %}active{% endif %}">
|
||||
Approved
|
||||
</a>
|
||||
<a href="{{ url_for('admin.webmentions', status='hidden') }}" class="{% if status_filter == 'hidden' %}active{% endif %}">
|
||||
Hidden
|
||||
</a>
|
||||
</div>
|
||||
|
||||
{% if webmentions %}
|
||||
<form method="POST" action="{{ url_for('admin.webmention_bulk_action') }}">
|
||||
<div class="bulk-actions">
|
||||
<button type="submit" name="action" value="approve" class="button">
|
||||
Approve Selected
|
||||
</button>
|
||||
<button type="submit" name="action" value="hide" class="button">
|
||||
Hide Selected
|
||||
</button>
|
||||
<button type="submit" name="action" value="delete" class="button button-danger"
|
||||
onclick="return confirm('Delete selected webmentions permanently?')">
|
||||
Delete Selected
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<table class="webmention-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th><input type="checkbox" id="select-all"></th>
|
||||
<th>Source</th>
|
||||
<th>Target</th>
|
||||
<th>Type</th>
|
||||
<th>Author</th>
|
||||
<th>Status</th>
|
||||
<th>Received</th>
|
||||
<th>Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for wm in webmentions %}
|
||||
<tr>
|
||||
<td><input type="checkbox" name="ids" value="{{ wm.id }}"></td>
|
||||
<td><a href="{{ wm.source_url }}" target="_blank">{{ wm.source_url|truncate(40) }}</a></td>
|
||||
<td>
|
||||
{% if wm.target_note_id %}
|
||||
<a href="{{ url_for('public.note', slug=wm.target_slug) }}">{{ wm.target_slug }}</a>
|
||||
{% else %}
|
||||
Homepage
|
||||
{% endif %}
|
||||
</td>
|
||||
<td><span class="badge badge-{{ wm.mention_type }}">{{ wm.mention_type }}</span></td>
|
||||
<td>{{ wm.author_name or 'Anonymous' }}</td>
|
||||
<td>
|
||||
<span class="status-badge status-{{ wm.moderation_status }}">
|
||||
{{ wm.moderation_status }}
|
||||
</span>
|
||||
</td>
|
||||
<td>{{ wm.received_at.strftime('%b %d, %Y %H:%M') }}</td>
|
||||
<td>
|
||||
<a href="{{ url_for('admin.webmention_detail', id=wm.id) }}" class="button button-small">
|
||||
View
|
||||
</a>
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</form>
|
||||
|
||||
{# Pagination #}
|
||||
{% if total_pages > 1 %}
|
||||
<nav class="pagination">
|
||||
{% if page > 1 %}
|
||||
<a href="?page={{ page - 1 }}{% if status_filter %}&status={{ status_filter }}{% endif %}" class="button">Previous</a>
|
||||
{% endif %}
|
||||
<span>Page {{ page }} of {{ total_pages }}</span>
|
||||
{% if page < total_pages %}
|
||||
<a href="?page={{ page + 1 }}{% if status_filter %}&status={{ status_filter }}{% endif %}" class="button">Next</a>
|
||||
{% endif %}
|
||||
</nav>
|
||||
{% endif %}
|
||||
|
||||
{% else %}
|
||||
{# Empty states #}
|
||||
{% if status_filter %}
|
||||
<div class="empty-state">
|
||||
<p>No webmentions match the current filters.</p>
|
||||
<a href="{{ url_for('admin.webmentions') }}" class="button">Clear Filters</a>
|
||||
</div>
|
||||
{% else %}
|
||||
<div class="empty-state">
|
||||
<p>No webmentions received yet.</p>
|
||||
<p class="text-muted">They will appear here when other sites mention your content.</p>
|
||||
</div>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
<script>
|
||||
// Select All functionality
|
||||
document.getElementById('select-all')?.addEventListener('change', function() {
|
||||
document.querySelectorAll('input[name="ids"]').forEach(cb => cb.checked = this.checked);
|
||||
});
|
||||
</script>
|
||||
{% endblock %}
|
||||
+8
-1
@@ -8,6 +8,11 @@
|
||||
<link rel="alternate" type="application/rss+xml" title="{{ config.SITE_NAME }} RSS Feed" href="{{ url_for('public.feed', _external=True) }}">
|
||||
<link rel="alternate" type="application/xml+opml" title="{{ config.SITE_NAME }} Feed Subscription List" href="{{ url_for('public.opml', _external=True) }}">
|
||||
|
||||
{# Webmention endpoint (v1.6.0) #}
|
||||
{% if config.WEBMENTION_ENABLED %}
|
||||
<link rel="webmention" href="/webmention">
|
||||
{% endif %}
|
||||
|
||||
{# rel-me links from discovered author profile (v1.2.0 Phase 2) #}
|
||||
{% if author and author.rel_me_links %}
|
||||
{% for profile_url in author.rel_me_links %}
|
||||
@@ -30,7 +35,9 @@
|
||||
<a href="/">Home</a>
|
||||
<a href="{{ url_for('public.feed') }}">RSS</a>
|
||||
{% if g.me %}
|
||||
<a href="{{ url_for('admin.dashboard') }}">Admin</a>
|
||||
<a href="{{ url_for('admin.dashboard') }}">Dashboard</a>
|
||||
{% else %}
|
||||
<a href="{{ url_for('auth.login_form') }}">Login</a>
|
||||
{% endif %}
|
||||
<form action="/search" method="get" role="search" style="margin-left: auto; display: flex; gap: var(--spacing-sm);">
|
||||
<input
|
||||
|
||||
@@ -13,6 +13,31 @@
|
||||
<h1 class="p-name">{{ note.title }}</h1>
|
||||
{% endif %}
|
||||
|
||||
{# Reply context (v1.6.0 Phase 4) - displayed if this is a reply post #}
|
||||
{% if note.in_reply_to %}
|
||||
<div class="reply-context">
|
||||
<span class="reply-label">In reply to:</span>
|
||||
<blockquote class="h-cite">
|
||||
{% if reply_context and reply_context.fetch_status == 'fetched' %}
|
||||
{% if reply_context.content_snippet %}
|
||||
<p class="p-content">{{ reply_context.content_snippet }}</p>
|
||||
{% endif %}
|
||||
<cite>
|
||||
<a class="u-in-reply-to" href="{{ note.in_reply_to }}">
|
||||
{% if reply_context.author_name %}{{ reply_context.author_name }}{% endif %}
|
||||
{% if reply_context.site_name %} on {{ reply_context.site_name }}{% endif %}
|
||||
{% if not reply_context.author_name and not reply_context.site_name %}{{ note.in_reply_to }}{% endif %}
|
||||
</a>
|
||||
</cite>
|
||||
{% else %}
|
||||
<cite>
|
||||
<a class="u-in-reply-to" href="{{ note.in_reply_to }}">{{ note.in_reply_to }}</a>
|
||||
</cite>
|
||||
{% endif %}
|
||||
</blockquote>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{# u-photo placement: Per draft spec, u-photo must be direct child of h-entry, #}
|
||||
{# NOT inside e-content. Media is rendered ABOVE e-content to meet this requirement. #}
|
||||
{{ display_media(note.media) }}
|
||||
@@ -68,5 +93,118 @@
|
||||
<nav class="note-nav">
|
||||
<a href="/">Back to all notes</a>
|
||||
</nav>
|
||||
|
||||
{# Webmentions section (v1.6.0 Phase 5) - only shown if approved webmentions exist #}
|
||||
{% if webmentions %}
|
||||
<section class="webmentions">
|
||||
<h2>Responses</h2>
|
||||
|
||||
{% if replies %}
|
||||
<div class="webmention-group webmention-replies">
|
||||
<h3>{{ replies|length }} {{ 'Reply' if replies|length == 1 else 'Replies' }}</h3>
|
||||
{% for reply in replies %}
|
||||
<article class="webmention webmention-reply h-cite">
|
||||
<div class="webmention-author">
|
||||
{% if reply.author_photo %}
|
||||
<img src="{{ reply.author_photo }}" alt="" width="48" height="48" loading="lazy">
|
||||
{% endif %}
|
||||
<a class="p-author" href="{{ reply.author_url or reply.source_url }}">
|
||||
{{ reply.author_name or 'Anonymous' }}
|
||||
</a>
|
||||
</div>
|
||||
{% if reply.content_snippet %}
|
||||
<div class="p-content">{{ reply.content_snippet|safe }}</div>
|
||||
{% endif %}
|
||||
<a class="u-url webmention-date" href="{{ reply.source_url }}">
|
||||
<time class="dt-published" datetime="{{ reply.received_at.isoformat() }}">
|
||||
{{ reply.received_at.strftime('%B %d, %Y') }}
|
||||
</time>
|
||||
</a>
|
||||
</article>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{% if likes %}
|
||||
<div class="webmention-group webmention-likes">
|
||||
<h3>{{ likes|length }} {{ 'Like' if likes|length == 1 else 'Likes' }}</h3>
|
||||
<div class="facepile">
|
||||
{% for wm in likes[:8] %}
|
||||
<a href="{{ wm.author_url or wm.source_url }}" class="face" title="{{ wm.author_name or 'Anonymous' }}">
|
||||
{% if wm.author_photo %}
|
||||
<img src="{{ wm.author_photo }}" alt="{{ wm.author_name or '' }}" width="32" height="32" loading="lazy">
|
||||
{% else %}
|
||||
<span class="face-initial" style="background-color: {{ wm.author_url|url_to_color }}">
|
||||
{{ (wm.author_name or '?')[0]|upper }}
|
||||
</span>
|
||||
{% endif %}
|
||||
</a>
|
||||
{% endfor %}
|
||||
{% if likes|length > 8 %}
|
||||
<span class="face-overflow">+{{ likes|length - 8 }} others</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{% if reposts %}
|
||||
<div class="webmention-group webmention-reposts">
|
||||
<h3>{{ reposts|length }} {{ 'Repost' if reposts|length == 1 else 'Reposts' }}</h3>
|
||||
<div class="facepile">
|
||||
{% for wm in reposts[:8] %}
|
||||
<a href="{{ wm.author_url or wm.source_url }}" class="face" title="{{ wm.author_name or 'Anonymous' }}">
|
||||
{% if wm.author_photo %}
|
||||
<img src="{{ wm.author_photo }}" alt="{{ wm.author_name or '' }}" width="32" height="32" loading="lazy">
|
||||
{% else %}
|
||||
<span class="face-initial" style="background-color: {{ wm.author_url|url_to_color }}">
|
||||
{{ (wm.author_name or '?')[0]|upper }}
|
||||
</span>
|
||||
{% endif %}
|
||||
</a>
|
||||
{% endfor %}
|
||||
{% if reposts|length > 8 %}
|
||||
<span class="face-overflow">+{{ reposts|length - 8 }} others</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{% if bookmarks %}
|
||||
<div class="webmention-group webmention-bookmarks">
|
||||
<h3>{{ bookmarks|length }} {{ 'Bookmark' if bookmarks|length == 1 else 'Bookmarks' }}</h3>
|
||||
<div class="facepile">
|
||||
{% for wm in bookmarks[:8] %}
|
||||
<a href="{{ wm.author_url or wm.source_url }}" class="face" title="{{ wm.author_name or 'Anonymous' }}">
|
||||
{% if wm.author_photo %}
|
||||
<img src="{{ wm.author_photo }}" alt="{{ wm.author_name or '' }}" width="32" height="32" loading="lazy">
|
||||
{% else %}
|
||||
<span class="face-initial" style="background-color: {{ wm.author_url|url_to_color }}">
|
||||
{{ (wm.author_name or '?')[0]|upper }}
|
||||
</span>
|
||||
{% endif %}
|
||||
</a>
|
||||
{% endfor %}
|
||||
{% if bookmarks|length > 8 %}
|
||||
<span class="face-overflow">+{{ bookmarks|length - 8 }} others</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{% if mentions %}
|
||||
<div class="webmention-group webmention-mentions">
|
||||
<h3>{{ mentions|length }} {{ 'Mention' if mentions|length == 1 else 'Mentions' }}</h3>
|
||||
{% for mention in mentions %}
|
||||
<div class="webmention webmention-mention">
|
||||
<a href="{{ mention.source_url }}">{{ mention.author_name or mention.source_url }}</a>
|
||||
{% if mention.content_snippet %}
|
||||
<p>{{ mention.content_snippet|truncate(200) }}</p>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% endif %}
|
||||
</section>
|
||||
{% endif %}
|
||||
</article>
|
||||
{% endblock %}
|
||||
Reference in new issue
Block a user