Files
StarPunk/starpunk/webmention_worker.py
T
PhilandClaude Opus 5.5 438465592c feat: Microformats2 review fixes (v2.8.0)
- Responses as u-comment/u-like/u-repost/u-bookmark/u-mention h-cites
  with author h-cards; source's own url and published (migration 014)
- dt-* and Micropub published in RFC 3339 UTC
- Title only for a real Markdown heading (hashtag notes stay notes)
- Ignore webmentions from a note's own syndicated copies
- Micropub/IndieAuth links independent of Webmention; rel=me/rel=author
- Representative h-card and authorship algorithm (starpunk/mf2_utils.py)
- Tag/likes h-feeds get url and author; previews get u-uid

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-03 09:36:49 -07:00

885 lines
31 KiB
Python

"""
Background worker thread for webmention processing
Per background-worker.md:
- Daemon thread that processes webmention queues
- Handles both incoming validation and outgoing sending
- Graceful shutdown on app termination
- Orphan detection and cleanup on startup
References:
- Background Worker Design: /docs/design/v1.6.0/background-worker.md
- ADR-063: Webmention Implementation
"""
import fcntl
import logging
import re
import threading
import time
import sqlite3
import httpx
from datetime import datetime, timedelta
from pathlib import Path
from typing import Dict, Any, Optional
from urllib.parse import urljoin
from lxml import html as lxml_html
from starpunk.webmention_processor import process_webmention, TransientError, ValidationError
from starpunk.webmention_sender import note_responses_changed
from starpunk.mf2_utils import normalize_url
from starpunk.webmention_ssrf import (
FetchTooLargeError, SSRFBlockedError, safe_get, safe_post, user_agent,
)
logger = logging.getLogger(__name__)
# Link header (RFC 8288): <uri> then ;-separated params; quoted values may
# contain , and ;
LINK_RE = re.compile(r'<([^>]*)>((?:\s*;\s*[^;,"=\s]+\s*(?:=\s*(?:"[^"]*"|[^;,]*))?)*)')
LINK_PARAM_RE = re.compile(r';\s*([^;,"=\s]+)\s*(?:=\s*(?:"([^"]*)"|([^;,]*)))?')
class WebmentionWorker(threading.Thread):
"""
Background thread for webmention queue processing
Follows MemoryMonitor pattern: daemon thread with stop event
"""
def __init__(
self,
app,
db_path: Path,
site_url: str,
poll_interval: int = 10,
error_backoff: int = 60,
batch_size: int = 5,
auto_approve: bool = True
):
"""
Initialize webmention worker thread
Args:
app: Flask app (the SSRF checks read its config)
db_path: Path to SQLite database
site_url: Site URL for User-Agent header
poll_interval: Seconds between queue checks (default: 10)
error_backoff: Seconds to wait after error (default: 60)
batch_size: Items to process per iteration (default: 5)
auto_approve: Auto-approve valid webmentions (default: True)
"""
super().__init__(daemon=True, name="webmention-worker")
self.app = app
self.db_path = db_path
self.site_url = site_url
self.poll_interval = poll_interval
self.error_backoff = error_backoff
self.batch_size = batch_size
self.auto_approve = auto_approve
self._stop_event = threading.Event()
self._last_processed = None
self._lock_file = None
def run(self):
"""
Main processing loop
Per background-worker.md:
1. Clean up orphaned items on startup
2. Process pending validations (incoming)
3. Process pending sends (outgoing) - Phase 3
4. Sleep if nothing to process
"""
try:
logger.info("Webmention worker started")
# Main processing loop
while not self._stop_event.is_set():
# Every gunicorn process starts a worker; only the one holding
# the lock processes queues, so nothing is sent twice
if not self._acquire_lock():
self._stop_event.wait(self.poll_interval)
continue
try:
processed = 0
with self.app.app_context():
# Process incoming webmentions
processed += self._process_pending_validations()
# Phase 3: Process outgoing webmentions
processed += self._process_pending_sends()
# Update last processed time
if processed > 0:
self._last_processed = datetime.utcnow()
# Sleep if nothing processed
if processed == 0:
self._stop_event.wait(self.poll_interval)
except Exception as e:
logger.error(f"Worker error: {e}", exc_info=True)
self._stop_event.wait(self.error_backoff)
except Exception as e:
logger.error(f"Webmention worker thread failed: {e}", exc_info=True)
finally:
self._release_lock()
logger.info("Webmention worker stopped")
def _acquire_lock(self) -> bool:
"""
Take the cross-process worker lock (non-blocking)
On first acquisition, also reset items left mid-send by a previous
holder. The OS releases the lock if this process dies.
Returns:
True if this worker holds the lock
"""
if self._lock_file:
return True
lock_file = open(Path(self.db_path).parent / "webmention-worker.lock", "w")
try:
fcntl.flock(lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB)
except OSError:
lock_file.close()
return False
self._lock_file = lock_file
logger.info("Webmention worker holds the queue lock")
self._cleanup_orphaned_items()
return True
def _release_lock(self):
"""Release the cross-process worker lock if held"""
if self._lock_file:
self._lock_file.close()
self._lock_file = None
def _cleanup_orphaned_items(self):
"""
Clean up orphaned 'sending' status items on startup
Per background-worker.md:
Reset items stuck in 'sending' status from previous crashes
"""
try:
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
# Reset orphaned outgoing webmentions
cursor.execute("""
UPDATE webmention_queue
SET status = 'pending',
started_at = NULL
WHERE status = 'sending'
AND started_at < datetime('now', '-5 minutes')
""")
orphaned = cursor.rowcount
conn.commit()
conn.close()
if orphaned > 0:
logger.info(f"Reset {orphaned} orphaned webmention queue items")
except Exception as e:
logger.error(f"Failed to cleanup orphaned items: {e}")
def _process_pending_validations(self) -> int:
"""
Process pending incoming webmentions
Returns:
Number of webmentions processed
"""
processed = 0
try:
conn = sqlite3.connect(self.db_path)
conn.row_factory = sqlite3.Row
cursor = conn.cursor()
# Get pending webmentions
cursor.execute("""
SELECT id, source_url, target_url, target_note_id
FROM webmentions
WHERE validation_status = 'pending'
ORDER BY received_at ASC
LIMIT ?
""", (self.batch_size,))
pending = cursor.fetchall()
conn.close()
# Process each webmention
for row in pending:
try:
self._validate_webmention(
row['id'],
row['source_url'],
row['target_url'],
row['target_note_id']
)
processed += 1
except Exception as e:
logger.error(
f"Failed to validate webmention {row['id']}: {e}",
exc_info=True
)
except Exception as e:
logger.error(f"Error fetching pending validations: {e}")
return processed
def _validate_webmention(
self,
webmention_id: int,
source_url: str,
target_url: str,
target_note_id: Optional[int]
):
"""
Validate a webmention, then re-send the target note's webmentions if
what the note shows changed (spec 3.1.4: displayed responses are an
update of the post). Unchanged re-validations send nothing, so two
sites doing this don't ping-pong.
"""
before = self._display_state(webmention_id)
try:
self._validate_and_store(webmention_id, source_url, target_url, target_note_id)
finally:
after = self._display_state(webmention_id)
if before != after and target_note_id is not None:
try:
note_responses_changed(self.db_path, [target_note_id])
except Exception as e:
logger.warning(f"Failed to re-send webmentions for note {target_note_id}: {e}")
def _is_own_syndicated_copy(self, note_id: Optional[int], source_url: str,
result: Dict[str, Any]) -> bool:
"""True if the source is one of the note's own u-syndication copies"""
if note_id is None:
return False
conn = sqlite3.connect(self.db_path)
try:
copies = {normalize_url(row[0]) for row in conn.execute(
"SELECT remote_url FROM syndications WHERE note_id = ? AND remote_url IS NOT NULL",
(note_id,),
)}
finally:
conn.close()
source = {normalize_url(u) for u in (
source_url, result.get('source_post_url'), result.get('source_uid')) if u}
return bool(copies & source)
def _display_state(self, webmention_id: int) -> Optional[tuple]:
"""
What the note page shows for this webmention, or None if hidden
A row reset to 'pending' by a re-send still counts as shown if it was
valid before (validated with no error), as it was until now.
"""
conn = sqlite3.connect(self.db_path)
try:
row = conn.execute("""
SELECT mention_type, author_name, author_url, author_photo, content_snippet
FROM webmentions
WHERE id = ?
AND moderation_status = 'approved'
AND (validation_status = 'valid'
OR (validation_status = 'pending' AND validated_at IS NOT NULL
AND validation_error IS NULL))
""", (webmention_id,)).fetchone()
finally:
conn.close()
return tuple(row) if row else None
def _validate_and_store(
self,
webmention_id: int,
source_url: str,
target_url: str,
target_note_id: Optional[int]
):
"""
Validate a single webmention
Args:
webmention_id: Database ID of webmention
source_url: Source URL
target_url: Target URL
target_note_id: Target note ID (if applicable)
"""
try:
# Process webmention (fetch, validate, extract)
result = process_webmention(source_url, target_url, self.site_url)
# A copy of this very note on a silo (e.g. backfed by Bridgy) is
# not a response to it
if self._is_own_syndicated_copy(target_note_id, source_url, result):
raise ValidationError("Source is this note's own syndicated copy")
# Update database with successful validation
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
moderation_status = 'approved' if self.auto_approve else 'pending'
# An update to an existing mention (spec 3.2.4) keeps the admin's
# moderation decision; only a first-time valid mention gets the
# default. Previously valid = validated with no error.
cursor.execute("""
UPDATE webmentions
SET moderation_status = CASE
WHEN moderation_status IN ('hidden', 'deleted') THEN moderation_status
WHEN validated_at IS NOT NULL AND validation_error IS NULL
THEN moderation_status
ELSE ?
END,
validation_status = 'valid',
validation_error = NULL,
validated_at = CURRENT_TIMESTAMP,
mention_type = ?,
author_name = ?,
author_url = ?,
author_photo = ?,
content_snippet = ?,
source_post_url = ?,
source_published = ?,
updated_at = CURRENT_TIMESTAMP
WHERE id = ?
""", (
moderation_status,
result['mention_type'],
result['author_name'],
result['author_url'],
result['author_photo'],
result['content_snippet'],
result.get('source_post_url'),
result.get('source_published'),
webmention_id
))
conn.commit()
conn.close()
logger.info(
f"Validated webmention: {source_url} -> {target_url} "
f"(type={result['mention_type']})"
)
except TransientError as e:
# Source unreachable right now: keep a previously valid mention
self._mark_validation_failed(webmention_id, source_url, target_url, str(e),
transient=True)
except ValidationError as e:
# Validation failed (410/4xx, no link): the mention is removed
# from display (spec 3.2.4)
self._mark_validation_failed(webmention_id, source_url, target_url, str(e))
except Exception as e:
# Unexpected error - mark as error
self._mark_validation_failed(
webmention_id,
source_url,
target_url,
f"Unexpected error: {e}"
)
raise
def _mark_validation_failed(
self,
webmention_id: int,
source_url: str,
target_url: str,
error: str,
transient: bool = False
):
"""
Mark webmention validation as failed/invalid
A transient failure of a mention that was valid before restores it to
valid with its previous data, rather than hiding it.
Args:
webmention_id: Database ID
source_url: Source URL
target_url: Target URL
error: Error message
"""
# Determine if error or invalid
# Per background-worker.md: HTTP 4xx and "no link" are invalid
# All other errors are 'error' status
if transient:
status = 'error'
elif any(x in error.lower() for x in ['does not link', 'returned 4', 'own syndicated copy']):
status = 'invalid'
else:
status = 'error'
try:
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
if transient:
cursor.execute("""
UPDATE webmentions
SET validation_status = 'valid',
updated_at = CURRENT_TIMESTAMP
WHERE id = ? AND validated_at IS NOT NULL AND validation_error IS NULL
""", (webmention_id,))
if cursor.rowcount:
conn.commit()
conn.close()
logger.warning(
f"Could not re-check webmention, keeping it: "
f"{source_url} -> {target_url}: {error}"
)
return
cursor.execute("""
UPDATE webmentions
SET validation_status = ?,
validation_error = ?,
validated_at = CURRENT_TIMESTAMP,
updated_at = CURRENT_TIMESTAMP
WHERE id = ?
""", (status, error, webmention_id))
conn.commit()
conn.close()
logger.warning(
f"Invalid webmention: {source_url} -> {target_url}: {error}"
)
except Exception as e:
logger.error(f"Failed to mark webmention as failed: {e}")
def _process_pending_sends(self) -> int:
"""
Process pending outgoing webmentions (Phase 3)
Returns:
Number of webmentions processed
"""
processed = 0
try:
conn = sqlite3.connect(self.db_path)
conn.row_factory = sqlite3.Row
cursor = conn.cursor()
# Get pending webmentions ready to send
cursor.execute("""
SELECT id, source_url, target_url, endpoint_url, retry_count
FROM webmention_queue
WHERE status = 'pending'
AND (next_retry_at IS NULL OR next_retry_at <= datetime('now'))
ORDER BY created_at ASC
LIMIT ?
""", (self.batch_size,))
pending = cursor.fetchall()
conn.close()
# Process each webmention
for row in pending:
try:
self._send_webmention(
row['id'],
row['source_url'],
row['target_url'],
row['endpoint_url'],
row['retry_count']
)
processed += 1
except Exception as e:
logger.error(
f"Failed to send webmention {row['id']}: {e}",
exc_info=True
)
except Exception as e:
logger.error(f"Error fetching pending sends: {e}")
return processed
def _send_webmention(
self,
queue_id: int,
source_url: str,
target_url: str,
cached_endpoint: Optional[str],
retry_count: int
):
"""
Send a single outgoing webmention
Args:
queue_id: Database ID of queue item
source_url: Source URL (our note)
target_url: Target URL to notify
cached_endpoint: Previously discovered endpoint (if any)
retry_count: Current retry count
"""
# Mark as sending
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
cursor.execute("""
UPDATE webmention_queue
SET status = 'sending',
started_at = CURRENT_TIMESTAMP
WHERE id = ?
""", (queue_id,))
conn.commit()
conn.close()
try:
# Discover endpoint if not cached
endpoint = cached_endpoint
if not endpoint:
endpoint = self._discover_endpoint(target_url)
if not endpoint:
# No endpoint found - permanent failure
self._mark_send_no_endpoint(queue_id, source_url, target_url)
return
# Cache the discovered endpoint in database
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
cursor.execute("""
UPDATE webmention_queue
SET endpoint_url = ?
WHERE id = ?
""", (endpoint, queue_id))
conn.commit()
conn.close()
# Send webmention POST (spec 3.1.3)
try:
# SSRF-checked and pinned to the checked address (spec 4.3).
# Redirects are not followed: a 301/302 would turn the POST
# into a GET without the body
response = safe_post(
endpoint,
data={
'source': source_url,
'target': target_url
},
headers={'User-Agent': user_agent(self.site_url)},
)
# Any 2xx is success (spec 3.1.3)
if 200 <= response.status_code < 300:
self._mark_send_success(queue_id, source_url, target_url)
elif response.status_code == 429 or response.status_code >= 500:
# Rate limited or server error - retry
error = f"HTTP {response.status_code}: {response.text[:200]}"
self._mark_send_retry(queue_id, source_url, target_url, retry_count, error)
else:
# Permanent failure (3xx, 4xx)
error = f"HTTP {response.status_code}: {response.text[:200]}"
self._mark_send_failed(queue_id, source_url, target_url, error)
except (httpx.TimeoutException, httpx.HTTPError) as e:
# Connection error - retry
self._mark_send_retry(queue_id, source_url, target_url, retry_count, str(e))
except (SSRFBlockedError, ValueError) as e:
# SSRF blocked - permanent failure
logger.warning(f"SSRF blocked endpoint {endpoint}: {e}")
self._mark_send_failed(queue_id, source_url, target_url, f"Blocked: {e}")
except Exception as e:
# Discovery failed - retry
self._mark_send_retry(queue_id, source_url, target_url, retry_count, str(e))
def _discover_endpoint(self, target_url: str) -> Optional[str]:
"""
Discover the webmention endpoint for a target URL (spec 3.1.2)
- GET the target, following redirects (each hop SSRF-checked)
- The first Link header with rel=webmention wins, then the first
<link> or <a> with rel=webmention and an href, in document order
- rel is a space-separated token list, matched case-insensitively
- Relative endpoints resolve against the final URL; an empty href
is the target itself; query strings are kept
- Not cached: an updated post must re-discover (spec 3.1.4)
Returns:
Endpoint URL, or None if the target has none (or is blocked/4xx)
Raises:
httpx.HTTPError: Network errors and 5xx, so the caller retries
"""
try:
result = safe_get(target_url, headers={'User-Agent': user_agent(self.site_url)})
except (SSRFBlockedError, ValueError) as e:
logger.warning(f"SSRF protection blocked target {target_url}: {e}")
return None
except FetchTooLargeError:
logger.info(f"Target too large for discovery: {target_url}")
return None
if result.status_code >= 500:
raise httpx.HTTPError(f"Target returned {result.status_code}")
if result.status_code >= 400:
logger.info(f"Target returned {result.status_code}: {target_url}")
return None
for link_header in result.headers.get_list('Link'):
endpoint = self._parse_link_header(link_header, result.url)
if endpoint is not None:
return endpoint
content_type = result.headers.get('Content-Type', '')
if 'html' in content_type.lower():
endpoint = self._parse_html_for_endpoint(result.text, result.url)
if endpoint is not None:
return endpoint
logger.info(f"No webmention endpoint found for {target_url}")
return None
@staticmethod
def _has_webmention_rel(rel: str) -> bool:
return 'webmention' in rel.lower().split()
def _parse_link_header(self, link_header: str, base_url: str) -> Optional[str]:
"""
Parse a Link header (RFC 8288) for a rel=webmention link
Args:
link_header: Link header value (may hold several links)
base_url: Base URL for resolving relative URLs
Returns:
Absolute endpoint URL of the first match, None otherwise
"""
for match in LINK_RE.finditer(link_header):
uri, params = match.group(1), match.group(2)
for param in LINK_PARAM_RE.finditer(params):
if param.group(1).lower() != 'rel':
continue
rel = param.group(2) if param.group(2) is not None else (param.group(3) or '')
if self._has_webmention_rel(rel):
return urljoin(base_url, uri.strip())
break # only the first rel parameter counts (RFC 8288)
return None
def _parse_html_for_endpoint(self, html_content: str, base_url: str) -> Optional[str]:
"""
Find the first <link> or <a> with rel=webmention, in document order
Args:
html_content: HTML content
base_url: Base URL for resolving relative URLs
Returns:
Absolute endpoint URL if found, None otherwise
"""
try:
doc = lxml_html.document_fromstring(html_content)
except Exception as e:
logger.debug(f"HTML parsing failed: {e}")
return None
for element in doc.iter('link', 'a'):
href = element.get('href')
if href is not None and self._has_webmention_rel(element.get('rel', '')):
return urljoin(base_url, href.strip())
return None
def _mark_send_success(self, queue_id: int, source_url: str, target_url: str):
"""Mark webmention send as successful"""
try:
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
cursor.execute("""
UPDATE webmention_queue
SET status = 'sent',
sent_at = CURRENT_TIMESTAMP
WHERE id = ?
""", (queue_id,))
conn.commit()
conn.close()
logger.info(f"Sent webmention: {source_url} -> {target_url}")
except Exception as e:
logger.error(f"Failed to mark send success: {e}")
def _mark_send_failed(self, queue_id: int, source_url: str, target_url: str, error: str):
"""Mark webmention send as permanently failed"""
try:
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
cursor.execute("""
UPDATE webmention_queue
SET status = 'failed',
last_error = ?
WHERE id = ?
""", (error, queue_id))
conn.commit()
conn.close()
logger.warning(f"Webmention send failed permanently: {source_url} -> {target_url}: {error}")
except Exception as e:
logger.error(f"Failed to mark send failed: {e}")
def _mark_send_no_endpoint(self, queue_id: int, source_url: str, target_url: str):
"""Mark webmention as no endpoint found"""
try:
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
cursor.execute("""
UPDATE webmention_queue
SET status = 'no_endpoint'
WHERE id = ?
""", (queue_id,))
conn.commit()
conn.close()
logger.info(f"No webmention endpoint for {target_url}")
except Exception as e:
logger.error(f"Failed to mark no endpoint: {e}")
def _mark_send_retry(
self,
queue_id: int,
source_url: str,
target_url: str,
current_retry_count: int,
error: str
):
"""
Mark webmention for retry with exponential backoff
Per architect's answer to Q5 and background-worker.md:
- Increment retry_count AFTER failed attempt
- Check retry_count >= 4 AFTER incrementing
- Retry schedule: 1h, 6h, 24h
"""
new_retry_count = current_retry_count + 1
# Calculate next retry time
next_retry_at = None
if new_retry_count < 4:
delays = {
1: timedelta(hours=1),
2: timedelta(hours=6),
3: timedelta(hours=24),
}
delay = delays.get(new_retry_count)
if delay:
next_retry_at = datetime.utcnow() + delay
try:
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
if new_retry_count >= 4:
# Max retries reached - mark as failed
cursor.execute("""
UPDATE webmention_queue
SET status = 'failed',
retry_count = ?,
last_error = ?
WHERE id = ?
""", (new_retry_count, error, queue_id))
logger.warning(
f"Webmention send failed permanently after {new_retry_count} attempts: "
f"{source_url} -> {target_url}: {error}"
)
else:
# Schedule retry
cursor.execute("""
UPDATE webmention_queue
SET status = 'pending',
retry_count = ?,
next_retry_at = ?,
last_error = ?
WHERE id = ?
""", (new_retry_count, next_retry_at, error, queue_id))
logger.warning(
f"Webmention send failed, retry {new_retry_count}: "
f"{source_url} -> {target_url}: {error}"
)
conn.commit()
conn.close()
except Exception as e:
logger.error(f"Failed to mark send retry: {e}")
def stop(self):
"""
Stop the worker thread gracefully
Sets the stop event to signal the thread to exit
"""
logger.info("Stopping webmention worker")
self._stop_event.set()
def get_stats(self) -> Dict[str, Any]:
"""
Get current worker statistics
Returns:
Dict with worker stats for health check
"""
try:
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
# Count pending validations
cursor.execute("""
SELECT COUNT(*) FROM webmentions
WHERE validation_status = 'pending'
""")
pending_validations = cursor.fetchone()[0]
# Count pending sends (Phase 3)
cursor.execute("""
SELECT COUNT(*) FROM webmention_queue
WHERE status = 'pending'
""")
pending_sends = cursor.fetchone()[0]
conn.close()
return {
'running': self.is_alive(),
'last_processed': self._last_processed.isoformat() if self._last_processed else None,
'pending_validations': pending_validations,
'pending_sends': pending_sends
}
except Exception as e:
logger.error(f"Failed to get worker stats: {e}")
return {
'running': self.is_alive(),
'error': str(e)
}