Links to the site itself, and to self-hosted apps like Gitea, timed out from inside the container and sat in the admin Log. They are now skipped when queueing; already queued or failed ones are dropped at worker start. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
966 lines
34 KiB
Python
966 lines
34 KiB
Python
"""
|
|
Background worker thread for webmention processing
|
|
|
|
Per background-worker.md:
|
|
- Daemon thread that processes webmention queues
|
|
- Handles both incoming validation and outgoing sending
|
|
- Graceful shutdown on app termination
|
|
- Orphan detection and cleanup on startup
|
|
|
|
References:
|
|
- Background Worker Design: /docs/design/v1.6.0/background-worker.md
|
|
- ADR-063: Webmention Implementation
|
|
"""
|
|
|
|
import fcntl
|
|
import logging
|
|
import re
|
|
import threading
|
|
import time
|
|
import sqlite3
|
|
import httpx
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
from typing import Dict, Any, Optional
|
|
from urllib.parse import urljoin
|
|
from lxml import html as lxml_html
|
|
|
|
from starpunk.webmention_processor import process_webmention, TransientError, ValidationError
|
|
from starpunk.webmention_sender import note_responses_changed
|
|
from starpunk.mf2_utils import normalize_url
|
|
from starpunk.webmention_ssrf import (
|
|
FetchTooLargeError, SSRFBlockedError, safe_get, safe_post, user_agent,
|
|
)
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Link header (RFC 8288): <uri> then ;-separated params; quoted values may
|
|
# contain , and ;
|
|
LINK_RE = re.compile(r'<([^>]*)>((?:\s*;\s*[^;,"=\s]+\s*(?:=\s*(?:"[^"]*"|[^;,]*))?)*)')
|
|
LINK_PARAM_RE = re.compile(r';\s*([^;,"=\s]+)\s*(?:=\s*(?:"([^"]*)"|([^;,]*)))?')
|
|
|
|
|
|
|
|
class WebmentionWorker(threading.Thread):
|
|
"""
|
|
Background thread for webmention queue processing
|
|
|
|
Follows MemoryMonitor pattern: daemon thread with stop event
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
app,
|
|
db_path: Path,
|
|
site_url: str,
|
|
poll_interval: int = 10,
|
|
error_backoff: int = 60,
|
|
batch_size: int = 5,
|
|
auto_approve: bool = True
|
|
):
|
|
"""
|
|
Initialize webmention worker thread
|
|
|
|
Args:
|
|
app: Flask app (the SSRF checks read its config)
|
|
db_path: Path to SQLite database
|
|
site_url: Site URL for User-Agent header
|
|
poll_interval: Seconds between queue checks (default: 10)
|
|
error_backoff: Seconds to wait after error (default: 60)
|
|
batch_size: Items to process per iteration (default: 5)
|
|
auto_approve: Auto-approve valid webmentions (default: True)
|
|
"""
|
|
super().__init__(daemon=True, name="webmention-worker")
|
|
self.app = app
|
|
self.db_path = db_path
|
|
self.site_url = site_url
|
|
self.poll_interval = poll_interval
|
|
self.error_backoff = error_backoff
|
|
self.batch_size = batch_size
|
|
self.auto_approve = auto_approve
|
|
self._stop_event = threading.Event()
|
|
self._last_processed = None
|
|
self._lock_file = None
|
|
from starpunk.avatars import Upkeep
|
|
from starpunk.notifications import Sweeper
|
|
self._avatar_upkeep = Upkeep()
|
|
self._notifier = Sweeper()
|
|
|
|
def run(self):
|
|
"""
|
|
Main processing loop
|
|
|
|
Per background-worker.md:
|
|
1. Clean up orphaned items on startup
|
|
2. Process pending validations (incoming)
|
|
3. Process pending sends (outgoing) - Phase 3
|
|
4. Sleep if nothing to process
|
|
"""
|
|
try:
|
|
logger.info("Webmention worker started")
|
|
|
|
# Main processing loop
|
|
while not self._stop_event.is_set():
|
|
# Every gunicorn process starts a worker; only the one holding
|
|
# the lock processes queues, so nothing is sent twice
|
|
if not self._acquire_lock():
|
|
self._stop_event.wait(self.poll_interval)
|
|
continue
|
|
|
|
try:
|
|
processed = 0
|
|
|
|
with self.app.app_context():
|
|
# Process incoming webmentions
|
|
processed += self._process_pending_validations()
|
|
|
|
# Phase 3: Process outgoing webmentions
|
|
processed += self._process_pending_sends()
|
|
# Local copies of response avatars (v2.13.0)
|
|
self._avatar_upkeep.run(self.db_path, self.app.config['DATA_PATH'],
|
|
self.site_url)
|
|
# New responses to the owner (v2.14.0, ADR-074)
|
|
try:
|
|
self._notifier.run(self.app.config, self.db_path, self.site_url)
|
|
except Exception as e:
|
|
logger.error(f"Notification sweep failed: {e}", exc_info=True)
|
|
|
|
# Update last processed time
|
|
if processed > 0:
|
|
self._last_processed = datetime.now(timezone.utc).replace(tzinfo=None)
|
|
|
|
# Sleep if nothing processed
|
|
if processed == 0:
|
|
self._stop_event.wait(self.poll_interval)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Worker error: {e}", exc_info=True)
|
|
self._stop_event.wait(self.error_backoff)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Webmention worker thread failed: {e}", exc_info=True)
|
|
finally:
|
|
self._release_lock()
|
|
logger.info("Webmention worker stopped")
|
|
|
|
def _acquire_lock(self) -> bool:
|
|
"""
|
|
Take the cross-process worker lock (non-blocking)
|
|
|
|
On first acquisition, also reset items left mid-send by a previous
|
|
holder. The OS releases the lock if this process dies.
|
|
|
|
Returns:
|
|
True if this worker holds the lock
|
|
"""
|
|
if self._lock_file:
|
|
return True
|
|
|
|
lock_file = open(Path(self.db_path).parent / "webmention-worker.lock", "w")
|
|
try:
|
|
fcntl.flock(lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except OSError:
|
|
lock_file.close()
|
|
return False
|
|
|
|
self._lock_file = lock_file
|
|
logger.info("Webmention worker holds the queue lock")
|
|
self._cleanup_orphaned_items()
|
|
return True
|
|
|
|
def _release_lock(self):
|
|
"""Release the cross-process worker lock if held"""
|
|
if self._lock_file:
|
|
self._lock_file.close()
|
|
self._lock_file = None
|
|
|
|
def _cleanup_orphaned_items(self):
|
|
"""
|
|
Clean up orphaned 'sending' status items on startup
|
|
|
|
Per background-worker.md:
|
|
Reset items stuck in 'sending' status from previous crashes
|
|
"""
|
|
try:
|
|
conn = sqlite3.connect(self.db_path)
|
|
cursor = conn.cursor()
|
|
|
|
# Reset orphaned outgoing webmentions
|
|
cursor.execute("""
|
|
UPDATE webmention_queue
|
|
SET status = 'pending',
|
|
started_at = NULL
|
|
WHERE status = 'sending'
|
|
AND started_at < datetime('now', '-5 minutes')
|
|
""")
|
|
|
|
orphaned = cursor.rowcount
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
if orphaned > 0:
|
|
logger.info(f"Reset {orphaned} orphaned webmention queue items")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to cleanup orphaned items: {e}")
|
|
|
|
self._drop_skipped_targets()
|
|
|
|
def _skipped_hosts(self) -> set:
|
|
from starpunk.webmention_sender import skipped_hosts
|
|
return skipped_hosts(self.app.config) if self.app is not None else set()
|
|
|
|
def _drop_skipped_targets(self):
|
|
"""
|
|
Stop sending to this site and WEBMENTION_SKIP_HOSTS (v2.18.3)
|
|
|
|
Queued or failed sends to those hosts are set to 'no_endpoint' (nothing
|
|
to send to), so they leave the retry queue and the admin Log.
|
|
"""
|
|
from starpunk.webmention_sender import SKIPPED_NOTE, is_skipped
|
|
|
|
hosts = self._skipped_hosts()
|
|
if not hosts:
|
|
return
|
|
try:
|
|
conn = sqlite3.connect(self.db_path)
|
|
rows = conn.execute(
|
|
"SELECT id, target_url FROM webmention_queue WHERE status IN ('pending', 'failed')"
|
|
).fetchall()
|
|
ids = [(SKIPPED_NOTE, i) for i, url in rows if is_skipped(url, hosts)]
|
|
if ids:
|
|
conn.executemany(
|
|
"UPDATE webmention_queue SET status = 'no_endpoint', last_error = ? WHERE id = ?",
|
|
ids)
|
|
conn.commit()
|
|
logger.info(f"Dropped {len(ids)} queued webmention(s) to skipped hosts")
|
|
conn.close()
|
|
except Exception as e:
|
|
logger.error(f"Failed to drop skipped webmention targets: {e}")
|
|
|
|
def _process_pending_validations(self) -> int:
|
|
"""
|
|
Process pending incoming webmentions
|
|
|
|
Returns:
|
|
Number of webmentions processed
|
|
"""
|
|
processed = 0
|
|
|
|
try:
|
|
conn = sqlite3.connect(self.db_path)
|
|
conn.row_factory = sqlite3.Row
|
|
cursor = conn.cursor()
|
|
|
|
# Get pending webmentions
|
|
cursor.execute("""
|
|
SELECT id, source_url, target_url, target_note_id
|
|
FROM webmentions
|
|
WHERE validation_status = 'pending'
|
|
ORDER BY received_at ASC
|
|
LIMIT ?
|
|
""", (self.batch_size,))
|
|
|
|
pending = cursor.fetchall()
|
|
conn.close()
|
|
|
|
# Process each webmention
|
|
for row in pending:
|
|
try:
|
|
self._validate_webmention(
|
|
row['id'],
|
|
row['source_url'],
|
|
row['target_url'],
|
|
row['target_note_id']
|
|
)
|
|
processed += 1
|
|
except Exception as e:
|
|
logger.error(
|
|
f"Failed to validate webmention {row['id']}: {e}",
|
|
exc_info=True
|
|
)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error fetching pending validations: {e}")
|
|
|
|
return processed
|
|
|
|
def _validate_webmention(
|
|
self,
|
|
webmention_id: int,
|
|
source_url: str,
|
|
target_url: str,
|
|
target_note_id: Optional[int]
|
|
):
|
|
"""
|
|
Validate a webmention, then re-send the target note's webmentions if
|
|
what the note shows changed (spec 3.1.4: displayed responses are an
|
|
update of the post). Unchanged re-validations send nothing, so two
|
|
sites doing this don't ping-pong.
|
|
"""
|
|
before = self._display_state(webmention_id)
|
|
try:
|
|
self._validate_and_store(webmention_id, source_url, target_url, target_note_id)
|
|
finally:
|
|
after = self._display_state(webmention_id)
|
|
if before != after and target_note_id is not None:
|
|
try:
|
|
note_responses_changed(self.db_path, [target_note_id])
|
|
except Exception as e:
|
|
logger.warning(f"Failed to re-send webmentions for note {target_note_id}: {e}")
|
|
|
|
def _is_own_syndicated_copy(self, note_id: Optional[int], source_url: str,
|
|
result: Dict[str, Any]) -> bool:
|
|
"""True if the source is one of the note's own u-syndication copies"""
|
|
if note_id is None:
|
|
return False
|
|
conn = sqlite3.connect(self.db_path)
|
|
try:
|
|
copies = {normalize_url(row[0]) for row in conn.execute(
|
|
"SELECT remote_url FROM syndications WHERE note_id = ? AND remote_url IS NOT NULL",
|
|
(note_id,),
|
|
)}
|
|
finally:
|
|
conn.close()
|
|
source = {normalize_url(u) for u in (
|
|
source_url, result.get('source_post_url'), result.get('source_uid')) if u}
|
|
return bool(copies & source)
|
|
|
|
def _is_backfed(self, note_id: Optional[int], source_url: str, result: Dict[str, Any]) -> bool:
|
|
"""True if the source is a response already stored by backfeed for this note"""
|
|
if note_id is None:
|
|
return False
|
|
conn = sqlite3.connect(self.db_path)
|
|
try:
|
|
backfed = {normalize_url(row[0]) for row in conn.execute(
|
|
"SELECT source_url FROM webmentions WHERE target_note_id = ? AND origin != 'webmention'",
|
|
(note_id,),
|
|
)}
|
|
finally:
|
|
conn.close()
|
|
source = {normalize_url(u) for u in (
|
|
source_url, result.get('source_post_url'), result.get('source_uid')) if u}
|
|
return bool(backfed & source)
|
|
|
|
def _display_state(self, webmention_id: int) -> Optional[tuple]:
|
|
"""
|
|
What the note page shows for this webmention, or None if hidden
|
|
|
|
A row reset to 'pending' by a re-send still counts as shown if it was
|
|
valid before (validated with no error), as it was until now.
|
|
"""
|
|
conn = sqlite3.connect(self.db_path)
|
|
try:
|
|
row = conn.execute("""
|
|
SELECT mention_type, author_name, author_url, author_photo, content_snippet
|
|
FROM webmentions
|
|
WHERE id = ?
|
|
AND moderation_status = 'approved'
|
|
AND (validation_status = 'valid'
|
|
OR (validation_status = 'pending' AND validated_at IS NOT NULL
|
|
AND validation_error IS NULL))
|
|
""", (webmention_id,)).fetchone()
|
|
finally:
|
|
conn.close()
|
|
return tuple(row) if row else None
|
|
|
|
def _validate_and_store(
|
|
self,
|
|
webmention_id: int,
|
|
source_url: str,
|
|
target_url: str,
|
|
target_note_id: Optional[int]
|
|
):
|
|
"""
|
|
Validate a single webmention
|
|
|
|
Args:
|
|
webmention_id: Database ID of webmention
|
|
source_url: Source URL
|
|
target_url: Target URL
|
|
target_note_id: Target note ID (if applicable)
|
|
"""
|
|
try:
|
|
# Process webmention (fetch, validate, extract)
|
|
result = process_webmention(source_url, target_url, self.site_url)
|
|
|
|
# A copy of this very note on a silo (e.g. backfed by Bridgy) is
|
|
# not a response to it
|
|
if self._is_own_syndicated_copy(target_note_id, source_url, result):
|
|
raise ValidationError("Source is this note's own syndicated copy")
|
|
|
|
# A response StarPunk already backfed from the silo (e.g. a reply
|
|
# also sent by Bridgy) would show twice (v2.13.0, ADR-073)
|
|
if self._is_backfed(target_note_id, source_url, result):
|
|
raise ValidationError("Already backfed from the silo")
|
|
|
|
# Update database with successful validation
|
|
conn = sqlite3.connect(self.db_path)
|
|
cursor = conn.cursor()
|
|
|
|
moderation_status = 'approved' if self.auto_approve else 'pending'
|
|
|
|
# An update to an existing mention (spec 3.2.4) keeps the admin's
|
|
# moderation decision; only a first-time valid mention gets the
|
|
# default. Previously valid = validated with no error.
|
|
cursor.execute("""
|
|
UPDATE webmentions
|
|
SET moderation_status = CASE
|
|
WHEN moderation_status IN ('hidden', 'deleted') THEN moderation_status
|
|
WHEN validated_at IS NOT NULL AND validation_error IS NULL
|
|
THEN moderation_status
|
|
ELSE ?
|
|
END,
|
|
validation_status = 'valid',
|
|
validation_error = NULL,
|
|
validated_at = CURRENT_TIMESTAMP,
|
|
mention_type = ?,
|
|
author_name = ?,
|
|
author_url = ?,
|
|
author_photo = ?,
|
|
content_snippet = ?,
|
|
source_post_url = ?,
|
|
source_published = ?,
|
|
updated_at = CURRENT_TIMESTAMP
|
|
WHERE id = ?
|
|
""", (
|
|
moderation_status,
|
|
result['mention_type'],
|
|
result['author_name'],
|
|
result['author_url'],
|
|
result['author_photo'],
|
|
result['content_snippet'],
|
|
result.get('source_post_url'),
|
|
result.get('source_published'),
|
|
webmention_id
|
|
))
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
# Pages show a local copy of the avatar (v2.13.0, ADR-073)
|
|
from starpunk import avatars
|
|
avatars.fetch(self.app.config['DATA_PATH'], result['author_photo'], self.site_url)
|
|
|
|
logger.info(
|
|
f"Validated webmention: {source_url} -> {target_url} "
|
|
f"(type={result['mention_type']})"
|
|
)
|
|
|
|
except TransientError as e:
|
|
# Source unreachable right now: keep a previously valid mention
|
|
self._mark_validation_failed(webmention_id, source_url, target_url, str(e),
|
|
transient=True)
|
|
|
|
except ValidationError as e:
|
|
# Validation failed (410/4xx, no link): the mention is removed
|
|
# from display (spec 3.2.4)
|
|
self._mark_validation_failed(webmention_id, source_url, target_url, str(e))
|
|
|
|
except Exception as e:
|
|
# Unexpected error - mark as error
|
|
self._mark_validation_failed(
|
|
webmention_id,
|
|
source_url,
|
|
target_url,
|
|
f"Unexpected error: {e}"
|
|
)
|
|
raise
|
|
|
|
def _mark_validation_failed(
|
|
self,
|
|
webmention_id: int,
|
|
source_url: str,
|
|
target_url: str,
|
|
error: str,
|
|
transient: bool = False
|
|
):
|
|
"""
|
|
Mark webmention validation as failed/invalid
|
|
|
|
A transient failure of a mention that was valid before restores it to
|
|
valid with its previous data, rather than hiding it.
|
|
|
|
Args:
|
|
webmention_id: Database ID
|
|
source_url: Source URL
|
|
target_url: Target URL
|
|
error: Error message
|
|
"""
|
|
# Determine if error or invalid
|
|
# Per background-worker.md: HTTP 4xx and "no link" are invalid
|
|
# All other errors are 'error' status
|
|
if transient:
|
|
status = 'error'
|
|
elif any(x in error.lower() for x in ['does not link', 'returned 4', 'own syndicated copy',
|
|
'already backfed']):
|
|
status = 'invalid'
|
|
else:
|
|
status = 'error'
|
|
|
|
try:
|
|
conn = sqlite3.connect(self.db_path)
|
|
cursor = conn.cursor()
|
|
|
|
if transient:
|
|
cursor.execute("""
|
|
UPDATE webmentions
|
|
SET validation_status = 'valid',
|
|
updated_at = CURRENT_TIMESTAMP
|
|
WHERE id = ? AND validated_at IS NOT NULL AND validation_error IS NULL
|
|
""", (webmention_id,))
|
|
if cursor.rowcount:
|
|
conn.commit()
|
|
conn.close()
|
|
logger.warning(
|
|
f"Could not re-check webmention, keeping it: "
|
|
f"{source_url} -> {target_url}: {error}"
|
|
)
|
|
return
|
|
|
|
cursor.execute("""
|
|
UPDATE webmentions
|
|
SET validation_status = ?,
|
|
validation_error = ?,
|
|
validated_at = CURRENT_TIMESTAMP,
|
|
updated_at = CURRENT_TIMESTAMP
|
|
WHERE id = ?
|
|
""", (status, error, webmention_id))
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
logger.warning(
|
|
f"Invalid webmention: {source_url} -> {target_url}: {error}"
|
|
)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to mark webmention as failed: {e}")
|
|
|
|
def _process_pending_sends(self) -> int:
|
|
"""
|
|
Process pending outgoing webmentions (Phase 3)
|
|
|
|
Returns:
|
|
Number of webmentions processed
|
|
"""
|
|
processed = 0
|
|
|
|
try:
|
|
conn = sqlite3.connect(self.db_path)
|
|
conn.row_factory = sqlite3.Row
|
|
cursor = conn.cursor()
|
|
|
|
# Get pending webmentions ready to send
|
|
cursor.execute("""
|
|
SELECT id, source_url, target_url, endpoint_url, retry_count
|
|
FROM webmention_queue
|
|
WHERE status = 'pending'
|
|
AND (next_retry_at IS NULL OR next_retry_at <= datetime('now'))
|
|
ORDER BY created_at ASC
|
|
LIMIT ?
|
|
""", (self.batch_size,))
|
|
|
|
pending = cursor.fetchall()
|
|
conn.close()
|
|
|
|
# Process each webmention
|
|
for row in pending:
|
|
try:
|
|
self._send_webmention(
|
|
row['id'],
|
|
row['source_url'],
|
|
row['target_url'],
|
|
row['endpoint_url'],
|
|
row['retry_count']
|
|
)
|
|
processed += 1
|
|
except Exception as e:
|
|
logger.error(
|
|
f"Failed to send webmention {row['id']}: {e}",
|
|
exc_info=True
|
|
)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error fetching pending sends: {e}")
|
|
|
|
return processed
|
|
|
|
def _send_webmention(
|
|
self,
|
|
queue_id: int,
|
|
source_url: str,
|
|
target_url: str,
|
|
cached_endpoint: Optional[str],
|
|
retry_count: int
|
|
):
|
|
"""
|
|
Send a single outgoing webmention
|
|
|
|
Args:
|
|
queue_id: Database ID of queue item
|
|
source_url: Source URL (our note)
|
|
target_url: Target URL to notify
|
|
cached_endpoint: Previously discovered endpoint (if any)
|
|
retry_count: Current retry count
|
|
"""
|
|
# Mark as sending
|
|
conn = sqlite3.connect(self.db_path)
|
|
cursor = conn.cursor()
|
|
cursor.execute("""
|
|
UPDATE webmention_queue
|
|
SET status = 'sending',
|
|
started_at = CURRENT_TIMESTAMP
|
|
WHERE id = ?
|
|
""", (queue_id,))
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
from starpunk.webmention_sender import SKIPPED_NOTE, is_skipped
|
|
if is_skipped(target_url, self._skipped_hosts()):
|
|
conn = sqlite3.connect(self.db_path)
|
|
conn.execute("UPDATE webmention_queue SET status = 'no_endpoint', last_error = ? WHERE id = ?",
|
|
(SKIPPED_NOTE, queue_id))
|
|
conn.commit()
|
|
conn.close()
|
|
return
|
|
|
|
try:
|
|
# Discover endpoint if not cached
|
|
endpoint = cached_endpoint
|
|
if not endpoint:
|
|
endpoint = self._discover_endpoint(target_url)
|
|
|
|
if not endpoint:
|
|
# No endpoint found - permanent failure
|
|
self._mark_send_no_endpoint(queue_id, source_url, target_url)
|
|
return
|
|
|
|
# Cache the discovered endpoint in database
|
|
conn = sqlite3.connect(self.db_path)
|
|
cursor = conn.cursor()
|
|
cursor.execute("""
|
|
UPDATE webmention_queue
|
|
SET endpoint_url = ?
|
|
WHERE id = ?
|
|
""", (endpoint, queue_id))
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
# Send webmention POST (spec 3.1.3)
|
|
try:
|
|
# SSRF-checked and pinned to the checked address (spec 4.3).
|
|
# Redirects are not followed: a 301/302 would turn the POST
|
|
# into a GET without the body
|
|
response = safe_post(
|
|
endpoint,
|
|
data={
|
|
'source': source_url,
|
|
'target': target_url
|
|
},
|
|
headers={'User-Agent': user_agent(self.site_url)},
|
|
)
|
|
|
|
# Any 2xx is success (spec 3.1.3)
|
|
if 200 <= response.status_code < 300:
|
|
self._mark_send_success(queue_id, source_url, target_url)
|
|
|
|
elif response.status_code == 429 or response.status_code >= 500:
|
|
# Rate limited or server error - retry
|
|
error = f"HTTP {response.status_code}: {response.text[:200]}"
|
|
self._mark_send_retry(queue_id, source_url, target_url, retry_count, error)
|
|
|
|
else:
|
|
# Permanent failure (3xx, 4xx)
|
|
error = f"HTTP {response.status_code}: {response.text[:200]}"
|
|
self._mark_send_failed(queue_id, source_url, target_url, error)
|
|
|
|
except (httpx.TimeoutException, httpx.HTTPError) as e:
|
|
# Connection error - retry
|
|
self._mark_send_retry(queue_id, source_url, target_url, retry_count, str(e))
|
|
|
|
except (SSRFBlockedError, ValueError) as e:
|
|
# SSRF blocked - permanent failure
|
|
logger.warning(f"SSRF blocked endpoint {endpoint}: {e}")
|
|
self._mark_send_failed(queue_id, source_url, target_url, f"Blocked: {e}")
|
|
|
|
except Exception as e:
|
|
# Discovery failed - retry
|
|
self._mark_send_retry(queue_id, source_url, target_url, retry_count, str(e))
|
|
|
|
def _discover_endpoint(self, target_url: str) -> Optional[str]:
|
|
"""
|
|
Discover the webmention endpoint for a target URL (spec 3.1.2)
|
|
|
|
- GET the target, following redirects (each hop SSRF-checked)
|
|
- The first Link header with rel=webmention wins, then the first
|
|
<link> or <a> with rel=webmention and an href, in document order
|
|
- rel is a space-separated token list, matched case-insensitively
|
|
- Relative endpoints resolve against the final URL; an empty href
|
|
is the target itself; query strings are kept
|
|
- Not cached: an updated post must re-discover (spec 3.1.4)
|
|
|
|
Returns:
|
|
Endpoint URL, or None if the target has none (or is blocked/4xx)
|
|
|
|
Raises:
|
|
httpx.HTTPError: Network errors and 5xx, so the caller retries
|
|
"""
|
|
try:
|
|
result = safe_get(target_url, headers={'User-Agent': user_agent(self.site_url)})
|
|
except (SSRFBlockedError, ValueError) as e:
|
|
logger.warning(f"SSRF protection blocked target {target_url}: {e}")
|
|
return None
|
|
except FetchTooLargeError:
|
|
logger.info(f"Target too large for discovery: {target_url}")
|
|
return None
|
|
|
|
if result.status_code >= 500:
|
|
raise httpx.HTTPError(f"Target returned {result.status_code}")
|
|
if result.status_code >= 400:
|
|
logger.info(f"Target returned {result.status_code}: {target_url}")
|
|
return None
|
|
|
|
for link_header in result.headers.get_list('Link'):
|
|
endpoint = self._parse_link_header(link_header, result.url)
|
|
if endpoint is not None:
|
|
return endpoint
|
|
|
|
content_type = result.headers.get('Content-Type', '')
|
|
if 'html' in content_type.lower():
|
|
endpoint = self._parse_html_for_endpoint(result.text, result.url)
|
|
if endpoint is not None:
|
|
return endpoint
|
|
|
|
logger.info(f"No webmention endpoint found for {target_url}")
|
|
return None
|
|
|
|
@staticmethod
|
|
def _has_webmention_rel(rel: str) -> bool:
|
|
return 'webmention' in rel.lower().split()
|
|
|
|
def _parse_link_header(self, link_header: str, base_url: str) -> Optional[str]:
|
|
"""
|
|
Parse a Link header (RFC 8288) for a rel=webmention link
|
|
|
|
Args:
|
|
link_header: Link header value (may hold several links)
|
|
base_url: Base URL for resolving relative URLs
|
|
|
|
Returns:
|
|
Absolute endpoint URL of the first match, None otherwise
|
|
"""
|
|
for match in LINK_RE.finditer(link_header):
|
|
uri, params = match.group(1), match.group(2)
|
|
for param in LINK_PARAM_RE.finditer(params):
|
|
if param.group(1).lower() != 'rel':
|
|
continue
|
|
rel = param.group(2) if param.group(2) is not None else (param.group(3) or '')
|
|
if self._has_webmention_rel(rel):
|
|
return urljoin(base_url, uri.strip())
|
|
break # only the first rel parameter counts (RFC 8288)
|
|
|
|
return None
|
|
|
|
def _parse_html_for_endpoint(self, html_content: str, base_url: str) -> Optional[str]:
|
|
"""
|
|
Find the first <link> or <a> with rel=webmention, in document order
|
|
|
|
Args:
|
|
html_content: HTML content
|
|
base_url: Base URL for resolving relative URLs
|
|
|
|
Returns:
|
|
Absolute endpoint URL if found, None otherwise
|
|
"""
|
|
try:
|
|
doc = lxml_html.document_fromstring(html_content)
|
|
except Exception as e:
|
|
logger.debug(f"HTML parsing failed: {e}")
|
|
return None
|
|
|
|
for element in doc.iter('link', 'a'):
|
|
href = element.get('href')
|
|
if href is not None and self._has_webmention_rel(element.get('rel', '')):
|
|
return urljoin(base_url, href.strip())
|
|
|
|
return None
|
|
|
|
def _mark_send_success(self, queue_id: int, source_url: str, target_url: str):
|
|
"""Mark webmention send as successful"""
|
|
try:
|
|
conn = sqlite3.connect(self.db_path)
|
|
cursor = conn.cursor()
|
|
cursor.execute("""
|
|
UPDATE webmention_queue
|
|
SET status = 'sent',
|
|
sent_at = CURRENT_TIMESTAMP
|
|
WHERE id = ?
|
|
""", (queue_id,))
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
logger.info(f"Sent webmention: {source_url} -> {target_url}")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to mark send success: {e}")
|
|
|
|
def _mark_send_failed(self, queue_id: int, source_url: str, target_url: str, error: str):
|
|
"""Mark webmention send as permanently failed"""
|
|
try:
|
|
conn = sqlite3.connect(self.db_path)
|
|
cursor = conn.cursor()
|
|
cursor.execute("""
|
|
UPDATE webmention_queue
|
|
SET status = 'failed',
|
|
last_error = ?
|
|
WHERE id = ?
|
|
""", (error, queue_id))
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
logger.warning(f"Webmention send failed permanently: {source_url} -> {target_url}: {error}")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to mark send failed: {e}")
|
|
|
|
def _mark_send_no_endpoint(self, queue_id: int, source_url: str, target_url: str):
|
|
"""Mark webmention as no endpoint found"""
|
|
try:
|
|
conn = sqlite3.connect(self.db_path)
|
|
cursor = conn.cursor()
|
|
cursor.execute("""
|
|
UPDATE webmention_queue
|
|
SET status = 'no_endpoint'
|
|
WHERE id = ?
|
|
""", (queue_id,))
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
logger.info(f"No webmention endpoint for {target_url}")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to mark no endpoint: {e}")
|
|
|
|
def _mark_send_retry(
|
|
self,
|
|
queue_id: int,
|
|
source_url: str,
|
|
target_url: str,
|
|
current_retry_count: int,
|
|
error: str
|
|
):
|
|
"""
|
|
Mark webmention for retry with exponential backoff
|
|
|
|
Per architect's answer to Q5 and background-worker.md:
|
|
- Increment retry_count AFTER failed attempt
|
|
- Check retry_count >= 4 AFTER incrementing
|
|
- Retry schedule: 1h, 6h, 24h
|
|
"""
|
|
new_retry_count = current_retry_count + 1
|
|
|
|
# Calculate next retry time
|
|
next_retry_at = None
|
|
if new_retry_count < 4:
|
|
delays = {
|
|
1: timedelta(hours=1),
|
|
2: timedelta(hours=6),
|
|
3: timedelta(hours=24),
|
|
}
|
|
delay = delays.get(new_retry_count)
|
|
if delay:
|
|
next_retry_at = datetime.now(timezone.utc).replace(tzinfo=None) + delay
|
|
|
|
try:
|
|
conn = sqlite3.connect(self.db_path)
|
|
cursor = conn.cursor()
|
|
|
|
if new_retry_count >= 4:
|
|
# Max retries reached - mark as failed
|
|
cursor.execute("""
|
|
UPDATE webmention_queue
|
|
SET status = 'failed',
|
|
retry_count = ?,
|
|
last_error = ?
|
|
WHERE id = ?
|
|
""", (new_retry_count, error, queue_id))
|
|
|
|
logger.warning(
|
|
f"Webmention send failed permanently after {new_retry_count} attempts: "
|
|
f"{source_url} -> {target_url}: {error}"
|
|
)
|
|
|
|
else:
|
|
# Schedule retry
|
|
cursor.execute("""
|
|
UPDATE webmention_queue
|
|
SET status = 'pending',
|
|
retry_count = ?,
|
|
next_retry_at = ?,
|
|
last_error = ?
|
|
WHERE id = ?
|
|
""", (new_retry_count, next_retry_at, error, queue_id))
|
|
|
|
logger.warning(
|
|
f"Webmention send failed, retry {new_retry_count}: "
|
|
f"{source_url} -> {target_url}: {error}"
|
|
)
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to mark send retry: {e}")
|
|
|
|
def stop(self):
|
|
"""
|
|
Stop the worker thread gracefully
|
|
|
|
Sets the stop event to signal the thread to exit
|
|
"""
|
|
logger.info("Stopping webmention worker")
|
|
self._stop_event.set()
|
|
|
|
def get_stats(self) -> Dict[str, Any]:
|
|
"""
|
|
Get current worker statistics
|
|
|
|
Returns:
|
|
Dict with worker stats for health check
|
|
"""
|
|
try:
|
|
conn = sqlite3.connect(self.db_path)
|
|
cursor = conn.cursor()
|
|
|
|
# Count pending validations
|
|
cursor.execute("""
|
|
SELECT COUNT(*) FROM webmentions
|
|
WHERE validation_status = 'pending'
|
|
""")
|
|
pending_validations = cursor.fetchone()[0]
|
|
|
|
# Count pending sends (Phase 3)
|
|
cursor.execute("""
|
|
SELECT COUNT(*) FROM webmention_queue
|
|
WHERE status = 'pending'
|
|
""")
|
|
pending_sends = cursor.fetchone()[0]
|
|
|
|
conn.close()
|
|
|
|
return {
|
|
'running': self.is_alive(),
|
|
'last_processed': self._last_processed.isoformat() if self._last_processed else None,
|
|
'pending_validations': pending_validations,
|
|
'pending_sends': pending_sends
|
|
}
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to get worker stats: {e}")
|
|
return {
|
|
'running': self.is_alive(),
|
|
'error': str(e)
|
|
}
|