Links to the site itself, and to self-hosted apps like Gitea, timed out from inside the container and sat in the admin Log. They are now skipped when queueing; already queued or failed ones are dropped at worker start. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
274 lines
9.0 KiB
Python
274 lines
9.0 KiB
Python
"""
|
|
Outgoing webmention link extraction and queueing
|
|
|
|
Per background-worker.md and architect's answers:
|
|
- Extract links from rendered HTML (not markdown)
|
|
- Extract only <a href> elements, exclude <pre> and <code> blocks
|
|
- Queue max 10 links per note per operation
|
|
- Priority: in_reply_to first, then document order
|
|
|
|
References:
|
|
- Background Worker Design: /docs/design/v1.6.0/background-worker.md
|
|
- API Contract: /docs/design/v1.6.0/api-contract.md
|
|
"""
|
|
|
|
import logging
|
|
import sqlite3
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
from lxml import html as lxml_html
|
|
from urllib.parse import urlparse
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
SKIPPED_NOTE = "Not sent: links to this site and WEBMENTION_SKIP_HOSTS are skipped"
|
|
|
|
|
|
def skipped_hosts(config) -> set[str]:
|
|
"""Hosts that never get webmentions: this site and WEBMENTION_SKIP_HOSTS (v2.18.3)"""
|
|
hosts = set(config.get("WEBMENTION_SKIP_HOSTS") or [])
|
|
site = urlparse(config.get("SITE_URL") or "").hostname
|
|
if site:
|
|
hosts.add(site.lower())
|
|
return hosts
|
|
|
|
|
|
def is_skipped(url: str, hosts: set[str]) -> bool:
|
|
"""True if url is on one of the skipped hosts (exact host, any port)"""
|
|
return (urlparse(url).hostname or "").lower() in hosts
|
|
|
|
|
|
def extract_links_from_html(html_content: str, exclude_code_blocks: bool = True) -> list[str]:
|
|
"""
|
|
Extract links from HTML content.
|
|
|
|
Per architect's answer to Q1:
|
|
- Extract from rendered HTML (not markdown)
|
|
- Extract <a href> elements only
|
|
- Exclude links in <pre> and <code> blocks
|
|
- Return in document order
|
|
|
|
Args:
|
|
html_content: Rendered HTML content
|
|
exclude_code_blocks: If True, skip links in <pre> and <code> (default: True)
|
|
|
|
Returns:
|
|
List of URLs in document order
|
|
"""
|
|
if not html_content or not html_content.strip():
|
|
return []
|
|
|
|
try:
|
|
# Parse HTML inside a wrapper, so a top-level <pre> can be removed too
|
|
doc = lxml_html.fragment_fromstring(html_content, create_parent='div')
|
|
|
|
if exclude_code_blocks:
|
|
# Remove <pre> and <code> elements before extracting links
|
|
for element in doc.xpath('.//pre | .//code'):
|
|
element.drop_tree()
|
|
|
|
# Extract all <a href> attributes in document order
|
|
links = doc.xpath('.//a/@href')
|
|
|
|
# Filter to valid absolute URLs (http/https)
|
|
absolute_links = []
|
|
for link in links:
|
|
parsed = urlparse(link)
|
|
if parsed.scheme in ('http', 'https') and parsed.netloc:
|
|
absolute_links.append(link)
|
|
|
|
return absolute_links
|
|
|
|
except Exception as e:
|
|
logger.warning(f"Failed to parse HTML for link extraction: {e}")
|
|
return []
|
|
|
|
|
|
def _dedupe(urls: list) -> list:
|
|
"""Drop empty and repeated URLs, keeping the first occurrence"""
|
|
seen = set()
|
|
result = []
|
|
for url in urls:
|
|
if url and url not in seen:
|
|
seen.add(url)
|
|
result.append(url)
|
|
return result
|
|
|
|
|
|
def previous_targets(db_path: Path, note_id: int) -> list[str]:
|
|
"""Targets this note has been queued to send webmentions to before"""
|
|
conn = sqlite3.connect(db_path)
|
|
try:
|
|
rows = conn.execute(
|
|
"SELECT target_url FROM webmention_queue WHERE source_note_id = ? ORDER BY id",
|
|
(note_id,),
|
|
).fetchall()
|
|
finally:
|
|
conn.close()
|
|
return [row[0] for row in rows]
|
|
|
|
|
|
def requeue_previous_webmentions(db_path: Path, note_id: int, detach: bool = False) -> int:
|
|
"""
|
|
Re-send every webmention this note has sent (spec 3.1.5 deleted posts)
|
|
|
|
Args:
|
|
db_path: Path to SQLite database
|
|
note_id: Note being deleted or restored
|
|
detach: Clear source_note_id so the rows survive a hard delete
|
|
(the foreign key cascades)
|
|
|
|
Returns:
|
|
Number of webmentions queued
|
|
"""
|
|
conn = sqlite3.connect(db_path)
|
|
try:
|
|
cursor = conn.execute(f"""
|
|
UPDATE webmention_queue
|
|
SET status = 'pending',
|
|
retry_count = 0,
|
|
next_retry_at = NULL,
|
|
last_error = NULL,
|
|
endpoint_url = NULL,
|
|
created_at = CURRENT_TIMESTAMP
|
|
{', source_note_id = NULL' if detach else ''}
|
|
WHERE source_note_id = ?
|
|
""", (note_id,))
|
|
conn.commit()
|
|
queued = cursor.rowcount
|
|
finally:
|
|
conn.close()
|
|
|
|
if queued:
|
|
logger.info(f"Re-queued {queued} webmentions for note {note_id}")
|
|
return queued
|
|
|
|
|
|
def note_responses_changed(db_path: Path, note_ids) -> int:
|
|
"""
|
|
Re-send a note's webmentions because the responses it shows changed
|
|
|
|
Spec 3.1.4: if the source post displays responses (e.g. comments), a
|
|
change to them is an update of the post. Only published, undeleted notes.
|
|
|
|
Args:
|
|
db_path: Path to SQLite database
|
|
note_ids: IDs of notes whose displayed responses changed
|
|
|
|
Returns:
|
|
Number of webmentions queued
|
|
"""
|
|
ids = sorted({i for i in note_ids if i is not None})
|
|
if not ids:
|
|
return 0
|
|
|
|
conn = sqlite3.connect(db_path)
|
|
try:
|
|
placeholders = ','.join('?' * len(ids))
|
|
live = [row[0] for row in conn.execute(
|
|
f"SELECT id FROM notes WHERE id IN ({placeholders}) "
|
|
"AND published = 1 AND deleted_at IS NULL", ids,
|
|
)]
|
|
finally:
|
|
conn.close()
|
|
|
|
return sum(requeue_previous_webmentions(db_path, note_id) for note_id in live)
|
|
|
|
|
|
def queue_webmentions_for_note(
|
|
db_path: Path,
|
|
note_id: int,
|
|
note_url: str,
|
|
content_html: str,
|
|
in_reply_to: Optional[str] = None,
|
|
old_content_html: Optional[str] = None # Phase 6: update semantics
|
|
) -> int:
|
|
"""
|
|
Extract links and queue webmentions for a note.
|
|
|
|
Per architect's answers (Phase 6):
|
|
- Called from notes.py AFTER successful database commit
|
|
- For creates: reply target first, then links in document order (max 10)
|
|
- For updates: the same, plus every previously sent target and links
|
|
removed by the edit (spec 3.1.4)
|
|
- Webmention sending is best-effort; failures do not affect note creation
|
|
|
|
Args:
|
|
db_path: Path to SQLite database
|
|
note_id: Database ID of the note
|
|
note_url: Full URL of the published note
|
|
content_html: Rendered HTML content (None if unpublishing)
|
|
in_reply_to: Reply-to URL (if reply post)
|
|
old_content_html: Previous HTML (for update diff, Phase 6)
|
|
|
|
Returns:
|
|
Number of webmentions queued
|
|
"""
|
|
# Links to this site or a skipped host are dropped before the cap (v2.18.3):
|
|
# this site only takes webmentions for its posts, and its own public
|
|
# address may not be reachable from inside the container
|
|
from flask import current_app, has_app_context
|
|
hosts = skipped_hosts(current_app.config) if has_app_context() else set()
|
|
|
|
def wanted(urls):
|
|
return [u for u in urls if not is_skipped(u, hosts)]
|
|
|
|
# Targets for the current content: reply target first, then links in
|
|
# document order, at most 10
|
|
links = extract_links_from_html(content_html) if content_html else []
|
|
current = wanted(_dedupe(([in_reply_to] if in_reply_to else []) + links))[:10]
|
|
|
|
if old_content_html is not None:
|
|
# Update (spec 3.1.4): also re-send to every target notified before,
|
|
# and to links the edit removed, so they can update or remove the
|
|
# mention. These are not capped.
|
|
removed = [u for u in extract_links_from_html(old_content_html) if u not in links]
|
|
target_urls = _dedupe(current + wanted(previous_targets(db_path, note_id) + removed))
|
|
else:
|
|
target_urls = current
|
|
|
|
if not target_urls:
|
|
logger.debug(f"No links to send webmentions for note {note_id}")
|
|
return 0
|
|
|
|
# Queue webmentions
|
|
queued = 0
|
|
try:
|
|
conn = sqlite3.connect(db_path)
|
|
cursor = conn.cursor()
|
|
|
|
for target_url in target_urls:
|
|
try:
|
|
# Use UPSERT to handle duplicate sends
|
|
# Phase 6: Force endpoint rediscovery per W3C MUST requirement
|
|
cursor.execute("""
|
|
INSERT INTO webmention_queue
|
|
(source_url, target_url, source_note_id, status, created_at)
|
|
VALUES (?, ?, ?, 'pending', CURRENT_TIMESTAMP)
|
|
ON CONFLICT(source_url, target_url) DO UPDATE SET
|
|
status = 'pending',
|
|
retry_count = 0,
|
|
next_retry_at = NULL,
|
|
last_error = NULL,
|
|
endpoint_url = NULL,
|
|
created_at = CURRENT_TIMESTAMP
|
|
""", (note_url, target_url, note_id))
|
|
|
|
queued += 1
|
|
|
|
except Exception as e:
|
|
logger.warning(
|
|
f"Failed to queue webmention {note_url} -> {target_url}: {e}"
|
|
)
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
logger.info(f"Queued {queued} webmentions for note {note_id}")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to queue webmentions for note {note_id}: {e}")
|
|
|
|
return queued
|