Files
StarPunk/starpunk/webmention_sender.py
PhilandClaude Opus 5.5 b9e5321df2 fix: don't send webmentions to this site or WEBMENTION_SKIP_HOSTS (v2.18.3)
Links to the site itself, and to self-hosted apps like Gitea, timed out
from inside the container and sat in the admin Log. They are now skipped
when queueing; already queued or failed ones are dropped at worker start.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-12 13:40:27 -07:00

274 lines
9.0 KiB
Python

"""
Outgoing webmention link extraction and queueing
Per background-worker.md and architect's answers:
- Extract links from rendered HTML (not markdown)
- Extract only <a href> elements, exclude <pre> and <code> blocks
- Queue max 10 links per note per operation
- Priority: in_reply_to first, then document order
References:
- Background Worker Design: /docs/design/v1.6.0/background-worker.md
- API Contract: /docs/design/v1.6.0/api-contract.md
"""
import logging
import sqlite3
from pathlib import Path
from typing import Optional
from lxml import html as lxml_html
from urllib.parse import urlparse
logger = logging.getLogger(__name__)
SKIPPED_NOTE = "Not sent: links to this site and WEBMENTION_SKIP_HOSTS are skipped"
def skipped_hosts(config) -> set[str]:
"""Hosts that never get webmentions: this site and WEBMENTION_SKIP_HOSTS (v2.18.3)"""
hosts = set(config.get("WEBMENTION_SKIP_HOSTS") or [])
site = urlparse(config.get("SITE_URL") or "").hostname
if site:
hosts.add(site.lower())
return hosts
def is_skipped(url: str, hosts: set[str]) -> bool:
"""True if url is on one of the skipped hosts (exact host, any port)"""
return (urlparse(url).hostname or "").lower() in hosts
def extract_links_from_html(html_content: str, exclude_code_blocks: bool = True) -> list[str]:
"""
Extract links from HTML content.
Per architect's answer to Q1:
- Extract from rendered HTML (not markdown)
- Extract <a href> elements only
- Exclude links in <pre> and <code> blocks
- Return in document order
Args:
html_content: Rendered HTML content
exclude_code_blocks: If True, skip links in <pre> and <code> (default: True)
Returns:
List of URLs in document order
"""
if not html_content or not html_content.strip():
return []
try:
# Parse HTML inside a wrapper, so a top-level <pre> can be removed too
doc = lxml_html.fragment_fromstring(html_content, create_parent='div')
if exclude_code_blocks:
# Remove <pre> and <code> elements before extracting links
for element in doc.xpath('.//pre | .//code'):
element.drop_tree()
# Extract all <a href> attributes in document order
links = doc.xpath('.//a/@href')
# Filter to valid absolute URLs (http/https)
absolute_links = []
for link in links:
parsed = urlparse(link)
if parsed.scheme in ('http', 'https') and parsed.netloc:
absolute_links.append(link)
return absolute_links
except Exception as e:
logger.warning(f"Failed to parse HTML for link extraction: {e}")
return []
def _dedupe(urls: list) -> list:
"""Drop empty and repeated URLs, keeping the first occurrence"""
seen = set()
result = []
for url in urls:
if url and url not in seen:
seen.add(url)
result.append(url)
return result
def previous_targets(db_path: Path, note_id: int) -> list[str]:
"""Targets this note has been queued to send webmentions to before"""
conn = sqlite3.connect(db_path)
try:
rows = conn.execute(
"SELECT target_url FROM webmention_queue WHERE source_note_id = ? ORDER BY id",
(note_id,),
).fetchall()
finally:
conn.close()
return [row[0] for row in rows]
def requeue_previous_webmentions(db_path: Path, note_id: int, detach: bool = False) -> int:
"""
Re-send every webmention this note has sent (spec 3.1.5 deleted posts)
Args:
db_path: Path to SQLite database
note_id: Note being deleted or restored
detach: Clear source_note_id so the rows survive a hard delete
(the foreign key cascades)
Returns:
Number of webmentions queued
"""
conn = sqlite3.connect(db_path)
try:
cursor = conn.execute(f"""
UPDATE webmention_queue
SET status = 'pending',
retry_count = 0,
next_retry_at = NULL,
last_error = NULL,
endpoint_url = NULL,
created_at = CURRENT_TIMESTAMP
{', source_note_id = NULL' if detach else ''}
WHERE source_note_id = ?
""", (note_id,))
conn.commit()
queued = cursor.rowcount
finally:
conn.close()
if queued:
logger.info(f"Re-queued {queued} webmentions for note {note_id}")
return queued
def note_responses_changed(db_path: Path, note_ids) -> int:
"""
Re-send a note's webmentions because the responses it shows changed
Spec 3.1.4: if the source post displays responses (e.g. comments), a
change to them is an update of the post. Only published, undeleted notes.
Args:
db_path: Path to SQLite database
note_ids: IDs of notes whose displayed responses changed
Returns:
Number of webmentions queued
"""
ids = sorted({i for i in note_ids if i is not None})
if not ids:
return 0
conn = sqlite3.connect(db_path)
try:
placeholders = ','.join('?' * len(ids))
live = [row[0] for row in conn.execute(
f"SELECT id FROM notes WHERE id IN ({placeholders}) "
"AND published = 1 AND deleted_at IS NULL", ids,
)]
finally:
conn.close()
return sum(requeue_previous_webmentions(db_path, note_id) for note_id in live)
def queue_webmentions_for_note(
db_path: Path,
note_id: int,
note_url: str,
content_html: str,
in_reply_to: Optional[str] = None,
old_content_html: Optional[str] = None # Phase 6: update semantics
) -> int:
"""
Extract links and queue webmentions for a note.
Per architect's answers (Phase 6):
- Called from notes.py AFTER successful database commit
- For creates: reply target first, then links in document order (max 10)
- For updates: the same, plus every previously sent target and links
removed by the edit (spec 3.1.4)
- Webmention sending is best-effort; failures do not affect note creation
Args:
db_path: Path to SQLite database
note_id: Database ID of the note
note_url: Full URL of the published note
content_html: Rendered HTML content (None if unpublishing)
in_reply_to: Reply-to URL (if reply post)
old_content_html: Previous HTML (for update diff, Phase 6)
Returns:
Number of webmentions queued
"""
# Links to this site or a skipped host are dropped before the cap (v2.18.3):
# this site only takes webmentions for its posts, and its own public
# address may not be reachable from inside the container
from flask import current_app, has_app_context
hosts = skipped_hosts(current_app.config) if has_app_context() else set()
def wanted(urls):
return [u for u in urls if not is_skipped(u, hosts)]
# Targets for the current content: reply target first, then links in
# document order, at most 10
links = extract_links_from_html(content_html) if content_html else []
current = wanted(_dedupe(([in_reply_to] if in_reply_to else []) + links))[:10]
if old_content_html is not None:
# Update (spec 3.1.4): also re-send to every target notified before,
# and to links the edit removed, so they can update or remove the
# mention. These are not capped.
removed = [u for u in extract_links_from_html(old_content_html) if u not in links]
target_urls = _dedupe(current + wanted(previous_targets(db_path, note_id) + removed))
else:
target_urls = current
if not target_urls:
logger.debug(f"No links to send webmentions for note {note_id}")
return 0
# Queue webmentions
queued = 0
try:
conn = sqlite3.connect(db_path)
cursor = conn.cursor()
for target_url in target_urls:
try:
# Use UPSERT to handle duplicate sends
# Phase 6: Force endpoint rediscovery per W3C MUST requirement
cursor.execute("""
INSERT INTO webmention_queue
(source_url, target_url, source_note_id, status, created_at)
VALUES (?, ?, ?, 'pending', CURRENT_TIMESTAMP)
ON CONFLICT(source_url, target_url) DO UPDATE SET
status = 'pending',
retry_count = 0,
next_retry_at = NULL,
last_error = NULL,
endpoint_url = NULL,
created_at = CURRENT_TIMESTAMP
""", (note_url, target_url, note_id))
queued += 1
except Exception as e:
logger.warning(
f"Failed to queue webmention {note_url} -> {target_url}: {e}"
)
conn.commit()
conn.close()
logger.info(f"Queued {queued} webmentions for note {note_id}")
except Exception as e:
logger.error(f"Failed to queue webmentions for note {note_id}: {e}")
return queued