- Connect to the address that passed the SSRF check (Host header and SNI carry the hostname, so TLS still verifies it), closing the DNS rebinding gap (spec 4.5). The webmention POST uses the same path. - Treat a change in the responses a note shows as an update and re-send its webmentions (spec 3.1.4), from validation and from admin moderation. Unchanged re-validations send nothing, so there is no ping-pong. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
249 lines
8.0 KiB
Python
249 lines
8.0 KiB
Python
"""
|
|
Outgoing webmention link extraction and queueing
|
|
|
|
Per background-worker.md and architect's answers:
|
|
- Extract links from rendered HTML (not markdown)
|
|
- Extract only <a href> elements, exclude <pre> and <code> blocks
|
|
- Queue max 10 links per note per operation
|
|
- Priority: in_reply_to first, then document order
|
|
|
|
References:
|
|
- Background Worker Design: /docs/design/v1.6.0/background-worker.md
|
|
- API Contract: /docs/design/v1.6.0/api-contract.md
|
|
"""
|
|
|
|
import logging
|
|
import sqlite3
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
from lxml import html as lxml_html
|
|
from urllib.parse import urlparse
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def extract_links_from_html(html_content: str, exclude_code_blocks: bool = True) -> list[str]:
|
|
"""
|
|
Extract links from HTML content.
|
|
|
|
Per architect's answer to Q1:
|
|
- Extract from rendered HTML (not markdown)
|
|
- Extract <a href> elements only
|
|
- Exclude links in <pre> and <code> blocks
|
|
- Return in document order
|
|
|
|
Args:
|
|
html_content: Rendered HTML content
|
|
exclude_code_blocks: If True, skip links in <pre> and <code> (default: True)
|
|
|
|
Returns:
|
|
List of URLs in document order
|
|
"""
|
|
if not html_content or not html_content.strip():
|
|
return []
|
|
|
|
try:
|
|
# Parse HTML inside a wrapper, so a top-level <pre> can be removed too
|
|
doc = lxml_html.fragment_fromstring(html_content, create_parent='div')
|
|
|
|
if exclude_code_blocks:
|
|
# Remove <pre> and <code> elements before extracting links
|
|
for element in doc.xpath('.//pre | .//code'):
|
|
element.drop_tree()
|
|
|
|
# Extract all <a href> attributes in document order
|
|
links = doc.xpath('.//a/@href')
|
|
|
|
# Filter to valid absolute URLs (http/https)
|
|
absolute_links = []
|
|
for link in links:
|
|
parsed = urlparse(link)
|
|
if parsed.scheme in ('http', 'https') and parsed.netloc:
|
|
absolute_links.append(link)
|
|
|
|
return absolute_links
|
|
|
|
except Exception as e:
|
|
logger.warning(f"Failed to parse HTML for link extraction: {e}")
|
|
return []
|
|
|
|
|
|
def _dedupe(urls: list) -> list:
|
|
"""Drop empty and repeated URLs, keeping the first occurrence"""
|
|
seen = set()
|
|
result = []
|
|
for url in urls:
|
|
if url and url not in seen:
|
|
seen.add(url)
|
|
result.append(url)
|
|
return result
|
|
|
|
|
|
def previous_targets(db_path: Path, note_id: int) -> list[str]:
|
|
"""Targets this note has been queued to send webmentions to before"""
|
|
conn = sqlite3.connect(db_path)
|
|
try:
|
|
rows = conn.execute(
|
|
"SELECT target_url FROM webmention_queue WHERE source_note_id = ? ORDER BY id",
|
|
(note_id,),
|
|
).fetchall()
|
|
finally:
|
|
conn.close()
|
|
return [row[0] for row in rows]
|
|
|
|
|
|
def requeue_previous_webmentions(db_path: Path, note_id: int, detach: bool = False) -> int:
|
|
"""
|
|
Re-send every webmention this note has sent (spec 3.1.5 deleted posts)
|
|
|
|
Args:
|
|
db_path: Path to SQLite database
|
|
note_id: Note being deleted or restored
|
|
detach: Clear source_note_id so the rows survive a hard delete
|
|
(the foreign key cascades)
|
|
|
|
Returns:
|
|
Number of webmentions queued
|
|
"""
|
|
conn = sqlite3.connect(db_path)
|
|
try:
|
|
cursor = conn.execute(f"""
|
|
UPDATE webmention_queue
|
|
SET status = 'pending',
|
|
retry_count = 0,
|
|
next_retry_at = NULL,
|
|
last_error = NULL,
|
|
endpoint_url = NULL,
|
|
created_at = CURRENT_TIMESTAMP
|
|
{', source_note_id = NULL' if detach else ''}
|
|
WHERE source_note_id = ?
|
|
""", (note_id,))
|
|
conn.commit()
|
|
queued = cursor.rowcount
|
|
finally:
|
|
conn.close()
|
|
|
|
if queued:
|
|
logger.info(f"Re-queued {queued} webmentions for note {note_id}")
|
|
return queued
|
|
|
|
|
|
def note_responses_changed(db_path: Path, note_ids) -> int:
|
|
"""
|
|
Re-send a note's webmentions because the responses it shows changed
|
|
|
|
Spec 3.1.4: if the source post displays responses (e.g. comments), a
|
|
change to them is an update of the post. Only published, undeleted notes.
|
|
|
|
Args:
|
|
db_path: Path to SQLite database
|
|
note_ids: IDs of notes whose displayed responses changed
|
|
|
|
Returns:
|
|
Number of webmentions queued
|
|
"""
|
|
ids = sorted({i for i in note_ids if i is not None})
|
|
if not ids:
|
|
return 0
|
|
|
|
conn = sqlite3.connect(db_path)
|
|
try:
|
|
placeholders = ','.join('?' * len(ids))
|
|
live = [row[0] for row in conn.execute(
|
|
f"SELECT id FROM notes WHERE id IN ({placeholders}) "
|
|
"AND published = 1 AND deleted_at IS NULL", ids,
|
|
)]
|
|
finally:
|
|
conn.close()
|
|
|
|
return sum(requeue_previous_webmentions(db_path, note_id) for note_id in live)
|
|
|
|
|
|
def queue_webmentions_for_note(
|
|
db_path: Path,
|
|
note_id: int,
|
|
note_url: str,
|
|
content_html: str,
|
|
in_reply_to: Optional[str] = None,
|
|
old_content_html: Optional[str] = None # Phase 6: update semantics
|
|
) -> int:
|
|
"""
|
|
Extract links and queue webmentions for a note.
|
|
|
|
Per architect's answers (Phase 6):
|
|
- Called from notes.py AFTER successful database commit
|
|
- For creates: reply target first, then links in document order (max 10)
|
|
- For updates: the same, plus every previously sent target and links
|
|
removed by the edit (spec 3.1.4)
|
|
- Webmention sending is best-effort; failures do not affect note creation
|
|
|
|
Args:
|
|
db_path: Path to SQLite database
|
|
note_id: Database ID of the note
|
|
note_url: Full URL of the published note
|
|
content_html: Rendered HTML content (None if unpublishing)
|
|
in_reply_to: Reply-to URL (if reply post)
|
|
old_content_html: Previous HTML (for update diff, Phase 6)
|
|
|
|
Returns:
|
|
Number of webmentions queued
|
|
"""
|
|
# Targets for the current content: reply target first, then links in
|
|
# document order, at most 10
|
|
links = extract_links_from_html(content_html) if content_html else []
|
|
current = _dedupe(([in_reply_to] if in_reply_to else []) + links)[:10]
|
|
|
|
if old_content_html is not None:
|
|
# Update (spec 3.1.4): also re-send to every target notified before,
|
|
# and to links the edit removed, so they can update or remove the
|
|
# mention. These are not capped.
|
|
removed = [u for u in extract_links_from_html(old_content_html) if u not in links]
|
|
target_urls = _dedupe(current + previous_targets(db_path, note_id) + removed)
|
|
else:
|
|
target_urls = current
|
|
|
|
if not target_urls:
|
|
logger.debug(f"No links to send webmentions for note {note_id}")
|
|
return 0
|
|
|
|
# Queue webmentions
|
|
queued = 0
|
|
try:
|
|
conn = sqlite3.connect(db_path)
|
|
cursor = conn.cursor()
|
|
|
|
for target_url in target_urls:
|
|
try:
|
|
# Use UPSERT to handle duplicate sends
|
|
# Phase 6: Force endpoint rediscovery per W3C MUST requirement
|
|
cursor.execute("""
|
|
INSERT INTO webmention_queue
|
|
(source_url, target_url, source_note_id, status, created_at)
|
|
VALUES (?, ?, ?, 'pending', CURRENT_TIMESTAMP)
|
|
ON CONFLICT(source_url, target_url) DO UPDATE SET
|
|
status = 'pending',
|
|
retry_count = 0,
|
|
next_retry_at = NULL,
|
|
last_error = NULL,
|
|
endpoint_url = NULL,
|
|
created_at = CURRENT_TIMESTAMP
|
|
""", (note_url, target_url, note_id))
|
|
|
|
queued += 1
|
|
|
|
except Exception as e:
|
|
logger.warning(
|
|
f"Failed to queue webmention {note_url} -> {target_url}: {e}"
|
|
)
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
logger.info(f"Queued {queued} webmentions for note {note_id}")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to queue webmentions for note {note_id}: {e}")
|
|
|
|
return queued
|