From 4aaa7a812267714c218965a753c456bb215deb31 Mon Sep 17 00:00:00 2001 From: Phil Date: Tue, 3 Nov 2026 08:07:04 -0700 Subject: [PATCH] feat: W3C Webmention compliance (v2.7.0) Sending: RFC 8288 Link header parsing, rel token matching and document order for /, endpoint re-discovery on every send, re-send to previous targets on update/delete/undelete, 410 Gone for deleted notes, no redirect following on the POST, 429 retried. Receiving: exact target match per media type, Accept header, final URL for relative links, only published notes accepted, fragments ignored, moderation kept on updates, transient errors keep valid mentions. Security: every redirect hop SSRF-checked, all resolved addresses checked, broader blocklist, total time and size limits. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 29 ++ .../2026-11-03-webmention-spec-compliance.md | 50 ++++ starpunk/__init__.py | 6 +- starpunk/errors.py | 5 + starpunk/notes.py | 17 ++ starpunk/routes/public.py | 25 +- starpunk/routes/webmention.py | 34 ++- starpunk/webmention_processor.py | 176 ++++++----- starpunk/webmention_sender.py | 141 +++++---- starpunk/webmention_ssrf.py | 194 ++++++++---- starpunk/webmention_worker.py | 265 ++++++++-------- templates/410.html | 11 + templates/note.html | 2 +- tests/test_micropub_v240.py | 2 +- tests/test_webmention_spec.py | 283 ++++++++++++++++++ tests/test_webmention_worker.py | 11 +- 16 files changed, 895 insertions(+), 356 deletions(-) create mode 100644 docs/design/v2.7.0/2026-11-03-webmention-spec-compliance.md create mode 100644 templates/410.html create mode 100644 tests/test_webmention_spec.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f37eae..5dec0a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,35 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [2.7.0] - 2026-11-03 + +Webmention now follows the [W3C Recommendation](https://www.w3.org/TR/webmention/) throughout. + +### Changed + +- Webmentions are only accepted for published notes. Mentions of the home page or other pages were stored but never shown; they're now refused with 400 +- Deleted notes answer 410 Gone with a "This post has been deleted" page instead of 404 +- A received mention that is sent again keeps its moderation: one you hid stays hidden, one you approved stays approved + +### Fixed + +- Sending: endpoint discovery now handles `rel` values with several words (`rel="webmention other"`), any capitalisation, `rel` after other Link parameters, and several Link headers, and takes the first `` or `` in page order +- Sending: the endpoint is looked up again every time (the 1–24 hour cache broke the spec's rule for updated posts) +- Sending: editing a note notifies every page it mentioned before, including an old reply target and links the edit removed. Deleting or restoring a note notifies them too, so they can remove or restore the mention +- Sending: a redirect from the endpoint is no longer treated as success (the POST was turned into a GET); "429 Too Many Requests" is retried +- Receiving: the source must link to the exact target URL (in a link, image, or other HTML attribute, any JSON value, or plain text). A longer URL that starts with the target, or the target mentioned in plain text or an HTML comment, no longer counts +- Receiving: relative links in the source are resolved, using the URL after redirects +- Receiving: the source is fetched with an `Accept` header and decoded in its declared character set +- Receiving: if the source can't be reached when a mention is sent again, the existing mention stays up instead of disappearing +- Receiving: the target's host is compared without regard to case or a default port, and `#fragment`s are dropped, so one mention isn't stored twice +- Mention snippets cut at 500 characters no longer leave HTML tags open, and the "Mentions" list no longer shows raw HTML tags +- The `Link: rel="webmention"` header is added alongside other Link headers instead of replacing them +- Links at the top level of a code block were still sent webmentions + +### Security + +- Fetches for webmentions check every redirect, not just the first URL, so a public page can't redirect StarPunk to a private address. Every address a hostname resolves to (IPv4 and IPv6) must be public; carrier-grade NAT, `0.0.0.0/8`, multicast, reserved and IPv4-mapped addresses are blocked too. Each fetch has a total time limit and a 1 MB size limit, including endpoint discovery + ## [2.6.9] - 2026-11-03 ### Fixed diff --git a/docs/design/v2.7.0/2026-11-03-webmention-spec-compliance.md b/docs/design/v2.7.0/2026-11-03-webmention-spec-compliance.md new file mode 100644 index 0000000..0e93921 --- /dev/null +++ b/docs/design/v2.7.0/2026-11-03-webmention-spec-compliance.md @@ -0,0 +1,50 @@ +# v2.7.0: Webmention W3C compliance + +Review of StarPunk's Webmention support against the W3C Recommendation +(https://www.w3.org/TR/webmention/, 12 January 2017), and the changes made. +v2.6.8 and v2.6.9 fixed the bugs that stopped webmentions working at all. + +## Sending + +| Section | Requirement | Change | +|---|---|---| +| 3.1.2 | Link header with rel=webmention first (MUST) | RFC 8288 parsing: several links per header, several headers, `rel` in any position, token list, case-insensitive | +| 3.1.2 | Then first ``/`` rel=webmention in document order (MUST) | Single pass over `link` and `a` in document order; rel token match; elements without `href` skipped | +| 3.1.2 | Relative URL resolved against target; empty href is the target; query kept (MUST) | `urljoin` against the final URL after redirects | +| 3.1.2 | Follow redirects | `safe_get` follows up to 10, checking each hop | +| 3.1.2 | User-Agent with "Webmention" (MAY) | `StarPunk/ Webmention (+site)` | +| 3.1.3 | Form-encoded POST; any 2xx success (MUST) | Unchanged. Redirects not followed (httpx turned 301/302 POST into GET); 429 retried | +| 3.1.4 | Re-discover endpoint on update (MUST) | Discovery cache removed | +| 3.1.4 | Re-send to previously sent targets, including removed links (SHOULD) | `previous_targets()` from the queue, plus removed links; not capped | +| 3.1.5 | Deleted post returns 410 (SHOULD), tombstone (SHOULD) | `public.note` aborts 410 for soft-deleted notes and hard-deleted notes that sent webmentions; `410.html` | +| 3.1.5 | Re-send on delete (SHOULD) | `requeue_previous_webmentions()`; for a hard delete the rows are detached (`source_note_id = NULL`) first, since the foreign key cascades. Also on undelete | + +## Receiving + +| Section | Requirement | Change | +|---|---|---| +| 3.2 | 202 for async processing (MUST) | Unchanged | +| 3.2.1 | Valid http(s) URLs; source ≠ target; 400 on error (MUST) | Unchanged | +| 3.2.1 | Target is a valid resource (SHOULD) | Only published notes; others 400 | +| 3.2.1 | Fragment ignored | Stripped before storing (no duplicate rows) | +| 3.2.2 | GET source, follow redirects, limit redirects | `safe_get` | +| 3.2.2 | Accept header (SHOULD) | Added | +| 3.2.2 | Exact match of target, per media type (MUST) | HTML: every link attribute (`lxml iterlinks`) resolved against the final URL; JSON: any string value; `text/*`: substring; other types: no match. Fragments ignored | +| 3.2.4 | Update existing data; 410 / no link removes it (SHOULD) | Unchanged in effect. Moderation decisions kept on update; transient errors keep a valid mention | + +## Security (section 4) + +- 4.1 XSS: author URL/photo must be http(s) (v2.6.9); snippets re-sanitized after truncation; Mentions list uses `striptags`. +- 4.2 / 4.5: `safe_get` checks every redirect hop, checks all `getaddrinfo` addresses, blocks private, loopback, link-local, multicast, reserved, unspecified, CGNAT, `0.0.0.0/8` and IPv4-mapped forms; 1 MB and 10 s total limits. +- 4.3: endpoint URL checked before POST, no redirects followed. + +## Known limits + +- DNS is checked before connecting, not pinned for the connection, so a + rebinding DNS server could still answer differently on connect. +- Homepage mentions are refused rather than stored. +- `error` rows (unexpected failures) are not retried automatically. + +## Tests + +`tests/test_webmention_spec.py` (section-numbered), `tests/test_webmention_worker.py`. diff --git a/starpunk/__init__.py b/starpunk/__init__.py index 41d31ad..0eb7fd9 100644 --- a/starpunk/__init__.py +++ b/starpunk/__init__.py @@ -302,7 +302,7 @@ def create_app(config=None): if (app.config.get('WEBMENTION_ENABLED', True) and response.content_type and 'text/html' in response.content_type): - response.headers['Link'] = '; rel="webmention"' + response.headers.add('Link', '; rel="webmention"') return response # Register centralized error handlers @@ -459,5 +459,5 @@ def create_app(config=None): # Package version (Semantic Versioning 2.0.0) # See docs/standards/versioning-strategy.md for details -__version__ = "2.6.9" -__version_info__ = (2, 6, 9) +__version__ = "2.7.0" +__version_info__ = (2, 7, 0) diff --git a/starpunk/errors.py b/starpunk/errors.py index 60fcb7a..5cfde8a 100644 --- a/starpunk/errors.py +++ b/starpunk/errors.py @@ -82,6 +82,11 @@ def register_error_handlers(app): return render_template('404.html'), 404 + @app.errorhandler(410) + def gone(error): + """Handle 410 Gone (deleted notes, Webmention 3.1.5)""" + return render_template('410.html'), 410 + @app.errorhandler(405) def method_not_allowed(error): """Handle 405 Method Not Allowed errors""" diff --git a/starpunk/notes.py b/starpunk/notes.py index cf5192d..cf80c03 100644 --- a/starpunk/notes.py +++ b/starpunk/notes.py @@ -989,6 +989,17 @@ def delete_note( "Path validation failed", ) + # Re-send webmentions so receivers learn the post is gone (spec 3.1.5). + # A hard delete would cascade the queue rows away, so detach them first. + if existing_note.published: + try: + from starpunk.webmention_sender import requeue_previous_webmentions + requeue_previous_webmentions( + Path(current_app.config["DATABASE_PATH"]), existing_note.id, detach=not soft + ) + except Exception as e: + current_app.logger.warning(f"Failed to queue webmentions for deleted note: {e}") + # 5. PERFORM DELETION db = get_db(current_app) @@ -1102,4 +1113,10 @@ def undelete_note(slug: str) -> Note: except Exception as e: current_app.logger.warning(f"Failed to queue syndication after undelete: {e}") + try: + from starpunk.webmention_sender import requeue_previous_webmentions + requeue_previous_webmentions(Path(current_app.config["DATABASE_PATH"]), row["id"]) + except Exception as e: + current_app.logger.warning(f"Failed to queue webmentions after undelete: {e}") + return get_note(slug=slug) diff --git a/starpunk/routes/public.py b/starpunk/routes/public.py index d1c3474..e2589b3 100644 --- a/starpunk/routes/public.py +++ b/starpunk/routes/public.py @@ -294,6 +294,27 @@ def likes(): return render_template("likes.html", notes=notes) +def _note_was_deleted(slug: str) -> bool: + """ + True if this slug was a note that has been deleted + + Soft-deleted notes keep their row; hard-deleted notes that sent + webmentions keep their queue rows (detached from the note). + """ + from starpunk.database import get_db + from starpunk.utils import absolute_url_for + + db = get_db(current_app) + if db.execute( + "SELECT 1 FROM notes WHERE slug = ? AND deleted_at IS NOT NULL", (slug,) + ).fetchone(): + return True + return db.execute( + "SELECT 1 FROM webmention_queue WHERE source_url = ? AND source_note_id IS NULL", + (absolute_url_for("public.note", slug=slug),), + ).fetchone() is not None + + @bp.route("/note/") def note(slug: str): """ @@ -319,8 +340,10 @@ def note(slug: str): # Get note by slug note_obj = get_note(slug=slug) - # Return 404 if note doesn't exist or isn't published + # Deleted notes are 410 Gone (Webmention 3.1.5); unknown or draft 404 if not note_obj or not note_obj.published: + if not note_obj and _note_was_deleted(slug): + abort(410) abort(404) # Get media for note (v1.2.0 Phase 3) diff --git a/starpunk/routes/webmention.py b/starpunk/routes/webmention.py index be8267c..7b35a2e 100644 --- a/starpunk/routes/webmention.py +++ b/starpunk/routes/webmention.py @@ -11,7 +11,7 @@ References: """ from flask import Blueprint, current_app, request, jsonify -from urllib.parse import urlparse +from urllib.parse import urldefrag, urlparse from starpunk.database import get_db from starpunk.notes import get_note, NoteNotFoundError from starpunk.rate_limiter import check_rate_limit @@ -107,17 +107,24 @@ def validate_url(url: str) -> tuple[bool, str | None]: return False, "URL is malformed" +def _host_port(url: str) -> tuple[str, int | None]: + parsed = urlparse(url) + default = {'http': 80, 'https': 443}.get(parsed.scheme.lower()) + try: + port = parsed.port or default + except ValueError: + port = None + return (parsed.hostname or '').lower(), port + + def is_target_on_site(target_url: str) -> bool: """ Check if target URL is on this site. - Compares hostname of target_url with SITE_URL configuration. + Compares host (case-insensitively) and effective port with SITE_URL. """ site_url = current_app.config.get('SITE_URL', '') - site_parsed = urlparse(site_url) - target_parsed = urlparse(target_url) - - return site_parsed.netloc == target_parsed.netloc + return _host_port(site_url) == _host_port(target_url) def resolve_target_note(target_url: str) -> int | None: @@ -228,16 +235,17 @@ def receive_webmention(): "error_description": "Target URL is not on this site" }), 400 - # Resolve target to note (may be None for homepage/other pages) - target_note_id = resolve_target_note(target) + # The fragment is ignored (spec 3.2.1) and not stored, so target#a and + # target#b are the same mention + target = urldefrag(target)[0] - # If target looks like a note URL but doesn't resolve, it's not found - target_parsed = urlparse(target) - target_path = target_parsed.path.rstrip('/') - if target_path.startswith('/note/') and target_note_id is None: + # Only published notes accept webmentions (spec 3.2.1: target must be a + # valid resource that accepts them) + target_note_id = resolve_target_note(target) + if target_note_id is None: return jsonify({ "error": "invalid_target", - "error_description": "Target URL not found" + "error_description": "Target URL does not accept webmentions" }), 400 # Insert webmention into database (queued for validation) diff --git a/starpunk/webmention_processor.py b/starpunk/webmention_processor.py index b6a0cb4..41843a3 100644 --- a/starpunk/webmention_processor.py +++ b/starpunk/webmention_processor.py @@ -14,15 +14,18 @@ References: - W3C Webmention: https://www.w3.org/TR/webmention/ """ +import json import logging import mf2py import nh3 import httpx from datetime import datetime from typing import Dict, Any, Optional -from urllib.parse import urlparse +from urllib.parse import urldefrag, urlparse -from starpunk.webmention_ssrf import validate_url_for_fetch, SSRFBlockedError +from starpunk.webmention_ssrf import ( + FetchTooLargeError, SSRFBlockedError, safe_get, user_agent, +) logger = logging.getLogger(__name__) @@ -33,103 +36,113 @@ class ValidationError(Exception): pass -def fetch_source(url: str, site_url: str, timeout: int = 10, max_size: int = 1048576) -> tuple[str, str]: +ACCEPT = ('text/html, application/xhtml+xml;q=0.9, application/json;q=0.8, ' + 'text/plain;q=0.7, */*;q=0.1') + + +class TransientError(ValidationError): + """Source could not be checked right now (network error, 5xx)""" + pass + + +def fetch_source(url: str, site_url: str) -> tuple[str, str, str]: """ Fetch source URL content with SSRF protection. + Per spec 3.2.2: GET following redirects (limited), with an Accept header. + Args: url: Source URL to fetch site_url: Site URL for User-Agent header - timeout: Request timeout in seconds (default: 10) - max_size: Maximum content size in bytes (default: 1MB) Returns: - Tuple of (content, content_type) + Tuple of (content, content_type, final_url) Raises: - SSRFBlockedError: If URL points to private IP - ValidationError: If fetch fails or content too large + ValidationError: Source is blocked, too large, or returned 4xx + TransientError: Network error, timeout or 5xx """ - # SSRF protection try: - validate_url_for_fetch(url) + result = safe_get(url, headers={'User-Agent': user_agent(site_url), 'Accept': ACCEPT}) except (SSRFBlockedError, ValueError) as e: logger.warning(f"SSRF protection blocked URL {url}: {e}") raise ValidationError(f"Blocked request: {e}") - - # Fetch URL - try: - # Use streaming to check size before downloading entire content - with httpx.stream( - 'GET', - url, - timeout=timeout, - headers={'User-Agent': f'StarPunk/1.6.0 (+{site_url})'}, - follow_redirects=True - ) as response: - response.raise_for_status() - - # Check content length if provided - content_length = response.headers.get('Content-Length') - if content_length and int(content_length) > max_size: - raise ValidationError(f"Source exceeds size limit ({max_size} bytes)") - - # Read content with size limit - content = b'' - for chunk in response.iter_bytes(chunk_size=8192): - content += chunk - if len(content) > max_size: - raise ValidationError(f"Source exceeds size limit ({max_size} bytes)") - - content_type = response.headers.get('Content-Type', '') - - return content.decode('utf-8', errors='replace'), content_type - + except FetchTooLargeError as e: + raise ValidationError(f"Source {e}") except httpx.TimeoutException: - raise ValidationError("Source fetch timed out") - except httpx.HTTPStatusError as e: - if 400 <= e.response.status_code < 500: - raise ValidationError(f"Source returned {e.response.status_code}") - else: - raise ValidationError(f"Source server error: {e.response.status_code}") + raise TransientError("Source fetch timed out") except httpx.HTTPError as e: - raise ValidationError(f"Failed to fetch source: {e}") - except UnicodeDecodeError: - raise ValidationError("Source content encoding is not supported") + raise TransientError(f"Failed to fetch source: {e}") + + if 400 <= result.status_code < 500: + raise ValidationError(f"Source returned {result.status_code}") + if result.status_code >= 300: + raise TransientError(f"Source server error: {result.status_code}") + + return result.text, result.headers.get('Content-Type', ''), result.url -def verify_link_presence(content: str, target_url: str, content_type: str) -> bool: +def _strip_fragment(url: str) -> str: + return urldefrag(url.strip())[0] + + +def _json_strings(value): + """Yield every string in a parsed JSON document""" + if isinstance(value, str): + yield value + elif isinstance(value, dict): + for item in value.values(): + yield from _json_strings(item) + elif isinstance(value, list): + for item in value: + yield from _json_strings(item) + + +def verify_link_presence(content: str, target_url: str, content_type: str, + source_url: str = '') -> bool: """ - Verify that source content contains a link to target URL. + Verify the source links to the target (spec 3.2.2). + + The source MUST have an exact match of the target URL, checked per media + type: in HTML, any link-bearing attribute (a/link href, img/video/audio + src, ...) resolved against the source URL; in JSON, any string value; in + other text, the URL anywhere. Fragments are ignored on both sides. Args: - content: Source content (HTML or text) + content: Source content target_url: Target URL to find content_type: Content type of source + source_url: Final source URL, for resolving relative links Returns: - True if target link found, False otherwise + True if the target is linked, False otherwise """ - # Normalize URLs for comparison (remove fragments, trailing slashes) - target_normalized = urlparse(target_url)._replace(fragment='').geturl().rstrip('/') + target = _strip_fragment(target_url) + media_type = content_type.split(';')[0].strip().lower() - # For HTML, parse and check all links - if 'html' in content_type.lower(): + if 'html' in media_type: + from lxml import html as lxml_html try: - from lxml import html as lxml_html - doc = lxml_html.fromstring(content) - links = doc.xpath('//a/@href') - - for link in links: - # Make absolute if relative - link_normalized = urlparse(link)._replace(fragment='').geturl().rstrip('/') - if link_normalized == target_normalized: - return True + doc = lxml_html.document_fromstring(content) except Exception as e: - logger.debug(f"HTML parsing failed, falling back to string search: {e}") + logger.debug(f"HTML parsing failed: {e}") + return False + if source_url: + doc.make_links_absolute(source_url, resolve_base_href=True, + handle_failures='ignore') + return any(_strip_fragment(link) == target for _, _, link, _ in doc.iterlinks()) - # Fallback: simple string search - return target_normalized in content + if media_type == 'application/json' or media_type.endswith('+json'): + try: + data = json.loads(content) + except ValueError: + return False + return any(_strip_fragment(s) == target for s in _json_strings(data)) + + if media_type.startswith('text/'): + return target in content + + return False def detect_mention_type(parsed_mf2: Dict, target_url: str) -> str: @@ -292,12 +305,7 @@ def extract_content(parsed_mf2: Dict) -> Optional[str]: content = content_data if content: - # Sanitize HTML - sanitized = sanitize_content(content) - # Truncate to 500 chars - if len(sanitized) > 500: - sanitized = sanitized[:497] + '...' - return sanitized + return _snippet(content) # Try p-summary as fallback summary = props.get('summary', []) @@ -306,14 +314,20 @@ def extract_content(parsed_mf2: Dict) -> Optional[str]: if isinstance(content, dict): content = content.get('value', '') if content: - sanitized = sanitize_content(content) - if len(sanitized) > 500: - sanitized = sanitized[:497] + '...' - return sanitized + return _snippet(content) return None +def _snippet(html: str, limit: int = 500) -> str: + """Sanitized HTML of at most ~limit characters, with tags closed""" + sanitized = sanitize_content(html) + if len(sanitized) > limit: + # Cutting can leave a tag or attribute open; sanitizing again closes it + sanitized = sanitize_content(sanitized[:limit - 3] + '...') + return sanitized + + def sanitize_content(html: str) -> str: """ Sanitize HTML content using nh3. @@ -362,17 +376,17 @@ def process_webmention(source_url: str, target_url: str, site_url: str) -> Dict[ ValidationError: If validation fails with specific error """ # Fetch source content - content, content_type = fetch_source(source_url, site_url) + content, content_type, final_url = fetch_source(source_url, site_url) # Verify target link presence - if not verify_link_presence(content, target_url, content_type): + if not verify_link_presence(content, target_url, content_type, final_url): raise ValidationError("Source does not link to target") # Parse microformats (only for HTML) parsed_mf2 = None if 'html' in content_type.lower(): try: - parsed_mf2 = mf2py.parse(doc=content, url=source_url) + parsed_mf2 = mf2py.parse(doc=content, url=final_url) except Exception as e: logger.debug(f"Microformat parsing failed for {source_url}: {e}") diff --git a/starpunk/webmention_sender.py b/starpunk/webmention_sender.py index f583374..11aec57 100644 --- a/starpunk/webmention_sender.py +++ b/starpunk/webmention_sender.py @@ -14,9 +14,8 @@ References: import logging import sqlite3 -from datetime import datetime from pathlib import Path -from typing import Optional, Set +from typing import Optional from lxml import html as lxml_html from urllib.parse import urlparse @@ -45,16 +44,16 @@ def extract_links_from_html(html_content: str, exclude_code_blocks: bool = True) return [] try: - # Parse HTML - doc = lxml_html.fromstring(html_content) + # Parse HTML inside a wrapper, so a top-level
 can be removed too
+        doc = lxml_html.fragment_fromstring(html_content, create_parent='div')
 
         if exclude_code_blocks:
             # Remove 
 and  elements before extracting links
-            for element in doc.xpath('//pre | //code'):
-                element.getparent().remove(element)
+            for element in doc.xpath('.//pre | .//code'):
+                element.drop_tree()
 
         # Extract all  attributes in document order
-        links = doc.xpath('//a/@href')
+        links = doc.xpath('.//a/@href')
 
         # Filter to valid absolute URLs (http/https)
         absolute_links = []
@@ -70,6 +69,66 @@ def extract_links_from_html(html_content: str, exclude_code_blocks: bool = True)
         return []
 
 
+def _dedupe(urls: list) -> list:
+    """Drop empty and repeated URLs, keeping the first occurrence"""
+    seen = set()
+    result = []
+    for url in urls:
+        if url and url not in seen:
+            seen.add(url)
+            result.append(url)
+    return result
+
+
+def previous_targets(db_path: Path, note_id: int) -> list[str]:
+    """Targets this note has been queued to send webmentions to before"""
+    conn = sqlite3.connect(db_path)
+    try:
+        rows = conn.execute(
+            "SELECT target_url FROM webmention_queue WHERE source_note_id = ? ORDER BY id",
+            (note_id,),
+        ).fetchall()
+    finally:
+        conn.close()
+    return [row[0] for row in rows]
+
+
+def requeue_previous_webmentions(db_path: Path, note_id: int, detach: bool = False) -> int:
+    """
+    Re-send every webmention this note has sent (spec 3.1.5 deleted posts)
+
+    Args:
+        db_path: Path to SQLite database
+        note_id: Note being deleted or restored
+        detach: Clear source_note_id so the rows survive a hard delete
+            (the foreign key cascades)
+
+    Returns:
+        Number of webmentions queued
+    """
+    conn = sqlite3.connect(db_path)
+    try:
+        cursor = conn.execute(f"""
+            UPDATE webmention_queue
+            SET status = 'pending',
+                retry_count = 0,
+                next_retry_at = NULL,
+                last_error = NULL,
+                endpoint_url = NULL,
+                created_at = CURRENT_TIMESTAMP
+                {', source_note_id = NULL' if detach else ''}
+            WHERE source_note_id = ?
+        """, (note_id,))
+        conn.commit()
+        queued = cursor.rowcount
+    finally:
+        conn.close()
+
+    if queued:
+        logger.info(f"Re-queued {queued} webmentions for note {note_id}")
+    return queued
+
+
 def queue_webmentions_for_note(
     db_path: Path,
     note_id: int,
@@ -83,10 +142,9 @@ def queue_webmentions_for_note(
 
     Per architect's answers (Phase 6):
     - Called from notes.py AFTER successful database commit
-    - For updates: diff old vs new URLs, queue for new/existing/removed
-    - For creates: queue for all URLs in content
-    - Max 10 webmentions per operation
-    - Priority: in_reply_to URL first, then document order
+    - For creates: reply target first, then links in document order (max 10)
+    - For updates: the same, plus every previously sent target and links
+      removed by the edit (spec 3.1.4)
     - Webmention sending is best-effort; failures do not affect note creation
 
     Args:
@@ -100,58 +158,19 @@ def queue_webmentions_for_note(
     Returns:
         Number of webmentions queued
     """
-    # Phase 6: Implement URL diffing for updates
+    # Targets for the current content: reply target first, then links in
+    # document order, at most 10
+    links = extract_links_from_html(content_html) if content_html else []
+    current = _dedupe(([in_reply_to] if in_reply_to else []) + links)[:10]
+
     if old_content_html is not None:
-        # This is an update - compute URL diff
-        old_links = set(extract_links_from_html(old_content_html))
-
-        # Handle unpublishing case (content_html is None)
-        if content_html is None:
-            # Unpublishing - queue for all old URLs (they're all removed)
-            new_links = set()
-        else:
-            new_links = set(extract_links_from_html(content_html))
-
-        # Compute sets for URL diffing
-        new_urls = new_links - old_links  # In new, not in old
-        existing_urls = new_links & old_links  # In both (re-validate)
-        removed_urls = old_links - new_links  # In old, not in new
-
-        # Build target list: new + existing + removed
-        # (All need webmentions per W3C spec)
-        all_update_urls = list(new_urls) + list(existing_urls) + list(removed_urls)
-
-        # Add in_reply_to if changed
-        target_urls = []
-        if in_reply_to and in_reply_to not in all_update_urls:
-            target_urls.append(in_reply_to)
-
-        # Add the diffed URLs
-        target_urls.extend(all_update_urls)
-
-        # Limit to 10
-        target_urls = target_urls[:10]
-
+        # Update (spec 3.1.4): also re-send to every target notified before,
+        # and to links the edit removed, so they can update or remove the
+        # mention. These are not capped.
+        removed = [u for u in extract_links_from_html(old_content_html) if u not in links]
+        target_urls = _dedupe(current + previous_targets(db_path, note_id) + removed)
     else:
-        # This is a create (no old content) - use original logic
-        links = extract_links_from_html(content_html) if content_html else []
-
-        # Build priority-ordered list of target URLs
-        target_urls = []
-
-        # 1. in_reply_to gets highest priority
-        if in_reply_to:
-            target_urls.append(in_reply_to)
-
-        # 2. Add extracted links in document order (deduplicated)
-        seen = {in_reply_to} if in_reply_to else set()
-        for link in links:
-            if link not in seen:
-                target_urls.append(link)
-                seen.add(link)
-
-        # 3. Limit to 10 webmentions per operation
-        target_urls = target_urls[:10]
+        target_urls = current
 
     if not target_urls:
         logger.debug(f"No links to send webmentions for note {note_id}")
diff --git a/starpunk/webmention_ssrf.py b/starpunk/webmention_ssrf.py
index 5f0c794..b0dadc6 100644
--- a/starpunk/webmention_ssrf.py
+++ b/starpunk/webmention_ssrf.py
@@ -1,113 +1,203 @@
 """
-SSRF protection utilities for webmention processing
+SSRF protection for webmention fetches
 
-Per ADR-063 Decision 9 and background-worker.md:
-- Block private IP address ranges
-- Resolve hostname before making requests
+Per ADR-063 Decision 9 and Webmention sections 4.2, 4.3 and 4.5:
+- Every address a hostname resolves to must be public
+- Redirects are followed by hand so each hop is checked
+- Redirect count, response size and total time are limited
 - DEV_MODE exception for localhost testing
-- Applied to all outgoing HTTP requests
 
 References:
     - ADR-063: Webmention Implementation
-    - Background Worker Design: /docs/design/v1.6.0/background-worker.md
+    - W3C Webmention: https://www.w3.org/TR/webmention/#security-considerations
 """
 
 import ipaddress
 import socket
+import time
+from dataclasses import dataclass
+from urllib.parse import urljoin, urlparse
+
+import httpx
 from flask import current_app
 
+MAX_REDIRECTS = 10
+MAX_BYTES = 1024 * 1024
+TIMEOUT = 10
+
+# Tests can set an httpx transport here (e.g. httpx.MockTransport)
+TRANSPORT = None
+
+CGNAT = ipaddress.ip_network('100.64.0.0/10')
+THIS_NETWORK = ipaddress.ip_network('0.0.0.0/8')
+
 
 class SSRFBlockedError(Exception):
     """Raised when SSRF protection blocks a request"""
     pass
 
 
-# Private IP ranges per RFC 1918 and link-local ranges
-BLOCKED_NETWORKS = [
-    ipaddress.ip_network('10.0.0.0/8'),         # RFC 1918 private
-    ipaddress.ip_network('172.16.0.0/12'),      # RFC 1918 private
-    ipaddress.ip_network('192.168.0.0/16'),     # RFC 1918 private
-    ipaddress.ip_network('127.0.0.0/8'),        # Localhost
-    ipaddress.ip_network('169.254.0.0/16'),     # Link-local
-    ipaddress.ip_network('::1/128'),            # IPv6 localhost
-    ipaddress.ip_network('fc00::/7'),           # IPv6 unique local
-    ipaddress.ip_network('fe80::/10'),          # IPv6 link-local
-]
+class FetchTooLargeError(Exception):
+    """Raised when a response is larger than the limit"""
+    pass
+
+
+def user_agent(site_url: str) -> str:
+    """User-Agent for webmention requests (spec 3.1.2 suggests including 'Webmention')"""
+    from starpunk import __version__
+    return f"StarPunk/{__version__} Webmention (+{site_url})"
 
 
 def is_private_ip(ip: str) -> bool:
     """
-    Check if IP address is in a private/blocked range.
+    Check if an IP address must not be fetched.
+
+    Blocks private, loopback, link-local, multicast, reserved, unspecified,
+    carrier-grade NAT and IPv4-mapped forms of these.
 
     Args:
         ip: IP address string (IPv4 or IPv6)
 
     Returns:
-        True if IP is private/blocked, False otherwise
+        True if IP is blocked, False otherwise
     """
     try:
-        addr = ipaddress.ip_address(ip)
-        return any(addr in network for network in BLOCKED_NETWORKS)
+        addr = ipaddress.ip_address(ip.split('%')[0])
     except ValueError:
-        # Invalid IP format, treat as blocked
         return True
 
+    if isinstance(addr, ipaddress.IPv6Address) and addr.ipv4_mapped:
+        addr = addr.ipv4_mapped
+
+    if isinstance(addr, ipaddress.IPv4Address) and (addr in CGNAT or addr in THIS_NETWORK):
+        return True
+
+    return (
+        addr.is_private
+        or addr.is_loopback
+        or addr.is_link_local
+        or addr.is_multicast
+        or addr.is_reserved
+        or addr.is_unspecified
+    )
+
 
 def safe_resolve(hostname: str) -> str:
     """
-    Resolve hostname and validate IP is not private.
+    Resolve hostname and check every address is public.
 
     Per background-worker.md: DEV_MODE exception for localhost testing.
 
-    Args:
-        hostname: Hostname to resolve
-
     Returns:
-        Resolved IP address if safe
+        First resolved IP address
 
     Raises:
-        SSRFBlockedError: If hostname resolves to private IP
+        SSRFBlockedError: If any address is blocked
         socket.gaierror: If hostname cannot be resolved
     """
-    # DEV_MODE exception for localhost
     if current_app.config.get('DEV_MODE') and hostname in ('localhost', '127.0.0.1', '::1'):
         return '127.0.0.1'
 
-    # Resolve hostname
-    ip = socket.gethostbyname(hostname)
+    addresses = sorted({info[4][0] for info in socket.getaddrinfo(hostname, None)})
+    for ip in addresses:
+        if is_private_ip(ip):
+            raise SSRFBlockedError(f"Blocked request to private IP: {ip}")
 
-    # Check if resolved IP is private
-    if is_private_ip(ip):
-        raise SSRFBlockedError(f"Blocked request to private IP: {ip}")
-
-    return ip
+    return addresses[0]
 
 
 def validate_url_for_fetch(url: str) -> None:
     """
-    Validate URL is safe to fetch (not private IP).
-
-    Args:
-        url: URL to validate
+    Validate URL is safe to fetch (http(s), public addresses only).
 
     Raises:
-        SSRFBlockedError: If URL points to private IP
-        ValueError: If URL is malformed
+        SSRFBlockedError: If URL points to a blocked address
+        ValueError: If URL is malformed or its host does not resolve
     """
-    from urllib.parse import urlparse
-
     parsed = urlparse(url)
 
-    if not parsed.scheme in ('http', 'https'):
+    if parsed.scheme not in ('http', 'https'):
         raise ValueError(f"Unsupported URL scheme: {parsed.scheme}")
 
-    if not parsed.netloc:
+    if not parsed.hostname:
         raise ValueError("URL must include a hostname")
 
-    # Extract hostname (remove port if present)
-    hostname = parsed.hostname
-    if not hostname:
-        raise ValueError("Could not extract hostname from URL")
+    try:
+        safe_resolve(parsed.hostname)
+    except socket.gaierror as e:
+        raise ValueError(f"Could not resolve {parsed.hostname}: {e}")
 
-    # Validate hostname resolves to public IP
-    safe_resolve(hostname)
+
+@dataclass
+class FetchResult:
+    """A fetched response, after redirects"""
+    url: str
+    status_code: int
+    headers: httpx.Headers
+    content: bytes
+    encoding: str
+
+    @property
+    def text(self) -> str:
+        try:
+            return self.content.decode(self.encoding, errors='replace')
+        except LookupError:
+            return self.content.decode('utf-8', errors='replace')
+
+
+def safe_get(url: str, headers: dict, max_bytes: int = MAX_BYTES,
+             max_redirects: int = MAX_REDIRECTS, timeout: float = TIMEOUT) -> FetchResult:
+    """
+    GET a URL, following redirects by hand and checking each hop.
+
+    Args:
+        url: URL to fetch
+        headers: Request headers
+        max_bytes: Response body limit
+        max_redirects: Redirect limit
+        timeout: Total time limit in seconds
+
+    Returns:
+        FetchResult for the final response (any status code)
+
+    Raises:
+        SSRFBlockedError, ValueError: URL or a redirect target is not allowed
+        FetchTooLargeError: Body exceeds max_bytes
+        httpx.HTTPError: Network errors, too many redirects, time limit
+    """
+    deadline = time.monotonic() + timeout
+
+    with httpx.Client(follow_redirects=False, transport=TRANSPORT) as client:
+        for _ in range(max_redirects + 1):
+            validate_url_for_fetch(url)
+            remaining = deadline - time.monotonic()
+            if remaining <= 0:
+                raise httpx.TimeoutException("Fetch exceeded time limit")
+
+            with client.stream('GET', url, headers=headers, timeout=remaining) as response:
+                location = response.headers.get('Location')
+                if response.is_redirect and location:
+                    url = urljoin(url, location)
+                    continue
+
+                content_length = response.headers.get('Content-Length', '')
+                if content_length.isdigit() and int(content_length) > max_bytes:
+                    raise FetchTooLargeError(f"Response exceeds size limit ({max_bytes} bytes)")
+
+                content = b''
+                for chunk in response.iter_bytes(chunk_size=8192):
+                    content += chunk
+                    if len(content) > max_bytes:
+                        raise FetchTooLargeError(f"Response exceeds size limit ({max_bytes} bytes)")
+                    if time.monotonic() > deadline:
+                        raise httpx.TimeoutException("Fetch exceeded time limit")
+
+                return FetchResult(
+                    url=url,
+                    status_code=response.status_code,
+                    headers=response.headers,
+                    content=content,
+                    encoding=response.charset_encoding or 'utf-8',
+                )
+
+    raise httpx.TooManyRedirects(f"More than {max_redirects} redirects")
diff --git a/starpunk/webmention_worker.py b/starpunk/webmention_worker.py
index a14d02b..22e7f02 100644
--- a/starpunk/webmention_worker.py
+++ b/starpunk/webmention_worker.py
@@ -14,28 +14,30 @@ References:
 
 import fcntl
 import logging
+import re
 import threading
 import time
 import sqlite3
 import httpx
 from datetime import datetime, timedelta
 from pathlib import Path
-from typing import Dict, Any, Optional, TypedDict
-from urllib.parse import urlparse
+from typing import Dict, Any, Optional
+from urllib.parse import urljoin
 from lxml import html as lxml_html
 
-from starpunk.webmention_processor import process_webmention, ValidationError
-from starpunk.webmention_ssrf import validate_url_for_fetch, SSRFBlockedError
+from starpunk.webmention_processor import process_webmention, TransientError, ValidationError
+from starpunk.webmention_ssrf import (
+    FetchTooLargeError, SSRFBlockedError, safe_get, user_agent, validate_url_for_fetch,
+)
 
 
 logger = logging.getLogger(__name__)
 
+# Link header (RFC 8288):  then ;-separated params; quoted values may
+# contain , and ;
+LINK_RE = re.compile(r'<([^>]*)>((?:\s*;\s*[^;,"=\s]+\s*(?:=\s*(?:"[^"]*"|[^;,]*))?)*)')
+LINK_PARAM_RE = re.compile(r';\s*([^;,"=\s]+)\s*(?:=\s*(?:"([^"]*)"|([^;,]*)))?')
 
-# Discovery cache entry type
-class DiscoveryCacheEntry(TypedDict):
-    """Cache entry for endpoint discovery results"""
-    endpoint: Optional[str]  # None = negative cache (no endpoint found)
-    expires_at: datetime
 
 
 class WebmentionWorker(threading.Thread):
@@ -79,10 +81,6 @@ class WebmentionWorker(threading.Thread):
         self._last_processed = None
         self._lock_file = None
 
-        # Phase 3: Discovery cache (worker instance variable)
-        # Per architect's answer to Q4
-        self._discovery_cache: Dict[str, DiscoveryCacheEntry] = {}
-
     def run(self):
         """
         Main processing loop
@@ -266,25 +264,34 @@ class WebmentionWorker(threading.Thread):
 
             moderation_status = 'approved' if self.auto_approve else 'pending'
 
+            # An update to an existing mention (spec 3.2.4) keeps the admin's
+            # moderation decision; only a first-time valid mention gets the
+            # default. Previously valid = validated with no error.
             cursor.execute("""
                 UPDATE webmentions
-                SET validation_status = 'valid',
+                SET moderation_status = CASE
+                        WHEN moderation_status IN ('hidden', 'deleted') THEN moderation_status
+                        WHEN validated_at IS NOT NULL AND validation_error IS NULL
+                            THEN moderation_status
+                        ELSE ?
+                    END,
+                    validation_status = 'valid',
+                    validation_error = NULL,
                     validated_at = CURRENT_TIMESTAMP,
                     mention_type = ?,
                     author_name = ?,
                     author_url = ?,
                     author_photo = ?,
                     content_snippet = ?,
-                    moderation_status = ?,
                     updated_at = CURRENT_TIMESTAMP
                 WHERE id = ?
             """, (
+                moderation_status,
                 result['mention_type'],
                 result['author_name'],
                 result['author_url'],
                 result['author_photo'],
                 result['content_snippet'],
-                moderation_status,
                 webmention_id
             ))
 
@@ -293,11 +300,17 @@ class WebmentionWorker(threading.Thread):
 
             logger.info(
                 f"Validated webmention: {source_url} -> {target_url} "
-                f"(type={result['mention_type']}, status={moderation_status})"
+                f"(type={result['mention_type']})"
             )
 
+        except TransientError as e:
+            # Source unreachable right now: keep a previously valid mention
+            self._mark_validation_failed(webmention_id, source_url, target_url, str(e),
+                                         transient=True)
+
         except ValidationError as e:
-            # Validation failed - update with error
+            # Validation failed (410/4xx, no link): the mention is removed
+            # from display (spec 3.2.4)
             self._mark_validation_failed(webmention_id, source_url, target_url, str(e))
 
         except Exception as e:
@@ -315,11 +328,15 @@ class WebmentionWorker(threading.Thread):
         webmention_id: int,
         source_url: str,
         target_url: str,
-        error: str
+        error: str,
+        transient: bool = False
     ):
         """
         Mark webmention validation as failed/invalid
 
+        A transient failure of a mention that was valid before restores it to
+        valid with its previous data, rather than hiding it.
+
         Args:
             webmention_id: Database ID
             source_url: Source URL
@@ -329,7 +346,9 @@ class WebmentionWorker(threading.Thread):
         # Determine if error or invalid
         # Per background-worker.md: HTTP 4xx and "no link" are invalid
         # All other errors are 'error' status
-        if any(x in error.lower() for x in ['does not link', 'returned 4']):
+        if transient:
+            status = 'error'
+        elif any(x in error.lower() for x in ['does not link', 'returned 4']):
             status = 'invalid'
         else:
             status = 'error'
@@ -338,6 +357,22 @@ class WebmentionWorker(threading.Thread):
             conn = sqlite3.connect(self.db_path)
             cursor = conn.cursor()
 
+            if transient:
+                cursor.execute("""
+                    UPDATE webmentions
+                    SET validation_status = 'valid',
+                        updated_at = CURRENT_TIMESTAMP
+                    WHERE id = ? AND validated_at IS NOT NULL AND validation_error IS NULL
+                """, (webmention_id,))
+                if cursor.rowcount:
+                    conn.commit()
+                    conn.close()
+                    logger.warning(
+                        f"Could not re-check webmention, keeping it: "
+                        f"{source_url} -> {target_url}: {error}"
+                    )
+                    return
+
             cursor.execute("""
                 UPDATE webmentions
                 SET validation_status = ?,
@@ -458,11 +493,13 @@ class WebmentionWorker(threading.Thread):
                 conn.commit()
                 conn.close()
 
-            # Send webmention POST
+            # Send webmention POST (spec 3.1.3)
             try:
-                # SSRF protection for endpoint URL
+                # SSRF protection for endpoint URL (spec 4.3)
                 validate_url_for_fetch(endpoint)
 
+                # Redirects are not followed: httpx would turn a 301/302 POST
+                # into a GET without the body
                 response = httpx.post(
                     endpoint,
                     data={
@@ -470,25 +507,24 @@ class WebmentionWorker(threading.Thread):
                         'target': target_url
                     },
                     timeout=10,
-                    headers={'User-Agent': f'StarPunk/1.6.0 (+{self.site_url})'},
-                    follow_redirects=True
+                    headers={'User-Agent': user_agent(self.site_url)},
+                    follow_redirects=False
                 )
 
-                # Check response
+                # Any 2xx is success (spec 3.1.3)
                 if 200 <= response.status_code < 300:
-                    # Success
                     self._mark_send_success(queue_id, source_url, target_url)
 
-                elif 400 <= response.status_code < 500:
-                    # Permanent failure (4xx)
-                    error = f"HTTP {response.status_code}: {response.text[:200]}"
-                    self._mark_send_failed(queue_id, source_url, target_url, error)
-
-                else:
-                    # Temporary failure (5xx) - retry
+                elif response.status_code == 429 or response.status_code >= 500:
+                    # Rate limited or server error - retry
                     error = f"HTTP {response.status_code}: {response.text[:200]}"
                     self._mark_send_retry(queue_id, source_url, target_url, retry_count, error)
 
+                else:
+                    # Permanent failure (3xx, 4xx)
+                    error = f"HTTP {response.status_code}: {response.text[:200]}"
+                    self._mark_send_failed(queue_id, source_url, target_url, error)
+
             except (httpx.TimeoutException, httpx.HTTPError) as e:
                 # Connection error - retry
                 self._mark_send_retry(queue_id, source_url, target_url, retry_count, str(e))
@@ -504,100 +540,81 @@ class WebmentionWorker(threading.Thread):
 
     def _discover_endpoint(self, target_url: str) -> Optional[str]:
         """
-        Discover webmention endpoint for target URL with caching
+        Discover the webmention endpoint for a target URL (spec 3.1.2)
 
-        Per architect's answers to Q3, Q4, Q10:
-        - GET only (no HEAD)
-        - Check Link header →  → 
-        - First match wins
-        - Cache results: 1h for found, 24h for not found
-        - SSRF protection on target URL
-
-        Args:
-            target_url: Target URL to discover endpoint for
+        - GET the target, following redirects (each hop SSRF-checked)
+        - The first Link header with rel=webmention wins, then the first
+           or  with rel=webmention and an href, in document order
+        - rel is a space-separated token list, matched case-insensitively
+        - Relative endpoints resolve against the final URL; an empty href
+          is the target itself; query strings are kept
+        - Not cached: an updated post must re-discover (spec 3.1.4)
 
         Returns:
-            Endpoint URL if found, None otherwise
-        """
-        # Check cache first
-        cached = self._discovery_cache.get(target_url)
-        if cached and cached['expires_at'] > datetime.utcnow():
-            logger.debug(f"Using cached endpoint for {target_url}: {cached['endpoint']}")
-            return cached['endpoint']
+            Endpoint URL, or None if the target has none (or is blocked/4xx)
 
-        # SSRF protection
+        Raises:
+            httpx.HTTPError: Network errors and 5xx, so the caller retries
+        """
         try:
-            validate_url_for_fetch(target_url)
+            result = safe_get(target_url, headers={'User-Agent': user_agent(self.site_url)})
         except (SSRFBlockedError, ValueError) as e:
             logger.warning(f"SSRF protection blocked target {target_url}: {e}")
-            # Cache negative result
-            self._cache_endpoint(target_url, None, hours=24)
+            return None
+        except FetchTooLargeError:
+            logger.info(f"Target too large for discovery: {target_url}")
             return None
 
-        # Fetch target URL
-        try:
-            response = httpx.get(
-                target_url,
-                timeout=10,
-                headers={'User-Agent': f'StarPunk/1.6.0 (+{self.site_url})'},
-                follow_redirects=True
-            )
-            response.raise_for_status()
-
-            # 1. Check Link header
-            link_header = response.headers.get('Link', '')
-            if link_header:
-                endpoint = self._parse_link_header(link_header, str(response.url))
-                if endpoint:
-                    self._cache_endpoint(target_url, endpoint, hours=1)
-                    return endpoint
-
-            # 2. Parse HTML for  and  elements
-            content_type = response.headers.get('Content-Type', '')
-            if 'html' in content_type.lower():
-                endpoint = self._parse_html_for_endpoint(response.text, str(response.url))
-                if endpoint:
-                    self._cache_endpoint(target_url, endpoint, hours=1)
-                    return endpoint
-
-            # No endpoint found - cache negative result
-            logger.info(f"No webmention endpoint found for {target_url}")
-            self._cache_endpoint(target_url, None, hours=24)
+        if result.status_code >= 500:
+            raise httpx.HTTPError(f"Target returned {result.status_code}")
+        if result.status_code >= 400:
+            logger.info(f"Target returned {result.status_code}: {target_url}")
             return None
 
-        except Exception as e:
-            # Don't cache; the caller schedules a retry
-            logger.warning(f"Failed to discover endpoint for {target_url}: {e}")
-            raise
+        for link_header in result.headers.get_list('Link'):
+            endpoint = self._parse_link_header(link_header, result.url)
+            if endpoint is not None:
+                return endpoint
+
+        content_type = result.headers.get('Content-Type', '')
+        if 'html' in content_type.lower():
+            endpoint = self._parse_html_for_endpoint(result.text, result.url)
+            if endpoint is not None:
+                return endpoint
+
+        logger.info(f"No webmention endpoint found for {target_url}")
+        return None
+
+    @staticmethod
+    def _has_webmention_rel(rel: str) -> bool:
+        return 'webmention' in rel.lower().split()
 
     def _parse_link_header(self, link_header: str, base_url: str) -> Optional[str]:
         """
-        Parse Link header for webmention endpoint
+        Parse a Link header (RFC 8288) for a rel=webmention link
 
         Args:
-            link_header: Raw Link header value
+            link_header: Link header value (may hold several links)
             base_url: Base URL for resolving relative URLs
 
         Returns:
-            Absolute endpoint URL if found, None otherwise
+            Absolute endpoint URL of the first match, None otherwise
         """
-        # Simple parsing - look for rel="webmention"
-        # Format: ; rel="webmention"
-        import re
-        pattern = r'<([^>]+)>\s*;\s*rel\s*=\s*["\']?webmention["\']?'
-        match = re.search(pattern, link_header, re.IGNORECASE)
-
-        if match:
-            endpoint = match.group(1)
-            # Make absolute if relative
-            from urllib.parse import urljoin
-            return urljoin(base_url, endpoint)
+        for match in LINK_RE.finditer(link_header):
+            uri, params = match.group(1), match.group(2)
+            for param in LINK_PARAM_RE.finditer(params):
+                if param.group(1).lower() != 'rel':
+                    continue
+                rel = param.group(2) if param.group(2) is not None else (param.group(3) or '')
+                if self._has_webmention_rel(rel):
+                    return urljoin(base_url, uri.strip())
+                break  # only the first rel parameter counts (RFC 8288)
 
         return None
 
     def _parse_html_for_endpoint(self, html_content: str, base_url: str) -> Optional[str]:
         """
-        Parse HTML for webmention endpoint in  or  elements
+        Find the first  or  with rel=webmention, in document order
 
         Args:
             html_content: HTML content
@@ -607,48 +624,18 @@ class WebmentionWorker(threading.Thread):
             Absolute endpoint URL if found, None otherwise
         """
         try:
-            doc = lxml_html.fromstring(html_content)
-
-            # Check  first
-            links = doc.xpath('//link[@rel="webmention"]/@href')
-            if links:
-                from urllib.parse import urljoin
-                return urljoin(base_url, links[0])
-
-            # Check 
-            anchors = doc.xpath('//a[@rel="webmention"]/@href')
-            if anchors:
-                from urllib.parse import urljoin
-                return urljoin(base_url, anchors[0])
-
+            doc = lxml_html.document_fromstring(html_content)
         except Exception as e:
             logger.debug(f"HTML parsing failed: {e}")
+            return None
+
+        for element in doc.iter('link', 'a'):
+            href = element.get('href')
+            if href is not None and self._has_webmention_rel(element.get('rel', '')):
+                return urljoin(base_url, href.strip())
 
         return None
 
-    def _cache_endpoint(self, target_url: str, endpoint: Optional[str], hours: int):
-        """
-        Cache endpoint discovery result
-
-        Args:
-            target_url: Target URL
-            endpoint: Discovered endpoint (None = negative cache)
-            hours: TTL in hours
-        """
-        self._discovery_cache[target_url] = {
-            'endpoint': endpoint,
-            'expires_at': datetime.utcnow() + timedelta(hours=hours)
-        }
-
-        # Optional: Evict old entries (keep cache bounded)
-        if len(self._discovery_cache) > 1000:
-            # Remove expired entries
-            now = datetime.utcnow()
-            self._discovery_cache = {
-                k: v for k, v in self._discovery_cache.items()
-                if v['expires_at'] > now
-            }
-
     def _mark_send_success(self, queue_id: int, source_url: str, target_url: str):
         """Mark webmention send as successful"""
         try:
diff --git a/templates/410.html b/templates/410.html
new file mode 100644
index 0000000..9d56c51
--- /dev/null
+++ b/templates/410.html
@@ -0,0 +1,11 @@
+{% extends "base.html" %}
+
+{% block title %}Post Deleted - {{ config.SITE_NAME }}{% endblock %}
+
+{% block content %}
+
+{% endblock %}
diff --git a/templates/note.html b/templates/note.html
index c111540..dc0c805 100644
--- a/templates/note.html
+++ b/templates/note.html
@@ -185,7 +185,7 @@
       
{{ mention.author_name or mention.source_url }} {% if mention.content_snippet %} -

{{ mention.content_snippet|truncate(200) }}

+

{{ mention.content_snippet|striptags|truncate(200) }}

{% endif %}
{% endfor %} diff --git a/tests/test_micropub_v240.py b/tests/test_micropub_v240.py index f370b17..788b2e3 100644 --- a/tests/test_micropub_v240.py +++ b/tests/test_micropub_v240.py @@ -161,7 +161,7 @@ class TestDelete: url = create(client, h, category=["kept"]) path = url.replace(SITE, "") assert client.post("/micropub", data={"action": "delete", "url": url}, headers=h).status_code == 204 - assert client.get(path).status_code == 404 + assert client.get(path).status_code == 410 # Gone (Webmention 3.1.5) response = client.post("/micropub", json={"action": "undelete", "url": url}, headers=h) assert response.status_code == 204 assert client.get(path).status_code == 200 diff --git a/tests/test_webmention_spec.py b/tests/test_webmention_spec.py new file mode 100644 index 0000000..086e96e --- /dev/null +++ b/tests/test_webmention_spec.py @@ -0,0 +1,283 @@ +""" +W3C Webmention conformance tests (v2.7.0) + +https://www.w3.org/TR/webmention/ — section numbers in test names. +""" + +import sqlite3 +from unittest.mock import patch + +import httpx +import pytest + +from starpunk import webmention_ssrf +from starpunk.webmention_processor import ( + TransientError, ValidationError, fetch_source, verify_link_presence, +) +from starpunk.webmention_worker import WebmentionWorker + +PUBLIC = [(2, 1, 6, '', ('93.184.216.34', 0))] +BASE = 'https://t.example/test/1?x=1' + + +@pytest.fixture +def worker(app): + return WebmentionWorker(app, db_path=app.config["DATABASE_PATH"], + site_url=app.config["SITE_URL"]) + + +@pytest.fixture +def mock_web(): + """Route safe_get through a MockTransport; all hosts resolve public""" + routes = {} + + def handler(request): + return routes[str(request.url)](request) + + with patch("starpunk.webmention_ssrf.socket.getaddrinfo", return_value=PUBLIC), \ + patch("starpunk.webmention_ssrf.TRANSPORT", httpx.MockTransport(handler)): + yield routes + + +# 3.1.2 Sender discovers receiver Webmention endpoint + +@pytest.mark.parametrize("header,expected", [ + ('; rel="webmention"', 'https://e.example/wm'), + ('; rel=webmention', 'https://t.example/wm'), + ('; rel="other webmention"', 'https://t.example/test/wm'), + ('; title="a;b,c"; rel="webmention"', 'https://t.example/test/wm'), + ('; REL="WebMention"', 'https://t.example/test/wm'), + ('; rel="other", ; rel="webmention"', 'https://t.example/wm?q=1'), + ('; rel="webmention-not"', None), + ('; rel="not-webmention"', None), +]) +def test_3_1_2_link_header(worker, header, expected): + assert worker._parse_link_header(header, BASE) == expected + + +@pytest.mark.parametrize("html,expected", [ + # before : first in document order wins + ('x', 'https://t.example/a'), + ('', 'https://t.example/l'), + ('', 'https://t.example/l'), + # empty href is the target itself + ('', BASE), + # no href: skipped + ('', 'https://t.example/a'), + # comments, escaped markup and other rels are ignored + ('' + '' + '<a rel="webmention" href="/bad">' + '', 'https://t.example/ok'), + ('

nothing

', None), +]) +def test_3_1_2_html(worker, html, expected): + assert worker._parse_html_for_endpoint(html, BASE) == expected + + +def test_3_1_2_header_before_html_and_relative_to_final_url(app, worker, mock_web): + mock_web['https://t.example/old'] = lambda r: httpx.Response( + 301, headers={'Location': 'https://t.example/new/page'}) + mock_web['https://t.example/new/page'] = lambda r: httpx.Response( + 200, headers={'Link': '; rel="webmention"', 'Content-Type': 'text/html'}, + text='') + with app.app_context(): + assert worker._discover_endpoint('https://t.example/old') == 'https://t.example/new/wm' + + +def test_3_1_2_multiple_link_headers(app, worker, mock_web): + mock_web['https://t.example/p'] = lambda r: httpx.Response( + 200, headers=[('Link', '; rel="other"'), ('Link', '; rel="webmention"')]) + with app.app_context(): + assert worker._discover_endpoint('https://t.example/p') == 'https://t.example/wm' + + +def test_3_1_2_server_error_is_retried_not_final(app, worker, mock_web): + mock_web['https://t.example/p'] = lambda r: httpx.Response(503) + with app.app_context(), pytest.raises(httpx.HTTPError): + worker._discover_endpoint('https://t.example/p') + + +# 4.5 SSRF: every redirect hop is checked + +def test_4_5_redirect_to_private_address_blocked(app): + def resolve(host, *a, **k): + ip = '169.254.169.254' if host == 'metadata.internal' else '93.184.216.34' + return [(2, 1, 6, '', (ip, 0))] + + def handler(request): + return httpx.Response(302, headers={'Location': 'http://metadata.internal/'}) + + with app.app_context(), \ + patch("starpunk.webmention_ssrf.socket.getaddrinfo", side_effect=resolve), \ + patch("starpunk.webmention_ssrf.TRANSPORT", httpx.MockTransport(handler)): + with pytest.raises(ValidationError, match="Blocked"): + fetch_source('https://evil.example/', 'http://localhost:5000') + + +@pytest.mark.parametrize("ip", [ + '127.0.0.1', '10.1.2.3', '169.254.169.254', '0.0.0.0', '100.64.0.1', + '::1', '::ffff:127.0.0.1', 'fe80::1', '224.0.0.1', +]) +def test_4_5_blocked_addresses(ip): + assert webmention_ssrf.is_private_ip(ip) + + +def test_4_5_any_blocked_address_blocks(app): + mixed = PUBLIC + [(10, 1, 6, '', ('::1', 0, 0, 0))] + with app.app_context(), patch("starpunk.webmention_ssrf.socket.getaddrinfo", return_value=mixed): + with pytest.raises(webmention_ssrf.SSRFBlockedError): + webmention_ssrf.validate_url_for_fetch('https://x.example/') + + +# 3.2.2 Webmention verification + +TARGET = 'https://me.example/note/abc' + + +@pytest.mark.parametrize("content,content_type,expected", [ + ('x', 'text/html', True), + ('', 'text/html; charset=utf-8', True), + ('x', 'text/html', False), + ('

https://me.example/note/abc

', 'text/html', False), + ('', 'text/html', False), + ('{"a": ["https://me.example/note/abc"]}', 'application/json', True), + ('{"a": "https://me.example/note/abcdef"}', 'application/json', False), + ('see https://me.example/note/abc', 'text/plain', True), +]) +def test_3_2_2_exact_match(content, content_type, expected): + assert verify_link_presence(content, TARGET, content_type, 'https://me.example/x') is expected + + +def test_3_2_2_accept_header_and_status(app, mock_web): + seen = {} + + def page(request): + seen['accept'] = request.headers.get('Accept') + return httpx.Response(410) + + mock_web['https://s.example/p'] = page + with app.app_context(): + with pytest.raises(ValidationError, match="410"): + fetch_source('https://s.example/p', 'http://localhost:5000') + assert 'text/html' in seen['accept'] + + mock_web['https://s.example/p'] = lambda r: httpx.Response(500) + with app.app_context(), pytest.raises(TransientError): + fetch_source('https://s.example/p', 'http://localhost:5000') + + +# 3.2.1 Request verification + +def _post(client, source, target): + return client.post('/webmention', data={'source': source, 'target': target}) + + +def test_3_2_1_target_must_be_a_note(app, client, sample_note): + note_url = f"http://localhost:5000/note/{sample_note.slug}" + assert _post(client, 'https://s.example/a', note_url).status_code == 202 + assert _post(client, 'https://s.example/a', 'http://localhost:5000/nope').status_code == 400 + assert _post(client, 'https://s.example/a', 'http://localhost:5000/').status_code == 400 + assert _post(client, 'https://s.example/a', 'https://other.example/note/x').status_code == 400 + assert _post(client, note_url, note_url).status_code == 400 + assert _post(client, 'ftp://s.example/a', note_url).status_code == 400 + + +def test_3_2_1_host_case_and_fragment(app, client, sample_note): + note_url = f"http://LOCALHOST:5000/note/{sample_note.slug}" + assert _post(client, 'https://s.example/a', note_url + '#one').status_code == 202 + assert _post(client, 'https://s.example/a', note_url + '#two').status_code == 202 + db = sqlite3.connect(app.config['DATABASE_PATH']) + rows = db.execute("SELECT target_url FROM webmentions").fetchall() + assert rows == [(note_url,)] + + +# 3.2.4 Updating existing Webmentions + +def _insert_mention(app, note_id, **cols): + db = sqlite3.connect(app.config['DATABASE_PATH']) + fields = {'source_url': 'https://s.example/a', 'target_url': 'http://localhost:5000/n', + 'target_note_id': note_id, **cols} + db.execute(f"INSERT INTO webmentions ({', '.join(fields)}) VALUES ({', '.join('?' * len(fields))})", + tuple(fields.values())) + db.commit() + return db + + +VALID = {'valid': True, 'mention_type': 'reply', 'author_name': 'A', 'author_url': 'https://a.example/', + 'author_photo': None, 'content_snippet': 'hi'} + + +def test_3_2_4_update_keeps_moderation(app, worker, sample_note): + db = _insert_mention(app, sample_note.id, validation_status='valid', moderation_status='hidden', + validated_at='2026-01-01') + with patch("starpunk.webmention_worker.process_webmention", return_value=VALID): + worker._validate_webmention(1, 'https://s.example/a', 'http://localhost:5000/n', sample_note.id) + assert db.execute("SELECT validation_status, moderation_status FROM webmentions").fetchone() == ('valid', 'hidden') + + +def test_3_2_4_new_mention_gets_default_moderation(app, worker, sample_note): + db = _insert_mention(app, sample_note.id) + with patch("starpunk.webmention_worker.process_webmention", return_value=VALID): + worker._validate_webmention(1, 'https://s.example/a', 'http://localhost:5000/n', sample_note.id) + assert db.execute("SELECT validation_status, moderation_status FROM webmentions").fetchone() == ('valid', 'approved') + + +def test_3_2_4_transient_error_keeps_valid_mention(app, worker, sample_note): + db = _insert_mention(app, sample_note.id, validation_status='pending', moderation_status='approved', + validated_at='2026-01-01', content_snippet='kept') + with patch("starpunk.webmention_worker.process_webmention", side_effect=TransientError("timeout")): + worker._validate_webmention(1, 'https://s.example/a', 'http://localhost:5000/n', sample_note.id) + assert db.execute("SELECT validation_status, content_snippet FROM webmentions").fetchone() == ('valid', 'kept') + + +def test_3_2_4_gone_source_removes_mention(app, worker, sample_note): + db = _insert_mention(app, sample_note.id, validation_status='pending', moderation_status='approved', + validated_at='2026-01-01') + with patch("starpunk.webmention_worker.process_webmention", + side_effect=ValidationError("Source returned 410")): + worker._validate_webmention(1, 'https://s.example/a', 'http://localhost:5000/n', sample_note.id) + assert db.execute("SELECT validation_status FROM webmentions").fetchone() == ('invalid',) + + +# 3.1.4 Updated posts / 3.1.5 Deleted posts + +def _queue(app): + db = sqlite3.connect(app.config['DATABASE_PATH']) + return db.execute( + "SELECT target_url, status, source_note_id FROM webmention_queue ORDER BY target_url" + ).fetchall() + + +def _sent(app): + db = sqlite3.connect(app.config['DATABASE_PATH']) + db.execute("UPDATE webmention_queue SET status = 'sent'") + db.commit() + + +def test_3_1_4_update_resends_previous_targets(app): + from starpunk.notes import create_note, update_note + + with app.test_request_context(): + note = create_note("Reply", published=True, in_reply_to="https://a.example/post") + _sent(app) + update_note(id=note.id, content="Now about [b](https://b.example/x)", in_reply_to="") + targets = {t: s for t, s, _ in _queue(app)} + assert targets == {'https://a.example/post': 'pending', 'https://b.example/x': 'pending'} + + +@pytest.mark.parametrize("soft", [True, False]) +def test_3_1_5_delete_resends_and_returns_410(app, client, soft): + from starpunk.notes import create_note, delete_note + + with app.test_request_context(): + note = create_note("See [a](https://a.example/x)", published=True) + _sent(app) + delete_note(id=note.id, soft=soft) + + rows = _queue(app) + assert [(t, s) for t, s, _ in rows] == [('https://a.example/x', 'pending')] + if not soft: + assert rows[0][2] is None # detached, so it survives the delete + assert client.get(f"/note/{note.slug}").status_code == 410 + assert client.get("/note/never-existed").status_code == 404 diff --git a/tests/test_webmention_worker.py b/tests/test_webmention_worker.py index 8c375d1..bfd3e09 100644 --- a/tests/test_webmention_worker.py +++ b/tests/test_webmention_worker.py @@ -14,6 +14,8 @@ from unittest.mock import MagicMock, patch from starpunk.webmention_worker import WebmentionWorker +PUBLIC = [(2, 1, 6, '', ('93.184.216.34', 0))] + def _wait_for(predicate, timeout=5): deadline = time.time() + timeout @@ -35,14 +37,15 @@ def test_worker_sends_queued_webmention(app): conn.commit() conn.close() - page = httpx.Response(200, headers={"Link": '
; rel="webmention"'}, - request=httpx.Request("GET", "https://example.org/post")) + def target_site(request): + return httpx.Response(200, headers={"Link": '
; rel="webmention"'}) + accepted = MagicMock(status_code=202, text="") worker = WebmentionWorker(app, db_path=db_path, site_url=app.config["SITE_URL"], poll_interval=0.05) - with patch("starpunk.webmention_ssrf.socket.gethostbyname", return_value="93.184.216.34"), \ - patch("starpunk.webmention_worker.httpx.get", return_value=page), \ + with patch("starpunk.webmention_ssrf.socket.getaddrinfo", return_value=PUBLIC), \ + patch("starpunk.webmention_ssrf.TRANSPORT", httpx.MockTransport(target_site)), \ patch("starpunk.webmention_worker.httpx.post", return_value=accepted) as post: worker.start() try: