410 - Post Deleted
+This post has been deleted.
+ + can be removed too
+ doc = lxml_html.fragment_fromstring(html_content, create_parent='div')
if exclude_code_blocks:
# Remove and elements before extracting links
- for element in doc.xpath('//pre | //code'):
- element.getparent().remove(element)
+ for element in doc.xpath('.//pre | .//code'):
+ element.drop_tree()
# Extract all attributes in document order
- links = doc.xpath('//a/@href')
+ links = doc.xpath('.//a/@href')
# Filter to valid absolute URLs (http/https)
absolute_links = []
@@ -70,6 +69,66 @@ def extract_links_from_html(html_content: str, exclude_code_blocks: bool = True)
return []
+def _dedupe(urls: list) -> list:
+ """Drop empty and repeated URLs, keeping the first occurrence"""
+ seen = set()
+ result = []
+ for url in urls:
+ if url and url not in seen:
+ seen.add(url)
+ result.append(url)
+ return result
+
+
+def previous_targets(db_path: Path, note_id: int) -> list[str]:
+ """Targets this note has been queued to send webmentions to before"""
+ conn = sqlite3.connect(db_path)
+ try:
+ rows = conn.execute(
+ "SELECT target_url FROM webmention_queue WHERE source_note_id = ? ORDER BY id",
+ (note_id,),
+ ).fetchall()
+ finally:
+ conn.close()
+ return [row[0] for row in rows]
+
+
+def requeue_previous_webmentions(db_path: Path, note_id: int, detach: bool = False) -> int:
+ """
+ Re-send every webmention this note has sent (spec 3.1.5 deleted posts)
+
+ Args:
+ db_path: Path to SQLite database
+ note_id: Note being deleted or restored
+ detach: Clear source_note_id so the rows survive a hard delete
+ (the foreign key cascades)
+
+ Returns:
+ Number of webmentions queued
+ """
+ conn = sqlite3.connect(db_path)
+ try:
+ cursor = conn.execute(f"""
+ UPDATE webmention_queue
+ SET status = 'pending',
+ retry_count = 0,
+ next_retry_at = NULL,
+ last_error = NULL,
+ endpoint_url = NULL,
+ created_at = CURRENT_TIMESTAMP
+ {', source_note_id = NULL' if detach else ''}
+ WHERE source_note_id = ?
+ """, (note_id,))
+ conn.commit()
+ queued = cursor.rowcount
+ finally:
+ conn.close()
+
+ if queued:
+ logger.info(f"Re-queued {queued} webmentions for note {note_id}")
+ return queued
+
+
def queue_webmentions_for_note(
db_path: Path,
note_id: int,
@@ -83,10 +142,9 @@ def queue_webmentions_for_note(
Per architect's answers (Phase 6):
- Called from notes.py AFTER successful database commit
- - For updates: diff old vs new URLs, queue for new/existing/removed
- - For creates: queue for all URLs in content
- - Max 10 webmentions per operation
- - Priority: in_reply_to URL first, then document order
+ - For creates: reply target first, then links in document order (max 10)
+ - For updates: the same, plus every previously sent target and links
+ removed by the edit (spec 3.1.4)
- Webmention sending is best-effort; failures do not affect note creation
Args:
@@ -100,58 +158,19 @@ def queue_webmentions_for_note(
Returns:
Number of webmentions queued
"""
- # Phase 6: Implement URL diffing for updates
+ # Targets for the current content: reply target first, then links in
+ # document order, at most 10
+ links = extract_links_from_html(content_html) if content_html else []
+ current = _dedupe(([in_reply_to] if in_reply_to else []) + links)[:10]
+
if old_content_html is not None:
- # This is an update - compute URL diff
- old_links = set(extract_links_from_html(old_content_html))
-
- # Handle unpublishing case (content_html is None)
- if content_html is None:
- # Unpublishing - queue for all old URLs (they're all removed)
- new_links = set()
- else:
- new_links = set(extract_links_from_html(content_html))
-
- # Compute sets for URL diffing
- new_urls = new_links - old_links # In new, not in old
- existing_urls = new_links & old_links # In both (re-validate)
- removed_urls = old_links - new_links # In old, not in new
-
- # Build target list: new + existing + removed
- # (All need webmentions per W3C spec)
- all_update_urls = list(new_urls) + list(existing_urls) + list(removed_urls)
-
- # Add in_reply_to if changed
- target_urls = []
- if in_reply_to and in_reply_to not in all_update_urls:
- target_urls.append(in_reply_to)
-
- # Add the diffed URLs
- target_urls.extend(all_update_urls)
-
- # Limit to 10
- target_urls = target_urls[:10]
-
+ # Update (spec 3.1.4): also re-send to every target notified before,
+ # and to links the edit removed, so they can update or remove the
+ # mention. These are not capped.
+ removed = [u for u in extract_links_from_html(old_content_html) if u not in links]
+ target_urls = _dedupe(current + previous_targets(db_path, note_id) + removed)
else:
- # This is a create (no old content) - use original logic
- links = extract_links_from_html(content_html) if content_html else []
-
- # Build priority-ordered list of target URLs
- target_urls = []
-
- # 1. in_reply_to gets highest priority
- if in_reply_to:
- target_urls.append(in_reply_to)
-
- # 2. Add extracted links in document order (deduplicated)
- seen = {in_reply_to} if in_reply_to else set()
- for link in links:
- if link not in seen:
- target_urls.append(link)
- seen.add(link)
-
- # 3. Limit to 10 webmentions per operation
- target_urls = target_urls[:10]
+ target_urls = current
if not target_urls:
logger.debug(f"No links to send webmentions for note {note_id}")
diff --git a/starpunk/webmention_ssrf.py b/starpunk/webmention_ssrf.py
index 5f0c794..b0dadc6 100644
--- a/starpunk/webmention_ssrf.py
+++ b/starpunk/webmention_ssrf.py
@@ -1,113 +1,203 @@
"""
-SSRF protection utilities for webmention processing
+SSRF protection for webmention fetches
-Per ADR-063 Decision 9 and background-worker.md:
-- Block private IP address ranges
-- Resolve hostname before making requests
+Per ADR-063 Decision 9 and Webmention sections 4.2, 4.3 and 4.5:
+- Every address a hostname resolves to must be public
+- Redirects are followed by hand so each hop is checked
+- Redirect count, response size and total time are limited
- DEV_MODE exception for localhost testing
-- Applied to all outgoing HTTP requests
References:
- ADR-063: Webmention Implementation
- - Background Worker Design: /docs/design/v1.6.0/background-worker.md
+ - W3C Webmention: https://www.w3.org/TR/webmention/#security-considerations
"""
import ipaddress
import socket
+import time
+from dataclasses import dataclass
+from urllib.parse import urljoin, urlparse
+
+import httpx
from flask import current_app
+MAX_REDIRECTS = 10
+MAX_BYTES = 1024 * 1024
+TIMEOUT = 10
+
+# Tests can set an httpx transport here (e.g. httpx.MockTransport)
+TRANSPORT = None
+
+CGNAT = ipaddress.ip_network('100.64.0.0/10')
+THIS_NETWORK = ipaddress.ip_network('0.0.0.0/8')
+
class SSRFBlockedError(Exception):
"""Raised when SSRF protection blocks a request"""
pass
-# Private IP ranges per RFC 1918 and link-local ranges
-BLOCKED_NETWORKS = [
- ipaddress.ip_network('10.0.0.0/8'), # RFC 1918 private
- ipaddress.ip_network('172.16.0.0/12'), # RFC 1918 private
- ipaddress.ip_network('192.168.0.0/16'), # RFC 1918 private
- ipaddress.ip_network('127.0.0.0/8'), # Localhost
- ipaddress.ip_network('169.254.0.0/16'), # Link-local
- ipaddress.ip_network('::1/128'), # IPv6 localhost
- ipaddress.ip_network('fc00::/7'), # IPv6 unique local
- ipaddress.ip_network('fe80::/10'), # IPv6 link-local
-]
+class FetchTooLargeError(Exception):
+ """Raised when a response is larger than the limit"""
+ pass
+
+
+def user_agent(site_url: str) -> str:
+ """User-Agent for webmention requests (spec 3.1.2 suggests including 'Webmention')"""
+ from starpunk import __version__
+ return f"StarPunk/{__version__} Webmention (+{site_url})"
def is_private_ip(ip: str) -> bool:
"""
- Check if IP address is in a private/blocked range.
+ Check if an IP address must not be fetched.
+
+ Blocks private, loopback, link-local, multicast, reserved, unspecified,
+ carrier-grade NAT and IPv4-mapped forms of these.
Args:
ip: IP address string (IPv4 or IPv6)
Returns:
- True if IP is private/blocked, False otherwise
+ True if IP is blocked, False otherwise
"""
try:
- addr = ipaddress.ip_address(ip)
- return any(addr in network for network in BLOCKED_NETWORKS)
+ addr = ipaddress.ip_address(ip.split('%')[0])
except ValueError:
- # Invalid IP format, treat as blocked
return True
+ if isinstance(addr, ipaddress.IPv6Address) and addr.ipv4_mapped:
+ addr = addr.ipv4_mapped
+
+ if isinstance(addr, ipaddress.IPv4Address) and (addr in CGNAT or addr in THIS_NETWORK):
+ return True
+
+ return (
+ addr.is_private
+ or addr.is_loopback
+ or addr.is_link_local
+ or addr.is_multicast
+ or addr.is_reserved
+ or addr.is_unspecified
+ )
+
def safe_resolve(hostname: str) -> str:
"""
- Resolve hostname and validate IP is not private.
+ Resolve hostname and check every address is public.
Per background-worker.md: DEV_MODE exception for localhost testing.
- Args:
- hostname: Hostname to resolve
-
Returns:
- Resolved IP address if safe
+ First resolved IP address
Raises:
- SSRFBlockedError: If hostname resolves to private IP
+ SSRFBlockedError: If any address is blocked
socket.gaierror: If hostname cannot be resolved
"""
- # DEV_MODE exception for localhost
if current_app.config.get('DEV_MODE') and hostname in ('localhost', '127.0.0.1', '::1'):
return '127.0.0.1'
- # Resolve hostname
- ip = socket.gethostbyname(hostname)
+ addresses = sorted({info[4][0] for info in socket.getaddrinfo(hostname, None)})
+ for ip in addresses:
+ if is_private_ip(ip):
+ raise SSRFBlockedError(f"Blocked request to private IP: {ip}")
- # Check if resolved IP is private
- if is_private_ip(ip):
- raise SSRFBlockedError(f"Blocked request to private IP: {ip}")
-
- return ip
+ return addresses[0]
def validate_url_for_fetch(url: str) -> None:
"""
- Validate URL is safe to fetch (not private IP).
-
- Args:
- url: URL to validate
+ Validate URL is safe to fetch (http(s), public addresses only).
Raises:
- SSRFBlockedError: If URL points to private IP
- ValueError: If URL is malformed
+ SSRFBlockedError: If URL points to a blocked address
+ ValueError: If URL is malformed or its host does not resolve
"""
- from urllib.parse import urlparse
-
parsed = urlparse(url)
- if not parsed.scheme in ('http', 'https'):
+ if parsed.scheme not in ('http', 'https'):
raise ValueError(f"Unsupported URL scheme: {parsed.scheme}")
- if not parsed.netloc:
+ if not parsed.hostname:
raise ValueError("URL must include a hostname")
- # Extract hostname (remove port if present)
- hostname = parsed.hostname
- if not hostname:
- raise ValueError("Could not extract hostname from URL")
+ try:
+ safe_resolve(parsed.hostname)
+ except socket.gaierror as e:
+ raise ValueError(f"Could not resolve {parsed.hostname}: {e}")
- # Validate hostname resolves to public IP
- safe_resolve(hostname)
+
+@dataclass
+class FetchResult:
+ """A fetched response, after redirects"""
+ url: str
+ status_code: int
+ headers: httpx.Headers
+ content: bytes
+ encoding: str
+
+ @property
+ def text(self) -> str:
+ try:
+ return self.content.decode(self.encoding, errors='replace')
+ except LookupError:
+ return self.content.decode('utf-8', errors='replace')
+
+
+def safe_get(url: str, headers: dict, max_bytes: int = MAX_BYTES,
+ max_redirects: int = MAX_REDIRECTS, timeout: float = TIMEOUT) -> FetchResult:
+ """
+ GET a URL, following redirects by hand and checking each hop.
+
+ Args:
+ url: URL to fetch
+ headers: Request headers
+ max_bytes: Response body limit
+ max_redirects: Redirect limit
+ timeout: Total time limit in seconds
+
+ Returns:
+ FetchResult for the final response (any status code)
+
+ Raises:
+ SSRFBlockedError, ValueError: URL or a redirect target is not allowed
+ FetchTooLargeError: Body exceeds max_bytes
+ httpx.HTTPError: Network errors, too many redirects, time limit
+ """
+ deadline = time.monotonic() + timeout
+
+ with httpx.Client(follow_redirects=False, transport=TRANSPORT) as client:
+ for _ in range(max_redirects + 1):
+ validate_url_for_fetch(url)
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ raise httpx.TimeoutException("Fetch exceeded time limit")
+
+ with client.stream('GET', url, headers=headers, timeout=remaining) as response:
+ location = response.headers.get('Location')
+ if response.is_redirect and location:
+ url = urljoin(url, location)
+ continue
+
+ content_length = response.headers.get('Content-Length', '')
+ if content_length.isdigit() and int(content_length) > max_bytes:
+ raise FetchTooLargeError(f"Response exceeds size limit ({max_bytes} bytes)")
+
+ content = b''
+ for chunk in response.iter_bytes(chunk_size=8192):
+ content += chunk
+ if len(content) > max_bytes:
+ raise FetchTooLargeError(f"Response exceeds size limit ({max_bytes} bytes)")
+ if time.monotonic() > deadline:
+ raise httpx.TimeoutException("Fetch exceeded time limit")
+
+ return FetchResult(
+ url=url,
+ status_code=response.status_code,
+ headers=response.headers,
+ content=content,
+ encoding=response.charset_encoding or 'utf-8',
+ )
+
+ raise httpx.TooManyRedirects(f"More than {max_redirects} redirects")
diff --git a/starpunk/webmention_worker.py b/starpunk/webmention_worker.py
index a14d02b..22e7f02 100644
--- a/starpunk/webmention_worker.py
+++ b/starpunk/webmention_worker.py
@@ -14,28 +14,30 @@ References:
import fcntl
import logging
+import re
import threading
import time
import sqlite3
import httpx
from datetime import datetime, timedelta
from pathlib import Path
-from typing import Dict, Any, Optional, TypedDict
-from urllib.parse import urlparse
+from typing import Dict, Any, Optional
+from urllib.parse import urljoin
from lxml import html as lxml_html
-from starpunk.webmention_processor import process_webmention, ValidationError
-from starpunk.webmention_ssrf import validate_url_for_fetch, SSRFBlockedError
+from starpunk.webmention_processor import process_webmention, TransientError, ValidationError
+from starpunk.webmention_ssrf import (
+ FetchTooLargeError, SSRFBlockedError, safe_get, user_agent, validate_url_for_fetch,
+)
logger = logging.getLogger(__name__)
+# Link header (RFC 8288): then ;-separated params; quoted values may
+# contain , and ;
+LINK_RE = re.compile(r'<([^>]*)>((?:\s*;\s*[^;,"=\s]+\s*(?:=\s*(?:"[^"]*"|[^;,]*))?)*)')
+LINK_PARAM_RE = re.compile(r';\s*([^;,"=\s]+)\s*(?:=\s*(?:"([^"]*)"|([^;,]*)))?')
-# Discovery cache entry type
-class DiscoveryCacheEntry(TypedDict):
- """Cache entry for endpoint discovery results"""
- endpoint: Optional[str] # None = negative cache (no endpoint found)
- expires_at: datetime
class WebmentionWorker(threading.Thread):
@@ -79,10 +81,6 @@ class WebmentionWorker(threading.Thread):
self._last_processed = None
self._lock_file = None
- # Phase 3: Discovery cache (worker instance variable)
- # Per architect's answer to Q4
- self._discovery_cache: Dict[str, DiscoveryCacheEntry] = {}
-
def run(self):
"""
Main processing loop
@@ -266,25 +264,34 @@ class WebmentionWorker(threading.Thread):
moderation_status = 'approved' if self.auto_approve else 'pending'
+ # An update to an existing mention (spec 3.2.4) keeps the admin's
+ # moderation decision; only a first-time valid mention gets the
+ # default. Previously valid = validated with no error.
cursor.execute("""
UPDATE webmentions
- SET validation_status = 'valid',
+ SET moderation_status = CASE
+ WHEN moderation_status IN ('hidden', 'deleted') THEN moderation_status
+ WHEN validated_at IS NOT NULL AND validation_error IS NULL
+ THEN moderation_status
+ ELSE ?
+ END,
+ validation_status = 'valid',
+ validation_error = NULL,
validated_at = CURRENT_TIMESTAMP,
mention_type = ?,
author_name = ?,
author_url = ?,
author_photo = ?,
content_snippet = ?,
- moderation_status = ?,
updated_at = CURRENT_TIMESTAMP
WHERE id = ?
""", (
+ moderation_status,
result['mention_type'],
result['author_name'],
result['author_url'],
result['author_photo'],
result['content_snippet'],
- moderation_status,
webmention_id
))
@@ -293,11 +300,17 @@ class WebmentionWorker(threading.Thread):
logger.info(
f"Validated webmention: {source_url} -> {target_url} "
- f"(type={result['mention_type']}, status={moderation_status})"
+ f"(type={result['mention_type']})"
)
+ except TransientError as e:
+ # Source unreachable right now: keep a previously valid mention
+ self._mark_validation_failed(webmention_id, source_url, target_url, str(e),
+ transient=True)
+
except ValidationError as e:
- # Validation failed - update with error
+ # Validation failed (410/4xx, no link): the mention is removed
+ # from display (spec 3.2.4)
self._mark_validation_failed(webmention_id, source_url, target_url, str(e))
except Exception as e:
@@ -315,11 +328,15 @@ class WebmentionWorker(threading.Thread):
webmention_id: int,
source_url: str,
target_url: str,
- error: str
+ error: str,
+ transient: bool = False
):
"""
Mark webmention validation as failed/invalid
+ A transient failure of a mention that was valid before restores it to
+ valid with its previous data, rather than hiding it.
+
Args:
webmention_id: Database ID
source_url: Source URL
@@ -329,7 +346,9 @@ class WebmentionWorker(threading.Thread):
# Determine if error or invalid
# Per background-worker.md: HTTP 4xx and "no link" are invalid
# All other errors are 'error' status
- if any(x in error.lower() for x in ['does not link', 'returned 4']):
+ if transient:
+ status = 'error'
+ elif any(x in error.lower() for x in ['does not link', 'returned 4']):
status = 'invalid'
else:
status = 'error'
@@ -338,6 +357,22 @@ class WebmentionWorker(threading.Thread):
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
+ if transient:
+ cursor.execute("""
+ UPDATE webmentions
+ SET validation_status = 'valid',
+ updated_at = CURRENT_TIMESTAMP
+ WHERE id = ? AND validated_at IS NOT NULL AND validation_error IS NULL
+ """, (webmention_id,))
+ if cursor.rowcount:
+ conn.commit()
+ conn.close()
+ logger.warning(
+ f"Could not re-check webmention, keeping it: "
+ f"{source_url} -> {target_url}: {error}"
+ )
+ return
+
cursor.execute("""
UPDATE webmentions
SET validation_status = ?,
@@ -458,11 +493,13 @@ class WebmentionWorker(threading.Thread):
conn.commit()
conn.close()
- # Send webmention POST
+ # Send webmention POST (spec 3.1.3)
try:
- # SSRF protection for endpoint URL
+ # SSRF protection for endpoint URL (spec 4.3)
validate_url_for_fetch(endpoint)
+ # Redirects are not followed: httpx would turn a 301/302 POST
+ # into a GET without the body
response = httpx.post(
endpoint,
data={
@@ -470,25 +507,24 @@ class WebmentionWorker(threading.Thread):
'target': target_url
},
timeout=10,
- headers={'User-Agent': f'StarPunk/1.6.0 (+{self.site_url})'},
- follow_redirects=True
+ headers={'User-Agent': user_agent(self.site_url)},
+ follow_redirects=False
)
- # Check response
+ # Any 2xx is success (spec 3.1.3)
if 200 <= response.status_code < 300:
- # Success
self._mark_send_success(queue_id, source_url, target_url)
- elif 400 <= response.status_code < 500:
- # Permanent failure (4xx)
- error = f"HTTP {response.status_code}: {response.text[:200]}"
- self._mark_send_failed(queue_id, source_url, target_url, error)
-
- else:
- # Temporary failure (5xx) - retry
+ elif response.status_code == 429 or response.status_code >= 500:
+ # Rate limited or server error - retry
error = f"HTTP {response.status_code}: {response.text[:200]}"
self._mark_send_retry(queue_id, source_url, target_url, retry_count, error)
+ else:
+ # Permanent failure (3xx, 4xx)
+ error = f"HTTP {response.status_code}: {response.text[:200]}"
+ self._mark_send_failed(queue_id, source_url, target_url, error)
+
except (httpx.TimeoutException, httpx.HTTPError) as e:
# Connection error - retry
self._mark_send_retry(queue_id, source_url, target_url, retry_count, str(e))
@@ -504,100 +540,81 @@ class WebmentionWorker(threading.Thread):
def _discover_endpoint(self, target_url: str) -> Optional[str]:
"""
- Discover webmention endpoint for target URL with caching
+ Discover the webmention endpoint for a target URL (spec 3.1.2)
- Per architect's answers to Q3, Q4, Q10:
- - GET only (no HEAD)
- - Check Link header → →
- - First match wins
- - Cache results: 1h for found, 24h for not found
- - SSRF protection on target URL
-
- Args:
- target_url: Target URL to discover endpoint for
+ - GET the target, following redirects (each hop SSRF-checked)
+ - The first Link header with rel=webmention wins, then the first
+ or with rel=webmention and an href, in document order
+ - rel is a space-separated token list, matched case-insensitively
+ - Relative endpoints resolve against the final URL; an empty href
+ is the target itself; query strings are kept
+ - Not cached: an updated post must re-discover (spec 3.1.4)
Returns:
- Endpoint URL if found, None otherwise
- """
- # Check cache first
- cached = self._discovery_cache.get(target_url)
- if cached and cached['expires_at'] > datetime.utcnow():
- logger.debug(f"Using cached endpoint for {target_url}: {cached['endpoint']}")
- return cached['endpoint']
+ Endpoint URL, or None if the target has none (or is blocked/4xx)
- # SSRF protection
+ Raises:
+ httpx.HTTPError: Network errors and 5xx, so the caller retries
+ """
try:
- validate_url_for_fetch(target_url)
+ result = safe_get(target_url, headers={'User-Agent': user_agent(self.site_url)})
except (SSRFBlockedError, ValueError) as e:
logger.warning(f"SSRF protection blocked target {target_url}: {e}")
- # Cache negative result
- self._cache_endpoint(target_url, None, hours=24)
+ return None
+ except FetchTooLargeError:
+ logger.info(f"Target too large for discovery: {target_url}")
return None
- # Fetch target URL
- try:
- response = httpx.get(
- target_url,
- timeout=10,
- headers={'User-Agent': f'StarPunk/1.6.0 (+{self.site_url})'},
- follow_redirects=True
- )
- response.raise_for_status()
-
- # 1. Check Link header
- link_header = response.headers.get('Link', '')
- if link_header:
- endpoint = self._parse_link_header(link_header, str(response.url))
- if endpoint:
- self._cache_endpoint(target_url, endpoint, hours=1)
- return endpoint
-
- # 2. Parse HTML for and elements
- content_type = response.headers.get('Content-Type', '')
- if 'html' in content_type.lower():
- endpoint = self._parse_html_for_endpoint(response.text, str(response.url))
- if endpoint:
- self._cache_endpoint(target_url, endpoint, hours=1)
- return endpoint
-
- # No endpoint found - cache negative result
- logger.info(f"No webmention endpoint found for {target_url}")
- self._cache_endpoint(target_url, None, hours=24)
+ if result.status_code >= 500:
+ raise httpx.HTTPError(f"Target returned {result.status_code}")
+ if result.status_code >= 400:
+ logger.info(f"Target returned {result.status_code}: {target_url}")
return None
- except Exception as e:
- # Don't cache; the caller schedules a retry
- logger.warning(f"Failed to discover endpoint for {target_url}: {e}")
- raise
+ for link_header in result.headers.get_list('Link'):
+ endpoint = self._parse_link_header(link_header, result.url)
+ if endpoint is not None:
+ return endpoint
+
+ content_type = result.headers.get('Content-Type', '')
+ if 'html' in content_type.lower():
+ endpoint = self._parse_html_for_endpoint(result.text, result.url)
+ if endpoint is not None:
+ return endpoint
+
+ logger.info(f"No webmention endpoint found for {target_url}")
+ return None
+
+ @staticmethod
+ def _has_webmention_rel(rel: str) -> bool:
+ return 'webmention' in rel.lower().split()
def _parse_link_header(self, link_header: str, base_url: str) -> Optional[str]:
"""
- Parse Link header for webmention endpoint
+ Parse a Link header (RFC 8288) for a rel=webmention link
Args:
- link_header: Raw Link header value
+ link_header: Link header value (may hold several links)
base_url: Base URL for resolving relative URLs
Returns:
- Absolute endpoint URL if found, None otherwise
+ Absolute endpoint URL of the first match, None otherwise
"""
- # Simple parsing - look for rel="webmention"
- # Format: ; rel="webmention"
- import re
- pattern = r'<([^>]+)>\s*;\s*rel\s*=\s*["\']?webmention["\']?'
- match = re.search(pattern, link_header, re.IGNORECASE)
-
- if match:
- endpoint = match.group(1)
- # Make absolute if relative
- from urllib.parse import urljoin
- return urljoin(base_url, endpoint)
+ for match in LINK_RE.finditer(link_header):
+ uri, params = match.group(1), match.group(2)
+ for param in LINK_PARAM_RE.finditer(params):
+ if param.group(1).lower() != 'rel':
+ continue
+ rel = param.group(2) if param.group(2) is not None else (param.group(3) or '')
+ if self._has_webmention_rel(rel):
+ return urljoin(base_url, uri.strip())
+ break # only the first rel parameter counts (RFC 8288)
return None
def _parse_html_for_endpoint(self, html_content: str, base_url: str) -> Optional[str]:
"""
- Parse HTML for webmention endpoint in or elements
+ Find the first or with rel=webmention, in document order
Args:
html_content: HTML content
@@ -607,48 +624,18 @@ class WebmentionWorker(threading.Thread):
Absolute endpoint URL if found, None otherwise
"""
try:
- doc = lxml_html.fromstring(html_content)
-
- # Check first
- links = doc.xpath('//link[@rel="webmention"]/@href')
- if links:
- from urllib.parse import urljoin
- return urljoin(base_url, links[0])
-
- # Check
- anchors = doc.xpath('//a[@rel="webmention"]/@href')
- if anchors:
- from urllib.parse import urljoin
- return urljoin(base_url, anchors[0])
-
+ doc = lxml_html.document_fromstring(html_content)
except Exception as e:
logger.debug(f"HTML parsing failed: {e}")
+ return None
+
+ for element in doc.iter('link', 'a'):
+ href = element.get('href')
+ if href is not None and self._has_webmention_rel(element.get('rel', '')):
+ return urljoin(base_url, href.strip())
return None
- def _cache_endpoint(self, target_url: str, endpoint: Optional[str], hours: int):
- """
- Cache endpoint discovery result
-
- Args:
- target_url: Target URL
- endpoint: Discovered endpoint (None = negative cache)
- hours: TTL in hours
- """
- self._discovery_cache[target_url] = {
- 'endpoint': endpoint,
- 'expires_at': datetime.utcnow() + timedelta(hours=hours)
- }
-
- # Optional: Evict old entries (keep cache bounded)
- if len(self._discovery_cache) > 1000:
- # Remove expired entries
- now = datetime.utcnow()
- self._discovery_cache = {
- k: v for k, v in self._discovery_cache.items()
- if v['expires_at'] > now
- }
-
def _mark_send_success(self, queue_id: int, source_url: str, target_url: str):
"""Mark webmention send as successful"""
try:
diff --git a/templates/410.html b/templates/410.html
new file mode 100644
index 0000000..9d56c51
--- /dev/null
+++ b/templates/410.html
@@ -0,0 +1,11 @@
+{% extends "base.html" %}
+
+{% block title %}Post Deleted - {{ config.SITE_NAME }}{% endblock %}
+
+{% block content %}
+
+ 410 - Post Deleted
+ This post has been deleted.
+
+
+{% endblock %}
diff --git a/templates/note.html b/templates/note.html
index c111540..dc0c805 100644
--- a/templates/note.html
+++ b/templates/note.html
@@ -185,7 +185,7 @@
{{ mention.author_name or mention.source_url }}
{% if mention.content_snippet %}
- {{ mention.content_snippet|truncate(200) }}
+ {{ mention.content_snippet|striptags|truncate(200) }}
{% endif %}
{% endfor %}
diff --git a/tests/test_micropub_v240.py b/tests/test_micropub_v240.py
index f370b17..788b2e3 100644
--- a/tests/test_micropub_v240.py
+++ b/tests/test_micropub_v240.py
@@ -161,7 +161,7 @@ class TestDelete:
url = create(client, h, category=["kept"])
path = url.replace(SITE, "")
assert client.post("/micropub", data={"action": "delete", "url": url}, headers=h).status_code == 204
- assert client.get(path).status_code == 404
+ assert client.get(path).status_code == 410 # Gone (Webmention 3.1.5)
response = client.post("/micropub", json={"action": "undelete", "url": url}, headers=h)
assert response.status_code == 204
assert client.get(path).status_code == 200
diff --git a/tests/test_webmention_spec.py b/tests/test_webmention_spec.py
new file mode 100644
index 0000000..086e96e
--- /dev/null
+++ b/tests/test_webmention_spec.py
@@ -0,0 +1,283 @@
+"""
+W3C Webmention conformance tests (v2.7.0)
+
+https://www.w3.org/TR/webmention/ — section numbers in test names.
+"""
+
+import sqlite3
+from unittest.mock import patch
+
+import httpx
+import pytest
+
+from starpunk import webmention_ssrf
+from starpunk.webmention_processor import (
+ TransientError, ValidationError, fetch_source, verify_link_presence,
+)
+from starpunk.webmention_worker import WebmentionWorker
+
+PUBLIC = [(2, 1, 6, '', ('93.184.216.34', 0))]
+BASE = 'https://t.example/test/1?x=1'
+
+
+@pytest.fixture
+def worker(app):
+ return WebmentionWorker(app, db_path=app.config["DATABASE_PATH"],
+ site_url=app.config["SITE_URL"])
+
+
+@pytest.fixture
+def mock_web():
+ """Route safe_get through a MockTransport; all hosts resolve public"""
+ routes = {}
+
+ def handler(request):
+ return routes[str(request.url)](request)
+
+ with patch("starpunk.webmention_ssrf.socket.getaddrinfo", return_value=PUBLIC), \
+ patch("starpunk.webmention_ssrf.TRANSPORT", httpx.MockTransport(handler)):
+ yield routes
+
+
+# 3.1.2 Sender discovers receiver Webmention endpoint
+
+@pytest.mark.parametrize("header,expected", [
+ ('; rel="webmention"', 'https://e.example/wm'),
+ ('; rel=webmention', 'https://t.example/wm'),
+ ('; rel="other webmention"', 'https://t.example/test/wm'),
+ ('; title="a;b,c"; rel="webmention"', 'https://t.example/test/wm'),
+ ('; REL="WebMention"', 'https://t.example/test/wm'),
+ ('; rel="other", ; rel="webmention"', 'https://t.example/wm?q=1'),
+ ('; rel="webmention-not"', None),
+ ('; rel="not-webmention"', None),
+])
+def test_3_1_2_link_header(worker, header, expected):
+ assert worker._parse_link_header(header, BASE) == expected
+
+
+@pytest.mark.parametrize("html,expected", [
+ # before : first in document order wins
+ ('x', 'https://t.example/a'),
+ ('', 'https://t.example/l'),
+ ('', 'https://t.example/l'),
+ # empty href is the target itself
+ ('', BASE),
+ # no href: skipped
+ ('', 'https://t.example/a'),
+ # comments, escaped markup and other rels are ignored
+ (''
+ ''
+ '<a rel="webmention" href="/bad">'
+ '', 'https://t.example/ok'),
+ ('nothing
', None),
+])
+def test_3_1_2_html(worker, html, expected):
+ assert worker._parse_html_for_endpoint(html, BASE) == expected
+
+
+def test_3_1_2_header_before_html_and_relative_to_final_url(app, worker, mock_web):
+ mock_web['https://t.example/old'] = lambda r: httpx.Response(
+ 301, headers={'Location': 'https://t.example/new/page'})
+ mock_web['https://t.example/new/page'] = lambda r: httpx.Response(
+ 200, headers={'Link': '; rel="webmention"', 'Content-Type': 'text/html'},
+ text='')
+ with app.app_context():
+ assert worker._discover_endpoint('https://t.example/old') == 'https://t.example/new/wm'
+
+
+def test_3_1_2_multiple_link_headers(app, worker, mock_web):
+ mock_web['https://t.example/p'] = lambda r: httpx.Response(
+ 200, headers=[('Link', '; rel="other"'), ('Link', ' ; rel="webmention"')])
+ with app.app_context():
+ assert worker._discover_endpoint('https://t.example/p') == 'https://t.example/wm'
+
+
+def test_3_1_2_server_error_is_retried_not_final(app, worker, mock_web):
+ mock_web['https://t.example/p'] = lambda r: httpx.Response(503)
+ with app.app_context(), pytest.raises(httpx.HTTPError):
+ worker._discover_endpoint('https://t.example/p')
+
+
+# 4.5 SSRF: every redirect hop is checked
+
+def test_4_5_redirect_to_private_address_blocked(app):
+ def resolve(host, *a, **k):
+ ip = '169.254.169.254' if host == 'metadata.internal' else '93.184.216.34'
+ return [(2, 1, 6, '', (ip, 0))]
+
+ def handler(request):
+ return httpx.Response(302, headers={'Location': 'http://metadata.internal/'})
+
+ with app.app_context(), \
+ patch("starpunk.webmention_ssrf.socket.getaddrinfo", side_effect=resolve), \
+ patch("starpunk.webmention_ssrf.TRANSPORT", httpx.MockTransport(handler)):
+ with pytest.raises(ValidationError, match="Blocked"):
+ fetch_source('https://evil.example/', 'http://localhost:5000')
+
+
+@pytest.mark.parametrize("ip", [
+ '127.0.0.1', '10.1.2.3', '169.254.169.254', '0.0.0.0', '100.64.0.1',
+ '::1', '::ffff:127.0.0.1', 'fe80::1', '224.0.0.1',
+])
+def test_4_5_blocked_addresses(ip):
+ assert webmention_ssrf.is_private_ip(ip)
+
+
+def test_4_5_any_blocked_address_blocks(app):
+ mixed = PUBLIC + [(10, 1, 6, '', ('::1', 0, 0, 0))]
+ with app.app_context(), patch("starpunk.webmention_ssrf.socket.getaddrinfo", return_value=mixed):
+ with pytest.raises(webmention_ssrf.SSRFBlockedError):
+ webmention_ssrf.validate_url_for_fetch('https://x.example/')
+
+
+# 3.2.2 Webmention verification
+
+TARGET = 'https://me.example/note/abc'
+
+
+@pytest.mark.parametrize("content,content_type,expected", [
+ ('x', 'text/html', True),
+ ('
', 'text/html; charset=utf-8', True),
+ ('x', 'text/html', False),
+ ('https://me.example/note/abc
', 'text/html', False),
+ ('', 'text/html', False),
+ ('{"a": ["https://me.example/note/abc"]}', 'application/json', True),
+ ('{"a": "https://me.example/note/abcdef"}', 'application/json', False),
+ ('see https://me.example/note/abc', 'text/plain', True),
+])
+def test_3_2_2_exact_match(content, content_type, expected):
+ assert verify_link_presence(content, TARGET, content_type, 'https://me.example/x') is expected
+
+
+def test_3_2_2_accept_header_and_status(app, mock_web):
+ seen = {}
+
+ def page(request):
+ seen['accept'] = request.headers.get('Accept')
+ return httpx.Response(410)
+
+ mock_web['https://s.example/p'] = page
+ with app.app_context():
+ with pytest.raises(ValidationError, match="410"):
+ fetch_source('https://s.example/p', 'http://localhost:5000')
+ assert 'text/html' in seen['accept']
+
+ mock_web['https://s.example/p'] = lambda r: httpx.Response(500)
+ with app.app_context(), pytest.raises(TransientError):
+ fetch_source('https://s.example/p', 'http://localhost:5000')
+
+
+# 3.2.1 Request verification
+
+def _post(client, source, target):
+ return client.post('/webmention', data={'source': source, 'target': target})
+
+
+def test_3_2_1_target_must_be_a_note(app, client, sample_note):
+ note_url = f"http://localhost:5000/note/{sample_note.slug}"
+ assert _post(client, 'https://s.example/a', note_url).status_code == 202
+ assert _post(client, 'https://s.example/a', 'http://localhost:5000/nope').status_code == 400
+ assert _post(client, 'https://s.example/a', 'http://localhost:5000/').status_code == 400
+ assert _post(client, 'https://s.example/a', 'https://other.example/note/x').status_code == 400
+ assert _post(client, note_url, note_url).status_code == 400
+ assert _post(client, 'ftp://s.example/a', note_url).status_code == 400
+
+
+def test_3_2_1_host_case_and_fragment(app, client, sample_note):
+ note_url = f"http://LOCALHOST:5000/note/{sample_note.slug}"
+ assert _post(client, 'https://s.example/a', note_url + '#one').status_code == 202
+ assert _post(client, 'https://s.example/a', note_url + '#two').status_code == 202
+ db = sqlite3.connect(app.config['DATABASE_PATH'])
+ rows = db.execute("SELECT target_url FROM webmentions").fetchall()
+ assert rows == [(note_url,)]
+
+
+# 3.2.4 Updating existing Webmentions
+
+def _insert_mention(app, note_id, **cols):
+ db = sqlite3.connect(app.config['DATABASE_PATH'])
+ fields = {'source_url': 'https://s.example/a', 'target_url': 'http://localhost:5000/n',
+ 'target_note_id': note_id, **cols}
+ db.execute(f"INSERT INTO webmentions ({', '.join(fields)}) VALUES ({', '.join('?' * len(fields))})",
+ tuple(fields.values()))
+ db.commit()
+ return db
+
+
+VALID = {'valid': True, 'mention_type': 'reply', 'author_name': 'A', 'author_url': 'https://a.example/',
+ 'author_photo': None, 'content_snippet': 'hi'}
+
+
+def test_3_2_4_update_keeps_moderation(app, worker, sample_note):
+ db = _insert_mention(app, sample_note.id, validation_status='valid', moderation_status='hidden',
+ validated_at='2026-01-01')
+ with patch("starpunk.webmention_worker.process_webmention", return_value=VALID):
+ worker._validate_webmention(1, 'https://s.example/a', 'http://localhost:5000/n', sample_note.id)
+ assert db.execute("SELECT validation_status, moderation_status FROM webmentions").fetchone() == ('valid', 'hidden')
+
+
+def test_3_2_4_new_mention_gets_default_moderation(app, worker, sample_note):
+ db = _insert_mention(app, sample_note.id)
+ with patch("starpunk.webmention_worker.process_webmention", return_value=VALID):
+ worker._validate_webmention(1, 'https://s.example/a', 'http://localhost:5000/n', sample_note.id)
+ assert db.execute("SELECT validation_status, moderation_status FROM webmentions").fetchone() == ('valid', 'approved')
+
+
+def test_3_2_4_transient_error_keeps_valid_mention(app, worker, sample_note):
+ db = _insert_mention(app, sample_note.id, validation_status='pending', moderation_status='approved',
+ validated_at='2026-01-01', content_snippet='kept')
+ with patch("starpunk.webmention_worker.process_webmention", side_effect=TransientError("timeout")):
+ worker._validate_webmention(1, 'https://s.example/a', 'http://localhost:5000/n', sample_note.id)
+ assert db.execute("SELECT validation_status, content_snippet FROM webmentions").fetchone() == ('valid', 'kept')
+
+
+def test_3_2_4_gone_source_removes_mention(app, worker, sample_note):
+ db = _insert_mention(app, sample_note.id, validation_status='pending', moderation_status='approved',
+ validated_at='2026-01-01')
+ with patch("starpunk.webmention_worker.process_webmention",
+ side_effect=ValidationError("Source returned 410")):
+ worker._validate_webmention(1, 'https://s.example/a', 'http://localhost:5000/n', sample_note.id)
+ assert db.execute("SELECT validation_status FROM webmentions").fetchone() == ('invalid',)
+
+
+# 3.1.4 Updated posts / 3.1.5 Deleted posts
+
+def _queue(app):
+ db = sqlite3.connect(app.config['DATABASE_PATH'])
+ return db.execute(
+ "SELECT target_url, status, source_note_id FROM webmention_queue ORDER BY target_url"
+ ).fetchall()
+
+
+def _sent(app):
+ db = sqlite3.connect(app.config['DATABASE_PATH'])
+ db.execute("UPDATE webmention_queue SET status = 'sent'")
+ db.commit()
+
+
+def test_3_1_4_update_resends_previous_targets(app):
+ from starpunk.notes import create_note, update_note
+
+ with app.test_request_context():
+ note = create_note("Reply", published=True, in_reply_to="https://a.example/post")
+ _sent(app)
+ update_note(id=note.id, content="Now about [b](https://b.example/x)", in_reply_to="")
+ targets = {t: s for t, s, _ in _queue(app)}
+ assert targets == {'https://a.example/post': 'pending', 'https://b.example/x': 'pending'}
+
+
+@pytest.mark.parametrize("soft", [True, False])
+def test_3_1_5_delete_resends_and_returns_410(app, client, soft):
+ from starpunk.notes import create_note, delete_note
+
+ with app.test_request_context():
+ note = create_note("See [a](https://a.example/x)", published=True)
+ _sent(app)
+ delete_note(id=note.id, soft=soft)
+
+ rows = _queue(app)
+ assert [(t, s) for t, s, _ in rows] == [('https://a.example/x', 'pending')]
+ if not soft:
+ assert rows[0][2] is None # detached, so it survives the delete
+ assert client.get(f"/note/{note.slug}").status_code == 410
+ assert client.get("/note/never-existed").status_code == 404
diff --git a/tests/test_webmention_worker.py b/tests/test_webmention_worker.py
index 8c375d1..bfd3e09 100644
--- a/tests/test_webmention_worker.py
+++ b/tests/test_webmention_worker.py
@@ -14,6 +14,8 @@ from unittest.mock import MagicMock, patch
from starpunk.webmention_worker import WebmentionWorker
+PUBLIC = [(2, 1, 6, '', ('93.184.216.34', 0))]
+
def _wait_for(predicate, timeout=5):
deadline = time.time() + timeout
@@ -35,14 +37,15 @@ def test_worker_sends_queued_webmention(app):
conn.commit()
conn.close()
- page = httpx.Response(200, headers={"Link": ' ; rel="webmention"'},
- request=httpx.Request("GET", "https://example.org/post"))
+ def target_site(request):
+ return httpx.Response(200, headers={"Link": ' ; rel="webmention"'})
+
accepted = MagicMock(status_code=202, text="")
worker = WebmentionWorker(app, db_path=db_path, site_url=app.config["SITE_URL"],
poll_interval=0.05)
- with patch("starpunk.webmention_ssrf.socket.gethostbyname", return_value="93.184.216.34"), \
- patch("starpunk.webmention_worker.httpx.get", return_value=page), \
+ with patch("starpunk.webmention_ssrf.socket.getaddrinfo", return_value=PUBLIC), \
+ patch("starpunk.webmention_ssrf.TRANSPORT", httpx.MockTransport(target_site)), \
patch("starpunk.webmention_worker.httpx.post", return_value=accepted) as post:
worker.start()
try: