From 106495d1141fc2fc41ea9882aa26b448b27cc4cd Mon Sep 17 00:00:00 2001 From: Phil Date: Tue, 3 Nov 2026 07:53:19 -0700 Subject: [PATCH] fix: Webmention processing works end to end (v2.6.9) - Normalise mf2 author values: u-photo with alt text is a dict in mf2py and broke the database update for most real senders - Pass str(response.url) to urljoin during endpoint discovery; the TypeError marked every target as having no endpoint. Retry on discovery errors - Hold a cross-process flock so only one gunicorn process runs the worker - Keep author URL/photo only if http(s) (XSS via javascript: URLs) Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 12 ++++ .../2026-11-03-webmention-processing-fixes.md | 36 +++++++++++ starpunk/__init__.py | 4 +- starpunk/webmention_processor.py | 39 +++++++++--- starpunk/webmention_worker.py | 51 +++++++++++++--- tests/test_webmention_worker.py | 60 ++++++++++++++++++- 6 files changed, 184 insertions(+), 18 deletions(-) create mode 100644 docs/design/v2.6.9/2026-11-03-webmention-processing-fixes.md diff --git a/CHANGELOG.md b/CHANGELOG.md index d235afa..2f37eae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [2.6.9] - 2026-11-03 + +### Fixed + +- Received webmentions failed when the sender's profile photo had alt text (Bridgy, Mastodon and most IndieWeb sites do this). The photo is now stored as its URL +- Outgoing webmentions were never sent: endpoint discovery crashed on every target that had an endpoint, and the target was then marked as having none. Discovery now works, and network errors during discovery are retried instead of giving up +- Each server process ran its own webmention worker, so a webmention could be processed or sent more than once. Only one process handles the queues at a time now; if it stops, another takes over + +### Security + +- A webmention sender could set its author link to a `javascript:` URL, which was shown as a clickable link on the note page. Author links and photos are now only kept if they are http or https URLs + ## [2.6.8] - 2026-11-03 ### Fixed diff --git a/docs/design/v2.6.9/2026-11-03-webmention-processing-fixes.md b/docs/design/v2.6.9/2026-11-03-webmention-processing-fixes.md new file mode 100644 index 0000000..9f17297 --- /dev/null +++ b/docs/design/v2.6.9/2026-11-03-webmention-processing-fixes.md @@ -0,0 +1,36 @@ +# v2.6.9: Webmention processing fixes + +Found after v2.6.8 let the worker run in production, and by a review of the +implementation against the W3C Webmention Recommendation. + +## Fixes + +1. **Author photo with alt text** (`webmention_processor.extract_author`). + mf2py returns `u-photo` with an `alt` attribute as + `{'value': url, 'alt': ...}`; the dict was bound to SQLite and the update + failed. `_first_str()` reduces mf2 values to their `value`. +2. **Endpoint discovery** (`WebmentionWorker._discover_endpoint`). + `response.url` is an `httpx.URL`; `urljoin(str, URL)` raised `TypeError`, + which was caught and returned as "no endpoint", so every target was marked + `no_endpoint`. Pass `str(response.url)`. Fetch errors now propagate so the + caller schedules a retry, instead of being treated as "no endpoint". +3. **Duplicate processing.** gunicorn runs 4 processes, each with a worker, + and queue items were claimed by SELECT then UPDATE. The worker now takes a + non-blocking `flock` on `/webmention-worker.lock` before processing; + others wait and retry each poll, so one takes over if the holder dies. + Orphan cleanup runs when the lock is acquired. +4. **XSS (spec 4.1).** `author_url` and `author_photo` are kept only if they + are absolute http(s) URLs; otherwise the source URL / no photo is used. + +## Tests + +`tests/test_webmention_worker.py`: the send test uses a real `httpx.Response` +(the old `MagicMock` with a string URL hid fix 2), plus tests for the lock, +an alt-text photo and a `javascript:` author URL. All fail on v2.6.8. + +## Remaining spec gaps + +Tracked separately: rel-token and document-order endpoint discovery, exact +target match during verification, Accept header, re-sending on delete, 410 +for deleted notes, SSRF re-validation on redirects, and the others listed in +the compliance review. diff --git a/starpunk/__init__.py b/starpunk/__init__.py index 11d5f79..41d31ad 100644 --- a/starpunk/__init__.py +++ b/starpunk/__init__.py @@ -459,5 +459,5 @@ def create_app(config=None): # Package version (Semantic Versioning 2.0.0) # See docs/standards/versioning-strategy.md for details -__version__ = "2.6.8" -__version_info__ = (2, 6, 8) +__version__ = "2.6.9" +__version_info__ = (2, 6, 9) diff --git a/starpunk/webmention_processor.py b/starpunk/webmention_processor.py index 291df24..b6a0cb4 100644 --- a/starpunk/webmention_processor.py +++ b/starpunk/webmention_processor.py @@ -182,6 +182,28 @@ def detect_mention_type(parsed_mf2: Dict, target_url: str) -> str: return 'mention' +def _first_str(props: Dict, name: str) -> Optional[str]: + """ + First value of an mf2 property as a string + + mf2py gives u-photo with alt text as {'value': url, 'alt': ...} and nested + microformats as dicts with a 'value'; reduce those to the plain value. + """ + values = props.get(name) or [None] + value = values[0] + if isinstance(value, dict): + value = value.get('value') + return value if isinstance(value, str) and value else None + + +def _is_http_url(value: Optional[str]) -> bool: + """True if value is an absolute http or https URL""" + if not value: + return False + parsed = urlparse(value.strip()) + return parsed.scheme.lower() in ('http', 'https') and bool(parsed.netloc) + + def extract_author(parsed_mf2: Dict, source_url: str) -> Dict[str, Optional[str]]: """ Extract author information from h-card. @@ -212,26 +234,29 @@ def extract_author(parsed_mf2: Dict, source_url: str) -> Dict[str, Optional[str] if isinstance(author_data, dict): # h-card object props = author_data.get('properties', {}) - author_name = props.get('name', [None])[0] - author_url = props.get('url', [None])[0] - author_photo = props.get('photo', [None])[0] + author_name = _first_str(props, 'name') + author_url = _first_str(props, 'url') + author_photo = _first_str(props, 'photo') # If no author in h-entry, find representative h-card if not author_name: for item in parsed_mf2.get('items', []): if 'h-card' in item.get('type', []): props = item.get('properties', {}) - author_name = props.get('name', [None])[0] - author_url = props.get('url', [None])[0] - author_photo = props.get('photo', [None])[0] + author_name = _first_str(props, 'name') + author_url = _first_str(props, 'url') + author_photo = _first_str(props, 'photo') break # Fallback to hostname if no author found if not author_name: author_name = urlparse(source_url).netloc - if not author_url: + # Only http(s) URLs are rendered as links (blocks javascript: and data:) + if not _is_http_url(author_url): author_url = source_url + if not _is_http_url(author_photo): + author_photo = None return { 'author_name': author_name, diff --git a/starpunk/webmention_worker.py b/starpunk/webmention_worker.py index 5becfca..a14d02b 100644 --- a/starpunk/webmention_worker.py +++ b/starpunk/webmention_worker.py @@ -12,6 +12,7 @@ References: - ADR-063: Webmention Implementation """ +import fcntl import logging import threading import time @@ -76,6 +77,7 @@ class WebmentionWorker(threading.Thread): self.auto_approve = auto_approve self._stop_event = threading.Event() self._last_processed = None + self._lock_file = None # Phase 3: Discovery cache (worker instance variable) # Per architect's answer to Q4 @@ -94,11 +96,14 @@ class WebmentionWorker(threading.Thread): try: logger.info("Webmention worker started") - # Cleanup orphaned items from previous crashes - self._cleanup_orphaned_items() - # Main processing loop while not self._stop_event.is_set(): + # Every gunicorn process starts a worker; only the one holding + # the lock processes queues, so nothing is sent twice + if not self._acquire_lock(): + self._stop_event.wait(self.poll_interval) + continue + try: processed = 0 @@ -124,8 +129,40 @@ class WebmentionWorker(threading.Thread): except Exception as e: logger.error(f"Webmention worker thread failed: {e}", exc_info=True) finally: + self._release_lock() logger.info("Webmention worker stopped") + def _acquire_lock(self) -> bool: + """ + Take the cross-process worker lock (non-blocking) + + On first acquisition, also reset items left mid-send by a previous + holder. The OS releases the lock if this process dies. + + Returns: + True if this worker holds the lock + """ + if self._lock_file: + return True + + lock_file = open(Path(self.db_path).parent / "webmention-worker.lock", "w") + try: + fcntl.flock(lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + lock_file.close() + return False + + self._lock_file = lock_file + logger.info("Webmention worker holds the queue lock") + self._cleanup_orphaned_items() + return True + + def _release_lock(self): + """Release the cross-process worker lock if held""" + if self._lock_file: + self._lock_file.close() + self._lock_file = None + def _cleanup_orphaned_items(self): """ Clean up orphaned 'sending' status items on startup @@ -510,7 +547,7 @@ class WebmentionWorker(threading.Thread): # 1. Check Link header link_header = response.headers.get('Link', '') if link_header: - endpoint = self._parse_link_header(link_header, response.url) + endpoint = self._parse_link_header(link_header, str(response.url)) if endpoint: self._cache_endpoint(target_url, endpoint, hours=1) return endpoint @@ -518,7 +555,7 @@ class WebmentionWorker(threading.Thread): # 2. Parse HTML for and elements content_type = response.headers.get('Content-Type', '') if 'html' in content_type.lower(): - endpoint = self._parse_html_for_endpoint(response.text, response.url) + endpoint = self._parse_html_for_endpoint(response.text, str(response.url)) if endpoint: self._cache_endpoint(target_url, endpoint, hours=1) return endpoint @@ -529,9 +566,9 @@ class WebmentionWorker(threading.Thread): return None except Exception as e: + # Don't cache; the caller schedules a retry logger.warning(f"Failed to discover endpoint for {target_url}: {e}") - # Don't cache on error - allow retry - return None + raise def _parse_link_header(self, link_header: str, base_url: str) -> Optional[str]: """ diff --git a/tests/test_webmention_worker.py b/tests/test_webmention_worker.py index 31a6694..8c375d1 100644 --- a/tests/test_webmention_worker.py +++ b/tests/test_webmention_worker.py @@ -7,6 +7,8 @@ whenever any app context ended. """ import sqlite3 + +import httpx import time from unittest.mock import MagicMock, patch @@ -33,8 +35,8 @@ def test_worker_sends_queued_webmention(app): conn.commit() conn.close() - page = MagicMock(status_code=200, url="https://example.org/post", - headers={"Link": '; rel="webmention"'}) + page = httpx.Response(200, headers={"Link": '; rel="webmention"'}, + request=httpx.Request("GET", "https://example.org/post")) accepted = MagicMock(status_code=202, text="") worker = WebmentionWorker(app, db_path=db_path, site_url=app.config["SITE_URL"], @@ -52,6 +54,7 @@ def test_worker_sends_queued_webmention(app): assert _wait_for(lambda: status()[0] != "pending"), status() assert status() == ("sent", None) post.assert_called_once() + assert post.call_args.args[0] == "https://example.org/wm" finally: worker.stop() worker.join(timeout=2) @@ -70,3 +73,56 @@ def test_worker_survives_app_context_teardown(app): finally: worker.stop() worker.join(timeout=2) + + +def test_only_one_worker_processes_queues(app): + """Each gunicorn process starts a worker; only one may hold the queue lock""" + kwargs = dict(db_path=app.config["DATABASE_PATH"], site_url=app.config["SITE_URL"]) + first = WebmentionWorker(app, **kwargs) + second = WebmentionWorker(app, **kwargs) + try: + assert first._acquire_lock() + assert not second._acquire_lock() + first._release_lock() + assert second._acquire_lock() + finally: + first._release_lock() + second._release_lock() + + +def test_author_photo_with_alt_text_is_a_string(): + """mf2py returns u-photo with alt as a dict; the database needs a URL string""" + import mf2py + from starpunk.webmention_processor import extract_author + + html = """ +
+ + A Person + + target +
""" + author = extract_author(mf2py.parse(doc=html, url="https://src.example/"), + "https://src.example/") + assert author == { + "author_name": "A Person", + "author_url": "https://a.example/", + "author_photo": "https://a.example/me.jpg", + } + + +def test_author_urls_must_be_http(): + """A javascript: author URL would be rendered as a clickable link (XSS)""" + import mf2py + from starpunk.webmention_processor import extract_author + + html = """ +
+ + Evil + +
""" + author = extract_author(mf2py.parse(doc=html, url="https://src.example/"), + "https://src.example/") + assert author["author_url"] == "https://src.example/" + assert author["author_photo"] is None