diff --git a/.env.example b/.env.example index cc9e41a..2645957 100644 --- a/.env.example +++ b/.env.example @@ -130,6 +130,9 @@ CSRF_ENABLED=true # Background worker that verifies received webmentions, sends outgoing ones, # and sends response notifications (v2.14.0) (default: true) # WEBMENTION_WORKER_ENABLED=true +# Hosts never sent webmentions, comma-separated, e.g. self-hosted apps that +# don't accept them. Links to this site itself are always skipped (v2.18.3) +# WEBMENTION_SKIP_HOSTS=git.example.com,photos.example.com # Seconds between queue checks (default: 10) and after an error (default: 60) # WEBMENTION_POLL_INTERVAL=10 # WEBMENTION_ERROR_BACKOFF=60 diff --git a/CHANGELOG.md b/CHANGELOG.md index e5e4e6d..0bd6274 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [2.18.3] - 2026-11-12 + +### Added + +- `WEBMENTION_SKIP_HOSTS`: comma-separated hosts that are never sent webmentions, for self-hosted apps that don't accept them (Gitea, say). The Settings page lists them + +### Fixed + +- StarPunk sent webmentions to links pointing at itself. It only takes webmentions for its posts, and its own public address can time out from inside the container, so these failed and sat in the Log. Links to this site are now skipped +- Sends to this site or a skipped host that were already queued or failed are dropped when the worker starts, so they leave the Log + ## [2.18.2] - 2026-11-12 ### Fixed diff --git a/docs/design/v2.18.3/2026-11-12-webmention-skip-hosts.md b/docs/design/v2.18.3/2026-11-12-webmention-skip-hosts.md new file mode 100644 index 0000000..ec21d49 --- /dev/null +++ b/docs/design/v2.18.3/2026-11-12-webmention-skip-hosts.md @@ -0,0 +1,34 @@ +# v2.18.3: Don't send webmentions to this site or skipped hosts + +## Why + +The admin Log showed two sends that could never succeed, both "timed out": + +- a note linking to `https://starpunk.thesatelliteoflove.com/` (this site), +- a note linking to a Gitea repository on `git.thesatelliteoflove.com`. + +From inside the container both names resolve to the server's public address, +which isn't reachable from there (no NAT hairpinning). Routing them to the +internal proxy with `extra_hosts` wouldn't help: the SSRF guard blocks +private addresses. And neither target wants the webmention: StarPunk only +takes webmentions for its posts, and Gitea advertises no endpoint. + +## What changed + +| Area | Change | +|---|---| +| Config | `WEBMENTION_SKIP_HOSTS` (comma-separated, lowercased) | +| Sender | `webmention_sender.skipped_hosts(config)` = `SITE_URL`'s host + the setting; `is_skipped(url, hosts)` matches the exact host, any port. `queue_webmentions_for_note` drops those links before the 10-link cap, and from re-sends on update | +| Worker | `_send_webmention` marks a skipped target `no_endpoint` without fetching it (rows queued before the setting changed). At startup `_drop_skipped_targets` does the same for `pending` and `failed` rows, so they leave the Log | +| Settings | A Webmentions section lists the skipped hosts | + +`no_endpoint` with `last_error` = "Not sent: …" is used rather than a new +status because `webmention_queue.status` has a CHECK constraint; changing it +would mean rebuilding the table. The Log only lists `pending` and `failed`. + +Exact host matching is deliberate: skipping `thesatelliteoflove.com` should +not skip its subdomains, or the reverse. + +## Tests + +`tests/test_webmention_skip_hosts.py`. diff --git a/starpunk/__init__.py b/starpunk/__init__.py index ff2c9cb..0e1bc0a 100644 --- a/starpunk/__init__.py +++ b/starpunk/__init__.py @@ -468,5 +468,5 @@ def create_app(config=None): # Package version (Semantic Versioning 2.0.0) # See docs/standards/versioning-strategy.md for details -__version__ = "2.18.2" -__version_info__ = (2, 18, 2) +__version__ = "2.18.3" +__version_info__ = (2, 18, 3) diff --git a/starpunk/config.py b/starpunk/config.py index 17fcf9d..5941f77 100644 --- a/starpunk/config.py +++ b/starpunk/config.py @@ -106,6 +106,10 @@ def load_config(app, config_override=None): # Webmention configuration (v1.6.0) app.config["WEBMENTION_ENABLED"] = os.getenv("WEBMENTION_ENABLED", "true").lower() == "true" + # Hosts never sent webmentions, besides this site itself (v2.18.3) + app.config["WEBMENTION_SKIP_HOSTS"] = [ + h.strip().lower() for h in os.getenv("WEBMENTION_SKIP_HOSTS", "").split(",") if h.strip() + ] # WebSub (v2.10.0, ADR-071): built-in hub unless WEBSUB_HUB_URL names another app.config["WEBSUB_ENABLED"] = os.getenv("WEBSUB_ENABLED", "true").lower() == "true" diff --git a/starpunk/routes/admin.py b/starpunk/routes/admin.py index 8546f05..56c2b76 100644 --- a/starpunk/routes/admin.py +++ b/starpunk/routes/admin.py @@ -728,7 +728,10 @@ def _notify_status() -> dict: @require_auth def settings(): """Site settings: notifications (v2.18.0). Template: templates/admin/settings.html""" - return render_template("admin/settings.html", notify=_notify_status(), user_me=g.me) + from starpunk.webmention_sender import skipped_hosts + + return render_template("admin/settings.html", notify=_notify_status(), user_me=g.me, + skip_hosts=sorted(skipped_hosts(current_app.config))) @bp.route("/log") diff --git a/starpunk/webmention_sender.py b/starpunk/webmention_sender.py index ab1ffe2..01fefaf 100644 --- a/starpunk/webmention_sender.py +++ b/starpunk/webmention_sender.py @@ -22,6 +22,22 @@ from urllib.parse import urlparse logger = logging.getLogger(__name__) +SKIPPED_NOTE = "Not sent: links to this site and WEBMENTION_SKIP_HOSTS are skipped" + + +def skipped_hosts(config) -> set[str]: + """Hosts that never get webmentions: this site and WEBMENTION_SKIP_HOSTS (v2.18.3)""" + hosts = set(config.get("WEBMENTION_SKIP_HOSTS") or []) + site = urlparse(config.get("SITE_URL") or "").hostname + if site: + hosts.add(site.lower()) + return hosts + + +def is_skipped(url: str, hosts: set[str]) -> bool: + """True if url is on one of the skipped hosts (exact host, any port)""" + return (urlparse(url).hostname or "").lower() in hosts + def extract_links_from_html(html_content: str, exclude_code_blocks: bool = True) -> list[str]: """ @@ -189,17 +205,26 @@ def queue_webmentions_for_note( Returns: Number of webmentions queued """ + # Links to this site or a skipped host are dropped before the cap (v2.18.3): + # this site only takes webmentions for its posts, and its own public + # address may not be reachable from inside the container + from flask import current_app, has_app_context + hosts = skipped_hosts(current_app.config) if has_app_context() else set() + + def wanted(urls): + return [u for u in urls if not is_skipped(u, hosts)] + # Targets for the current content: reply target first, then links in # document order, at most 10 links = extract_links_from_html(content_html) if content_html else [] - current = _dedupe(([in_reply_to] if in_reply_to else []) + links)[:10] + current = wanted(_dedupe(([in_reply_to] if in_reply_to else []) + links))[:10] if old_content_html is not None: # Update (spec 3.1.4): also re-send to every target notified before, # and to links the edit removed, so they can update or remove the # mention. These are not capped. removed = [u for u in extract_links_from_html(old_content_html) if u not in links] - target_urls = _dedupe(current + previous_targets(db_path, note_id) + removed) + target_urls = _dedupe(current + wanted(previous_targets(db_path, note_id) + removed)) else: target_urls = current diff --git a/starpunk/webmention_worker.py b/starpunk/webmention_worker.py index 4eb7639..552ec82 100644 --- a/starpunk/webmention_worker.py +++ b/starpunk/webmention_worker.py @@ -205,6 +205,40 @@ class WebmentionWorker(threading.Thread): except Exception as e: logger.error(f"Failed to cleanup orphaned items: {e}") + self._drop_skipped_targets() + + def _skipped_hosts(self) -> set: + from starpunk.webmention_sender import skipped_hosts + return skipped_hosts(self.app.config) if self.app is not None else set() + + def _drop_skipped_targets(self): + """ + Stop sending to this site and WEBMENTION_SKIP_HOSTS (v2.18.3) + + Queued or failed sends to those hosts are set to 'no_endpoint' (nothing + to send to), so they leave the retry queue and the admin Log. + """ + from starpunk.webmention_sender import SKIPPED_NOTE, is_skipped + + hosts = self._skipped_hosts() + if not hosts: + return + try: + conn = sqlite3.connect(self.db_path) + rows = conn.execute( + "SELECT id, target_url FROM webmention_queue WHERE status IN ('pending', 'failed')" + ).fetchall() + ids = [(SKIPPED_NOTE, i) for i, url in rows if is_skipped(url, hosts)] + if ids: + conn.executemany( + "UPDATE webmention_queue SET status = 'no_endpoint', last_error = ? WHERE id = ?", + ids) + conn.commit() + logger.info(f"Dropped {len(ids)} queued webmention(s) to skipped hosts") + conn.close() + except Exception as e: + logger.error(f"Failed to drop skipped webmention targets: {e}") + def _process_pending_validations(self) -> int: """ Process pending incoming webmentions @@ -583,6 +617,15 @@ class WebmentionWorker(threading.Thread): conn.commit() conn.close() + from starpunk.webmention_sender import SKIPPED_NOTE, is_skipped + if is_skipped(target_url, self._skipped_hosts()): + conn = sqlite3.connect(self.db_path) + conn.execute("UPDATE webmention_queue SET status = 'no_endpoint', last_error = ? WHERE id = ?", + (SKIPPED_NOTE, queue_id)) + conn.commit() + conn.close() + return + try: # Discover endpoint if not cached endpoint = cached_endpoint diff --git a/templates/admin/settings.html b/templates/admin/settings.html index 56f4f73..0390337 100644 --- a/templates/admin/settings.html +++ b/templates/admin/settings.html @@ -19,5 +19,12 @@

Off. Set NOTIFY_URLS to one or more Apprise URLs to be notified of new responses.

{% endif %} + + {# Outgoing webmentions (v2.18.3) #} +
+

Webmentions

+

Never sent to: {% for h in skip_hosts %}{{ h }}{{ ", " if not loop.last }}{% endfor %}. + This site is always skipped; add others with WEBMENTION_SKIP_HOSTS.

+
{% endblock %} diff --git a/tests/test_webmention_skip_hosts.py b/tests/test_webmention_skip_hosts.py new file mode 100644 index 0000000..5d8f20e --- /dev/null +++ b/tests/test_webmention_skip_hosts.py @@ -0,0 +1,88 @@ +""" +Webmentions are never sent to this site or WEBMENTION_SKIP_HOSTS (v2.18.3) + +From inside the container the site's own public address can time out, and +self-hosted apps like Gitea don't take webmentions, so these sends only +failed and stayed in the admin Log. +""" + +import sqlite3 + +from starpunk.webmention_sender import ( + SKIPPED_NOTE, is_skipped, queue_webmentions_for_note, skipped_hosts) +from starpunk.webmention_worker import WebmentionWorker + +SKIP = "git.example.com" + + +def queued(db_path): + conn = sqlite3.connect(db_path) + rows = conn.execute("SELECT target_url, status, last_error FROM webmention_queue ORDER BY id").fetchall() + conn.close() + return rows + + +def test_hosts_include_this_site(app): + app.config["WEBMENTION_SKIP_HOSTS"] = [SKIP] + hosts = skipped_hosts(app.config) + assert hosts == {"localhost", SKIP} + assert is_skipped("https://GIT.example.com:443/phil/x", hosts) + assert not is_skipped("https://example.com/", hosts) # exact host only + + +def test_links_to_skipped_hosts_are_not_queued(app): + app.config["WEBMENTION_SKIP_HOSTS"] = [SKIP] + html = "".join(f'

x

' for u in [ + "http://localhost:5000/note/other", f"https://{SKIP}/phil/StarPunk"] + + [f"https://site{i}.example/" for i in range(10)]) + with app.app_context(): + n = queue_webmentions_for_note(app.config["DATABASE_PATH"], 1, "http://localhost:5000/note/a", + html, in_reply_to="http://localhost:5000/note/b") + targets = [t for t, _, _ in queued(app.config["DATABASE_PATH"])] + assert n == 10 and targets == [f"https://site{i}.example/" for i in range(10)] # skipped ones don't use the cap + + +def test_worker_drops_queued_and_failed_sends(app): + app.config["WEBMENTION_SKIP_HOSTS"] = [SKIP] + db_path = app.config["DATABASE_PATH"] + conn = sqlite3.connect(db_path) + conn.executemany( + "INSERT INTO webmention_queue (source_url, target_url, status, last_error) VALUES (?, ?, ?, ?)", + [("http://localhost:5000/note/1", f"https://{SKIP}/phil/StarPunk", "failed", "timed out"), + ("http://localhost:5000/note/1", "http://localhost:5000/", "pending", "timed out"), + ("http://localhost:5000/note/1", "https://example.org/post", "failed", "timed out"), + ("http://localhost:5000/note/2", "http://localhost:5000/x", "sent", None)]) + conn.commit() + conn.close() + + WebmentionWorker(app, db_path=db_path, site_url=app.config["SITE_URL"])._drop_skipped_targets() + + assert [(s, e) for _, s, e in queued(db_path)] == [ + ("no_endpoint", SKIPPED_NOTE), ("no_endpoint", SKIPPED_NOTE), + ("failed", "timed out"), # other sites are left alone + ("sent", None)] + + +def test_worker_skips_without_fetching(app): + db_path = app.config["DATABASE_PATH"] + conn = sqlite3.connect(db_path) + conn.execute("INSERT INTO webmention_queue (source_url, target_url, status) VALUES (?, ?, 'pending')", + ("http://localhost:5000/note/1", "http://localhost:5000/")) + conn.commit() + (queue_id,) = conn.execute("SELECT id FROM webmention_queue").fetchone() + conn.close() + + worker = WebmentionWorker(app, db_path=db_path, site_url=app.config["SITE_URL"]) + worker._discover_endpoint = lambda url: (_ for _ in ()).throw(AssertionError("fetched")) + worker._send_webmention(queue_id, "http://localhost:5000/note/1", "http://localhost:5000/", None, 0) + assert queued(db_path)[0][1:] == ("no_endpoint", SKIPPED_NOTE) + + +def test_settings_page_lists_skipped_hosts(app, client): + from starpunk.auth import create_session + + app.config["WEBMENTION_SKIP_HOSTS"] = [SKIP] + with app.test_request_context(): + client.set_cookie("starpunk_session", create_session(app.config["ADMIN_ME"])) + html = client.get("/admin/settings").data.decode() + assert f"{SKIP}" in html and "localhost" in html