From 65fc246d559e70e4ab16f25a69e1c39048ca9666 Mon Sep 17 00:00:00 2001 From: Phil Date: Tue, 10 Nov 2026 12:56:04 -0700 Subject: [PATCH] fix: save avatars from own GoToSocial and Bluesky PDSes (v2.13.1) Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 7 ++ docs/design/INDEX.md | 6 +- .../design/v2.13.1/2026-11-10-avatar-fetch.md | 33 ++++++++ docs/standards/versioning-strategy.md | 2 +- starpunk/__init__.py | 4 +- starpunk/avatars.py | 82 +++++++++++++++++-- tests/test_backfeed.py | 22 +++++ 7 files changed, 145 insertions(+), 11 deletions(-) create mode 100644 docs/design/v2.13.1/2026-11-10-avatar-fetch.md diff --git a/CHANGELOG.md b/CHANGELOG.md index e4b4275..1f766c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [2.13.1] - 2026-11-10 + +### Fixed + +- Avatars from your own GoToSocial server are saved again. When StarPunk reaches it through the same proxy, its name resolves to an internal address, which the safe fetcher refused. That one server (`GOTOSOCIAL_URL`) is now fetched directly, as crossposting already does +- Bluesky avatars are saved again. Bluesky's image CDN refuses some servers' IPv4 addresses (HTTP 403), so avatars are now downloaded from the account's own server (`com.atproto.sync.getBlob`) + ## [2.13.0] - 2026-11-10 Backfeed: responses to crossposts come back to the note (ADR-073). diff --git a/docs/design/INDEX.md b/docs/design/INDEX.md index cd0531d..45a2c06 100644 --- a/docs/design/INDEX.md +++ b/docs/design/INDEX.md @@ -21,7 +21,7 @@ is in `../architecture/`; decisions are in `../decisions/` (ADRs). ## Contents -[v1.0.0](#v100) · [v1.1.1](#v111) · [v1.1.2](#v112) · [v1.2.0](#v120) · [v1.3.0](#v130) · [v1.3.1](#v131) · [v1.4.0](#v140) · [v1.4.1](#v141) · [v1.4.2](#v142) · [v1.5.0](#v150) · [v1.6.0](#v160) · [v2.0.0](#v200) · [v2.0.1](#v201) · [v2.0.2](#v202) · [v2.1.0](#v210) · [v2.2.0](#v220) · [v2.3.0](#v230) · [v2.3.1](#v231) · [v2.3.2](#v232) · [v2.3.3](#v233) · [v2.4.0](#v240) · [v2.5.0](#v250) · [v2.6.0](#v260) · [v2.6.1](#v261) · [v2.6.2](#v262) · [v2.6.3](#v263) · [v2.6.4](#v264) · [v2.6.5](#v265) · [v2.6.6](#v266) · [v2.6.7](#v267) · [v2.6.8](#v268) · [v2.6.9](#v269) · [v2.7.0](#v270) · [v2.7.1](#v271) · [v2.7.2](#v272) · [v2.8.0](#v280) · [v2.8.1](#v281) · [v2.9.0](#v290) · [v2.10.0](#v2100) · [v2.10.1](#v2101) · [v2.10.2](#v2102) · [v2.11.0](#v2110) · [v2.12.0](#v2120) · [v2.12.1](#v2121) · [v2.13.0](#v2130) · [hotfix](#hotfix) +[v1.0.0](#v100) · [v1.1.1](#v111) · [v1.1.2](#v112) · [v1.2.0](#v120) · [v1.3.0](#v130) · [v1.3.1](#v131) · [v1.4.0](#v140) · [v1.4.1](#v141) · [v1.4.2](#v142) · [v1.5.0](#v150) · [v1.6.0](#v160) · [v2.0.0](#v200) · [v2.0.1](#v201) · [v2.0.2](#v202) · [v2.1.0](#v210) · [v2.2.0](#v220) · [v2.3.0](#v230) · [v2.3.1](#v231) · [v2.3.2](#v232) · [v2.3.3](#v233) · [v2.4.0](#v240) · [v2.5.0](#v250) · [v2.6.0](#v260) · [v2.6.1](#v261) · [v2.6.2](#v262) · [v2.6.3](#v263) · [v2.6.4](#v264) · [v2.6.5](#v265) · [v2.6.6](#v266) · [v2.6.7](#v267) · [v2.6.8](#v268) · [v2.6.9](#v269) · [v2.7.0](#v270) · [v2.7.1](#v271) · [v2.7.2](#v272) · [v2.8.0](#v280) · [v2.8.1](#v281) · [v2.9.0](#v290) · [v2.10.0](#v2100) · [v2.10.1](#v2101) · [v2.10.2](#v2102) · [v2.11.0](#v2110) · [v2.12.0](#v2120) · [v2.12.1](#v2121) · [v2.13.0](#v2130) · [v2.13.1](#v2131) · [hotfix](#hotfix) ## v1.0.0 @@ -426,6 +426,10 @@ OIDC admin login and built-in IndieAuth server. - [2026-11-10-backfeed.md](v2.13.0/2026-11-10-backfeed.md) - Backfeed of responses to crossposts (ADR-073), avatar cache, delete tombstones +## v2.13.1 + +- [2026-11-10-avatar-fetch.md](v2.13.1/2026-11-10-avatar-fetch.md) - Avatars from the own GoToSocial server and from Bluesky PDSes + ## hotfix Not a release folder. diff --git a/docs/design/v2.13.1/2026-11-10-avatar-fetch.md b/docs/design/v2.13.1/2026-11-10-avatar-fetch.md new file mode 100644 index 0000000..6a2ad4c --- /dev/null +++ b/docs/design/v2.13.1/2026-11-10-avatar-fetch.md @@ -0,0 +1,33 @@ +# v2.13.1: Avatar fetching in production + +## Problem + +After deploying v2.13.0, backfeed stored responses but no avatar was saved: + +- GoToSocial avatars (`https://social.../fileserver/...`): + `SSRFBlockedError: Blocked request to private IP: 172.20.0.5`. Inside the + container the GoToSocial host resolves to the reverse proxy's internal + address (the deployment's hairpin `extra_hosts` entry). +- Bluesky avatars (`https://cdn.bsky.app/img/avatar/...`): HTTP 403 from + BunnyCDN. From the server, the CDN answers 403 over IPv4 and 200 over + IPv6; the container has no IPv6 route. From elsewhere IPv4 works. + +## Fix (`starpunk/avatars.py`, `_download`) + +- The host of `GOTOSOCIAL_URL` is fetched with a plain HTTP client (no + redirects, 1 MB limit), as `MastodonClient` already talks to it. Every + other host still goes through `safe_get`, so private addresses stay + blocked. +- `cdn.bsky.app/img//plain//` URLs are fetched from the + account's PDS with `com.atproto.sync.getBlob`; the PDS comes from the DID + document (`plc.directory` for `did:plc`, `/.well-known/did.json` for + `did:web`). Both requests use `safe_get`. The cache file is still named + after the original URL. + +## Verification + +- `tests/test_backfeed.py::test_avatars_from_own_gotosocial_and_bluesky_pds`: + own GoToSocial on a private address is fetched, a Bluesky CDN URL is + fetched from its PDS, another private address is still refused. +- Checked against the real PLC directory and a bsky.network PDS. +- From the production server: getBlob over IPv4 returns the image (200). diff --git a/docs/standards/versioning-strategy.md b/docs/standards/versioning-strategy.md index 4261cff..901396d 100644 --- a/docs/standards/versioning-strategy.md +++ b/docs/standards/versioning-strategy.md @@ -4,7 +4,7 @@ **Last Updated**: 2026-11-03 (v2.10.1) StarPunk uses [Semantic Versioning 2.0.0](https://semver.org/): -`MAJOR.MINOR.PATCH`, currently **2.13.0**. Every release is a git tag +`MAJOR.MINOR.PATCH`, currently **2.13.1**. Every release is a git tag `vX.Y.Z` and a `CHANGELOG.md` entry. Where the number is stored and how to write the changelog entry is in [version-implementation-guide.md](version-implementation-guide.md); the diff --git a/starpunk/__init__.py b/starpunk/__init__.py index 3541b3c..39461a2 100644 --- a/starpunk/__init__.py +++ b/starpunk/__init__.py @@ -465,5 +465,5 @@ def create_app(config=None): # Package version (Semantic Versioning 2.0.0) # See docs/standards/versioning-strategy.md for details -__version__ = "2.13.0" -__version_info__ = (2, 13, 0) +__version__ = "2.13.1" +__version_info__ = (2, 13, 1) diff --git a/starpunk/avatars.py b/starpunk/avatars.py index e1fa719..89260da 100644 --- a/starpunk/avatars.py +++ b/starpunk/avatars.py @@ -24,6 +24,9 @@ logger = logging.getLogger(__name__) SIZE = 96 MAX_BYTES = 1024 * 1024 +# Bluesky avatar on its CDN: .../img//plain//[@jpeg] +BSKY_CDN_RE = re.compile( + r"^https://cdn\.bsky\.app/img/[a-z_]+/plain/(did:(?:plc|web):[A-Za-z0-9._:%-]+)/([a-z2-7]+)(?:@\w+)?$") NAME_RE = re.compile(r"^[0-9a-f]{64}\.webp$") CLEANUP_EVERY = 7 * 24 * 3600 @@ -56,14 +59,8 @@ def fetch(data_path, url: Optional[str], site_url: str) -> bool: return True from PIL import Image - from starpunk.webmention_ssrf import safe_get, user_agent - try: - result = safe_get(url, headers={"User-Agent": user_agent(site_url)}, - max_bytes=MAX_BYTES, max_redirects=5, timeout=10.0, truncate=False) - if result.status_code >= 400: - raise ValueError(f"HTTP {result.status_code}") - with Image.open(io.BytesIO(result.content)) as image: + with Image.open(io.BytesIO(_download(url, site_url))) as image: image.load() image = image.convert("RGBA" if "A" in image.getbands() else "RGB") side = min(image.size) @@ -80,6 +77,77 @@ def fetch(data_path, url: Optional[str], site_url: str) -> bool: return False +def _own_instance_host() -> Optional[str]: + """Host of the GoToSocial server StarPunk posts to (it may resolve to an internal address)""" + from urllib.parse import urlparse + + from flask import current_app + return urlparse(current_app.config.get("GOTOSOCIAL_URL") or "").hostname + + +def _get(url: str, headers: dict) -> bytes: + """GET a public URL through the SSRF-safe fetcher""" + from starpunk.webmention_ssrf import safe_get + + result = safe_get(url, headers=headers, max_bytes=MAX_BYTES, max_redirects=5, + timeout=10.0, truncate=False) + if result.status_code >= 400: + raise ValueError(f"HTTP {result.status_code}") + return result.content + + +def _pds(did: str, headers: dict) -> str: + """The PDS that hosts a Bluesky account, from its DID document""" + if did.startswith("did:plc:"): + doc_url = f"https://plc.directory/{did}" + else: # did:web: + doc_url = f"https://{did[len('did:web:'):]}/.well-known/did.json" + import json + + for service in json.loads(_get(doc_url, headers)).get("service") or []: + if service.get("id", "").endswith("#atproto_pds") and service.get("serviceEndpoint"): + return service["serviceEndpoint"].rstrip("/") + raise ValueError(f"no PDS for {did}") + + +def _download(url: str, site_url: str) -> bytes: + """ + The avatar's bytes (v2.13.1): + - from StarPunk's own GoToSocial directly, as the crossposting client + does: behind the same proxy it resolves to an internal address, which + the SSRF-safe fetcher refuses; + - Bluesky CDN avatars from the account's PDS (com.atproto.sync.getBlob), + because the CDN refuses some servers' IPv4 addresses; + - anything else through the SSRF-safe fetcher. + """ + from urllib.parse import quote, urlparse + + import httpx + + from starpunk import webmention_ssrf + from starpunk.webmention_ssrf import user_agent + + headers = {"User-Agent": user_agent(site_url)} + own = _own_instance_host() + if own and urlparse(url).hostname == own: + with httpx.Client(timeout=10.0, transport=webmention_ssrf.TRANSPORT) as http: + with http.stream("GET", url, headers=headers) as response: + if response.status_code >= 400: + raise ValueError(f"HTTP {response.status_code}") + data = b"" + for chunk in response.iter_bytes(): + data += chunk + if len(data) > MAX_BYTES: + raise ValueError("too large") + return data + match = BSKY_CDN_RE.match(url) + if match: + did, cid = match.groups() + url = (f"{_pds(did, headers)}/xrpc/com.atproto.sync.getBlob" + f"?did={quote(did, safe=':')}&cid={cid}") + return _get(url, headers) + + def fill_missing(db_path, data_path, site_url: str, limit: int = 20) -> int: """Save avatars of shown responses that have no copy yet. Returns how many were tried.""" conn = sqlite3.connect(db_path) diff --git a/tests/test_backfeed.py b/tests/test_backfeed.py index 4a78dbe..cefce9f 100644 --- a/tests/test_backfeed.py +++ b/tests/test_backfeed.py @@ -562,3 +562,25 @@ def test_threads_avatar_kept_while_its_copy_exists(app, client, web): client.harvest = Harvest([again]) run(app) assert rows(app, note.id)[0]["author_photo"] == "https://cdn.example/p.jpg?sig=1" + + +def test_avatars_from_own_gotosocial_and_bluesky_pds(app, web, monkeypatch): + """v2.13.1: own GoToSocial may resolve to a private address; Bluesky CDN avatars come from the PDS""" + did, cid = "did:plc:abc123", "bafkreiabc" + web.files[f"{GTS}/fileserver/a/avatar.png"] = png() + web.files[f"https://plc.directory/{did}"] = json.dumps( + {"service": [{"id": "#atproto_pds", "serviceEndpoint": "https://pds.example"}]}).encode() + web.files[f"https://pds.example/xrpc/com.atproto.sync.getBlob?did={did}&cid={cid}"] = png("blue") + real_resolve = resolve + monkeypatch.setattr("starpunk.webmention_ssrf.socket.getaddrinfo", + lambda host, *a, **k: [(2, 1, 6, "", ("172.20.0.5", 0))] + if host == "social.example" else real_resolve(host)) + data = app.config["DATA_PATH"] + with app.app_context(): + assert avatars.fetch(data, f"{GTS}/fileserver/a/avatar.png", SITE) + assert avatars.fetch(data, f"https://cdn.bsky.app/img/avatar/plain/{did}/{cid}@jpeg", SITE) + # Other private addresses stay blocked + web.files["https://internal.example/a.png"] = png() + monkeypatch.setattr("starpunk.webmention_ssrf.socket.getaddrinfo", + lambda host, *a, **k: [(2, 1, 6, "", ("10.0.0.5", 0))]) + assert not avatars.fetch(data, "https://internal.example/a.png", SITE)