Files
StarPunk/starpunk/avatars.py
T
PhilandClaude Opus 5.5 62e257e0e0 feat: reply contexts that show the post (v2.19.0)
Replies to Mastodon posts showed only the server's domain: the page has no
h-entry, so nothing was found and the host stood in for the author.

- Context sources in order: Bluesky AppView API, microformats, the
  origin's Mastodon API or this site's GoToSocial instance, OpenGraph
- Migration 024: author_handle, photo, photo_alt, source; rows without a
  source are fetched again once by the webmention worker
- Note page: a card with avatar, name, handle, text, image and date;
  home/tag/likes: "In reply to <author>" with a short quote
- Avatars and images saved locally (CSP allows only this site's images);
  avatar upkeep now covers reply contexts so their avatars aren't removed

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-13 15:10:46 -07:00

285 lines
10 KiB
Python

"""
Local copies of response authors' avatars (v2.13.0, ADR-073 Decision 6)
Pages used to show other sites' avatar URLs directly, which tells those
sites who is reading, and Bluesky and Threads avatar URLs change or expire.
Each avatar is fetched once (SSRF-safe), shrunk to a 96 px WebP and saved
as DATA_PATH/avatars/<sha256 of its URL>.webp. Pages show the saved copy,
or the coloured initial until there is one.
The image of a post being replied to is kept the same way (v2.19.0): at most
PREVIEW_WIDTH wide, in DATA_PATH/previews/.
Paths and URLs are passed in, so the webmention worker can use it. Fetching
needs an app context (the SSRF-safe fetcher reads DEV_MODE).
"""
import hashlib
import io
import logging
import re
import sqlite3
import time
from pathlib import Path
from typing import Optional
logger = logging.getLogger(__name__)
SIZE = 96
MAX_BYTES = 1024 * 1024
PREVIEW_WIDTH = 640
PREVIEW_MAX_BYTES = 8 * 1024 * 1024
# Bluesky avatar on its CDN: .../img/<kind>/plain/<did>/<cid>[@jpeg]
BSKY_CDN_RE = re.compile(
r"^https://cdn\.bsky\.app/img/[a-z_]+/plain/(did:(?:plc|web):[A-Za-z0-9._:%-]+)/([a-z2-7]+)(?:@\w+)?$")
NAME_RE = re.compile(r"^[0-9a-f]{64}\.webp$")
CLEANUP_EVERY = 7 * 24 * 3600
def name_for(url: str) -> str:
return hashlib.sha256(url.encode()).hexdigest() + ".webp"
def folder(data_path) -> Path:
return Path(data_path) / "avatars"
def preview_folder(data_path) -> Path:
return Path(data_path) / "previews"
def _cached(directory: Path, url: Optional[str]) -> Optional[str]:
if not url or not url.startswith(("http://", "https://")):
return None
name = name_for(url)
return name if (directory / name).is_file() else None
def cached_name(data_path, url: Optional[str]) -> Optional[str]:
"""File name of the saved copy of this avatar, if there is one"""
return _cached(folder(data_path), url)
def cached_preview(data_path, url: Optional[str]) -> Optional[str]:
"""File name of the saved copy of a replied-to post's image, if there is one"""
return _cached(preview_folder(data_path), url)
def _square(image):
from PIL import Image
side = min(image.size)
left, top = (image.width - side) // 2, (image.height - side) // 2
image = image.crop((left, top, left + side, top + side))
return image.resize((SIZE, SIZE), Image.LANCZOS)
def _narrow(image):
from PIL import Image
if image.width <= PREVIEW_WIDTH:
return image
height = max(1, round(image.height * PREVIEW_WIDTH / image.width))
return image.resize((PREVIEW_WIDTH, height), Image.LANCZOS)
def _save(path: Path, url: Optional[str], site_url: str, shape, max_bytes: int,
what: str) -> bool:
if not url or not url.startswith(("http://", "https://")):
return False
if path.is_file():
return True
from PIL import Image
try:
with Image.open(io.BytesIO(_download(url, site_url, max_bytes))) as image:
image.load()
image = image.convert("RGBA" if "A" in image.getbands() else "RGB")
image = shape(image)
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_suffix(".tmp")
image.save(tmp, "WEBP", quality=80)
tmp.replace(path)
return True
except Exception as e:
logger.info(f"Could not save {what} {url}: {type(e).__name__}: {e}")
return False
def fetch(data_path, url: Optional[str], site_url: str) -> bool:
"""
Save a copy of an avatar unless there is one. Returns True if one exists
afterwards. Failures are logged and the page keeps showing the initial.
"""
if not url:
return False
return _save(folder(data_path) / name_for(url), url, site_url, _square,
MAX_BYTES, "avatar")
def fetch_preview(data_path, url: Optional[str], site_url: str) -> bool:
"""Save a copy of a replied-to post's image unless there is one (v2.19.0)"""
if not url:
return False
return _save(preview_folder(data_path) / name_for(url), url, site_url, _narrow,
PREVIEW_MAX_BYTES, "preview")
def _own_instance_host() -> Optional[str]:
"""Host of the GoToSocial server StarPunk posts to (it may resolve to an internal address)"""
from urllib.parse import urlparse
from flask import current_app
return urlparse(current_app.config.get("GOTOSOCIAL_URL") or "").hostname
def _get(url: str, headers: dict, max_bytes: int = MAX_BYTES) -> bytes:
"""GET a public URL through the SSRF-safe fetcher"""
from starpunk.webmention_ssrf import safe_get
result = safe_get(url, headers=headers, max_bytes=max_bytes, max_redirects=5,
timeout=10.0, truncate=False)
if result.status_code >= 400:
raise ValueError(f"HTTP {result.status_code}")
return result.content
def _pds(did: str, headers: dict) -> str:
"""The PDS that hosts a Bluesky account, from its DID document"""
if did.startswith("did:plc:"):
doc_url = f"https://plc.directory/{did}"
else: # did:web:<host>
doc_url = f"https://{did[len('did:web:'):]}/.well-known/did.json"
import json
for service in json.loads(_get(doc_url, headers)).get("service") or []:
if service.get("id", "").endswith("#atproto_pds") and service.get("serviceEndpoint"):
return service["serviceEndpoint"].rstrip("/")
raise ValueError(f"no PDS for {did}")
def _download(url: str, site_url: str, max_bytes: int = MAX_BYTES) -> bytes:
"""
The avatar's bytes (v2.13.1):
- from StarPunk's own GoToSocial directly, as the crossposting client
does: behind the same proxy it resolves to an internal address, which
the SSRF-safe fetcher refuses;
- Bluesky CDN avatars from the account's PDS (com.atproto.sync.getBlob),
because the CDN refuses some servers' IPv4 addresses;
- anything else through the SSRF-safe fetcher.
"""
from urllib.parse import quote, urlparse
import httpx
from starpunk import webmention_ssrf
from starpunk.webmention_ssrf import user_agent
headers = {"User-Agent": user_agent(site_url)}
own = _own_instance_host()
if own and urlparse(url).hostname == own:
with httpx.Client(timeout=10.0, transport=webmention_ssrf.TRANSPORT) as http:
with http.stream("GET", url, headers=headers) as response:
if response.is_redirect:
# GoToSocial sends media it hasn't cached to the original
# server: that one is fetched like any outside URL (v2.13.2)
from urllib.parse import urljoin
return _get(urljoin(url, response.headers["location"]), headers, max_bytes)
if response.status_code >= 400:
raise ValueError(f"HTTP {response.status_code}")
data = b""
for chunk in response.iter_bytes():
data += chunk
if len(data) > max_bytes:
raise ValueError("too large")
return data
match = BSKY_CDN_RE.match(url)
if match:
did, cid = match.groups()
url = (f"{_pds(did, headers)}/xrpc/com.atproto.sync.getBlob"
f"?did={quote(did, safe=':')}&cid={cid}")
return _get(url, headers, max_bytes)
def _context_images(conn) -> tuple[list, list]:
"""Avatars and post images of reply contexts that notes use (v2.19.0)"""
rows = conn.execute(
"""
SELECT rc.author_photo, rc.photo FROM reply_contexts rc
WHERE rc.fetch_status = 'fetched' AND EXISTS (
SELECT 1 FROM notes n WHERE n.deleted_at IS NULL AND rc.url IN
(n.in_reply_to, n.like_of, n.repost_of, n.bookmark_of))
ORDER BY rc.id DESC
"""
).fetchall()
return [r[0] for r in rows if r[0]], [r[1] for r in rows if r[1]]
def fill_missing(db_path, data_path, site_url: str, limit: int = 20) -> int:
"""Save avatars of shown responses and reply contexts, and the images of
replied-to posts, that have no copy yet. Returns how many were tried."""
conn = sqlite3.connect(db_path)
try:
urls = [row[0] for row in conn.execute(
"""
SELECT DISTINCT author_photo FROM webmentions
WHERE author_photo IS NOT NULL AND author_photo != ''
AND validation_status = 'valid' AND moderation_status = 'approved'
ORDER BY id DESC
"""
)]
context_avatars, context_photos = _context_images(conn)
finally:
conn.close()
wanted = [(url, cached_name, fetch) for url in dict.fromkeys(urls + context_avatars)]
wanted += [(url, cached_preview, fetch_preview) for url in dict.fromkeys(context_photos)]
tried = 0
for url, cached, save in wanted:
if tried >= limit:
break
if cached(data_path, url) or not url.startswith(("http://", "https://")):
continue
save(data_path, url, site_url)
tried += 1
return tried
def remove_unused(db_path, data_path) -> int:
"""Delete saved avatars and previews nothing uses. Returns how many were removed."""
conn = sqlite3.connect(db_path)
try:
used = {name_for(row[0]) for row in conn.execute(
"SELECT DISTINCT author_photo FROM webmentions WHERE author_photo IS NOT NULL")}
context_avatars, context_photos = _context_images(conn)
finally:
conn.close()
used |= {name_for(url) for url in context_avatars}
used_previews = {name_for(url) for url in context_photos}
removed = 0
for directory, keep in ((folder(data_path), used), (preview_folder(data_path), used_previews)):
if not directory.is_dir():
continue
for path in directory.iterdir():
if NAME_RE.match(path.name) and path.name not in keep:
path.unlink(missing_ok=True)
removed += 1
return removed
class Upkeep:
"""Runs fill_missing every `fill_every` seconds and remove_unused weekly"""
def __init__(self, fill_every: int = 600):
self.fill_every = fill_every
self._filled_at = 0.0
self._cleaned_at = 0.0
def run(self, db_path, data_path, site_url: str) -> None:
now = time.time()
if now - self._filled_at >= self.fill_every:
self._filled_at = now
fill_missing(db_path, data_path, site_url)
if now - self._cleaned_at >= CLEANUP_EVERY:
self._cleaned_at = now
removed = remove_unused(db_path, data_path)
if removed:
logger.info(f"Removed {removed} unused avatars")