Replies to Mastodon posts showed only the server's domain: the page has no h-entry, so nothing was found and the host stood in for the author. - Context sources in order: Bluesky AppView API, microformats, the origin's Mastodon API or this site's GoToSocial instance, OpenGraph - Migration 024: author_handle, photo, photo_alt, source; rows without a source are fetched again once by the webmention worker - Note page: a card with avatar, name, handle, text, image and date; home/tag/likes: "In reply to <author>" with a short quote - Avatars and images saved locally (CSP allows only this site's images); avatar upkeep now covers reply contexts so their avatars aren't removed Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
285 lines
10 KiB
Python
285 lines
10 KiB
Python
"""
|
|
Local copies of response authors' avatars (v2.13.0, ADR-073 Decision 6)
|
|
|
|
Pages used to show other sites' avatar URLs directly, which tells those
|
|
sites who is reading, and Bluesky and Threads avatar URLs change or expire.
|
|
Each avatar is fetched once (SSRF-safe), shrunk to a 96 px WebP and saved
|
|
as DATA_PATH/avatars/<sha256 of its URL>.webp. Pages show the saved copy,
|
|
or the coloured initial until there is one.
|
|
|
|
The image of a post being replied to is kept the same way (v2.19.0): at most
|
|
PREVIEW_WIDTH wide, in DATA_PATH/previews/.
|
|
|
|
Paths and URLs are passed in, so the webmention worker can use it. Fetching
|
|
needs an app context (the SSRF-safe fetcher reads DEV_MODE).
|
|
"""
|
|
|
|
import hashlib
|
|
import io
|
|
import logging
|
|
import re
|
|
import sqlite3
|
|
import time
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
SIZE = 96
|
|
MAX_BYTES = 1024 * 1024
|
|
PREVIEW_WIDTH = 640
|
|
PREVIEW_MAX_BYTES = 8 * 1024 * 1024
|
|
# Bluesky avatar on its CDN: .../img/<kind>/plain/<did>/<cid>[@jpeg]
|
|
BSKY_CDN_RE = re.compile(
|
|
r"^https://cdn\.bsky\.app/img/[a-z_]+/plain/(did:(?:plc|web):[A-Za-z0-9._:%-]+)/([a-z2-7]+)(?:@\w+)?$")
|
|
NAME_RE = re.compile(r"^[0-9a-f]{64}\.webp$")
|
|
CLEANUP_EVERY = 7 * 24 * 3600
|
|
|
|
|
|
def name_for(url: str) -> str:
|
|
return hashlib.sha256(url.encode()).hexdigest() + ".webp"
|
|
|
|
|
|
def folder(data_path) -> Path:
|
|
return Path(data_path) / "avatars"
|
|
|
|
|
|
def preview_folder(data_path) -> Path:
|
|
return Path(data_path) / "previews"
|
|
|
|
|
|
def _cached(directory: Path, url: Optional[str]) -> Optional[str]:
|
|
if not url or not url.startswith(("http://", "https://")):
|
|
return None
|
|
name = name_for(url)
|
|
return name if (directory / name).is_file() else None
|
|
|
|
|
|
def cached_name(data_path, url: Optional[str]) -> Optional[str]:
|
|
"""File name of the saved copy of this avatar, if there is one"""
|
|
return _cached(folder(data_path), url)
|
|
|
|
|
|
def cached_preview(data_path, url: Optional[str]) -> Optional[str]:
|
|
"""File name of the saved copy of a replied-to post's image, if there is one"""
|
|
return _cached(preview_folder(data_path), url)
|
|
|
|
|
|
def _square(image):
|
|
from PIL import Image
|
|
side = min(image.size)
|
|
left, top = (image.width - side) // 2, (image.height - side) // 2
|
|
image = image.crop((left, top, left + side, top + side))
|
|
return image.resize((SIZE, SIZE), Image.LANCZOS)
|
|
|
|
|
|
def _narrow(image):
|
|
from PIL import Image
|
|
if image.width <= PREVIEW_WIDTH:
|
|
return image
|
|
height = max(1, round(image.height * PREVIEW_WIDTH / image.width))
|
|
return image.resize((PREVIEW_WIDTH, height), Image.LANCZOS)
|
|
|
|
|
|
def _save(path: Path, url: Optional[str], site_url: str, shape, max_bytes: int,
|
|
what: str) -> bool:
|
|
if not url or not url.startswith(("http://", "https://")):
|
|
return False
|
|
if path.is_file():
|
|
return True
|
|
from PIL import Image
|
|
|
|
try:
|
|
with Image.open(io.BytesIO(_download(url, site_url, max_bytes))) as image:
|
|
image.load()
|
|
image = image.convert("RGBA" if "A" in image.getbands() else "RGB")
|
|
image = shape(image)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
tmp = path.with_suffix(".tmp")
|
|
image.save(tmp, "WEBP", quality=80)
|
|
tmp.replace(path)
|
|
return True
|
|
except Exception as e:
|
|
logger.info(f"Could not save {what} {url}: {type(e).__name__}: {e}")
|
|
return False
|
|
|
|
|
|
def fetch(data_path, url: Optional[str], site_url: str) -> bool:
|
|
"""
|
|
Save a copy of an avatar unless there is one. Returns True if one exists
|
|
afterwards. Failures are logged and the page keeps showing the initial.
|
|
"""
|
|
if not url:
|
|
return False
|
|
return _save(folder(data_path) / name_for(url), url, site_url, _square,
|
|
MAX_BYTES, "avatar")
|
|
|
|
|
|
def fetch_preview(data_path, url: Optional[str], site_url: str) -> bool:
|
|
"""Save a copy of a replied-to post's image unless there is one (v2.19.0)"""
|
|
if not url:
|
|
return False
|
|
return _save(preview_folder(data_path) / name_for(url), url, site_url, _narrow,
|
|
PREVIEW_MAX_BYTES, "preview")
|
|
|
|
|
|
def _own_instance_host() -> Optional[str]:
|
|
"""Host of the GoToSocial server StarPunk posts to (it may resolve to an internal address)"""
|
|
from urllib.parse import urlparse
|
|
|
|
from flask import current_app
|
|
return urlparse(current_app.config.get("GOTOSOCIAL_URL") or "").hostname
|
|
|
|
|
|
def _get(url: str, headers: dict, max_bytes: int = MAX_BYTES) -> bytes:
|
|
"""GET a public URL through the SSRF-safe fetcher"""
|
|
from starpunk.webmention_ssrf import safe_get
|
|
|
|
result = safe_get(url, headers=headers, max_bytes=max_bytes, max_redirects=5,
|
|
timeout=10.0, truncate=False)
|
|
if result.status_code >= 400:
|
|
raise ValueError(f"HTTP {result.status_code}")
|
|
return result.content
|
|
|
|
|
|
def _pds(did: str, headers: dict) -> str:
|
|
"""The PDS that hosts a Bluesky account, from its DID document"""
|
|
if did.startswith("did:plc:"):
|
|
doc_url = f"https://plc.directory/{did}"
|
|
else: # did:web:<host>
|
|
doc_url = f"https://{did[len('did:web:'):]}/.well-known/did.json"
|
|
import json
|
|
|
|
for service in json.loads(_get(doc_url, headers)).get("service") or []:
|
|
if service.get("id", "").endswith("#atproto_pds") and service.get("serviceEndpoint"):
|
|
return service["serviceEndpoint"].rstrip("/")
|
|
raise ValueError(f"no PDS for {did}")
|
|
|
|
|
|
def _download(url: str, site_url: str, max_bytes: int = MAX_BYTES) -> bytes:
|
|
"""
|
|
The avatar's bytes (v2.13.1):
|
|
- from StarPunk's own GoToSocial directly, as the crossposting client
|
|
does: behind the same proxy it resolves to an internal address, which
|
|
the SSRF-safe fetcher refuses;
|
|
- Bluesky CDN avatars from the account's PDS (com.atproto.sync.getBlob),
|
|
because the CDN refuses some servers' IPv4 addresses;
|
|
- anything else through the SSRF-safe fetcher.
|
|
"""
|
|
from urllib.parse import quote, urlparse
|
|
|
|
import httpx
|
|
|
|
from starpunk import webmention_ssrf
|
|
from starpunk.webmention_ssrf import user_agent
|
|
|
|
headers = {"User-Agent": user_agent(site_url)}
|
|
own = _own_instance_host()
|
|
if own and urlparse(url).hostname == own:
|
|
with httpx.Client(timeout=10.0, transport=webmention_ssrf.TRANSPORT) as http:
|
|
with http.stream("GET", url, headers=headers) as response:
|
|
if response.is_redirect:
|
|
# GoToSocial sends media it hasn't cached to the original
|
|
# server: that one is fetched like any outside URL (v2.13.2)
|
|
from urllib.parse import urljoin
|
|
return _get(urljoin(url, response.headers["location"]), headers, max_bytes)
|
|
if response.status_code >= 400:
|
|
raise ValueError(f"HTTP {response.status_code}")
|
|
data = b""
|
|
for chunk in response.iter_bytes():
|
|
data += chunk
|
|
if len(data) > max_bytes:
|
|
raise ValueError("too large")
|
|
return data
|
|
match = BSKY_CDN_RE.match(url)
|
|
if match:
|
|
did, cid = match.groups()
|
|
url = (f"{_pds(did, headers)}/xrpc/com.atproto.sync.getBlob"
|
|
f"?did={quote(did, safe=':')}&cid={cid}")
|
|
return _get(url, headers, max_bytes)
|
|
|
|
|
|
def _context_images(conn) -> tuple[list, list]:
|
|
"""Avatars and post images of reply contexts that notes use (v2.19.0)"""
|
|
rows = conn.execute(
|
|
"""
|
|
SELECT rc.author_photo, rc.photo FROM reply_contexts rc
|
|
WHERE rc.fetch_status = 'fetched' AND EXISTS (
|
|
SELECT 1 FROM notes n WHERE n.deleted_at IS NULL AND rc.url IN
|
|
(n.in_reply_to, n.like_of, n.repost_of, n.bookmark_of))
|
|
ORDER BY rc.id DESC
|
|
"""
|
|
).fetchall()
|
|
return [r[0] for r in rows if r[0]], [r[1] for r in rows if r[1]]
|
|
|
|
|
|
def fill_missing(db_path, data_path, site_url: str, limit: int = 20) -> int:
|
|
"""Save avatars of shown responses and reply contexts, and the images of
|
|
replied-to posts, that have no copy yet. Returns how many were tried."""
|
|
conn = sqlite3.connect(db_path)
|
|
try:
|
|
urls = [row[0] for row in conn.execute(
|
|
"""
|
|
SELECT DISTINCT author_photo FROM webmentions
|
|
WHERE author_photo IS NOT NULL AND author_photo != ''
|
|
AND validation_status = 'valid' AND moderation_status = 'approved'
|
|
ORDER BY id DESC
|
|
"""
|
|
)]
|
|
context_avatars, context_photos = _context_images(conn)
|
|
finally:
|
|
conn.close()
|
|
wanted = [(url, cached_name, fetch) for url in dict.fromkeys(urls + context_avatars)]
|
|
wanted += [(url, cached_preview, fetch_preview) for url in dict.fromkeys(context_photos)]
|
|
tried = 0
|
|
for url, cached, save in wanted:
|
|
if tried >= limit:
|
|
break
|
|
if cached(data_path, url) or not url.startswith(("http://", "https://")):
|
|
continue
|
|
save(data_path, url, site_url)
|
|
tried += 1
|
|
return tried
|
|
|
|
|
|
def remove_unused(db_path, data_path) -> int:
|
|
"""Delete saved avatars and previews nothing uses. Returns how many were removed."""
|
|
conn = sqlite3.connect(db_path)
|
|
try:
|
|
used = {name_for(row[0]) for row in conn.execute(
|
|
"SELECT DISTINCT author_photo FROM webmentions WHERE author_photo IS NOT NULL")}
|
|
context_avatars, context_photos = _context_images(conn)
|
|
finally:
|
|
conn.close()
|
|
used |= {name_for(url) for url in context_avatars}
|
|
used_previews = {name_for(url) for url in context_photos}
|
|
removed = 0
|
|
for directory, keep in ((folder(data_path), used), (preview_folder(data_path), used_previews)):
|
|
if not directory.is_dir():
|
|
continue
|
|
for path in directory.iterdir():
|
|
if NAME_RE.match(path.name) and path.name not in keep:
|
|
path.unlink(missing_ok=True)
|
|
removed += 1
|
|
return removed
|
|
|
|
|
|
class Upkeep:
|
|
"""Runs fill_missing every `fill_every` seconds and remove_unused weekly"""
|
|
|
|
def __init__(self, fill_every: int = 600):
|
|
self.fill_every = fill_every
|
|
self._filled_at = 0.0
|
|
self._cleaned_at = 0.0
|
|
|
|
def run(self, db_path, data_path, site_url: str) -> None:
|
|
now = time.time()
|
|
if now - self._filled_at >= self.fill_every:
|
|
self._filled_at = now
|
|
fill_missing(db_path, data_path, site_url)
|
|
if now - self._cleaned_at >= CLEANUP_EVERY:
|
|
self._cleaned_at = now
|
|
removed = remove_unused(db_path, data_path)
|
|
if removed:
|
|
logger.info(f"Removed {removed} unused avatars")
|