Files
StarPunk/starpunk/avatars.py
T

218 lines
7.9 KiB
Python

"""
Local copies of response authors' avatars (v2.13.0, ADR-073 Decision 6)
Pages used to show other sites' avatar URLs directly, which tells those
sites who is reading, and Bluesky and Threads avatar URLs change or expire.
Each avatar is fetched once (SSRF-safe), shrunk to a 96 px WebP and saved
as DATA_PATH/avatars/<sha256 of its URL>.webp. Pages show the saved copy,
or the coloured initial until there is one.
Paths and URLs are passed in, so the webmention worker can use it. Fetching
needs an app context (the SSRF-safe fetcher reads DEV_MODE).
"""
import hashlib
import io
import logging
import re
import sqlite3
import time
from pathlib import Path
from typing import Optional
logger = logging.getLogger(__name__)
SIZE = 96
MAX_BYTES = 1024 * 1024
# Bluesky avatar on its CDN: .../img/<kind>/plain/<did>/<cid>[@jpeg]
BSKY_CDN_RE = re.compile(
r"^https://cdn\.bsky\.app/img/[a-z_]+/plain/(did:(?:plc|web):[A-Za-z0-9._:%-]+)/([a-z2-7]+)(?:@\w+)?$")
NAME_RE = re.compile(r"^[0-9a-f]{64}\.webp$")
CLEANUP_EVERY = 7 * 24 * 3600
def name_for(url: str) -> str:
return hashlib.sha256(url.encode()).hexdigest() + ".webp"
def folder(data_path) -> Path:
return Path(data_path) / "avatars"
def cached_name(data_path, url: Optional[str]) -> Optional[str]:
"""File name of the saved copy of this avatar, if there is one"""
if not url or not url.startswith(("http://", "https://")):
return None
name = name_for(url)
return name if (folder(data_path) / name).is_file() else None
def fetch(data_path, url: Optional[str], site_url: str) -> bool:
"""
Save a copy of an avatar unless there is one. Returns True if one exists
afterwards. Failures are logged and the page keeps showing the initial.
"""
if not url or not url.startswith(("http://", "https://")):
return False
path = folder(data_path) / name_for(url)
if path.is_file():
return True
from PIL import Image
try:
with Image.open(io.BytesIO(_download(url, site_url))) as image:
image.load()
image = image.convert("RGBA" if "A" in image.getbands() else "RGB")
side = min(image.size)
left, top = (image.width - side) // 2, (image.height - side) // 2
image = image.crop((left, top, left + side, top + side))
image = image.resize((SIZE, SIZE), Image.LANCZOS)
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_suffix(".tmp")
image.save(tmp, "WEBP", quality=80)
tmp.replace(path)
return True
except Exception as e:
logger.info(f"Could not save avatar {url}: {type(e).__name__}: {e}")
return False
def _own_instance_host() -> Optional[str]:
"""Host of the GoToSocial server StarPunk posts to (it may resolve to an internal address)"""
from urllib.parse import urlparse
from flask import current_app
return urlparse(current_app.config.get("GOTOSOCIAL_URL") or "").hostname
def _get(url: str, headers: dict) -> bytes:
"""GET a public URL through the SSRF-safe fetcher"""
from starpunk.webmention_ssrf import safe_get
result = safe_get(url, headers=headers, max_bytes=MAX_BYTES, max_redirects=5,
timeout=10.0, truncate=False)
if result.status_code >= 400:
raise ValueError(f"HTTP {result.status_code}")
return result.content
def _pds(did: str, headers: dict) -> str:
"""The PDS that hosts a Bluesky account, from its DID document"""
if did.startswith("did:plc:"):
doc_url = f"https://plc.directory/{did}"
else: # did:web:<host>
doc_url = f"https://{did[len('did:web:'):]}/.well-known/did.json"
import json
for service in json.loads(_get(doc_url, headers)).get("service") or []:
if service.get("id", "").endswith("#atproto_pds") and service.get("serviceEndpoint"):
return service["serviceEndpoint"].rstrip("/")
raise ValueError(f"no PDS for {did}")
def _download(url: str, site_url: str) -> bytes:
"""
The avatar's bytes (v2.13.1):
- from StarPunk's own GoToSocial directly, as the crossposting client
does: behind the same proxy it resolves to an internal address, which
the SSRF-safe fetcher refuses;
- Bluesky CDN avatars from the account's PDS (com.atproto.sync.getBlob),
because the CDN refuses some servers' IPv4 addresses;
- anything else through the SSRF-safe fetcher.
"""
from urllib.parse import quote, urlparse
import httpx
from starpunk import webmention_ssrf
from starpunk.webmention_ssrf import user_agent
headers = {"User-Agent": user_agent(site_url)}
own = _own_instance_host()
if own and urlparse(url).hostname == own:
with httpx.Client(timeout=10.0, transport=webmention_ssrf.TRANSPORT) as http:
with http.stream("GET", url, headers=headers) as response:
if response.is_redirect:
# GoToSocial sends media it hasn't cached to the original
# server: that one is fetched like any outside URL (v2.13.2)
from urllib.parse import urljoin
return _get(urljoin(url, response.headers["location"]), headers)
if response.status_code >= 400:
raise ValueError(f"HTTP {response.status_code}")
data = b""
for chunk in response.iter_bytes():
data += chunk
if len(data) > MAX_BYTES:
raise ValueError("too large")
return data
match = BSKY_CDN_RE.match(url)
if match:
did, cid = match.groups()
url = (f"{_pds(did, headers)}/xrpc/com.atproto.sync.getBlob"
f"?did={quote(did, safe=':')}&cid={cid}")
return _get(url, headers)
def fill_missing(db_path, data_path, site_url: str, limit: int = 20) -> int:
"""Save avatars of shown responses that have no copy yet. Returns how many were tried."""
conn = sqlite3.connect(db_path)
try:
urls = [row[0] for row in conn.execute(
"""
SELECT DISTINCT author_photo FROM webmentions
WHERE author_photo IS NOT NULL AND author_photo != ''
AND validation_status = 'valid' AND moderation_status = 'approved'
ORDER BY id DESC
"""
)]
finally:
conn.close()
tried = 0
for url in urls:
if tried >= limit:
break
if cached_name(data_path, url) or not url.startswith(("http://", "https://")):
continue
fetch(data_path, url, site_url)
tried += 1
return tried
def remove_unused(db_path, data_path) -> int:
"""Delete saved avatars no stored response uses. Returns how many were removed."""
directory = folder(data_path)
if not directory.is_dir():
return 0
conn = sqlite3.connect(db_path)
try:
used = {name_for(row[0]) for row in conn.execute(
"SELECT DISTINCT author_photo FROM webmentions WHERE author_photo IS NOT NULL")}
finally:
conn.close()
removed = 0
for path in directory.iterdir():
if NAME_RE.match(path.name) and path.name not in used:
path.unlink(missing_ok=True)
removed += 1
return removed
class Upkeep:
"""Runs fill_missing every `fill_every` seconds and remove_unused weekly"""
def __init__(self, fill_every: int = 600):
self.fill_every = fill_every
self._filled_at = 0.0
self._cleaned_at = 0.0
def run(self, db_path, data_path, site_url: str) -> None:
now = time.time()
if now - self._filled_at >= self.fill_every:
self._filled_at = now
fill_missing(db_path, data_path, site_url)
if now - self._cleaned_at >= CLEANUP_EVERY:
self._cleaned_at = now
removed = remove_unused(db_path, data_path)
if removed:
logger.info(f"Removed {removed} unused avatars")