Likes, reposts, quotes and replies to a note's GoToSocial, Bluesky and Threads copies are read back from the silos and stored as webmentions (ADR-073): age-based schedule to 90 days, pruning of withdrawn responses, #nobridge/#nobot opt-outs, admin backfill and Check now, Threads counts, a Quotes group, local avatar copies, and delete tombstones. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
145 lines
4.9 KiB
Python
145 lines
4.9 KiB
Python
"""
|
|
Local copies of response authors' avatars (v2.13.0, ADR-073 Decision 6)
|
|
|
|
Pages used to show other sites' avatar URLs directly, which tells those
|
|
sites who is reading, and Bluesky and Threads avatar URLs change or expire.
|
|
Each avatar is fetched once (SSRF-safe), shrunk to a 96 px WebP and saved
|
|
as DATA_PATH/avatars/<sha256 of its URL>.webp. Pages show the saved copy,
|
|
or the coloured initial until there is one.
|
|
|
|
Paths and URLs are passed in, so the webmention worker can use it. Fetching
|
|
needs an app context (the SSRF-safe fetcher reads DEV_MODE).
|
|
"""
|
|
|
|
import hashlib
|
|
import io
|
|
import logging
|
|
import re
|
|
import sqlite3
|
|
import time
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
SIZE = 96
|
|
MAX_BYTES = 1024 * 1024
|
|
NAME_RE = re.compile(r"^[0-9a-f]{64}\.webp$")
|
|
CLEANUP_EVERY = 7 * 24 * 3600
|
|
|
|
|
|
def name_for(url: str) -> str:
|
|
return hashlib.sha256(url.encode()).hexdigest() + ".webp"
|
|
|
|
|
|
def folder(data_path) -> Path:
|
|
return Path(data_path) / "avatars"
|
|
|
|
|
|
def cached_name(data_path, url: Optional[str]) -> Optional[str]:
|
|
"""File name of the saved copy of this avatar, if there is one"""
|
|
if not url or not url.startswith(("http://", "https://")):
|
|
return None
|
|
name = name_for(url)
|
|
return name if (folder(data_path) / name).is_file() else None
|
|
|
|
|
|
def fetch(data_path, url: Optional[str], site_url: str) -> bool:
|
|
"""
|
|
Save a copy of an avatar unless there is one. Returns True if one exists
|
|
afterwards. Failures are logged and the page keeps showing the initial.
|
|
"""
|
|
if not url or not url.startswith(("http://", "https://")):
|
|
return False
|
|
path = folder(data_path) / name_for(url)
|
|
if path.is_file():
|
|
return True
|
|
from PIL import Image
|
|
|
|
from starpunk.webmention_ssrf import safe_get, user_agent
|
|
|
|
try:
|
|
result = safe_get(url, headers={"User-Agent": user_agent(site_url)},
|
|
max_bytes=MAX_BYTES, max_redirects=5, timeout=10.0, truncate=False)
|
|
if result.status_code >= 400:
|
|
raise ValueError(f"HTTP {result.status_code}")
|
|
with Image.open(io.BytesIO(result.content)) as image:
|
|
image.load()
|
|
image = image.convert("RGBA" if "A" in image.getbands() else "RGB")
|
|
side = min(image.size)
|
|
left, top = (image.width - side) // 2, (image.height - side) // 2
|
|
image = image.crop((left, top, left + side, top + side))
|
|
image = image.resize((SIZE, SIZE), Image.LANCZOS)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
tmp = path.with_suffix(".tmp")
|
|
image.save(tmp, "WEBP", quality=80)
|
|
tmp.replace(path)
|
|
return True
|
|
except Exception as e:
|
|
logger.info(f"Could not save avatar {url}: {type(e).__name__}: {e}")
|
|
return False
|
|
|
|
|
|
def fill_missing(db_path, data_path, site_url: str, limit: int = 20) -> int:
|
|
"""Save avatars of shown responses that have no copy yet. Returns how many were tried."""
|
|
conn = sqlite3.connect(db_path)
|
|
try:
|
|
urls = [row[0] for row in conn.execute(
|
|
"""
|
|
SELECT DISTINCT author_photo FROM webmentions
|
|
WHERE author_photo IS NOT NULL AND author_photo != ''
|
|
AND validation_status = 'valid' AND moderation_status = 'approved'
|
|
ORDER BY id DESC
|
|
"""
|
|
)]
|
|
finally:
|
|
conn.close()
|
|
tried = 0
|
|
for url in urls:
|
|
if tried >= limit:
|
|
break
|
|
if cached_name(data_path, url) or not url.startswith(("http://", "https://")):
|
|
continue
|
|
fetch(data_path, url, site_url)
|
|
tried += 1
|
|
return tried
|
|
|
|
|
|
def remove_unused(db_path, data_path) -> int:
|
|
"""Delete saved avatars no stored response uses. Returns how many were removed."""
|
|
directory = folder(data_path)
|
|
if not directory.is_dir():
|
|
return 0
|
|
conn = sqlite3.connect(db_path)
|
|
try:
|
|
used = {name_for(row[0]) for row in conn.execute(
|
|
"SELECT DISTINCT author_photo FROM webmentions WHERE author_photo IS NOT NULL")}
|
|
finally:
|
|
conn.close()
|
|
removed = 0
|
|
for path in directory.iterdir():
|
|
if NAME_RE.match(path.name) and path.name not in used:
|
|
path.unlink(missing_ok=True)
|
|
removed += 1
|
|
return removed
|
|
|
|
|
|
class Upkeep:
|
|
"""Runs fill_missing every `fill_every` seconds and remove_unused weekly"""
|
|
|
|
def __init__(self, fill_every: int = 600):
|
|
self.fill_every = fill_every
|
|
self._filled_at = 0.0
|
|
self._cleaned_at = 0.0
|
|
|
|
def run(self, db_path, data_path, site_url: str) -> None:
|
|
now = time.time()
|
|
if now - self._filled_at >= self.fill_every:
|
|
self._filled_at = now
|
|
fill_missing(db_path, data_path, site_url)
|
|
if now - self._cleaned_at >= CLEANUP_EVERY:
|
|
self._cleaned_at = now
|
|
removed = remove_unused(db_path, data_path)
|
|
if removed:
|
|
logger.info(f"Removed {removed} unused avatars")
|