Files
StarPunk/starpunk/reply_context.py
T
PhilandClaude Opus 5.5 952ca300c0 fix: keep GIFs moving in reply contexts (v2.20.1)
Mastodon turns GIFs into silent looping MP4s (gifv); only the first-frame
preview was kept. The MP4 is now saved locally and played muted and
looping with the still as poster. Animated GIFs from other sites are saved
as animated WebP. Migration 025 adds reply_contexts.video and has
fediverse contexts with an image fetched again once.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-13 20:28:20 -07:00

577 lines
22 KiB
Python

"""
Reply context fetching for in-reply-to URLs
Per database-schema.md Story 2.4:
- Fetch context from in_reply_to URL when note is reply
- Parse microformats for author, content
- Store in reply_contexts table
- Graceful degradation on fetch failure
Fediverse and Bluesky pages carry no h-entry, so from v2.19.0 the context
comes from the first source that knows the post:
1. Bluesky post URLs: the public Bluesky AppView API
2. Pages with an h-entry: microformats (IndieWeb sites)
3. Fediverse posts: the origin's public Mastodon API, else this site's
GoToSocial instance resolving the URL (as likes and boosts already do;
it fetches with signed requests, so it works for GoToSocial and for
Mastodon servers that refuse anonymous API calls)
4. Anything else: the page's OpenGraph tags, with the page's h-card as author
The source used is stored in reply_contexts.source. Rows from before v2.19.0
have none; refresh_stale() fetches those again once.
References:
- Database Schema: /docs/design/v1.6.0/database-schema.md
- Background Worker: /docs/design/v1.6.0/background-worker.md
- Design: /docs/design/v2.19.0/2026-11-13-reply-context-sources.md
"""
import html
import json
import logging
import re
import sqlite3
from pathlib import Path
from typing import Optional, Dict, Any
from urllib.parse import quote, urljoin, urlparse
import mf2py
from starpunk import mf2_utils
from starpunk.webmention_processor import (
fetch_source,
extract_content,
sanitize_content,
ValidationError,
_is_http_url,
)
logger = logging.getLogger(__name__)
SNIPPET_LIMIT = 500
BLUESKY_POST_RE = re.compile(r"^https://bsky\.app/profile/([^/]+)/post/([^/?#]+)")
BLUESKY_API = "https://public.api.bsky.app/xrpc/app.bsky.feed.getPostThread"
# Mastodon: https://host/@user/123456 (numeric ids; the public API may answer)
MASTODON_STATUS_RE = re.compile(r"^/@[^/]+/(\d+)/?$")
# Fediverse post paths: Mastodon, GoToSocial, ActivityPub ids, Misskey,
# Pleroma/Akkoma
FEDIVERSE_PATH_RE = re.compile(
r"^/(?:@[^/]+/(?:statuses/)?[A-Za-z0-9]+|users/[^/]+/statuses/[A-Za-z0-9]+|"
r"notes/[A-Za-z0-9]+|notice/[A-Za-z0-9]+|objects/[A-Za-z0-9-]+)/?$")
# og:title of fediverse posts: "Display Name (@user@host)"
FEDI_TITLE_RE = re.compile(r"^(.*?)\s*\((@[^)\s]+)\)\s*$")
FIELDS = ('title', 'content_snippet', 'published_at', 'author_name', 'author_handle',
'author_url', 'author_photo', 'site_name', 'photo', 'photo_alt', 'video')
def fetch_reply_context(
db_path: Path,
reply_url: str,
site_url: str,
refresh: bool = False
) -> Dict[str, Any]:
"""
Fetch and parse reply context from a URL.
Per database-schema.md:
- Fetch URL content with SSRF protection
- Parse h-entry, a Mastodon status, a Bluesky post or OpenGraph tags
- Store in reply_contexts table
- Graceful degradation: errors don't prevent reply creation
Args:
db_path: Path to SQLite database
reply_url: URL being replied to
site_url: Site URL for User-Agent header
refresh: Fetch again even if a context is stored
Returns:
Dict with fetch status and context data
"""
existing = get_reply_context(db_path, reply_url)
# Contexts stored before v2.19.0 (no source) are fetched again
if existing and existing.get('source') and not refresh:
logger.debug(f"Reply context already cached for {reply_url}")
return existing
try:
found = _find_context(reply_url, site_url)
except Exception as e:
if isinstance(e, ValidationError):
logger.warning(f"Failed to fetch reply context for {reply_url}: {e}")
error = str(e)
else:
logger.error(f"Error fetching reply context for {reply_url}: {e}", exc_info=True)
error = f"Unexpected error: {e}"
if existing and existing.get('fetch_status') == 'fetched':
# Keep what an earlier fetch found; don't try again
_mark_source(db_path, reply_url, existing.get('source') or 'legacy')
return existing
return _store_minimal_context(db_path, reply_url, error)
context = _store_context(db_path, reply_url, found)
_save_images(context, site_url)
logger.info(f"Fetched reply context for {reply_url} ({context['source']})")
return context
def refresh_stale(db_path: Path, site_url: str, limit: int = 5) -> int:
"""
Fetch again the contexts stored before v2.19.0 that notes still use.
The webmention worker calls this; each row is fetched once, since the
new fetch records its source. Returns how many were fetched.
"""
conn = sqlite3.connect(db_path)
try:
urls = [row[0] for row in conn.execute(
"""
SELECT rc.url FROM reply_contexts rc
WHERE rc.source IS NULL AND EXISTS (
SELECT 1 FROM notes n WHERE n.deleted_at IS NULL AND rc.url IN
(n.in_reply_to, n.like_of, n.repost_of, n.bookmark_of))
ORDER BY rc.id DESC LIMIT ?
""", (limit,))]
finally:
conn.close()
for url in urls:
fetch_reply_context(db_path, url, site_url, refresh=True)
return len(urls)
# --- Sources ----------------------------------------------------------------
def _find_context(url: str, site_url: str) -> Dict[str, Any]:
"""Context fields and 'source' for a URL; raises ValidationError if nothing could be fetched"""
if BLUESKY_POST_RE.match(url):
try:
found = _from_bluesky(url, site_url)
except ValidationError as e:
logger.info(f"Bluesky API had no post for {url}: {e}")
found = None
if found:
return found
page_error = None
content = content_type = final_url = None
try:
content, content_type, final_url = fetch_source(url, site_url)
except ValidationError as e:
page_error = e
og: Dict[str, Optional[str]] = {}
card: Dict[str, Optional[str]] = {}
is_html = content is not None and 'html' in (content_type or '').lower()
if is_html:
parsed = mf2py.parse(doc=content, url=final_url)
entry = mf2_utils.responding_entry(parsed, '')
og = _opengraph(content, final_url)
if entry:
return _from_mf2(parsed, entry, final_url, og)
card = _page_card(parsed, final_url)
if _looks_fediverse(url, og):
status = _mastodon_api(url, site_url) or _resolve_with_instance(url)
if status:
return _from_status(status, url)
if page_error:
raise page_error
if not is_html:
raise ValidationError("Not HTML content")
return _from_opengraph(og, card)
def _from_mf2(parsed: dict, entry: dict, page_url: str, og: dict) -> Dict[str, Any]:
"""An IndieWeb post's h-entry"""
author = mf2_utils.entry_author(parsed, entry, page_url)
snippet = extract_content(parsed, entry)
title = mf2_utils.first(entry, 'name')
if title and _repeats(title, snippet):
title = None # implied name: the content again
photo, photo_alt = mf2_utils.first(entry, 'photo'), None
for value in entry.get('properties', {}).get('photo', [])[:1]:
if isinstance(value, dict):
photo_alt = value.get('alt') or None
if not _is_http_url(photo):
photo, photo_alt = og.get('image'), og.get('image_alt')
return {
'source': 'mf2',
'title': _clip(title, 200),
'content_snippet': snippet,
'published_at': _iso(mf2_utils.first(entry, 'published')),
'author_name': author['name'],
'author_url': author['url'] if _is_http_url(author['url']) else None,
'author_photo': author['photo'] if _is_http_url(author['photo']) else None,
'site_name': og.get('site_name'),
'photo': photo if _is_http_url(photo) else None,
'photo_alt': photo_alt,
}
def _from_bluesky(url: str, site_url: str) -> Optional[Dict[str, Any]]:
"""A Bluesky post from the public AppView API (no login needed)"""
actor, rkey = BLUESKY_POST_RE.match(url).groups()
uri = quote(f"at://{actor}/app.bsky.feed.post/{rkey}", safe='')
data = _get_json(f"{BLUESKY_API}?uri={uri}&depth=0&parentHeight=0", site_url)
post = ((data or {}).get('thread') or {}).get('post') if isinstance(data, dict) else None
if not post:
return None
author = post.get('author') or {}
record = post.get('record') or {}
handle = author.get('handle')
photo, photo_alt = _bluesky_image(post.get('embed') or {})
return {
'source': 'bluesky',
'title': None,
'content_snippet': _text_snippet(record.get('text')),
'published_at': _iso(record.get('createdAt')),
'author_name': author.get('displayName') or handle,
'author_handle': f"@{handle}" if handle else None,
'author_url': f"https://bsky.app/profile/{handle}" if handle else None,
'author_photo': author.get('avatar') if _is_http_url(author.get('avatar')) else None,
'site_name': 'Bluesky',
'photo': photo,
'photo_alt': photo_alt,
}
def _bluesky_image(embed: dict) -> tuple:
"""First image of a post's embed view: (url, alt)"""
kind = embed.get('$type', '')
if kind.startswith('app.bsky.embed.recordWithMedia'):
return _bluesky_image(embed.get('media') or {})
if kind.startswith('app.bsky.embed.images'):
for image in embed.get('images') or []:
if _is_http_url(image.get('thumb')):
return image['thumb'], image.get('alt') or None
if kind.startswith('app.bsky.embed.video') and _is_http_url(embed.get('thumbnail')):
return embed['thumbnail'], embed.get('alt') or None
if kind.startswith('app.bsky.embed.external'):
external = embed.get('external') or {}
if _is_http_url(external.get('thumb')):
return external['thumb'], None
return None, None
def _looks_fediverse(url: str, og: dict) -> bool:
return bool(FEDIVERSE_PATH_RE.match(urlparse(url).path) or og.get('username'))
def _mastodon_api(url: str, site_url: str) -> Optional[dict]:
"""The status from its own server's public API (Mastodon post URLs only)"""
parts = urlparse(url)
m = MASTODON_STATUS_RE.match(parts.path)
if not m:
return None
try:
status = _get_json(f"{parts.scheme}://{parts.netloc}/api/v1/statuses/{m.group(1)}", site_url)
except ValidationError:
return None
return status if isinstance(status, dict) and status.get('account') else None
def _resolve_with_instance(url: str) -> Optional[dict]:
"""The status as this site's GoToSocial instance sees it, if one is configured"""
try:
from starpunk.syndication import get_targets
target = get_targets().get('gotosocial')
if target is None:
return None
return target.client.resolve_status(url)
except Exception as e:
logger.info(f"Instance could not resolve {url}: {e}")
return None
def _from_status(status: dict, url: str) -> Dict[str, Any]:
"""A Mastodon API status entity"""
account = status.get('account') or {}
acct = account.get('acct') or account.get('username')
host = urlparse(account.get('url') or url).hostname
if acct and '@' not in acct and host:
acct = f"{acct}@{host}"
spoiler = (status.get('spoiler_text') or '').strip()
if spoiler:
# Keep the content warning: show it, not what it covers
snippet = _text_snippet(f"CW: {spoiler}")
else:
snippet = _snippet(status.get('content') or '') or None
photo = photo_alt = video = None
if not status.get('sensitive'):
for media in status.get('media_attachments') or []:
if media.get('type') in ('image', 'gifv', 'video') and _is_http_url(media.get('preview_url')):
photo, photo_alt = media['preview_url'], media.get('description') or None
# A "GIF" on Mastodon is a silent looping MP4; the preview is its first frame
if media['type'] == 'gifv' and _is_http_url(media.get('url')):
video = media['url']
break
avatar = account.get('avatar_static') or account.get('avatar')
return {
'source': 'fediverse',
'title': None,
'content_snippet': snippet,
'published_at': _iso(status.get('created_at')),
'author_name': account.get('display_name') or account.get('username'),
'author_handle': f"@{acct}" if acct else None,
'author_url': account.get('url') if _is_http_url(account.get('url')) else None,
'author_photo': avatar if _is_http_url(avatar) else None,
'site_name': urlparse(url).hostname,
'photo': photo,
'photo_alt': photo_alt,
'video': video,
}
def _from_opengraph(og: dict, card: dict) -> Dict[str, Any]:
"""Any other page: OpenGraph tags, the page's h-card as author"""
title = og.get('title')
author_name, handle = card.get('name') or og.get('author'), None
m = FEDI_TITLE_RE.match(title or '')
if m and (og.get('username') or not author_name):
# A fediverse post's og:title is its author, not a title
author_name, handle, title = m.group(1) or author_name, m.group(2), None
elif og.get('username') and title:
author_name, handle, title = title, f"@{og['username'].lstrip('@')}", None
snippet = _text_snippet(og.get('description'))
if title and snippet and _repeats(title, snippet):
snippet = None
return {
'source': 'opengraph',
'title': _clip(title, 200),
'content_snippet': snippet,
'published_at': _iso(og.get('published')),
'author_name': author_name,
'author_handle': handle,
'author_url': card.get('url') if _is_http_url(card.get('url')) else None,
'author_photo': card.get('photo') if _is_http_url(card.get('photo')) else None,
'site_name': og.get('site_name'),
'photo': og.get('image') if _is_http_url(og.get('image')) else None,
'photo_alt': og.get('image_alt'),
}
OG_FIELDS = {
'og:title': 'title', 'twitter:title': 'title',
'og:description': 'description', 'twitter:description': 'description',
'description': 'description',
'og:image': 'image', 'og:image:url': 'image', 'twitter:image': 'image',
'og:image:alt': 'image_alt', 'twitter:image:alt': 'image_alt',
'og:site_name': 'site_name',
'article:published_time': 'published', 'og:article:published_time': 'published',
'og:published_time': 'published',
'profile:username': 'username', 'og:profile:username': 'username',
'author': 'author', 'article:author': 'author',
}
def _opengraph(content: str, page_url: str) -> Dict[str, Optional[str]]:
"""OpenGraph (and plain) meta tags of a page; the first of each wins"""
from lxml import html as lxml_html
try:
doc = lxml_html.document_fromstring(content)
except Exception:
return {}
found: Dict[str, Optional[str]] = {}
for meta in doc.iter('meta'):
key = (meta.get('property') or meta.get('name') or '').strip().lower()
value = (meta.get('content') or '').strip()
field = OG_FIELDS.get(key)
if not field or not value:
continue
if field == 'image_alt':
# Only the alt text of the image we keep: the one right after it
if 'image' in found and 'image_alt' not in found and not found.get('_alt_done'):
found['image_alt'] = value
continue
if field == 'image':
if 'image' in found:
found['_alt_done'] = '1'
continue
value = urljoin(page_url, value)
if field == 'author' and _is_http_url(value):
continue # article:author is often a profile URL
found.setdefault(field, value)
found.pop('_alt_done', None)
if 'title' not in found:
titles = doc.findall('.//title')
if titles and (titles[0].text_content() or '').strip():
found['title'] = titles[0].text_content().strip()
return found
def _page_card(parsed: dict, page_url: str) -> Dict[str, Optional[str]]:
"""The page's representative h-card, else its first h-card (GoToSocial pages)"""
card = mf2_utils.representative_hcard(parsed, page_url)
if card is None:
card = next((i for i in mf2_utils.walk(parsed.get('items', []))
if mf2_utils.is_type(i, 'h-card')), None)
if not card:
return {}
return {'name': mf2_utils.first(card, 'name'), 'url': mf2_utils.first(card, 'url'),
'photo': mf2_utils.first(card, 'photo')}
# --- Helpers ----------------------------------------------------------------
def _get_json(url: str, site_url: str):
"""GET JSON through the SSRF-safe fetcher; None if it isn't JSON"""
content, _, _ = fetch_source(url, site_url)
try:
return json.loads(content)
except ValueError:
return None
def _snippet(markup: str) -> str:
"""Sanitized HTML of at most ~SNIPPET_LIMIT characters, with tags closed"""
sanitized = sanitize_content(markup).strip()
if len(sanitized) > SNIPPET_LIMIT:
# Cutting can leave a tag or attribute open; sanitizing again closes it
sanitized = sanitize_content(sanitized[:SNIPPET_LIMIT - 3] + '...')
return sanitized
def _text_snippet(text: Optional[str]) -> Optional[str]:
"""Plain text as a sanitized HTML snippet"""
text = (text or '').strip()
if not text:
return None
if len(text) > SNIPPET_LIMIT:
text = text[:SNIPPET_LIMIT - 1].rstrip() + '…'
paragraphs = [p.strip() for p in re.split(r'\n\s*\n', text) if p.strip()]
return ''.join(f"<p>{html.escape(p).replace(chr(10), '<br>')}</p>" for p in paragraphs)
def _plain(markup: Optional[str]) -> str:
return ' '.join(html.unescape(re.sub(r'<[^>]+>', ' ', markup or '')).split())
def _repeats(title: str, snippet: Optional[str]) -> bool:
"""True if a title is just the start of the content (mf2 implied names, og:description)"""
a, b = ' '.join(title.split()).rstrip('.…'), _plain(snippet).rstrip('.…')
return bool(a and b) and (b.startswith(a) or a.startswith(b))
def _clip(text: Optional[str], limit: int) -> Optional[str]:
if not text:
return None
text = ' '.join(text.split())
return text if len(text) <= limit else text[:limit - 3] + '...'
def _iso(value: Optional[str]) -> Optional[str]:
"""An ISO 8601 date string, normalized; None if it doesn't parse"""
if not value:
return None
from dateutil import parser as dateutil_parser
try:
return dateutil_parser.isoparse(value).isoformat()
except (ValueError, OverflowError):
try:
return dateutil_parser.parse(value).isoformat()
except (ValueError, OverflowError):
return None
def _save_images(context: Dict[str, Any], site_url: str) -> None:
"""Local copies of the author's avatar and the post's image (best effort;
the webmention worker retries what's missing)"""
try:
from flask import current_app
from starpunk import avatars
data_path = current_app.config['DATA_PATH']
avatars.fetch(data_path, context.get('author_photo'), site_url)
avatars.fetch_preview(data_path, context.get('photo'), site_url)
avatars.fetch_video(data_path, context.get('video'), site_url)
except Exception as e:
logger.info(f"Could not save reply context images: {e}")
# --- Storage ----------------------------------------------------------------
def _store_context(db_path: Path, url: str, found: Dict[str, Any]) -> Dict[str, Any]:
"""Store a fetched reply context; returns it as stored"""
values = [found.get(field) for field in FIELDS]
conn = sqlite3.connect(db_path)
try:
conn.execute(f"""
INSERT OR REPLACE INTO reply_contexts
(url, {', '.join(FIELDS)}, source, fetch_status, fetch_error, fetched_at)
VALUES (?, {', '.join('?' for _ in FIELDS)}, ?, 'fetched', NULL, CURRENT_TIMESTAMP)
""", (url, *values, found['source']))
conn.commit()
except Exception as e:
logger.error(f"Failed to store reply context: {e}")
raise
finally:
conn.close()
return {'url': url, **dict(zip(FIELDS, values)), 'source': found['source'],
'fetch_status': 'fetched'}
def _mark_source(db_path: Path, url: str, source: str) -> None:
conn = sqlite3.connect(db_path)
try:
conn.execute("UPDATE reply_contexts SET source = ? WHERE url = ?", (source, url))
conn.commit()
finally:
conn.close()
def _store_minimal_context(
db_path: Path,
url: str,
error: str
) -> Dict[str, Any]:
"""
Store minimal context with error status.
Args:
db_path: Database path
url: Reply-to URL
error: Error message
Returns:
Minimal context dict with error
"""
context = {'url': url, 'fetch_status': 'error', 'fetch_error': error, 'source': 'error'}
try:
conn = sqlite3.connect(db_path)
conn.execute("""
INSERT OR REPLACE INTO reply_contexts
(url, fetch_status, fetch_error, source, created_at)
VALUES (?, 'error', ?, 'error', CURRENT_TIMESTAMP)
""", (url, error))
conn.commit()
conn.close()
except Exception as e:
# Return the in-memory dict even if storage fails
logger.error(f"Failed to store minimal context: {e}")
return context
def get_reply_context(db_path: Path, reply_url: str) -> Optional[Dict[str, Any]]:
"""
Get cached reply context from database.
Args:
db_path: Database path
reply_url: Reply-to URL
Returns:
Context dict if exists, None otherwise
"""
try:
conn = sqlite3.connect(db_path)
conn.row_factory = sqlite3.Row
row = conn.execute("SELECT * FROM reply_contexts WHERE url = ?", (reply_url,)).fetchone()
conn.close()
return dict(row) if row else None
except Exception as e:
logger.error(f"Failed to get reply context: {e}")
return None