""" Reply context fetching for in-reply-to URLs Per database-schema.md Story 2.4: - Fetch context from in_reply_to URL when note is reply - Parse microformats for author, content - Store in reply_contexts table - Graceful degradation on fetch failure Fediverse and Bluesky pages carry no h-entry, so from v2.19.0 the context comes from the first source that knows the post: 1. Bluesky post URLs: the public Bluesky AppView API 2. Pages with an h-entry: microformats (IndieWeb sites) 3. Fediverse posts: the origin's public Mastodon API, else this site's GoToSocial instance resolving the URL (as likes and boosts already do; it fetches with signed requests, so it works for GoToSocial and for Mastodon servers that refuse anonymous API calls) 4. Anything else: the page's OpenGraph tags, with the page's h-card as author The source used is stored in reply_contexts.source. Rows from before v2.19.0 have none; refresh_stale() fetches those again once. References: - Database Schema: /docs/design/v1.6.0/database-schema.md - Background Worker: /docs/design/v1.6.0/background-worker.md - Design: /docs/design/v2.19.0/2026-11-13-reply-context-sources.md """ import html import json import logging import re import sqlite3 from pathlib import Path from typing import Optional, Dict, Any from urllib.parse import quote, urljoin, urlparse import mf2py from starpunk import mf2_utils from starpunk.webmention_processor import ( fetch_source, extract_content, sanitize_content, ValidationError, _is_http_url, ) logger = logging.getLogger(__name__) SNIPPET_LIMIT = 500 BLUESKY_POST_RE = re.compile(r"^https://bsky\.app/profile/([^/]+)/post/([^/?#]+)") BLUESKY_API = "https://public.api.bsky.app/xrpc/app.bsky.feed.getPostThread" # Mastodon: https://host/@user/123456 (numeric ids; the public API may answer) MASTODON_STATUS_RE = re.compile(r"^/@[^/]+/(\d+)/?$") # Fediverse post paths: Mastodon, GoToSocial, ActivityPub ids, Misskey, # Pleroma/Akkoma FEDIVERSE_PATH_RE = re.compile( r"^/(?:@[^/]+/(?:statuses/)?[A-Za-z0-9]+|users/[^/]+/statuses/[A-Za-z0-9]+|" r"notes/[A-Za-z0-9]+|notice/[A-Za-z0-9]+|objects/[A-Za-z0-9-]+)/?$") # og:title of fediverse posts: "Display Name (@user@host)" FEDI_TITLE_RE = re.compile(r"^(.*?)\s*\((@[^)\s]+)\)\s*$") FIELDS = ('title', 'content_snippet', 'published_at', 'author_name', 'author_handle', 'author_url', 'author_photo', 'site_name', 'photo', 'photo_alt', 'video') def fetch_reply_context( db_path: Path, reply_url: str, site_url: str, refresh: bool = False ) -> Dict[str, Any]: """ Fetch and parse reply context from a URL. Per database-schema.md: - Fetch URL content with SSRF protection - Parse h-entry, a Mastodon status, a Bluesky post or OpenGraph tags - Store in reply_contexts table - Graceful degradation: errors don't prevent reply creation Args: db_path: Path to SQLite database reply_url: URL being replied to site_url: Site URL for User-Agent header refresh: Fetch again even if a context is stored Returns: Dict with fetch status and context data """ existing = get_reply_context(db_path, reply_url) # Contexts stored before v2.19.0 (no source) are fetched again if existing and existing.get('source') and not refresh: logger.debug(f"Reply context already cached for {reply_url}") return existing try: found = _find_context(reply_url, site_url) except Exception as e: if isinstance(e, ValidationError): logger.warning(f"Failed to fetch reply context for {reply_url}: {e}") error = str(e) else: logger.error(f"Error fetching reply context for {reply_url}: {e}", exc_info=True) error = f"Unexpected error: {e}" if existing and existing.get('fetch_status') == 'fetched': # Keep what an earlier fetch found; don't try again _mark_source(db_path, reply_url, existing.get('source') or 'legacy') return existing return _store_minimal_context(db_path, reply_url, error) context = _store_context(db_path, reply_url, found) _save_images(context, site_url) logger.info(f"Fetched reply context for {reply_url} ({context['source']})") return context def refresh_stale(db_path: Path, site_url: str, limit: int = 5) -> int: """ Fetch again the contexts stored before v2.19.0 that notes still use. The webmention worker calls this; each row is fetched once, since the new fetch records its source. Returns how many were fetched. """ conn = sqlite3.connect(db_path) try: urls = [row[0] for row in conn.execute( """ SELECT rc.url FROM reply_contexts rc WHERE rc.source IS NULL AND EXISTS ( SELECT 1 FROM notes n WHERE n.deleted_at IS NULL AND rc.url IN (n.in_reply_to, n.like_of, n.repost_of, n.bookmark_of)) ORDER BY rc.id DESC LIMIT ? """, (limit,))] finally: conn.close() for url in urls: fetch_reply_context(db_path, url, site_url, refresh=True) return len(urls) # --- Sources ---------------------------------------------------------------- def _find_context(url: str, site_url: str) -> Dict[str, Any]: """Context fields and 'source' for a URL; raises ValidationError if nothing could be fetched""" if BLUESKY_POST_RE.match(url): try: found = _from_bluesky(url, site_url) except ValidationError as e: logger.info(f"Bluesky API had no post for {url}: {e}") found = None if found: return found page_error = None content = content_type = final_url = None try: content, content_type, final_url = fetch_source(url, site_url) except ValidationError as e: page_error = e og: Dict[str, Optional[str]] = {} card: Dict[str, Optional[str]] = {} is_html = content is not None and 'html' in (content_type or '').lower() if is_html: parsed = mf2py.parse(doc=content, url=final_url) entry = mf2_utils.responding_entry(parsed, '') og = _opengraph(content, final_url) if entry: return _from_mf2(parsed, entry, final_url, og) card = _page_card(parsed, final_url) if _looks_fediverse(url, og): status = _mastodon_api(url, site_url) or _resolve_with_instance(url) if status: return _from_status(status, url) if page_error: raise page_error if not is_html: raise ValidationError("Not HTML content") return _from_opengraph(og, card) def _from_mf2(parsed: dict, entry: dict, page_url: str, og: dict) -> Dict[str, Any]: """An IndieWeb post's h-entry""" author = mf2_utils.entry_author(parsed, entry, page_url) snippet = extract_content(parsed, entry) title = mf2_utils.first(entry, 'name') if title and _repeats(title, snippet): title = None # implied name: the content again photo, photo_alt = mf2_utils.first(entry, 'photo'), None for value in entry.get('properties', {}).get('photo', [])[:1]: if isinstance(value, dict): photo_alt = value.get('alt') or None if not _is_http_url(photo): photo, photo_alt = og.get('image'), og.get('image_alt') return { 'source': 'mf2', 'title': _clip(title, 200), 'content_snippet': snippet, 'published_at': _iso(mf2_utils.first(entry, 'published')), 'author_name': author['name'], 'author_url': author['url'] if _is_http_url(author['url']) else None, 'author_photo': author['photo'] if _is_http_url(author['photo']) else None, 'site_name': og.get('site_name'), 'photo': photo if _is_http_url(photo) else None, 'photo_alt': photo_alt, } def _from_bluesky(url: str, site_url: str) -> Optional[Dict[str, Any]]: """A Bluesky post from the public AppView API (no login needed)""" actor, rkey = BLUESKY_POST_RE.match(url).groups() uri = quote(f"at://{actor}/app.bsky.feed.post/{rkey}", safe='') data = _get_json(f"{BLUESKY_API}?uri={uri}&depth=0&parentHeight=0", site_url) post = ((data or {}).get('thread') or {}).get('post') if isinstance(data, dict) else None if not post: return None author = post.get('author') or {} record = post.get('record') or {} handle = author.get('handle') photo, photo_alt = _bluesky_image(post.get('embed') or {}) return { 'source': 'bluesky', 'title': None, 'content_snippet': _text_snippet(record.get('text')), 'published_at': _iso(record.get('createdAt')), 'author_name': author.get('displayName') or handle, 'author_handle': f"@{handle}" if handle else None, 'author_url': f"https://bsky.app/profile/{handle}" if handle else None, 'author_photo': author.get('avatar') if _is_http_url(author.get('avatar')) else None, 'site_name': 'Bluesky', 'photo': photo, 'photo_alt': photo_alt, } def _bluesky_image(embed: dict) -> tuple: """First image of a post's embed view: (url, alt)""" kind = embed.get('$type', '') if kind.startswith('app.bsky.embed.recordWithMedia'): return _bluesky_image(embed.get('media') or {}) if kind.startswith('app.bsky.embed.images'): for image in embed.get('images') or []: if _is_http_url(image.get('thumb')): return image['thumb'], image.get('alt') or None if kind.startswith('app.bsky.embed.video') and _is_http_url(embed.get('thumbnail')): return embed['thumbnail'], embed.get('alt') or None if kind.startswith('app.bsky.embed.external'): external = embed.get('external') or {} if _is_http_url(external.get('thumb')): return external['thumb'], None return None, None def _looks_fediverse(url: str, og: dict) -> bool: return bool(FEDIVERSE_PATH_RE.match(urlparse(url).path) or og.get('username')) def _mastodon_api(url: str, site_url: str) -> Optional[dict]: """The status from its own server's public API (Mastodon post URLs only)""" parts = urlparse(url) m = MASTODON_STATUS_RE.match(parts.path) if not m: return None try: status = _get_json(f"{parts.scheme}://{parts.netloc}/api/v1/statuses/{m.group(1)}", site_url) except ValidationError: return None return status if isinstance(status, dict) and status.get('account') else None def _resolve_with_instance(url: str) -> Optional[dict]: """The status as this site's GoToSocial instance sees it, if one is configured""" try: from starpunk.syndication import get_targets target = get_targets().get('gotosocial') if target is None: return None return target.client.resolve_status(url) except Exception as e: logger.info(f"Instance could not resolve {url}: {e}") return None def _from_status(status: dict, url: str) -> Dict[str, Any]: """A Mastodon API status entity""" account = status.get('account') or {} acct = account.get('acct') or account.get('username') host = urlparse(account.get('url') or url).hostname if acct and '@' not in acct and host: acct = f"{acct}@{host}" spoiler = (status.get('spoiler_text') or '').strip() if spoiler: # Keep the content warning: show it, not what it covers snippet = _text_snippet(f"CW: {spoiler}") else: snippet = _snippet(status.get('content') or '') or None photo = photo_alt = video = None if not status.get('sensitive'): for media in status.get('media_attachments') or []: if media.get('type') in ('image', 'gifv', 'video') and _is_http_url(media.get('preview_url')): photo, photo_alt = media['preview_url'], media.get('description') or None # A "GIF" on Mastodon is a silent looping MP4; the preview is its first frame if media['type'] == 'gifv' and _is_http_url(media.get('url')): video = media['url'] break avatar = account.get('avatar_static') or account.get('avatar') return { 'source': 'fediverse', 'title': None, 'content_snippet': snippet, 'published_at': _iso(status.get('created_at')), 'author_name': account.get('display_name') or account.get('username'), 'author_handle': f"@{acct}" if acct else None, 'author_url': account.get('url') if _is_http_url(account.get('url')) else None, 'author_photo': avatar if _is_http_url(avatar) else None, 'site_name': urlparse(url).hostname, 'photo': photo, 'photo_alt': photo_alt, 'video': video, } def _from_opengraph(og: dict, card: dict) -> Dict[str, Any]: """Any other page: OpenGraph tags, the page's h-card as author""" title = og.get('title') author_name, handle = card.get('name') or og.get('author'), None m = FEDI_TITLE_RE.match(title or '') if m and (og.get('username') or not author_name): # A fediverse post's og:title is its author, not a title author_name, handle, title = m.group(1) or author_name, m.group(2), None elif og.get('username') and title: author_name, handle, title = title, f"@{og['username'].lstrip('@')}", None snippet = _text_snippet(og.get('description')) if title and snippet and _repeats(title, snippet): snippet = None return { 'source': 'opengraph', 'title': _clip(title, 200), 'content_snippet': snippet, 'published_at': _iso(og.get('published')), 'author_name': author_name, 'author_handle': handle, 'author_url': card.get('url') if _is_http_url(card.get('url')) else None, 'author_photo': card.get('photo') if _is_http_url(card.get('photo')) else None, 'site_name': og.get('site_name'), 'photo': og.get('image') if _is_http_url(og.get('image')) else None, 'photo_alt': og.get('image_alt'), } OG_FIELDS = { 'og:title': 'title', 'twitter:title': 'title', 'og:description': 'description', 'twitter:description': 'description', 'description': 'description', 'og:image': 'image', 'og:image:url': 'image', 'twitter:image': 'image', 'og:image:alt': 'image_alt', 'twitter:image:alt': 'image_alt', 'og:site_name': 'site_name', 'article:published_time': 'published', 'og:article:published_time': 'published', 'og:published_time': 'published', 'profile:username': 'username', 'og:profile:username': 'username', 'author': 'author', 'article:author': 'author', } def _opengraph(content: str, page_url: str) -> Dict[str, Optional[str]]: """OpenGraph (and plain) meta tags of a page; the first of each wins""" from lxml import html as lxml_html try: doc = lxml_html.document_fromstring(content) except Exception: return {} found: Dict[str, Optional[str]] = {} for meta in doc.iter('meta'): key = (meta.get('property') or meta.get('name') or '').strip().lower() value = (meta.get('content') or '').strip() field = OG_FIELDS.get(key) if not field or not value: continue if field == 'image_alt': # Only the alt text of the image we keep: the one right after it if 'image' in found and 'image_alt' not in found and not found.get('_alt_done'): found['image_alt'] = value continue if field == 'image': if 'image' in found: found['_alt_done'] = '1' continue value = urljoin(page_url, value) if field == 'author' and _is_http_url(value): continue # article:author is often a profile URL found.setdefault(field, value) found.pop('_alt_done', None) if 'title' not in found: titles = doc.findall('.//title') if titles and (titles[0].text_content() or '').strip(): found['title'] = titles[0].text_content().strip() return found def _page_card(parsed: dict, page_url: str) -> Dict[str, Optional[str]]: """The page's representative h-card, else its first h-card (GoToSocial pages)""" card = mf2_utils.representative_hcard(parsed, page_url) if card is None: card = next((i for i in mf2_utils.walk(parsed.get('items', [])) if mf2_utils.is_type(i, 'h-card')), None) if not card: return {} return {'name': mf2_utils.first(card, 'name'), 'url': mf2_utils.first(card, 'url'), 'photo': mf2_utils.first(card, 'photo')} # --- Helpers ---------------------------------------------------------------- def _get_json(url: str, site_url: str): """GET JSON through the SSRF-safe fetcher; None if it isn't JSON""" content, _, _ = fetch_source(url, site_url) try: return json.loads(content) except ValueError: return None def _snippet(markup: str) -> str: """Sanitized HTML of at most ~SNIPPET_LIMIT characters, with tags closed""" sanitized = sanitize_content(markup).strip() if len(sanitized) > SNIPPET_LIMIT: # Cutting can leave a tag or attribute open; sanitizing again closes it sanitized = sanitize_content(sanitized[:SNIPPET_LIMIT - 3] + '...') return sanitized def _text_snippet(text: Optional[str]) -> Optional[str]: """Plain text as a sanitized HTML snippet""" text = (text or '').strip() if not text: return None if len(text) > SNIPPET_LIMIT: text = text[:SNIPPET_LIMIT - 1].rstrip() + '…' paragraphs = [p.strip() for p in re.split(r'\n\s*\n', text) if p.strip()] return ''.join(f"
{html.escape(p).replace(chr(10), '
')}