From 62e257e0e0449199dbe43873ecb02bd20295a2f1 Mon Sep 17 00:00:00 2001 From: Phil Date: Fri, 13 Nov 2026 15:10:46 -0700 Subject: [PATCH] feat: reply contexts that show the post (v2.19.0) Replies to Mastodon posts showed only the server's domain: the page has no h-entry, so nothing was found and the host stood in for the author. - Context sources in order: Bluesky AppView API, microformats, the origin's Mastodon API or this site's GoToSocial instance, OpenGraph - Migration 024: author_handle, photo, photo_alt, source; rows without a source are fetched again once by the webmention worker - Note page: a card with avatar, name, handle, text, image and date; home/tag/likes: "In reply to " with a short quote - Avatars and images saved locally (CSP allows only this site's images); avatar upkeep now covers reply contexts so their avatars aren't removed Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 17 + .../2026-11-13-reply-context-sources.md | 108 +++ migrations/024_reply_context_details.sql | 9 + starpunk/__init__.py | 12 +- starpunk/avatars.py | 149 ++-- starpunk/reply_context.py | 664 ++++++++++++------ starpunk/routes/public.py | 21 +- starpunk/syndication.py | 11 +- starpunk/webmention_worker.py | 6 + static/css/style.css | 45 +- templates/note.html | 26 +- templates/partials/note_preview.html | 3 +- templates/partials/reply_context.html | 62 ++ tests/test_reply_context_sources.py | 270 +++++++ 14 files changed, 1094 insertions(+), 309 deletions(-) create mode 100644 docs/design/v2.19.0/2026-11-13-reply-context-sources.md create mode 100644 migrations/024_reply_context_details.sql create mode 100644 templates/partials/reply_context.html create mode 100644 tests/test_reply_context_sources.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0bd6274..088b997 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [2.19.0] - 2026-11-13 + +### Added + +- Replies show what they reply to. A note's page shows a card with the post's author (avatar, name, handle), its text, its first image and when it was posted, linking to the original. The home, tag and likes pages show "↩ In reply to " with a short quote of the post +- Context for fediverse and Bluesky posts. These pages have no microformats, so StarPunk asks the post's Mastodon server's API, then this site's GoToSocial instance (which also reaches GoToSocial and locked-down Mastodon servers), or the public Bluesky API. Other pages without microformats use their OpenGraph tags +- The avatars and images in reply contexts are saved on this site, like response avatars (the CSP doesn't allow images from other sites). Images are kept at most 640 px wide in `DATA_PATH/previews/` + +### Fixed + +- A reply to a Mastodon post showed only the server's name ("defcon.social"): the page had no h-entry, so the fetch found nothing and the domain stood in for the author. Contexts fetched before this release are fetched once more by the webmention worker +- The quoted text of a replied-to post was shown with its HTML tags escaped + +### Changed + +- Migration 024 adds `author_handle`, `photo`, `photo_alt` and `source` to `reply_contexts` + ## [2.18.3] - 2026-11-12 ### Added diff --git a/docs/design/v2.19.0/2026-11-13-reply-context-sources.md b/docs/design/v2.19.0/2026-11-13-reply-context-sources.md new file mode 100644 index 0000000..8ff0c50 --- /dev/null +++ b/docs/design/v2.19.0/2026-11-13-reply-context-sources.md @@ -0,0 +1,108 @@ +# v2.19.0: Reply contexts that show the post + +## Why + +A reply to `https://defcon.social/@deviantollam/117611186265622833` showed +"In reply to: defcon.social" and nothing else. Reply contexts were read only +from microformats. Mastodon's status page is a JavaScript app shell with no +h-entry, so `fetch_reply_context` stored `fetch_status='fetched'` with no +content, and `extract_author` fell back to the URL's host for the author name. +GoToSocial pages have an h-card but no h-entry; Bluesky pages have neither. +The home page didn't use the context at all: it printed the raw URL. + +The data was available elsewhere: + +- Mastodon's public API (`/api/v1/statuses/`) returned the author, text, + attachments (with alt text) and date without authentication. +- GoToSocial's API needs a token, but this site's own GoToSocial instance can + resolve any fediverse URL with signed requests + (`/api/v2/search?resolve=true`). The crossposter already does this for + native likes and boosts (`MastodonClient.find_status`). +- `public.api.bsky.app` answers `app.bsky.feed.getPostThread` for + `at:///app.bsky.feed.post/` without logging in. (`getPosts` + rejects handles in the URI; `getPostThread` resolves them.) +- Most pages carry OpenGraph tags: `og:title`, `og:description`, `og:image` + (with `og:image:alt`), `og:site_name`, a published time. Fediverse servers + put "Name (@user@host)" in `og:title` and the handle in + `profile:username`. + +## Sources, in order + +| # | When | Source | `source` | +|---|---|---|---| +| 1 | `bsky.app/profile//post/` | Bluesky AppView API | `bluesky` | +| 2 | The page has an h-entry | Microformats (as before), image from `u-photo` or `og:image` | `mf2` | +| 3 | Fediverse path (`/@user/`, `/@user/statuses/`, `/users/…/statuses/…`, `/notes/…`, `/notice/…`, `/objects/…`) or a `profile:username` tag | The origin's Mastodon API (numeric Mastodon ids only), else this site's GoToSocial instance | `fediverse` | +| 4 | Any other HTML page | OpenGraph tags; author from the page's h-card or `` | `opengraph` | + +All requests other than the instance lookup go through `fetch_source`, so +they get the SSRF checks, size limit and timeouts. The instance lookup uses +the crossposting client (`MastodonClient.resolve_status`, which +`find_status` now calls too) and is only tried when GoToSocial crossposting +is configured. + +Mastodon statuses keep their content warnings: when `spoiler_text` is set, +the snippet is "CW: …" and a `sensitive` post's image is left out. Content +HTML is sanitized with the webmention sanitizer (nh3; `p br strong em b i a +code pre blockquote`, `a[href]`) and cut to about 500 characters. Plain text +(Bluesky, OpenGraph) is escaped and split into paragraphs. + +An `mf2` or `opengraph` title that is only the start of the content +(an mf2 implied name, a description that repeats the title) is dropped. +The URL's host is no longer used as the author's name. + +## Storage (migration 024) + +`reply_contexts` gains `author_handle`, `photo`, `photo_alt` and `source`. +Error rows get `source='error'`. + +A row with no `source` was stored before v2.19.0. `fetch_reply_context` +fetches such rows again instead of returning them, and the webmention +worker calls `refresh_stale()` on every pass, which fetches up to five rows +that a note still uses. Each row is fetched once: success stores a source, +and a failure keeps the old data and marks it `legacy`. + +## Images + +The CSP only allows images from this site (`img-src 'self' data:`), so +remote avatars and images can't be shown directly. As with response avatars +(ADR-073 Decision 6), they are saved locally: + +- Author avatars go into the existing `DATA_PATH/avatars/` (96 px squares). +- Post images go into `DATA_PATH/previews/`, at most 640 px wide, as WebP, + served from `/previews/.webp`. Downloads are capped at 8 MB. + +The images are saved right after the fetch. `avatars.fill_missing` also +covers reply contexts that notes use, so the worker retries what failed, and +`remove_unused` keeps those files (before this release it would have deleted +any avatar that wasn't a webmention's). Until there is a copy, the card shows +the coloured initial and leaves the image out. + +## Display + +`templates/partials/reply_context.html`: + +- `reply_card(url, ctx)` on the note page: "↩ In reply to", then a card with + the avatar, name (linked to the author) and handle, the title if there is + one, the content (sanitized HTML), the image with its alt text, and a footer + linking to the original with its date and site. Without a fetched context + it is the bare link, as before. +- `reply_line(url, ctx)` on the home, tag and likes pages: "↩ In reply to + " (or the title, "by "), then up to 140 characters of the + post as a quote. Without a context it is the raw URL, as before. + +`attach_contexts` now gives replies their context as well as likes and +reposts, and the note page uses that when webmentions are disabled. + +Microformats are unchanged for consumers: the card is still the note's +`u-in-reply-to h-cite`. Its author h-card and `e-content` are nested inside +the h-cite, so they don't become the note's properties. Post Type Discovery +still reads the note as a reply (`tests/test_reply_context_sources.py`). + +## Tests + +`tests/test_reply_context_sources.py`, through the mocked network: the +Mastodon API, content warnings, resolving through the instance, the +OpenGraph fallback for fediverse and other pages, Bluesky, the once-only +refresh of old rows (including keeping data when the refresh fails), and the +note and home pages (content, local image and avatar, mf2 reply). diff --git a/migrations/024_reply_context_details.sql b/migrations/024_reply_context_details.sql new file mode 100644 index 0000000..2c8ae18 --- /dev/null +++ b/migrations/024_reply_context_details.sql @@ -0,0 +1,9 @@ +-- Migration 024: Richer reply contexts (v2.19.0) +-- Fediverse and Bluesky posts have no microformats, so their context now +-- comes from the Mastodon or Bluesky API, or the page's OpenGraph tags. +-- source records which one was used; rows from before have none, and the +-- webmention worker fetches those again once. +ALTER TABLE reply_contexts ADD COLUMN author_handle TEXT; +ALTER TABLE reply_contexts ADD COLUMN photo TEXT; +ALTER TABLE reply_contexts ADD COLUMN photo_alt TEXT; +ALTER TABLE reply_contexts ADD COLUMN source TEXT; diff --git a/starpunk/__init__.py b/starpunk/__init__.py index 0e1bc0a..0dab7f2 100644 --- a/starpunk/__init__.py +++ b/starpunk/__init__.py @@ -225,6 +225,14 @@ def create_app(config=None): name = cached_name(app.config.get('DATA_PATH', 'data'), url) return url_for('public.avatar', name=name) if name else None + # Template filter - local copy of a replied-to post's image, or None (v2.19.0) + @app.template_filter('local_preview') + def local_preview_filter(url): + from flask import url_for + from starpunk.avatars import cached_preview + name = cached_preview(app.config.get('DATA_PATH', 'data'), url) + return url_for('public.preview', name=name) if name else None + # Template filter - URL to color for facepile (v1.6.0) @app.template_filter('url_to_color') def url_to_color_filter(url): @@ -468,5 +476,5 @@ def create_app(config=None): # Package version (Semantic Versioning 2.0.0) # See docs/standards/versioning-strategy.md for details -__version__ = "2.18.3" -__version_info__ = (2, 18, 3) +__version__ = "2.19.0" +__version_info__ = (2, 19, 0) diff --git a/starpunk/avatars.py b/starpunk/avatars.py index fb8861e..3de5b1d 100644 --- a/starpunk/avatars.py +++ b/starpunk/avatars.py @@ -7,6 +7,9 @@ Each avatar is fetched once (SSRF-safe), shrunk to a 96 px WebP and saved as DATA_PATH/avatars/.webp. Pages show the saved copy, or the coloured initial until there is one. +The image of a post being replied to is kept the same way (v2.19.0): at most +PREVIEW_WIDTH wide, in DATA_PATH/previews/. + Paths and URLs are passed in, so the webmention worker can use it. Fetching needs an app context (the SSRF-safe fetcher reads DEV_MODE). """ @@ -24,6 +27,8 @@ logger = logging.getLogger(__name__) SIZE = 96 MAX_BYTES = 1024 * 1024 +PREVIEW_WIDTH = 640 +PREVIEW_MAX_BYTES = 8 * 1024 * 1024 # Bluesky avatar on its CDN: .../img//plain//[@jpeg] BSKY_CDN_RE = re.compile( r"^https://cdn\.bsky\.app/img/[a-z_]+/plain/(did:(?:plc|web):[A-Za-z0-9._:%-]+)/([a-z2-7]+)(?:@\w+)?$") @@ -39,12 +44,64 @@ def folder(data_path) -> Path: return Path(data_path) / "avatars" -def cached_name(data_path, url: Optional[str]) -> Optional[str]: - """File name of the saved copy of this avatar, if there is one""" +def preview_folder(data_path) -> Path: + return Path(data_path) / "previews" + + +def _cached(directory: Path, url: Optional[str]) -> Optional[str]: if not url or not url.startswith(("http://", "https://")): return None name = name_for(url) - return name if (folder(data_path) / name).is_file() else None + return name if (directory / name).is_file() else None + + +def cached_name(data_path, url: Optional[str]) -> Optional[str]: + """File name of the saved copy of this avatar, if there is one""" + return _cached(folder(data_path), url) + + +def cached_preview(data_path, url: Optional[str]) -> Optional[str]: + """File name of the saved copy of a replied-to post's image, if there is one""" + return _cached(preview_folder(data_path), url) + + +def _square(image): + from PIL import Image + side = min(image.size) + left, top = (image.width - side) // 2, (image.height - side) // 2 + image = image.crop((left, top, left + side, top + side)) + return image.resize((SIZE, SIZE), Image.LANCZOS) + + +def _narrow(image): + from PIL import Image + if image.width <= PREVIEW_WIDTH: + return image + height = max(1, round(image.height * PREVIEW_WIDTH / image.width)) + return image.resize((PREVIEW_WIDTH, height), Image.LANCZOS) + + +def _save(path: Path, url: Optional[str], site_url: str, shape, max_bytes: int, + what: str) -> bool: + if not url or not url.startswith(("http://", "https://")): + return False + if path.is_file(): + return True + from PIL import Image + + try: + with Image.open(io.BytesIO(_download(url, site_url, max_bytes))) as image: + image.load() + image = image.convert("RGBA" if "A" in image.getbands() else "RGB") + image = shape(image) + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(".tmp") + image.save(tmp, "WEBP", quality=80) + tmp.replace(path) + return True + except Exception as e: + logger.info(f"Could not save {what} {url}: {type(e).__name__}: {e}") + return False def fetch(data_path, url: Optional[str], site_url: str) -> bool: @@ -52,29 +109,18 @@ def fetch(data_path, url: Optional[str], site_url: str) -> bool: Save a copy of an avatar unless there is one. Returns True if one exists afterwards. Failures are logged and the page keeps showing the initial. """ - if not url or not url.startswith(("http://", "https://")): + if not url: return False - path = folder(data_path) / name_for(url) - if path.is_file(): - return True - from PIL import Image + return _save(folder(data_path) / name_for(url), url, site_url, _square, + MAX_BYTES, "avatar") - try: - with Image.open(io.BytesIO(_download(url, site_url))) as image: - image.load() - image = image.convert("RGBA" if "A" in image.getbands() else "RGB") - side = min(image.size) - left, top = (image.width - side) // 2, (image.height - side) // 2 - image = image.crop((left, top, left + side, top + side)) - image = image.resize((SIZE, SIZE), Image.LANCZOS) - path.parent.mkdir(parents=True, exist_ok=True) - tmp = path.with_suffix(".tmp") - image.save(tmp, "WEBP", quality=80) - tmp.replace(path) - return True - except Exception as e: - logger.info(f"Could not save avatar {url}: {type(e).__name__}: {e}") + +def fetch_preview(data_path, url: Optional[str], site_url: str) -> bool: + """Save a copy of a replied-to post's image unless there is one (v2.19.0)""" + if not url: return False + return _save(preview_folder(data_path) / name_for(url), url, site_url, _narrow, + PREVIEW_MAX_BYTES, "preview") def _own_instance_host() -> Optional[str]: @@ -85,11 +131,11 @@ def _own_instance_host() -> Optional[str]: return urlparse(current_app.config.get("GOTOSOCIAL_URL") or "").hostname -def _get(url: str, headers: dict) -> bytes: +def _get(url: str, headers: dict, max_bytes: int = MAX_BYTES) -> bytes: """GET a public URL through the SSRF-safe fetcher""" from starpunk.webmention_ssrf import safe_get - result = safe_get(url, headers=headers, max_bytes=MAX_BYTES, max_redirects=5, + result = safe_get(url, headers=headers, max_bytes=max_bytes, max_redirects=5, timeout=10.0, truncate=False) if result.status_code >= 400: raise ValueError(f"HTTP {result.status_code}") @@ -110,7 +156,7 @@ def _pds(did: str, headers: dict) -> str: raise ValueError(f"no PDS for {did}") -def _download(url: str, site_url: str) -> bytes: +def _download(url: str, site_url: str, max_bytes: int = MAX_BYTES) -> bytes: """ The avatar's bytes (v2.13.1): - from StarPunk's own GoToSocial directly, as the crossposting client @@ -136,13 +182,13 @@ def _download(url: str, site_url: str) -> bytes: # GoToSocial sends media it hasn't cached to the original # server: that one is fetched like any outside URL (v2.13.2) from urllib.parse import urljoin - return _get(urljoin(url, response.headers["location"]), headers) + return _get(urljoin(url, response.headers["location"]), headers, max_bytes) if response.status_code >= 400: raise ValueError(f"HTTP {response.status_code}") data = b"" for chunk in response.iter_bytes(): data += chunk - if len(data) > MAX_BYTES: + if len(data) > max_bytes: raise ValueError("too large") return data match = BSKY_CDN_RE.match(url) @@ -150,11 +196,26 @@ def _download(url: str, site_url: str) -> bytes: did, cid = match.groups() url = (f"{_pds(did, headers)}/xrpc/com.atproto.sync.getBlob" f"?did={quote(did, safe=':')}&cid={cid}") - return _get(url, headers) + return _get(url, headers, max_bytes) + + +def _context_images(conn) -> tuple[list, list]: + """Avatars and post images of reply contexts that notes use (v2.19.0)""" + rows = conn.execute( + """ + SELECT rc.author_photo, rc.photo FROM reply_contexts rc + WHERE rc.fetch_status = 'fetched' AND EXISTS ( + SELECT 1 FROM notes n WHERE n.deleted_at IS NULL AND rc.url IN + (n.in_reply_to, n.like_of, n.repost_of, n.bookmark_of)) + ORDER BY rc.id DESC + """ + ).fetchall() + return [r[0] for r in rows if r[0]], [r[1] for r in rows if r[1]] def fill_missing(db_path, data_path, site_url: str, limit: int = 20) -> int: - """Save avatars of shown responses that have no copy yet. Returns how many were tried.""" + """Save avatars of shown responses and reply contexts, and the images of + replied-to posts, that have no copy yet. Returns how many were tried.""" conn = sqlite3.connect(db_path) try: urls = [row[0] for row in conn.execute( @@ -165,35 +226,41 @@ def fill_missing(db_path, data_path, site_url: str, limit: int = 20) -> int: ORDER BY id DESC """ )] + context_avatars, context_photos = _context_images(conn) finally: conn.close() + wanted = [(url, cached_name, fetch) for url in dict.fromkeys(urls + context_avatars)] + wanted += [(url, cached_preview, fetch_preview) for url in dict.fromkeys(context_photos)] tried = 0 - for url in urls: + for url, cached, save in wanted: if tried >= limit: break - if cached_name(data_path, url) or not url.startswith(("http://", "https://")): + if cached(data_path, url) or not url.startswith(("http://", "https://")): continue - fetch(data_path, url, site_url) + save(data_path, url, site_url) tried += 1 return tried def remove_unused(db_path, data_path) -> int: - """Delete saved avatars no stored response uses. Returns how many were removed.""" - directory = folder(data_path) - if not directory.is_dir(): - return 0 + """Delete saved avatars and previews nothing uses. Returns how many were removed.""" conn = sqlite3.connect(db_path) try: used = {name_for(row[0]) for row in conn.execute( "SELECT DISTINCT author_photo FROM webmentions WHERE author_photo IS NOT NULL")} + context_avatars, context_photos = _context_images(conn) finally: conn.close() + used |= {name_for(url) for url in context_avatars} + used_previews = {name_for(url) for url in context_photos} removed = 0 - for path in directory.iterdir(): - if NAME_RE.match(path.name) and path.name not in used: - path.unlink(missing_ok=True) - removed += 1 + for directory, keep in ((folder(data_path), used), (preview_folder(data_path), used_previews)): + if not directory.is_dir(): + continue + for path in directory.iterdir(): + if NAME_RE.match(path.name) and path.name not in keep: + path.unlink(missing_ok=True) + removed += 1 return removed diff --git a/starpunk/reply_context.py b/starpunk/reply_context.py index affbab4..9dd6611 100644 --- a/starpunk/reply_context.py +++ b/starpunk/reply_context.py @@ -7,41 +7,79 @@ Per database-schema.md Story 2.4: - Store in reply_contexts table - Graceful degradation on fetch failure +Fediverse and Bluesky pages carry no h-entry, so from v2.19.0 the context +comes from the first source that knows the post: + +1. Bluesky post URLs: the public Bluesky AppView API +2. Pages with an h-entry: microformats (IndieWeb sites) +3. Fediverse posts: the origin's public Mastodon API, else this site's + GoToSocial instance resolving the URL (as likes and boosts already do; + it fetches with signed requests, so it works for GoToSocial and for + Mastodon servers that refuse anonymous API calls) +4. Anything else: the page's OpenGraph tags, with the page's h-card as author + +The source used is stored in reply_contexts.source. Rows from before v2.19.0 +have none; refresh_stale() fetches those again once. + References: - Database Schema: /docs/design/v1.6.0/database-schema.md - Background Worker: /docs/design/v1.6.0/background-worker.md + - Design: /docs/design/v2.19.0/2026-11-13-reply-context-sources.md """ +import html +import json import logging +import re import sqlite3 -from datetime import datetime from pathlib import Path from typing import Optional, Dict, Any +from urllib.parse import quote, urljoin, urlparse +import mf2py + +from starpunk import mf2_utils from starpunk.webmention_processor import ( fetch_source, - extract_author, extract_content, - ValidationError + sanitize_content, + ValidationError, + _is_http_url, ) -import mf2py logger = logging.getLogger(__name__) +SNIPPET_LIMIT = 500 + +BLUESKY_POST_RE = re.compile(r"^https://bsky\.app/profile/([^/]+)/post/([^/?#]+)") +BLUESKY_API = "https://public.api.bsky.app/xrpc/app.bsky.feed.getPostThread" +# Mastodon: https://host/@user/123456 (numeric ids; the public API may answer) +MASTODON_STATUS_RE = re.compile(r"^/@[^/]+/(\d+)/?$") +# Fediverse post paths: Mastodon, GoToSocial, ActivityPub ids, Misskey, +# Pleroma/Akkoma +FEDIVERSE_PATH_RE = re.compile( + r"^/(?:@[^/]+/(?:statuses/)?[A-Za-z0-9]+|users/[^/]+/statuses/[A-Za-z0-9]+|" + r"notes/[A-Za-z0-9]+|notice/[A-Za-z0-9]+|objects/[A-Za-z0-9-]+)/?$") +# og:title of fediverse posts: "Display Name (@user@host)" +FEDI_TITLE_RE = re.compile(r"^(.*?)\s*\((@[^)\s]+)\)\s*$") + +FIELDS = ('title', 'content_snippet', 'published_at', 'author_name', 'author_handle', + 'author_url', 'author_photo', 'site_name', 'photo', 'photo_alt') + def fetch_reply_context( db_path: Path, reply_url: str, - site_url: str + site_url: str, + refresh: bool = False ) -> Dict[str, Any]: """ Fetch and parse reply context from a URL. Per database-schema.md: - Fetch URL content with SSRF protection - - Parse h-entry for title, content, published date - - Parse h-card for author information + - Parse h-entry, a Mastodon status, a Bluesky post or OpenGraph tags - Store in reply_contexts table - Graceful degradation: errors don't prevent reply creation @@ -49,208 +87,434 @@ def fetch_reply_context( db_path: Path to SQLite database reply_url: URL being replied to site_url: Site URL for User-Agent header + refresh: Fetch again even if a context is stored Returns: Dict with fetch status and context data """ - # Check if context already exists + existing = get_reply_context(db_path, reply_url) + # Contexts stored before v2.19.0 (no source) are fetched again + if existing and existing.get('source') and not refresh: + logger.debug(f"Reply context already cached for {reply_url}") + return existing + try: - conn = sqlite3.connect(db_path) - conn.row_factory = sqlite3.Row - cursor = conn.cursor() + found = _find_context(reply_url, site_url) + except Exception as e: + if isinstance(e, ValidationError): + logger.warning(f"Failed to fetch reply context for {reply_url}: {e}") + error = str(e) + else: + logger.error(f"Error fetching reply context for {reply_url}: {e}", exc_info=True) + error = f"Unexpected error: {e}" + if existing and existing.get('fetch_status') == 'fetched': + # Keep what an earlier fetch found; don't try again + _mark_source(db_path, reply_url, existing.get('source') or 'legacy') + return existing + return _store_minimal_context(db_path, reply_url, error) - cursor.execute(""" - SELECT * FROM reply_contexts WHERE url = ? - """, (reply_url,)) + context = _store_context(db_path, reply_url, found) + _save_images(context, site_url) + logger.info(f"Fetched reply context for {reply_url} ({context['source']})") + return context - existing = cursor.fetchone() + +def refresh_stale(db_path: Path, site_url: str, limit: int = 5) -> int: + """ + Fetch again the contexts stored before v2.19.0 that notes still use. + The webmention worker calls this; each row is fetched once, since the + new fetch records its source. Returns how many were fetched. + """ + conn = sqlite3.connect(db_path) + try: + urls = [row[0] for row in conn.execute( + """ + SELECT rc.url FROM reply_contexts rc + WHERE rc.source IS NULL AND EXISTS ( + SELECT 1 FROM notes n WHERE n.deleted_at IS NULL AND rc.url IN + (n.in_reply_to, n.like_of, n.repost_of, n.bookmark_of)) + ORDER BY rc.id DESC LIMIT ? + """, (limit,))] + finally: conn.close() + for url in urls: + fetch_reply_context(db_path, url, site_url, refresh=True) + return len(urls) - if existing: - logger.debug(f"Reply context already cached for {reply_url}") - return dict(existing) - except Exception as e: - logger.warning(f"Failed to check existing context: {e}") +# --- Sources ---------------------------------------------------------------- - # Fetch and parse context +def _find_context(url: str, site_url: str) -> Dict[str, Any]: + """Context fields and 'source' for a URL; raises ValidationError if nothing could be fetched""" + if BLUESKY_POST_RE.match(url): + try: + found = _from_bluesky(url, site_url) + except ValidationError as e: + logger.info(f"Bluesky API had no post for {url}: {e}") + found = None + if found: + return found + + page_error = None + content = content_type = final_url = None try: - # Fetch URL content - # fetch_source also returns the URL after redirects (v2.7.0); relative - # links in the page resolve against it (v2.10.2) - content, content_type, final_url = fetch_source(reply_url, site_url) - - # Parse microformats (only for HTML) - if 'html' not in content_type.lower(): - # Not HTML - store minimal context - return _store_minimal_context(db_path, reply_url, "Not HTML content") - - parsed_mf2 = mf2py.parse(doc=content, url=final_url) - - # Extract h-entry data - entry_data = _extract_entry_data(parsed_mf2) - - # Extract author data - author_data = extract_author(parsed_mf2, reply_url) - - # Store in database - context = _store_context( - db_path, - reply_url, - entry_data.get('title'), - entry_data.get('content_snippet'), - entry_data.get('published_at'), - author_data.get('author_name'), - author_data.get('author_url'), - author_data.get('author_photo'), - entry_data.get('site_name') - ) - - logger.info(f"Fetched reply context for {reply_url}") - return context - + content, content_type, final_url = fetch_source(url, site_url) except ValidationError as e: - # Fetch/validation error - store error status - logger.warning(f"Failed to fetch reply context for {reply_url}: {e}") - return _store_minimal_context(db_path, reply_url, str(e)) + page_error = e - except Exception as e: - # Unexpected error - store error status - logger.error(f"Error fetching reply context for {reply_url}: {e}", exc_info=True) - return _store_minimal_context(db_path, reply_url, f"Unexpected error: {e}") + og: Dict[str, Optional[str]] = {} + card: Dict[str, Optional[str]] = {} + is_html = content is not None and 'html' in (content_type or '').lower() + if is_html: + parsed = mf2py.parse(doc=content, url=final_url) + entry = mf2_utils.responding_entry(parsed, '') + og = _opengraph(content, final_url) + if entry: + return _from_mf2(parsed, entry, final_url, og) + card = _page_card(parsed, final_url) + + if _looks_fediverse(url, og): + status = _mastodon_api(url, site_url) or _resolve_with_instance(url) + if status: + return _from_status(status, url) + + if page_error: + raise page_error + if not is_html: + raise ValidationError("Not HTML content") + return _from_opengraph(og, card) -def _extract_entry_data(parsed_mf2: Dict) -> Dict[str, Any]: - """ - Extract entry data from parsed microformats. - - Args: - parsed_mf2: Parsed microformats data from mf2py - - Returns: - Dict with title, content_snippet, published_at, site_name - """ - title = None - content_snippet = None - published_at = None - site_name = None - - # Find h-entry - for item in parsed_mf2.get('items', []): - if 'h-entry' not in item.get('type', []): - continue - - props = item.get('properties', {}) - - # Extract title (p-name) - name = props.get('name', []) - if name: - title = name[0] - if isinstance(title, dict): - title = title.get('value', '') - # Truncate if too long - if title and len(title) > 200: - title = title[:197] + '...' - - # Extract content (reuse processor function) - content_snippet = extract_content(parsed_mf2) - - # Extract published date - published = props.get('published', []) - if published: - pub_str = published[0] - if isinstance(pub_str, dict): - pub_str = pub_str.get('value', '') - # Try to parse ISO 8601 datetime - try: - from dateutil import parser as dateutil_parser - published_at = dateutil_parser.parse(pub_str) - except Exception: - # Parsing failed, leave as None - pass - - break # Use first h-entry - - # Try to extract site name from h-card - for item in parsed_mf2.get('items', []): - if 'h-card' in item.get('type', []): - props = item.get('properties', {}) - name = props.get('name', []) - if name: - site_name = name[0] - if isinstance(site_name, dict): - site_name = site_name.get('value', '') - break - +def _from_mf2(parsed: dict, entry: dict, page_url: str, og: dict) -> Dict[str, Any]: + """An IndieWeb post's h-entry""" + author = mf2_utils.entry_author(parsed, entry, page_url) + snippet = extract_content(parsed, entry) + title = mf2_utils.first(entry, 'name') + if title and _repeats(title, snippet): + title = None # implied name: the content again + photo, photo_alt = mf2_utils.first(entry, 'photo'), None + for value in entry.get('properties', {}).get('photo', [])[:1]: + if isinstance(value, dict): + photo_alt = value.get('alt') or None + if not _is_http_url(photo): + photo, photo_alt = og.get('image'), og.get('image_alt') return { - 'title': title, - 'content_snippet': content_snippet, - 'published_at': published_at, - 'site_name': site_name + 'source': 'mf2', + 'title': _clip(title, 200), + 'content_snippet': snippet, + 'published_at': _iso(mf2_utils.first(entry, 'published')), + 'author_name': author['name'], + 'author_url': author['url'] if _is_http_url(author['url']) else None, + 'author_photo': author['photo'] if _is_http_url(author['photo']) else None, + 'site_name': og.get('site_name'), + 'photo': photo if _is_http_url(photo) else None, + 'photo_alt': photo_alt, } -def _store_context( - db_path: Path, - url: str, - title: Optional[str], - content_snippet: Optional[str], - published_at: Optional[datetime], - author_name: Optional[str], - author_url: Optional[str], - author_photo: Optional[str], - site_name: Optional[str] -) -> Dict[str, Any]: - """ - Store reply context in database. +def _from_bluesky(url: str, site_url: str) -> Optional[Dict[str, Any]]: + """A Bluesky post from the public AppView API (no login needed)""" + actor, rkey = BLUESKY_POST_RE.match(url).groups() + uri = quote(f"at://{actor}/app.bsky.feed.post/{rkey}", safe='') + data = _get_json(f"{BLUESKY_API}?uri={uri}&depth=0&parentHeight=0", site_url) + post = ((data or {}).get('thread') or {}).get('post') if isinstance(data, dict) else None + if not post: + return None + author = post.get('author') or {} + record = post.get('record') or {} + handle = author.get('handle') + photo, photo_alt = _bluesky_image(post.get('embed') or {}) + return { + 'source': 'bluesky', + 'title': None, + 'content_snippet': _text_snippet(record.get('text')), + 'published_at': _iso(record.get('createdAt')), + 'author_name': author.get('displayName') or handle, + 'author_handle': f"@{handle}" if handle else None, + 'author_url': f"https://bsky.app/profile/{handle}" if handle else None, + 'author_photo': author.get('avatar') if _is_http_url(author.get('avatar')) else None, + 'site_name': 'Bluesky', + 'photo': photo, + 'photo_alt': photo_alt, + } - Args: - db_path: Database path - url: Reply-to URL - title: Entry title - content_snippet: Content snippet - published_at: Published datetime - author_name: Author name - author_url: Author URL - author_photo: Author photo URL - site_name: Site name - Returns: - Stored context dict - """ +def _bluesky_image(embed: dict) -> tuple: + """First image of a post's embed view: (url, alt)""" + kind = embed.get('$type', '') + if kind.startswith('app.bsky.embed.recordWithMedia'): + return _bluesky_image(embed.get('media') or {}) + if kind.startswith('app.bsky.embed.images'): + for image in embed.get('images') or []: + if _is_http_url(image.get('thumb')): + return image['thumb'], image.get('alt') or None + if kind.startswith('app.bsky.embed.video') and _is_http_url(embed.get('thumbnail')): + return embed['thumbnail'], embed.get('alt') or None + if kind.startswith('app.bsky.embed.external'): + external = embed.get('external') or {} + if _is_http_url(external.get('thumb')): + return external['thumb'], None + return None, None + + +def _looks_fediverse(url: str, og: dict) -> bool: + return bool(FEDIVERSE_PATH_RE.match(urlparse(url).path) or og.get('username')) + + +def _mastodon_api(url: str, site_url: str) -> Optional[dict]: + """The status from its own server's public API (Mastodon post URLs only)""" + parts = urlparse(url) + m = MASTODON_STATUS_RE.match(parts.path) + if not m: + return None try: - conn = sqlite3.connect(db_path) - cursor = conn.cursor() + status = _get_json(f"{parts.scheme}://{parts.netloc}/api/v1/statuses/{m.group(1)}", site_url) + except ValidationError: + return None + return status if isinstance(status, dict) and status.get('account') else None - cursor.execute(""" + +def _resolve_with_instance(url: str) -> Optional[dict]: + """The status as this site's GoToSocial instance sees it, if one is configured""" + try: + from starpunk.syndication import get_targets + target = get_targets().get('gotosocial') + if target is None: + return None + return target.client.resolve_status(url) + except Exception as e: + logger.info(f"Instance could not resolve {url}: {e}") + return None + + +def _from_status(status: dict, url: str) -> Dict[str, Any]: + """A Mastodon API status entity""" + account = status.get('account') or {} + acct = account.get('acct') or account.get('username') + host = urlparse(account.get('url') or url).hostname + if acct and '@' not in acct and host: + acct = f"{acct}@{host}" + spoiler = (status.get('spoiler_text') or '').strip() + if spoiler: + # Keep the content warning: show it, not what it covers + snippet = _text_snippet(f"CW: {spoiler}") + else: + snippet = _snippet(status.get('content') or '') or None + photo = photo_alt = None + if not status.get('sensitive'): + for media in status.get('media_attachments') or []: + if media.get('type') in ('image', 'gifv', 'video') and _is_http_url(media.get('preview_url')): + photo, photo_alt = media['preview_url'], media.get('description') or None + break + avatar = account.get('avatar_static') or account.get('avatar') + return { + 'source': 'fediverse', + 'title': None, + 'content_snippet': snippet, + 'published_at': _iso(status.get('created_at')), + 'author_name': account.get('display_name') or account.get('username'), + 'author_handle': f"@{acct}" if acct else None, + 'author_url': account.get('url') if _is_http_url(account.get('url')) else None, + 'author_photo': avatar if _is_http_url(avatar) else None, + 'site_name': urlparse(url).hostname, + 'photo': photo, + 'photo_alt': photo_alt, + } + + +def _from_opengraph(og: dict, card: dict) -> Dict[str, Any]: + """Any other page: OpenGraph tags, the page's h-card as author""" + title = og.get('title') + author_name, handle = card.get('name') or og.get('author'), None + m = FEDI_TITLE_RE.match(title or '') + if m and (og.get('username') or not author_name): + # A fediverse post's og:title is its author, not a title + author_name, handle, title = m.group(1) or author_name, m.group(2), None + elif og.get('username') and title: + author_name, handle, title = title, f"@{og['username'].lstrip('@')}", None + snippet = _text_snippet(og.get('description')) + if title and snippet and _repeats(title, snippet): + snippet = None + return { + 'source': 'opengraph', + 'title': _clip(title, 200), + 'content_snippet': snippet, + 'published_at': _iso(og.get('published')), + 'author_name': author_name, + 'author_handle': handle, + 'author_url': card.get('url') if _is_http_url(card.get('url')) else None, + 'author_photo': card.get('photo') if _is_http_url(card.get('photo')) else None, + 'site_name': og.get('site_name'), + 'photo': og.get('image') if _is_http_url(og.get('image')) else None, + 'photo_alt': og.get('image_alt'), + } + + +OG_FIELDS = { + 'og:title': 'title', 'twitter:title': 'title', + 'og:description': 'description', 'twitter:description': 'description', + 'description': 'description', + 'og:image': 'image', 'og:image:url': 'image', 'twitter:image': 'image', + 'og:image:alt': 'image_alt', 'twitter:image:alt': 'image_alt', + 'og:site_name': 'site_name', + 'article:published_time': 'published', 'og:article:published_time': 'published', + 'og:published_time': 'published', + 'profile:username': 'username', 'og:profile:username': 'username', + 'author': 'author', 'article:author': 'author', +} + + +def _opengraph(content: str, page_url: str) -> Dict[str, Optional[str]]: + """OpenGraph (and plain) meta tags of a page; the first of each wins""" + from lxml import html as lxml_html + try: + doc = lxml_html.document_fromstring(content) + except Exception: + return {} + found: Dict[str, Optional[str]] = {} + for meta in doc.iter('meta'): + key = (meta.get('property') or meta.get('name') or '').strip().lower() + value = (meta.get('content') or '').strip() + field = OG_FIELDS.get(key) + if not field or not value: + continue + if field == 'image_alt': + # Only the alt text of the image we keep: the one right after it + if 'image' in found and 'image_alt' not in found and not found.get('_alt_done'): + found['image_alt'] = value + continue + if field == 'image': + if 'image' in found: + found['_alt_done'] = '1' + continue + value = urljoin(page_url, value) + if field == 'author' and _is_http_url(value): + continue # article:author is often a profile URL + found.setdefault(field, value) + found.pop('_alt_done', None) + if 'title' not in found: + titles = doc.findall('.//title') + if titles and (titles[0].text_content() or '').strip(): + found['title'] = titles[0].text_content().strip() + return found + + +def _page_card(parsed: dict, page_url: str) -> Dict[str, Optional[str]]: + """The page's representative h-card, else its first h-card (GoToSocial pages)""" + card = mf2_utils.representative_hcard(parsed, page_url) + if card is None: + card = next((i for i in mf2_utils.walk(parsed.get('items', [])) + if mf2_utils.is_type(i, 'h-card')), None) + if not card: + return {} + return {'name': mf2_utils.first(card, 'name'), 'url': mf2_utils.first(card, 'url'), + 'photo': mf2_utils.first(card, 'photo')} + + +# --- Helpers ---------------------------------------------------------------- + +def _get_json(url: str, site_url: str): + """GET JSON through the SSRF-safe fetcher; None if it isn't JSON""" + content, _, _ = fetch_source(url, site_url) + try: + return json.loads(content) + except ValueError: + return None + + +def _snippet(markup: str) -> str: + """Sanitized HTML of at most ~SNIPPET_LIMIT characters, with tags closed""" + sanitized = sanitize_content(markup).strip() + if len(sanitized) > SNIPPET_LIMIT: + # Cutting can leave a tag or attribute open; sanitizing again closes it + sanitized = sanitize_content(sanitized[:SNIPPET_LIMIT - 3] + '...') + return sanitized + + +def _text_snippet(text: Optional[str]) -> Optional[str]: + """Plain text as a sanitized HTML snippet""" + text = (text or '').strip() + if not text: + return None + if len(text) > SNIPPET_LIMIT: + text = text[:SNIPPET_LIMIT - 1].rstrip() + '…' + paragraphs = [p.strip() for p in re.split(r'\n\s*\n', text) if p.strip()] + return ''.join(f"

{html.escape(p).replace(chr(10), '
')}

" for p in paragraphs) + + +def _plain(markup: Optional[str]) -> str: + return ' '.join(html.unescape(re.sub(r'<[^>]+>', ' ', markup or '')).split()) + + +def _repeats(title: str, snippet: Optional[str]) -> bool: + """True if a title is just the start of the content (mf2 implied names, og:description)""" + a, b = ' '.join(title.split()).rstrip('.…'), _plain(snippet).rstrip('.…') + return bool(a and b) and (b.startswith(a) or a.startswith(b)) + + +def _clip(text: Optional[str], limit: int) -> Optional[str]: + if not text: + return None + text = ' '.join(text.split()) + return text if len(text) <= limit else text[:limit - 3] + '...' + + +def _iso(value: Optional[str]) -> Optional[str]: + """An ISO 8601 date string, normalized; None if it doesn't parse""" + if not value: + return None + from dateutil import parser as dateutil_parser + try: + return dateutil_parser.isoparse(value).isoformat() + except (ValueError, OverflowError): + try: + return dateutil_parser.parse(value).isoformat() + except (ValueError, OverflowError): + return None + + +def _save_images(context: Dict[str, Any], site_url: str) -> None: + """Local copies of the author's avatar and the post's image (best effort; + the webmention worker retries what's missing)""" + try: + from flask import current_app + from starpunk import avatars + data_path = current_app.config['DATA_PATH'] + avatars.fetch(data_path, context.get('author_photo'), site_url) + avatars.fetch_preview(data_path, context.get('photo'), site_url) + except Exception as e: + logger.info(f"Could not save reply context images: {e}") + + +# --- Storage ---------------------------------------------------------------- + +def _store_context(db_path: Path, url: str, found: Dict[str, Any]) -> Dict[str, Any]: + """Store a fetched reply context; returns it as stored""" + values = [found.get(field) for field in FIELDS] + conn = sqlite3.connect(db_path) + try: + conn.execute(f""" INSERT OR REPLACE INTO reply_contexts - (url, title, content_snippet, published_at, - author_name, author_url, author_photo, site_name, - fetch_status, fetched_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'fetched', CURRENT_TIMESTAMP) - """, ( - url, title, content_snippet, published_at, - author_name, author_url, author_photo, site_name - )) - + (url, {', '.join(FIELDS)}, source, fetch_status, fetch_error, fetched_at) + VALUES (?, {', '.join('?' for _ in FIELDS)}, ?, 'fetched', NULL, CURRENT_TIMESTAMP) + """, (url, *values, found['source'])) conn.commit() - conn.close() - - return { - 'url': url, - 'title': title, - 'content_snippet': content_snippet, - 'published_at': published_at, - 'author_name': author_name, - 'author_url': author_url, - 'author_photo': author_photo, - 'site_name': site_name, - 'fetch_status': 'fetched' - } - except Exception as e: logger.error(f"Failed to store reply context: {e}") raise + finally: + conn.close() + return {'url': url, **dict(zip(FIELDS, values)), 'source': found['source'], + 'fetch_status': 'fetched'} + + +def _mark_source(db_path: Path, url: str, source: str) -> None: + conn = sqlite3.connect(db_path) + try: + conn.execute("UPDATE reply_contexts SET source = ? WHERE url = ?", (source, url)) + conn.commit() + finally: + conn.close() def _store_minimal_context( @@ -269,33 +533,20 @@ def _store_minimal_context( Returns: Minimal context dict with error """ + context = {'url': url, 'fetch_status': 'error', 'fetch_error': error, 'source': 'error'} try: conn = sqlite3.connect(db_path) - cursor = conn.cursor() - - cursor.execute(""" + conn.execute(""" INSERT OR REPLACE INTO reply_contexts - (url, fetch_status, fetch_error, created_at) - VALUES (?, 'error', ?, CURRENT_TIMESTAMP) + (url, fetch_status, fetch_error, source, created_at) + VALUES (?, 'error', ?, 'error', CURRENT_TIMESTAMP) """, (url, error)) - conn.commit() conn.close() - - return { - 'url': url, - 'fetch_status': 'error', - 'fetch_error': error - } - except Exception as e: + # Return the in-memory dict even if storage fails logger.error(f"Failed to store minimal context: {e}") - # Return in-memory dict even if storage fails - return { - 'url': url, - 'fetch_status': 'error', - 'fetch_error': error - } + return context def get_reply_context(db_path: Path, reply_url: str) -> Optional[Dict[str, Any]]: @@ -312,20 +563,9 @@ def get_reply_context(db_path: Path, reply_url: str) -> Optional[Dict[str, Any]] try: conn = sqlite3.connect(db_path) conn.row_factory = sqlite3.Row - cursor = conn.cursor() - - cursor.execute(""" - SELECT * FROM reply_contexts WHERE url = ? - """, (reply_url,)) - - row = cursor.fetchone() + row = conn.execute("SELECT * FROM reply_contexts WHERE url = ?", (reply_url,)).fetchone() conn.close() - - if row: - return dict(row) - - return None - + return dict(row) if row else None except Exception as e: logger.error(f"Failed to get reply context: {e}") return None diff --git a/starpunk/routes/public.py b/starpunk/routes/public.py index ea7d79f..8efb5b0 100644 --- a/starpunk/routes/public.py +++ b/starpunk/routes/public.py @@ -264,6 +264,20 @@ def avatar(name: str): return response +@bp.route("/previews/") +def preview(name: str): + """A saved copy of a replied-to post's image (v2.19.0)""" + from pathlib import Path + from starpunk.avatars import NAME_RE, preview_folder + + if not NAME_RE.match(name): + abort(404) + response = send_from_directory( + preview_folder(Path(current_app.config.get('DATA_PATH', 'data'))), name) + response.headers['Cache-Control'] = 'public, max-age=604800' + return response + + @bp.route("/") def index(): """ @@ -310,14 +324,15 @@ def attach_response_counts(notes) -> None: def attach_contexts(notes) -> None: - """Give likes and reposts the fetched context of the post they point at""" + """Give replies, likes and reposts the fetched context of the post they point at""" from starpunk.webmention_queries import get_reply_context for note in notes: context = None - if note.reacts_to: + url = note.in_reply_to or note.reacts_to + if url: try: - context = get_reply_context(note.reacts_to) + context = get_reply_context(url) except Exception: context = None object.__setattr__(note, 'context', context) diff --git a/starpunk/syndication.py b/starpunk/syndication.py index f15b952..92646b8 100644 --- a/starpunk/syndication.py +++ b/starpunk/syndication.py @@ -127,8 +127,8 @@ class MastodonClient: can_edit = True NATIVE = ("favourite", "reblog") # remote_id prefixes for likes/boosts (v2.5.0) - def find_status(self, url: str) -> Optional[str]: - """Local id of a fediverse post, fetching it if needed; None if the URL isn't one""" + def resolve_status(self, url: str) -> Optional[dict]: + """A fediverse post as a Mastodon status, fetching it if needed; None if the URL isn't one""" try: response = self._call("GET", "/api/v2/search", params={ "q": url, "resolve": "true", "type": "statuses", "limit": 1, @@ -136,7 +136,12 @@ class MastodonClient: except RemoteGone: return None statuses = response.json().get("statuses") or [] - return statuses[0]["id"] if statuses else None + return statuses[0] if statuses else None + + def find_status(self, url: str) -> Optional[str]: + """Local id of a fediverse post, fetching it if needed; None if the URL isn't one""" + status = self.resolve_status(url) + return status["id"] if status else None def _prepare(self, post: "Post") -> tuple: if post.kind == "repost": diff --git a/starpunk/webmention_worker.py b/starpunk/webmention_worker.py index 552ec82..32dc79a 100644 --- a/starpunk/webmention_worker.py +++ b/starpunk/webmention_worker.py @@ -117,6 +117,12 @@ class WebmentionWorker(threading.Thread): # Phase 3: Process outgoing webmentions processed += self._process_pending_sends() + # Reply contexts stored before v2.19.0 are fetched once more + try: + from starpunk.reply_context import refresh_stale + refresh_stale(self.db_path, self.site_url) + except Exception as e: + logger.error(f"Reply context refresh failed: {e}", exc_info=True) # Local copies of response avatars (v2.13.0) self._avatar_upkeep.run(self.db_path, self.app.config['DATA_PATH'], self.site_url) diff --git a/static/css/style.css b/static/css/style.css index 3d0e572..130a424 100644 --- a/static/css/style.css +++ b/static/css/style.css @@ -278,30 +278,27 @@ small { display: block; margin-top: var(--spacing-xs); color: var(--color-text-l /* Webmention Styles (v1.6.0 Phase 5) */ -/* Reply Context */ -.reply-context { - margin-bottom: var(--spacing-md); - padding: var(--spacing-sm); - border-left: 3px solid var(--color-link); - background: var(--color-bg-alt); -} - -.reply-context .reply-label { - font-size: 0.875rem; - color: var(--color-text-light); - display: block; - margin-bottom: var(--spacing-xs); -} - -.reply-context blockquote { - margin: 0; - padding: 0; -} - -.reply-context .p-content { - font-style: italic; - margin-bottom: var(--spacing-xs); -} +/* Reply Context (card v2.19.0) */ +.reply-context { margin-bottom: var(--spacing-md); } +.reply-context .reply-label { display: block; margin-bottom: var(--spacing-xs); font-size: 0.875rem; font-weight: 600; color: var(--color-text-light); } +.reply-card { margin: 0; padding: var(--spacing-sm) var(--spacing-md); border-left: 3px solid var(--color-link); background: var(--color-bg-alt); border-radius: 0 var(--border-radius) var(--border-radius) 0; overflow-wrap: anywhere; } +.reply-author { margin-bottom: var(--spacing-xs); } +.reply-author .h-card { display: flex; align-items: center; gap: var(--spacing-sm); } +.reply-author img, .reply-author .face-initial { width: 40px; height: 40px; border-radius: 50%; flex: none; } +.reply-author .face-initial { display: inline-flex; align-items: center; justify-content: center; color: #fff; font-weight: 700; } +.reply-who { display: flex; flex-wrap: wrap; column-gap: 0.4rem; line-height: 1.3; min-width: 0; } +.reply-who .p-name { font-weight: 600; color: var(--color-text); } +.reply-who a.p-name:hover { color: var(--color-link); } +.reply-handle { font-size: 0.8125rem; color: var(--color-text-light); } +.reply-title { margin: 0 0 var(--spacing-xs); font-weight: 600; } +.reply-content { font-size: 0.9375rem; } +.reply-content p { margin-bottom: var(--spacing-xs); } +.reply-content > :last-child { margin-bottom: 0; } +.reply-photo { display: block; max-width: 100%; max-height: 20rem; width: auto; height: auto; margin-top: var(--spacing-sm); border-radius: var(--border-radius); } +.reply-source { margin-top: var(--spacing-xs); font-size: 0.8125rem; } +.reply-source a { color: var(--color-text-light); } +.reply-source a:hover { color: var(--color-link); } +.reply-quote { display: block; margin-top: 0.15rem; padding-left: var(--spacing-sm); border-left: 2px solid var(--color-border); color: var(--color-text-light); font-style: italic; quotes: none; } /* Webmentions Section */ .webmentions { diff --git a/templates/note.html b/templates/note.html index 651d0b2..4351db1 100644 --- a/templates/note.html +++ b/templates/note.html @@ -1,6 +1,7 @@ {% extends "base.html" %} {% from "partials/media.html" import display_media %} {% from "partials/reaction.html" import reaction %} +{% from "partials/reply_context.html" import reply_card %} {% import "partials/responses.html" as responses with context %} {% block title %}{% if note.title %}{{ note.title }}{% elif note.excerpt.strip() %}{{ note.excerpt|truncate(60) }}{% elif note.repost_of %}Repost{% elif note.like_of %}Like{% elif note.bookmark_of %}Bookmark{% elif note.in_reply_to %}Reply{% else %}Photo{% endif %} · {{ config.SITE_NAME }}{% endblock %} @@ -15,30 +16,9 @@

{{ note.summary }}

{% endif %} - {# Reply context (v1.6.0 Phase 4) - displayed if this is a reply post #} + {# Reply context (v1.6.0 Phase 4; card v2.19.0) - displayed if this is a reply post #} {% if note.in_reply_to %} -
- In reply to: - {# u-in-reply-to on the h-cite itself makes it the entry's property (Post Type Discovery) #} -
- {% if reply_context and reply_context.fetch_status == 'fetched' %} - {% if reply_context.content_snippet %} -

{{ reply_context.content_snippet }}

- {% endif %} - - - {% if reply_context.author_name %}{{ reply_context.author_name }}{% endif %} - {% if reply_context.site_name %} on {{ reply_context.site_name }}{% endif %} - {% if not reply_context.author_name and not reply_context.site_name %}{{ note.in_reply_to }}{% endif %} - - - {% else %} - - {{ note.in_reply_to }} - - {% endif %} -
-
+ {{ reply_card(note.in_reply_to, reply_context or (note.context if note.context is defined else None)) }} {% endif %} {{ reaction(note) }} diff --git a/templates/partials/note_preview.html b/templates/partials/note_preview.html index 3b6ae7d..8dd22ca 100644 --- a/templates/partials/note_preview.html +++ b/templates/partials/note_preview.html @@ -1,6 +1,7 @@ {# One note in a list (home, tag pages). v2.3.0 #} {% from "partials/media.html" import display_media %} {% from "partials/reaction.html" import reaction %} +{% from "partials/reply_context.html" import reply_line %} {% from "partials/responses.html" import list_counts %} {% macro note_preview(note, author) %}
@@ -10,7 +11,7 @@ {% endif %} {% if note.in_reply_to %} -

↩ In reply to {{ note.in_reply_to }}

+ {{ reply_line(note.in_reply_to, note.context if note.context is defined else None) }} {% endif %} {{ reaction(note) }} diff --git a/templates/partials/reply_context.html b/templates/partials/reply_context.html new file mode 100644 index 0000000..d63c7f5 --- /dev/null +++ b/templates/partials/reply_context.html @@ -0,0 +1,62 @@ +{# The post a reply answers (v2.19.0). + The card is the note's u-in-reply-to h-cite (Post Type Discovery reads it + as a reply). Avatars and images are this site's saved copies; until there + is one, the initial is shown and the image left out. #} + +{% macro _who(ctx, url) %}{{ ctx.author_name or ctx.site_name or url.split('/')[2] }}{% endmacro %} + +{% macro _when(ctx, fmt) %} +{%- set when = ctx.published_at -%} +{%- if when and when.strftime is defined -%} + +{%- endif -%} +{% endmacro %} + +{# Full card, on the note's page #} +{% macro reply_card(url, ctx) %} +{% set fetched = ctx and ctx.fetch_status == 'fetched' %} +
+ ↩ In reply to +
+ {% if fetched and (ctx.author_name or ctx.content_snippet or ctx.title) %} +
+ {% set avatar = ctx.author_photo|local_avatar %} + + {%- if avatar %} + {%- else %}{% endif %} + + {%- if ctx.author_url %}{{ _who(ctx, url) }} + {%- else %}{{ _who(ctx, url) }}{% endif %} + {%- if ctx.author_handle %} {{ ctx.author_handle }}{% endif -%} + + +
+ {% if ctx.title %}

{{ ctx.title }}

{% endif %} + {% if ctx.content_snippet %}
{{ ctx.content_snippet|safe }}
{% endif %} + {% set image = ctx.photo|local_preview %} + {% if image %}{{ ctx.photo_alt or '' }}{% endif %} + + {% else %} + {{ url }} + {% endif %} +
+
+{% endmacro %} + +{# One line and a short quote, in lists of notes #} +{% macro reply_line(url, ctx) %} +{% set fetched = ctx and ctx.fetch_status == 'fetched' %} +
+ ↩ In reply to + {% if fetched and (ctx.author_name or ctx.title) %} + {{ ctx.title or ctx.author_name }} + {%- if ctx.title and ctx.author_name %} by {{ ctx.author_name }}{% endif %} + {% set quote = ctx.content_snippet|striptags|replace('\n', ' ')|trim if ctx.content_snippet else '' %} + {% if quote and not ctx.title %}{{ quote|truncate(140) }}{% endif %} + {% else %} + {{ url }} + {% endif %} +
+{% endmacro %} diff --git a/tests/test_reply_context_sources.py b/tests/test_reply_context_sources.py new file mode 100644 index 0000000..1b4cd4b --- /dev/null +++ b/tests/test_reply_context_sources.py @@ -0,0 +1,270 @@ +""" +Reply contexts from fediverse, Bluesky and OpenGraph sources (v2.19.0) + +Mastodon and GoToSocial pages have no h-entry, so a reply to one used to show +only the server's domain. The context now comes from the Mastodon API, this +site's GoToSocial instance, the Bluesky AppView API or OpenGraph tags, and the +note page shows it as a card; lists show the author and a short quote. +""" + +import io +import json +import sqlite3 + +import httpx +import mf2py +import pytest +from PIL import Image + +from starpunk import avatars, syndication +from starpunk.reply_context import fetch_reply_context, refresh_stale +from starpunk.syndication import MastodonClient + +PUBLIC = [(2, 1, 6, "", ("93.184.216.34", 0))] + +TOOT = "https://masto.example/@ada/117611186265622833" +GTS_POST = "https://gts.example/@bob/statuses/01M7BE51ABF9HP2G0CPH5DKKGG" +SKEET = "https://bsky.app/profile/carol.example/post/3l6oveex3ii2l" +ARTICLE = "https://news.example/2026/11/13/ground-stop" + +# Mastodon's status page: an app shell with OpenGraph tags, no microformats +TOOT_PAGE = """ + + + +
""" + +STATUS = { + "id": "117611186265622833", + "created_at": "2026-11-13T21:11:22.624Z", + "spoiler_text": "", + "sensitive": False, + "content": '

All aircraft are ground stopped.

' + '', + "account": {"username": "ada", "acct": "ada", "display_name": "Ada Lovelace", + "url": "https://masto.example/@ada", + "avatar_static": "https://masto.example/avatars/ada.png"}, + "media_attachments": [{"type": "image", "preview_url": "https://masto.example/media/small.png", + "description": "An advisory"}], +} + +GTS_PAGE = """ + + +
+Bob +

Posted on GoToSocial

+
""" + +ARTICLE_PAGE = """Ignored + + + + + + + +

Story

""" + +THREAD = {"thread": {"post": { + "author": {"handle": "carol.example", "displayName": "Carol", + "avatar": "https://cdn.example/avatar/carol.png"}, + "record": {"text": "Hello \n\nSecond paragraph", "createdAt": "2026-11-12T08:00:00.000Z"}, + "embed": {"$type": "app.bsky.embed.images#view", + "images": [{"thumb": "https://cdn.example/thumb/1.png", "alt": "A cat"}]}, +}}} + + +def png(color="red", size=(800, 400)): + buffer = io.BytesIO() + Image.new("RGB", size, color).save(buffer, "PNG") + return buffer.getvalue() + + +@pytest.fixture +def web(monkeypatch): + """The mocked internet; tests change `routes` and read `seen`""" + seen = [] + routes = { + TOOT: lambda r: httpx.Response(200, html=TOOT_PAGE), + "https://masto.example/api/v1/statuses/117611186265622833": lambda r: httpx.Response(200, json=STATUS), + GTS_POST: lambda r: httpx.Response(200, html=GTS_PAGE), + ARTICLE: lambda r: httpx.Response(200, html=ARTICLE_PAGE), + } + + def handler(request): + # The SSRF-safe fetcher connects to the checked IP; the name is in Host + url = f"{request.url.scheme}://{request.headers['host']}{request.url.raw_path.decode()}" + seen.append(url) + if url.startswith("https://public.api.bsky.app/"): + return httpx.Response(200, json=THREAD) + if url.endswith(".png"): + return httpx.Response(200, content=png(), headers={"Content-Type": "image/png"}) + route = routes.get(url) + return route(request) if route else httpx.Response(404) + + monkeypatch.setattr("starpunk.webmention_ssrf.socket.getaddrinfo", lambda *a, **k: PUBLIC) + monkeypatch.setattr("starpunk.webmention_ssrf.TRANSPORT", httpx.MockTransport(handler)) + return type("Web", (), {"routes": routes, "seen": seen}) + + +def fetch(app, url, **kwargs): + with app.app_context(): + return fetch_reply_context(app.config["DATABASE_PATH"], url, app.config["SITE_URL"], **kwargs) + + +def test_mastodon_post_from_its_public_api(app, web): + context = fetch(app, TOOT) + + assert context["source"] == "fediverse" + assert context["author_name"] == "Ada Lovelace" + assert context["author_handle"] == "@ada@masto.example" + assert context["author_url"] == "https://masto.example/@ada" + assert "ground stopped" in context["content_snippet"] + assert "script" not in context["content_snippet"] + assert context["photo"] == "https://masto.example/media/small.png" + assert context["photo_alt"] == "An advisory" + assert context["published_at"].startswith("2026-11-13T21:11:22") + # Avatar and image are saved locally (the CSP allows only this site's images) + data = app.config["DATA_PATH"] + assert avatars.cached_name(data, context["author_photo"]) + preview = avatars.preview_folder(data) / avatars.cached_preview(data, context["photo"]) + with Image.open(preview) as image: + assert image.width == avatars.PREVIEW_WIDTH + + # Stored with its source, so it isn't fetched again + web.seen.clear() + assert fetch(app, TOOT)["author_name"] == "Ada Lovelace" + assert web.seen == [] + + +def test_content_warning_is_shown_instead_of_the_post(app, web): + web.routes["https://masto.example/api/v1/statuses/117611186265622833"] = lambda r: httpx.Response( + 200, json={**STATUS, "spoiler_text": "flight news", "sensitive": True}) + context = fetch(app, TOOT) + assert context["content_snippet"] == "

CW: flight news

" + assert context["photo"] is None + + +def test_fediverse_post_through_own_instance(app, web): + """GoToSocial has no anonymous API; this site's instance resolves the post""" + asked = [] + + def gts(request): + asked.append(request.url.params.get("q")) + status = {**STATUS, "content": "

Posted on GoToSocial

", "media_attachments": [], + "account": {**STATUS["account"], "acct": "bob@gts.example", "display_name": "Bob", + "url": "https://gts.example/@bob"}} + return httpx.Response(200, json={"statuses": [status]}) + + own = "https://own.example" + app.config.update(GOTOSOCIAL_URL=own, GOTOSOCIAL_ACCESS_TOKEN="t") + syndication._client_cache[own] = MastodonClient( + own, "t", http=httpx.Client(transport=httpx.MockTransport(gts))) + try: + with app.test_request_context(): + context = fetch_reply_context(app.config["DATABASE_PATH"], GTS_POST, app.config["SITE_URL"]) + finally: + syndication._client_cache.clear() + + assert asked == [GTS_POST] + assert context["source"] == "fediverse" + assert context["author_name"] == "Bob" + assert context["author_handle"] == "@bob@gts.example" + assert "Posted on GoToSocial" in context["content_snippet"] + + +def test_fediverse_post_without_an_api_uses_opengraph(app, web): + context = fetch(app, GTS_POST) + assert context["source"] == "opengraph" + assert context["author_name"] == "Bob" + assert context["author_handle"] == "@bob@gts.example" + assert context["author_url"] == "https://gts.example/@bob" + assert context["title"] is None + + +def test_bluesky_post_from_the_public_api(app, web): + context = fetch(app, SKEET) + assert context["source"] == "bluesky" + assert context["author_name"] == "Carol" + assert context["author_handle"] == "@carol.example" + assert context["author_url"] == "https://bsky.app/profile/carol.example" + assert context["content_snippet"] == "

Hello <world>

Second paragraph

" + assert (context["photo"], context["photo_alt"]) == ("https://cdn.example/thumb/1.png", "A cat") + assert any("uri=at%3A%2F%2Fcarol.example%2Fapp.bsky.feed.post%2F3l6oveex3ii2l" in u for u in web.seen) + + +def test_other_pages_use_opengraph(app, web): + context = fetch(app, ARTICLE) + assert context["source"] == "opengraph" + assert context["title"] == "The FAA stops all departures" + assert context["author_name"] == "Dee Reporter" + assert context["site_name"] == "News Example" + assert "held on the ground" in context["content_snippet"] + assert context["photo"] == "https://news.example/img/lead.png" + assert context["photo_alt"] == "A departures board" + + +def _old_row(app, url, **values): + """A context stored before v2.19.0: no source, the domain as author""" + conn = sqlite3.connect(app.config["DATABASE_PATH"]) + conn.execute("DELETE FROM reply_contexts WHERE url = ?", (url,)) + conn.execute("INSERT INTO reply_contexts (url, author_name, fetch_status) VALUES (?, ?, 'fetched')", + (url, values.get("author_name", "masto.example"))) + conn.commit() + conn.close() + + +def _reply(app, url, content="Agreed"): + from starpunk.notes import create_note + with app.test_request_context(): + return create_note(content, published=True, in_reply_to=url) + + +def test_contexts_from_before_are_fetched_again_once(app, web): + _reply(app, TOOT) + _old_row(app, TOOT) + + with app.app_context(): + assert refresh_stale(app.config["DATABASE_PATH"], app.config["SITE_URL"]) == 1 + assert refresh_stale(app.config["DATABASE_PATH"], app.config["SITE_URL"]) == 0 + assert fetch(app, TOOT)["author_name"] == "Ada Lovelace" + + +def test_failed_refetch_keeps_what_was_there(app, web): + _reply(app, TOOT) + _old_row(app, TOOT, author_name="Ada (old)") + web.routes.clear() # everything 404s now + + with app.app_context(): + refresh_stale(app.config["DATABASE_PATH"], app.config["SITE_URL"]) + assert refresh_stale(app.config["DATABASE_PATH"], app.config["SITE_URL"]) == 0 + context = fetch(app, TOOT) + assert (context["fetch_status"], context["author_name"]) == ("fetched", "Ada (old)") + + +def test_reply_page_and_home_page_show_the_context(app, web): + note = _reply(app, TOOT, "This cannot be good") + client = app.test_client() + + page = client.get(f"/note/{note.slug}").data.decode() + assert "Ada Lovelace" in page and "@ada@masto.example" in page + assert "ground stopped" in page + assert "/previews/" in page and 'alt="An advisory"' in page + assert "/avatars/" in page + + home = client.get("/").data.decode() + assert "↩ In reply to" in home + assert ">Ada Lovelace" in home + assert "All aircraft are ground stopped." in home + + # Still a reply to the toot (Post Type Discovery), on both pages, and the + # card's author and content stay inside the h-cite + for html in (page, home): + items = mf2py.parse(doc=html, url="http://localhost:5000/")["items"] + entries = [i for i in items if "h-entry" in i["type"]] or [ + c for i in items for c in i.get("children", []) if "h-entry" in c["type"]] + (entry,) = [e for e in entries if "This cannot be good" in json.dumps(e)] + reply_to = entry["properties"]["in-reply-to"][0] + assert (reply_to if isinstance(reply_to, str) else reply_to["properties"]["url"][0]) == TOOT + assert "Ada Lovelace" not in json.dumps(entry["properties"].get("author", ""))