From ab6ff60f5c3894ce64b745368b3e2391ca5dbf83 Mon Sep 17 00:00:00 2001 From: Phil Date: Tue, 3 Nov 2026 10:11:23 -0700 Subject: [PATCH] feat: Micropub review fixes (v2.9.0) - mp-syndicate-to/mp-slug read from JSON properties; absent means no crossposting - Sanitize rendered HTML (nh3 allowlist) - q=source/q=category need a posting scope; draft scope creates drafts; private/unlisted visibility saved as drafts - Photos that can't be fetched or found fail the request; remote photos via the SSRF-checked, pinned fetcher - JSON scalars wrapped, update delete values must be arrays - bookmark-of stored; HTML content returned as {html} (migration 015) - published stored as UTC, bad dates 400; malformed JSON 400 - Auth: case-insensitive Bearer, header+body 400, WWW-Authenticate Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 31 +++ .../v2.9.0/2026-11-03-micropub-review.md | 51 ++++ .../015_bookmarks_and_content_format.sql | 7 + starpunk/__init__.py | 4 +- starpunk/media.py | 8 +- starpunk/micropub.py | 235 +++++++++++------ starpunk/models.py | 56 +++- starpunk/notes.py | 38 ++- starpunk/routes/indieauth.py | 8 +- starpunk/routes/micropub.py | 15 +- starpunk/syndication.py | 3 + templates/admin/edit.html | 2 +- templates/note.html | 2 +- templates/partials/reaction.html | 4 +- tests/test_likes_reposts.py | 5 + tests/test_media_upload.py | 2 +- tests/test_micropub_v240.py | 7 +- tests/test_micropub_v290.py | 249 ++++++++++++++++++ tests/test_syndication.py | 5 +- 19 files changed, 610 insertions(+), 122 deletions(-) create mode 100644 docs/design/v2.9.0/2026-11-03-micropub-review.md create mode 100644 migrations/015_bookmarks_and_content_format.sql create mode 100644 tests/test_micropub_v290.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e65a77c..9dc8927 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,37 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [2.9.0] - 2026-11-03 + +Micropub review against the [W3C Recommendation](https://www.w3.org/TR/micropub/). + +### Changed + +- **Micropub posts are only crossposted to the targets the client asks for.** If a client doesn't send `mp-syndicate-to`, nothing is crossposted (before, everything was). Posts made in the admin editor are unchanged +- Posts sent with `visibility` private or unlisted are saved as drafts instead of being published (StarPunk has no private posts) +- A token with only the `draft` scope can create drafts, and nothing else +- `q=source` and `q=category` need a token that can post (`create`, `update`, `delete` or `draft`); before, any token could list every post, drafts included +- Sending the access token in both the header and the body is refused (400) + +### Added + +- Bookmarks: Micropub `bookmark-of` is stored and shown ("๐Ÿ”– Bookmarked โ€ฆ", `u-bookmark-of`), returned by `q=source`, listed in `q=config`, and crossposts lead with the bookmarked link (migration 015) +- A post with only a `summary` uses it as the text; a post with only a `name` is a titled post with no body + +### Fixed + +- Quill and other JSON clients put `mp-syndicate-to` and `mp-slug` inside `properties`; they were ignored, so posts crossposted everywhere and slugs were not used +- HTML in posts is sanitized: scripts, event handlers, iframes and `javascript:` links are removed when notes are shown. Formatting, images, code highlighting, footnotes and microformats are kept +- A photo that can't be downloaded or found made the post without it while reporting success. Create and update now fail with 400 and change nothing +- Remote photos are fetched the same way as webmentions: every redirect is checked for private addresses and the connection goes to the checked address +- JSON values that weren't lists were read a character at a time (`"category": "solo"` became tags s, o, l). Single values are now treated as a one-item list; `delete` values in updates must be lists +- Posts written as HTML are returned by `q=source` as `{"html": โ€ฆ}`, as the spec requires; empty `content` is left out +- A client-supplied `published` time is stored as UTC like every other post; a date that can't be read is refused instead of silently replaced with now +- Malformed JSON gets 400 instead of 500, and error messages no longer include Python internals +- `video` and `audio` (not supported) are refused with 400 instead of being dropped; person tags in `category` are ignored instead of failing the post +- An update with no `replace`, `add` or `delete` is refused +- `Authorization: bearer โ€ฆ` is accepted (the scheme is case-insensitive), and 401 responses include `WWW-Authenticate` + ## [2.8.1] - 2026-11-03 ### Fixed diff --git a/docs/design/v2.9.0/2026-11-03-micropub-review.md b/docs/design/v2.9.0/2026-11-03-micropub-review.md new file mode 100644 index 0000000..85a8576 --- /dev/null +++ b/docs/design/v2.9.0/2026-11-03-micropub-review.md @@ -0,0 +1,51 @@ +# v2.9.0: Micropub review + +Review against the W3C Micropub Recommendation (https://www.w3.org/TR/micropub/), +run through the Flask test client and mapped to the micropub.rocks server +tests. Everyday create/update/delete/query/media/auth already passed; the +gaps were privacy, security and silent data loss. + +## Decisions (user) + +- No `mp-syndicate-to` from a Micropub client means no crossposting + (was: all targets, v2.1.0). The admin editor's default is unchanged. +- `visibility` private/unlisted are saved as drafts. +- Rendered HTML is sanitized with an allowlist (nh3 defaults plus class, + title, id, lang, dir on all tags; video/audio/source/track/picture). + +## Changes + +| Area | Before | After | +|---|---|---| +| `mp-*` in JSON `properties` | ignored (all targets, timestamp slug) | read as commands, never stored | +| absent `mp-syndicate-to` | all targets | none | +| `q=source` / `q=category` | any token | create, update, delete or draft scope | +| HTML content | rendered raw (stored XSS) | `render_markdown()` = Markdown + nh3 | +| `visibility` | ignored (published) | private/unlisted -> draft; unknown -> 400 | +| `draft` scope | 403 | creates drafts only | +| photo fetch/lookup failure | dropped, 201 | 400, nothing created or changed | +| remote photo fetch | first host checked, redirects followed | `webmention_ssrf.safe_get` (per hop, pinned) | +| JSON scalars | indexed per character | wrapped on create; update `delete` values must be arrays | +| `bookmark-of` | dropped | `notes.bookmark_of`, kind `bookmark` (migration 015) | +| HTML-authored content in `q=source` | string | `{"html": ...}` via `notes.content_format` | +| `summary` / `name` only | 400 | used as text / titled post | +| `video` / `audio` | dropped | 400 | +| `published` | stored with offset; garbage -> now | naive UTC; garbage -> 400 | +| malformed JSON | 500 | 400 | +| update without operations | 204 | 400 | +| auth | `Bearer` only; header+body accepted; no challenge | case-insensitive; both -> 400; `WWW-Authenticate` | + +## Kept as is + +- 403 (RFC 6750) for insufficient_scope; the Micropub table says 401. +- `add` only for category and photo; `published`, `syndication`, + `location` not updatable. +- `location`/`checkin` ignored (3.3.5 allows). +- GET `access_token` query parameter still accepted. + +## Tests + +`tests/test_micropub_v290.py` (31 of 34 fail on v2.8.1). Updated: +likes/reposts helper sends `mp-syndicate-to` in properties; +`test_micropub_default_is_no_targets`; remote photo test uses the pinned +fetcher; corrupted-image message. diff --git a/migrations/015_bookmarks_and_content_format.sql b/migrations/015_bookmarks_and_content_format.sql new file mode 100644 index 0000000..1346773 --- /dev/null +++ b/migrations/015_bookmarks_and_content_format.sql @@ -0,0 +1,7 @@ +-- Migration 015: Bookmarks and HTML-authored content (v2.9.0) +-- bookmark_of: URL bookmarked (Micropub bookmark-of); the note is a bookmark. +-- content_format: 'html' when a Micropub client sent the content as HTML, +-- so q=source returns it as {"html": ...} (Micropub 3.7.2). NULL = Markdown. + +ALTER TABLE notes ADD COLUMN bookmark_of TEXT; +ALTER TABLE notes ADD COLUMN content_format TEXT; diff --git a/starpunk/__init__.py b/starpunk/__init__.py index 9329421..e94c402 100644 --- a/starpunk/__init__.py +++ b/starpunk/__init__.py @@ -463,5 +463,5 @@ def create_app(config=None): # Package version (Semantic Versioning 2.0.0) # See docs/standards/versioning-strategy.md for details -__version__ = "2.8.1" -__version_info__ = (2, 8, 1) +__version__ = "2.9.0" +__version_info__ = (2, 9, 0) diff --git a/starpunk/media.py b/starpunk/media.py index e8fe378..35ec64a 100644 --- a/starpunk/media.py +++ b/starpunk/media.py @@ -93,6 +93,10 @@ def get_optimization_params(file_size: int) -> Tuple[int, int]: return (1280, 85) +# Shown to clients; the parser's own message names internal objects (v2.9.0) +UNSUPPORTED_IMAGE = "Not a supported image: use JPEG, PNG, GIF, WebP or HEIC" + + def validate_image(file_data: bytes, filename: str) -> Tuple[Image.Image, str, int, int]: """ Validate image file and return PIL Image for memory-efficient processing @@ -158,9 +162,9 @@ def validate_image(file_data: bytes, filename: str) -> Tuple[Image.Image, str, i current_app.logger.info(f'Saved failed upload for analysis: {debug_file}') except RuntimeError: pass # Outside app context (e.g., tests) - raise ValueError(f"Invalid or corrupted image: {e}") + raise ValueError(UNSUPPORTED_IMAGE) from e else: - raise ValueError(f"Invalid or corrupted image: {e}") + raise ValueError(UNSUPPORTED_IMAGE) from e # Store original format before any conversions original_format = img.format diff --git a/starpunk/micropub.py b/starpunk/micropub.py index 19f6390..ae024d4 100644 --- a/starpunk/micropub.py +++ b/starpunk/micropub.py @@ -25,7 +25,7 @@ References: - IndieAuth Specification: https://www.w3.org/TR/indieauth/ """ -from datetime import datetime +from datetime import datetime, timezone from typing import Optional from flask import Request, current_app, jsonify @@ -81,9 +81,17 @@ def error_response(error: str, error_description: str, status_code: int = 400): Returns: Tuple of (response, status_code) """ + headers = {} + if status_code == 401: + # RFC 6750 3: tell the client how to authenticate (v2.9.0) + challenge = 'Bearer realm="micropub"' + if error == "unauthorized" and "Invalid" in error_description: + challenge += ', error="invalid_token"' + headers["WWW-Authenticate"] = challenge return ( jsonify({"error": error, "error_description": error_description}), status_code, + headers, ) @@ -104,18 +112,21 @@ def extract_bearer_token(request: Request) -> Optional[str]: Returns: Token string if found, None otherwise """ - # Try Authorization header first - auth_header = request.headers.get("Authorization", "") - if auth_header.startswith("Bearer "): - return auth_header[7:] # Remove "Bearer " prefix + # The scheme is case-insensitive (RFC 7235) + scheme, _, credentials = request.headers.get("Authorization", "").partition(" ") + header_token = credentials.strip() if scheme.lower() == "bearer" else None - # Try form parameter + body_token = None if request.method == "POST": - return request.form.get("access_token") + body_token = request.form.get("access_token") elif request.method == "GET": - return request.args.get("access_token") + body_token = request.args.get("access_token") - return None + # Only one method may be used (RFC 6750 2) (v2.9.0) + if header_token and body_token: + raise MicropubValidationError("Send the access token in the header or the body, not both") + + return header_token or body_token # Property Normalization @@ -135,9 +146,18 @@ def normalize_properties(data: dict) -> dict: Returns: Normalized properties dict with all values as lists """ - # JSON format has properties nested + # JSON format has properties nested. Values MUST be arrays (3.3.2); a + # bare value is wrapped rather than read character by character. + # mp-* commands and access_token are not properties (3.2). if "properties" in data: - return data["properties"] + properties = data["properties"] + if not isinstance(properties, dict): + raise MicropubValidationError("properties must be an object") + return { + key: value if isinstance(value, list) else [value] + for key, value in properties.items() + if not key.startswith("mp-") and key != "access_token" + } # Form format - convert to properties dict properties = {} @@ -176,7 +196,7 @@ def extract_content(properties: dict) -> str: Raises: MicropubValidationError: If content is missing or empty """ - content_list = properties.get("content", []) + content_list = properties.get("content") or properties.get("summary") or [] # Handle both plain text and HTML/text objects if not content_list: @@ -189,12 +209,20 @@ def extract_content(properties: dict) -> str: # Prefer text over html for markdown storage content = content.get("text") or content.get("html", "") - if not content or not content.strip(): + if not isinstance(content, str) or not content.strip(): raise MicropubValidationError("Content cannot be empty") return content.strip() +def content_format(properties: dict) -> Optional[str]: + """'html' if the content was sent only as HTML (returned as such by q=source)""" + content = _first(properties.get("content")) + if isinstance(content, dict) and content.get("html") and not content.get("text"): + return "html" + return None + + def _has_text(content) -> bool: """Whether a content property has any text. Likes, reposts (v2.5.0) and photo posts (v2.6.5) may come without.""" @@ -245,8 +273,9 @@ def extract_tags(properties: dict) -> list[str]: List of tag strings """ categories = properties.get("category", []) - # Filter out empty strings and strip whitespace - return [tag.strip() for tag in categories if tag and tag.strip()] + # Strings only: person tags (h-card objects) aren't supported and are + # ignored, as Micropub says to ignore what isn't recognised (3.3.5) + return [tag.strip() for tag in categories if isinstance(tag, str) and tag.strip()] def extract_published_date(properties: dict) -> Optional[datetime]: @@ -259,18 +288,18 @@ def extract_published_date(properties: dict) -> Optional[datetime]: Returns: Datetime object if published date provided, None otherwise """ - published = properties.get("published", [""])[0] + published = _first(properties.get("published")) if not published: return None try: - # Parse ISO 8601 datetime - # datetime.fromisoformat handles most ISO formats - return datetime.fromisoformat(published.replace("Z", "+00:00")) - except (ValueError, AttributeError): - # If parsing fails, log and return None (will use current time) - current_app.logger.warning(f"Failed to parse published date: {published}") - return None + value = datetime.fromisoformat(str(published).strip().replace("Z", "+00:00")) + except ValueError: + raise MicropubValidationError(f"published is not an ISO 8601 date: {published}") + # Stored times are naive UTC, like every other note (v2.9.0) + if value.tzinfo is not None: + value = value.astimezone(timezone.utc).replace(tzinfo=None) + return value # Action Handlers @@ -306,6 +335,24 @@ def extract_post_status(properties: dict) -> bool: raise MicropubValidationError(f"Unsupported post-status: {status}") +def extract_visibility_draft(properties: dict) -> bool: + """ + True if visibility asks for a non-public post (v2.9.0) + + StarPunk only has public posts and drafts, so private and unlisted posts + are kept as drafts rather than published. + """ + visibility = _first(properties.get("visibility")) + if visibility in (None, "", "public"): + return False + if visibility in ("private", "unlisted"): + return True + raise MicropubValidationError(f"Unsupported visibility: {visibility}") + + +UNSUPPORTED_MEDIA = ("video", "audio") + + def extract_url_property(properties: dict, name: str) -> Optional[str]: """A URL property (in-reply-to, like-of, repost-of): string or h-cite""" value = _first(properties.get(name)) @@ -361,27 +408,25 @@ def _public_address(host: str) -> bool: def _fetch_remote_photo(url: str) -> Optional[int]: - """Download a photo hosted elsewhere into the media library. Returns media id.""" + """ + Download a photo hosted elsewhere into the media library. Returns media id. + + Uses the webmention fetcher (v2.9.0): every redirect is checked against + private addresses and the connection is pinned to the checked address. + """ from urllib.parse import urlparse - import httpx from starpunk.media import save_media + from starpunk.webmention_ssrf import safe_get, user_agent - parsed = urlparse(url) - if parsed.scheme not in ("http", "https") or not parsed.hostname: - return None - if not current_app.config.get("TESTING") and not _public_address(parsed.hostname): - current_app.logger.warning(f"Refusing to fetch photo from non-public address: {url}") - return None try: - with httpx.stream("GET", url, timeout=20.0, follow_redirects=True) as response: - response.raise_for_status() - data = bytearray() - for chunk in response.iter_bytes(): - data += chunk - if len(data) > MAX_REMOTE_PHOTO_BYTES: - raise ValueError("photo too large") - name = parsed.path.rsplit("/", 1)[-1] or "photo.jpg" - return save_media(bytes(data), name)["id"] + result = safe_get( + url, headers={"User-Agent": user_agent(current_app.config.get("SITE_URL", ""))}, + max_bytes=MAX_REMOTE_PHOTO_BYTES, timeout=20, + ) + if result.status_code != 200: + raise ValueError(f"HTTP {result.status_code}") + name = urlparse(result.url).path.rsplit("/", 1)[-1] or "photo.jpg" + return save_media(result.content, name)["id"] except Exception as e: current_app.logger.warning(f"Could not fetch photo {url}: {e}") return None @@ -430,8 +475,11 @@ def _resolve_photos(photos: list[dict]) -> tuple[list[int], list[str]]: if len(media_ids) == MAX_PHOTOS: break media_id = photo.get("media_id") or _media_id_for_url(photo["url"]) + if not media_id: + # Don't create or change the post without it (v2.9.0) + raise MicropubValidationError(f"Could not get photo {photo.get('url')}") # The same photo twice is shown once (v2.6.5) - if media_id and media_id not in media_ids: + if media_id not in media_ids: media_ids.append(media_id) captions.append(photo.get("alt", "")) return media_ids, captions @@ -442,8 +490,7 @@ def _attach_photos_to_note(note_id: int, photos: list[dict]) -> None: from starpunk.media import attach_media_to_note media_ids, captions = _resolve_photos(photos) - if media_ids or not photos: - attach_media_to_note(note_id, media_ids, captions) + attach_media_to_note(note_id, media_ids, captions) def _note_photos(note) -> list[dict]: @@ -508,47 +555,54 @@ def handle_create(data: dict, token_info: dict, files=None): Returns: Tuple of (response_body, status_code, headers) """ - _require_scope(token_info, "create") + _require_scope(token_info, "create", "draft") + # A token with only the draft scope may create drafts (v2.9.0) + drafts_only = not check_scope("create", token_info.get("scope", "")) _check_type(data) - # mp-slug and mp-syndicate-to are server commands, not properties - custom_slug = None - if isinstance(data, dict) and 'mp-slug' in data: - slug_value = data.get('mp-slug') - if isinstance(slug_value, list) and slug_value: - custom_slug = slug_value[0] - elif isinstance(slug_value, str): - custom_slug = slug_value + # mp-slug and mp-syndicate-to are server commands, not properties. Form + # posts send them at the top level; JSON clients inside properties too. + commands = dict(data) if isinstance(data, dict) else {} + if isinstance(commands.get("properties"), dict): + commands.update({k: v for k, v in commands["properties"].items() if k.startswith("mp-")}) - # Absent means "all targets"; present means exactly those (v2.1.0) - requested_targets = None - if isinstance(data, dict): - for key in ("mp-syndicate-to", "mp-syndicate-to[]"): - if key in data: - value = data[key] - requested_targets = (requested_targets or []) + ( - value if isinstance(value, list) else [value] - ) + custom_slug = _first(commands.get("mp-slug")) + if custom_slug is not None and not isinstance(custom_slug, str): + raise MicropubValidationError("mp-slug must be a string") + + # Crossposting only to the targets asked for: absent means none (v2.9.0) + requested_targets = [] + for key in ("mp-syndicate-to", "mp-syndicate-to[]"): + if key in commands: + value = commands[key] + requested_targets += value if isinstance(value, list) else [value] try: properties = normalize_properties(data) like_of = extract_url_property(properties, "like-of") repost_of = extract_url_property(properties, "repost-of") - if like_of and repost_of: - raise MicropubValidationError("A post can't be both a like and a repost") - has_content = _has_text(properties.get("content")) + bookmark_of = extract_url_property(properties, "bookmark-of") + if sum(bool(u) for u in (like_of, repost_of, bookmark_of)) > 1: + raise MicropubValidationError("A post can only be one of a like, repost or bookmark") + for prop in UNSUPPORTED_MEDIA: + if properties.get(prop) or (files and (files.getlist(prop) or files.getlist(f"{prop}[]"))): + raise MicropubValidationError(f"{prop} posts are not supported") + has_content = _has_text(properties.get("content")) or _has_text(properties.get("summary")) photos = extract_photos(properties) + _save_uploads(files) - # Photo posts may have no text (v2.6.5): Micropub doesn't require content + title = extract_title(properties) + # Photo posts may have no text (v2.6.5), nor titled posts (v2.9.0) content = ( extract_content(properties) - if has_content or not (like_of or repost_of or photos) + if has_content or not (like_of or repost_of or bookmark_of or photos or title) else "" ) - title = extract_title(properties) tags = extract_tags(properties) published_date = extract_published_date(properties) - published = extract_post_status(properties) + published = extract_post_status(properties) and not extract_visibility_draft(properties) + if drafts_only: + published = False in_reply_to = extract_in_reply_to(properties) + fmt = content_format(properties) if has_content else None # Look up and download photos before the note exists, so a bad photo # can't leave a half-made post behind (v2.6.5) media_ids, captions = _resolve_photos(photos) @@ -556,7 +610,7 @@ def handle_create(data: dict, token_info: dict, files=None): raise except Exception as e: current_app.logger.error(f"Property extraction failed: {e}") - raise MicropubValidationError(f"Failed to parse request: {str(e)}") + raise MicropubValidationError("Could not read the post's properties") note = None try: @@ -569,7 +623,9 @@ def handle_create(data: dict, token_info: dict, files=None): in_reply_to=in_reply_to, like_of=like_of, repost_of=repost_of, + bookmark_of=bookmark_of, has_media=bool(media_ids), + content_format=fmt, ) if media_ids: @@ -577,11 +633,8 @@ def handle_create(data: dict, token_info: dict, files=None): attach_media_to_note(note.id, media_ids, captions) # Crosspost after photos are attached (v2.1.0); drafts wait until published - from starpunk.syndication import get_targets, resolve_targets, select_targets - if requested_targets is None: - select_targets(note.id, list(get_targets())) - else: - select_targets(note.id, resolve_targets(requested_targets)) + from starpunk.syndication import resolve_targets, select_targets + select_targets(note.id, resolve_targets(requested_targets)) return "", 201, {"Location": absolute_url_for("public.note", slug=note.slug)} @@ -618,9 +671,12 @@ def handle_update(data: dict, token_info: dict): delete = data.get("delete") or {} if not isinstance(replace, dict) or not isinstance(add, dict) or not isinstance(delete, (dict, list)): raise MicropubValidationError("replace and add must be objects; delete an object or list") - for prop, values in list(replace.items()) + list(add.items()): + for prop, values in list(replace.items()) + list(add.items()) + ( + list(delete.items()) if isinstance(delete, dict) else []): if not isinstance(values, list): raise MicropubValidationError(f"Values for {prop} must be an array") + if not (replace or add or delete): + raise MicropubValidationError("An update needs replace, add or delete") touched = set(replace) | set(add) | set(delete) unsupported = sorted(touched - UPDATABLE) if unsupported: @@ -664,6 +720,7 @@ def handle_update(data: dict, token_info: dict): if "content" in replace: # Empty text is fine if the note keeps its photos; update_note checks (v2.6.5) body = extract_content(replace) if _has_text(replace["content"]) else "" + changes["content_format"] = content_format(replace) if body else None if "name" in replace: title = extract_title(replace) if "category" in replace: @@ -681,13 +738,19 @@ def handle_update(data: dict, token_info: dict): if "photo" in add: photos += extract_photos(add) - # Text may go only if photos stay (or it's a like or repost) (v2.6.5) - if not (body or "").strip() and not (photos or note.like_of or note.repost_of): + # Text may go only if photos stay (or it's a like, repost or bookmark) (v2.6.5) + if not (body or "").strip() and not (photos or note.reacts_to): raise MicropubValidationError("A post needs text or a photo") + # Photos must all resolve before anything changes (v2.9.0) + if "photo" in touched: + media_ids, captions = _resolve_photos(photos) + new_content = add_title_heading(body, title) if new_content != note.content: changes["content"] = new_content + elif "content_format" in changes and changes["content_format"] == note.content_format: + del changes["content_format"] if {"category"} & touched: changes["tags"] = tags @@ -695,7 +758,8 @@ def handle_update(data: dict, token_info: dict): if changes: update_note(id=note.id, has_media=bool(photos), **changes) if "photo" in touched: - _attach_photos_to_note(note.id, photos) + from starpunk.media import attach_media_to_note + attach_media_to_note(note.id, media_ids, captions) note_changed(note.id) except InvalidNoteDataError as e: raise MicropubValidationError(str(e)) @@ -742,7 +806,8 @@ def note_to_mf2(note, wanted: Optional[list] = None) -> dict: title, content = split_title(note.content) props = { - "content": [content], + # HTML-authored content comes back as HTML (3.7.2, v2.9.0) + "content": [{"html": content} if note.content_format == "html" else content], "published": [iso_utc(note.created_at)], "url": [absolute_url_for("public.note", slug=note.slug)], "post-status": ["published" if note.published else "draft"], @@ -760,6 +825,10 @@ def note_to_mf2(note, wanted: Optional[list] = None) -> dict: props["like-of"] = [note.like_of] if getattr(note, "repost_of", None): props["repost-of"] = [note.repost_of] + if getattr(note, "bookmark_of", None): + props["bookmark-of"] = [note.bookmark_of] + if not content.strip(): + del props["content"] # photo posts, likes, reposts and bookmarks without text photos = _note_photos(note) if photos: props["photo"] = [{"value": p["url"], "alt": p["alt"]} for p in photos] @@ -812,6 +881,11 @@ def handle_query(args, token_info: dict): q = args.get("q") + # Posts (drafts included) and tags are for apps that may post (v2.9.0); + # config and syndicate-to are fine for any token + if q in ("source", "category"): + _require_scope(token_info, "create", "update", "delete", "draft") + if q == "config": site_url = current_app.config.get("SITE_URL", "http://localhost:5000").rstrip('/') return jsonify({ @@ -825,6 +899,7 @@ def handle_query(args, token_info: dict): {"type": "reply", "name": "Reply", "properties": ["in-reply-to", "content"]}, {"type": "like", "name": "Like", "properties": ["like-of"]}, {"type": "repost", "name": "Repost", "properties": ["repost-of", "content"]}, + {"type": "bookmark", "name": "Bookmark", "properties": ["bookmark-of", "content"]}, ], }), 200 @@ -853,4 +928,4 @@ def handle_query(args, token_info: dict): if q == "category": return jsonify({"categories": _all_categories(args.get("filter") or "")}), 200 - return error_response("invalid_request", f"Unknown query: {q}") + return error_response("invalid_request", f"Unknown query: {q}" if q else "Missing q parameter") diff --git a/starpunk/models.py b/starpunk/models.py index a2cd7fb..ef10df2 100644 --- a/starpunk/models.py +++ b/starpunk/models.py @@ -16,6 +16,7 @@ import sqlite3 # Third-party imports import markdown +import nh3 # Local imports from starpunk.utils import ( @@ -42,6 +43,32 @@ MARKDOWN_EXTENSIONS = ["extra", "codehilite", "nl2br", BareUrlExtension()] # Constants - Content limits MAX_TITLE_LENGTH = 200 +# HTML allowed in rendered notes (v2.9.0). Markdown passes raw HTML through, +# and Micropub clients may send HTML, so scripts, event handlers, iframes and +# javascript: links are stripped. class keeps code highlighting and +# microformats; id keeps footnote links. +ALLOWED_TAGS = set(nh3.ALLOWED_TAGS) | {"video", "audio", "source", "track", "picture"} +ALLOWED_ATTRIBUTES = { + **{tag: set(attrs) for tag, attrs in nh3.ALLOWED_ATTRIBUTES.items()}, + "*": {"class", "title", "id", "lang", "dir"}, + "img": set(nh3.ALLOWED_ATTRIBUTES.get("img", ())) | {"loading", "srcset", "sizes"}, + "video": {"src", "controls", "poster", "width", "height", "loop", "muted", "playsinline", "preload"}, + "audio": {"src", "controls", "loop", "muted", "preload"}, + "source": {"src", "type", "srcset", "media", "sizes"}, + "track": {"src", "kind", "label", "srclang", "default"}, +} + + +def render_markdown(text: str) -> str: + """Note Markdown (or HTML) to sanitized HTML""" + html = markdown.markdown(text, extensions=MARKDOWN_EXTENSIONS) + return nh3.clean(html, tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRIBUTES) + + +# Likes, reposts and bookmarks (v2.5.0, v2.9.0) +REACTION_VERBS = {"like": "Liked", "repost": "Reposted", "bookmark": "Bookmarked"} +REACTION_ICONS = {"like": "โ˜…", "repost": "๐Ÿ”", "bookmark": "๐Ÿ”–"} + # A Markdown ATX heading: up to 6 '#', whitespace, then text TITLE_RE = re.compile(r"#{1,6}[ \t]+(\S.*)$") EXCERPT_LENGTH = 200 @@ -142,6 +169,8 @@ class Note: in_reply_to: Optional[str] = None # v1.6.0: URL being replied to like_of: Optional[str] = None # v2.5.0: URL liked (this note is a like) repost_of: Optional[str] = None # v2.5.0: URL reposted (this note is a repost) + bookmark_of: Optional[str] = None # v2.9.0: URL bookmarked (this note is a bookmark) + content_format: Optional[str] = None # v2.9.0: 'html' if sent as HTML via Micropub _cached_content: Optional[str] = field( default=None, repr=False, compare=False, init=False ) @@ -202,6 +231,8 @@ class Note: in_reply_to=data.get("in_reply_to"), like_of=data.get("like_of"), repost_of=data.get("repost_of"), + bookmark_of=data.get("bookmark_of"), + content_format=data.get("content_format"), ) @property @@ -260,11 +291,8 @@ class Note:

This is my note content...

""" if self._cached_html is None: - # Render markdown to HTML - html = markdown.markdown( - self.content, # This triggers lazy load of content - extensions=MARKDOWN_EXTENSIONS, - ) + # Render markdown to sanitized HTML (v2.9.0) + html = render_markdown(self.content) # triggers lazy load of content # Cache it object.__setattr__(self, "_cached_html", html) @@ -282,7 +310,7 @@ class Note: title, body = split_title(self.content) if title is None: return self.html - return markdown.markdown(body, extensions=MARKDOWN_EXTENSIONS) + return render_markdown(body) @property def has_title(self) -> bool: @@ -386,9 +414,8 @@ class Note: excerpt += "..." if not excerpt and self.reacts_to: - # A like or repost with no comment - verb = "Liked" if self.kind == "like" else "Reposted" - return f"{verb} {self.reacts_to}" + # A like, repost or bookmark with no comment + return f"{REACTION_VERBS[self.kind]} {self.reacts_to}" return excerpt @@ -398,17 +425,19 @@ class Note: @property def kind(self) -> str: - """'like', 'repost' or 'note' (v2.5.0)""" + """'like', 'repost', 'bookmark' (v2.9.0) or 'note' (v2.5.0)""" if self.like_of: return "like" if self.repost_of: return "repost" + if self.bookmark_of: + return "bookmark" return "note" @property def reacts_to(self) -> Optional[str]: - """The URL a like or repost points at""" - return self.like_of or self.repost_of + """The URL a like, repost or bookmark points at""" + return self.like_of or self.repost_of or self.bookmark_of @property def feed_html(self) -> str: @@ -417,9 +446,8 @@ class Note: if not self.reacts_to: return self.html - verb = "Liked" if self.kind == "like" else "Reposted" url = escape(self.reacts_to, quote=True) - line = f'

{"โ˜…" if self.kind == "like" else "๐Ÿ”"} {verb} {url}

' + line = f'

{REACTION_ICONS[self.kind]} {REACTION_VERBS[self.kind]} {url}

' return line + (self.html or "") @property diff --git a/starpunk/notes.py b/starpunk/notes.py index cf80c03..9059432 100644 --- a/starpunk/notes.py +++ b/starpunk/notes.py @@ -133,6 +133,10 @@ def _get_existing_slugs(db) -> set[str]: # Core CRUD Functions +# update_note: leave a field as it is +UNCHANGED = object() + + def create_note( content: str, published: bool = False, @@ -143,6 +147,8 @@ def create_note( like_of: Optional[str] = None, repost_of: Optional[str] = None, has_media: bool = False, + bookmark_of: Optional[str] = None, + content_format: Optional[str] = None, ) -> Note: """ Create a new note @@ -207,10 +213,13 @@ def create_note( """ # 1. VALIDATION (before any changes) # Likes and reposts may have no comment (v2.5.0), photo posts no text (v2.6.5) - if like_of and repost_of: - raise InvalidNoteDataError("like_of", like_of, "A note can't be both a like and a repost") + # Bookmarks too (v2.9.0) + if sum(bool(u) for u in (like_of, repost_of, bookmark_of)) > 1: + raise InvalidNoteDataError( + "like_of", like_of, "A note can only be one of a like, repost or bookmark" + ) content = content or "" - if not content.strip() and not (like_of or repost_of or has_media): + if not content.strip() and not (like_of or repost_of or bookmark_of or has_media): raise InvalidNoteDataError( "content", content, "Content cannot be empty or whitespace-only" ) @@ -271,10 +280,12 @@ def create_note( try: db.execute( """ - INSERT INTO notes (slug, file_path, published, created_at, updated_at, content_hash, in_reply_to, like_of, repost_of) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + INSERT INTO notes (slug, file_path, published, created_at, updated_at, content_hash, + in_reply_to, like_of, repost_of, bookmark_of, content_format) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, - (slug, file_path_rel, published, created_at, updated_at, content_hash, in_reply_to, like_of, repost_of), + (slug, file_path_rel, published, created_at, updated_at, content_hash, + in_reply_to, like_of, repost_of, bookmark_of, content_format), ) db.commit() except Exception as e: @@ -342,7 +353,7 @@ def create_note( note_id=note_id, note_url=note_url, content_html=html_content, - in_reply_to=in_reply_to or like_of or repost_of + in_reply_to=in_reply_to or like_of or repost_of or bookmark_of ) except Exception as e: # Webmention queueing failure should not prevent note creation @@ -350,7 +361,7 @@ def create_note( # 12. FETCH REPLY CONTEXT (v1.6.0 Phase 3) # Fetch context for the post being replied to, liked or reposted - context_url = in_reply_to or like_of or repost_of + context_url = in_reply_to or like_of or repost_of or bookmark_of if context_url: try: from starpunk.reply_context import fetch_reply_context @@ -607,10 +618,13 @@ def update_note( tags: Optional[list[str]] = None, in_reply_to: Optional[str] = None, has_media: bool = False, + content_format=UNCHANGED, ) -> Note: """ Update a note's content and/or published status + content_format (v2.9.0): 'html' or None (Markdown); left alone unless given + Updates note content and/or metadata, maintaining atomic synchronization between file and database. At least one of content or published must be provided. @@ -692,7 +706,7 @@ def update_note( identifier = slug if slug is not None else id raise NoteNotFoundError(identifier) - if empty_content and not (existing_note.like_of or existing_note.repost_of or has_media): + if empty_content and not (existing_note.reacts_to or has_media): raise InvalidNoteDataError( "content", content, "Content cannot be empty or whitespace-only" ) @@ -756,6 +770,10 @@ def update_note( update_fields.append("in_reply_to = ?") params.append(in_reply_to) + if content_format is not UNCHANGED: + update_fields.append("content_format = ?") + params.append(content_format) + # Add WHERE clause parameter if slug is not None: where_clause = "slug = ?" @@ -841,7 +859,7 @@ def update_note( html_content = updated_note.html if updated_note.published else None # Determine in_reply_to (use new value if provided, otherwise existing) - reply_to = (in_reply_to if in_reply_to is not None else updated_note.in_reply_to) or updated_note.like_of or updated_note.repost_of + reply_to = (in_reply_to if in_reply_to is not None else updated_note.in_reply_to) or updated_note.reacts_to # Queue webmentions (best-effort) # Pass old_html for URL diffing (Phase 6) diff --git a/starpunk/routes/indieauth.py b/starpunk/routes/indieauth.py index efb2cc5..0d8d4fa 100644 --- a/starpunk/routes/indieauth.py +++ b/starpunk/routes/indieauth.py @@ -37,7 +37,7 @@ from starpunk.indieauth import ( validate_authorization_request, verify_access_token, ) -from starpunk.micropub import extract_bearer_token +from starpunk.micropub import MicropubError, extract_bearer_token from starpunk.utils import absolute_url_for bp = Blueprint("indieauth", __name__) @@ -215,7 +215,11 @@ def token(): @bp.route("/auth/token", methods=["GET"]) def token_verify(): """Legacy token verification for clients that still call it""" - info = verify_access_token(extract_bearer_token(request)) + try: + token = extract_bearer_token(request) + except MicropubError as e: + return _oauth_error("invalid_request", e.error_description) + info = verify_access_token(token) if not info: return _oauth_error("unauthorized", "Invalid or expired access token", 401) return _json(info) diff --git a/starpunk/routes/micropub.py b/starpunk/routes/micropub.py index 56659bf..f0d5e59 100644 --- a/starpunk/routes/micropub.py +++ b/starpunk/routes/micropub.py @@ -70,7 +70,10 @@ def micropub_endpoint(): } """ # Extract and verify token - token = extract_bearer_token(request) + try: + token = extract_bearer_token(request) + except MicropubError as e: + return error_response(e.error, e.error_description, e.status_code) if not token: return error_response("unauthorized", "No access token provided", 401) @@ -94,7 +97,10 @@ def micropub_endpoint(): try: # Parse request based on content type if "application/json" in content_type: - data = request.get_json() or {} + # Malformed JSON is the client's error, not ours (v2.9.0) + data = request.get_json(silent=True) + if not isinstance(data, dict): + return error_response("invalid_request", "Request body must be a JSON object", 400) action = data.get("action", "create") else: # Form-encoded or multipart (files handled separately) @@ -136,7 +142,10 @@ def media_endpoint(): from starpunk.media import save_media # Extract and verify token - token = extract_bearer_token(request) + try: + token = extract_bearer_token(request) + except MicropubError as e: + return error_response(e.error, e.error_description, e.status_code) if not token: return error_response("unauthorized", "No access token provided", 401) diff --git a/starpunk/syndication.py b/starpunk/syndication.py index 365f78b..8f0afd8 100644 --- a/starpunk/syndication.py +++ b/starpunk/syndication.py @@ -1002,6 +1002,9 @@ def compose_text(note, permalink: str, tags: List[str], max_chars: int, hashtags = f"{hashtags} {h}".strip() tail = f"\n\n{permalink}" + (f"\n\n{hashtags}" if hashtags else "") body = (note.content if body is None else body).strip() + if getattr(note, "bookmark_of", None): + # A bookmark leads with what it bookmarks (v2.9.0) + body = f"๐Ÿ”– {note.bookmark_of}\n\n{body}".strip() if len(body) + len(tail) > max_chars: room = max_chars - len(tail) - 1 body = body[:room].rsplit(" ", 1)[0].rstrip() + "โ€ฆ" diff --git a/templates/admin/edit.html b/templates/admin/edit.html index 6bd5901..16c9ec6 100644 --- a/templates/admin/edit.html +++ b/templates/admin/edit.html @@ -13,7 +13,7 @@
{% if note.reacts_to %} -

{{ 'โ˜… Like of' if note.kind == 'like' else '๐Ÿ” Repost of' }} {{ note.reacts_to }}

+

{{ {'like': 'โ˜… Like of', 'repost': '๐Ÿ” Repost of', 'bookmark': '๐Ÿ”– Bookmark of'}[note.kind] }} {{ note.reacts_to }}

{% endif %}