Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
409b8b31b4 | ||
|
|
b520b8b4d3 | ||
|
|
720a1e6d6f | ||
|
|
c89c2cd5ca | ||
|
|
ab6ff60f5c | ||
|
|
57682165a4 | ||
|
|
8c51b52ed6 | ||
|
|
42c77e34b7 | ||
|
|
438465592c | ||
|
|
c8a7ef309f | ||
|
|
0640730416 | ||
|
|
51f711ea4a | ||
|
|
6a76ee26af | ||
|
|
43ec7cb8be | ||
|
|
4aaa7a8122 | ||
|
|
0cfc9965df | ||
|
|
106495d114 | ||
|
|
3c1e109358 | ||
|
|
d43361d505 | ||
|
|
f924413934 | ||
|
|
4b2ae4c7fc | ||
|
|
d3148063e3 | ||
|
|
39c8259f3f | ||
|
|
1ce88b38a5 | ||
|
|
4ffe36f9d3 | ||
|
|
a551770c76 | ||
|
|
d0bfc80c2e | ||
|
|
d16463ef8a | ||
|
|
4b56f7cd4b | ||
|
|
2474628646 | ||
|
|
0fe341161e | ||
|
|
a62077ab30 | ||
|
|
fade75fd2d | ||
|
|
0dd59fbc18 | ||
|
|
11e09a0d02 | ||
|
|
565adaeecf | ||
|
|
53d10ed271 | ||
|
|
e8c1b03980 | ||
|
|
0d3a5cd519 |
No files matched your search
@@ -63,6 +63,14 @@ GOTOSOCIAL_NAME=GoToSocial
|
||||
# public, unlisted, private (default: public)
|
||||
GOTOSOCIAL_VISIBILITY=public
|
||||
|
||||
# Threads account (v2.6.0). A long-lived user token with threads_basic,
|
||||
# threads_content_publish and threads_delete; StarPunk refreshes it daily and
|
||||
# keeps the current one in DATA_PATH/threads_token.json.
|
||||
# How to get one: docs/design/v2.6.0/2026-10-31-threads-crossposting.md
|
||||
THREADS_ACCESS_TOKEN=
|
||||
# Your Threads username (for the Micropub syndicate-to uid)
|
||||
THREADS_USERNAME=
|
||||
|
||||
# =============================================================================
|
||||
# DATA STORAGE
|
||||
# =============================================================================
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Gitea Actions workflow for StarPunk
|
||||
# Builds and pushes container images on version tags.
|
||||
# Runs the tests, then builds and pushes container images on version tags.
|
||||
#
|
||||
# The runner has no Docker daemon (that would give workflows root on the
|
||||
# server). Images are built by the rootless BuildKit container next to it,
|
||||
@@ -30,6 +30,10 @@ jobs:
|
||||
run: echo "${{ secrets.REGISTRY_TOKEN }}" | docker login "$REGISTRY" -u "${{ github.repository_owner }}" --password-stdin
|
||||
- name: Use the BuildKit socket
|
||||
run: docker buildx create --name ci --driver remote "$BUILDKIT" --use
|
||||
# Builds the Containerfile's test stage, which runs pytest; a failure
|
||||
# stops the workflow before anything is pushed
|
||||
- name: Run tests
|
||||
run: docker buildx build --target test --progress plain -f Containerfile .
|
||||
- name: Build and push
|
||||
run: |
|
||||
image="$REGISTRY/$(echo '${{ github.repository }}' | tr '[:upper:]' '[:lower:]')"
|
||||
|
||||
+247
@@ -7,6 +7,253 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [2.10.0] - 2026-11-03
|
||||
|
||||
### Added
|
||||
|
||||
- **WebSub** ([ADR-071](docs/decisions/ADR-071-websub.md)): readers that support it get new posts, edits and deletions within seconds instead of polling. StarPunk runs its own hub at `/websub` for its own feeds: the home page, `/feed.atom`, `/feed.rss`, `/feed.json`, `/likes` and each tag page. Each advertises the hub and itself (`Link` headers, and links inside the page or feed). Subscriptions are verified with a challenge, leases expire, deliveries carry the full feed, signed with `X-Hub-Signature` when the subscriber gave a secret, and failures are retried for about nine hours. Requests to subscribers go through the same private-address protections as webmentions, and there are limits per subscriber host and in total. Set `WEBSUB_HUB_URL` to use another hub instead; `WEBSUB_ENABLED=false` turns it off
|
||||
- The admin dashboard shows WebSub subscribers per feed
|
||||
|
||||
### Fixed
|
||||
|
||||
- Feeds could be up to five minutes out of date after posting, editing or deleting, and each server process cached its own copy. Any change to a public note now refreshes the feeds everywhere at once
|
||||
- The memory monitor stopped after the first request (the same bug the webmention worker had before v2.6.8)
|
||||
|
||||
### Changed
|
||||
|
||||
- The RSS feed calls itself `/feed.rss` instead of the older `/feed.xml`. Both addresses still work
|
||||
|
||||
## [2.9.0] - 2026-11-03
|
||||
|
||||
Micropub review against the [W3C Recommendation](https://www.w3.org/TR/micropub/).
|
||||
|
||||
### Changed
|
||||
|
||||
- **Micropub posts are only crossposted to the targets the client asks for.** If a client doesn't send `mp-syndicate-to`, nothing is crossposted (before, everything was). Posts made in the admin editor are unchanged
|
||||
- Posts sent with `visibility` private or unlisted are saved as drafts instead of being published (StarPunk has no private posts)
|
||||
- A token with only the `draft` scope can create drafts, and nothing else
|
||||
- `q=source` and `q=category` need a token that can post (`create`, `update`, `delete` or `draft`); before, any token could list every post, drafts included
|
||||
- Sending the access token in both the header and the body is refused (400)
|
||||
|
||||
### Added
|
||||
|
||||
- Bookmarks: Micropub `bookmark-of` is stored and shown ("🔖 Bookmarked …", `u-bookmark-of`), returned by `q=source`, listed in `q=config`, and crossposts lead with the bookmarked link (migration 015)
|
||||
- A post with only a `summary` uses it as the text; a post with only a `name` is a titled post with no body
|
||||
|
||||
### Fixed
|
||||
|
||||
- Quill and other JSON clients put `mp-syndicate-to` and `mp-slug` inside `properties`; they were ignored, so posts crossposted everywhere and slugs were not used
|
||||
- HTML in posts is sanitized: scripts, event handlers, iframes and `javascript:` links are removed when notes are shown. Formatting, images, code highlighting, footnotes and microformats are kept
|
||||
- A photo that can't be downloaded or found made the post without it while reporting success. Create and update now fail with 400 and change nothing
|
||||
- Remote photos are fetched the same way as webmentions: every redirect is checked for private addresses and the connection goes to the checked address
|
||||
- JSON values that weren't lists were read a character at a time (`"category": "solo"` became tags s, o, l). Single values are now treated as a one-item list; `delete` values in updates must be lists
|
||||
- Posts written as HTML are returned by `q=source` as `{"html": …}`, as the spec requires; empty `content` is left out
|
||||
- A client-supplied `published` time is stored as UTC like every other post; a date that can't be read is refused instead of silently replaced with now
|
||||
- Malformed JSON gets 400 instead of 500, and error messages no longer include Python internals
|
||||
- `video` and `audio` (not supported) are refused with 400 instead of being dropped; person tags in `category` are ignored instead of failing the post
|
||||
- An update with no `replace`, `add` or `delete` is refused
|
||||
- `Authorization: bearer …` is accepted (the scheme is case-insensitive), and 401 responses include `WWW-Authenticate`
|
||||
|
||||
## [2.8.1] - 2026-11-03
|
||||
|
||||
### Fixed
|
||||
|
||||
- Starting the server after an upgrade with a database migration could fail: several server processes applied the same migration at once and one stopped with "duplicate column name", taking the server down until it restarted (this happened deploying v2.8.0, for about two seconds). The migration lock was released right after it was taken, and again before each migration was recorded. Migrations now run entirely inside the lock
|
||||
|
||||
## [2.8.0] - 2026-11-03
|
||||
|
||||
Microformats2 review: the markup now matches the [microformats2](https://microformats.org/wiki/microformats2) vocabularies, so IndieWeb readers and tools see what the page shows.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Replies to a note weren't marked up as its comments, so readers and threading tools saw none. They are now `u-comment h-cite`s with the replier's h-card (name, link, photo) and the reply's HTML
|
||||
- Likes, reposts, bookmarks and mentions on a note had no microformats at all. Each is now a `u-like`, `u-repost`, `u-bookmark` or `u-mention` h-cite with its author, including likes past the eight faces shown
|
||||
- Responses link to and are dated by the original post (its own URL and published time) instead of the address that sent the webmention and when it arrived. New database columns hold these (migration 014)
|
||||
- Post times had no time zone (and production showed microseconds), so readers could shift them by hours. Times on pages and in Micropub `q=source` are now UTC with a `Z`, matching the Atom feed
|
||||
- A note starting with a hashtag (`#indieweb is great…`) was treated as a titled article: its text became a heading and the body was empty, and a note starting with a bare `#` line got its slug as a title. Only a Markdown heading (`#` then a space) makes a title now
|
||||
- A webmention from one of a note's own crossposts (for example your GoToSocial copy, sent back through Bridgy) showed up as a response on the note. It is now ignored
|
||||
- Micropub and IndieAuth discovery links disappeared from pages when webmentions were turned off
|
||||
- A new site had no `rel="me"` link to its author, so readers couldn't tell whose site it was; pages now always link `rel="me"` and `rel="author"` to the author's home page
|
||||
- Finding the author's profile could pick someone else's h-card from their home page. It now follows the [representative h-card](https://microformats.org/wiki/representative-h-card-parsing) rules
|
||||
- A webmention's author is found by the [authorship algorithm](https://indieweb.org/authorship-spec), from the entry on the source page that actually responds to the note: an author given as a name or link, and the author of a feed the entry is in, now count
|
||||
|
||||
### Added
|
||||
|
||||
- Tag pages and the likes page have a feed URL and author, like the home page
|
||||
- Note previews have `u-uid`
|
||||
|
||||
## [2.7.2] - 2026-11-03
|
||||
|
||||
### Added
|
||||
|
||||
- When the replies, likes and other responses shown on a note change (a webmention arrives, is updated or removed, or you approve, hide or delete one), the note's webmentions are sent again, as the Webmention spec recommends, so the sites it links to can pick up the conversation. A response that is checked again without changing sends nothing
|
||||
|
||||
### Security
|
||||
|
||||
- Webmention fetches and sends connect to the exact address that passed the private-address check. Before, the hostname was looked up a second time when connecting, so a DNS server that changed its answer could still point StarPunk at a private address. TLS certificates are still checked against the hostname. IPv4 addresses are preferred when a host has both
|
||||
|
||||
## [2.7.1] - 2026-11-03
|
||||
|
||||
### Changed
|
||||
|
||||
- Releases run the full test suite first: CI builds the Containerfile's new `test` stage (same Python image as production) and stops before pushing an image if any test fails
|
||||
|
||||
### Fixed
|
||||
|
||||
- Eight tests that had been failing: the image validation tests still expected bytes (validation has returned an image object since v1.5.0), and a migration test started the app in production mode
|
||||
|
||||
## [2.7.0] - 2026-11-03
|
||||
|
||||
Webmention now follows the [W3C Recommendation](https://www.w3.org/TR/webmention/) throughout.
|
||||
|
||||
### Changed
|
||||
|
||||
- Webmentions are only accepted for published notes. Mentions of the home page or other pages were stored but never shown; they're now refused with 400
|
||||
- Deleted notes answer 410 Gone with a "This post has been deleted" page instead of 404
|
||||
- A received mention that is sent again keeps its moderation: one you hid stays hidden, one you approved stays approved
|
||||
|
||||
### Fixed
|
||||
|
||||
- Sending: endpoint discovery now handles `rel` values with several words (`rel="webmention other"`), any capitalisation, `rel` after other Link parameters, and several Link headers, and takes the first `<link>` or `<a>` in page order
|
||||
- Sending: the endpoint is looked up again every time (the 1–24 hour cache broke the spec's rule for updated posts)
|
||||
- Sending: editing a note notifies every page it mentioned before, including an old reply target and links the edit removed. Deleting or restoring a note notifies them too, so they can remove or restore the mention
|
||||
- Sending: a redirect from the endpoint is no longer treated as success (the POST was turned into a GET); "429 Too Many Requests" is retried
|
||||
- Receiving: the source must link to the exact target URL (in a link, image, or other HTML attribute, any JSON value, or plain text). A longer URL that starts with the target, or the target mentioned in plain text or an HTML comment, no longer counts
|
||||
- Receiving: relative links in the source are resolved, using the URL after redirects
|
||||
- Receiving: the source is fetched with an `Accept` header and decoded in its declared character set
|
||||
- Receiving: if the source can't be reached when a mention is sent again, the existing mention stays up instead of disappearing
|
||||
- Receiving: the target's host is compared without regard to case or a default port, and `#fragment`s are dropped, so one mention isn't stored twice
|
||||
- Mention snippets cut at 500 characters no longer leave HTML tags open, and the "Mentions" list no longer shows raw HTML tags
|
||||
- The `Link: rel="webmention"` header is added alongside other Link headers instead of replacing them
|
||||
- Links at the top level of a code block were still sent webmentions
|
||||
|
||||
### Security
|
||||
|
||||
- Fetches for webmentions check every redirect, not just the first URL, so a public page can't redirect StarPunk to a private address. Every address a hostname resolves to (IPv4 and IPv6) must be public; carrier-grade NAT, `0.0.0.0/8`, multicast, reserved and IPv4-mapped addresses are blocked too. Each fetch has a total time limit and a 1 MB size limit, including endpoint discovery
|
||||
|
||||
## [2.6.9] - 2026-11-03
|
||||
|
||||
### Fixed
|
||||
|
||||
- Received webmentions failed when the sender's profile photo had alt text (Bridgy, Mastodon and most IndieWeb sites do this). The photo is now stored as its URL
|
||||
- Outgoing webmentions were never sent: endpoint discovery crashed on every target that had an endpoint, and the target was then marked as having none. Discovery now works, and network errors during discovery are retried instead of giving up
|
||||
- Each server process ran its own webmention worker, so a webmention could be processed or sent more than once. Only one process handles the queues at a time now; if it stops, another takes over
|
||||
|
||||
### Security
|
||||
|
||||
- A webmention sender could set its author link to a `javascript:` URL, which was shown as a clickable link on the note page. Author links and photos are now only kept if they are http or https URLs
|
||||
|
||||
## [2.6.8] - 2026-11-03
|
||||
|
||||
### Fixed
|
||||
|
||||
- Webmentions were never processed. The background worker checked URLs against this site's settings without the app loaded, so every incoming webmention was marked "error" ("Working outside of application context") and every outgoing one kept failing and retrying. The worker now runs with the app loaded. A received webmention showed up in the admin list but never on the note
|
||||
- The webmention worker also stopped as soon as any request finished, so after startup it often wasn't running at all. It now runs for the life of the server process
|
||||
|
||||
## [2.6.7] - 2026-11-02
|
||||
|
||||
### Fixed
|
||||
|
||||
- Admin pages were misaligned: the notes table ran past the right edge of the header and admin menu, because every page sat in the 40rem reading column and the table couldn't shrink to fit it. Admin pages now use a 60rem frame for the header, menu, content and footer alike, long post previews wrap, and on narrow screens tables scroll sideways instead of spilling out
|
||||
|
||||
## [2.6.6] - 2026-11-02
|
||||
|
||||
### Fixed
|
||||
|
||||
- Crossposts left out photos that were embedded in the post text instead of attached. Some Micropub clients (OutPost, for one) upload a photo to the media endpoint and then put it in `content` as an image rather than sending it as `photo`; both are valid Micropub. Images in the text that are in this site's media library are now sent to GoToSocial, Bluesky and Threads like attached photos, with their alt text, after any attached ones and within each target's photo limit. GoToSocial's text no longer includes the image Markdown. Images hosted on other sites are still left out
|
||||
|
||||
## [2.6.5] - 2026-11-01
|
||||
|
||||
### Fixed
|
||||
|
||||
- A Micropub post that failed partway (for example, the same photo listed twice) answered "Failed to create post" but stayed published without its photos, so a client retrying would post it twice. Photos are now looked up before the post is made, and a post that fails after that is removed again
|
||||
- After such a failure, every request failed with "database is locked" for about 10 seconds: database connections now roll back unfinished writes when they're returned to the pool
|
||||
- The same photo listed twice in one post is shown once instead of failing the post
|
||||
- Replies didn't read as replies to IndieWeb readers and webmention receivers: on note pages `u-in-reply-to` sat inside an unrelated `h-cite`, and the home page didn't mark replies at all, so [Post Type Discovery](https://www.w3.org/TR/post-type-discovery/) saw notes (or photo posts). The reply context is now the entry's `u-in-reply-to h-cite`, and home page entries show "↩ In reply to"
|
||||
- Photos without alt text used "Image" as their description, which IndieWeb readers passed on as if you'd written it. They now use the post's text (up to 150 characters), and fall back to "Image" only when the post has no text
|
||||
- Home page entries always have an `e-content`, so a photo post without text never gets an implied name (which readers would show as its caption)
|
||||
|
||||
### Changed
|
||||
|
||||
- Photo posts no longer need text. Micropub doesn't require `content`, and `q=config` already offered a Photo post type with only `photo`. A Micropub update may also clear the text of a post that keeps its photos. Crossposts of a photo post without text start with its link, and its page is titled "Photo"
|
||||
|
||||
## [2.6.4] - 2026-11-01
|
||||
|
||||
### Fixed
|
||||
|
||||
- Tags keep the order you entered them in. The first tag is the note's main tag: it's the Threads topic, and Bluesky keeps the first tags when space runs out. Before, both used the alphabetically first tag. Public pages still list tags alphabetically
|
||||
- Opening a note in the editor or updating it via Micropub reordered its tags alphabetically on save; the editor and Micropub `q=source` now show, and keep, the entered order
|
||||
|
||||
## [2.6.3] - 2026-11-01
|
||||
|
||||
### Changed
|
||||
|
||||
- **A note's own link gets the preview card** (ADR-070). When a note links somewhere (say, an article), the crosspost's card shows that page instead of the note. Bluesky cards are built from the page's title, description and image; Threads previews it from the text. Notes without links keep their own card
|
||||
|
||||
### Fixed
|
||||
|
||||
- Markdown links (`[text](url)`) lost their URL on Bluesky and Threads. On Bluesky the link text is now clickable; on Threads it's written "text (url)"
|
||||
- Notes with many links failed on Threads (Meta allows 5 per post); extra links now keep just their text
|
||||
|
||||
## [2.6.2] - 2026-11-01
|
||||
|
||||
### Fixed
|
||||
|
||||
- New notes failed their first Threads crosspost with "Invalid Link Attachment" (Meta rejects an explicit link preview for a page it hasn't fetched yet) and only went out on the retry a minute later. StarPunk no longer sends the explicit link preview; Threads still shows one, built from the note's link in the text
|
||||
|
||||
## [2.6.1] - 2026-11-01
|
||||
|
||||
### Fixed
|
||||
|
||||
- Threads crossposts gave up after Meta's publish step returned a one-off `HTTP 400 Fatal`; publish errors are now retried (a retry checks for an existing post first, so nothing is posted twice). Only a bad token or missing permission stops retries
|
||||
- Threads errors now include Meta's error code, subcode, user message and trace id, so the editor shows why a post failed
|
||||
- Wait a moment before the first readiness check on a Threads post, since Meta can report it ready too early
|
||||
|
||||
## [2.6.0] - 2026-11-01
|
||||
|
||||
### Added
|
||||
|
||||
- **Crossposting to Threads** (ADR-069): set `THREADS_ACCESS_TOKEN` (and optionally `THREADS_USERNAME`) to add a Threads target. Text posts get a link preview; 1 image posts as an image, 2–4 as a carousel, with alt text; the note's first tag becomes the post's topic. Shortened to 500 characters. Edits don't follow (the API can't edit); unpublish and delete remove the post. Likes aren't crossposted to Threads; reposts post "🔁 title url" (with the comment, if any)
|
||||
- The Threads token (60-day lifetime) is refreshed automatically once a day and kept in `data/threads_token.json`, so it survives restarts; setting a new token in the config replaces it. Setup steps: `docs/design/v2.6.0/`
|
||||
|
||||
## [2.5.0] - 2026-11-01
|
||||
|
||||
### Added
|
||||
|
||||
- **Likes and reposts** via Micropub `like-of` / `repost-of` (ADR-068); a comment is optional
|
||||
- Likes are kept off the home page and feeds and listed on a new `/likes` page (linked in the header)
|
||||
- Reposts appear in the stream and feeds with a "🔁 Reposted" line
|
||||
- The liked/reposted post's title and author are fetched and shown; it is sent a webmention
|
||||
- GoToSocial: likes/reposts of fediverse posts become native favourites/boosts; reposts of other pages (or with a comment) post "🔁 title url"
|
||||
- Bluesky: likes/reposts of Bluesky posts become native likes/reposts; a repost with a comment is a quote post; reposts of other pages post with a link card
|
||||
- Likes of ordinary web pages are not crossposted; deleting a like/repost undoes it everywhere
|
||||
- Migration 013: `notes.like_of`, `notes.repost_of`
|
||||
|
||||
### Fixed
|
||||
|
||||
- Outgoing webmentions named posts `https://site//note/<slug>` (double slash)
|
||||
|
||||
### Notes
|
||||
|
||||
- The GoToSocial token needs the extra scopes `read:search` and `write:favourites`
|
||||
|
||||
## [2.4.0] - 2026-10-31
|
||||
|
||||
### Added
|
||||
|
||||
- **Micropub update** (`action=update`, JSON): replace/add/delete for `content`, `name`, `category`, `photo`, `in-reply-to`, `post-status`; edits follow to crossposts (ADR-067)
|
||||
- **Micropub delete and undelete**: soft delete, restorable; crossposts are removed and re-posted
|
||||
- **Drafts** via `post-status: draft`; **replies** via `in-reply-to`
|
||||
- Photos uploaded in a multipart create request; photos hosted elsewhere are downloaded (public addresses only, 20 MB cap)
|
||||
- `q=source` without `url` lists recent posts (drafts included, `limit`/`offset`); `properties[]` filter; returns photos, syndication links, reply and post status
|
||||
- `q=category` (tag suggestions, optional `filter`); `q=config` lists supported queries and post types
|
||||
- Media endpoint accepts the `media` scope
|
||||
|
||||
### Fixed
|
||||
|
||||
- Media endpoint was advertised at `/micropub/media` but served at `/media` (404 for clients); now both work
|
||||
- Unsupported Micropub types (e.g. `h=event`) were silently stored as notes; now rejected
|
||||
|
||||
## [2.3.3] - 2026-10-31
|
||||
|
||||
### Fixed
|
||||
|
||||
+21
-1
@@ -23,7 +23,27 @@ RUN uv venv /opt/venv && \
|
||||
uv pip install --no-cache -r requirements.txt
|
||||
|
||||
# ============================================================================
|
||||
# Runtime Stage - Minimal production image
|
||||
# Test Stage - Runs the test suite (CI builds it with --target test)
|
||||
# ============================================================================
|
||||
FROM python:3.11-slim AS test
|
||||
|
||||
COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
|
||||
COPY --from=builder /opt/venv /opt/venv
|
||||
RUN uv pip install --no-cache --python /opt/venv/bin/python pytest pytest-mock
|
||||
|
||||
ENV PATH="/opt/venv/bin:$PATH" \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
SESSION_SECRET=ci-test-secret \
|
||||
SITE_URL=http://localhost:5000 \
|
||||
ADMIN_ME=https://test.example.com
|
||||
|
||||
WORKDIR /app
|
||||
COPY . .
|
||||
RUN python -m pytest -q -p no:cacheprovider tests
|
||||
|
||||
# ============================================================================
|
||||
# Runtime Stage - Minimal production image (default target: keep it last)
|
||||
# ============================================================================
|
||||
FROM python:3.11-slim
|
||||
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
# ADR-067: Micropub update, delete, drafts and replies
|
||||
|
||||
## Status
|
||||
|
||||
Accepted (v2.4.0, 2026-10-31)
|
||||
|
||||
## Context
|
||||
|
||||
The Micropub endpoint could only create published notes. Apps could not edit
|
||||
or delete posts, drafts and replies were dropped, and photos sent with the
|
||||
post or hosted elsewhere were lost.
|
||||
|
||||
## Decision
|
||||
|
||||
1. **Update** (JSON only, `update` scope). Supported properties are the ones a
|
||||
note can store: `content`, `name` (the `# Heading` first line),
|
||||
`category`, `photo`, `in-reply-to`, `post-status`. Anything else is a 400,
|
||||
not silently ignored. Order: delete, then replace, then add (`add` only for
|
||||
`category` and `photo`). Success is 204.
|
||||
2. **Delete is soft** (`delete` scope) so `undelete` can restore the note and
|
||||
its file from `.trash/`. Crossposts are removed on delete and posted again
|
||||
on undelete.
|
||||
3. **Drafts**: `post-status: draft` creates an unpublished note; crossposts
|
||||
wait until it is published.
|
||||
4. **Photos**: multipart uploads are saved like media-endpoint uploads; remote
|
||||
URLs are downloaded only from public IP addresses, up to 20 MB, max 4.
|
||||
5. **Queries**: `q=source` returns everything create accepts (round-trip);
|
||||
without `url` it lists recent posts, drafts included, for editing apps.
|
||||
`q=category` lists tags by use.
|
||||
|
||||
## Consequences
|
||||
|
||||
- `published` (backdating) can be set on create but not changed by update.
|
||||
- Likes, reposts and bookmarks remain unsupported (round 2).
|
||||
@@ -0,0 +1,32 @@
|
||||
# ADR-068: Likes and reposts
|
||||
|
||||
## Status
|
||||
|
||||
Accepted (v2.5.0, 2026-11-01). Bookmarks were considered and declined.
|
||||
|
||||
## Decision
|
||||
|
||||
1. **Storage.** `notes.like_of` / `notes.repost_of` (migration 013). The note's
|
||||
content is an optional comment and may be empty.
|
||||
2. **Display.** Likes stay off the home page and feeds and are listed on
|
||||
`/likes`. Reposts appear in the stream and feeds. Both show the target's
|
||||
fetched title and author (the reply-context cache) and carry
|
||||
`u-like-of` / `u-repost-of`. The target is sent a webmention.
|
||||
3. **Crossposting, per target.**
|
||||
|
||||
| | Target is a post on that network | Anything else |
|
||||
|---|---|---|
|
||||
| Like | native favourite / like | nothing |
|
||||
| Repost, no comment | native boost / repost | "🔁 title url" post |
|
||||
| Repost, with comment | Bluesky quote post; GoToSocial status | comment + "🔁 title url" post |
|
||||
|
||||
GoToSocial finds fediverse posts with `/api/v2/search?resolve=true`
|
||||
(needs `read:search`; favourites need `write:favourites`). Bluesky
|
||||
recognises `bsky.app/profile/<actor>/post/<rkey>` URLs. Native actions
|
||||
are stored as `favourite:<id>` / `reblog:<id>` or the like/repost record
|
||||
URI, and deleting the note undoes them.
|
||||
|
||||
## Consequences
|
||||
|
||||
- Editing a like/repost doesn't change a native favourite/boost.
|
||||
- A repost with a comment can't be a GoToSocial boost (no quote posts there).
|
||||
@@ -0,0 +1,57 @@
|
||||
# ADR-069: Crossposting to Threads
|
||||
|
||||
## Status
|
||||
|
||||
Accepted (v2.6.0, 2026-10-31)
|
||||
|
||||
## Context
|
||||
|
||||
ADR-065 and ADR-066 crosspost notes to GoToSocial and Bluesky. Threads
|
||||
(Meta) has a publishing API, so it can be a third target on the same loop.
|
||||
|
||||
## Decision
|
||||
|
||||
1. **Third client, same loop.** `ThreadsClient` is target id `threads`
|
||||
behind the `syndications` table and reconcile loop from ADR-065.
|
||||
2. **Auth.** A long-lived Threads user token (`THREADS_ACCESS_TOKEN`) with
|
||||
`threads_basic`, `threads_content_publish` and `threads_delete`. These
|
||||
tokens expire after 60 days, so StarPunk refreshes the token about once a
|
||||
day (`th_refresh_token`) from the worker loop, even when nothing is being
|
||||
posted. The current token is kept in `DATA_PATH/threads_token.json`
|
||||
(mode 0600) with a hash of the configured token. A different configured
|
||||
token replaces the saved one. A file lock stops gunicorn workers from
|
||||
refreshing at the same time. The file avoids a migration for one value.
|
||||
3. **Posting.** Create a container, poll its status until `FINISHED`, then
|
||||
`threads_publish`. Text-only notes are `TEXT` posts. Threads previews
|
||||
the first link in the text (the permalink) by itself. An explicit
|
||||
`link_attachment` is not sent, because Meta rejects one for a page it
|
||||
hasn't fetched yet ("Invalid Link Attachment", subcode 4279047), which
|
||||
is always the case for a new note (v2.6.2). One image is an `IMAGE` post and 2–4 images are a
|
||||
`CAROUSEL`. Meta fetches images from their public `/media/` URL. Only
|
||||
JPEG and PNG up to 8 MB are allowed, so GIF and WebP images are left off.
|
||||
Alt text goes in `alt_text`.
|
||||
4. **Content.** Plain text (as for Bluesky), then the permalink, shortened
|
||||
to 500 characters. Non-ASCII characters are counted as UTF-8 bytes,
|
||||
because Threads counts emoji that way. Threads allows one tag per post,
|
||||
so the note's first tag is sent as `topic_tag` and no hashtags are added
|
||||
to the text.
|
||||
5. **Edits don't follow.** The API has no way to edit a post. As with
|
||||
Bluesky, the post stays as first sent, and unpublishing, unticking the
|
||||
target or deleting the note removes it.
|
||||
6. **Likes and reposts (ADR-068).** The API can't like or repost outside
|
||||
content, so likes are not crossposted. Reposts are text posts with
|
||||
"🔁 title" and the URL, plus the comment if there is one, and the
|
||||
reposted page as the link preview.
|
||||
7. **No double posts.** A retried create first looks for a recent post that
|
||||
already contains the permalink.
|
||||
|
||||
## Consequences
|
||||
|
||||
- Meta limits accounts to 250 posts and 100 deletes per day. Rate-limit
|
||||
errors (Graph codes 4, 17, 32, 613) are retried.
|
||||
- If StarPunk is down for more than 60 days, the token expires and a new
|
||||
one is needed.
|
||||
- Getting the first token requires a Meta developer app with the Threads
|
||||
use case. See the setup notes in `docs/design/v2.6.0/`.
|
||||
- Notes with more than five links fail on Threads (a Meta limit). The
|
||||
error is shown in the editor.
|
||||
@@ -0,0 +1,44 @@
|
||||
# ADR-070: Links and link cards in crossposts
|
||||
|
||||
## Status
|
||||
|
||||
Accepted (v2.6.3, 2026-11-01)
|
||||
|
||||
## Context
|
||||
|
||||
Every crosspost carries the note's permalink. Notes often link elsewhere
|
||||
too, usually to the article being discussed. Before this change:
|
||||
|
||||
- Bluesky and Threads get plain text, and a Markdown link `[text](url)`
|
||||
lost its URL, so readers there couldn't follow it.
|
||||
- The Bluesky card always showed the note, and the Threads preview showed
|
||||
whichever link came first.
|
||||
- Threads rejects posts with more than 5 different links.
|
||||
|
||||
## Decision
|
||||
|
||||
1. **The card shows what the note links to.** If the note has its own link
|
||||
(a Markdown link, `<url>` or a bare URL), the first one gets the
|
||||
preview card. A note without links gets a card for itself, as before.
|
||||
Images still take the place of a card on Bluesky.
|
||||
2. **Bluesky.** The card is built by StarPunk: it fetches the page's Open
|
||||
Graph, Twitter or plain HTML title, description and image, and uploads
|
||||
the image (shrunk to under 1 MB) as the thumbnail. If the page can't
|
||||
be fetched, the card has just the URL. Markdown link text becomes a
|
||||
clickable link facet, so no characters are spent on the URL.
|
||||
3. **Threads.** It has no clickable link text, so Markdown links are
|
||||
written "text (url)". Threads previews the first link in the text, which
|
||||
is now the linked page. To stay within Meta's 5-link limit, after 4
|
||||
different links in the body a Markdown link keeps only its text.
|
||||
4. **GoToSocial** already gets Markdown, so its links are unchanged.
|
||||
Receiving servers build their own cards.
|
||||
5. **Fetching is careful.** Only http(s) URLs on public addresses are
|
||||
fetched, and this is checked again at every redirect. At most 512 KB of
|
||||
a page and 5 MB of an image are read. A failed fetch never fails the
|
||||
post.
|
||||
|
||||
## Consequences
|
||||
|
||||
- One or two extra requests to the linked site when a note is crossposted
|
||||
to Bluesky.
|
||||
- Reposts (ADR-068) are unchanged: their card is the reposted page.
|
||||
@@ -0,0 +1,166 @@
|
||||
# ADR-071: WebSub publishing with a built-in hub
|
||||
|
||||
## Status
|
||||
|
||||
Accepted (v2.10.0, 2026-11-03)
|
||||
|
||||
## Context
|
||||
|
||||
Readers poll StarPunk's feeds. WebSub (W3C Recommendation,
|
||||
https://www.w3.org/TR/websub/) lets readers that support it (Monocle,
|
||||
Aperture, Inoreader, NewsBlur, Miniflux and others) subscribe once and
|
||||
have new content pushed to them within seconds.
|
||||
|
||||
WebSub has three roles. StarPunk is the **publisher**. Someone has to run
|
||||
the **hub** that keeps subscriptions and delivers content. Public hubs
|
||||
exist (switchboard.p3k.io, pubsubhubbub.appspot.com), but they are a
|
||||
third-party dependency and some have shut down. The rest of this site is
|
||||
self-hosted.
|
||||
|
||||
ADR-014 deferred WebSub for v1 as "complexity, external dependency".
|
||||
|
||||
Two existing problems matter here:
|
||||
|
||||
- Feeds are cached for `FEED_CACHE_SECONDS` (5 minutes) and nothing clears
|
||||
the cache when a note changes. A hub fetching right after a change would
|
||||
get stale content. Regular readers are also up to 5 minutes behind.
|
||||
- The RSS feed is served at `/feed.rss` and `/feed.xml` (legacy alias), and
|
||||
calls itself `/feed.xml`.
|
||||
|
||||
## Decision
|
||||
|
||||
StarPunk runs its own hub, **for its own topics only**, at `/websub`.
|
||||
`WEBSUB_HUB_URL` can point at an external hub instead; then StarPunk
|
||||
advertises that hub, pings it on changes, and its own `/websub` is off.
|
||||
|
||||
### 1. Topics
|
||||
|
||||
Absolute URLs under `SITE_URL`:
|
||||
|
||||
| Topic | Content type |
|
||||
|---|---|
|
||||
| `/` (h-feed) | text/html |
|
||||
| `/feed.atom` | application/atom+xml |
|
||||
| `/feed.rss` | application/rss+xml |
|
||||
| `/feed.json` | application/feed+json |
|
||||
| `/likes` | text/html |
|
||||
| `/tag/<name>` | text/html |
|
||||
|
||||
`/feed.xml` and the negotiated `/feed` advertise the topic of the format
|
||||
they serve as `rel=self` (`/feed.rss`, `/feed.atom`, `/feed.json`). The
|
||||
RSS feed's own self link changes from `/feed.xml` to `/feed.rss`.
|
||||
|
||||
### 2. Discovery (publisher, spec 4)
|
||||
|
||||
Every topic response carries `Link: <hub>; rel="hub"` and exactly one
|
||||
`Link: <topic>; rel="self"`. Embedded links as well: `<link rel="hub">` and
|
||||
`<link rel="self">` in the HTML head of topic pages, `<link rel="hub">`
|
||||
next to the existing self link in Atom, `<atom:link rel="hub">` in RSS, and
|
||||
`"hubs": [{"type": "WebSub", "url": ...}]` in JSON Feed.
|
||||
|
||||
### 3. What changed, and when
|
||||
|
||||
`websub.note_changed(note, old_tags=())` is called wherever
|
||||
`syndication.note_changed` is, plus on create. It:
|
||||
|
||||
1. clears the feed cache (all formats), and
|
||||
2. if the note is or was public, queues delivery for `/`, the three
|
||||
feeds, `/likes` if it is a like, and each `/tag/<name>` for its tags
|
||||
before and after the change.
|
||||
|
||||
Drafts that stay drafts change nothing.
|
||||
|
||||
### 4. Hub endpoint (spec 5)
|
||||
|
||||
`POST /websub`, `application/x-www-form-urlencoded`:
|
||||
|
||||
- `hub.mode` subscribe or unsubscribe; `hub.topic` must be one of the
|
||||
topics above (after normalising), else 400 with a plain-text reason;
|
||||
`hub.callback` must be an absolute http(s) URL that passes the SSRF
|
||||
check; `hub.secret` under 200 bytes; `hub.lease_seconds` clamped to
|
||||
1 day to 30 days (default 10 days). Unknown parameters are ignored.
|
||||
- Response: 202 Accepted, independent of what verification later finds.
|
||||
- Limits: the Webmention rate limiter (per IP); at most 25 subscriptions
|
||||
per callback host and 1,000 in total; beyond that, 429.
|
||||
|
||||
### 5. Verification of intent (spec 5.3)
|
||||
|
||||
A background worker sends `GET <callback>` with the callback's query
|
||||
string kept and `hub.mode`, `hub.topic`, `hub.challenge` (random, 32
|
||||
bytes) and `hub.lease_seconds` appended. Success is a 2xx with the body
|
||||
equal to the challenge; anything else, including redirects, fails and
|
||||
leaves the subscription state unchanged. On success a subscribe becomes
|
||||
active with `expires_at = now + lease`, and an unsubscribe deletes it.
|
||||
A re-subscribe replaces the secret and lease only once verified.
|
||||
|
||||
### 6. Content distribution (spec 6, 7)
|
||||
|
||||
For each change, one delivery per active, unexpired subscription to the
|
||||
topic. The worker renders the topic inside the app (no HTTP round trip
|
||||
through Caddy), so subscribers get exactly what the public URL serves,
|
||||
then `POST`s it to the callback with:
|
||||
|
||||
- `Content-Type` of the topic,
|
||||
- `Link` headers with `rel=hub` and `rel=self`,
|
||||
- `X-Hub-Signature: sha256=<HMAC-SHA256(secret, body)>` if a secret was
|
||||
given.
|
||||
|
||||
A 2xx is success. 410 Gone removes the subscription. Anything else is
|
||||
retried after 1 min, 5 min, 30 min, 2 h and 6 h, then dropped; a
|
||||
subscription whose deliveries fail for 3 days is removed. Only the latest
|
||||
content is sent: a queued delivery for a topic is replaced by a newer one.
|
||||
|
||||
### 7. Outbound safety
|
||||
|
||||
Callbacks are arbitrary URLs, so verification and delivery go through
|
||||
`webmention_ssrf` (`safe_get`/`safe_post`): public addresses only, the
|
||||
connection pinned to the checked address, no redirects, 10 s time limit,
|
||||
1 MB response cap.
|
||||
|
||||
### 8. Data (migration 016)
|
||||
|
||||
- `websub_subscriptions(id, topic, callback, secret, lease_seconds,
|
||||
state ['pending','active'], expires_at, created_at, verified_at,
|
||||
last_success_at, last_error)`, unique on `(topic, callback)`.
|
||||
- `websub_intents(id, topic, callback, mode, secret, lease_seconds,
|
||||
attempts, next_attempt_at, created_at)`: verifications to do.
|
||||
- `websub_deliveries(id, subscription_id, attempts, next_attempt_at,
|
||||
last_error, created_at)`, unique on `subscription_id`.
|
||||
|
||||
### 9. Worker
|
||||
|
||||
The same pattern as the Webmention worker: one thread per gunicorn
|
||||
process, a file lock so only one process works the queues, app context
|
||||
around each poll. Expired subscriptions are pruned once a day.
|
||||
|
||||
### 10. Admin
|
||||
|
||||
The admin dashboard shows active subscribers per topic and failing
|
||||
deliveries.
|
||||
|
||||
### 11. Configuration
|
||||
|
||||
`WEBSUB_ENABLED` (default true), `WEBSUB_HUB_URL` (default: built-in),
|
||||
`WEBSUB_MAX_SUBSCRIPTIONS` (1000), `WEBSUB_MAX_PER_HOST` (25).
|
||||
|
||||
## Consequences
|
||||
|
||||
- Readers that support WebSub get posts and edits within seconds.
|
||||
- Feeds are fresh right after a change for every reader, because the cache
|
||||
is cleared.
|
||||
- StarPunk sends HTTP requests to subscriber-chosen URLs. The SSRF
|
||||
protections, per-host and total limits, and rate limiting bound that.
|
||||
- websub.rocks' hub tests use websub.rocks' own topics, which this hub
|
||||
refuses by design. Testing uses unit tests of each spec section and a
|
||||
real subscription from a reader after deploy.
|
||||
- The RSS self link changes to `/feed.rss`. Readers keep whichever URL
|
||||
they subscribed to; both keep working.
|
||||
|
||||
## Alternatives considered
|
||||
|
||||
- **Public hub only.** Least code, but a third-party dependency for a
|
||||
self-hosted site. Still available through `WEBSUB_HUB_URL`.
|
||||
- **A separate hub service** (another container). More to deploy and
|
||||
keep updated, and few maintained hubs exist.
|
||||
- **Accepting any topic** (a general-purpose hub). Not needed, and it
|
||||
would let the hub be used to fetch and push arbitrary URLs.
|
||||
@@ -0,0 +1,62 @@
|
||||
# v2.6.0: Crossposting to Threads
|
||||
|
||||
Implements ADR-069.
|
||||
|
||||
## What changed
|
||||
|
||||
- `starpunk/syndication.py`: `ThreadsClient` (target `threads`); media
|
||||
entries now carry their public `url`; `process_due()` calls each client's
|
||||
optional `maintain()` hook, which Threads uses to refresh its token.
|
||||
- `starpunk/config.py`: `THREADS_ACCESS_TOKEN`, `THREADS_USERNAME`,
|
||||
`THREADS_NAME`, `THREADS_MAX_CHARS` (500), `THREADS_MAX_MEDIA` (4).
|
||||
- `tests/test_syndication_threads.py`: a fake Graph API covering text,
|
||||
image and carousel posts; container polling; edits not following; deletes
|
||||
(including posts already deleted by hand); a lost publish reply;
|
||||
permanent and rate-limit errors; and token refresh, persistence and
|
||||
replacement.
|
||||
|
||||
## Configuration
|
||||
|
||||
| Variable | Meaning |
|
||||
|---|---|
|
||||
| `THREADS_ACCESS_TOKEN` | Long-lived Threads user token. Enables the target. |
|
||||
| `THREADS_USERNAME` | Optional. Used for the Micropub `syndicate-to` uid (`https://www.threads.com/@name`) and for fallback links. |
|
||||
| `THREADS_NAME` | Label in the editor (default `Threads`) |
|
||||
|
||||
After the first daily refresh, the working token lives in
|
||||
`DATA_PATH/threads_token.json`. The configured value only seeds it.
|
||||
Changing `THREADS_ACCESS_TOKEN` switches to the new token.
|
||||
|
||||
## Getting a token (one time)
|
||||
|
||||
1. At <https://developers.facebook.com/apps>, create an app with the
|
||||
**Access the Threads API** use case.
|
||||
2. Under the use case's permissions, add `threads_basic`,
|
||||
`threads_content_publish` and `threads_delete`.
|
||||
3. Under **App roles → Roles**, add your Threads account as a Threads
|
||||
tester. Accept the invite in Threads (Settings → Account → Website
|
||||
permissions → Invites). The app can stay in development mode.
|
||||
4. In the use case settings, add `https://<site>/` as a redirect URI. Then
|
||||
open:
|
||||
|
||||
https://threads.net/oauth/authorize?client_id=APP_ID&redirect_uri=https://<site>/&scope=threads_basic,threads_content_publish,threads_delete&response_type=code
|
||||
|
||||
After you approve, the browser lands on `https://<site>/?code=...`. Copy
|
||||
the code (drop a trailing `#_`).
|
||||
5. Exchange the code for a short-lived token, then exchange that for a
|
||||
long-lived one:
|
||||
|
||||
curl -s -X POST https://graph.threads.net/oauth/access_token \
|
||||
-F client_id=APP_ID -F client_secret=APP_SECRET \
|
||||
-F grant_type=authorization_code -F redirect_uri=https://<site>/ -F code=CODE
|
||||
curl -s "https://graph.threads.net/access_token?grant_type=th_exchange_token&client_secret=APP_SECRET&access_token=SHORT_TOKEN"
|
||||
|
||||
6. Set `THREADS_ACCESS_TOKEN` to the long-lived `access_token`.
|
||||
|
||||
The app secret is only needed for these steps. StarPunk doesn't use it.
|
||||
|
||||
## Not done
|
||||
|
||||
- Video, GIF and WebP images (Threads takes video by URL. StarPunk only
|
||||
stores still images, and Threads rejects GIF and WebP).
|
||||
- Edits (no API for them).
|
||||
@@ -0,0 +1,89 @@
|
||||
# v2.6.5: Micropub media fixes
|
||||
|
||||
Found while adding media uploads to tuipub, a Micropub client written to the
|
||||
spec rather than to StarPunk.
|
||||
|
||||
## Problems
|
||||
|
||||
1. **Failed creates left a post behind.** `handle_create` made the note, then
|
||||
attached photos. Listing one photo twice broke the `note_media` unique key,
|
||||
so the client got a 500 while the note stayed published without photos.
|
||||
A retry would post it twice.
|
||||
2. **Then the database stayed locked.** `attach_media_to_note` didn't roll
|
||||
back, and `ConnectionPool.return_connection` pooled the connection with
|
||||
its write transaction still open. Other requests got "database is locked"
|
||||
until the 10 s timeout.
|
||||
3. **Photo posts needed text.** The Micropub spec doesn't require `content`,
|
||||
and `q=config` advertised a `photo` post type with only `photo`, yet
|
||||
creates without content got 400 "Content is required".
|
||||
|
||||
## Changes
|
||||
|
||||
- `database/pool.py`: `return_connection` rolls back an open transaction
|
||||
(or closes the connection if that fails).
|
||||
- `media.py`: `attach_media_to_note` rolls back on error.
|
||||
- `micropub.py`:
|
||||
- `handle_create` resolves photos (by media URL, downloading remote ones)
|
||||
before `create_note`. If attaching or crosspost selection fails, it
|
||||
hard-deletes the note before answering 500.
|
||||
- `_resolve_photos` skips repeated media.
|
||||
- Content is optional when photos resolve. Content-less posts whose photos
|
||||
all fail still get a 400 from `create_note`.
|
||||
- `handle_update` allows `replace: {content: [""]}` and `delete: ["content"]`
|
||||
while the note keeps photos (or is a like or repost). Otherwise it answers
|
||||
400 "A post needs text or a photo".
|
||||
- `notes.py`: `create_note` and `update_note` take `has_media`, so
|
||||
empty content is allowed for photo posts.
|
||||
- `syndication.py`: `compose_text` drops the leading blank line when there's
|
||||
no body.
|
||||
|
||||
## Post types (IndieWeb standards check)
|
||||
|
||||
Post Type Discovery (W3C Note) has no separate "photo" type to store. A post
|
||||
with a valid `photo` is a photo post, unless it's first a repost, like or
|
||||
reply (step order: repost, like, reply, video, photo). Otherwise it's a note,
|
||||
or an article when its `name` isn't a prefix of its content. h-entry treats
|
||||
`u-photo` as the primary content, with `name` as a caption. mf2 implies a
|
||||
`name` only when an entry has no other `p-*`/`e-*` properties and no nested
|
||||
microformats.
|
||||
|
||||
StarPunk already stores photo posts as notes with media and renders
|
||||
`u-photo` outside `e-content`, which is what the standards ask for. Checking
|
||||
the rendered pages with mf2py found:
|
||||
|
||||
- **Replies weren't replies.** In `note.html`, `u-in-reply-to` was inside a
|
||||
bare `h-cite`, so it was the h-cite's property, not the entry's.
|
||||
`note_preview.html` had no reply markup at all. Fixed: the blockquote is
|
||||
`u-in-reply-to h-cite` (with `u-url` inside), and previews show
|
||||
"↩ In reply to" with `u-in-reply-to`.
|
||||
- **Implied names were possible.** Previews left out `e-content` for posts
|
||||
without text. They worked only because the author h-card blocked an
|
||||
implied name. Previews now always emit `e-content`.
|
||||
- Page title for text-free posts follows PTD order: Repost, Like, Reply,
|
||||
Photo.
|
||||
|
||||
Photos without a caption take the post's text as alt (`partials/media.html`
|
||||
`fallback_alt`) instead of the made-up "Image". The Micropub `q=source`
|
||||
still reports the stored caption (`""`).
|
||||
|
||||
`tests/test_post_types.py` parses note pages and the home page with mf2py and
|
||||
runs PTD for notes, articles, photo posts (no text, caption, title), replies
|
||||
(with and without a photo) and reposts.
|
||||
|
||||
## Not changed (noted for later)
|
||||
|
||||
- Text-free photos without alt text still render `alt="Image"`. Captioned
|
||||
photos use their caption, and others now use the post's text (`display_media`
|
||||
`fallback_alt`, the excerpt cut to 150 characters).
|
||||
|
||||
- More than 4 photos are still cut to 4 without an error (ADR-057).
|
||||
- `video` and `audio` are ignored. This is allowed by the spec, and they
|
||||
aren't in `q=config` post-types.
|
||||
- Media endpoint errors for non-images expose Python text
|
||||
("cannot identify image file <_io.BytesIO …>").
|
||||
|
||||
## Tests
|
||||
|
||||
`tests/test_micropub_v265.py`. The full suite runs with
|
||||
`SESSION_SECRET=test`. 8 tests in `test_media_upload.py` and
|
||||
`test_migration_race_condition.py` already fail on main, and still fail here.
|
||||
@@ -0,0 +1,37 @@
|
||||
# v2.6.8: Webmention worker app context
|
||||
|
||||
## Problem
|
||||
|
||||
Production webmention #1 (Bridgy, reply from Mastodon to note 20261101162512)
|
||||
was stored with `validation_status = 'error'`:
|
||||
"Working outside of application context". Both outgoing queue items had the
|
||||
same error and were on their last retry.
|
||||
|
||||
## Causes
|
||||
|
||||
1. `webmention_ssrf.safe_resolve()` reads `current_app.config['DEV_MODE']`.
|
||||
`WebmentionWorker` is a plain thread with no Flask app context, so every
|
||||
`validate_url_for_fetch()` call raised `RuntimeError`, for both
|
||||
validation (incoming) and endpoint discovery and sending (outgoing).
|
||||
2. `create_app()` registered a `teardown_appcontext` handler that called
|
||||
`webmention_worker.stop()`. Teardown runs whenever any app context ends
|
||||
(every request), so the worker stopped soon after startup.
|
||||
|
||||
## Fix
|
||||
|
||||
- `WebmentionWorker` takes the Flask app and runs each poll inside
|
||||
`app.app_context()`, as `SyndicationWorker` already does.
|
||||
- The teardown handler is removed. The worker is a daemon thread and ends with
|
||||
the process.
|
||||
|
||||
## Tests
|
||||
|
||||
`tests/test_webmention_worker.py`: an outgoing send completes through the
|
||||
worker thread, and the worker is still alive after an app context and a
|
||||
request end. Both fail without the fix.
|
||||
|
||||
## Production data
|
||||
|
||||
After deploying, reset webmention 1 to `validation_status = 'pending'` and the
|
||||
two queue items to `retry_count = 0, next_retry_at = NULL` so the worker
|
||||
processes them again.
|
||||
@@ -0,0 +1,36 @@
|
||||
# v2.6.9: Webmention processing fixes
|
||||
|
||||
Found after v2.6.8 let the worker run in production, and by a review of the
|
||||
implementation against the W3C Webmention Recommendation.
|
||||
|
||||
## Fixes
|
||||
|
||||
1. **Author photo with alt text** (`webmention_processor.extract_author`).
|
||||
mf2py returns `u-photo` with an `alt` attribute as
|
||||
`{'value': url, 'alt': ...}`; the dict was bound to SQLite and the update
|
||||
failed. `_first_str()` reduces mf2 values to their `value`.
|
||||
2. **Endpoint discovery** (`WebmentionWorker._discover_endpoint`).
|
||||
`response.url` is an `httpx.URL`; `urljoin(str, URL)` raised `TypeError`,
|
||||
which was caught and returned as "no endpoint", so every target was marked
|
||||
`no_endpoint`. Pass `str(response.url)`. Fetch errors now propagate so the
|
||||
caller schedules a retry, instead of being treated as "no endpoint".
|
||||
3. **Duplicate processing.** gunicorn runs 4 processes, each with a worker,
|
||||
and queue items were claimed by SELECT then UPDATE. The worker now takes a
|
||||
non-blocking `flock` on `<data>/webmention-worker.lock` before processing;
|
||||
others wait and retry each poll, so one takes over if the holder dies.
|
||||
Orphan cleanup runs when the lock is acquired.
|
||||
4. **XSS (spec 4.1).** `author_url` and `author_photo` are kept only if they
|
||||
are absolute http(s) URLs; otherwise the source URL / no photo is used.
|
||||
|
||||
## Tests
|
||||
|
||||
`tests/test_webmention_worker.py`: the send test uses a real `httpx.Response`
|
||||
(the old `MagicMock` with a string URL hid fix 2), plus tests for the lock,
|
||||
an alt-text photo and a `javascript:` author URL. All fail on v2.6.8.
|
||||
|
||||
## Remaining spec gaps
|
||||
|
||||
Tracked separately: rel-token and document-order endpoint discovery, exact
|
||||
target match during verification, Accept header, re-sending on delete, 410
|
||||
for deleted notes, SSRF re-validation on redirects, and the others listed in
|
||||
the compliance review.
|
||||
@@ -0,0 +1,51 @@
|
||||
# v2.7.0: Webmention W3C compliance
|
||||
|
||||
Review of StarPunk's Webmention support against the W3C Recommendation
|
||||
(https://www.w3.org/TR/webmention/, 12 January 2017), and the changes made.
|
||||
v2.6.8 and v2.6.9 fixed the bugs that stopped webmentions working at all.
|
||||
|
||||
## Sending
|
||||
|
||||
| Section | Requirement | Change |
|
||||
|---|---|---|
|
||||
| 3.1.2 | Link header with rel=webmention first (MUST) | RFC 8288 parsing: several links per header, several headers, `rel` in any position, token list, case-insensitive |
|
||||
| 3.1.2 | Then first `<link>`/`<a>` rel=webmention in document order (MUST) | Single pass over `link` and `a` in document order; rel token match; elements without `href` skipped |
|
||||
| 3.1.2 | Relative URL resolved against target; empty href is the target; query kept (MUST) | `urljoin` against the final URL after redirects |
|
||||
| 3.1.2 | Follow redirects | `safe_get` follows up to 10, checking each hop |
|
||||
| 3.1.2 | User-Agent with "Webmention" (MAY) | `StarPunk/<version> Webmention (+site)` |
|
||||
| 3.1.3 | Form-encoded POST; any 2xx success (MUST) | Unchanged. Redirects not followed (httpx turned 301/302 POST into GET); 429 retried |
|
||||
| 3.1.4 | Re-discover endpoint on update (MUST) | Discovery cache removed |
|
||||
| 3.1.4 | Re-send to previously sent targets, including removed links (SHOULD) | `previous_targets()` from the queue, plus removed links; not capped |
|
||||
| 3.1.5 | Deleted post returns 410 (SHOULD), tombstone (SHOULD) | `public.note` aborts 410 for soft-deleted notes and hard-deleted notes that sent webmentions; `410.html` |
|
||||
| 3.1.5 | Re-send on delete (SHOULD) | `requeue_previous_webmentions()`; for a hard delete the rows are detached (`source_note_id = NULL`) first, since the foreign key cascades. Also on undelete |
|
||||
|
||||
## Receiving
|
||||
|
||||
| Section | Requirement | Change |
|
||||
|---|---|---|
|
||||
| 3.2 | 202 for async processing (MUST) | Unchanged |
|
||||
| 3.2.1 | Valid http(s) URLs; source ≠ target; 400 on error (MUST) | Unchanged |
|
||||
| 3.2.1 | Target is a valid resource (SHOULD) | Only published notes; others 400 |
|
||||
| 3.2.1 | Fragment ignored | Stripped before storing (no duplicate rows) |
|
||||
| 3.2.2 | GET source, follow redirects, limit redirects | `safe_get` |
|
||||
| 3.2.2 | Accept header (SHOULD) | Added |
|
||||
| 3.2.2 | Exact match of target, per media type (MUST) | HTML: every link attribute (`lxml iterlinks`) resolved against the final URL; JSON: any string value; `text/*`: substring; other types: no match. Fragments ignored |
|
||||
| 3.2.4 | Update existing data; 410 / no link removes it (SHOULD) | Unchanged in effect. Moderation decisions kept on update; transient errors keep a valid mention |
|
||||
|
||||
## Security (section 4)
|
||||
|
||||
- 4.1 XSS: author URL/photo must be http(s) (v2.6.9); snippets re-sanitized after truncation; Mentions list uses `striptags`.
|
||||
- 4.2 / 4.5: `safe_get` checks every redirect hop, checks all `getaddrinfo` addresses, blocks private, loopback, link-local, multicast, reserved, unspecified, CGNAT, `0.0.0.0/8` and IPv4-mapped forms; 1 MB and 10 s total limits.
|
||||
- 4.3: endpoint URL checked before POST, no redirects followed.
|
||||
|
||||
## Known limits
|
||||
|
||||
- Homepage mentions are refused rather than stored.
|
||||
- Discovery does not apply HTTP cache headers (5.3); targets are only
|
||||
fetched when a note is created, edited or deleted.
|
||||
- The 410 page is a generic tombstone without microformats.
|
||||
- `error` rows (unexpected failures) are not retried automatically.
|
||||
|
||||
## Tests
|
||||
|
||||
`tests/test_webmention_spec.py` (section-numbered), `tests/test_webmention_worker.py`.
|
||||
@@ -0,0 +1,36 @@
|
||||
# v2.7.2: Remaining Webmention SHOULDs
|
||||
|
||||
Follow-up to the v2.7.0 compliance review. Every SHOULD in the
|
||||
Recommendation was checked; two were not met.
|
||||
|
||||
## 4.5 DNS pinning
|
||||
|
||||
`validate_url_for_fetch()` now returns the checked address.
|
||||
`_pinned_request()` builds each request against that IP, with the original
|
||||
host in the `Host` header and, for https, in the `sni_hostname` extension.
|
||||
httpcore uses `sni_hostname` as `server_hostname` with `check_hostname`
|
||||
on, so the certificate is verified against the real hostname. Used by
|
||||
`safe_get()` (every redirect hop) and the new `safe_post()` (the webmention
|
||||
POST). IPv4 is preferred when a host has both families.
|
||||
|
||||
## 3.1.4 Displayed responses are an update
|
||||
|
||||
"If the source post displays responses... then sender SHOULD treat that as
|
||||
an update of the source URL and re-send any previously sent Webmentions."
|
||||
|
||||
`note_responses_changed()` re-queues a published note's previous webmentions.
|
||||
It is called when:
|
||||
|
||||
- the worker validates a webmention and what the note shows for it changed
|
||||
(`_display_state()` before and after: shown or not, type, author, snippet).
|
||||
A re-send from the source resets the row to `pending`; a previously valid
|
||||
row still counts as shown, so an unchanged re-validation sends nothing.
|
||||
This keeps two sites that both implement this from ping-ponging.
|
||||
- an admin approves, hides or deletes webmentions and a valid one changes
|
||||
between shown and not shown.
|
||||
|
||||
## Tests
|
||||
|
||||
`tests/test_webmention_spec.py`: pinning (URL, Host, SNI, one lookup per
|
||||
request), IPv4 preference, re-send on a new response, none on an unchanged
|
||||
one, re-send on moderation changes.
|
||||
@@ -0,0 +1,38 @@
|
||||
# v2.8.0: Microformats2 review
|
||||
|
||||
Review of StarPunk's published and consumed microformats against
|
||||
microformats2, h-entry, h-card, h-feed, h-cite, representative h-card
|
||||
parsing, the authorship spec and Post Type Discovery. Every page type was
|
||||
rendered, parsed with mf2py and checked with mf2util.
|
||||
|
||||
Posts (h-entry) were largely correct: no implied names, PTD right for
|
||||
note/article/reply/like/repost/photo, author h-card, u-url/u-uid,
|
||||
p-category, u-syndication.
|
||||
|
||||
## Fixes
|
||||
|
||||
| # | Problem | Fix |
|
||||
|---|---|---|
|
||||
| 1 | Replies were unattached `h-cite` children with a plain-text author | `u-comment h-cite`, `p-author h-card` (u-url on an inner link, u-photo, p-name), `e-content` (`templates/partials/responses.html`) |
|
||||
| 2 | Likes/reposts/bookmarks/mentions had no mf2 | `u-like` / `u-repost` / `u-bookmark` / `u-mention` h-cites; faces past 8 are `hidden`, not omitted |
|
||||
| 3 | `dt-*` naive, with microseconds | `iso_utc` filter/helper: RFC 3339 UTC `Z`, no fraction; Micropub `q=source` too |
|
||||
| 4 | `#hashtag` first line = title; bare `#` = slug title | `split_title` requires an ATX heading (`#{1,6}` + space); templates use `Note.has_title` |
|
||||
| 5 | Note's own syndicated copy shown as a mention | Worker rejects a source whose url/uid (or URL) is one of the note's `syndications.remote_url` as invalid |
|
||||
| 6 | Micropub/IndieAuth links gated on WEBMENTION_ENABLED | Only `rel=webmention` is gated |
|
||||
| 7 | No rel=me on a fresh install | `rel=me` and `rel=author` to `author.url or author.me or ADMIN_ME`, always |
|
||||
| 8 | Author discovery guessed h-cards | `mf2_utils.representative_hcard` (uid+url, rel=me, single h-card); parsed with the URL after redirects |
|
||||
| 9 | Webmention entry/author/time | `mf2_utils.responding_entry` (entry whose in-reply-to/like-of/... or content refers to the target), `entry_author` (author property as h-card/URL/name, parent h-feed author, rel=author, representative h-card), `source_post_url` and `source_published` stored (migration 014) |
|
||||
|
||||
Also: tag and likes h-feeds have `u-url` and `p-author`; previews have `u-uid`.
|
||||
|
||||
## Not done
|
||||
|
||||
- Reply-context h-cite does not carry the cached author h-card, name or published time.
|
||||
- Photo alt falls back to "Image" when a photo-only post has no caption.
|
||||
- Person tags (`p-category h-card`) are not supported.
|
||||
- Existing webmentions keep NULL source details until they are re-validated.
|
||||
|
||||
## Tests
|
||||
|
||||
`tests/test_microformats_v280.py`. `tests/test_microformats.py`
|
||||
`test_no_relme_without_author` became `test_relme_to_admin_me_without_author`.
|
||||
@@ -0,0 +1,33 @@
|
||||
# v2.8.1: Migration lock held for the whole run
|
||||
|
||||
## Incident
|
||||
|
||||
Deploying v2.8.0 (migration 014 adds two columns): one gunicorn worker
|
||||
applied 014, another failed with `duplicate column name: source_post_url`,
|
||||
gunicorn shut down ("Worker failed to boot") and Docker restarted the
|
||||
container. About two seconds of downtime; schema correct afterwards.
|
||||
|
||||
## Causes
|
||||
|
||||
`run_migrations` takes `BEGIN IMMEDIATE` so one worker migrates at a time,
|
||||
but the lock was released early twice:
|
||||
|
||||
1. `create_migrations_table()`, called right after `BEGIN IMMEDIATE`,
|
||||
ended with `conn.commit()`. Everything after it ran without the lock.
|
||||
2. `conn.executescript()` COMMITs any open transaction before running the
|
||||
script, so the lock was dropped before the migration was recorded in
|
||||
`schema_migrations`.
|
||||
|
||||
## Fix
|
||||
|
||||
- `create_migrations_table(conn, commit=True)`; `run_migrations` passes
|
||||
`commit=False`.
|
||||
- `execute_in_transaction()` runs each statement from
|
||||
`split_sql_statements()` (built on `sqlite3.complete_statement`, so
|
||||
trigger bodies stay whole) with `conn.execute`, inside the transaction.
|
||||
|
||||
## Verification
|
||||
|
||||
Four processes migrating a database at 013 simultaneously: 6/30 trials
|
||||
failed before, 0/30 after. `tests/test_migration_concurrency.py` runs this
|
||||
five times, plus a rollback test showing a migration commits nothing early.
|
||||
@@ -0,0 +1,51 @@
|
||||
# v2.9.0: Micropub review
|
||||
|
||||
Review against the W3C Micropub Recommendation (https://www.w3.org/TR/micropub/),
|
||||
run through the Flask test client and mapped to the micropub.rocks server
|
||||
tests. Everyday create/update/delete/query/media/auth already passed; the
|
||||
gaps were privacy, security and silent data loss.
|
||||
|
||||
## Decisions (user)
|
||||
|
||||
- No `mp-syndicate-to` from a Micropub client means no crossposting
|
||||
(was: all targets, v2.1.0). The admin editor's default is unchanged.
|
||||
- `visibility` private/unlisted are saved as drafts.
|
||||
- Rendered HTML is sanitized with an allowlist (nh3 defaults plus class,
|
||||
title, id, lang, dir on all tags; video/audio/source/track/picture).
|
||||
|
||||
## Changes
|
||||
|
||||
| Area | Before | After |
|
||||
|---|---|---|
|
||||
| `mp-*` in JSON `properties` | ignored (all targets, timestamp slug) | read as commands, never stored |
|
||||
| absent `mp-syndicate-to` | all targets | none |
|
||||
| `q=source` / `q=category` | any token | create, update, delete or draft scope |
|
||||
| HTML content | rendered raw (stored XSS) | `render_markdown()` = Markdown + nh3 |
|
||||
| `visibility` | ignored (published) | private/unlisted -> draft; unknown -> 400 |
|
||||
| `draft` scope | 403 | creates drafts only |
|
||||
| photo fetch/lookup failure | dropped, 201 | 400, nothing created or changed |
|
||||
| remote photo fetch | first host checked, redirects followed | `webmention_ssrf.safe_get` (per hop, pinned) |
|
||||
| JSON scalars | indexed per character | wrapped on create; update `delete` values must be arrays |
|
||||
| `bookmark-of` | dropped | `notes.bookmark_of`, kind `bookmark` (migration 015) |
|
||||
| HTML-authored content in `q=source` | string | `{"html": ...}` via `notes.content_format` |
|
||||
| `summary` / `name` only | 400 | used as text / titled post |
|
||||
| `video` / `audio` | dropped | 400 |
|
||||
| `published` | stored with offset; garbage -> now | naive UTC; garbage -> 400 |
|
||||
| malformed JSON | 500 | 400 |
|
||||
| update without operations | 204 | 400 |
|
||||
| auth | `Bearer` only; header+body accepted; no challenge | case-insensitive; both -> 400; `WWW-Authenticate` |
|
||||
|
||||
## Kept as is
|
||||
|
||||
- 403 (RFC 6750) for insufficient_scope; the Micropub table says 401.
|
||||
- `add` only for category and photo; `published`, `syndication`,
|
||||
`location` not updatable.
|
||||
- `location`/`checkin` ignored (3.3.5 allows).
|
||||
- GET `access_token` query parameter still accepted.
|
||||
|
||||
## Tests
|
||||
|
||||
`tests/test_micropub_v290.py` (31 of 34 fail on v2.8.1). Updated:
|
||||
likes/reposts helper sends `mp-syndicate-to` in properties;
|
||||
`test_micropub_default_is_no_targets`; remote photo test uses the pinned
|
||||
fetcher; corrupted-image message.
|
||||
@@ -0,0 +1,8 @@
|
||||
-- Migration 013: Likes and reposts (v2.5.0, ADR-068)
|
||||
-- A note with like_of is a like; with repost_of, a repost. Content is the
|
||||
-- optional comment and may be empty.
|
||||
|
||||
ALTER TABLE notes ADD COLUMN like_of TEXT;
|
||||
ALTER TABLE notes ADD COLUMN repost_of TEXT;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_notes_like_of ON notes(like_of);
|
||||
@@ -0,0 +1,6 @@
|
||||
-- Migration 014: Source details for received webmentions (v2.8.0)
|
||||
-- The source's own u-url and dt-published, so a response links to and dates
|
||||
-- the original post rather than the URL that sent it and when it arrived.
|
||||
|
||||
ALTER TABLE webmentions ADD COLUMN source_post_url TEXT;
|
||||
ALTER TABLE webmentions ADD COLUMN source_published TIMESTAMP;
|
||||
@@ -0,0 +1,7 @@
|
||||
-- Migration 015: Bookmarks and HTML-authored content (v2.9.0)
|
||||
-- bookmark_of: URL bookmarked (Micropub bookmark-of); the note is a bookmark.
|
||||
-- content_format: 'html' when a Micropub client sent the content as HTML,
|
||||
-- so q=source returns it as {"html": ...} (Micropub 3.7.2). NULL = Markdown.
|
||||
|
||||
ALTER TABLE notes ADD COLUMN bookmark_of TEXT;
|
||||
ALTER TABLE notes ADD COLUMN content_format TEXT;
|
||||
@@ -0,0 +1,55 @@
|
||||
-- Migration 016: WebSub hub (v2.10.0, ADR-071)
|
||||
|
||||
-- Active and pending subscriptions to this site's topics
|
||||
CREATE TABLE IF NOT EXISTS websub_subscriptions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
topic TEXT NOT NULL,
|
||||
callback TEXT NOT NULL,
|
||||
secret TEXT,
|
||||
lease_seconds INTEGER NOT NULL,
|
||||
expires_at TIMESTAMP NOT NULL,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
verified_at TIMESTAMP,
|
||||
last_success_at TIMESTAMP,
|
||||
last_error TEXT,
|
||||
UNIQUE(topic, callback)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_websub_subscriptions_topic ON websub_subscriptions(topic);
|
||||
|
||||
-- Subscribe/unsubscribe requests waiting for verification of intent
|
||||
CREATE TABLE IF NOT EXISTS websub_intents (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
topic TEXT NOT NULL,
|
||||
callback TEXT NOT NULL,
|
||||
mode TEXT NOT NULL CHECK (mode IN ('subscribe', 'unsubscribe')),
|
||||
secret TEXT,
|
||||
lease_seconds INTEGER,
|
||||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
next_attempt_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(topic, callback)
|
||||
);
|
||||
|
||||
-- Content to push: one row per subscription (built-in hub) or per topic
|
||||
-- (ping to an external hub, subscription_id NULL)
|
||||
CREATE TABLE IF NOT EXISTS websub_deliveries (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
subscription_id INTEGER REFERENCES websub_subscriptions(id) ON DELETE CASCADE,
|
||||
topic TEXT NOT NULL,
|
||||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
next_attempt_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
last_error TEXT,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_websub_deliveries_subscription
|
||||
ON websub_deliveries(subscription_id) WHERE subscription_id IS NOT NULL;
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_websub_deliveries_ping
|
||||
ON websub_deliveries(topic) WHERE subscription_id IS NULL;
|
||||
|
||||
-- Site-wide values. content_version changes whenever a public note does, so
|
||||
-- every gunicorn process knows its cached feed note list is stale.
|
||||
CREATE TABLE IF NOT EXISTS site_state (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
INSERT OR IGNORE INTO site_state (key, value) VALUES ('content_version', '0');
|
||||
+20
-14
@@ -219,6 +219,10 @@ def create_app(config=None):
|
||||
|
||||
return {'author': author}
|
||||
|
||||
# Template filter - RFC 3339 UTC for dt-* properties (v2.8.0)
|
||||
from starpunk.utils import iso_utc
|
||||
app.add_template_filter(iso_utc, 'iso_utc')
|
||||
|
||||
# Template filter - URL to color for facepile (v1.6.0)
|
||||
@app.template_filter('url_to_color')
|
||||
def url_to_color_filter(url):
|
||||
@@ -302,9 +306,14 @@ def create_app(config=None):
|
||||
if (app.config.get('WEBMENTION_ENABLED', True) and
|
||||
response.content_type and
|
||||
'text/html' in response.content_type):
|
||||
response.headers['Link'] = '</webmention>; rel="webmention"'
|
||||
response.headers.add('Link', '</webmention>; rel="webmention"')
|
||||
return response
|
||||
|
||||
# WebSub discovery: Link rel=hub and rel=self on topics (v2.10.0)
|
||||
from starpunk.websub import add_discovery_headers, template_links
|
||||
app.after_request(add_discovery_headers)
|
||||
app.add_template_global(template_links, 'websub_links')
|
||||
|
||||
# Register centralized error handlers
|
||||
from starpunk.errors import register_error_handlers
|
||||
|
||||
@@ -318,18 +327,15 @@ def create_app(config=None):
|
||||
memory_monitor.start()
|
||||
app.memory_monitor = memory_monitor
|
||||
app.logger.info(f"Memory monitor started (interval={memory_monitor.interval}s)")
|
||||
|
||||
# Register cleanup handler
|
||||
@app.teardown_appcontext
|
||||
def cleanup_memory_monitor(error=None):
|
||||
if hasattr(app, 'memory_monitor') and app.memory_monitor.is_alive():
|
||||
app.memory_monitor.stop()
|
||||
# No teardown_appcontext stop: that ran after every request and stopped
|
||||
# the monitor straight away (v2.10.0). It is a daemon thread.
|
||||
|
||||
# Start webmention worker thread (v1.6.0 Phase 2)
|
||||
# Per background-worker.md: Skip in test mode
|
||||
if app.config.get('WEBMENTION_WORKER_ENABLED', True) and not app.config.get('TESTING', False):
|
||||
from starpunk.webmention_worker import WebmentionWorker
|
||||
webmention_worker = WebmentionWorker(
|
||||
app,
|
||||
db_path=app.config['DATABASE_PATH'],
|
||||
site_url=app.config['SITE_URL'],
|
||||
poll_interval=app.config.get('WEBMENTION_POLL_INTERVAL', 10),
|
||||
@@ -345,11 +351,11 @@ def create_app(config=None):
|
||||
f"batch_size={webmention_worker.batch_size})"
|
||||
)
|
||||
|
||||
# Register cleanup handler
|
||||
@app.teardown_appcontext
|
||||
def cleanup_webmention_worker(error=None):
|
||||
if hasattr(app, 'webmention_worker') and app.webmention_worker.is_alive():
|
||||
app.webmention_worker.stop()
|
||||
# Start WebSub worker (v2.10.0)
|
||||
from starpunk.websub_worker import start_worker
|
||||
app.websub_worker = start_worker(app)
|
||||
if app.websub_worker:
|
||||
app.logger.info("WebSub worker started")
|
||||
|
||||
# Start syndication worker when a target is configured (v2.1.0)
|
||||
from starpunk.syndication import SyndicationWorker, targets_configured
|
||||
@@ -464,5 +470,5 @@ def create_app(config=None):
|
||||
|
||||
# Package version (Semantic Versioning 2.0.0)
|
||||
# See docs/standards/versioning-strategy.md for details
|
||||
__version__ = "2.3.3"
|
||||
__version_info__ = (2, 1, 0)
|
||||
__version__ = "2.10.0"
|
||||
__version_info__ = (2, 10, 0)
|
||||
@@ -82,10 +82,12 @@ def discover_author_profile(me_url: str) -> Optional[Dict]:
|
||||
response.raise_for_status()
|
||||
|
||||
# Parse microformats from HTML
|
||||
parsed = mf2py.parse(doc=response.text, url=me_url)
|
||||
# Relative URLs and the representative h-card use the URL after redirects
|
||||
page_url = str(response.url) if isinstance(response.url, (str, httpx.URL)) else me_url
|
||||
parsed = mf2py.parse(doc=response.text, url=page_url)
|
||||
|
||||
# Extract h-card (per Q16: first representative h-card)
|
||||
hcard = _find_representative_hcard(parsed, me_url)
|
||||
# Representative h-card (v2.8.0: per the parsing spec)
|
||||
hcard = _find_representative_hcard(parsed, page_url)
|
||||
|
||||
if not hcard:
|
||||
current_app.logger.warning(f"No h-card found at {me_url}")
|
||||
@@ -132,47 +134,21 @@ def discover_author_profile(me_url: str) -> Optional[Dict]:
|
||||
|
||||
def _find_representative_hcard(parsed: dict, me_url: str) -> Optional[dict]:
|
||||
"""
|
||||
Find representative h-card from parsed microformats
|
||||
Find the representative h-card of the author's page (v2.8.0)
|
||||
|
||||
Per Q16: First representative h-card = first h-card with p-name
|
||||
Per Q18: First h-card with url property matching profile URL
|
||||
Follows https://microformats.org/wiki/representative-h-card-parsing
|
||||
(uid+url match, rel=me match, or the only h-card with a matching url).
|
||||
Other h-cards on the page may be other people, so none is guessed.
|
||||
|
||||
Args:
|
||||
parsed: Parsed microformats data from mf2py
|
||||
me_url: Profile URL for matching
|
||||
me_url: Profile URL (the page that was parsed)
|
||||
|
||||
Returns:
|
||||
h-card dict or None if not found
|
||||
h-card dict or None if the page has no representative h-card
|
||||
"""
|
||||
items = parsed.get('items', [])
|
||||
|
||||
# First try: h-card with matching URL (most specific)
|
||||
for item in items:
|
||||
if 'h-card' in item.get('type', []):
|
||||
properties = item.get('properties', {})
|
||||
urls = properties.get('url', [])
|
||||
|
||||
# Check if any URL matches the profile URL
|
||||
for url in urls:
|
||||
if isinstance(url, dict):
|
||||
url = url.get('value', '')
|
||||
if _normalize_url(url) == _normalize_url(me_url):
|
||||
# Found matching h-card
|
||||
return item
|
||||
|
||||
# Second try: First h-card with p-name (representative h-card)
|
||||
for item in items:
|
||||
if 'h-card' in item.get('type', []):
|
||||
properties = item.get('properties', {})
|
||||
if properties.get('name'):
|
||||
return item
|
||||
|
||||
# Third try: Just use first h-card if any
|
||||
for item in items:
|
||||
if 'h-card' in item.get('type', []):
|
||||
return item
|
||||
|
||||
return None
|
||||
from starpunk.mf2_utils import representative_hcard
|
||||
return representative_hcard(parsed, me_url)
|
||||
|
||||
|
||||
def _get_property(hcard: dict, prop_name: str) -> Optional[str]:
|
||||
|
||||
@@ -111,6 +111,13 @@ def load_config(app, config_override=None):
|
||||
|
||||
# Webmention configuration (v1.6.0)
|
||||
app.config["WEBMENTION_ENABLED"] = os.getenv("WEBMENTION_ENABLED", "true").lower() == "true"
|
||||
|
||||
# WebSub (v2.10.0, ADR-071): built-in hub unless WEBSUB_HUB_URL names another
|
||||
app.config["WEBSUB_ENABLED"] = os.getenv("WEBSUB_ENABLED", "true").lower() == "true"
|
||||
app.config["WEBSUB_HUB_URL"] = os.getenv("WEBSUB_HUB_URL", "").strip()
|
||||
app.config["WEBSUB_MAX_SUBSCRIPTIONS"] = int(os.getenv("WEBSUB_MAX_SUBSCRIPTIONS", "1000"))
|
||||
app.config["WEBSUB_MAX_PER_HOST"] = int(os.getenv("WEBSUB_MAX_PER_HOST", "25"))
|
||||
app.config["WEBSUB_POLL_INTERVAL"] = int(os.getenv("WEBSUB_POLL_INTERVAL", "5"))
|
||||
app.config["WEBMENTION_WORKER_ENABLED"] = os.getenv("WEBMENTION_WORKER_ENABLED", "true").lower() == "true"
|
||||
app.config["WEBMENTION_POLL_INTERVAL"] = int(os.getenv("WEBMENTION_POLL_INTERVAL", "10"))
|
||||
app.config["WEBMENTION_ERROR_BACKOFF"] = int(os.getenv("WEBMENTION_ERROR_BACKOFF", "60"))
|
||||
@@ -137,6 +144,12 @@ def load_config(app, config_override=None):
|
||||
app.config["BLUESKY_SERVICE"] = os.getenv("BLUESKY_SERVICE", "https://bsky.social")
|
||||
app.config["BLUESKY_NAME"] = os.getenv("BLUESKY_NAME", "Bluesky")
|
||||
app.config["BLUESKY_MAX_CHARS"] = int(os.getenv("BLUESKY_MAX_CHARS", "300"))
|
||||
# Crossposting to Threads (v2.6.0, ADR-069): long-lived token, refreshed automatically
|
||||
app.config["THREADS_ACCESS_TOKEN"] = os.getenv("THREADS_ACCESS_TOKEN", "")
|
||||
app.config["THREADS_USERNAME"] = os.getenv("THREADS_USERNAME", "")
|
||||
app.config["THREADS_NAME"] = os.getenv("THREADS_NAME", "Threads")
|
||||
app.config["THREADS_MAX_CHARS"] = int(os.getenv("THREADS_MAX_CHARS", "500"))
|
||||
app.config["THREADS_MAX_MEDIA"] = int(os.getenv("THREADS_MAX_MEDIA", "4"))
|
||||
app.config["SYNDICATION_POLL_INTERVAL"] = int(os.getenv("SYNDICATION_POLL_INTERVAL", "10"))
|
||||
|
||||
# Apply overrides if provided
|
||||
|
||||
@@ -107,6 +107,16 @@ class ConnectionPool:
|
||||
if not conn:
|
||||
return
|
||||
|
||||
# A request that failed mid-write must not hand an open transaction
|
||||
# (and its write lock) to the next request (v2.6.5)
|
||||
if conn.in_transaction:
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception:
|
||||
conn.close()
|
||||
self._stats['connections_closed'] += 1
|
||||
return
|
||||
|
||||
with self._lock:
|
||||
if len(self._pool) < self.pool_size:
|
||||
# Return to pool
|
||||
|
||||
@@ -82,6 +82,11 @@ def register_error_handlers(app):
|
||||
|
||||
return render_template('404.html'), 404
|
||||
|
||||
@app.errorhandler(410)
|
||||
def gone(error):
|
||||
"""Handle 410 Gone (deleted notes, Webmention 3.1.5)"""
|
||||
return render_template('410.html'), 410
|
||||
|
||||
@app.errorhandler(405)
|
||||
def method_not_allowed(error):
|
||||
"""Handle 405 Method Not Allowed errors"""
|
||||
|
||||
@@ -198,7 +198,7 @@ def generate_atom_streaming(
|
||||
yield f' <link rel="enclosure" type="{_escape_xml(mime_type)}" href="{_escape_xml(media_url)}" length="{size}"{title_attr}/>\n'
|
||||
|
||||
# Content - include media as HTML (per Q24)
|
||||
if note.html:
|
||||
if note.feed_html:
|
||||
# Build HTML content with media at top
|
||||
html_content = ""
|
||||
|
||||
@@ -212,7 +212,7 @@ def generate_atom_streaming(
|
||||
html_content += '</div>'
|
||||
|
||||
# Add text content below media
|
||||
html_content += note.html
|
||||
html_content += note.feed_html
|
||||
|
||||
# HTML content - escaped
|
||||
yield ' <content type="html">'
|
||||
|
||||
@@ -267,7 +267,7 @@ def _build_item_object(site_url: str, note: Note) -> Dict[str, Any]:
|
||||
|
||||
# Add content (HTML or text)
|
||||
# Per Q24: Include media as HTML in content_html
|
||||
if note.html:
|
||||
if note.feed_html:
|
||||
content_html = ""
|
||||
|
||||
# Add media at top if present (v1.2.0 Phase 3)
|
||||
@@ -280,7 +280,7 @@ def _build_item_object(site_url: str, note: Note) -> Dict[str, Any]:
|
||||
content_html += '</div>'
|
||||
|
||||
# Add text content below media
|
||||
content_html += note.html
|
||||
content_html += note.feed_html
|
||||
item["content_html"] = content_html
|
||||
else:
|
||||
item["content_text"] = note.content
|
||||
|
||||
@@ -93,7 +93,7 @@ def generate_rss(
|
||||
fg.language("en")
|
||||
|
||||
# Add self-link for feed discovery (Atom namespace)
|
||||
fg.link(href=f"{site_url}/feed.xml", rel="self", type="application/rss+xml")
|
||||
fg.link(href=f"{site_url}/feed.rss", rel="self", type="application/rss+xml")
|
||||
|
||||
# Set last build date to now
|
||||
fg.lastBuildDate(datetime.now(timezone.utc))
|
||||
@@ -137,7 +137,7 @@ def generate_rss(
|
||||
html_content += '</div>'
|
||||
|
||||
# Add text content below media
|
||||
html_content += clean_html_for_rss(note.html)
|
||||
html_content += clean_html_for_rss(note.feed_html)
|
||||
|
||||
# feedgen automatically wraps content in CDATA for RSS
|
||||
fe.description(html_content)
|
||||
@@ -246,7 +246,7 @@ def generate_rss_streaming(
|
||||
yield f" <description>{_escape_xml(site_description or site_name)}</description>\n"
|
||||
yield " <language>en</language>\n"
|
||||
yield f" <lastBuildDate>{last_build}</lastBuildDate>\n"
|
||||
yield f' <atom:link href="{_escape_xml(site_url)}/feed.xml" rel="self" type="application/rss+xml"/>\n'
|
||||
yield f' <atom:link href="{_escape_xml(site_url)}/feed.rss" rel="self" type="application/rss+xml"/>\n'
|
||||
|
||||
# Yield items (newest first)
|
||||
# Notes from database are already in DESC order (newest first)
|
||||
@@ -278,7 +278,7 @@ def generate_rss_streaming(
|
||||
html_content += '</div>'
|
||||
|
||||
# Add text content below media
|
||||
html_content += clean_html_for_rss(note.html)
|
||||
html_content += clean_html_for_rss(note.feed_html)
|
||||
|
||||
# Build item XML
|
||||
item_xml = f""" <item>
|
||||
|
||||
+22
-14
@@ -93,6 +93,10 @@ def get_optimization_params(file_size: int) -> Tuple[int, int]:
|
||||
return (1280, 85)
|
||||
|
||||
|
||||
# Shown to clients; the parser's own message names internal objects (v2.9.0)
|
||||
UNSUPPORTED_IMAGE = "Not a supported image: use JPEG, PNG, GIF, WebP or HEIC"
|
||||
|
||||
|
||||
def validate_image(file_data: bytes, filename: str) -> Tuple[Image.Image, str, int, int]:
|
||||
"""
|
||||
Validate image file and return PIL Image for memory-efficient processing
|
||||
@@ -158,9 +162,9 @@ def validate_image(file_data: bytes, filename: str) -> Tuple[Image.Image, str, i
|
||||
current_app.logger.info(f'Saved failed upload for analysis: {debug_file}')
|
||||
except RuntimeError:
|
||||
pass # Outside app context (e.g., tests)
|
||||
raise ValueError(f"Invalid or corrupted image: {e}")
|
||||
raise ValueError(UNSUPPORTED_IMAGE) from e
|
||||
else:
|
||||
raise ValueError(f"Invalid or corrupted image: {e}")
|
||||
raise ValueError(UNSUPPORTED_IMAGE) from e
|
||||
|
||||
# Store original format before any conversions
|
||||
original_format = img.format
|
||||
@@ -868,20 +872,24 @@ def attach_media_to_note(note_id: int, media_ids: List[int], captions: List[str]
|
||||
|
||||
db = get_db(current_app)
|
||||
|
||||
# Delete existing associations (for edit case)
|
||||
db.execute("DELETE FROM note_media WHERE note_id = ?", (note_id,))
|
||||
try:
|
||||
# Delete existing associations (for edit case)
|
||||
db.execute("DELETE FROM note_media WHERE note_id = ?", (note_id,))
|
||||
|
||||
# Insert new associations
|
||||
for i, (media_id, caption) in enumerate(zip(media_ids, captions)):
|
||||
db.execute(
|
||||
"""
|
||||
INSERT INTO note_media (note_id, media_id, display_order, caption)
|
||||
VALUES (?, ?, ?, ?)
|
||||
""",
|
||||
(note_id, media_id, i, caption or None)
|
||||
)
|
||||
# Insert new associations
|
||||
for i, (media_id, caption) in enumerate(zip(media_ids, captions)):
|
||||
db.execute(
|
||||
"""
|
||||
INSERT INTO note_media (note_id, media_id, display_order, caption)
|
||||
VALUES (?, ?, ?, ?)
|
||||
""",
|
||||
(note_id, media_id, i, caption or None)
|
||||
)
|
||||
|
||||
db.commit()
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
|
||||
|
||||
def get_note_media(note_id: int) -> List[Dict]:
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
"""
|
||||
Microformats2 consuming helpers (v2.8.0)
|
||||
|
||||
Shared by Webmention processing and author discovery:
|
||||
- representative h-card of a page
|
||||
- the h-entry on a page that responds to a given URL
|
||||
- the author of an h-entry (authorship algorithm)
|
||||
|
||||
References:
|
||||
- https://microformats.org/wiki/representative-h-card-parsing
|
||||
- https://indieweb.org/authorship-spec
|
||||
"""
|
||||
|
||||
from typing import Iterator, Optional
|
||||
from urllib.parse import urldefrag, urlparse
|
||||
|
||||
RESPONSE_PROPERTIES = ('in-reply-to', 'like-of', 'repost-of', 'bookmark-of')
|
||||
|
||||
|
||||
def normalize_url(url: str) -> str:
|
||||
"""URL for comparison: no fragment, no trailing slash, lowercase scheme/host"""
|
||||
if not url:
|
||||
return ''
|
||||
url = urldefrag(url.strip())[0]
|
||||
parsed = urlparse(url)
|
||||
url = parsed._replace(scheme=parsed.scheme.lower(), netloc=parsed.netloc.lower()).geturl()
|
||||
return url.rstrip('/')
|
||||
|
||||
|
||||
def value_of(value) -> Optional[str]:
|
||||
"""Plain value of an mf2 property value (strings, {'value':...}, {'html':...})"""
|
||||
if isinstance(value, dict):
|
||||
value = value.get('value')
|
||||
return value if isinstance(value, str) and value else None
|
||||
|
||||
|
||||
def first(item: dict, prop: str) -> Optional[str]:
|
||||
"""First plain value of an item's property"""
|
||||
for value in item.get('properties', {}).get(prop, []):
|
||||
result = value_of(value)
|
||||
if result:
|
||||
return result
|
||||
return None
|
||||
|
||||
|
||||
def values(item: dict, prop: str) -> list[str]:
|
||||
"""All plain values of an item's property"""
|
||||
return [v for v in (value_of(x) for x in item.get('properties', {}).get(prop, [])) if v]
|
||||
|
||||
|
||||
def walk(items: list) -> Iterator[dict]:
|
||||
"""Every microformat in items, depth first: top level, children, nested properties"""
|
||||
for item in items:
|
||||
if not isinstance(item, dict) or 'type' not in item:
|
||||
continue
|
||||
yield item
|
||||
yield from walk(item.get('children', []))
|
||||
for prop_values in item.get('properties', {}).values():
|
||||
yield from walk([v for v in prop_values if isinstance(v, dict)])
|
||||
|
||||
|
||||
def is_type(item: dict, mf_type: str) -> bool:
|
||||
return mf_type in item.get('type', [])
|
||||
|
||||
|
||||
def representative_hcard(parsed: dict, page_url: str) -> Optional[dict]:
|
||||
"""
|
||||
The page's representative h-card, per the parsing spec, in order:
|
||||
|
||||
1. An h-card whose uid and url both match the page URL
|
||||
2. An h-card whose url matches one of the page's rel=me URLs
|
||||
3. The only h-card on the page, if its url matches the page URL
|
||||
|
||||
Otherwise the page has none.
|
||||
"""
|
||||
page = normalize_url(page_url)
|
||||
cards = [item for item in walk(parsed.get('items', [])) if is_type(item, 'h-card')]
|
||||
|
||||
for card in cards:
|
||||
if page in map(normalize_url, values(card, 'uid')) and \
|
||||
page in map(normalize_url, values(card, 'url')):
|
||||
return card
|
||||
|
||||
rel_me = {normalize_url(u) for u in parsed.get('rels', {}).get('me', [])}
|
||||
for card in cards:
|
||||
if rel_me & set(map(normalize_url, values(card, 'url'))):
|
||||
return card
|
||||
|
||||
if len(cards) == 1 and page in map(normalize_url, values(cards[0], 'url')):
|
||||
return cards[0]
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def responding_entry(parsed: dict, target_url: str) -> Optional[dict]:
|
||||
"""
|
||||
The h-entry on a page that refers to target_url
|
||||
|
||||
Prefers an entry whose in-reply-to / like-of / repost-of / bookmark-of
|
||||
is the target, then one whose content links to it, then the first
|
||||
top-level h-entry.
|
||||
"""
|
||||
target = normalize_url(target_url)
|
||||
entries = [item for item in walk(parsed.get('items', [])) if is_type(item, 'h-entry')]
|
||||
top = [item for item in parsed.get('items', []) if is_type(item, 'h-entry')]
|
||||
|
||||
if not target:
|
||||
return top[0] if top else (entries[0] if entries else None)
|
||||
|
||||
for entry in entries:
|
||||
for prop in RESPONSE_PROPERTIES:
|
||||
if target in map(normalize_url, values(entry, prop)):
|
||||
return entry
|
||||
|
||||
for entry in entries:
|
||||
for content in entry.get('properties', {}).get('content', []):
|
||||
html = content.get('html', '') if isinstance(content, dict) else str(content)
|
||||
if target_url in html or target in html:
|
||||
return entry
|
||||
|
||||
return top[0] if top else (entries[0] if entries else None)
|
||||
|
||||
|
||||
def response_type(entry: dict, target_url: str) -> str:
|
||||
"""reply / like / repost / bookmark if the entry is one to target_url, else mention"""
|
||||
target = normalize_url(target_url)
|
||||
for prop, kind in zip(RESPONSE_PROPERTIES, ('reply', 'like', 'repost', 'bookmark')):
|
||||
if target in map(normalize_url, values(entry, prop)):
|
||||
return kind
|
||||
return 'mention'
|
||||
|
||||
|
||||
def entry_author(parsed: dict, entry: Optional[dict], page_url: str) -> dict:
|
||||
"""
|
||||
Author of an h-entry, following the authorship algorithm without fetching
|
||||
|
||||
1. The entry's author property (an h-card, a URL, or a name)
|
||||
2. The author of an h-feed that contains the entry
|
||||
3. The page's rel=author link
|
||||
4. The page's representative h-card
|
||||
|
||||
Returns:
|
||||
{'name', 'url', 'photo'}; values may be None
|
||||
"""
|
||||
author = None
|
||||
if entry:
|
||||
author = (entry.get('properties', {}).get('author') or [None])[0]
|
||||
if author is None:
|
||||
for feed in walk(parsed.get('items', [])):
|
||||
if is_type(feed, 'h-feed') and entry in feed.get('children', []):
|
||||
author = (feed.get('properties', {}).get('author') or [None])[0]
|
||||
break
|
||||
|
||||
if author is None:
|
||||
rel_author = parsed.get('rels', {}).get('author', [])
|
||||
author = rel_author[0] if rel_author else representative_hcard(parsed, page_url)
|
||||
|
||||
if isinstance(author, dict) and 'type' in author:
|
||||
return {
|
||||
'name': first(author, 'name'),
|
||||
'url': first(author, 'url'),
|
||||
'photo': first(author, 'photo'),
|
||||
}
|
||||
|
||||
text = value_of(author)
|
||||
if text and urlparse(text).scheme in ('http', 'https'):
|
||||
return {'name': None, 'url': text, 'photo': None}
|
||||
return {'name': text, 'url': None, 'photo': None}
|
||||
+599
-207
@@ -10,6 +10,8 @@ Functions:
|
||||
extract_title: Get the explicit title (name) from Micropub properties
|
||||
extract_tags: Get category tags from Micropub properties
|
||||
handle_create: Process Micropub create action
|
||||
handle_update: Process Micropub update action (v2.4.0)
|
||||
handle_delete: Process Micropub delete / undelete actions (v2.4.0)
|
||||
handle_query: Process Micropub query endpoints
|
||||
extract_bearer_token: Get token from Authorization header or form
|
||||
|
||||
@@ -23,15 +25,18 @@ References:
|
||||
- IndieAuth Specification: https://www.w3.org/TR/indieauth/
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
from flask import Request, current_app, jsonify
|
||||
|
||||
from starpunk.models import split_title
|
||||
from starpunk.notes import create_note, get_note, InvalidNoteDataError, NoteNotFoundError
|
||||
from starpunk.notes import (
|
||||
create_note, delete_note, get_note, list_notes, undelete_note, update_note,
|
||||
InvalidNoteDataError, NoteNotFoundError,
|
||||
)
|
||||
from starpunk.indieauth import check_scope
|
||||
from starpunk.utils import absolute_url_for
|
||||
from starpunk.utils import absolute_url_for, iso_utc
|
||||
|
||||
|
||||
# Custom Exceptions
|
||||
@@ -76,9 +81,17 @@ def error_response(error: str, error_description: str, status_code: int = 400):
|
||||
Returns:
|
||||
Tuple of (response, status_code)
|
||||
"""
|
||||
headers = {}
|
||||
if status_code == 401:
|
||||
# RFC 6750 3: tell the client how to authenticate (v2.9.0)
|
||||
challenge = 'Bearer realm="micropub"'
|
||||
if error == "unauthorized" and "Invalid" in error_description:
|
||||
challenge += ', error="invalid_token"'
|
||||
headers["WWW-Authenticate"] = challenge
|
||||
return (
|
||||
jsonify({"error": error, "error_description": error_description}),
|
||||
status_code,
|
||||
headers,
|
||||
)
|
||||
|
||||
|
||||
@@ -99,18 +112,21 @@ def extract_bearer_token(request: Request) -> Optional[str]:
|
||||
Returns:
|
||||
Token string if found, None otherwise
|
||||
"""
|
||||
# Try Authorization header first
|
||||
auth_header = request.headers.get("Authorization", "")
|
||||
if auth_header.startswith("Bearer "):
|
||||
return auth_header[7:] # Remove "Bearer " prefix
|
||||
# The scheme is case-insensitive (RFC 7235)
|
||||
scheme, _, credentials = request.headers.get("Authorization", "").partition(" ")
|
||||
header_token = credentials.strip() if scheme.lower() == "bearer" else None
|
||||
|
||||
# Try form parameter
|
||||
body_token = None
|
||||
if request.method == "POST":
|
||||
return request.form.get("access_token")
|
||||
body_token = request.form.get("access_token")
|
||||
elif request.method == "GET":
|
||||
return request.args.get("access_token")
|
||||
body_token = request.args.get("access_token")
|
||||
|
||||
return None
|
||||
# Only one method may be used (RFC 6750 2) (v2.9.0)
|
||||
if header_token and body_token:
|
||||
raise MicropubValidationError("Send the access token in the header or the body, not both")
|
||||
|
||||
return header_token or body_token
|
||||
|
||||
|
||||
# Property Normalization
|
||||
@@ -130,13 +146,24 @@ def normalize_properties(data: dict) -> dict:
|
||||
Returns:
|
||||
Normalized properties dict with all values as lists
|
||||
"""
|
||||
# JSON format has properties nested
|
||||
# JSON format has properties nested. Values MUST be arrays (3.3.2); a
|
||||
# bare value is wrapped rather than read character by character.
|
||||
# mp-* commands and access_token are not properties (3.2).
|
||||
if "properties" in data:
|
||||
return data["properties"]
|
||||
properties = data["properties"]
|
||||
if not isinstance(properties, dict):
|
||||
raise MicropubValidationError("properties must be an object")
|
||||
return {
|
||||
key: value if isinstance(value, list) else [value]
|
||||
for key, value in properties.items()
|
||||
if not key.startswith("mp-") and key != "access_token"
|
||||
}
|
||||
|
||||
# Form format - convert to properties dict
|
||||
properties = {}
|
||||
for key, value in data.items():
|
||||
if hasattr(value, "filename") or (isinstance(value, list) and value and hasattr(value[0], "filename")):
|
||||
continue # uploaded files are handled separately
|
||||
# Skip reserved Micropub parameters
|
||||
if key.startswith("mp-") or key in ["action", "url", "access_token", "h"]:
|
||||
continue
|
||||
@@ -169,7 +196,7 @@ def extract_content(properties: dict) -> str:
|
||||
Raises:
|
||||
MicropubValidationError: If content is missing or empty
|
||||
"""
|
||||
content_list = properties.get("content", [])
|
||||
content_list = properties.get("content") or properties.get("summary") or []
|
||||
|
||||
# Handle both plain text and HTML/text objects
|
||||
if not content_list:
|
||||
@@ -182,12 +209,30 @@ def extract_content(properties: dict) -> str:
|
||||
# Prefer text over html for markdown storage
|
||||
content = content.get("text") or content.get("html", "")
|
||||
|
||||
if not content or not content.strip():
|
||||
if not isinstance(content, str) or not content.strip():
|
||||
raise MicropubValidationError("Content cannot be empty")
|
||||
|
||||
return content.strip()
|
||||
|
||||
|
||||
def content_format(properties: dict) -> Optional[str]:
|
||||
"""'html' if the content was sent only as HTML (returned as such by q=source)"""
|
||||
content = _first(properties.get("content"))
|
||||
if isinstance(content, dict) and content.get("html") and not content.get("text"):
|
||||
return "html"
|
||||
return None
|
||||
|
||||
|
||||
def _has_text(content) -> bool:
|
||||
"""Whether a content property has any text. Likes, reposts (v2.5.0) and
|
||||
photo posts (v2.6.5) may come without."""
|
||||
values = content if isinstance(content, list) else [content]
|
||||
return any(
|
||||
(c.strip() if isinstance(c, str) else (c.get("text") or c.get("html") or "").strip())
|
||||
for c in values if c
|
||||
)
|
||||
|
||||
|
||||
def extract_title(properties: dict) -> Optional[str]:
|
||||
"""
|
||||
Extract the explicit title (Micropub 'name' property)
|
||||
@@ -228,8 +273,9 @@ def extract_tags(properties: dict) -> list[str]:
|
||||
List of tag strings
|
||||
"""
|
||||
categories = properties.get("category", [])
|
||||
# Filter out empty strings and strip whitespace
|
||||
return [tag.strip() for tag in categories if tag and tag.strip()]
|
||||
# Strings only: person tags (h-card objects) aren't supported and are
|
||||
# ignored, as Micropub says to ignore what isn't recognised (3.3.5)
|
||||
return [tag.strip() for tag in categories if isinstance(tag, str) and tag.strip()]
|
||||
|
||||
|
||||
def extract_published_date(properties: dict) -> Optional[datetime]:
|
||||
@@ -242,22 +288,85 @@ def extract_published_date(properties: dict) -> Optional[datetime]:
|
||||
Returns:
|
||||
Datetime object if published date provided, None otherwise
|
||||
"""
|
||||
published = properties.get("published", [""])[0]
|
||||
published = _first(properties.get("published"))
|
||||
if not published:
|
||||
return None
|
||||
|
||||
try:
|
||||
# Parse ISO 8601 datetime
|
||||
# datetime.fromisoformat handles most ISO formats
|
||||
return datetime.fromisoformat(published.replace("Z", "+00:00"))
|
||||
except (ValueError, AttributeError):
|
||||
# If parsing fails, log and return None (will use current time)
|
||||
current_app.logger.warning(f"Failed to parse published date: {published}")
|
||||
return None
|
||||
value = datetime.fromisoformat(str(published).strip().replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
raise MicropubValidationError(f"published is not an ISO 8601 date: {published}")
|
||||
# Stored times are naive UTC, like every other note (v2.9.0)
|
||||
if value.tzinfo is not None:
|
||||
value = value.astimezone(timezone.utc).replace(tzinfo=None)
|
||||
return value
|
||||
|
||||
|
||||
# Action Handlers
|
||||
|
||||
MAX_PHOTOS = 4 # ADR-057
|
||||
MAX_REMOTE_PHOTO_BYTES = 20 * 1024 * 1024
|
||||
|
||||
# Properties we can store, and so can be updated (v2.4.0)
|
||||
UPDATABLE = {"content", "name", "category", "in-reply-to", "post-status", "photo"}
|
||||
|
||||
|
||||
def _first(value):
|
||||
if isinstance(value, list):
|
||||
return value[0] if value else None
|
||||
return value
|
||||
|
||||
|
||||
def _require_scope(token_info: dict, *scopes: str) -> None:
|
||||
granted = token_info.get("scope", "")
|
||||
if not any(check_scope(s, granted) for s in scopes):
|
||||
raise MicropubError(
|
||||
"insufficient_scope", f"Token lacks {' or '.join(scopes)} scope", status_code=403
|
||||
)
|
||||
|
||||
|
||||
def extract_post_status(properties: dict) -> bool:
|
||||
"""post-status: 'draft' -> unpublished; 'published' or absent -> published"""
|
||||
status = _first(properties.get("post-status"))
|
||||
if status in (None, "", "published"):
|
||||
return True
|
||||
if status == "draft":
|
||||
return False
|
||||
raise MicropubValidationError(f"Unsupported post-status: {status}")
|
||||
|
||||
|
||||
def extract_visibility_draft(properties: dict) -> bool:
|
||||
"""
|
||||
True if visibility asks for a non-public post (v2.9.0)
|
||||
|
||||
StarPunk only has public posts and drafts, so private and unlisted posts
|
||||
are kept as drafts rather than published.
|
||||
"""
|
||||
visibility = _first(properties.get("visibility"))
|
||||
if visibility in (None, "", "public"):
|
||||
return False
|
||||
if visibility in ("private", "unlisted"):
|
||||
return True
|
||||
raise MicropubValidationError(f"Unsupported visibility: {visibility}")
|
||||
|
||||
|
||||
UNSUPPORTED_MEDIA = ("video", "audio")
|
||||
|
||||
|
||||
def extract_url_property(properties: dict, name: str) -> Optional[str]:
|
||||
"""A URL property (in-reply-to, like-of, repost-of): string or h-cite"""
|
||||
value = _first(properties.get(name))
|
||||
if isinstance(value, dict): # h-cite
|
||||
value = _first((value.get("properties") or {}).get("url")) or value.get("value")
|
||||
value = (value or "").strip() if isinstance(value, str) else ""
|
||||
if value and not value.startswith(("http://", "https://")):
|
||||
raise MicropubValidationError(f"{name} must be an http(s) URL")
|
||||
return value or None
|
||||
|
||||
|
||||
def extract_in_reply_to(properties: dict) -> Optional[str]:
|
||||
return extract_url_property(properties, "in-reply-to")
|
||||
|
||||
|
||||
def extract_photos(properties: dict) -> list[dict[str, str]]:
|
||||
"""
|
||||
@@ -265,101 +374,173 @@ def extract_photos(properties: dict) -> list[dict[str, str]]:
|
||||
|
||||
Handles both simple URL strings and structured photo objects with alt text.
|
||||
|
||||
Args:
|
||||
properties: Normalized Micropub properties dict
|
||||
|
||||
Returns:
|
||||
List of dicts with 'url' and optional 'alt' keys
|
||||
|
||||
Examples:
|
||||
>>> # Simple URL
|
||||
>>> extract_photos({'photo': ['https://example.com/photo.jpg']})
|
||||
[{'url': 'https://example.com/photo.jpg', 'alt': ''}]
|
||||
|
||||
>>> # With alt text
|
||||
>>> extract_photos({'photo': [{'value': 'https://example.com/photo.jpg', 'alt': 'Sunset'}]})
|
||||
[{'url': 'https://example.com/photo.jpg', 'alt': 'Sunset'}]
|
||||
"""
|
||||
photos = properties.get("photo", [])
|
||||
result = []
|
||||
|
||||
for photo in photos:
|
||||
for photo in properties.get("photo", []):
|
||||
if isinstance(photo, str):
|
||||
# Simple URL string
|
||||
result.append({'url': photo, 'alt': ''})
|
||||
elif isinstance(photo, dict):
|
||||
# Structured object with value and alt
|
||||
url = photo.get('value') or photo.get('url', '')
|
||||
alt = photo.get('alt', '')
|
||||
if url:
|
||||
result.append({'url': url, 'alt': alt})
|
||||
|
||||
result.append({'url': url, 'alt': photo.get('alt', '')})
|
||||
return result
|
||||
|
||||
|
||||
def _attach_photos_to_note(note_id: int, photos: list[dict[str, str]]) -> None:
|
||||
"""
|
||||
Attach photos to a note by URL
|
||||
def _public_address(host: str) -> bool:
|
||||
"""True if every address the host resolves to is public (no SSRF into the LAN)"""
|
||||
import ipaddress
|
||||
import socket
|
||||
|
||||
Photos must already exist on this server (uploaded via media endpoint).
|
||||
External URLs are accepted but stored as-is (no download).
|
||||
try:
|
||||
infos = socket.getaddrinfo(host, None)
|
||||
except socket.gaierror:
|
||||
return False
|
||||
for info in infos:
|
||||
ip = ipaddress.ip_address(info[4][0])
|
||||
if not ip.is_global:
|
||||
return False
|
||||
return bool(infos)
|
||||
|
||||
Args:
|
||||
note_id: ID of the note to attach to
|
||||
photos: List of dicts with 'url' and 'alt' keys
|
||||
|
||||
def _fetch_remote_photo(url: str) -> Optional[int]:
|
||||
"""
|
||||
Download a photo hosted elsewhere into the media library. Returns media id.
|
||||
|
||||
Uses the webmention fetcher (v2.9.0): every redirect is checked against
|
||||
private addresses and the connection is pinned to the checked address.
|
||||
"""
|
||||
from urllib.parse import urlparse
|
||||
from starpunk.media import save_media
|
||||
from starpunk.webmention_ssrf import safe_get, user_agent
|
||||
|
||||
try:
|
||||
result = safe_get(
|
||||
url, headers={"User-Agent": user_agent(current_app.config.get("SITE_URL", ""))},
|
||||
max_bytes=MAX_REMOTE_PHOTO_BYTES, timeout=20,
|
||||
)
|
||||
if result.status_code != 200:
|
||||
raise ValueError(f"HTTP {result.status_code}")
|
||||
name = urlparse(result.url).path.rsplit("/", 1)[-1] or "photo.jpg"
|
||||
return save_media(result.content, name)["id"]
|
||||
except Exception as e:
|
||||
current_app.logger.warning(f"Could not fetch photo {url}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def _media_id_for_url(url: str) -> Optional[int]:
|
||||
"""Media id for a photo URL: ours by path, or downloaded if hosted elsewhere"""
|
||||
from starpunk.database import get_db
|
||||
|
||||
site_url = current_app.config.get("SITE_URL", "http://localhost:5000").rstrip('/')
|
||||
if url.startswith(site_url) or url.startswith('/media/'):
|
||||
path = url[len(site_url):] if url.startswith(site_url) else url
|
||||
if path.startswith('/media/'):
|
||||
path = path[len('/media/'):]
|
||||
row = get_db(current_app).execute("SELECT id FROM media WHERE path = ?", (path,)).fetchone()
|
||||
if row:
|
||||
return row[0]
|
||||
current_app.logger.warning(f"Photo URL not found in media: {url}")
|
||||
return None
|
||||
return _fetch_remote_photo(url)
|
||||
|
||||
|
||||
def _save_uploads(files) -> list[dict]:
|
||||
"""Photos uploaded in a multipart create request -> [{'media_id', 'alt'}]"""
|
||||
from starpunk.media import save_media
|
||||
|
||||
result = []
|
||||
for key in ("photo", "photo[]"):
|
||||
for upload in (files.getlist(key) if files else []):
|
||||
if upload and upload.filename:
|
||||
try:
|
||||
result.append({"media_id": save_media(upload.read(), upload.filename)["id"], "alt": ""})
|
||||
except ValueError as e:
|
||||
raise MicropubValidationError(f"Invalid photo {upload.filename}: {e}")
|
||||
return result
|
||||
|
||||
|
||||
def _resolve_photos(photos: list[dict]) -> tuple[list[int], list[str]]:
|
||||
"""Photo dicts ({'url'|'media_id', 'alt'}) -> (media ids, captions), max 4"""
|
||||
if len(photos) > MAX_PHOTOS:
|
||||
current_app.logger.warning(
|
||||
f"Micropub received {len(photos)} photos, truncating to {MAX_PHOTOS} per ADR-057"
|
||||
)
|
||||
media_ids, captions = [], []
|
||||
for photo in photos:
|
||||
if len(media_ids) == MAX_PHOTOS:
|
||||
break
|
||||
media_id = photo.get("media_id") or _media_id_for_url(photo["url"])
|
||||
if not media_id:
|
||||
# Don't create or change the post without it (v2.9.0)
|
||||
raise MicropubValidationError(f"Could not get photo {photo.get('url')}")
|
||||
# The same photo twice is shown once (v2.6.5)
|
||||
if media_id not in media_ids:
|
||||
media_ids.append(media_id)
|
||||
captions.append(photo.get("alt", ""))
|
||||
return media_ids, captions
|
||||
|
||||
|
||||
def _attach_photos_to_note(note_id: int, photos: list[dict]) -> None:
|
||||
"""Attach photos (by URL or uploaded media id) to a note, replacing any others"""
|
||||
from starpunk.media import attach_media_to_note
|
||||
|
||||
# Normalize SITE_URL by stripping trailing slash for consistent comparison
|
||||
media_ids, captions = _resolve_photos(photos)
|
||||
attach_media_to_note(note_id, media_ids, captions)
|
||||
|
||||
|
||||
def _note_photos(note) -> list[dict]:
|
||||
"""A note's current photos as Micropub photo dicts"""
|
||||
from starpunk.media import get_note_media
|
||||
|
||||
site_url = current_app.config.get("SITE_URL", "http://localhost:5000").rstrip('/')
|
||||
db = get_db(current_app)
|
||||
|
||||
media_ids = []
|
||||
captions = []
|
||||
|
||||
# Log warning if photos are being truncated
|
||||
if len(photos) > 4:
|
||||
current_app.logger.warning(
|
||||
f"Micropub create received {len(photos)} photos, truncating to 4 per ADR-057"
|
||||
)
|
||||
|
||||
for photo in photos[:4]: # Max 4 photos per ADR-057
|
||||
url = photo['url']
|
||||
alt = photo.get('alt', '')
|
||||
|
||||
# Check if URL is on our server
|
||||
if url.startswith(site_url) or url.startswith('/media/'):
|
||||
# Extract path from URL
|
||||
if url.startswith(site_url):
|
||||
path = url[len(site_url):]
|
||||
else:
|
||||
path = url
|
||||
|
||||
# Remove leading /media/ if present
|
||||
if path.startswith('/media/'):
|
||||
path = path[7:]
|
||||
|
||||
# Look up media by path
|
||||
row = db.execute(
|
||||
"SELECT id FROM media WHERE path = ?",
|
||||
(path,)
|
||||
).fetchone()
|
||||
|
||||
if row:
|
||||
media_ids.append(row[0])
|
||||
captions.append(alt)
|
||||
else:
|
||||
current_app.logger.warning(f"Photo URL not found in media: {url}")
|
||||
else:
|
||||
# External URL - log but don't fail
|
||||
current_app.logger.info(f"External photo URL ignored: {url}")
|
||||
|
||||
if media_ids:
|
||||
attach_media_to_note(note_id, media_ids, captions)
|
||||
return [
|
||||
{"url": f"{site_url}/media/{m['path']}", "alt": m.get("caption") or "", "media_id": m["id"]}
|
||||
for m in get_note_media(note.id)
|
||||
]
|
||||
|
||||
|
||||
def handle_create(data: dict, token_info: dict):
|
||||
def _check_type(data: dict) -> None:
|
||||
"""Only h-entry is supported"""
|
||||
h = _first(data.get("h"))
|
||||
kind = _first(data.get("type"))
|
||||
if (h and h != "entry") or (kind and kind != "h-entry"):
|
||||
raise MicropubValidationError(f"Only h-entry posts are supported (got {h or kind})")
|
||||
|
||||
|
||||
def _note_from_url(url: Optional[str], deleted: bool = False):
|
||||
"""Find the note a URL points at (/note/<slug> on this site)"""
|
||||
from urllib.parse import urlparse
|
||||
from starpunk.database import get_db
|
||||
|
||||
if not url:
|
||||
raise MicropubValidationError("No url provided")
|
||||
site = urlparse(current_app.config.get("SITE_URL", ""))
|
||||
parsed = urlparse(url)
|
||||
if parsed.netloc and site.netloc and parsed.netloc != site.netloc:
|
||||
raise MicropubValidationError("URL is not on this site")
|
||||
parts = [p for p in parsed.path.split("/") if p]
|
||||
if len(parts) != 2 or parts[0] not in ("note", "notes"):
|
||||
raise MicropubValidationError("URL is not a post on this site")
|
||||
slug = parts[1]
|
||||
if deleted:
|
||||
row = get_db(current_app).execute(
|
||||
"SELECT id FROM notes WHERE slug = ? AND deleted_at IS NOT NULL", (slug,)
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise MicropubError("not_found", "No deleted post at that URL", 404)
|
||||
return slug
|
||||
note = get_note(slug=slug)
|
||||
if note is None:
|
||||
raise MicropubError("not_found", "Post not found", 404)
|
||||
return note
|
||||
|
||||
|
||||
def handle_create(data: dict, token_info: dict, files=None):
|
||||
"""
|
||||
Handle Micropub create action
|
||||
|
||||
@@ -369,89 +550,246 @@ def handle_create(data: dict, token_info: dict):
|
||||
Args:
|
||||
data: Raw request data (form or JSON)
|
||||
token_info: Authenticated token information (me, client_id, scope)
|
||||
files: Uploaded files for multipart requests (v2.4.0)
|
||||
|
||||
Returns:
|
||||
Tuple of (response_body, status_code, headers)
|
||||
|
||||
Raises:
|
||||
MicropubError: If scope insufficient or creation fails
|
||||
"""
|
||||
# Check scope
|
||||
if not check_scope("create", token_info.get("scope", "")):
|
||||
raise MicropubError(
|
||||
"insufficient_scope", "Token lacks create scope", status_code=403
|
||||
)
|
||||
_require_scope(token_info, "create", "draft")
|
||||
# A token with only the draft scope may create drafts (v2.9.0)
|
||||
drafts_only = not check_scope("create", token_info.get("scope", ""))
|
||||
_check_type(data)
|
||||
|
||||
# Extract mp-slug BEFORE normalizing properties (it's not a property!)
|
||||
# mp-slug is a Micropub server extension parameter that gets filtered during normalization
|
||||
custom_slug = None
|
||||
if isinstance(data, dict) and 'mp-slug' in data:
|
||||
# Handle both form-encoded (list) and JSON (could be string or list)
|
||||
slug_value = data.get('mp-slug')
|
||||
if isinstance(slug_value, list) and slug_value:
|
||||
custom_slug = slug_value[0]
|
||||
elif isinstance(slug_value, str):
|
||||
custom_slug = slug_value
|
||||
# mp-slug and mp-syndicate-to are server commands, not properties. Form
|
||||
# posts send them at the top level; JSON clients inside properties too.
|
||||
commands = dict(data) if isinstance(data, dict) else {}
|
||||
if isinstance(commands.get("properties"), dict):
|
||||
commands.update({k: v for k, v in commands["properties"].items() if k.startswith("mp-")})
|
||||
|
||||
# mp-syndicate-to is also a server command, not a property (v2.1.0).
|
||||
# Absent means "all targets"; present means exactly those.
|
||||
requested_targets = None
|
||||
if isinstance(data, dict):
|
||||
for key in ("mp-syndicate-to", "mp-syndicate-to[]"):
|
||||
if key in data:
|
||||
value = data[key]
|
||||
requested_targets = (requested_targets or []) + (
|
||||
value if isinstance(value, list) else [value]
|
||||
)
|
||||
custom_slug = _first(commands.get("mp-slug"))
|
||||
if custom_slug is not None and not isinstance(custom_slug, str):
|
||||
raise MicropubValidationError("mp-slug must be a string")
|
||||
|
||||
# Crossposting only to the targets asked for: absent means none (v2.9.0)
|
||||
requested_targets = []
|
||||
for key in ("mp-syndicate-to", "mp-syndicate-to[]"):
|
||||
if key in commands:
|
||||
value = commands[key]
|
||||
requested_targets += value if isinstance(value, list) else [value]
|
||||
|
||||
# Normalize and extract properties
|
||||
try:
|
||||
properties = normalize_properties(data)
|
||||
content = extract_content(properties)
|
||||
like_of = extract_url_property(properties, "like-of")
|
||||
repost_of = extract_url_property(properties, "repost-of")
|
||||
bookmark_of = extract_url_property(properties, "bookmark-of")
|
||||
if sum(bool(u) for u in (like_of, repost_of, bookmark_of)) > 1:
|
||||
raise MicropubValidationError("A post can only be one of a like, repost or bookmark")
|
||||
for prop in UNSUPPORTED_MEDIA:
|
||||
if properties.get(prop) or (files and (files.getlist(prop) or files.getlist(f"{prop}[]"))):
|
||||
raise MicropubValidationError(f"{prop} posts are not supported")
|
||||
has_content = _has_text(properties.get("content")) or _has_text(properties.get("summary"))
|
||||
photos = extract_photos(properties) + _save_uploads(files)
|
||||
title = extract_title(properties)
|
||||
# Photo posts may have no text (v2.6.5), nor titled posts (v2.9.0)
|
||||
content = (
|
||||
extract_content(properties)
|
||||
if has_content or not (like_of or repost_of or bookmark_of or photos or title)
|
||||
else ""
|
||||
)
|
||||
tags = extract_tags(properties)
|
||||
published_date = extract_published_date(properties)
|
||||
photos = extract_photos(properties) # v1.4.0
|
||||
|
||||
except MicropubValidationError as e:
|
||||
raise e
|
||||
published = extract_post_status(properties) and not extract_visibility_draft(properties)
|
||||
if drafts_only:
|
||||
published = False
|
||||
in_reply_to = extract_in_reply_to(properties)
|
||||
fmt = content_format(properties) if has_content else None
|
||||
# Look up and download photos before the note exists, so a bad photo
|
||||
# can't leave a half-made post behind (v2.6.5)
|
||||
media_ids, captions = _resolve_photos(photos)
|
||||
except MicropubError:
|
||||
raise
|
||||
except Exception as e:
|
||||
current_app.logger.error(f"Property extraction failed: {e}")
|
||||
raise MicropubValidationError(f"Failed to parse request: {str(e)}")
|
||||
raise MicropubValidationError("Could not read the post's properties")
|
||||
|
||||
# Create note using existing CRUD
|
||||
note = None
|
||||
try:
|
||||
note = create_note(
|
||||
content=add_title_heading(content, title),
|
||||
published=True, # Micropub posts are published by default
|
||||
published=published,
|
||||
created_at=published_date,
|
||||
custom_slug=custom_slug,
|
||||
tags=tags if tags else None # Pass tags to create_note (v1.3.0)
|
||||
tags=tags if tags else None,
|
||||
in_reply_to=in_reply_to,
|
||||
like_of=like_of,
|
||||
repost_of=repost_of,
|
||||
bookmark_of=bookmark_of,
|
||||
has_media=bool(media_ids),
|
||||
content_format=fmt,
|
||||
)
|
||||
|
||||
# Attach photos if present (v1.4.0)
|
||||
if photos:
|
||||
_attach_photos_to_note(note.id, photos)
|
||||
if media_ids:
|
||||
from starpunk.media import attach_media_to_note
|
||||
attach_media_to_note(note.id, media_ids, captions)
|
||||
|
||||
# Crosspost after photos are attached (v2.1.0)
|
||||
from starpunk.syndication import get_targets, resolve_targets, select_targets
|
||||
if requested_targets is None:
|
||||
select_targets(note.id, list(get_targets()))
|
||||
else:
|
||||
select_targets(note.id, resolve_targets(requested_targets))
|
||||
# Crosspost after photos are attached (v2.1.0); drafts wait until published
|
||||
from starpunk.syndication import resolve_targets, select_targets
|
||||
select_targets(note.id, resolve_targets(requested_targets))
|
||||
|
||||
permalink = absolute_url_for("public.note", slug=note.slug)
|
||||
|
||||
# Return 201 Created with Location header
|
||||
return "", 201, {"Location": permalink}
|
||||
return "", 201, {"Location": absolute_url_for("public.note", slug=note.slug)}
|
||||
|
||||
except InvalidNoteDataError as e:
|
||||
raise MicropubValidationError(str(e))
|
||||
except MicropubError:
|
||||
raise
|
||||
except Exception as e:
|
||||
current_app.logger.error(f"Failed to create note via Micropub: {e}")
|
||||
raise MicropubError(
|
||||
"server_error", "Failed to create post", status_code=500
|
||||
)
|
||||
# The client is told it failed, so don't leave the post up (v2.6.5)
|
||||
if note is not None:
|
||||
try:
|
||||
from starpunk.notes import delete_note
|
||||
delete_note(id=note.id, soft=False)
|
||||
except Exception as cleanup:
|
||||
current_app.logger.error(f"Could not remove failed note {note.slug}: {cleanup}")
|
||||
raise MicropubError("server_error", "Failed to create post", status_code=500)
|
||||
|
||||
|
||||
def handle_update(data: dict, token_info: dict):
|
||||
"""
|
||||
Handle Micropub update action (v2.4.0)
|
||||
|
||||
JSON body: {"action": "update", "url": ..., "replace": {...},
|
||||
"add": {...}, "delete": [...] or {...}}. Edits follow to crossposts.
|
||||
"""
|
||||
from starpunk.syndication import note_changed
|
||||
|
||||
_require_scope(token_info, "update")
|
||||
note = _note_from_url(_first(data.get("url")))
|
||||
|
||||
replace = data.get("replace") or {}
|
||||
add = data.get("add") or {}
|
||||
delete = data.get("delete") or {}
|
||||
if not isinstance(replace, dict) or not isinstance(add, dict) or not isinstance(delete, (dict, list)):
|
||||
raise MicropubValidationError("replace and add must be objects; delete an object or list")
|
||||
for prop, values in list(replace.items()) + list(add.items()) + (
|
||||
list(delete.items()) if isinstance(delete, dict) else []):
|
||||
if not isinstance(values, list):
|
||||
raise MicropubValidationError(f"Values for {prop} must be an array")
|
||||
if not (replace or add or delete):
|
||||
raise MicropubValidationError("An update needs replace, add or delete")
|
||||
touched = set(replace) | set(add) | set(delete)
|
||||
unsupported = sorted(touched - UPDATABLE)
|
||||
if unsupported:
|
||||
raise MicropubValidationError(f"Updating {', '.join(unsupported)} is not supported")
|
||||
if set(add) - {"category", "photo"}:
|
||||
raise MicropubValidationError("Only category and photo can be added to")
|
||||
|
||||
from starpunk.tags import get_note_tags
|
||||
|
||||
title, body = split_title(note.content)
|
||||
tags = [t["display_name"] for t in get_note_tags(note.id, entered_order=True)]
|
||||
photos = _note_photos(note)
|
||||
changes = {}
|
||||
|
||||
# Removals first, then replacements, then additions
|
||||
if isinstance(delete, list):
|
||||
for prop in delete:
|
||||
if prop == "name":
|
||||
title = None
|
||||
elif prop == "content":
|
||||
body = ""
|
||||
elif prop == "category":
|
||||
tags = []
|
||||
elif prop == "photo":
|
||||
photos = []
|
||||
elif prop == "in-reply-to":
|
||||
changes["in_reply_to"] = ""
|
||||
elif prop == "post-status":
|
||||
changes["published"] = True
|
||||
else:
|
||||
for prop, values in delete.items():
|
||||
if prop == "category":
|
||||
drop = {v.lower() for v in values}
|
||||
tags = [t for t in tags if t.lower() not in drop]
|
||||
elif prop == "photo":
|
||||
drop = {p["url"] for p in extract_photos({"photo": values})}
|
||||
photos = [p for p in photos if p["url"] not in drop]
|
||||
else:
|
||||
raise MicropubValidationError(f"Can't delete individual values of {prop}")
|
||||
|
||||
if "content" in replace:
|
||||
# Empty text is fine if the note keeps its photos; update_note checks (v2.6.5)
|
||||
body = extract_content(replace) if _has_text(replace["content"]) else ""
|
||||
changes["content_format"] = content_format(replace) if body else None
|
||||
if "name" in replace:
|
||||
title = extract_title(replace)
|
||||
if "category" in replace:
|
||||
tags = extract_tags(replace)
|
||||
if "photo" in replace:
|
||||
photos = extract_photos(replace)
|
||||
if "in-reply-to" in replace:
|
||||
changes["in_reply_to"] = extract_in_reply_to(replace) or ""
|
||||
if "post-status" in replace:
|
||||
changes["published"] = extract_post_status(replace)
|
||||
|
||||
if "category" in add:
|
||||
known = {t.lower() for t in tags}
|
||||
tags += [t for t in extract_tags(add) if t.lower() not in known]
|
||||
if "photo" in add:
|
||||
photos += extract_photos(add)
|
||||
|
||||
# Text may go only if photos stay (or it's a like, repost or bookmark) (v2.6.5)
|
||||
if not (body or "").strip() and not (photos or note.reacts_to):
|
||||
raise MicropubValidationError("A post needs text or a photo")
|
||||
|
||||
# Photos must all resolve before anything changes (v2.9.0)
|
||||
if "photo" in touched:
|
||||
media_ids, captions = _resolve_photos(photos)
|
||||
|
||||
new_content = add_title_heading(body, title)
|
||||
if new_content != note.content:
|
||||
changes["content"] = new_content
|
||||
elif "content_format" in changes and changes["content_format"] == note.content_format:
|
||||
del changes["content_format"]
|
||||
if {"category"} & touched:
|
||||
changes["tags"] = tags
|
||||
|
||||
try:
|
||||
if changes:
|
||||
update_note(id=note.id, has_media=bool(photos), **changes)
|
||||
if "photo" in touched:
|
||||
from starpunk.media import attach_media_to_note
|
||||
attach_media_to_note(note.id, media_ids, captions)
|
||||
note_changed(note.id)
|
||||
if not changes: # update_note already told WebSub otherwise
|
||||
from starpunk import websub
|
||||
websub.note_changed(note, was_public=note.published)
|
||||
except InvalidNoteDataError as e:
|
||||
raise MicropubValidationError(str(e))
|
||||
except MicropubError:
|
||||
raise
|
||||
except Exception as e:
|
||||
current_app.logger.error(f"Failed to update note via Micropub: {e}")
|
||||
raise MicropubError("server_error", "Failed to update post", status_code=500)
|
||||
|
||||
return "", 204
|
||||
|
||||
|
||||
def handle_delete(data: dict, token_info: dict, undelete: bool = False):
|
||||
"""Handle Micropub delete / undelete (v2.4.0). Deletes are soft so they can be undone."""
|
||||
_require_scope(token_info, "delete")
|
||||
url = _first(data.get("url"))
|
||||
try:
|
||||
if undelete:
|
||||
undelete_note(_note_from_url(url, deleted=True))
|
||||
else:
|
||||
delete_note(id=_note_from_url(url).id, soft=True)
|
||||
except MicropubError:
|
||||
raise
|
||||
except Exception as e:
|
||||
current_app.logger.error(f"Micropub {'undelete' if undelete else 'delete'} failed: {e}")
|
||||
raise MicropubError("server_error", "Failed to change post", status_code=500)
|
||||
return "", 204
|
||||
|
||||
|
||||
def _syndicate_to() -> list:
|
||||
@@ -461,82 +799,136 @@ def _syndicate_to() -> list:
|
||||
return [{"uid": t.uid, "name": t.name} for t in get_targets().values()]
|
||||
|
||||
|
||||
def handle_query(args: dict, token_info: dict):
|
||||
def note_to_mf2(note, wanted: Optional[list] = None) -> dict:
|
||||
"""
|
||||
A note as Microformats2 JSON, the inverse of create
|
||||
|
||||
A '# Heading' first line is the title, returned as name.
|
||||
"""
|
||||
from starpunk.syndication import get_note_syndications
|
||||
|
||||
title, content = split_title(note.content)
|
||||
props = {
|
||||
# HTML-authored content comes back as HTML (3.7.2, v2.9.0)
|
||||
"content": [{"html": content} if note.content_format == "html" else content],
|
||||
"published": [iso_utc(note.created_at)],
|
||||
"url": [absolute_url_for("public.note", slug=note.slug)],
|
||||
"post-status": ["published" if note.published else "draft"],
|
||||
}
|
||||
if title:
|
||||
props["name"] = [title]
|
||||
from starpunk.tags import get_note_tags
|
||||
|
||||
tags = get_note_tags(note.id, entered_order=True)
|
||||
if tags:
|
||||
props["category"] = [tag["display_name"] for tag in tags]
|
||||
if getattr(note, "in_reply_to", None):
|
||||
props["in-reply-to"] = [note.in_reply_to]
|
||||
if getattr(note, "like_of", None):
|
||||
props["like-of"] = [note.like_of]
|
||||
if getattr(note, "repost_of", None):
|
||||
props["repost-of"] = [note.repost_of]
|
||||
if getattr(note, "bookmark_of", None):
|
||||
props["bookmark-of"] = [note.bookmark_of]
|
||||
if not content.strip():
|
||||
del props["content"] # photo posts, likes, reposts and bookmarks without text
|
||||
photos = _note_photos(note)
|
||||
if photos:
|
||||
props["photo"] = [{"value": p["url"], "alt": p["alt"]} for p in photos]
|
||||
synced = [s["remote_url"] for s in get_note_syndications(note.id) if s.get("remote_url")]
|
||||
if synced:
|
||||
props["syndication"] = synced
|
||||
|
||||
if wanted:
|
||||
return {"properties": {k: v for k, v in props.items() if k in wanted}}
|
||||
return {"type": ["h-entry"], "properties": props}
|
||||
|
||||
|
||||
def _all_categories(prefix: str = "") -> list:
|
||||
"""Tags in use, most used first (q=category)"""
|
||||
from starpunk.database import get_db
|
||||
|
||||
rows = get_db(current_app).execute(
|
||||
"""
|
||||
SELECT t.display_name, COUNT(nt.note_id) AS uses
|
||||
FROM tags t LEFT JOIN note_tags nt ON nt.tag_id = t.id
|
||||
GROUP BY t.id ORDER BY uses DESC, t.display_name
|
||||
"""
|
||||
).fetchall()
|
||||
prefix = prefix.lower()
|
||||
return [r["display_name"] for r in rows if not prefix or prefix in r["display_name"].lower()]
|
||||
|
||||
|
||||
QUERIES = ["config", "source", "syndicate-to", "category"]
|
||||
|
||||
|
||||
def handle_query(args, token_info: dict):
|
||||
"""
|
||||
Handle Micropub query endpoints
|
||||
|
||||
Supports:
|
||||
- q=config: Return server configuration
|
||||
- q=source: Return post source in Microformats2 JSON
|
||||
- q=syndicate-to: Return syndication targets
|
||||
- q=config: Server configuration
|
||||
- q=source&url=...[&properties[]=...]: One post in Microformats2 JSON
|
||||
- q=source (no url) [&limit=N&offset=N]: Recent posts, drafts included
|
||||
- q=syndicate-to: Crosspost targets
|
||||
- q=category[&filter=...]: Tags in use
|
||||
|
||||
Args:
|
||||
args: Query string arguments
|
||||
token_info: Authenticated token information
|
||||
|
||||
Returns:
|
||||
Tuple of (response, status_code)
|
||||
args: Query arguments (werkzeug MultiDict or plain dict)
|
||||
"""
|
||||
def get_list(key):
|
||||
if hasattr(args, "getlist"):
|
||||
return args.getlist(key) + args.getlist(f"{key}[]")
|
||||
value = args.get(key) or args.get(f"{key}[]")
|
||||
return value if isinstance(value, list) else ([value] if value else [])
|
||||
|
||||
q = args.get("q")
|
||||
|
||||
# Posts (drafts included) and tags are for apps that may post (v2.9.0);
|
||||
# config and syndicate-to are fine for any token
|
||||
if q in ("source", "category"):
|
||||
_require_scope(token_info, "create", "update", "delete", "draft")
|
||||
|
||||
if q == "config":
|
||||
# Return server configuration with media endpoint (v1.4.0)
|
||||
site_url = current_app.config.get("SITE_URL", "http://localhost:5000").rstrip('/')
|
||||
config = {
|
||||
return jsonify({
|
||||
"media-endpoint": f"{site_url}/micropub/media",
|
||||
"syndicate-to": _syndicate_to(),
|
||||
"q": QUERIES,
|
||||
"post-types": [
|
||||
{"type": "note", "name": "Note", "properties": ["content"]},
|
||||
{"type": "photo", "name": "Photo", "properties": ["photo"]}
|
||||
{"type": "article", "name": "Article", "properties": ["name", "content"]},
|
||||
{"type": "photo", "name": "Photo", "properties": ["photo"]},
|
||||
{"type": "reply", "name": "Reply", "properties": ["in-reply-to", "content"]},
|
||||
{"type": "like", "name": "Like", "properties": ["like-of"]},
|
||||
{"type": "repost", "name": "Repost", "properties": ["repost-of", "content"]},
|
||||
{"type": "bookmark", "name": "Bookmark", "properties": ["bookmark-of", "content"]},
|
||||
],
|
||||
}
|
||||
return jsonify(config), 200
|
||||
}), 200
|
||||
|
||||
elif q == "source":
|
||||
# Return source of a specific post
|
||||
if q == "source":
|
||||
url = args.get("url")
|
||||
if not url:
|
||||
return error_response("invalid_request", "No URL provided")
|
||||
|
||||
# Extract slug from URL
|
||||
wanted = get_list("properties")
|
||||
if url:
|
||||
try:
|
||||
note = _note_from_url(url)
|
||||
except MicropubError as e:
|
||||
return error_response("invalid_request" if e.status_code == 400 else e.error,
|
||||
e.error_description, e.status_code)
|
||||
return jsonify(note_to_mf2(note, wanted)), 200
|
||||
try:
|
||||
# URL format: https://example.com/note/{slug}
|
||||
slug = url.rstrip("/").split("/")[-1]
|
||||
note = get_note(slug)
|
||||
limit = max(1, min(int(args.get("limit", 10)), 50))
|
||||
offset = max(0, int(args.get("offset", 0)))
|
||||
except ValueError:
|
||||
return error_response("invalid_request", "limit and offset must be numbers")
|
||||
notes = list_notes(published_only=False, limit=limit, offset=offset)
|
||||
items = [note_to_mf2(n, wanted) for n in notes]
|
||||
return jsonify({"items": items}), 200
|
||||
|
||||
# Check if note exists
|
||||
if note is None:
|
||||
return error_response("invalid_request", "Post not found")
|
||||
|
||||
except NoteNotFoundError:
|
||||
return error_response("invalid_request", "Post not found")
|
||||
except Exception as e:
|
||||
current_app.logger.error(f"Failed to get note source: {e}")
|
||||
return error_response("server_error", "Failed to retrieve post")
|
||||
|
||||
# Convert note to Micropub Microformats2 format. A '# Heading' first
|
||||
# line is the title, returned as name (mirrors how create stores it).
|
||||
title, content = split_title(note.content)
|
||||
mf2 = {
|
||||
"type": ["h-entry"],
|
||||
"properties": {
|
||||
"content": [content],
|
||||
"published": [note.created_at.isoformat()],
|
||||
"url": [absolute_url_for("public.note", slug=note.slug)],
|
||||
},
|
||||
}
|
||||
|
||||
if title:
|
||||
mf2["properties"]["name"] = [title]
|
||||
|
||||
# Add tags if present (v1.3.0)
|
||||
if note.tags:
|
||||
mf2["properties"]["category"] = [tag["display_name"] for tag in note.tags]
|
||||
|
||||
return jsonify(mf2), 200
|
||||
|
||||
elif q == "syndicate-to":
|
||||
if q == "syndicate-to":
|
||||
return jsonify({"syndicate-to": _syndicate_to()}), 200
|
||||
|
||||
else:
|
||||
return error_response("invalid_request", f"Unknown query: {q}")
|
||||
if q == "category":
|
||||
return jsonify({"categories": _all_categories(args.get("filter") or "")}), 200
|
||||
|
||||
return error_response("invalid_request", f"Unknown query: {q}" if q else "Missing q parameter")
|
||||
+42
-6
@@ -31,12 +31,47 @@ class MigrationError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def create_migrations_table(conn):
|
||||
def split_sql_statements(script: str) -> list[str]:
|
||||
"""
|
||||
Split a SQL script into complete statements (v2.8.1)
|
||||
|
||||
Uses sqlite3.complete_statement, so semicolons inside strings and
|
||||
trigger bodies (BEGIN ... END) don't split a statement.
|
||||
"""
|
||||
statements = []
|
||||
current = ""
|
||||
for line in script.splitlines(keepends=True):
|
||||
current += line
|
||||
if sqlite3.complete_statement(current):
|
||||
if current.strip().strip(";").strip():
|
||||
statements.append(current.strip())
|
||||
current = ""
|
||||
if current.strip():
|
||||
statements.append(current.strip())
|
||||
return statements
|
||||
|
||||
|
||||
def execute_in_transaction(conn, script: str) -> None:
|
||||
"""
|
||||
Run a SQL script inside the connection's open transaction
|
||||
|
||||
Not conn.executescript(): that COMMITs first, which released the
|
||||
migration lock mid-migration and let another gunicorn worker apply the
|
||||
same migration again ("duplicate column name", v2.8.0 deploy).
|
||||
"""
|
||||
for statement in split_sql_statements(script):
|
||||
conn.execute(statement)
|
||||
|
||||
|
||||
def create_migrations_table(conn, commit=True):
|
||||
"""
|
||||
Create schema_migrations tracking table if it doesn't exist
|
||||
|
||||
Args:
|
||||
conn: SQLite connection
|
||||
commit: Commit afterwards. run_migrations passes False: committing
|
||||
there released its BEGIN IMMEDIATE lock straight away, so
|
||||
gunicorn workers applied migrations concurrently (v2.8.1)
|
||||
"""
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
@@ -51,7 +86,8 @@ def create_migrations_table(conn):
|
||||
ON schema_migrations(migration_name)
|
||||
""")
|
||||
|
||||
conn.commit()
|
||||
if commit:
|
||||
conn.commit()
|
||||
|
||||
|
||||
def is_schema_current(conn):
|
||||
@@ -287,7 +323,7 @@ def apply_migration(conn, migration_name, migration_path, logger=None):
|
||||
|
||||
# Execute migration in transaction
|
||||
conn.execute("BEGIN")
|
||||
conn.executescript(migration_sql)
|
||||
execute_in_transaction(conn, migration_sql)
|
||||
|
||||
# Record migration as applied
|
||||
conn.execute(
|
||||
@@ -374,7 +410,7 @@ def run_migrations(db_path, logger=None):
|
||||
|
||||
try:
|
||||
# Ensure migrations tracking table exists
|
||||
create_migrations_table(conn)
|
||||
create_migrations_table(conn, commit=False)
|
||||
|
||||
# Quick check: have migrations already been applied by another worker?
|
||||
cursor = conn.execute("SELECT COUNT(*) FROM schema_migrations")
|
||||
@@ -472,8 +508,8 @@ def run_migrations(db_path, logger=None):
|
||||
|
||||
logger.debug(f"Applying migration: {migration_name}")
|
||||
|
||||
# Execute migration (already in transaction)
|
||||
conn.executescript(migration_sql)
|
||||
# Execute migration inside the locked transaction
|
||||
execute_in_transaction(conn, migration_sql)
|
||||
|
||||
# Record migration as applied
|
||||
conn.execute(
|
||||
|
||||
+88
-11
@@ -11,10 +11,12 @@ from dataclasses import dataclass, field, replace
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
import re
|
||||
import sqlite3
|
||||
|
||||
# Third-party imports
|
||||
import markdown
|
||||
import nh3
|
||||
|
||||
# Local imports
|
||||
from starpunk.utils import (
|
||||
@@ -40,6 +42,35 @@ MARKDOWN_EXTENSIONS = ["extra", "codehilite", "nl2br", BareUrlExtension()]
|
||||
|
||||
# Constants - Content limits
|
||||
MAX_TITLE_LENGTH = 200
|
||||
|
||||
# HTML allowed in rendered notes (v2.9.0). Markdown passes raw HTML through,
|
||||
# and Micropub clients may send HTML, so scripts, event handlers, iframes and
|
||||
# javascript: links are stripped. class keeps code highlighting and
|
||||
# microformats; id keeps footnote links.
|
||||
ALLOWED_TAGS = set(nh3.ALLOWED_TAGS) | {"video", "audio", "source", "track", "picture"}
|
||||
ALLOWED_ATTRIBUTES = {
|
||||
**{tag: set(attrs) for tag, attrs in nh3.ALLOWED_ATTRIBUTES.items()},
|
||||
"*": {"class", "title", "id", "lang", "dir"},
|
||||
"img": set(nh3.ALLOWED_ATTRIBUTES.get("img", ())) | {"loading", "srcset", "sizes"},
|
||||
"video": {"src", "controls", "poster", "width", "height", "loop", "muted", "playsinline", "preload"},
|
||||
"audio": {"src", "controls", "loop", "muted", "preload"},
|
||||
"source": {"src", "type", "srcset", "media", "sizes"},
|
||||
"track": {"src", "kind", "label", "srclang", "default"},
|
||||
}
|
||||
|
||||
|
||||
def render_markdown(text: str) -> str:
|
||||
"""Note Markdown (or HTML) to sanitized HTML"""
|
||||
html = markdown.markdown(text, extensions=MARKDOWN_EXTENSIONS)
|
||||
return nh3.clean(html, tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRIBUTES)
|
||||
|
||||
|
||||
# Likes, reposts and bookmarks (v2.5.0, v2.9.0)
|
||||
REACTION_VERBS = {"like": "Liked", "repost": "Reposted", "bookmark": "Bookmarked"}
|
||||
REACTION_ICONS = {"like": "★", "repost": "🔁", "bookmark": "🔖"}
|
||||
|
||||
# A Markdown ATX heading: up to 6 '#', whitespace, then text
|
||||
TITLE_RE = re.compile(r"#{1,6}[ \t]+(\S.*)$")
|
||||
EXCERPT_LENGTH = 200
|
||||
|
||||
|
||||
@@ -47,17 +78,19 @@ def split_title(content: str) -> tuple[Optional[str], str]:
|
||||
"""
|
||||
Split a note's '# Heading' first line (its title, per Q22) off its content
|
||||
|
||||
The first line must be a Markdown heading: 1-6 '#' then a space and text.
|
||||
'#indieweb ...' (a hashtag) or a bare '#' is not a title. This is the one
|
||||
place that decides whether a note has a title (templates use has_title).
|
||||
|
||||
Returns:
|
||||
(title, remaining content); title is None if the note has no heading
|
||||
"""
|
||||
stripped = content.lstrip()
|
||||
if not stripped.startswith("#"):
|
||||
return None, content
|
||||
first_line, _, rest = stripped.partition("\n")
|
||||
title = first_line.lstrip("#").strip()
|
||||
if not title:
|
||||
match = TITLE_RE.match(first_line)
|
||||
if not match:
|
||||
return None, content
|
||||
return title, rest.strip()
|
||||
return match.group(1).strip().rstrip("#").strip(), rest.strip()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -134,6 +167,10 @@ class Note:
|
||||
deleted_at: Optional[datetime] = None
|
||||
content_hash: Optional[str] = None
|
||||
in_reply_to: Optional[str] = None # v1.6.0: URL being replied to
|
||||
like_of: Optional[str] = None # v2.5.0: URL liked (this note is a like)
|
||||
repost_of: Optional[str] = None # v2.5.0: URL reposted (this note is a repost)
|
||||
bookmark_of: Optional[str] = None # v2.9.0: URL bookmarked (this note is a bookmark)
|
||||
content_format: Optional[str] = None # v2.9.0: 'html' if sent as HTML via Micropub
|
||||
_cached_content: Optional[str] = field(
|
||||
default=None, repr=False, compare=False, init=False
|
||||
)
|
||||
@@ -192,6 +229,10 @@ class Note:
|
||||
_data_dir=data_dir,
|
||||
content_hash=data.get("content_hash"),
|
||||
in_reply_to=data.get("in_reply_to"),
|
||||
like_of=data.get("like_of"),
|
||||
repost_of=data.get("repost_of"),
|
||||
bookmark_of=data.get("bookmark_of"),
|
||||
content_format=data.get("content_format"),
|
||||
)
|
||||
|
||||
@property
|
||||
@@ -250,11 +291,8 @@ class Note:
|
||||
<p>This is my note content...</p>
|
||||
"""
|
||||
if self._cached_html is None:
|
||||
# Render markdown to HTML
|
||||
html = markdown.markdown(
|
||||
self.content, # This triggers lazy load of content
|
||||
extensions=MARKDOWN_EXTENSIONS,
|
||||
)
|
||||
# Render markdown to sanitized HTML (v2.9.0)
|
||||
html = render_markdown(self.content) # triggers lazy load of content
|
||||
|
||||
# Cache it
|
||||
object.__setattr__(self, "_cached_html", html)
|
||||
@@ -272,7 +310,15 @@ class Note:
|
||||
title, body = split_title(self.content)
|
||||
if title is None:
|
||||
return self.html
|
||||
return markdown.markdown(body, extensions=MARKDOWN_EXTENSIONS)
|
||||
return render_markdown(body)
|
||||
|
||||
@property
|
||||
def has_title(self) -> bool:
|
||||
"""True if the note starts with a '# Heading' title (an article)"""
|
||||
try:
|
||||
return split_title(self.content)[0] is not None
|
||||
except (FileNotFoundError, OSError):
|
||||
return False
|
||||
|
||||
@property
|
||||
def title(self) -> str:
|
||||
@@ -367,12 +413,43 @@ class Note:
|
||||
excerpt = excerpt[:last_space]
|
||||
excerpt += "..."
|
||||
|
||||
if not excerpt and self.reacts_to:
|
||||
# A like, repost or bookmark with no comment
|
||||
return f"{REACTION_VERBS[self.kind]} {self.reacts_to}"
|
||||
|
||||
return excerpt
|
||||
|
||||
except (FileNotFoundError, OSError):
|
||||
# If file doesn't exist, return slug
|
||||
return self.slug
|
||||
|
||||
@property
|
||||
def kind(self) -> str:
|
||||
"""'like', 'repost', 'bookmark' (v2.9.0) or 'note' (v2.5.0)"""
|
||||
if self.like_of:
|
||||
return "like"
|
||||
if self.repost_of:
|
||||
return "repost"
|
||||
if self.bookmark_of:
|
||||
return "bookmark"
|
||||
return "note"
|
||||
|
||||
@property
|
||||
def reacts_to(self) -> Optional[str]:
|
||||
"""The URL a like, repost or bookmark points at"""
|
||||
return self.like_of or self.repost_of or self.bookmark_of
|
||||
|
||||
@property
|
||||
def feed_html(self) -> str:
|
||||
"""HTML for feeds: a repost/like line, then the note's own HTML"""
|
||||
from html import escape
|
||||
|
||||
if not self.reacts_to:
|
||||
return self.html
|
||||
url = escape(self.reacts_to, quote=True)
|
||||
line = f'<p>{REACTION_ICONS[self.kind]} {REACTION_VERBS[self.kind]} <a href="{url}">{url}</a></p>'
|
||||
return line + (self.html or "")
|
||||
|
||||
@property
|
||||
def permalink(self) -> str:
|
||||
"""
|
||||
|
||||
+157
-21
@@ -133,13 +133,22 @@ def _get_existing_slugs(db) -> set[str]:
|
||||
# Core CRUD Functions
|
||||
|
||||
|
||||
# update_note: leave a field as it is
|
||||
UNCHANGED = object()
|
||||
|
||||
|
||||
def create_note(
|
||||
content: str,
|
||||
published: bool = False,
|
||||
created_at: Optional[datetime] = None,
|
||||
custom_slug: Optional[str] = None,
|
||||
tags: Optional[list[str]] = None,
|
||||
in_reply_to: Optional[str] = None
|
||||
in_reply_to: Optional[str] = None,
|
||||
like_of: Optional[str] = None,
|
||||
repost_of: Optional[str] = None,
|
||||
has_media: bool = False,
|
||||
bookmark_of: Optional[str] = None,
|
||||
content_format: Optional[str] = None,
|
||||
) -> Note:
|
||||
"""
|
||||
Create a new note
|
||||
@@ -203,7 +212,14 @@ def create_note(
|
||||
- created_at and updated_at set to same value initially
|
||||
"""
|
||||
# 1. VALIDATION (before any changes)
|
||||
if not content or not content.strip():
|
||||
# Likes and reposts may have no comment (v2.5.0), photo posts no text (v2.6.5)
|
||||
# Bookmarks too (v2.9.0)
|
||||
if sum(bool(u) for u in (like_of, repost_of, bookmark_of)) > 1:
|
||||
raise InvalidNoteDataError(
|
||||
"like_of", like_of, "A note can only be one of a like, repost or bookmark"
|
||||
)
|
||||
content = content or ""
|
||||
if not content.strip() and not (like_of or repost_of or bookmark_of or has_media):
|
||||
raise InvalidNoteDataError(
|
||||
"content", content, "Content cannot be empty or whitespace-only"
|
||||
)
|
||||
@@ -264,10 +280,12 @@ def create_note(
|
||||
try:
|
||||
db.execute(
|
||||
"""
|
||||
INSERT INTO notes (slug, file_path, published, created_at, updated_at, content_hash, in_reply_to)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
INSERT INTO notes (slug, file_path, published, created_at, updated_at, content_hash,
|
||||
in_reply_to, like_of, repost_of, bookmark_of, content_format)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(slug, file_path_rel, published, created_at, updated_at, content_hash, in_reply_to),
|
||||
(slug, file_path_rel, published, created_at, updated_at, content_hash,
|
||||
in_reply_to, like_of, repost_of, bookmark_of, content_format),
|
||||
)
|
||||
db.commit()
|
||||
except Exception as e:
|
||||
@@ -322,40 +340,62 @@ def create_note(
|
||||
from starpunk.webmention_sender import queue_webmentions_for_note
|
||||
db_path = Path(current_app.config["DATABASE_PATH"])
|
||||
site_url = current_app.config["SITE_URL"]
|
||||
note_url = f"{site_url}/note/{slug}"
|
||||
from starpunk.utils import absolute_url_for
|
||||
note_url = absolute_url_for("public.note", slug=slug)
|
||||
|
||||
# Get rendered HTML from note
|
||||
html_content = note.html
|
||||
|
||||
# Queue webmentions (best-effort, failures logged but don't fail note creation)
|
||||
# The reply/like/repost target is notified first (v2.5.0)
|
||||
queue_webmentions_for_note(
|
||||
db_path=db_path,
|
||||
note_id=note_id,
|
||||
note_url=note_url,
|
||||
content_html=html_content,
|
||||
in_reply_to=in_reply_to
|
||||
in_reply_to=in_reply_to or like_of or repost_of or bookmark_of
|
||||
)
|
||||
except Exception as e:
|
||||
# Webmention queueing failure should not prevent note creation
|
||||
current_app.logger.warning(f"Failed to queue webmentions for note {slug}: {e}")
|
||||
|
||||
# 12. FETCH REPLY CONTEXT (v1.6.0 Phase 3)
|
||||
# Fetch context if this is a reply
|
||||
if in_reply_to:
|
||||
# Fetch context for the post being replied to, liked or reposted
|
||||
context_url = in_reply_to or like_of or repost_of or bookmark_of
|
||||
if context_url:
|
||||
try:
|
||||
from starpunk.reply_context import fetch_reply_context
|
||||
db_path = Path(current_app.config["DATABASE_PATH"])
|
||||
site_url = current_app.config["SITE_URL"]
|
||||
|
||||
# Fetch context (best-effort, failures logged but don't fail note creation)
|
||||
fetch_reply_context(db_path, in_reply_to, site_url)
|
||||
fetch_reply_context(db_path, context_url, site_url)
|
||||
except Exception as e:
|
||||
# Reply context fetch failure should not prevent note creation
|
||||
current_app.logger.warning(f"Failed to fetch reply context for {in_reply_to}: {e}")
|
||||
|
||||
_websub_changed(note)
|
||||
return note
|
||||
|
||||
|
||||
def _tag_names(note_id: int) -> list[str]:
|
||||
"""A note's tag names, for WebSub tag topics (best effort)"""
|
||||
try:
|
||||
from starpunk.tags import get_note_tags
|
||||
return [t["name"] for t in get_note_tags(note_id)]
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
def _websub_changed(note, was_public: bool = False, old_tags=()) -> None:
|
||||
"""Clear cached feeds and queue WebSub deliveries (v2.10.0, best effort)"""
|
||||
try:
|
||||
from starpunk import websub
|
||||
websub.note_changed(note, was_public=was_public, old_tags=old_tags)
|
||||
except Exception as e:
|
||||
current_app.logger.warning(f"Failed to queue WebSub update for note {note.slug}: {e}")
|
||||
|
||||
|
||||
def get_note(
|
||||
slug: Optional[str] = None, id: Optional[int] = None, load_content: bool = True
|
||||
) -> Optional[Note]:
|
||||
@@ -472,6 +512,7 @@ def list_notes(
|
||||
offset: int = 0,
|
||||
order_by: str = "created_at",
|
||||
order_dir: str = "DESC",
|
||||
likes: Optional[bool] = None,
|
||||
) -> list[Note]:
|
||||
"""
|
||||
List notes with filtering and pagination
|
||||
@@ -564,6 +605,11 @@ def list_notes(
|
||||
params = []
|
||||
if published_only:
|
||||
query += " AND published = 1"
|
||||
# likes: None = all notes, False = leave likes out, True = only likes (v2.5.0)
|
||||
if likes is True:
|
||||
query += " AND like_of IS NOT NULL"
|
||||
elif likes is False:
|
||||
query += " AND like_of IS NULL"
|
||||
|
||||
# Add ordering (safe because order_by validated above)
|
||||
query += f" ORDER BY {order_by} {order_dir}"
|
||||
@@ -589,11 +635,15 @@ def update_note(
|
||||
content: Optional[str] = None,
|
||||
published: Optional[bool] = None,
|
||||
tags: Optional[list[str]] = None,
|
||||
in_reply_to: Optional[str] = None
|
||||
in_reply_to: Optional[str] = None,
|
||||
has_media: bool = False,
|
||||
content_format=UNCHANGED,
|
||||
) -> Note:
|
||||
"""
|
||||
Update a note's content and/or published status
|
||||
|
||||
content_format (v2.9.0): 'html' or None (Markdown); left alone unless given
|
||||
|
||||
Updates note content and/or metadata, maintaining atomic synchronization
|
||||
between file and database. At least one of content or published must
|
||||
be provided.
|
||||
@@ -663,12 +713,9 @@ def update_note(
|
||||
if content is None and published is None and tags is None and in_reply_to is None:
|
||||
raise ValueError("Must provide at least one of content, published, tags, or in_reply_to to update")
|
||||
|
||||
# Validate content if provided
|
||||
if content is not None:
|
||||
if not content or not content.strip():
|
||||
raise InvalidNoteDataError(
|
||||
"content", content, "Content cannot be empty or whitespace-only"
|
||||
)
|
||||
# Validate content if provided (checked against the note below for
|
||||
# likes/reposts, whose comment may be empty)
|
||||
empty_content = content is not None and not content.strip()
|
||||
|
||||
# 2. GET EXISTING NOTE
|
||||
# Phase 6: Load content to get old HTML for webmention URL diffing
|
||||
@@ -677,6 +724,12 @@ def update_note(
|
||||
if existing_note is None:
|
||||
identifier = slug if slug is not None else id
|
||||
raise NoteNotFoundError(identifier)
|
||||
old_tag_names = _tag_names(existing_note.id) # for WebSub tag topics (v2.10.0)
|
||||
|
||||
if empty_content and not (existing_note.reacts_to or has_media):
|
||||
raise InvalidNoteDataError(
|
||||
"content", content, "Content cannot be empty or whitespace-only"
|
||||
)
|
||||
|
||||
# 3. SETUP
|
||||
updated_at = datetime.utcnow()
|
||||
@@ -737,6 +790,10 @@ def update_note(
|
||||
update_fields.append("in_reply_to = ?")
|
||||
params.append(in_reply_to)
|
||||
|
||||
if content_format is not UNCHANGED:
|
||||
update_fields.append("content_format = ?")
|
||||
params.append(content_format)
|
||||
|
||||
# Add WHERE clause parameter
|
||||
if slug is not None:
|
||||
where_clause = "slug = ?"
|
||||
@@ -815,13 +872,14 @@ def update_note(
|
||||
from starpunk.webmention_sender import queue_webmentions_for_note
|
||||
db_path = Path(current_app.config["DATABASE_PATH"])
|
||||
site_url = current_app.config["SITE_URL"]
|
||||
note_url = f"{site_url}/note/{updated_note.slug}"
|
||||
from starpunk.utils import absolute_url_for
|
||||
note_url = absolute_url_for("public.note", slug=updated_note.slug)
|
||||
|
||||
# Get rendered HTML from updated note (if published)
|
||||
html_content = updated_note.html if updated_note.published else None
|
||||
|
||||
# Determine in_reply_to (use new value if provided, otherwise existing)
|
||||
reply_to = in_reply_to if in_reply_to is not None else updated_note.in_reply_to
|
||||
reply_to = (in_reply_to if in_reply_to is not None else updated_note.in_reply_to) or updated_note.reacts_to
|
||||
|
||||
# Queue webmentions (best-effort)
|
||||
# Pass old_html for URL diffing (Phase 6)
|
||||
@@ -838,8 +896,8 @@ def update_note(
|
||||
current_app.logger.warning(f"Failed to queue webmentions for note {updated_note.slug}: {e}")
|
||||
|
||||
# 9. FETCH REPLY CONTEXT (v1.6.0 Phase 3)
|
||||
# Fetch context if in_reply_to changed and is not None
|
||||
if in_reply_to is not None:
|
||||
# Fetch context if in_reply_to changed and is set ("" clears it)
|
||||
if in_reply_to:
|
||||
try:
|
||||
from starpunk.reply_context import fetch_reply_context
|
||||
db_path = Path(current_app.config["DATABASE_PATH"])
|
||||
@@ -858,6 +916,8 @@ def update_note(
|
||||
except Exception as e:
|
||||
current_app.logger.warning(f"Failed to queue syndication update: {e}")
|
||||
|
||||
_websub_changed(updated_note, was_public=existing_note.published, old_tags=old_tag_names)
|
||||
|
||||
# 10. RETURN UPDATED NOTE
|
||||
return updated_note
|
||||
|
||||
@@ -957,6 +1017,10 @@ def delete_note(
|
||||
# For idempotency, don't raise error - just return
|
||||
return
|
||||
|
||||
# What feeds showed before, for WebSub (v2.10.0)
|
||||
was_public = bool(existing_note.published) and not getattr(existing_note, "deleted_at", None)
|
||||
deleted_tags = _tag_names(existing_note.id)
|
||||
|
||||
# 4. SETUP
|
||||
data_dir = Path(current_app.config["DATA_PATH"])
|
||||
note_path = data_dir / existing_note.file_path
|
||||
@@ -969,6 +1033,17 @@ def delete_note(
|
||||
"Path validation failed",
|
||||
)
|
||||
|
||||
# Re-send webmentions so receivers learn the post is gone (spec 3.1.5).
|
||||
# A hard delete would cascade the queue rows away, so detach them first.
|
||||
if existing_note.published:
|
||||
try:
|
||||
from starpunk.webmention_sender import requeue_previous_webmentions
|
||||
requeue_previous_webmentions(
|
||||
Path(current_app.config["DATABASE_PATH"]), existing_note.id, detach=not soft
|
||||
)
|
||||
except Exception as e:
|
||||
current_app.logger.warning(f"Failed to queue webmentions for deleted note: {e}")
|
||||
|
||||
# 5. PERFORM DELETION
|
||||
db = get_db(current_app)
|
||||
|
||||
@@ -1032,5 +1107,66 @@ def delete_note(
|
||||
except Exception as e:
|
||||
current_app.logger.warning(f"Failed to queue syndication delete: {e}")
|
||||
|
||||
# Feeds and pages no longer show it (v2.10.0)
|
||||
object.__setattr__(existing_note, "published", False)
|
||||
_websub_changed(existing_note, was_public=was_public, old_tags=deleted_tags)
|
||||
|
||||
# 6. RETURN (no value)
|
||||
return None
|
||||
|
||||
|
||||
def undelete_note(slug: str) -> Note:
|
||||
"""
|
||||
Restore a soft-deleted note (v2.4.0, Micropub undelete)
|
||||
|
||||
Moves the file back from .trash/ and clears deleted_at. Crossposts are
|
||||
queued again, so a note deleted from them is posted again.
|
||||
|
||||
Raises:
|
||||
NoteNotFoundError: If there is no soft-deleted note with this slug
|
||||
"""
|
||||
import shutil
|
||||
from starpunk.utils import TRASH_DIR_NAME
|
||||
|
||||
db = get_db(current_app)
|
||||
row = db.execute(
|
||||
"SELECT id, file_path, created_at FROM notes WHERE slug = ? AND deleted_at IS NOT NULL",
|
||||
(slug,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise NoteNotFoundError(slug)
|
||||
|
||||
data_dir = Path(current_app.config["DATA_PATH"])
|
||||
note_path = data_dir / row["file_path"]
|
||||
if not note_path.exists():
|
||||
# delete_note_file put it in .trash/<year>/<month>/<name>
|
||||
parts = Path(row["file_path"]).parts
|
||||
candidates = []
|
||||
if "notes" in parts and len(parts) > parts.index("notes") + 2:
|
||||
i = parts.index("notes")
|
||||
candidates.append(data_dir / TRASH_DIR_NAME / parts[i + 1] / parts[i + 2] / note_path.name)
|
||||
candidates += sorted((data_dir / TRASH_DIR_NAME).glob(f"*/*/{note_path.name}"))
|
||||
source = next((c for c in candidates if c.exists()), None)
|
||||
if source is None:
|
||||
raise NoteSyncError("undelete", f"Trashed file not found for {slug}", "Cannot restore note file")
|
||||
note_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.move(str(source), str(note_path))
|
||||
|
||||
db.execute("UPDATE notes SET deleted_at = NULL WHERE id = ?", (row["id"],))
|
||||
db.commit()
|
||||
|
||||
try:
|
||||
from starpunk.syndication import note_changed
|
||||
note_changed(row["id"])
|
||||
except Exception as e:
|
||||
current_app.logger.warning(f"Failed to queue syndication after undelete: {e}")
|
||||
|
||||
try:
|
||||
from starpunk.webmention_sender import requeue_previous_webmentions
|
||||
requeue_previous_webmentions(Path(current_app.config["DATABASE_PATH"]), row["id"])
|
||||
except Exception as e:
|
||||
current_app.logger.warning(f"Failed to queue webmentions after undelete: {e}")
|
||||
|
||||
restored = get_note(slug=slug)
|
||||
_websub_changed(restored)
|
||||
return restored
|
||||
@@ -7,7 +7,7 @@ admin, auth, and (conditionally) dev auth routes.
|
||||
|
||||
from flask import Flask
|
||||
|
||||
from starpunk.routes import admin, auth, indieauth, micropub, public, search, webmention
|
||||
from starpunk.routes import admin, auth, indieauth, micropub, public, search, webmention, websub
|
||||
|
||||
|
||||
def register_routes(app: Flask) -> None:
|
||||
@@ -44,6 +44,9 @@ def register_routes(app: Flask) -> None:
|
||||
if app.config.get("WEBMENTION_ENABLED", True):
|
||||
app.register_blueprint(webmention.bp)
|
||||
|
||||
# WebSub hub (v2.10.0); refuses requests when disabled or using another hub
|
||||
app.register_blueprint(websub.bp)
|
||||
|
||||
# Conditionally register dev auth routes
|
||||
if app.config.get("DEV_MODE"):
|
||||
app.logger.warning(
|
||||
|
||||
@@ -72,12 +72,23 @@ def dashboard():
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# WebSub subscribers (v2.10.0)
|
||||
websub_stats = None
|
||||
if current_app.config.get('WEBSUB_ENABLED', True):
|
||||
try:
|
||||
from starpunk import websub
|
||||
from starpunk.database import get_db
|
||||
websub_stats = websub.stats(get_db(current_app))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return render_template(
|
||||
"admin/dashboard.html",
|
||||
notes=notes,
|
||||
page=page,
|
||||
total_pages=total_pages,
|
||||
webmention_pending_count=webmention_pending_count,
|
||||
websub_stats=websub_stats,
|
||||
user_me=g.me
|
||||
)
|
||||
|
||||
@@ -226,8 +237,8 @@ def edit_note_form(note_id: int):
|
||||
flash("Note not found", "error")
|
||||
return redirect(url_for("admin.dashboard")), 404
|
||||
|
||||
# Pre-load tags for the edit form (v1.3.0 Phase 3)
|
||||
tags = get_note_tags(note.id)
|
||||
# Pre-load tags for the edit form (v1.3.0 Phase 3), as entered so saving keeps the order
|
||||
tags = get_note_tags(note.id, entered_order=True)
|
||||
object.__setattr__(note, '_cached_tags', tags)
|
||||
|
||||
from starpunk.syndication import get_note_syndications, get_targets
|
||||
@@ -277,7 +288,7 @@ def update_note_submit(note_id: int):
|
||||
published = "published" in request.form
|
||||
tags_input = request.form.get("tags", "")
|
||||
|
||||
if not content:
|
||||
if not content and not existing_note.reacts_to: # likes/reposts may have no comment
|
||||
flash("Content cannot be empty", "error")
|
||||
return redirect(url_for("admin.edit_note_form", note_id=note_id))
|
||||
|
||||
|
||||
@@ -37,7 +37,7 @@ from starpunk.indieauth import (
|
||||
validate_authorization_request,
|
||||
verify_access_token,
|
||||
)
|
||||
from starpunk.micropub import extract_bearer_token
|
||||
from starpunk.micropub import MicropubError, extract_bearer_token
|
||||
from starpunk.utils import absolute_url_for
|
||||
|
||||
bp = Blueprint("indieauth", __name__)
|
||||
@@ -215,7 +215,11 @@ def token():
|
||||
@bp.route("/auth/token", methods=["GET"])
|
||||
def token_verify():
|
||||
"""Legacy token verification for clients that still call it"""
|
||||
info = verify_access_token(extract_bearer_token(request))
|
||||
try:
|
||||
token = extract_bearer_token(request)
|
||||
except MicropubError as e:
|
||||
return _oauth_error("invalid_request", e.error_description)
|
||||
info = verify_access_token(token)
|
||||
if not info:
|
||||
return _oauth_error("unauthorized", "Invalid or expired access token", 401)
|
||||
return _json(info)
|
||||
|
||||
+30
-21
@@ -27,6 +27,8 @@ from starpunk.micropub import (
|
||||
extract_bearer_token,
|
||||
error_response,
|
||||
handle_create,
|
||||
handle_delete,
|
||||
handle_update,
|
||||
handle_query,
|
||||
)
|
||||
from starpunk.indieauth import verify_access_token, check_scope
|
||||
@@ -49,8 +51,8 @@ def micropub_endpoint():
|
||||
POST requests:
|
||||
Handle action operations (form-encoded or JSON):
|
||||
- action=create (or no action): Create new post
|
||||
- action=update: Update existing post (not supported in V1)
|
||||
- action=delete: Delete post (not supported in V1)
|
||||
- action=update: Update existing post (JSON; replace/add/delete)
|
||||
- action=delete / action=undelete: Delete or restore a post
|
||||
|
||||
Authentication:
|
||||
Requires valid bearer token in Authorization header or
|
||||
@@ -68,7 +70,10 @@ def micropub_endpoint():
|
||||
}
|
||||
"""
|
||||
# Extract and verify token
|
||||
token = extract_bearer_token(request)
|
||||
try:
|
||||
token = extract_bearer_token(request)
|
||||
except MicropubError as e:
|
||||
return error_response(e.error, e.error_description, e.status_code)
|
||||
if not token:
|
||||
return error_response("unauthorized", "No access token provided", 401)
|
||||
|
||||
@@ -79,7 +84,7 @@ def micropub_endpoint():
|
||||
# Handle query endpoints (GET requests)
|
||||
if request.method == "GET":
|
||||
try:
|
||||
return handle_query(request.args.to_dict(), token_info)
|
||||
return handle_query(request.args, token_info)
|
||||
except MicropubError as e:
|
||||
return error_response(e.error, e.error_description, e.status_code)
|
||||
except Exception as e:
|
||||
@@ -92,26 +97,25 @@ def micropub_endpoint():
|
||||
try:
|
||||
# Parse request based on content type
|
||||
if "application/json" in content_type:
|
||||
data = request.get_json() or {}
|
||||
# Malformed JSON is the client's error, not ours (v2.9.0)
|
||||
data = request.get_json(silent=True)
|
||||
if not isinstance(data, dict):
|
||||
return error_response("invalid_request", "Request body must be a JSON object", 400)
|
||||
action = data.get("action", "create")
|
||||
else:
|
||||
# Form-encoded or multipart (V1 only supports form-encoded)
|
||||
# Form-encoded or multipart (files handled separately)
|
||||
data = request.form.to_dict(flat=False)
|
||||
action = data.get("action", ["create"])[0]
|
||||
|
||||
# Route to appropriate handler
|
||||
if action == "create":
|
||||
return handle_create(data, token_info)
|
||||
return handle_create(data, token_info, files=request.files)
|
||||
elif action == "update":
|
||||
# V1: Update not supported
|
||||
return error_response(
|
||||
"invalid_request", "Update action not supported in V1", 400
|
||||
)
|
||||
elif action == "delete":
|
||||
# V1: Delete not supported
|
||||
return error_response(
|
||||
"invalid_request", "Delete action not supported in V1", 400
|
||||
)
|
||||
if "application/json" not in content_type:
|
||||
return error_response("invalid_request", "Updates must be sent as JSON", 400)
|
||||
return handle_update(data, token_info)
|
||||
elif action in ("delete", "undelete"):
|
||||
return handle_delete(data, token_info, undelete=(action == "undelete"))
|
||||
else:
|
||||
return error_response("invalid_request", f"Unknown action: {action}", 400)
|
||||
|
||||
@@ -122,7 +126,8 @@ def micropub_endpoint():
|
||||
return error_response("server_error", "An unexpected error occurred", 500)
|
||||
|
||||
|
||||
@bp.route('/media', methods=['POST'])
|
||||
@bp.route('/micropub/media', methods=['POST'])
|
||||
@bp.route('/media', methods=['POST']) # old path, kept for existing clients
|
||||
def media_endpoint():
|
||||
"""
|
||||
Micropub media endpoint for file uploads
|
||||
@@ -137,7 +142,10 @@ def media_endpoint():
|
||||
from starpunk.media import save_media
|
||||
|
||||
# Extract and verify token
|
||||
token = extract_bearer_token(request)
|
||||
try:
|
||||
token = extract_bearer_token(request)
|
||||
except MicropubError as e:
|
||||
return error_response(e.error, e.error_description, e.status_code)
|
||||
if not token:
|
||||
return error_response("unauthorized", "No access token provided", 401)
|
||||
|
||||
@@ -145,11 +153,12 @@ def media_endpoint():
|
||||
if not token_info:
|
||||
return error_response("unauthorized", "Invalid or expired access token", 401)
|
||||
|
||||
# Check scope (create scope allows media upload)
|
||||
if not check_scope("create", token_info.get("scope", "")):
|
||||
# Check scope: "media", or "create" for older clients
|
||||
scope = token_info.get("scope", "")
|
||||
if not (check_scope("media", scope) or check_scope("create", scope)):
|
||||
return error_response(
|
||||
"insufficient_scope",
|
||||
"Token lacks create scope",
|
||||
"Token lacks media or create scope",
|
||||
403
|
||||
)
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ from datetime import datetime, timedelta
|
||||
from flask import Blueprint, abort, render_template, Response, current_app, request, send_from_directory
|
||||
|
||||
from starpunk.notes import list_notes, get_note
|
||||
from starpunk.websub import embed_hub
|
||||
from starpunk.feed import generate_feed_streaming # Legacy RSS
|
||||
from starpunk.feeds import (
|
||||
generate_rss,
|
||||
@@ -55,8 +56,14 @@ def _get_cached_notes():
|
||||
cache_duration = timedelta(seconds=cache_seconds)
|
||||
now = datetime.utcnow()
|
||||
|
||||
# The cached list is stale once any public note changed, in any process
|
||||
# (v2.10.0): every change bumps content_version in the database
|
||||
from starpunk.database import get_db
|
||||
from starpunk.websub import get_content_version
|
||||
version = (str(current_app.config["DATABASE_PATH"]), get_content_version(get_db(current_app)))
|
||||
|
||||
# Check if note list cache is valid
|
||||
if _feed_cache["notes"] and _feed_cache["timestamp"]:
|
||||
if _feed_cache["notes"] and _feed_cache["timestamp"] and _feed_cache.get("version") == version:
|
||||
cache_age = now - _feed_cache["timestamp"]
|
||||
if cache_age < cache_duration:
|
||||
# Use cached note list
|
||||
@@ -64,7 +71,7 @@ def _get_cached_notes():
|
||||
|
||||
# Cache expired or empty, fetch fresh notes
|
||||
max_items = current_app.config.get("FEED_MAX_ITEMS", 50)
|
||||
notes = list_notes(published_only=True, limit=max_items)
|
||||
notes = list_notes(published_only=True, limit=max_items, likes=False)
|
||||
|
||||
# Batch load media and tags for all notes (v1.5.0 Phase 3)
|
||||
# Reduces query count from O(n) to O(1) for both media and tags
|
||||
@@ -82,6 +89,7 @@ def _get_cached_notes():
|
||||
|
||||
_feed_cache["notes"] = notes
|
||||
_feed_cache["timestamp"] = now
|
||||
_feed_cache["version"] = version
|
||||
|
||||
return notes
|
||||
|
||||
@@ -122,6 +130,7 @@ def _generate_feed_with_cache(format_name: str, non_streaming_generator):
|
||||
notes=notes,
|
||||
limit=max_items,
|
||||
)
|
||||
content = embed_hub(format_name, content) # WebSub hub link (v2.10.0)
|
||||
|
||||
# Create response with proper Content-Type including charset
|
||||
mime_type = get_mime_type(format_name)
|
||||
@@ -170,6 +179,7 @@ def _generate_feed_with_cache(format_name: str, non_streaming_generator):
|
||||
notes=notes,
|
||||
limit=max_items,
|
||||
)
|
||||
content = embed_hub(format_name, content) # WebSub hub link (v2.10.0)
|
||||
|
||||
# Store in cache and get ETag
|
||||
etag = feed_cache.set(format_name, content, notes_checksum)
|
||||
@@ -249,8 +259,8 @@ def index():
|
||||
from starpunk.media import get_note_media
|
||||
from starpunk.tags import get_note_tags
|
||||
|
||||
# Get recent published notes (limit 20)
|
||||
notes = list_notes(published_only=True, limit=20)
|
||||
# Get recent published notes (limit 20); likes have their own page (v2.5.0)
|
||||
notes = list_notes(published_only=True, limit=20, likes=False)
|
||||
|
||||
# Attach media and tags to each note for display
|
||||
for note in notes:
|
||||
@@ -262,9 +272,59 @@ def index():
|
||||
tags = get_note_tags(note.id)
|
||||
object.__setattr__(note, '_cached_tags', tags)
|
||||
|
||||
attach_contexts(notes)
|
||||
return render_template("index.html", notes=notes)
|
||||
|
||||
|
||||
def attach_contexts(notes) -> None:
|
||||
"""Give likes and reposts the fetched context of the post they point at"""
|
||||
from starpunk.webmention_queries import get_reply_context
|
||||
|
||||
for note in notes:
|
||||
context = None
|
||||
if note.reacts_to:
|
||||
try:
|
||||
context = get_reply_context(note.reacts_to)
|
||||
except Exception:
|
||||
context = None
|
||||
object.__setattr__(note, 'context', context)
|
||||
|
||||
|
||||
@bp.route("/likes")
|
||||
def likes():
|
||||
"""Likes, kept off the home page and feeds (v2.5.0)"""
|
||||
from starpunk.media import get_note_media
|
||||
from starpunk.tags import get_note_tags
|
||||
|
||||
notes = list_notes(published_only=True, limit=50, likes=True)
|
||||
for note in notes:
|
||||
object.__setattr__(note, 'media', get_note_media(note.id))
|
||||
object.__setattr__(note, '_cached_tags', get_note_tags(note.id))
|
||||
attach_contexts(notes)
|
||||
return render_template("likes.html", notes=notes)
|
||||
|
||||
|
||||
def _note_was_deleted(slug: str) -> bool:
|
||||
"""
|
||||
True if this slug was a note that has been deleted
|
||||
|
||||
Soft-deleted notes keep their row; hard-deleted notes that sent
|
||||
webmentions keep their queue rows (detached from the note).
|
||||
"""
|
||||
from starpunk.database import get_db
|
||||
from starpunk.utils import absolute_url_for
|
||||
|
||||
db = get_db(current_app)
|
||||
if db.execute(
|
||||
"SELECT 1 FROM notes WHERE slug = ? AND deleted_at IS NOT NULL", (slug,)
|
||||
).fetchone():
|
||||
return True
|
||||
return db.execute(
|
||||
"SELECT 1 FROM webmention_queue WHERE source_url = ? AND source_note_id IS NULL",
|
||||
(absolute_url_for("public.note", slug=slug),),
|
||||
).fetchone() is not None
|
||||
|
||||
|
||||
@bp.route("/note/<slug>")
|
||||
def note(slug: str):
|
||||
"""
|
||||
@@ -290,8 +350,10 @@ def note(slug: str):
|
||||
# Get note by slug
|
||||
note_obj = get_note(slug=slug)
|
||||
|
||||
# Return 404 if note doesn't exist or isn't published
|
||||
# Deleted notes are 410 Gone (Webmention 3.1.5); unknown or draft 404
|
||||
if not note_obj or not note_obj.published:
|
||||
if not note_obj and _note_was_deleted(slug):
|
||||
abort(410)
|
||||
abort(404)
|
||||
|
||||
# Get media for note (v1.2.0 Phase 3)
|
||||
@@ -334,6 +396,7 @@ def note(slug: str):
|
||||
from starpunk.syndication import get_note_syndications
|
||||
|
||||
syndications = [s for s in get_note_syndications(note_obj.id) if s["state"] == "synced"]
|
||||
attach_contexts([note_obj])
|
||||
|
||||
return render_template(
|
||||
"note.html",
|
||||
@@ -384,6 +447,7 @@ def tag(tag: str):
|
||||
for note in notes:
|
||||
media = get_note_media(note.id)
|
||||
object.__setattr__(note, 'media', media)
|
||||
attach_contexts(notes)
|
||||
|
||||
return render_template(
|
||||
"tag.html",
|
||||
|
||||
@@ -11,7 +11,7 @@ References:
|
||||
"""
|
||||
|
||||
from flask import Blueprint, current_app, request, jsonify
|
||||
from urllib.parse import urlparse
|
||||
from urllib.parse import urldefrag, urlparse
|
||||
from starpunk.database import get_db
|
||||
from starpunk.notes import get_note, NoteNotFoundError
|
||||
from starpunk.rate_limiter import check_rate_limit
|
||||
@@ -107,17 +107,24 @@ def validate_url(url: str) -> tuple[bool, str | None]:
|
||||
return False, "URL is malformed"
|
||||
|
||||
|
||||
def _host_port(url: str) -> tuple[str, int | None]:
|
||||
parsed = urlparse(url)
|
||||
default = {'http': 80, 'https': 443}.get(parsed.scheme.lower())
|
||||
try:
|
||||
port = parsed.port or default
|
||||
except ValueError:
|
||||
port = None
|
||||
return (parsed.hostname or '').lower(), port
|
||||
|
||||
|
||||
def is_target_on_site(target_url: str) -> bool:
|
||||
"""
|
||||
Check if target URL is on this site.
|
||||
|
||||
Compares hostname of target_url with SITE_URL configuration.
|
||||
Compares host (case-insensitively) and effective port with SITE_URL.
|
||||
"""
|
||||
site_url = current_app.config.get('SITE_URL', '')
|
||||
site_parsed = urlparse(site_url)
|
||||
target_parsed = urlparse(target_url)
|
||||
|
||||
return site_parsed.netloc == target_parsed.netloc
|
||||
return _host_port(site_url) == _host_port(target_url)
|
||||
|
||||
|
||||
def resolve_target_note(target_url: str) -> int | None:
|
||||
@@ -228,16 +235,17 @@ def receive_webmention():
|
||||
"error_description": "Target URL is not on this site"
|
||||
}), 400
|
||||
|
||||
# Resolve target to note (may be None for homepage/other pages)
|
||||
target_note_id = resolve_target_note(target)
|
||||
# The fragment is ignored (spec 3.2.1) and not stored, so target#a and
|
||||
# target#b are the same mention
|
||||
target = urldefrag(target)[0]
|
||||
|
||||
# If target looks like a note URL but doesn't resolve, it's not found
|
||||
target_parsed = urlparse(target)
|
||||
target_path = target_parsed.path.rstrip('/')
|
||||
if target_path.startswith('/note/') and target_note_id is None:
|
||||
# Only published notes accept webmentions (spec 3.2.1: target must be a
|
||||
# valid resource that accepts them)
|
||||
target_note_id = resolve_target_note(target)
|
||||
if target_note_id is None:
|
||||
return jsonify({
|
||||
"error": "invalid_target",
|
||||
"error_description": "Target URL not found"
|
||||
"error_description": "Target URL does not accept webmentions"
|
||||
}), 400
|
||||
|
||||
# Insert webmention into database (queued for validation)
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
"""
|
||||
WebSub hub endpoint (v2.10.0, ADR-071)
|
||||
|
||||
POST /websub with hub.mode subscribe/unsubscribe, hub.topic, hub.callback,
|
||||
optional hub.secret and hub.lease_seconds. Answers 202 and verifies later
|
||||
(spec 5.1); errors are 4xx with a plain-text reason.
|
||||
"""
|
||||
|
||||
from flask import Blueprint, current_app, request
|
||||
|
||||
from starpunk.database import get_db
|
||||
from starpunk.rate_limiter import RateLimiter
|
||||
from starpunk import websub
|
||||
|
||||
bp = Blueprint("websub", __name__)
|
||||
|
||||
# Subscription requests per client IP per minute
|
||||
_limiter = RateLimiter(limit=30, window_seconds=60)
|
||||
|
||||
|
||||
def _text(message: str, status: int):
|
||||
return message + "\n", status, {"Content-Type": "text/plain; charset=utf-8"}
|
||||
|
||||
|
||||
@bp.route("/websub", methods=["POST"])
|
||||
def hub():
|
||||
if not websub.enabled():
|
||||
return _text("WebSub is not enabled on this site", 404)
|
||||
|
||||
from starpunk.routes.webmention import get_client_ip
|
||||
if not _limiter.is_allowed(get_client_ip()):
|
||||
return _text("Too many requests", 429)
|
||||
|
||||
if "application/x-www-form-urlencoded" not in request.headers.get("Content-Type", ""):
|
||||
return _text("Send an application/x-www-form-urlencoded body", 400)
|
||||
|
||||
try:
|
||||
websub.accept_subscription_request(request.form, get_db(current_app))
|
||||
except websub.SubscriptionError as e:
|
||||
return _text(str(e), e.status)
|
||||
|
||||
return _text("Accepted; the subscription will be verified", 202)
|
||||
|
||||
|
||||
@bp.route("/websub", methods=["GET"])
|
||||
def hub_info():
|
||||
return _text(
|
||||
"WebSub hub for this site's own feeds. Subscribe with a POST "
|
||||
"(https://www.w3.org/TR/websub/).", 200)
|
||||
+713
-33
@@ -1,10 +1,10 @@
|
||||
"""
|
||||
Crossposting (POSSE) for StarPunk (v2.1.0 ADR-065, v2.2.0 ADR-066)
|
||||
Crossposting (POSSE) for StarPunk (v2.1.0 ADR-065, v2.2.0 ADR-066, v2.6.0 ADR-069, v2.6.3 ADR-070)
|
||||
|
||||
Published notes are copied to syndication targets, and later edits and
|
||||
deletes follow them. Two target kinds exist: a GoToSocial (or any
|
||||
Mastodon-API) account, and a Bluesky account. Bluesky posts can't be edited,
|
||||
so there only deletes follow.
|
||||
deletes follow them. Three target kinds exist: a GoToSocial (or any
|
||||
Mastodon-API) account, a Bluesky account and a Threads account. Bluesky and
|
||||
Threads posts can't be edited, so there only deletes follow.
|
||||
|
||||
Each (note, target) pair has a row in `syndications`. Changing a note marks
|
||||
its rows pending; a background worker then reconciles the remote post with
|
||||
@@ -17,9 +17,15 @@ the note:
|
||||
Rows carry a version so an edit made while a send is in flight is not lost.
|
||||
"""
|
||||
|
||||
import fcntl
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
@@ -28,6 +34,9 @@ from typing import Dict, List, Optional
|
||||
import httpx
|
||||
from flask import current_app
|
||||
|
||||
# httpx logs request URLs at INFO, and Threads takes its token in the URL
|
||||
logging.getLogger("httpx").setLevel(logging.WARNING)
|
||||
|
||||
from starpunk.database import get_db
|
||||
|
||||
MAX_ATTEMPTS = 8
|
||||
@@ -113,21 +122,58 @@ class MastodonClient:
|
||||
|
||||
# Interface used by the reconcile loop
|
||||
can_edit = True
|
||||
NATIVE = ("favourite", "reblog") # remote_id prefixes for likes/boosts (v2.5.0)
|
||||
|
||||
def find_status(self, url: str) -> Optional[str]:
|
||||
"""Local id of a fediverse post, fetching it if needed; None if the URL isn't one"""
|
||||
try:
|
||||
response = self._call("GET", "/api/v2/search", params={
|
||||
"q": url, "resolve": "true", "type": "statuses", "limit": 1,
|
||||
})
|
||||
except RemoteGone:
|
||||
return None
|
||||
statuses = response.json().get("statuses") or []
|
||||
return statuses[0]["id"] if statuses else None
|
||||
|
||||
def _prepare(self, post: "Post") -> tuple:
|
||||
text = compose_text(post.note, post.permalink, post.tags, post.max_chars)
|
||||
if post.kind == "repost":
|
||||
return compose_repost_text(post, post.note.content, post.max_chars), []
|
||||
# Images in the text are sent as attachments instead (v2.6.6)
|
||||
text = compose_text(post.note, post.permalink, post.tags, post.max_chars,
|
||||
body=strip_images(post.note.content))
|
||||
media_ids = [self.upload_media(f["path"], f["mime_type"], f["description"]) for f in post.media]
|
||||
return text, media_ids
|
||||
|
||||
def create(self, post: "Post", key: str, retrying: bool = False) -> dict:
|
||||
if post.kind in ("like", "repost") and post.target_url:
|
||||
# Native favourite/boost when the target is a fediverse post; a
|
||||
# repost with a comment is posted as a status so the comment stays
|
||||
commented = bool(post.note.content.strip())
|
||||
if post.kind == "like" or not commented:
|
||||
status_id = self.find_status(post.target_url)
|
||||
if status_id:
|
||||
action = "favourite" if post.kind == "like" else "reblog"
|
||||
self._call("POST", f"/api/v1/statuses/{status_id}/{action}")
|
||||
return {"id": f"{action}:{status_id}", "url": None}
|
||||
if post.kind == "like":
|
||||
return {"id": None, "url": None} # nothing to like here
|
||||
text, media_ids = self._prepare(post)
|
||||
return self.create_status(text, media_ids, idempotency_key=key)
|
||||
|
||||
def update(self, remote_id: str, post: "Post") -> dict:
|
||||
if remote_id.split(":", 1)[0] in self.NATIVE:
|
||||
return {"id": remote_id, "url": None} # a favourite/boost has nothing to edit
|
||||
text, media_ids = self._prepare(post)
|
||||
return self.update_status(remote_id, text, media_ids)
|
||||
|
||||
def delete(self, remote_id: str) -> None:
|
||||
action, _, status_id = remote_id.partition(":")
|
||||
if action in self.NATIVE and status_id:
|
||||
try:
|
||||
self._call("POST", f"/api/v1/statuses/{status_id}/un{action}")
|
||||
except RemoteGone:
|
||||
pass
|
||||
return
|
||||
self.delete_status(remote_id)
|
||||
|
||||
|
||||
@@ -229,20 +275,42 @@ class BlueskyClient:
|
||||
image["aspectRatio"] = {"width": width, "height": height}
|
||||
return image
|
||||
|
||||
def _record(self, post: "Post") -> dict:
|
||||
text = compose_text(post.note, post.permalink, post.tags, post.max_chars,
|
||||
body=plain_text(post.note.content))
|
||||
record = {
|
||||
"$type": self.POST,
|
||||
"text": text,
|
||||
"createdAt": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z",
|
||||
}
|
||||
facets = bluesky_facets(text)
|
||||
@staticmethod
|
||||
def _now() -> str:
|
||||
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
|
||||
|
||||
def _link_card(self, url: str) -> dict:
|
||||
"""External embed for a page, with its title, description and image when we can get them"""
|
||||
card = page_card(self.http, url)
|
||||
external = {"uri": url, "title": card["title"][:300], "description": card["description"][:1000]}
|
||||
if card["image"]:
|
||||
try:
|
||||
data, mime_type, _ = _fetch_public(self.http, card["image"], CARD_IMAGE_LIMIT,
|
||||
truncate=False)
|
||||
data, mime_type, _, _ = _fit_image(data, mime_type.split(";")[0] or "image/jpeg",
|
||||
BLUESKY_BLOB_LIMIT)
|
||||
external["thumb"] = self.upload_blob(data, mime_type)
|
||||
except Exception as e: # a card without a picture is fine
|
||||
current_app.logger.info(f"No link card image for {url}: {e}")
|
||||
return {"$type": "app.bsky.embed.external", "external": external}
|
||||
|
||||
def _record(self, post: "Post", text: Optional[str] = None, embed: Optional[dict] = None,
|
||||
anchors: List[tuple] = ()) -> dict:
|
||||
if text is None:
|
||||
body, anchors = plain_text_links(post.note.content, links="anchor")
|
||||
text = compose_text(post.note, post.permalink, post.tags, post.max_chars, body=body)
|
||||
record = {"$type": self.POST, "text": text, "createdAt": self._now()}
|
||||
facets = bluesky_facets(text, anchors)
|
||||
if facets:
|
||||
record["facets"] = facets
|
||||
if post.media:
|
||||
if embed is not None:
|
||||
record["embed"] = embed
|
||||
elif post.media:
|
||||
record["embed"] = {"$type": "app.bsky.embed.images",
|
||||
"images": [self._image(m) for m in post.media]}
|
||||
elif note_links(post.note.content):
|
||||
# The note links somewhere: the card shows that page, not the note
|
||||
record["embed"] = self._link_card(note_links(post.note.content)[0])
|
||||
else:
|
||||
record["embed"] = {"$type": "app.bsky.embed.external", "external": {
|
||||
"uri": post.permalink,
|
||||
@@ -251,6 +319,56 @@ class BlueskyClient:
|
||||
}}
|
||||
return record
|
||||
|
||||
_POST_URL = re.compile(r"^https://bsky\.app/profile/([^/]+)/post/([^/?#]+)")
|
||||
|
||||
def resolve_post(self, url: str) -> Optional[dict]:
|
||||
"""{'uri', 'cid'} of a Bluesky post URL; None if the URL isn't one"""
|
||||
m = self._POST_URL.match(url or "")
|
||||
if not m:
|
||||
return None
|
||||
actor, rkey = m.groups()
|
||||
if not actor.startswith("did:"):
|
||||
actor = self._call("GET", "com.atproto.identity.resolveHandle",
|
||||
params={"handle": actor}).json()["did"]
|
||||
uri = f"at://{actor}/{self.POST}/{rkey}"
|
||||
posts = self._call("GET", "app.bsky.feed.getPosts", params={"uris": uri}).json().get("posts") or []
|
||||
return {"uri": uri, "cid": posts[0]["cid"]} if posts else None
|
||||
|
||||
def _create(self, collection: str, record: dict) -> str:
|
||||
response = self._call("POST", "com.atproto.repo.createRecord",
|
||||
json={"repo": self.did or self.identifier, "collection": collection,
|
||||
"record": record})
|
||||
return response.json()["uri"]
|
||||
|
||||
def _react(self, post: "Post") -> dict:
|
||||
"""Like, repost or quote a Bluesky post; fall back to a link post for reposts"""
|
||||
comment, anchors = (plain_text_links(post.note.content, links="anchor")
|
||||
if post.note.content.strip() else ("", []))
|
||||
subject = self.resolve_post(post.target_url)
|
||||
if subject and post.kind == "like":
|
||||
uri = self._create("app.bsky.feed.like", {
|
||||
"$type": "app.bsky.feed.like", "subject": subject, "createdAt": self._now()})
|
||||
return {"id": uri, "url": None}
|
||||
if subject and not comment:
|
||||
uri = self._create("app.bsky.feed.repost", {
|
||||
"$type": "app.bsky.feed.repost", "subject": subject, "createdAt": self._now()})
|
||||
return {"id": uri, "url": None}
|
||||
if subject: # repost with a comment -> quote post
|
||||
text = comment if len(comment) <= post.max_chars else \
|
||||
comment[:post.max_chars - 1].rsplit(" ", 1)[0].rstrip() + "…"
|
||||
uri = self._create(self.POST, self._record(
|
||||
post, text=text, embed={"$type": "app.bsky.embed.record", "record": subject},
|
||||
anchors=anchors))
|
||||
return {"id": uri, "url": self._web_url(uri)}
|
||||
if post.kind == "like":
|
||||
return {"id": None, "url": None} # not a Bluesky post: nothing to like
|
||||
text = compose_repost_text(post, comment, post.max_chars)
|
||||
embed = {"$type": "app.bsky.embed.external", "external": {
|
||||
"uri": post.target_url, "title": (post.target_title or post.target_url)[:300],
|
||||
"description": ""}}
|
||||
uri = self._create(self.POST, self._record(post, text=text, embed=embed))
|
||||
return {"id": uri, "url": self._web_url(uri)}
|
||||
|
||||
def _web_url(self, uri: str) -> str:
|
||||
rkey = uri.rsplit("/", 1)[1]
|
||||
return f"https://bsky.app/profile/{self.handle or self.did}/post/{rkey}"
|
||||
@@ -265,6 +383,11 @@ class BlueskyClient:
|
||||
return None
|
||||
|
||||
def create(self, post: "Post", key: str, retrying: bool = False) -> dict:
|
||||
if post.kind in ("like", "repost") and post.target_url:
|
||||
if not self.did:
|
||||
with self._lock:
|
||||
self._login()
|
||||
return self._react(post)
|
||||
if retrying:
|
||||
if not self.did:
|
||||
with self._lock:
|
||||
@@ -289,11 +412,15 @@ class BlueskyClient:
|
||||
json={"repo": repo, "collection": collection, "rkey": rkey})
|
||||
|
||||
|
||||
def _fit_image(path: Path, mime_type: str, limit: int) -> tuple:
|
||||
"""Image bytes no bigger than `limit`, re-encoded as JPEG if needed. Returns (data, mime, w, h)."""
|
||||
def _fit_image(path, mime_type: str, limit: int) -> tuple:
|
||||
"""
|
||||
Image (a path, or bytes) no bigger than `limit`, re-encoded as JPEG if
|
||||
needed. Returns (data, mime, w, h).
|
||||
"""
|
||||
from PIL import Image
|
||||
|
||||
data = Path(path).read_bytes()
|
||||
data = path if isinstance(path, bytes) else Path(path).read_bytes()
|
||||
name = "image" if isinstance(path, bytes) else Path(path).name
|
||||
with Image.open(io.BytesIO(data)) as img:
|
||||
width, height = img.size
|
||||
if len(data) <= limit:
|
||||
@@ -306,7 +433,355 @@ def _fit_image(path: Path, mime_type: str, limit: int) -> tuple:
|
||||
copy.save(buf, format="JPEG", quality=quality, optimize=True)
|
||||
if buf.tell() <= limit:
|
||||
return buf.getvalue(), "image/jpeg", copy.size[0], copy.size[1]
|
||||
raise SyndicationError(f"{Path(path).name}: image too large for Bluesky", permanent=True)
|
||||
raise SyndicationError(f"{name}: image too large for Bluesky", permanent=True)
|
||||
|
||||
|
||||
# --- Link cards ---------------------------------------------------------------
|
||||
|
||||
CARD_PAGE_LIMIT = 512 * 1024 # the <head> is near the start; read no more
|
||||
CARD_IMAGE_LIMIT = 5 * 1024 * 1024
|
||||
|
||||
|
||||
def _fetch_public(http: httpx.Client, url: str, limit: int, truncate: bool) -> tuple:
|
||||
"""
|
||||
GET a public web URL, checking every redirect hop (no requests into the
|
||||
LAN). Reads at most `limit` bytes: stops there if `truncate`, else fails.
|
||||
Returns (data, content_type, final_url).
|
||||
"""
|
||||
from urllib.parse import urljoin, urlparse
|
||||
from starpunk.micropub import _public_address
|
||||
|
||||
site = current_app.config.get("SITE_URL", "")
|
||||
for _ in range(5):
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme not in ("http", "https") or not parsed.hostname:
|
||||
raise SyndicationError(f"not a web URL: {url}", permanent=True)
|
||||
if not current_app.config.get("TESTING") and not _public_address(parsed.hostname):
|
||||
raise SyndicationError(f"not a public address: {url}", permanent=True)
|
||||
try:
|
||||
with http.stream("GET", url, follow_redirects=False, timeout=10.0,
|
||||
headers={"User-Agent": f"StarPunk (+{site})"}) as response:
|
||||
if response.is_redirect and "location" in response.headers:
|
||||
url = urljoin(url, response.headers["location"])
|
||||
continue
|
||||
if response.status_code >= 400:
|
||||
raise SyndicationError(f"{url}: HTTP {response.status_code}")
|
||||
data = bytearray()
|
||||
for chunk in response.iter_bytes():
|
||||
data += chunk
|
||||
if len(data) > limit:
|
||||
if truncate:
|
||||
break
|
||||
raise SyndicationError(f"{url}: too large")
|
||||
return bytes(data), response.headers.get("content-type", ""), url
|
||||
except httpx.HTTPError as e:
|
||||
raise SyndicationError(f"{url}: {type(e).__name__}")
|
||||
raise SyndicationError(f"{url}: too many redirects")
|
||||
|
||||
|
||||
def page_card(http: httpx.Client, url: str) -> dict:
|
||||
"""
|
||||
A page's title, description and image URL (Open Graph, Twitter or plain
|
||||
HTML tags), for a link card. Missing parts come back empty; a page that
|
||||
can't be fetched gives just the URL as the title.
|
||||
"""
|
||||
from urllib.parse import urljoin
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
card = {"title": url, "description": "", "image": ""}
|
||||
try:
|
||||
data, content_type, final = _fetch_public(http, url, CARD_PAGE_LIMIT, truncate=True)
|
||||
except SyndicationError as e:
|
||||
current_app.logger.info(f"No link card details for {url}: {e}")
|
||||
return card
|
||||
if "html" not in content_type.lower():
|
||||
return card
|
||||
soup = BeautifulSoup(data, "html.parser")
|
||||
|
||||
def meta(*names):
|
||||
for name in names:
|
||||
tag = soup.find("meta", attrs={"property": name}) or soup.find("meta", attrs={"name": name})
|
||||
if tag and (tag.get("content") or "").strip():
|
||||
return " ".join(tag["content"].split())
|
||||
return ""
|
||||
|
||||
title = meta("og:title", "twitter:title")
|
||||
if not title and soup.title:
|
||||
title = " ".join(soup.title.get_text(" ").split())
|
||||
image = meta("og:image:secure_url", "og:image", "og:image:url", "twitter:image")
|
||||
card.update(title=title or url,
|
||||
description=meta("og:description", "twitter:description", "description"),
|
||||
image=urljoin(final, image) if image else "")
|
||||
return card
|
||||
|
||||
|
||||
THREADS_API = "https://graph.threads.net/v1.0"
|
||||
THREADS_IMAGE_TYPES = {"image/jpeg", "image/png"}
|
||||
THREADS_IMAGE_LIMIT = 8_000_000
|
||||
THREADS_MAX_LINKS = 5
|
||||
THREADS_REFRESH_EVERY = 24 * 3600 # tokens last 60 days; Meta allows a refresh once one is a day old
|
||||
# Graph API error codes that are temporary (rate limits, service hiccups)
|
||||
_GRAPH_TRANSIENT = {1, 2, 4, 17, 32, 341, 613}
|
||||
_GRAPH_AUTH = {10, 190, 200} # bad token or missing permission: retrying won't help
|
||||
|
||||
|
||||
def threads_length(text: str) -> int:
|
||||
"""Length as Threads counts it: emoji (and, to be safe, all non-ASCII) by UTF-8 bytes"""
|
||||
return sum(1 if ord(ch) < 128 else len(ch.encode()) for ch in text)
|
||||
|
||||
|
||||
def threads_topic(tags: List[str]) -> str:
|
||||
"""The first tag usable as a Threads topic (one per post; no '.' or '&', 1-50 chars)"""
|
||||
for tag in tags:
|
||||
topic = re.sub(r"[.&]", "", tag).strip()[:50]
|
||||
if topic:
|
||||
return topic
|
||||
return ""
|
||||
|
||||
|
||||
class ThreadsClient:
|
||||
"""
|
||||
Minimal Threads API client (Meta Graph API)
|
||||
|
||||
Posting is two steps: create a container, wait for it to be ready, then
|
||||
publish it. Meta fetches images from their public URL on this site.
|
||||
Posts can't be edited through the API, so only deletes follow.
|
||||
|
||||
Long-lived tokens expire after 60 days unless refreshed. `maintain()`
|
||||
refreshes the token once a day and keeps it in `token_file`, so it
|
||||
survives restarts. A different token in the config replaces it.
|
||||
"""
|
||||
|
||||
can_edit = False
|
||||
|
||||
def __init__(self, token: str, token_file: Optional[Path] = None, api: str = THREADS_API,
|
||||
http: Optional[httpx.Client] = None, sleep=time.sleep):
|
||||
self.config_token = token
|
||||
self.token_file = Path(token_file) if token_file else None
|
||||
self.api = api.rstrip("/")
|
||||
self.http = http or httpx.Client(timeout=30.0)
|
||||
self.sleep = sleep
|
||||
self.user_id: Optional[str] = None
|
||||
self.username: Optional[str] = None
|
||||
self._lock = threading.Lock()
|
||||
self._token = token
|
||||
self._checked_at = 0.0
|
||||
saved = self._read_file()
|
||||
if saved:
|
||||
self._token = saved["token"]
|
||||
|
||||
# --- Token ---
|
||||
|
||||
def _seed(self) -> str:
|
||||
return hashlib.sha256(self.config_token.encode()).hexdigest()
|
||||
|
||||
def _read_file(self) -> Optional[dict]:
|
||||
"""The saved token, if it descends from the configured one"""
|
||||
if not self.token_file or not self.token_file.exists():
|
||||
return None
|
||||
try:
|
||||
data = json.loads(self.token_file.read_text())
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
return data if data.get("seed") == self._seed() and data.get("token") else None
|
||||
|
||||
def _write_file(self, token: str, refreshed_at: float) -> None:
|
||||
tmp = self.token_file.with_suffix(".tmp")
|
||||
fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
||||
with os.fdopen(fd, "w") as f:
|
||||
json.dump({"seed": self._seed(), "token": token, "refreshed_at": refreshed_at}, f)
|
||||
os.replace(tmp, self.token_file)
|
||||
|
||||
def maintain(self) -> None:
|
||||
"""Refresh the token if a day has passed since the last try (cheap to call often)"""
|
||||
now = time.time()
|
||||
if now - self._checked_at < THREADS_REFRESH_EVERY:
|
||||
return
|
||||
self._checked_at = now
|
||||
if not self.token_file:
|
||||
return
|
||||
# Gunicorn workers share the file: one refreshes, the others pick up its result
|
||||
self.token_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(self.token_file.with_suffix(".lock"), "w") as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX)
|
||||
saved = self._read_file()
|
||||
if saved:
|
||||
self._token = saved["token"]
|
||||
if now - saved.get("refreshed_at", 0) < THREADS_REFRESH_EVERY:
|
||||
self._checked_at = saved["refreshed_at"]
|
||||
return
|
||||
base = self.api.rsplit("/", 1)[0] # refresh lives outside the versioned path
|
||||
try:
|
||||
response = self.http.get(f"{base}/refresh_access_token", params={
|
||||
"grant_type": "th_refresh_token", "access_token": self._token})
|
||||
except httpx.HTTPError as e:
|
||||
current_app.logger.warning(f"Threads token refresh failed: {type(e).__name__}")
|
||||
self._checked_at = now - THREADS_REFRESH_EVERY + 3600 # try again in an hour
|
||||
return
|
||||
if response.status_code < 400:
|
||||
self._token = response.json()["access_token"]
|
||||
current_app.logger.info("Threads access token refreshed")
|
||||
else:
|
||||
# e.g. a token less than a day old: keep it, try again tomorrow
|
||||
current_app.logger.warning(
|
||||
f"Threads token refresh failed: HTTP {response.status_code} {response.text[:200]}")
|
||||
self._write_file(self._token, now)
|
||||
|
||||
# --- Graph plumbing ---
|
||||
|
||||
def _call(self, method: str, path: str, **params) -> dict:
|
||||
params["access_token"] = self._token
|
||||
kwargs = {"params": params} if method in ("GET", "DELETE") else {"data": params}
|
||||
try:
|
||||
response = self.http.request(method, f"{self.api}/{path}", **kwargs)
|
||||
except httpx.HTTPError as e:
|
||||
raise SyndicationError(f"{method} {path}: {type(e).__name__}") # never echo the URL/token
|
||||
try:
|
||||
body = response.json()
|
||||
except ValueError:
|
||||
body = {}
|
||||
error = body.get("error") if isinstance(body, dict) else None
|
||||
if response.status_code >= 400 or error:
|
||||
error = error if isinstance(error, dict) else {}
|
||||
code = error.get("code")
|
||||
detail = " / ".join(str(error[k]) for k in ("error_user_title", "error_user_msg")
|
||||
if error.get(k))
|
||||
message = (f"{method} {path}: HTTP {response.status_code} "
|
||||
f"{error.get('message') or response.text[:200]}"
|
||||
f" (code {code}, subcode {error.get('error_subcode')}"
|
||||
f"{', ' + detail if detail else ''}, fbtrace {error.get('fbtrace_id')})")
|
||||
if response.status_code == 404 or (code == 100 and error.get("error_subcode") == 33):
|
||||
raise RemoteGone(message, permanent=True)
|
||||
permanent = (400 <= response.status_code < 500 and response.status_code != 429
|
||||
and code not in _GRAPH_TRANSIENT)
|
||||
exc = SyndicationError(message, permanent=permanent)
|
||||
exc.code = code
|
||||
raise exc
|
||||
return body
|
||||
|
||||
def _me(self) -> str:
|
||||
with self._lock:
|
||||
if not self.user_id:
|
||||
me = self._call("GET", "me", fields="id,username")
|
||||
self.user_id, self.username = me["id"], me.get("username")
|
||||
return self.user_id
|
||||
|
||||
def _wait(self, container_id: str) -> None:
|
||||
"""Wait until Meta has processed a container (images take a few seconds)"""
|
||||
delay = 1
|
||||
self.sleep(2) # Meta can report a container ready a moment too early
|
||||
for _ in range(8):
|
||||
status = self._call("GET", container_id, fields="status,error_message")
|
||||
state = status.get("status")
|
||||
if state in ("FINISHED", None):
|
||||
return
|
||||
if state in ("ERROR", "EXPIRED"):
|
||||
raise SyndicationError(
|
||||
f"Threads container {state}: {status.get('error_message') or 'no detail'}",
|
||||
permanent=True)
|
||||
self.sleep(delay)
|
||||
delay = min(delay * 2, 15)
|
||||
raise SyndicationError("Threads container still processing")
|
||||
|
||||
# --- Posts ---
|
||||
|
||||
def _text(self, post: "Post") -> str:
|
||||
# Links written "text (url)"; the first one gets Threads' preview card.
|
||||
# Meta allows 5 different links per post and the permalink is one.
|
||||
body = plain_text(post.note.content, links="inline", max_urls=THREADS_MAX_LINKS - 1)
|
||||
limit = post.max_chars
|
||||
while True:
|
||||
text = compose_text(post.note, post.permalink, [], limit, body=body)
|
||||
over = threads_length(text) - post.max_chars
|
||||
if over <= 0:
|
||||
return text
|
||||
limit -= over
|
||||
|
||||
def _image_container(self, uid: str, media: dict, **extra) -> str:
|
||||
params = {"media_type": "IMAGE", "image_url": media["url"], **extra}
|
||||
if media["description"]:
|
||||
params["alt_text"] = media["description"][:1000]
|
||||
return self._call("POST", f"{uid}/threads", **params)["id"]
|
||||
|
||||
def _repost_text(self, post: "Post") -> str:
|
||||
comment = plain_text(post.note.content, links="inline", max_urls=THREADS_MAX_LINKS - 1)
|
||||
limit = post.max_chars
|
||||
while True:
|
||||
text = compose_repost_text(post, comment, limit)
|
||||
over = threads_length(text) - post.max_chars
|
||||
if over <= 0:
|
||||
return text
|
||||
limit -= over
|
||||
|
||||
def _container(self, uid: str, post: "Post") -> str:
|
||||
if post.kind == "repost":
|
||||
# No native repost of outside content: post "🔁 title url"
|
||||
return self._call("POST", f"{uid}/threads", media_type="TEXT",
|
||||
text=self._repost_text(post))["id"]
|
||||
common = {"text": self._text(post)}
|
||||
topic = threads_topic(post.tags)
|
||||
if topic:
|
||||
common["topic_tag"] = topic
|
||||
# Threads takes JPEG/PNG up to 8 MB; others (GIF, WebP) are left off
|
||||
images = [m for m in post.media if m["mime_type"] in THREADS_IMAGE_TYPES
|
||||
and Path(m["path"]).stat().st_size <= THREADS_IMAGE_LIMIT]
|
||||
if not images:
|
||||
# No explicit link_attachment: Meta rejects one for a page it hasn't
|
||||
# seen yet ("Invalid Link Attachment"), but previews the text's
|
||||
# first link by itself without failing
|
||||
return self._call("POST", f"{uid}/threads", media_type="TEXT", **common)["id"]
|
||||
if len(images) == 1:
|
||||
return self._image_container(uid, images[0], **common)
|
||||
children = [self._image_container(uid, m, is_carousel_item="true") for m in images]
|
||||
for child in children:
|
||||
self._wait(child)
|
||||
return self._call("POST", f"{uid}/threads", media_type="CAROUSEL",
|
||||
children=",".join(children), **common)["id"]
|
||||
|
||||
def _profile_url(self) -> str:
|
||||
return f"https://www.threads.com/@{self.username}" if self.username else "https://www.threads.com/"
|
||||
|
||||
def _find_existing(self, uid: str, link: str) -> Optional[dict]:
|
||||
"""A recent post that already has this link (an earlier send whose reply was lost)"""
|
||||
listing = self._call("GET", f"{uid}/threads", fields="id,text,permalink", limit=25)
|
||||
for item in listing.get("data", []):
|
||||
if link in (item.get("text") or ""):
|
||||
return {"id": item["id"], "url": item.get("permalink") or self._profile_url()}
|
||||
return None
|
||||
|
||||
def create(self, post: "Post", key: str, retrying: bool = False) -> dict:
|
||||
if post.kind == "like" and post.target_url:
|
||||
return {"id": None, "url": None} # the API can't like things; likes stay on the site
|
||||
uid = self._me()
|
||||
if retrying:
|
||||
marker = post.target_url if post.kind == "repost" and post.target_url else post.permalink
|
||||
existing = self._find_existing(uid, marker)
|
||||
if existing:
|
||||
return existing
|
||||
container = self._container(uid, post)
|
||||
self._wait(container)
|
||||
try:
|
||||
media_id = self._call("POST", f"{uid}/threads_publish", creation_id=container)["id"]
|
||||
except SyndicationError as e:
|
||||
if getattr(e, "code", None) in _GRAPH_AUTH:
|
||||
raise
|
||||
# Meta's publish step fails now and then ("Fatal") and works on a
|
||||
# later try; the retry checks for an existing post first
|
||||
raise SyndicationError(str(e))
|
||||
try:
|
||||
url = self._call("GET", media_id, fields="permalink").get("permalink")
|
||||
except SyndicationError:
|
||||
url = None # it's published; don't fail (and post twice) over the link
|
||||
return {"id": media_id, "url": url or self._profile_url()}
|
||||
|
||||
def update(self, remote_id: str, post: "Post") -> dict:
|
||||
raise NotImplementedError("Threads posts can't be edited through the API")
|
||||
|
||||
def delete(self, remote_id: str) -> None:
|
||||
try:
|
||||
self._call("DELETE", remote_id)
|
||||
except RemoteGone:
|
||||
pass # already gone
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -327,6 +802,24 @@ class Post:
|
||||
tags: List[str]
|
||||
max_chars: int
|
||||
media: List[dict] = field(default_factory=list)
|
||||
kind: str = "note" # note, like or repost (v2.5.0)
|
||||
target_url: Optional[str] = None # what a like/repost points at
|
||||
target_title: Optional[str] = None # its fetched title, if any
|
||||
|
||||
|
||||
def compose_repost_text(post: "Post", comment: str, max_chars: int) -> str:
|
||||
"""'comment' then '🔁 title' and the URL; the comment is shortened to fit"""
|
||||
title = post.target_title or "Reposted"
|
||||
tail = f"🔁 {title}\n{post.target_url}"
|
||||
if len(tail) > max_chars:
|
||||
tail = f"🔁 {post.target_url}"
|
||||
comment = (comment or "").strip()
|
||||
if not comment:
|
||||
return tail
|
||||
room = max_chars - len(tail) - 2
|
||||
if len(comment) > room:
|
||||
comment = comment[:max(room - 1, 0)].rsplit(" ", 1)[0].rstrip() + "…"
|
||||
return f"{comment}\n\n{tail}" if room > 1 else tail
|
||||
|
||||
|
||||
_client_cache: Dict[str, object] = {}
|
||||
@@ -337,6 +830,7 @@ def targets_configured(config) -> bool:
|
||||
return bool(
|
||||
(config.get("GOTOSOCIAL_URL") and config.get("GOTOSOCIAL_ACCESS_TOKEN"))
|
||||
or (config.get("BLUESKY_HANDLE") and config.get("BLUESKY_APP_PASSWORD"))
|
||||
or config.get("THREADS_ACCESS_TOKEN")
|
||||
)
|
||||
|
||||
|
||||
@@ -378,6 +872,23 @@ def get_targets() -> Dict[str, Target]:
|
||||
max_chars=int(config.get("BLUESKY_MAX_CHARS", 300)),
|
||||
max_media=4,
|
||||
)
|
||||
|
||||
token = config.get("THREADS_ACCESS_TOKEN") or ""
|
||||
if token:
|
||||
client = _client_cache.get("threads")
|
||||
if client is None or client.config_token != token:
|
||||
client = ThreadsClient(token, Path(config["DATA_PATH"]) / "threads_token.json",
|
||||
api=config.get("THREADS_API") or THREADS_API)
|
||||
_client_cache["threads"] = client
|
||||
username = (config.get("THREADS_USERNAME") or "").lstrip("@")
|
||||
targets["threads"] = Target(
|
||||
id="threads",
|
||||
uid=f"https://www.threads.com/@{username}" if username else "https://www.threads.com/",
|
||||
name=config.get("THREADS_NAME", "Threads"),
|
||||
client=client,
|
||||
max_chars=int(config.get("THREADS_MAX_CHARS", 500)),
|
||||
max_media=int(config.get("THREADS_MAX_MEDIA", 4)),
|
||||
)
|
||||
return targets
|
||||
|
||||
|
||||
@@ -491,23 +1002,88 @@ def compose_text(note, permalink: str, tags: List[str], max_chars: int,
|
||||
hashtags = f"{hashtags} {h}".strip()
|
||||
tail = f"\n\n{permalink}" + (f"\n\n{hashtags}" if hashtags else "")
|
||||
body = (note.content if body is None else body).strip()
|
||||
if getattr(note, "bookmark_of", None):
|
||||
# A bookmark leads with what it bookmarks (v2.9.0)
|
||||
body = f"🔖 {note.bookmark_of}\n\n{body}".strip()
|
||||
if len(body) + len(tail) > max_chars:
|
||||
room = max_chars - len(tail) - 1
|
||||
body = body[:room].rsplit(" ", 1)[0].rstrip() + "…"
|
||||
return body + tail
|
||||
# A photo post may have no text: start with the link (v2.6.5)
|
||||
return (body + tail).lstrip()
|
||||
|
||||
|
||||
_BLOCKS = ["p", "h1", "h2", "h3", "h4", "h5", "h6", "pre", "blockquote", "ul", "ol", "hr", "table"]
|
||||
|
||||
|
||||
def plain_text(markdown_text: str) -> str:
|
||||
"""Markdown as readable plain text (for targets without Markdown)"""
|
||||
def _note_soup(markdown_text: str):
|
||||
import markdown
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
soup = BeautifulSoup(markdown.markdown(markdown_text, extensions=["fenced_code"]), "html.parser")
|
||||
return BeautifulSoup(markdown.markdown(markdown_text, extensions=["fenced_code"]), "html.parser")
|
||||
|
||||
|
||||
def _clean_url(url: str) -> str:
|
||||
return url.rstrip(".,;:!?)\"'…")
|
||||
|
||||
|
||||
def _is_web(url: str) -> bool:
|
||||
return url.startswith(("http://", "https://"))
|
||||
|
||||
|
||||
def note_links(markdown_text: str) -> List[str]:
|
||||
"""Web links in a note, in reading order: Markdown links, <autolinks> and bare URLs"""
|
||||
from bs4 import NavigableString
|
||||
|
||||
links = []
|
||||
for node in _note_soup(markdown_text).descendants:
|
||||
if getattr(node, "name", None) == "a" and _is_web(node.get("href", "")):
|
||||
links.append(node["href"])
|
||||
elif isinstance(node, NavigableString) and not node.find_parent("a"):
|
||||
links.extend(_clean_url(m.group(0)) for m in _URL_RE.finditer(str(node)))
|
||||
return list(dict.fromkeys(links))
|
||||
|
||||
|
||||
def plain_text(markdown_text: str, links: str = "text", max_urls: Optional[int] = None) -> str:
|
||||
"""Markdown as readable plain text (for targets without Markdown). See `plain_text_links`."""
|
||||
return plain_text_links(markdown_text, links, max_urls)[0]
|
||||
|
||||
|
||||
def plain_text_links(markdown_text: str, links: str = "text",
|
||||
max_urls: Optional[int] = None) -> tuple:
|
||||
"""
|
||||
Markdown as plain text, and the links whose text stands in for a URL.
|
||||
|
||||
A Markdown link `[text](url)` becomes:
|
||||
links="text" just its text
|
||||
links="inline" "text (url)", for targets with no clickable link text;
|
||||
after `max_urls` different URLs, just the text
|
||||
links="anchor" its text, listed in the returned anchors as (text, url)
|
||||
so the target can make that text clickable
|
||||
A link whose text is the URL itself stays a bare URL.
|
||||
"""
|
||||
from bs4 import NavigableString
|
||||
|
||||
soup = _note_soup(markdown_text)
|
||||
for img in soup.find_all("img"):
|
||||
img.decompose()
|
||||
anchors = []
|
||||
seen = set()
|
||||
for node in list(soup.descendants): # bare URLs count towards max_urls first
|
||||
if isinstance(node, NavigableString) and not node.find_parent("a"):
|
||||
seen.update(_clean_url(m.group(0)) for m in _URL_RE.finditer(str(node)))
|
||||
for a in soup.find_all("a"):
|
||||
href, label = a.get("href", ""), a.get_text()
|
||||
if not _is_web(href):
|
||||
continue
|
||||
bare = re.sub(r"^https?://", "", href).rstrip("/")
|
||||
if label.strip().rstrip("/") in (href.rstrip("/"), bare):
|
||||
a.replace_with(href)
|
||||
seen.add(href)
|
||||
elif links == "inline" and (href in seen or max_urls is None or len(seen) < max_urls):
|
||||
a.replace_with(f"{label} ({href})")
|
||||
seen.add(href)
|
||||
elif links == "anchor":
|
||||
anchors.append((label, href))
|
||||
for node in soup.find_all(string=True):
|
||||
if not node.strip() and "\n" in node and node.parent.name != "pre":
|
||||
node.extract() # layout newlines between tags; we add our own
|
||||
@@ -520,24 +1096,41 @@ def plain_text(markdown_text: str) -> str:
|
||||
block.append("\n\n")
|
||||
text = soup.get_text()
|
||||
text = re.sub(r"[ \t]+\n", "\n", text)
|
||||
return re.sub(r"\n{3,}", "\n\n", text).strip()
|
||||
return re.sub(r"\n{3,}", "\n\n", text).strip(), anchors
|
||||
|
||||
|
||||
_URL_RE = re.compile(r"https?://[^\s<>\"]+")
|
||||
_TAG_RE = re.compile(r"(?<![\w#])#(\w*[^\W\d]\w*)")
|
||||
|
||||
|
||||
def bluesky_facets(text: str) -> List[dict]:
|
||||
"""Link and hashtag facets (UTF-8 byte ranges) for a Bluesky post"""
|
||||
def bluesky_facets(text: str, anchors: List[tuple] = ()) -> List[dict]:
|
||||
"""
|
||||
Link and hashtag facets (UTF-8 byte ranges) for a Bluesky post.
|
||||
`anchors` are (text, url) pairs: that text, where it appears, links to url.
|
||||
"""
|
||||
def span(start: int, end: int) -> dict:
|
||||
return {"byteStart": len(text[:start].encode()), "byteEnd": len(text[:end].encode())}
|
||||
|
||||
facets = []
|
||||
for m in _URL_RE.finditer(text):
|
||||
url = m.group(0).rstrip(".,;:!?)\"'…")
|
||||
url = _clean_url(m.group(0))
|
||||
facets.append({"index": span(m.start(), m.start() + len(url)),
|
||||
"features": [{"$type": "app.bsky.richtext.facet#link", "uri": url}]})
|
||||
taken = [(m.start(), m.end()) for m in _URL_RE.finditer(text)]
|
||||
pos = 0
|
||||
for label, url in anchors:
|
||||
start = text.find(label, pos) if label.strip() else -1
|
||||
if start < 0:
|
||||
continue # shortened away
|
||||
end = start + len(label)
|
||||
if any(a < end and start < b for a, b in taken):
|
||||
continue
|
||||
taken.append((start, end))
|
||||
pos = end
|
||||
facets.append({"index": span(start, end),
|
||||
"features": [{"$type": "app.bsky.richtext.facet#link", "uri": url}]})
|
||||
urls = [(f["index"]["byteStart"], f["index"]["byteEnd"]) for f in facets]
|
||||
facets.sort(key=lambda f: f["index"]["byteStart"])
|
||||
for m in _TAG_RE.finditer(text):
|
||||
index = span(m.start(), m.end())
|
||||
if any(a <= index["byteStart"] < b for a, b in urls):
|
||||
@@ -547,23 +1140,84 @@ def bluesky_facets(text: str) -> List[dict]:
|
||||
return facets
|
||||
|
||||
|
||||
def _media_files(note_id: int, limit: int) -> List[dict]:
|
||||
"""Image files for a note: the 'large' variant when there is one"""
|
||||
_MD_IMAGE_RE = re.compile(r"!\[[^\]]*\]\([^)]*\)")
|
||||
_HTML_IMAGE_RE = re.compile(r"<img\b[^>]*>", re.IGNORECASE)
|
||||
|
||||
|
||||
def inline_images(markdown_text: str) -> List[tuple]:
|
||||
"""(url, alt) for each image embedded in a note's text, in reading order"""
|
||||
images = []
|
||||
for img in _note_soup(markdown_text).find_all("img"):
|
||||
if img.get("src"):
|
||||
images.append((img["src"], img.get("alt") or ""))
|
||||
return images
|
||||
|
||||
|
||||
def strip_images(markdown_text: str) -> str:
|
||||
"""A note's Markdown without its embedded images"""
|
||||
text = _HTML_IMAGE_RE.sub("", _MD_IMAGE_RE.sub("", markdown_text))
|
||||
return re.sub(r"\n{3,}", "\n\n", text).strip()
|
||||
|
||||
|
||||
def _own_media_path(url: str) -> Optional[str]:
|
||||
"""Path in the media library for one of this site's /media/ URLs"""
|
||||
site = current_app.config["SITE_URL"].rstrip("/")
|
||||
if url.startswith(site):
|
||||
url = url[len(site):]
|
||||
return url[len("/media/"):] if url.startswith("/media/") else None
|
||||
|
||||
|
||||
def _media_files(note_id: int, limit: int, content: str = "") -> List[dict]:
|
||||
"""
|
||||
Image files for a note: the 'large' variant when there is one.
|
||||
|
||||
Attached photos come first, then images embedded in the text that are in
|
||||
this site's media library. Some Micropub clients (OutPost, for one)
|
||||
upload a photo and put it in the content rather than in `photo` (v2.6.6).
|
||||
"""
|
||||
from starpunk.media import get_note_media
|
||||
|
||||
media_dir = Path(current_app.config["DATA_PATH"]) / "media"
|
||||
site = current_app.config["SITE_URL"].rstrip("/")
|
||||
items = [(m, m.get("caption") or "") for m in get_note_media(note_id)]
|
||||
seen = {m["path"] for m, _ in items}
|
||||
for url, alt in inline_images(content) if content else []:
|
||||
path = _own_media_path(url)
|
||||
if not path or path in seen:
|
||||
continue
|
||||
media = _media_by_path(path)
|
||||
if media:
|
||||
seen.add(path)
|
||||
items.append((media, alt or media.get("caption") or ""))
|
||||
|
||||
files = []
|
||||
for media in get_note_media(note_id)[:limit]:
|
||||
for media, description in items[:limit]:
|
||||
variant = (media.get("variants") or {}).get("large")
|
||||
rel = variant["path"] if variant else media["path"]
|
||||
files.append({
|
||||
"path": media_dir / rel,
|
||||
"url": f"{site}/media/{rel}", # for targets that fetch images themselves
|
||||
"mime_type": media.get("mime_type") or "image/jpeg",
|
||||
"description": media.get("caption") or "",
|
||||
"description": description,
|
||||
})
|
||||
return files
|
||||
|
||||
|
||||
def _media_by_path(path: str) -> Optional[dict]:
|
||||
"""A media row with its variants, found by its path in the library"""
|
||||
db = get_db(current_app)
|
||||
row = db.execute("SELECT id, path, mime_type FROM media WHERE path = ?", (path,)).fetchone()
|
||||
if not row:
|
||||
return None
|
||||
variants = {
|
||||
v["variant_type"]: {"path": v["path"]}
|
||||
for v in db.execute(
|
||||
"SELECT variant_type, path FROM media_variants WHERE media_id = ?", (row["id"],)
|
||||
)
|
||||
}
|
||||
return {"path": row["path"], "mime_type": row["mime_type"], "variants": variants, "caption": ""}
|
||||
|
||||
|
||||
# --- Worker -----------------------------------------------------------------
|
||||
|
||||
def _claim(db, row_id: int) -> bool:
|
||||
@@ -615,6 +1269,21 @@ def _fail(db, row, error: SyndicationError) -> None:
|
||||
)
|
||||
|
||||
|
||||
def _target_title(url: Optional[str]) -> Optional[str]:
|
||||
"""Fetched title of a liked/reposted post, if we have one"""
|
||||
if not url:
|
||||
return None
|
||||
try:
|
||||
from starpunk.webmention_queries import get_reply_context
|
||||
context = get_reply_context(url) or {}
|
||||
except Exception:
|
||||
return None
|
||||
title = context.get("title") or ""
|
||||
if not title and context.get("author_name"):
|
||||
title = f"Post by {context['author_name']}"
|
||||
return " ".join(title.split())[:200] or None
|
||||
|
||||
|
||||
def _reconcile(row, target: Target) -> tuple:
|
||||
"""Bring the remote post in line with the note. Returns (remote_id, remote_url)."""
|
||||
from starpunk.notes import get_note
|
||||
@@ -635,9 +1304,13 @@ def _reconcile(row, target: Target) -> tuple:
|
||||
post = Post(
|
||||
note=note,
|
||||
permalink=f"{site}/note/{note.slug}",
|
||||
tags=[t["display_name"] for t in get_note_tags(note.id)],
|
||||
tags=[t["display_name"] for t in get_note_tags(note.id, entered_order=True)],
|
||||
max_chars=target.max_chars,
|
||||
media=_media_files(note.id, target.max_media),
|
||||
media=_media_files(note.id, target.max_media, note.content)
|
||||
if note.kind == "note" else [],
|
||||
kind=note.kind,
|
||||
target_url=note.reacts_to,
|
||||
target_title=_target_title(note.reacts_to),
|
||||
)
|
||||
|
||||
if row["remote_id"]:
|
||||
@@ -658,6 +1331,13 @@ def process_due(limit: int = 5) -> int:
|
||||
targets = get_targets()
|
||||
if not targets:
|
||||
return 0
|
||||
for target in targets.values():
|
||||
maintain = getattr(target.client, "maintain", None)
|
||||
if maintain:
|
||||
try:
|
||||
maintain()
|
||||
except Exception as e:
|
||||
current_app.logger.error(f"Syndication {target.id} upkeep failed: {e}")
|
||||
db = get_db(current_app)
|
||||
rows = db.execute(
|
||||
"""
|
||||
|
||||
+11
-6
@@ -115,27 +115,32 @@ def add_tags_to_note(note_id: int, tags: list[str]) -> None:
|
||||
db.commit()
|
||||
|
||||
|
||||
def get_note_tags(note_id: int) -> list[dict]:
|
||||
def get_note_tags(note_id: int, entered_order: bool = False) -> list[dict]:
|
||||
"""
|
||||
Get all tags for a note
|
||||
|
||||
Args:
|
||||
note_id: Note database ID
|
||||
entered_order: In the order they were entered (the first is the
|
||||
note's main tag, e.g. its Threads topic) instead of alphabetical.
|
||||
add_tags_to_note() rewrites the links in entry order on every
|
||||
save, so note_tags.id records it. Use this wherever the tags are
|
||||
saved back (editor, Micropub update) so the order survives.
|
||||
|
||||
Returns:
|
||||
List of tag dicts with 'name' and 'display_name'
|
||||
Ordered alphabetically by display_name (case-insensitive)
|
||||
|
||||
Query should use: ORDER BY LOWER(tags.display_name) ASC
|
||||
Ordered alphabetically by display_name (case-insensitive), unless
|
||||
entered_order
|
||||
"""
|
||||
db = get_db(current_app)
|
||||
order = "note_tags.id" if entered_order else "LOWER(tags.display_name)"
|
||||
tags = db.execute(
|
||||
"""
|
||||
f"""
|
||||
SELECT tags.name, tags.display_name
|
||||
FROM tags
|
||||
JOIN note_tags ON tags.id = note_tags.tag_id
|
||||
WHERE note_tags.note_id = ?
|
||||
ORDER BY LOWER(tags.display_name) ASC
|
||||
ORDER BY {order} ASC
|
||||
""",
|
||||
(note_id,)
|
||||
).fetchall()
|
||||
|
||||
+24
-1
@@ -11,7 +11,7 @@ import hashlib
|
||||
import re
|
||||
import secrets
|
||||
import shutil
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
@@ -693,3 +693,26 @@ def absolute_url_for(endpoint: str, **values) -> str:
|
||||
from flask import current_app, url_for
|
||||
|
||||
return current_app.config["SITE_URL"].rstrip("/") + url_for(endpoint, **values)
|
||||
|
||||
|
||||
def iso_utc(value) -> str:
|
||||
"""
|
||||
Format a timestamp as RFC 3339 UTC for dt-* and Micropub (v2.8.0)
|
||||
|
||||
Stored times are naive UTC. Without an offset, microformats parsers read
|
||||
them as local time, so this adds 'Z' and drops microseconds (HTML
|
||||
<time datetime> allows at most 3 fractional digits).
|
||||
|
||||
Args:
|
||||
value: datetime (naive = UTC) or ISO 8601 string
|
||||
|
||||
Returns:
|
||||
e.g. '2026-11-01T16:25:12Z', or '' for empty values
|
||||
"""
|
||||
if not value:
|
||||
return ''
|
||||
if isinstance(value, str):
|
||||
value = datetime.fromisoformat(value.strip().replace('Z', '+00:00'))
|
||||
if value.tzinfo is None:
|
||||
value = value.replace(tzinfo=timezone.utc)
|
||||
return value.astimezone(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
|
||||
+171
-200
@@ -14,15 +14,19 @@ References:
|
||||
- W3C Webmention: https://www.w3.org/TR/webmention/
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import mf2py
|
||||
import nh3
|
||||
import httpx
|
||||
from datetime import datetime
|
||||
from typing import Dict, Any, Optional
|
||||
from urllib.parse import urlparse
|
||||
from urllib.parse import urldefrag, urlparse
|
||||
|
||||
from starpunk.webmention_ssrf import validate_url_for_fetch, SSRFBlockedError
|
||||
from starpunk import mf2_utils
|
||||
from starpunk.utils import iso_utc
|
||||
from starpunk.webmention_ssrf import (
|
||||
FetchTooLargeError, SSRFBlockedError, safe_get, user_agent,
|
||||
)
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -33,205 +37,155 @@ class ValidationError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def fetch_source(url: str, site_url: str, timeout: int = 10, max_size: int = 1048576) -> tuple[str, str]:
|
||||
ACCEPT = ('text/html, application/xhtml+xml;q=0.9, application/json;q=0.8, '
|
||||
'text/plain;q=0.7, */*;q=0.1')
|
||||
|
||||
|
||||
class TransientError(ValidationError):
|
||||
"""Source could not be checked right now (network error, 5xx)"""
|
||||
pass
|
||||
|
||||
|
||||
def fetch_source(url: str, site_url: str) -> tuple[str, str, str]:
|
||||
"""
|
||||
Fetch source URL content with SSRF protection.
|
||||
|
||||
Per spec 3.2.2: GET following redirects (limited), with an Accept header.
|
||||
|
||||
Args:
|
||||
url: Source URL to fetch
|
||||
site_url: Site URL for User-Agent header
|
||||
timeout: Request timeout in seconds (default: 10)
|
||||
max_size: Maximum content size in bytes (default: 1MB)
|
||||
|
||||
Returns:
|
||||
Tuple of (content, content_type)
|
||||
Tuple of (content, content_type, final_url)
|
||||
|
||||
Raises:
|
||||
SSRFBlockedError: If URL points to private IP
|
||||
ValidationError: If fetch fails or content too large
|
||||
ValidationError: Source is blocked, too large, or returned 4xx
|
||||
TransientError: Network error, timeout or 5xx
|
||||
"""
|
||||
# SSRF protection
|
||||
try:
|
||||
validate_url_for_fetch(url)
|
||||
result = safe_get(url, headers={'User-Agent': user_agent(site_url), 'Accept': ACCEPT})
|
||||
except (SSRFBlockedError, ValueError) as e:
|
||||
logger.warning(f"SSRF protection blocked URL {url}: {e}")
|
||||
raise ValidationError(f"Blocked request: {e}")
|
||||
|
||||
# Fetch URL
|
||||
try:
|
||||
# Use streaming to check size before downloading entire content
|
||||
with httpx.stream(
|
||||
'GET',
|
||||
url,
|
||||
timeout=timeout,
|
||||
headers={'User-Agent': f'StarPunk/1.6.0 (+{site_url})'},
|
||||
follow_redirects=True
|
||||
) as response:
|
||||
response.raise_for_status()
|
||||
|
||||
# Check content length if provided
|
||||
content_length = response.headers.get('Content-Length')
|
||||
if content_length and int(content_length) > max_size:
|
||||
raise ValidationError(f"Source exceeds size limit ({max_size} bytes)")
|
||||
|
||||
# Read content with size limit
|
||||
content = b''
|
||||
for chunk in response.iter_bytes(chunk_size=8192):
|
||||
content += chunk
|
||||
if len(content) > max_size:
|
||||
raise ValidationError(f"Source exceeds size limit ({max_size} bytes)")
|
||||
|
||||
content_type = response.headers.get('Content-Type', '')
|
||||
|
||||
return content.decode('utf-8', errors='replace'), content_type
|
||||
|
||||
except FetchTooLargeError as e:
|
||||
raise ValidationError(f"Source {e}")
|
||||
except httpx.TimeoutException:
|
||||
raise ValidationError("Source fetch timed out")
|
||||
except httpx.HTTPStatusError as e:
|
||||
if 400 <= e.response.status_code < 500:
|
||||
raise ValidationError(f"Source returned {e.response.status_code}")
|
||||
else:
|
||||
raise ValidationError(f"Source server error: {e.response.status_code}")
|
||||
raise TransientError("Source fetch timed out")
|
||||
except httpx.HTTPError as e:
|
||||
raise ValidationError(f"Failed to fetch source: {e}")
|
||||
except UnicodeDecodeError:
|
||||
raise ValidationError("Source content encoding is not supported")
|
||||
raise TransientError(f"Failed to fetch source: {e}")
|
||||
|
||||
if 400 <= result.status_code < 500:
|
||||
raise ValidationError(f"Source returned {result.status_code}")
|
||||
if result.status_code >= 300:
|
||||
raise TransientError(f"Source server error: {result.status_code}")
|
||||
|
||||
return result.text, result.headers.get('Content-Type', ''), result.url
|
||||
|
||||
|
||||
def verify_link_presence(content: str, target_url: str, content_type: str) -> bool:
|
||||
def _strip_fragment(url: str) -> str:
|
||||
return urldefrag(url.strip())[0]
|
||||
|
||||
|
||||
def _json_strings(value):
|
||||
"""Yield every string in a parsed JSON document"""
|
||||
if isinstance(value, str):
|
||||
yield value
|
||||
elif isinstance(value, dict):
|
||||
for item in value.values():
|
||||
yield from _json_strings(item)
|
||||
elif isinstance(value, list):
|
||||
for item in value:
|
||||
yield from _json_strings(item)
|
||||
|
||||
|
||||
def verify_link_presence(content: str, target_url: str, content_type: str,
|
||||
source_url: str = '') -> bool:
|
||||
"""
|
||||
Verify that source content contains a link to target URL.
|
||||
Verify the source links to the target (spec 3.2.2).
|
||||
|
||||
The source MUST have an exact match of the target URL, checked per media
|
||||
type: in HTML, any link-bearing attribute (a/link href, img/video/audio
|
||||
src, ...) resolved against the source URL; in JSON, any string value; in
|
||||
other text, the URL anywhere. Fragments are ignored on both sides.
|
||||
|
||||
Args:
|
||||
content: Source content (HTML or text)
|
||||
content: Source content
|
||||
target_url: Target URL to find
|
||||
content_type: Content type of source
|
||||
source_url: Final source URL, for resolving relative links
|
||||
|
||||
Returns:
|
||||
True if target link found, False otherwise
|
||||
True if the target is linked, False otherwise
|
||||
"""
|
||||
# Normalize URLs for comparison (remove fragments, trailing slashes)
|
||||
target_normalized = urlparse(target_url)._replace(fragment='').geturl().rstrip('/')
|
||||
target = _strip_fragment(target_url)
|
||||
media_type = content_type.split(';')[0].strip().lower()
|
||||
|
||||
# For HTML, parse and check all links
|
||||
if 'html' in content_type.lower():
|
||||
if 'html' in media_type:
|
||||
from lxml import html as lxml_html
|
||||
try:
|
||||
from lxml import html as lxml_html
|
||||
doc = lxml_html.fromstring(content)
|
||||
links = doc.xpath('//a/@href')
|
||||
|
||||
for link in links:
|
||||
# Make absolute if relative
|
||||
link_normalized = urlparse(link)._replace(fragment='').geturl().rstrip('/')
|
||||
if link_normalized == target_normalized:
|
||||
return True
|
||||
doc = lxml_html.document_fromstring(content)
|
||||
except Exception as e:
|
||||
logger.debug(f"HTML parsing failed, falling back to string search: {e}")
|
||||
logger.debug(f"HTML parsing failed: {e}")
|
||||
return False
|
||||
if source_url:
|
||||
doc.make_links_absolute(source_url, resolve_base_href=True,
|
||||
handle_failures='ignore')
|
||||
return any(_strip_fragment(link) == target for _, _, link, _ in doc.iterlinks())
|
||||
|
||||
# Fallback: simple string search
|
||||
return target_normalized in content
|
||||
if media_type == 'application/json' or media_type.endswith('+json'):
|
||||
try:
|
||||
data = json.loads(content)
|
||||
except ValueError:
|
||||
return False
|
||||
return any(_strip_fragment(s) == target for s in _json_strings(data))
|
||||
|
||||
if media_type.startswith('text/'):
|
||||
return target in content
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def detect_mention_type(parsed_mf2: Dict, target_url: str) -> str:
|
||||
"""
|
||||
Detect mention type from microformats.
|
||||
Detect mention type from the h-entry that responds to the target
|
||||
|
||||
Priority: reply > like > repost > bookmark > mention
|
||||
Returns:
|
||||
'reply', 'like', 'repost', 'bookmark', or 'mention'
|
||||
"""
|
||||
entry = mf2_utils.responding_entry(parsed_mf2, target_url)
|
||||
return mf2_utils.response_type(entry, target_url) if entry else 'mention'
|
||||
|
||||
|
||||
def _is_http_url(value: Optional[str]) -> bool:
|
||||
"""True if value is an absolute http or https URL"""
|
||||
if not value:
|
||||
return False
|
||||
parsed = urlparse(value.strip())
|
||||
return parsed.scheme.lower() in ('http', 'https') and bool(parsed.netloc)
|
||||
|
||||
|
||||
def extract_author(parsed_mf2: Dict, source_url: str,
|
||||
entry: Optional[Dict] = None) -> Dict[str, Optional[str]]:
|
||||
"""
|
||||
Author of the source's h-entry (authorship algorithm, without fetching)
|
||||
|
||||
Args:
|
||||
parsed_mf2: Parsed microformats data from mf2py
|
||||
target_url: Target URL being mentioned
|
||||
source_url: Source URL (fallback for author URL and name)
|
||||
entry: The h-entry to attribute (default: first top-level h-entry)
|
||||
|
||||
Returns:
|
||||
Mention type: 'reply', 'like', 'repost', 'bookmark', or 'mention'
|
||||
Dict with author_name, author_url, author_photo; only http(s) URLs
|
||||
are kept, so they are safe to render as links (XSS)
|
||||
"""
|
||||
target_normalized = urlparse(target_url)._replace(fragment='').geturl().rstrip('/')
|
||||
if entry is None:
|
||||
entry = mf2_utils.responding_entry(parsed_mf2, '')
|
||||
author = mf2_utils.entry_author(parsed_mf2, entry, source_url)
|
||||
|
||||
# Find h-entry that references target
|
||||
for item in parsed_mf2.get('items', []):
|
||||
if 'h-entry' not in item.get('type', []):
|
||||
continue
|
||||
|
||||
props = item.get('properties', {})
|
||||
|
||||
# Check in-reply-to (highest priority)
|
||||
for reply_to in props.get('in-reply-to', []):
|
||||
url = reply_to if isinstance(reply_to, str) else reply_to.get('value', '')
|
||||
if urlparse(url)._replace(fragment='').geturl().rstrip('/') == target_normalized:
|
||||
return 'reply'
|
||||
|
||||
# Check like-of
|
||||
for like in props.get('like-of', []):
|
||||
url = like if isinstance(like, str) else like.get('value', '')
|
||||
if urlparse(url)._replace(fragment='').geturl().rstrip('/') == target_normalized:
|
||||
return 'like'
|
||||
|
||||
# Check repost-of
|
||||
for repost in props.get('repost-of', []):
|
||||
url = repost if isinstance(repost, str) else repost.get('value', '')
|
||||
if urlparse(url)._replace(fragment='').geturl().rstrip('/') == target_normalized:
|
||||
return 'repost'
|
||||
|
||||
# Check bookmark-of
|
||||
for bookmark in props.get('bookmark-of', []):
|
||||
url = bookmark if isinstance(bookmark, str) else bookmark.get('value', '')
|
||||
if urlparse(url)._replace(fragment='').geturl().rstrip('/') == target_normalized:
|
||||
return 'bookmark'
|
||||
|
||||
# Default fallback
|
||||
return 'mention'
|
||||
|
||||
|
||||
def extract_author(parsed_mf2: Dict, source_url: str) -> Dict[str, Optional[str]]:
|
||||
"""
|
||||
Extract author information from h-card.
|
||||
|
||||
Args:
|
||||
parsed_mf2: Parsed microformats data from mf2py
|
||||
source_url: Source URL (fallback for author URL)
|
||||
|
||||
Returns:
|
||||
Dict with author_name, author_url, author_photo
|
||||
"""
|
||||
author_name = None
|
||||
author_url = None
|
||||
author_photo = None
|
||||
|
||||
# Find h-entry first
|
||||
entry = None
|
||||
for item in parsed_mf2.get('items', []):
|
||||
if 'h-entry' in item.get('type', []):
|
||||
entry = item
|
||||
break
|
||||
|
||||
# Check for author property in h-entry
|
||||
if entry:
|
||||
author_prop = entry.get('properties', {}).get('author', [])
|
||||
if author_prop:
|
||||
author_data = author_prop[0]
|
||||
if isinstance(author_data, dict):
|
||||
# h-card object
|
||||
props = author_data.get('properties', {})
|
||||
author_name = props.get('name', [None])[0]
|
||||
author_url = props.get('url', [None])[0]
|
||||
author_photo = props.get('photo', [None])[0]
|
||||
|
||||
# If no author in h-entry, find representative h-card
|
||||
if not author_name:
|
||||
for item in parsed_mf2.get('items', []):
|
||||
if 'h-card' in item.get('type', []):
|
||||
props = item.get('properties', {})
|
||||
author_name = props.get('name', [None])[0]
|
||||
author_url = props.get('url', [None])[0]
|
||||
author_photo = props.get('photo', [None])[0]
|
||||
break
|
||||
|
||||
# Fallback to hostname if no author found
|
||||
if not author_name:
|
||||
author_name = urlparse(source_url).netloc
|
||||
|
||||
if not author_url:
|
||||
author_url = source_url
|
||||
author_url = author['url'] if _is_http_url(author['url']) else source_url
|
||||
author_photo = author['photo'] if _is_http_url(author['photo']) else None
|
||||
author_name = author['name'] or urlparse(author_url).netloc
|
||||
|
||||
return {
|
||||
'author_name': author_name,
|
||||
@@ -240,53 +194,65 @@ def extract_author(parsed_mf2: Dict, source_url: str) -> Dict[str, Optional[str]
|
||||
}
|
||||
|
||||
|
||||
def extract_content(parsed_mf2: Dict) -> Optional[str]:
|
||||
def extract_content(parsed_mf2: Dict, entry: Optional[Dict] = None) -> Optional[str]:
|
||||
"""
|
||||
Extract content snippet from h-entry.
|
||||
Content snippet of an h-entry (e-content, else p-summary)
|
||||
|
||||
Args:
|
||||
parsed_mf2: Parsed microformats data from mf2py
|
||||
entry: The h-entry (default: first top-level h-entry)
|
||||
|
||||
Returns:
|
||||
Sanitized content snippet (max 500 chars) or None
|
||||
"""
|
||||
# Find h-entry
|
||||
for item in parsed_mf2.get('items', []):
|
||||
if 'h-entry' not in item.get('type', []):
|
||||
continue
|
||||
if entry is None:
|
||||
entry = mf2_utils.responding_entry(parsed_mf2, '')
|
||||
if not entry:
|
||||
return None
|
||||
|
||||
props = item.get('properties', {})
|
||||
props = entry.get('properties', {})
|
||||
for content in props.get('content', []):
|
||||
html = (content.get('html') or content.get('value')) if isinstance(content, dict) else content
|
||||
if html:
|
||||
return _snippet(html)
|
||||
|
||||
# Try e-content first (HTML)
|
||||
content_prop = props.get('content', [])
|
||||
if content_prop:
|
||||
content_data = content_prop[0]
|
||||
if isinstance(content_data, dict):
|
||||
content = content_data.get('html', '') or content_data.get('value', '')
|
||||
else:
|
||||
content = content_data
|
||||
summary = mf2_utils.first(entry, 'summary')
|
||||
return _snippet(summary) if summary else None
|
||||
|
||||
if content:
|
||||
# Sanitize HTML
|
||||
sanitized = sanitize_content(content)
|
||||
# Truncate to 500 chars
|
||||
if len(sanitized) > 500:
|
||||
sanitized = sanitized[:497] + '...'
|
||||
return sanitized
|
||||
|
||||
# Try p-summary as fallback
|
||||
summary = props.get('summary', [])
|
||||
if summary:
|
||||
content = summary[0]
|
||||
if isinstance(content, dict):
|
||||
content = content.get('value', '')
|
||||
if content:
|
||||
sanitized = sanitize_content(content)
|
||||
if len(sanitized) > 500:
|
||||
sanitized = sanitized[:497] + '...'
|
||||
return sanitized
|
||||
def extract_source_details(entry: Optional[Dict]) -> Dict[str, Optional[str]]:
|
||||
"""
|
||||
The source post's own URL, uid and published time
|
||||
|
||||
return None
|
||||
Returns:
|
||||
source_post_url, source_uid (http(s) only) and source_published
|
||||
(RFC 3339 UTC, or None if missing or unparseable)
|
||||
"""
|
||||
if not entry:
|
||||
return {'source_post_url': None, 'source_uid': None, 'source_published': None}
|
||||
|
||||
url = mf2_utils.first(entry, 'url')
|
||||
uid = mf2_utils.first(entry, 'uid')
|
||||
published = mf2_utils.first(entry, 'published')
|
||||
try:
|
||||
published = iso_utc(published) or None
|
||||
except (ValueError, TypeError):
|
||||
published = None
|
||||
|
||||
return {
|
||||
'source_post_url': url if _is_http_url(url) else None,
|
||||
'source_uid': uid,
|
||||
'source_published': published,
|
||||
}
|
||||
|
||||
|
||||
def _snippet(html: str, limit: int = 500) -> str:
|
||||
"""Sanitized HTML of at most ~limit characters, with tags closed"""
|
||||
sanitized = sanitize_content(html)
|
||||
if len(sanitized) > limit:
|
||||
# Cutting can leave a tag or attribute open; sanitizing again closes it
|
||||
sanitized = sanitize_content(sanitized[:limit - 3] + '...')
|
||||
return sanitized
|
||||
|
||||
|
||||
def sanitize_content(html: str) -> str:
|
||||
@@ -337,17 +303,17 @@ def process_webmention(source_url: str, target_url: str, site_url: str) -> Dict[
|
||||
ValidationError: If validation fails with specific error
|
||||
"""
|
||||
# Fetch source content
|
||||
content, content_type = fetch_source(source_url, site_url)
|
||||
content, content_type, final_url = fetch_source(source_url, site_url)
|
||||
|
||||
# Verify target link presence
|
||||
if not verify_link_presence(content, target_url, content_type):
|
||||
if not verify_link_presence(content, target_url, content_type, final_url):
|
||||
raise ValidationError("Source does not link to target")
|
||||
|
||||
# Parse microformats (only for HTML)
|
||||
parsed_mf2 = None
|
||||
if 'html' in content_type.lower():
|
||||
try:
|
||||
parsed_mf2 = mf2py.parse(doc=content, url=source_url)
|
||||
parsed_mf2 = mf2py.parse(doc=content, url=final_url)
|
||||
except Exception as e:
|
||||
logger.debug(f"Microformat parsing failed for {source_url}: {e}")
|
||||
|
||||
@@ -360,14 +326,19 @@ def process_webmention(source_url: str, target_url: str, site_url: str) -> Dict[
|
||||
}
|
||||
content_snippet = None
|
||||
|
||||
source_details = extract_source_details(None)
|
||||
|
||||
if parsed_mf2:
|
||||
mention_type = detect_mention_type(parsed_mf2, target_url)
|
||||
author_info = extract_author(parsed_mf2, source_url)
|
||||
content_snippet = extract_content(parsed_mf2)
|
||||
entry = mf2_utils.responding_entry(parsed_mf2, target_url)
|
||||
mention_type = mf2_utils.response_type(entry, target_url) if entry else 'mention'
|
||||
author_info = extract_author(parsed_mf2, final_url, entry)
|
||||
content_snippet = extract_content(parsed_mf2, entry)
|
||||
source_details = extract_source_details(entry)
|
||||
|
||||
return {
|
||||
'valid': True,
|
||||
'mention_type': mention_type,
|
||||
**author_info,
|
||||
'content_snippet': content_snippet
|
||||
'content_snippet': content_snippet,
|
||||
**source_details,
|
||||
}
|
||||
@@ -17,7 +17,10 @@ Functions:
|
||||
import logging
|
||||
import math
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Any, Optional
|
||||
|
||||
from flask import current_app
|
||||
from starpunk.database import get_db
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -71,9 +74,13 @@ def get_webmentions_for_note(note_id: int) -> Dict[str, List[Dict[str, Any]]]:
|
||||
|
||||
for row in rows:
|
||||
wm = dict(row)
|
||||
# Parse received_at timestamp
|
||||
if isinstance(wm.get('received_at'), str):
|
||||
wm['received_at'] = datetime.fromisoformat(wm['received_at'].replace('Z', '+00:00'))
|
||||
# Parse timestamps (source_published may be missing or malformed)
|
||||
for key in ('received_at', 'source_published'):
|
||||
if isinstance(wm.get(key), str):
|
||||
try:
|
||||
wm[key] = datetime.fromisoformat(wm[key].replace('Z', '+00:00'))
|
||||
except ValueError:
|
||||
wm[key] = None
|
||||
|
||||
mention_type = wm.get('mention_type', 'mention')
|
||||
if mention_type == 'reply':
|
||||
@@ -233,10 +240,19 @@ def update_webmention_status(webmention_ids: List[int], status: str) -> int:
|
||||
WHERE id IN ({placeholders})
|
||||
"""
|
||||
|
||||
# Notes whose displayed mentions change (valid ones switching between
|
||||
# approved and not)
|
||||
changed_notes = _visibility_changes(
|
||||
db, webmention_ids,
|
||||
"validation_status = 'valid' AND (moderation_status = 'approved') != (? = 'approved')",
|
||||
[status],
|
||||
)
|
||||
|
||||
params = [status] + webmention_ids
|
||||
cursor = db.execute(query, params)
|
||||
db.commit()
|
||||
|
||||
_notify_responses_changed(changed_notes)
|
||||
return cursor.rowcount
|
||||
|
||||
|
||||
@@ -255,15 +271,42 @@ def delete_webmentions(webmention_ids: List[int]) -> int:
|
||||
|
||||
db = get_db()
|
||||
|
||||
changed_notes = _visibility_changes(
|
||||
db, webmention_ids, "validation_status = 'valid' AND moderation_status = 'approved'", [],
|
||||
)
|
||||
|
||||
placeholders = ','.join('?' * len(webmention_ids))
|
||||
query = f"DELETE FROM webmentions WHERE id IN ({placeholders})"
|
||||
|
||||
cursor = db.execute(query, webmention_ids)
|
||||
db.commit()
|
||||
|
||||
_notify_responses_changed(changed_notes)
|
||||
return cursor.rowcount
|
||||
|
||||
|
||||
def _visibility_changes(db, webmention_ids: List[int], condition: str, params: list) -> List[int]:
|
||||
"""Target note IDs of the given webmentions that match condition"""
|
||||
placeholders = ','.join('?' * len(webmention_ids))
|
||||
rows = db.execute(
|
||||
f"SELECT DISTINCT target_note_id FROM webmentions "
|
||||
f"WHERE id IN ({placeholders}) AND {condition}",
|
||||
list(webmention_ids) + params,
|
||||
).fetchall()
|
||||
return [row[0] for row in rows]
|
||||
|
||||
|
||||
def _notify_responses_changed(note_ids: List[int]) -> None:
|
||||
"""Re-send webmentions for notes whose shown responses changed (best effort)"""
|
||||
if not note_ids:
|
||||
return
|
||||
try:
|
||||
from starpunk.webmention_sender import note_responses_changed
|
||||
note_responses_changed(Path(current_app.config['DATABASE_PATH']), note_ids)
|
||||
except Exception as e:
|
||||
current_app.logger.warning(f"Failed to re-send webmentions after moderation: {e}")
|
||||
|
||||
|
||||
def get_reply_context(url: str) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Get cached reply context for a URL.
|
||||
|
||||
+111
-61
@@ -14,9 +14,8 @@ References:
|
||||
|
||||
import logging
|
||||
import sqlite3
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Optional, Set
|
||||
from typing import Optional
|
||||
from lxml import html as lxml_html
|
||||
from urllib.parse import urlparse
|
||||
|
||||
@@ -45,16 +44,16 @@ def extract_links_from_html(html_content: str, exclude_code_blocks: bool = True)
|
||||
return []
|
||||
|
||||
try:
|
||||
# Parse HTML
|
||||
doc = lxml_html.fromstring(html_content)
|
||||
# Parse HTML inside a wrapper, so a top-level <pre> can be removed too
|
||||
doc = lxml_html.fragment_fromstring(html_content, create_parent='div')
|
||||
|
||||
if exclude_code_blocks:
|
||||
# Remove <pre> and <code> elements before extracting links
|
||||
for element in doc.xpath('//pre | //code'):
|
||||
element.getparent().remove(element)
|
||||
for element in doc.xpath('.//pre | .//code'):
|
||||
element.drop_tree()
|
||||
|
||||
# Extract all <a href> attributes in document order
|
||||
links = doc.xpath('//a/@href')
|
||||
links = doc.xpath('.//a/@href')
|
||||
|
||||
# Filter to valid absolute URLs (http/https)
|
||||
absolute_links = []
|
||||
@@ -70,6 +69,97 @@ def extract_links_from_html(html_content: str, exclude_code_blocks: bool = True)
|
||||
return []
|
||||
|
||||
|
||||
def _dedupe(urls: list) -> list:
|
||||
"""Drop empty and repeated URLs, keeping the first occurrence"""
|
||||
seen = set()
|
||||
result = []
|
||||
for url in urls:
|
||||
if url and url not in seen:
|
||||
seen.add(url)
|
||||
result.append(url)
|
||||
return result
|
||||
|
||||
|
||||
def previous_targets(db_path: Path, note_id: int) -> list[str]:
|
||||
"""Targets this note has been queued to send webmentions to before"""
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
rows = conn.execute(
|
||||
"SELECT target_url FROM webmention_queue WHERE source_note_id = ? ORDER BY id",
|
||||
(note_id,),
|
||||
).fetchall()
|
||||
finally:
|
||||
conn.close()
|
||||
return [row[0] for row in rows]
|
||||
|
||||
|
||||
def requeue_previous_webmentions(db_path: Path, note_id: int, detach: bool = False) -> int:
|
||||
"""
|
||||
Re-send every webmention this note has sent (spec 3.1.5 deleted posts)
|
||||
|
||||
Args:
|
||||
db_path: Path to SQLite database
|
||||
note_id: Note being deleted or restored
|
||||
detach: Clear source_note_id so the rows survive a hard delete
|
||||
(the foreign key cascades)
|
||||
|
||||
Returns:
|
||||
Number of webmentions queued
|
||||
"""
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
cursor = conn.execute(f"""
|
||||
UPDATE webmention_queue
|
||||
SET status = 'pending',
|
||||
retry_count = 0,
|
||||
next_retry_at = NULL,
|
||||
last_error = NULL,
|
||||
endpoint_url = NULL,
|
||||
created_at = CURRENT_TIMESTAMP
|
||||
{', source_note_id = NULL' if detach else ''}
|
||||
WHERE source_note_id = ?
|
||||
""", (note_id,))
|
||||
conn.commit()
|
||||
queued = cursor.rowcount
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
if queued:
|
||||
logger.info(f"Re-queued {queued} webmentions for note {note_id}")
|
||||
return queued
|
||||
|
||||
|
||||
def note_responses_changed(db_path: Path, note_ids) -> int:
|
||||
"""
|
||||
Re-send a note's webmentions because the responses it shows changed
|
||||
|
||||
Spec 3.1.4: if the source post displays responses (e.g. comments), a
|
||||
change to them is an update of the post. Only published, undeleted notes.
|
||||
|
||||
Args:
|
||||
db_path: Path to SQLite database
|
||||
note_ids: IDs of notes whose displayed responses changed
|
||||
|
||||
Returns:
|
||||
Number of webmentions queued
|
||||
"""
|
||||
ids = sorted({i for i in note_ids if i is not None})
|
||||
if not ids:
|
||||
return 0
|
||||
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
placeholders = ','.join('?' * len(ids))
|
||||
live = [row[0] for row in conn.execute(
|
||||
f"SELECT id FROM notes WHERE id IN ({placeholders}) "
|
||||
"AND published = 1 AND deleted_at IS NULL", ids,
|
||||
)]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
return sum(requeue_previous_webmentions(db_path, note_id) for note_id in live)
|
||||
|
||||
|
||||
def queue_webmentions_for_note(
|
||||
db_path: Path,
|
||||
note_id: int,
|
||||
@@ -83,10 +173,9 @@ def queue_webmentions_for_note(
|
||||
|
||||
Per architect's answers (Phase 6):
|
||||
- Called from notes.py AFTER successful database commit
|
||||
- For updates: diff old vs new URLs, queue for new/existing/removed
|
||||
- For creates: queue for all URLs in content
|
||||
- Max 10 webmentions per operation
|
||||
- Priority: in_reply_to URL first, then document order
|
||||
- For creates: reply target first, then links in document order (max 10)
|
||||
- For updates: the same, plus every previously sent target and links
|
||||
removed by the edit (spec 3.1.4)
|
||||
- Webmention sending is best-effort; failures do not affect note creation
|
||||
|
||||
Args:
|
||||
@@ -100,58 +189,19 @@ def queue_webmentions_for_note(
|
||||
Returns:
|
||||
Number of webmentions queued
|
||||
"""
|
||||
# Phase 6: Implement URL diffing for updates
|
||||
# Targets for the current content: reply target first, then links in
|
||||
# document order, at most 10
|
||||
links = extract_links_from_html(content_html) if content_html else []
|
||||
current = _dedupe(([in_reply_to] if in_reply_to else []) + links)[:10]
|
||||
|
||||
if old_content_html is not None:
|
||||
# This is an update - compute URL diff
|
||||
old_links = set(extract_links_from_html(old_content_html))
|
||||
|
||||
# Handle unpublishing case (content_html is None)
|
||||
if content_html is None:
|
||||
# Unpublishing - queue for all old URLs (they're all removed)
|
||||
new_links = set()
|
||||
else:
|
||||
new_links = set(extract_links_from_html(content_html))
|
||||
|
||||
# Compute sets for URL diffing
|
||||
new_urls = new_links - old_links # In new, not in old
|
||||
existing_urls = new_links & old_links # In both (re-validate)
|
||||
removed_urls = old_links - new_links # In old, not in new
|
||||
|
||||
# Build target list: new + existing + removed
|
||||
# (All need webmentions per W3C spec)
|
||||
all_update_urls = list(new_urls) + list(existing_urls) + list(removed_urls)
|
||||
|
||||
# Add in_reply_to if changed
|
||||
target_urls = []
|
||||
if in_reply_to and in_reply_to not in all_update_urls:
|
||||
target_urls.append(in_reply_to)
|
||||
|
||||
# Add the diffed URLs
|
||||
target_urls.extend(all_update_urls)
|
||||
|
||||
# Limit to 10
|
||||
target_urls = target_urls[:10]
|
||||
|
||||
# Update (spec 3.1.4): also re-send to every target notified before,
|
||||
# and to links the edit removed, so they can update or remove the
|
||||
# mention. These are not capped.
|
||||
removed = [u for u in extract_links_from_html(old_content_html) if u not in links]
|
||||
target_urls = _dedupe(current + previous_targets(db_path, note_id) + removed)
|
||||
else:
|
||||
# This is a create (no old content) - use original logic
|
||||
links = extract_links_from_html(content_html) if content_html else []
|
||||
|
||||
# Build priority-ordered list of target URLs
|
||||
target_urls = []
|
||||
|
||||
# 1. in_reply_to gets highest priority
|
||||
if in_reply_to:
|
||||
target_urls.append(in_reply_to)
|
||||
|
||||
# 2. Add extracted links in document order (deduplicated)
|
||||
seen = {in_reply_to} if in_reply_to else set()
|
||||
for link in links:
|
||||
if link not in seen:
|
||||
target_urls.append(link)
|
||||
seen.add(link)
|
||||
|
||||
# 3. Limit to 10 webmentions per operation
|
||||
target_urls = target_urls[:10]
|
||||
target_urls = current
|
||||
|
||||
if not target_urls:
|
||||
logger.debug(f"No links to send webmentions for note {note_id}")
|
||||
|
||||
+205
-52
@@ -1,113 +1,266 @@
|
||||
"""
|
||||
SSRF protection utilities for webmention processing
|
||||
SSRF protection for webmention fetches
|
||||
|
||||
Per ADR-063 Decision 9 and background-worker.md:
|
||||
- Block private IP address ranges
|
||||
- Resolve hostname before making requests
|
||||
Per ADR-063 Decision 9 and Webmention sections 4.2, 4.3 and 4.5:
|
||||
- Every address a hostname resolves to must be public
|
||||
- Redirects are followed by hand so each hop is checked
|
||||
- Each request connects to the address that was checked (no second DNS
|
||||
lookup that a rebinding DNS server could answer differently)
|
||||
- Redirect count, response size and total time are limited
|
||||
- DEV_MODE exception for localhost testing
|
||||
- Applied to all outgoing HTTP requests
|
||||
|
||||
References:
|
||||
- ADR-063: Webmention Implementation
|
||||
- Background Worker Design: /docs/design/v1.6.0/background-worker.md
|
||||
- W3C Webmention: https://www.w3.org/TR/webmention/#security-considerations
|
||||
"""
|
||||
|
||||
import ipaddress
|
||||
import socket
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
import httpx
|
||||
from flask import current_app
|
||||
|
||||
MAX_REDIRECTS = 10
|
||||
MAX_BYTES = 1024 * 1024
|
||||
TIMEOUT = 10
|
||||
|
||||
# Tests can set an httpx transport here (e.g. httpx.MockTransport)
|
||||
TRANSPORT = None
|
||||
|
||||
CGNAT = ipaddress.ip_network('100.64.0.0/10')
|
||||
THIS_NETWORK = ipaddress.ip_network('0.0.0.0/8')
|
||||
|
||||
|
||||
class SSRFBlockedError(Exception):
|
||||
"""Raised when SSRF protection blocks a request"""
|
||||
pass
|
||||
|
||||
|
||||
# Private IP ranges per RFC 1918 and link-local ranges
|
||||
BLOCKED_NETWORKS = [
|
||||
ipaddress.ip_network('10.0.0.0/8'), # RFC 1918 private
|
||||
ipaddress.ip_network('172.16.0.0/12'), # RFC 1918 private
|
||||
ipaddress.ip_network('192.168.0.0/16'), # RFC 1918 private
|
||||
ipaddress.ip_network('127.0.0.0/8'), # Localhost
|
||||
ipaddress.ip_network('169.254.0.0/16'), # Link-local
|
||||
ipaddress.ip_network('::1/128'), # IPv6 localhost
|
||||
ipaddress.ip_network('fc00::/7'), # IPv6 unique local
|
||||
ipaddress.ip_network('fe80::/10'), # IPv6 link-local
|
||||
]
|
||||
class FetchTooLargeError(Exception):
|
||||
"""Raised when a response is larger than the limit"""
|
||||
pass
|
||||
|
||||
|
||||
def user_agent(site_url: str) -> str:
|
||||
"""User-Agent for webmention requests (spec 3.1.2 suggests including 'Webmention')"""
|
||||
from starpunk import __version__
|
||||
return f"StarPunk/{__version__} Webmention (+{site_url})"
|
||||
|
||||
|
||||
def is_private_ip(ip: str) -> bool:
|
||||
"""
|
||||
Check if IP address is in a private/blocked range.
|
||||
Check if an IP address must not be fetched.
|
||||
|
||||
Blocks private, loopback, link-local, multicast, reserved, unspecified,
|
||||
carrier-grade NAT and IPv4-mapped forms of these.
|
||||
|
||||
Args:
|
||||
ip: IP address string (IPv4 or IPv6)
|
||||
|
||||
Returns:
|
||||
True if IP is private/blocked, False otherwise
|
||||
True if IP is blocked, False otherwise
|
||||
"""
|
||||
try:
|
||||
addr = ipaddress.ip_address(ip)
|
||||
return any(addr in network for network in BLOCKED_NETWORKS)
|
||||
addr = ipaddress.ip_address(ip.split('%')[0])
|
||||
except ValueError:
|
||||
# Invalid IP format, treat as blocked
|
||||
return True
|
||||
|
||||
if isinstance(addr, ipaddress.IPv6Address) and addr.ipv4_mapped:
|
||||
addr = addr.ipv4_mapped
|
||||
|
||||
if isinstance(addr, ipaddress.IPv4Address) and (addr in CGNAT or addr in THIS_NETWORK):
|
||||
return True
|
||||
|
||||
return (
|
||||
addr.is_private
|
||||
or addr.is_loopback
|
||||
or addr.is_link_local
|
||||
or addr.is_multicast
|
||||
or addr.is_reserved
|
||||
or addr.is_unspecified
|
||||
)
|
||||
|
||||
|
||||
def safe_resolve(hostname: str) -> str:
|
||||
"""
|
||||
Resolve hostname and validate IP is not private.
|
||||
Resolve hostname and check every address is public.
|
||||
|
||||
Per background-worker.md: DEV_MODE exception for localhost testing.
|
||||
|
||||
Args:
|
||||
hostname: Hostname to resolve
|
||||
|
||||
Returns:
|
||||
Resolved IP address if safe
|
||||
Address to connect to (IPv4 preferred)
|
||||
|
||||
Raises:
|
||||
SSRFBlockedError: If hostname resolves to private IP
|
||||
SSRFBlockedError: If any address is blocked
|
||||
socket.gaierror: If hostname cannot be resolved
|
||||
"""
|
||||
# DEV_MODE exception for localhost
|
||||
if current_app.config.get('DEV_MODE') and hostname in ('localhost', '127.0.0.1', '::1'):
|
||||
return '127.0.0.1'
|
||||
|
||||
# Resolve hostname
|
||||
ip = socket.gethostbyname(hostname)
|
||||
addresses = sorted({info[4][0] for info in socket.getaddrinfo(hostname, None)},
|
||||
key=lambda ip: (':' in ip, ip))
|
||||
for ip in addresses:
|
||||
if is_private_ip(ip):
|
||||
raise SSRFBlockedError(f"Blocked request to private IP: {ip}")
|
||||
|
||||
# Check if resolved IP is private
|
||||
if is_private_ip(ip):
|
||||
raise SSRFBlockedError(f"Blocked request to private IP: {ip}")
|
||||
|
||||
return ip
|
||||
return addresses[0]
|
||||
|
||||
|
||||
def validate_url_for_fetch(url: str) -> None:
|
||||
def validate_url_for_fetch(url: str) -> str:
|
||||
"""
|
||||
Validate URL is safe to fetch (not private IP).
|
||||
Validate URL is safe to fetch (http(s), public addresses only).
|
||||
|
||||
Args:
|
||||
url: URL to validate
|
||||
Returns:
|
||||
The checked IP address to connect to
|
||||
|
||||
Raises:
|
||||
SSRFBlockedError: If URL points to private IP
|
||||
ValueError: If URL is malformed
|
||||
SSRFBlockedError: If URL points to a blocked address
|
||||
ValueError: If URL is malformed or its host does not resolve
|
||||
"""
|
||||
from urllib.parse import urlparse
|
||||
|
||||
parsed = urlparse(url)
|
||||
|
||||
if not parsed.scheme in ('http', 'https'):
|
||||
if parsed.scheme not in ('http', 'https'):
|
||||
raise ValueError(f"Unsupported URL scheme: {parsed.scheme}")
|
||||
|
||||
if not parsed.netloc:
|
||||
if not parsed.hostname:
|
||||
raise ValueError("URL must include a hostname")
|
||||
|
||||
# Extract hostname (remove port if present)
|
||||
hostname = parsed.hostname
|
||||
if not hostname:
|
||||
raise ValueError("Could not extract hostname from URL")
|
||||
try:
|
||||
return safe_resolve(parsed.hostname)
|
||||
except socket.gaierror as e:
|
||||
raise ValueError(f"Could not resolve {parsed.hostname}: {e}")
|
||||
|
||||
# Validate hostname resolves to public IP
|
||||
safe_resolve(hostname)
|
||||
|
||||
def _pinned_request(client: httpx.Client, method: str, url: str, headers: dict,
|
||||
**kwargs) -> httpx.Request:
|
||||
"""
|
||||
Build a request that connects to the checked address for url's host.
|
||||
|
||||
The URL's host is replaced by the IP that passed the SSRF check; the
|
||||
real host goes in the Host header and, for https, in SNI, so virtual
|
||||
hosting and certificate verification still use the hostname.
|
||||
"""
|
||||
ip = validate_url_for_fetch(url)
|
||||
parsed = urlparse(url)
|
||||
ip_host = f"[{ip}]" if ':' in ip else ip
|
||||
port = f":{parsed.port}" if parsed.port else ''
|
||||
pinned_url = parsed._replace(netloc=f"{ip_host}{port}").geturl()
|
||||
host_header = f"{parsed.hostname}{port}"
|
||||
|
||||
extensions = {'sni_hostname': parsed.hostname} if parsed.scheme == 'https' else {}
|
||||
return client.build_request(
|
||||
method, pinned_url, headers={**headers, 'Host': host_header},
|
||||
extensions=extensions, **kwargs,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class FetchResult:
|
||||
"""A fetched response, after redirects"""
|
||||
url: str
|
||||
status_code: int
|
||||
headers: httpx.Headers
|
||||
content: bytes
|
||||
encoding: str
|
||||
|
||||
@property
|
||||
def text(self) -> str:
|
||||
try:
|
||||
return self.content.decode(self.encoding, errors='replace')
|
||||
except LookupError:
|
||||
return self.content.decode('utf-8', errors='replace')
|
||||
|
||||
|
||||
def safe_post(url: str, data: Optional[dict] = None, headers: Optional[dict] = None,
|
||||
timeout: float = TIMEOUT, content: Optional[bytes] = None,
|
||||
max_bytes: int = 64 * 1024) -> FetchResult:
|
||||
"""
|
||||
POST form data (or a raw body) to a checked, pinned address.
|
||||
|
||||
Redirects are not followed. Only the first max_bytes of the response are
|
||||
read (callers need the status and maybe an error message).
|
||||
|
||||
Raises:
|
||||
SSRFBlockedError, ValueError: URL is not allowed
|
||||
httpx.HTTPError: Network errors
|
||||
"""
|
||||
kwargs = {'content': content} if content is not None else {'data': data}
|
||||
with httpx.Client(follow_redirects=False, transport=TRANSPORT, timeout=timeout) as client:
|
||||
request = _pinned_request(client, 'POST', url, headers or {}, **kwargs)
|
||||
response = client.send(request, stream=True)
|
||||
try:
|
||||
body = b''
|
||||
for chunk in response.iter_bytes(chunk_size=8192):
|
||||
body += chunk
|
||||
if len(body) >= max_bytes:
|
||||
body = body[:max_bytes]
|
||||
break
|
||||
return FetchResult(url=url, status_code=response.status_code, headers=response.headers,
|
||||
content=body, encoding=response.charset_encoding or 'utf-8')
|
||||
finally:
|
||||
response.close()
|
||||
|
||||
|
||||
def safe_get(url: str, headers: dict, max_bytes: int = MAX_BYTES,
|
||||
max_redirects: int = MAX_REDIRECTS, timeout: float = TIMEOUT) -> FetchResult:
|
||||
"""
|
||||
GET a URL, following redirects by hand and checking each hop.
|
||||
|
||||
Args:
|
||||
url: URL to fetch
|
||||
headers: Request headers
|
||||
max_bytes: Response body limit
|
||||
max_redirects: Redirect limit
|
||||
timeout: Total time limit in seconds
|
||||
|
||||
Returns:
|
||||
FetchResult for the final response (any status code)
|
||||
|
||||
Raises:
|
||||
SSRFBlockedError, ValueError: URL or a redirect target is not allowed
|
||||
FetchTooLargeError: Body exceeds max_bytes
|
||||
httpx.HTTPError: Network errors, too many redirects, time limit
|
||||
"""
|
||||
deadline = time.monotonic() + timeout
|
||||
|
||||
with httpx.Client(follow_redirects=False, transport=TRANSPORT) as client:
|
||||
for _ in range(max_redirects + 1):
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
raise httpx.TimeoutException("Fetch exceeded time limit")
|
||||
|
||||
request = _pinned_request(client, 'GET', url, headers, timeout=remaining)
|
||||
response = client.send(request, stream=True)
|
||||
try:
|
||||
location = response.headers.get('Location')
|
||||
if response.is_redirect and location:
|
||||
url = urljoin(url, location)
|
||||
continue
|
||||
|
||||
content_length = response.headers.get('Content-Length', '')
|
||||
if content_length.isdigit() and int(content_length) > max_bytes:
|
||||
raise FetchTooLargeError(f"Response exceeds size limit ({max_bytes} bytes)")
|
||||
|
||||
content = b''
|
||||
for chunk in response.iter_bytes(chunk_size=8192):
|
||||
content += chunk
|
||||
if len(content) > max_bytes:
|
||||
raise FetchTooLargeError(f"Response exceeds size limit ({max_bytes} bytes)")
|
||||
if time.monotonic() > deadline:
|
||||
raise httpx.TimeoutException("Fetch exceeded time limit")
|
||||
|
||||
return FetchResult(
|
||||
url=url,
|
||||
status_code=response.status_code,
|
||||
headers=response.headers,
|
||||
content=content,
|
||||
encoding=response.charset_encoding or 'utf-8',
|
||||
)
|
||||
finally:
|
||||
response.close()
|
||||
|
||||
raise httpx.TooManyRedirects(f"More than {max_redirects} redirects")
|
||||
+248
-151
@@ -12,30 +12,34 @@ References:
|
||||
- ADR-063: Webmention Implementation
|
||||
"""
|
||||
|
||||
import fcntl
|
||||
import logging
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
import sqlite3
|
||||
import httpx
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, Optional, TypedDict
|
||||
from urllib.parse import urlparse
|
||||
from typing import Dict, Any, Optional
|
||||
from urllib.parse import urljoin
|
||||
from lxml import html as lxml_html
|
||||
|
||||
from starpunk.webmention_processor import process_webmention, ValidationError
|
||||
from starpunk.webmention_ssrf import validate_url_for_fetch, SSRFBlockedError
|
||||
from flask import has_app_context
|
||||
from starpunk.webmention_processor import process_webmention, TransientError, ValidationError
|
||||
from starpunk.webmention_sender import note_responses_changed
|
||||
from starpunk.mf2_utils import normalize_url
|
||||
from starpunk.webmention_ssrf import (
|
||||
FetchTooLargeError, SSRFBlockedError, safe_get, safe_post, user_agent,
|
||||
)
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Link header (RFC 8288): <uri> then ;-separated params; quoted values may
|
||||
# contain , and ;
|
||||
LINK_RE = re.compile(r'<([^>]*)>((?:\s*;\s*[^;,"=\s]+\s*(?:=\s*(?:"[^"]*"|[^;,]*))?)*)')
|
||||
LINK_PARAM_RE = re.compile(r';\s*([^;,"=\s]+)\s*(?:=\s*(?:"([^"]*)"|([^;,]*)))?')
|
||||
|
||||
# Discovery cache entry type
|
||||
class DiscoveryCacheEntry(TypedDict):
|
||||
"""Cache entry for endpoint discovery results"""
|
||||
endpoint: Optional[str] # None = negative cache (no endpoint found)
|
||||
expires_at: datetime
|
||||
|
||||
|
||||
class WebmentionWorker(threading.Thread):
|
||||
@@ -47,6 +51,7 @@ class WebmentionWorker(threading.Thread):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
app,
|
||||
db_path: Path,
|
||||
site_url: str,
|
||||
poll_interval: int = 10,
|
||||
@@ -58,6 +63,7 @@ class WebmentionWorker(threading.Thread):
|
||||
Initialize webmention worker thread
|
||||
|
||||
Args:
|
||||
app: Flask app (the SSRF checks read its config)
|
||||
db_path: Path to SQLite database
|
||||
site_url: Site URL for User-Agent header
|
||||
poll_interval: Seconds between queue checks (default: 10)
|
||||
@@ -66,6 +72,7 @@ class WebmentionWorker(threading.Thread):
|
||||
auto_approve: Auto-approve valid webmentions (default: True)
|
||||
"""
|
||||
super().__init__(daemon=True, name="webmention-worker")
|
||||
self.app = app
|
||||
self.db_path = db_path
|
||||
self.site_url = site_url
|
||||
self.poll_interval = poll_interval
|
||||
@@ -74,10 +81,7 @@ class WebmentionWorker(threading.Thread):
|
||||
self.auto_approve = auto_approve
|
||||
self._stop_event = threading.Event()
|
||||
self._last_processed = None
|
||||
|
||||
# Phase 3: Discovery cache (worker instance variable)
|
||||
# Per architect's answer to Q4
|
||||
self._discovery_cache: Dict[str, DiscoveryCacheEntry] = {}
|
||||
self._lock_file = None
|
||||
|
||||
def run(self):
|
||||
"""
|
||||
@@ -92,19 +96,23 @@ class WebmentionWorker(threading.Thread):
|
||||
try:
|
||||
logger.info("Webmention worker started")
|
||||
|
||||
# Cleanup orphaned items from previous crashes
|
||||
self._cleanup_orphaned_items()
|
||||
|
||||
# Main processing loop
|
||||
while not self._stop_event.is_set():
|
||||
# Every gunicorn process starts a worker; only the one holding
|
||||
# the lock processes queues, so nothing is sent twice
|
||||
if not self._acquire_lock():
|
||||
self._stop_event.wait(self.poll_interval)
|
||||
continue
|
||||
|
||||
try:
|
||||
processed = 0
|
||||
|
||||
# Process incoming webmentions
|
||||
processed += self._process_pending_validations()
|
||||
with self.app.app_context():
|
||||
# Process incoming webmentions
|
||||
processed += self._process_pending_validations()
|
||||
|
||||
# Phase 3: Process outgoing webmentions
|
||||
processed += self._process_pending_sends()
|
||||
# Phase 3: Process outgoing webmentions
|
||||
processed += self._process_pending_sends()
|
||||
|
||||
# Update last processed time
|
||||
if processed > 0:
|
||||
@@ -121,8 +129,40 @@ class WebmentionWorker(threading.Thread):
|
||||
except Exception as e:
|
||||
logger.error(f"Webmention worker thread failed: {e}", exc_info=True)
|
||||
finally:
|
||||
self._release_lock()
|
||||
logger.info("Webmention worker stopped")
|
||||
|
||||
def _acquire_lock(self) -> bool:
|
||||
"""
|
||||
Take the cross-process worker lock (non-blocking)
|
||||
|
||||
On first acquisition, also reset items left mid-send by a previous
|
||||
holder. The OS releases the lock if this process dies.
|
||||
|
||||
Returns:
|
||||
True if this worker holds the lock
|
||||
"""
|
||||
if self._lock_file:
|
||||
return True
|
||||
|
||||
lock_file = open(Path(self.db_path).parent / "webmention-worker.lock", "w")
|
||||
try:
|
||||
fcntl.flock(lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except OSError:
|
||||
lock_file.close()
|
||||
return False
|
||||
|
||||
self._lock_file = lock_file
|
||||
logger.info("Webmention worker holds the queue lock")
|
||||
self._cleanup_orphaned_items()
|
||||
return True
|
||||
|
||||
def _release_lock(self):
|
||||
"""Release the cross-process worker lock if held"""
|
||||
if self._lock_file:
|
||||
self._lock_file.close()
|
||||
self._lock_file = None
|
||||
|
||||
def _cleanup_orphaned_items(self):
|
||||
"""
|
||||
Clean up orphaned 'sending' status items on startup
|
||||
@@ -206,6 +246,69 @@ class WebmentionWorker(threading.Thread):
|
||||
source_url: str,
|
||||
target_url: str,
|
||||
target_note_id: Optional[int]
|
||||
):
|
||||
"""
|
||||
Validate a webmention, then re-send the target note's webmentions if
|
||||
what the note shows changed (spec 3.1.4: displayed responses are an
|
||||
update of the post). Unchanged re-validations send nothing, so two
|
||||
sites doing this don't ping-pong.
|
||||
"""
|
||||
before = self._display_state(webmention_id)
|
||||
try:
|
||||
self._validate_and_store(webmention_id, source_url, target_url, target_note_id)
|
||||
finally:
|
||||
after = self._display_state(webmention_id)
|
||||
if before != after and target_note_id is not None:
|
||||
try:
|
||||
note_responses_changed(self.db_path, [target_note_id])
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to re-send webmentions for note {target_note_id}: {e}")
|
||||
|
||||
def _is_own_syndicated_copy(self, note_id: Optional[int], source_url: str,
|
||||
result: Dict[str, Any]) -> bool:
|
||||
"""True if the source is one of the note's own u-syndication copies"""
|
||||
if note_id is None:
|
||||
return False
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
try:
|
||||
copies = {normalize_url(row[0]) for row in conn.execute(
|
||||
"SELECT remote_url FROM syndications WHERE note_id = ? AND remote_url IS NOT NULL",
|
||||
(note_id,),
|
||||
)}
|
||||
finally:
|
||||
conn.close()
|
||||
source = {normalize_url(u) for u in (
|
||||
source_url, result.get('source_post_url'), result.get('source_uid')) if u}
|
||||
return bool(copies & source)
|
||||
|
||||
def _display_state(self, webmention_id: int) -> Optional[tuple]:
|
||||
"""
|
||||
What the note page shows for this webmention, or None if hidden
|
||||
|
||||
A row reset to 'pending' by a re-send still counts as shown if it was
|
||||
valid before (validated with no error), as it was until now.
|
||||
"""
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
try:
|
||||
row = conn.execute("""
|
||||
SELECT mention_type, author_name, author_url, author_photo, content_snippet
|
||||
FROM webmentions
|
||||
WHERE id = ?
|
||||
AND moderation_status = 'approved'
|
||||
AND (validation_status = 'valid'
|
||||
OR (validation_status = 'pending' AND validated_at IS NOT NULL
|
||||
AND validation_error IS NULL))
|
||||
""", (webmention_id,)).fetchone()
|
||||
finally:
|
||||
conn.close()
|
||||
return tuple(row) if row else None
|
||||
|
||||
def _validate_and_store(
|
||||
self,
|
||||
webmention_id: int,
|
||||
source_url: str,
|
||||
target_url: str,
|
||||
target_note_id: Optional[int]
|
||||
):
|
||||
"""
|
||||
Validate a single webmention
|
||||
@@ -220,31 +323,49 @@ class WebmentionWorker(threading.Thread):
|
||||
# Process webmention (fetch, validate, extract)
|
||||
result = process_webmention(source_url, target_url, self.site_url)
|
||||
|
||||
# A copy of this very note on a silo (e.g. backfed by Bridgy) is
|
||||
# not a response to it
|
||||
if self._is_own_syndicated_copy(target_note_id, source_url, result):
|
||||
raise ValidationError("Source is this note's own syndicated copy")
|
||||
|
||||
# Update database with successful validation
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
cursor = conn.cursor()
|
||||
|
||||
moderation_status = 'approved' if self.auto_approve else 'pending'
|
||||
|
||||
# An update to an existing mention (spec 3.2.4) keeps the admin's
|
||||
# moderation decision; only a first-time valid mention gets the
|
||||
# default. Previously valid = validated with no error.
|
||||
cursor.execute("""
|
||||
UPDATE webmentions
|
||||
SET validation_status = 'valid',
|
||||
SET moderation_status = CASE
|
||||
WHEN moderation_status IN ('hidden', 'deleted') THEN moderation_status
|
||||
WHEN validated_at IS NOT NULL AND validation_error IS NULL
|
||||
THEN moderation_status
|
||||
ELSE ?
|
||||
END,
|
||||
validation_status = 'valid',
|
||||
validation_error = NULL,
|
||||
validated_at = CURRENT_TIMESTAMP,
|
||||
mention_type = ?,
|
||||
author_name = ?,
|
||||
author_url = ?,
|
||||
author_photo = ?,
|
||||
content_snippet = ?,
|
||||
moderation_status = ?,
|
||||
source_post_url = ?,
|
||||
source_published = ?,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?
|
||||
""", (
|
||||
moderation_status,
|
||||
result['mention_type'],
|
||||
result['author_name'],
|
||||
result['author_url'],
|
||||
result['author_photo'],
|
||||
result['content_snippet'],
|
||||
moderation_status,
|
||||
result.get('source_post_url'),
|
||||
result.get('source_published'),
|
||||
webmention_id
|
||||
))
|
||||
|
||||
@@ -253,11 +374,17 @@ class WebmentionWorker(threading.Thread):
|
||||
|
||||
logger.info(
|
||||
f"Validated webmention: {source_url} -> {target_url} "
|
||||
f"(type={result['mention_type']}, status={moderation_status})"
|
||||
f"(type={result['mention_type']})"
|
||||
)
|
||||
|
||||
except TransientError as e:
|
||||
# Source unreachable right now: keep a previously valid mention
|
||||
self._mark_validation_failed(webmention_id, source_url, target_url, str(e),
|
||||
transient=True)
|
||||
|
||||
except ValidationError as e:
|
||||
# Validation failed - update with error
|
||||
# Validation failed (410/4xx, no link): the mention is removed
|
||||
# from display (spec 3.2.4)
|
||||
self._mark_validation_failed(webmention_id, source_url, target_url, str(e))
|
||||
|
||||
except Exception as e:
|
||||
@@ -275,11 +402,15 @@ class WebmentionWorker(threading.Thread):
|
||||
webmention_id: int,
|
||||
source_url: str,
|
||||
target_url: str,
|
||||
error: str
|
||||
error: str,
|
||||
transient: bool = False
|
||||
):
|
||||
"""
|
||||
Mark webmention validation as failed/invalid
|
||||
|
||||
A transient failure of a mention that was valid before restores it to
|
||||
valid with its previous data, rather than hiding it.
|
||||
|
||||
Args:
|
||||
webmention_id: Database ID
|
||||
source_url: Source URL
|
||||
@@ -289,7 +420,9 @@ class WebmentionWorker(threading.Thread):
|
||||
# Determine if error or invalid
|
||||
# Per background-worker.md: HTTP 4xx and "no link" are invalid
|
||||
# All other errors are 'error' status
|
||||
if any(x in error.lower() for x in ['does not link', 'returned 4']):
|
||||
if transient:
|
||||
status = 'error'
|
||||
elif any(x in error.lower() for x in ['does not link', 'returned 4', 'own syndicated copy']):
|
||||
status = 'invalid'
|
||||
else:
|
||||
status = 'error'
|
||||
@@ -298,6 +431,22 @@ class WebmentionWorker(threading.Thread):
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
cursor = conn.cursor()
|
||||
|
||||
if transient:
|
||||
cursor.execute("""
|
||||
UPDATE webmentions
|
||||
SET validation_status = 'valid',
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ? AND validated_at IS NOT NULL AND validation_error IS NULL
|
||||
""", (webmention_id,))
|
||||
if cursor.rowcount:
|
||||
conn.commit()
|
||||
conn.close()
|
||||
logger.warning(
|
||||
f"Could not re-check webmention, keeping it: "
|
||||
f"{source_url} -> {target_url}: {error}"
|
||||
)
|
||||
return
|
||||
|
||||
cursor.execute("""
|
||||
UPDATE webmentions
|
||||
SET validation_status = ?,
|
||||
@@ -418,37 +567,34 @@ class WebmentionWorker(threading.Thread):
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
# Send webmention POST
|
||||
# Send webmention POST (spec 3.1.3)
|
||||
try:
|
||||
# SSRF protection for endpoint URL
|
||||
validate_url_for_fetch(endpoint)
|
||||
|
||||
response = httpx.post(
|
||||
# SSRF-checked and pinned to the checked address (spec 4.3).
|
||||
# Redirects are not followed: a 301/302 would turn the POST
|
||||
# into a GET without the body
|
||||
response = safe_post(
|
||||
endpoint,
|
||||
data={
|
||||
'source': source_url,
|
||||
'target': target_url
|
||||
},
|
||||
timeout=10,
|
||||
headers={'User-Agent': f'StarPunk/1.6.0 (+{self.site_url})'},
|
||||
follow_redirects=True
|
||||
headers={'User-Agent': user_agent(self.site_url)},
|
||||
)
|
||||
|
||||
# Check response
|
||||
# Any 2xx is success (spec 3.1.3)
|
||||
if 200 <= response.status_code < 300:
|
||||
# Success
|
||||
self._mark_send_success(queue_id, source_url, target_url)
|
||||
|
||||
elif 400 <= response.status_code < 500:
|
||||
# Permanent failure (4xx)
|
||||
error = f"HTTP {response.status_code}: {response.text[:200]}"
|
||||
self._mark_send_failed(queue_id, source_url, target_url, error)
|
||||
|
||||
else:
|
||||
# Temporary failure (5xx) - retry
|
||||
elif response.status_code == 429 or response.status_code >= 500:
|
||||
# Rate limited or server error - retry
|
||||
error = f"HTTP {response.status_code}: {response.text[:200]}"
|
||||
self._mark_send_retry(queue_id, source_url, target_url, retry_count, error)
|
||||
|
||||
else:
|
||||
# Permanent failure (3xx, 4xx)
|
||||
error = f"HTTP {response.status_code}: {response.text[:200]}"
|
||||
self._mark_send_failed(queue_id, source_url, target_url, error)
|
||||
|
||||
except (httpx.TimeoutException, httpx.HTTPError) as e:
|
||||
# Connection error - retry
|
||||
self._mark_send_retry(queue_id, source_url, target_url, retry_count, str(e))
|
||||
@@ -464,100 +610,81 @@ class WebmentionWorker(threading.Thread):
|
||||
|
||||
def _discover_endpoint(self, target_url: str) -> Optional[str]:
|
||||
"""
|
||||
Discover webmention endpoint for target URL with caching
|
||||
Discover the webmention endpoint for a target URL (spec 3.1.2)
|
||||
|
||||
Per architect's answers to Q3, Q4, Q10:
|
||||
- GET only (no HEAD)
|
||||
- Check Link header → <link rel="webmention"> → <a rel="webmention">
|
||||
- First match wins
|
||||
- Cache results: 1h for found, 24h for not found
|
||||
- SSRF protection on target URL
|
||||
|
||||
Args:
|
||||
target_url: Target URL to discover endpoint for
|
||||
- GET the target, following redirects (each hop SSRF-checked)
|
||||
- The first Link header with rel=webmention wins, then the first
|
||||
<link> or <a> with rel=webmention and an href, in document order
|
||||
- rel is a space-separated token list, matched case-insensitively
|
||||
- Relative endpoints resolve against the final URL; an empty href
|
||||
is the target itself; query strings are kept
|
||||
- Not cached: an updated post must re-discover (spec 3.1.4)
|
||||
|
||||
Returns:
|
||||
Endpoint URL if found, None otherwise
|
||||
"""
|
||||
# Check cache first
|
||||
cached = self._discovery_cache.get(target_url)
|
||||
if cached and cached['expires_at'] > datetime.utcnow():
|
||||
logger.debug(f"Using cached endpoint for {target_url}: {cached['endpoint']}")
|
||||
return cached['endpoint']
|
||||
Endpoint URL, or None if the target has none (or is blocked/4xx)
|
||||
|
||||
# SSRF protection
|
||||
Raises:
|
||||
httpx.HTTPError: Network errors and 5xx, so the caller retries
|
||||
"""
|
||||
try:
|
||||
validate_url_for_fetch(target_url)
|
||||
result = safe_get(target_url, headers={'User-Agent': user_agent(self.site_url)})
|
||||
except (SSRFBlockedError, ValueError) as e:
|
||||
logger.warning(f"SSRF protection blocked target {target_url}: {e}")
|
||||
# Cache negative result
|
||||
self._cache_endpoint(target_url, None, hours=24)
|
||||
return None
|
||||
except FetchTooLargeError:
|
||||
logger.info(f"Target too large for discovery: {target_url}")
|
||||
return None
|
||||
|
||||
# Fetch target URL
|
||||
try:
|
||||
response = httpx.get(
|
||||
target_url,
|
||||
timeout=10,
|
||||
headers={'User-Agent': f'StarPunk/1.6.0 (+{self.site_url})'},
|
||||
follow_redirects=True
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
# 1. Check Link header
|
||||
link_header = response.headers.get('Link', '')
|
||||
if link_header:
|
||||
endpoint = self._parse_link_header(link_header, response.url)
|
||||
if endpoint:
|
||||
self._cache_endpoint(target_url, endpoint, hours=1)
|
||||
return endpoint
|
||||
|
||||
# 2. Parse HTML for <link> and <a> elements
|
||||
content_type = response.headers.get('Content-Type', '')
|
||||
if 'html' in content_type.lower():
|
||||
endpoint = self._parse_html_for_endpoint(response.text, response.url)
|
||||
if endpoint:
|
||||
self._cache_endpoint(target_url, endpoint, hours=1)
|
||||
return endpoint
|
||||
|
||||
# No endpoint found - cache negative result
|
||||
logger.info(f"No webmention endpoint found for {target_url}")
|
||||
self._cache_endpoint(target_url, None, hours=24)
|
||||
if result.status_code >= 500:
|
||||
raise httpx.HTTPError(f"Target returned {result.status_code}")
|
||||
if result.status_code >= 400:
|
||||
logger.info(f"Target returned {result.status_code}: {target_url}")
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to discover endpoint for {target_url}: {e}")
|
||||
# Don't cache on error - allow retry
|
||||
return None
|
||||
for link_header in result.headers.get_list('Link'):
|
||||
endpoint = self._parse_link_header(link_header, result.url)
|
||||
if endpoint is not None:
|
||||
return endpoint
|
||||
|
||||
content_type = result.headers.get('Content-Type', '')
|
||||
if 'html' in content_type.lower():
|
||||
endpoint = self._parse_html_for_endpoint(result.text, result.url)
|
||||
if endpoint is not None:
|
||||
return endpoint
|
||||
|
||||
logger.info(f"No webmention endpoint found for {target_url}")
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _has_webmention_rel(rel: str) -> bool:
|
||||
return 'webmention' in rel.lower().split()
|
||||
|
||||
def _parse_link_header(self, link_header: str, base_url: str) -> Optional[str]:
|
||||
"""
|
||||
Parse Link header for webmention endpoint
|
||||
Parse a Link header (RFC 8288) for a rel=webmention link
|
||||
|
||||
Args:
|
||||
link_header: Raw Link header value
|
||||
link_header: Link header value (may hold several links)
|
||||
base_url: Base URL for resolving relative URLs
|
||||
|
||||
Returns:
|
||||
Absolute endpoint URL if found, None otherwise
|
||||
Absolute endpoint URL of the first match, None otherwise
|
||||
"""
|
||||
# Simple parsing - look for rel="webmention"
|
||||
# Format: <URL>; rel="webmention"
|
||||
import re
|
||||
pattern = r'<([^>]+)>\s*;\s*rel\s*=\s*["\']?webmention["\']?'
|
||||
match = re.search(pattern, link_header, re.IGNORECASE)
|
||||
|
||||
if match:
|
||||
endpoint = match.group(1)
|
||||
# Make absolute if relative
|
||||
from urllib.parse import urljoin
|
||||
return urljoin(base_url, endpoint)
|
||||
for match in LINK_RE.finditer(link_header):
|
||||
uri, params = match.group(1), match.group(2)
|
||||
for param in LINK_PARAM_RE.finditer(params):
|
||||
if param.group(1).lower() != 'rel':
|
||||
continue
|
||||
rel = param.group(2) if param.group(2) is not None else (param.group(3) or '')
|
||||
if self._has_webmention_rel(rel):
|
||||
return urljoin(base_url, uri.strip())
|
||||
break # only the first rel parameter counts (RFC 8288)
|
||||
|
||||
return None
|
||||
|
||||
def _parse_html_for_endpoint(self, html_content: str, base_url: str) -> Optional[str]:
|
||||
"""
|
||||
Parse HTML for webmention endpoint in <link> or <a> elements
|
||||
Find the first <link> or <a> with rel=webmention, in document order
|
||||
|
||||
Args:
|
||||
html_content: HTML content
|
||||
@@ -567,48 +694,18 @@ class WebmentionWorker(threading.Thread):
|
||||
Absolute endpoint URL if found, None otherwise
|
||||
"""
|
||||
try:
|
||||
doc = lxml_html.fromstring(html_content)
|
||||
|
||||
# Check <link rel="webmention"> first
|
||||
links = doc.xpath('//link[@rel="webmention"]/@href')
|
||||
if links:
|
||||
from urllib.parse import urljoin
|
||||
return urljoin(base_url, links[0])
|
||||
|
||||
# Check <a rel="webmention">
|
||||
anchors = doc.xpath('//a[@rel="webmention"]/@href')
|
||||
if anchors:
|
||||
from urllib.parse import urljoin
|
||||
return urljoin(base_url, anchors[0])
|
||||
|
||||
doc = lxml_html.document_fromstring(html_content)
|
||||
except Exception as e:
|
||||
logger.debug(f"HTML parsing failed: {e}")
|
||||
return None
|
||||
|
||||
for element in doc.iter('link', 'a'):
|
||||
href = element.get('href')
|
||||
if href is not None and self._has_webmention_rel(element.get('rel', '')):
|
||||
return urljoin(base_url, href.strip())
|
||||
|
||||
return None
|
||||
|
||||
def _cache_endpoint(self, target_url: str, endpoint: Optional[str], hours: int):
|
||||
"""
|
||||
Cache endpoint discovery result
|
||||
|
||||
Args:
|
||||
target_url: Target URL
|
||||
endpoint: Discovered endpoint (None = negative cache)
|
||||
hours: TTL in hours
|
||||
"""
|
||||
self._discovery_cache[target_url] = {
|
||||
'endpoint': endpoint,
|
||||
'expires_at': datetime.utcnow() + timedelta(hours=hours)
|
||||
}
|
||||
|
||||
# Optional: Evict old entries (keep cache bounded)
|
||||
if len(self._discovery_cache) > 1000:
|
||||
# Remove expired entries
|
||||
now = datetime.utcnow()
|
||||
self._discovery_cache = {
|
||||
k: v for k, v in self._discovery_cache.items()
|
||||
if v['expires_at'] > now
|
||||
}
|
||||
|
||||
def _mark_send_success(self, queue_id: int, source_url: str, target_url: str):
|
||||
"""Mark webmention send as successful"""
|
||||
try:
|
||||
|
||||
@@ -0,0 +1,340 @@
|
||||
"""
|
||||
WebSub publishing and the built-in hub (v2.10.0, ADR-071)
|
||||
|
||||
Publisher: topic responses advertise the hub and their own URL; a change to a
|
||||
public note clears cached feeds and queues delivery of every affected topic.
|
||||
|
||||
Hub: subscriptions to this site's own topics only. Requests are accepted
|
||||
here (202) and verified, delivered and retried by the WebSub worker.
|
||||
|
||||
References:
|
||||
- W3C WebSub: https://www.w3.org/TR/websub/
|
||||
- ADR-071: WebSub publishing with a built-in hub
|
||||
"""
|
||||
|
||||
import json
|
||||
import secrets
|
||||
from typing import Iterable, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from flask import current_app, request
|
||||
|
||||
|
||||
# Topic endpoints and their fixed paths (tag pages are added per tag)
|
||||
FEED_TOPICS = {
|
||||
"public.feed_atom": "/feed.atom",
|
||||
"public.feed_rss": "/feed.rss",
|
||||
"public.feed_xml_legacy": "/feed.rss", # legacy alias of the RSS topic
|
||||
"public.feed_json": "/feed.json",
|
||||
}
|
||||
PAGE_TOPICS = {"public.index": "/", "public.likes": "/likes"}
|
||||
NEGOTIATED = { # /feed: the topic of whichever format was served
|
||||
"application/atom+xml": "/feed.atom",
|
||||
"application/rss+xml": "/feed.rss",
|
||||
"application/feed+json": "/feed.json",
|
||||
"application/json": "/feed.json",
|
||||
}
|
||||
|
||||
LEASE_DEFAULT = 10 * 24 * 3600
|
||||
LEASE_MIN = 24 * 3600
|
||||
LEASE_MAX = 30 * 24 * 3600
|
||||
MAX_SECRET_BYTES = 200
|
||||
|
||||
|
||||
class SubscriptionError(Exception):
|
||||
"""A subscription request the hub refuses (4xx with a plain-text reason)"""
|
||||
|
||||
def __init__(self, message: str, status: int = 400):
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
|
||||
|
||||
# --- Configuration -----------------------------------------------------------
|
||||
|
||||
def enabled() -> bool:
|
||||
return bool(current_app.config.get("WEBSUB_ENABLED", True))
|
||||
|
||||
|
||||
def external_hub() -> Optional[str]:
|
||||
"""URL of an external hub, if one is configured instead of the built-in one"""
|
||||
return current_app.config.get("WEBSUB_HUB_URL") or None
|
||||
|
||||
|
||||
def hub_url() -> str:
|
||||
# Built from SITE_URL so the worker can use it outside a request
|
||||
return external_hub() or site_url() + "/websub"
|
||||
|
||||
|
||||
def site_url() -> str:
|
||||
return current_app.config["SITE_URL"].rstrip("/")
|
||||
|
||||
|
||||
# --- Topics ------------------------------------------------------------------
|
||||
|
||||
def topic_url(path: str) -> str:
|
||||
return site_url() + path
|
||||
|
||||
|
||||
def tag_topic(tag_name: str) -> str:
|
||||
from urllib.parse import quote
|
||||
return f"{site_url()}/tag/{quote(tag_name, safe='')}"
|
||||
|
||||
|
||||
def topic_for_request(response=None) -> Optional[str]:
|
||||
"""The topic URL the current request serves, or None if it isn't a topic"""
|
||||
endpoint = request.endpoint
|
||||
if endpoint in PAGE_TOPICS:
|
||||
return topic_url(PAGE_TOPICS[endpoint])
|
||||
if endpoint in FEED_TOPICS:
|
||||
return topic_url(FEED_TOPICS[endpoint])
|
||||
if endpoint == "public.feed" and response is not None:
|
||||
mime = (response.mimetype or "").lower()
|
||||
return topic_url(NEGOTIATED[mime]) if mime in NEGOTIATED else None
|
||||
if endpoint == "public.tag":
|
||||
from starpunk.tags import normalize_tag
|
||||
tag = (request.view_args or {}).get("tag", "")
|
||||
return tag_topic(normalize_tag(tag)[0]) if tag else None
|
||||
return None
|
||||
|
||||
|
||||
def canonical_topic(url: str) -> Optional[str]:
|
||||
"""
|
||||
A subscription's hub.topic as one of this site's topics, or None
|
||||
|
||||
Tag topics must name an existing tag.
|
||||
"""
|
||||
if not url:
|
||||
return None
|
||||
site = urlparse(site_url())
|
||||
parsed = urlparse(url.strip())
|
||||
if (parsed.scheme, parsed.netloc.lower()) != (site.scheme, site.netloc.lower()):
|
||||
return None
|
||||
path = parsed.path or "/"
|
||||
if parsed.query or parsed.fragment:
|
||||
return None
|
||||
if path in set(PAGE_TOPICS.values()) | set(FEED_TOPICS.values()):
|
||||
return topic_url(path)
|
||||
if path.startswith("/tag/"):
|
||||
from starpunk.tags import get_tag_by_name, normalize_tag
|
||||
name = normalize_tag(path[len("/tag/"):])[0]
|
||||
if name and get_tag_by_name(name):
|
||||
return tag_topic(name)
|
||||
return None
|
||||
|
||||
|
||||
def link_header_values(topic: str) -> list[str]:
|
||||
return [f'<{hub_url()}>; rel="hub"', f'<{topic}>; rel="self"']
|
||||
|
||||
|
||||
def add_discovery_headers(response):
|
||||
"""after_request: Link rel=hub and rel=self on topic responses (spec 4)"""
|
||||
if not enabled() or response.status_code not in (200, 304):
|
||||
return response
|
||||
topic = topic_for_request(response)
|
||||
if topic:
|
||||
for value in link_header_values(topic):
|
||||
response.headers.add("Link", value)
|
||||
return response
|
||||
|
||||
|
||||
def template_links() -> Optional[dict]:
|
||||
"""For base.html: {'hub', 'self'} on topic pages"""
|
||||
if not enabled() or not request:
|
||||
return None
|
||||
topic = topic_for_request()
|
||||
return {"hub": hub_url(), "self": topic} if topic else None
|
||||
|
||||
|
||||
def embed_hub(format_name: str, content: str) -> str:
|
||||
"""Add the hub link inside a generated feed, next to its self link"""
|
||||
if not enabled():
|
||||
return content
|
||||
hub = hub_url()
|
||||
if format_name == "atom":
|
||||
marker = '<link rel="self"'
|
||||
return content.replace(marker, f'<link rel="hub" href="{_xml(hub)}"/>\n {marker}', 1)
|
||||
if format_name == "rss":
|
||||
marker = "<atom:link "
|
||||
return content.replace(marker, f'<atom:link href="{_xml(hub)}" rel="hub"/>\n {marker}', 1)
|
||||
if format_name == "json":
|
||||
feed = json.loads(content)
|
||||
feed["hubs"] = [{"type": "WebSub", "url": hub}]
|
||||
return json.dumps(feed, ensure_ascii=False, indent=2)
|
||||
return content
|
||||
|
||||
|
||||
def _xml(value: str) -> str:
|
||||
return (value.replace("&", "&").replace("<", "<")
|
||||
.replace(">", ">").replace('"', """))
|
||||
|
||||
|
||||
# --- Changes -------------------------------------------------------------------
|
||||
|
||||
def get_content_version(db) -> str:
|
||||
row = db.execute("SELECT value FROM site_state WHERE key = 'content_version'").fetchone()
|
||||
return row[0] if row else "0"
|
||||
|
||||
|
||||
def bump_content_version(db) -> None:
|
||||
db.execute(
|
||||
"INSERT INTO site_state (key, value) VALUES ('content_version', '1') "
|
||||
"ON CONFLICT(key) DO UPDATE SET value = CAST(value AS INTEGER) + 1"
|
||||
)
|
||||
|
||||
|
||||
def note_changed(note, was_public: bool = False, old_tags: Iterable[str] = ()) -> None:
|
||||
"""
|
||||
A note was created, edited, deleted, restored or (un)published
|
||||
|
||||
Clears cached feeds in every process and queues delivery of the topics
|
||||
that show it. Changes that are never public (a draft edited) do nothing.
|
||||
|
||||
Args:
|
||||
note: The note (after the change; for a delete, before it)
|
||||
was_public: Whether it was public before the change
|
||||
old_tags: Its tag names before the change
|
||||
"""
|
||||
from starpunk.database import get_db
|
||||
|
||||
is_public = bool(note.published) and not getattr(note, "deleted_at", None)
|
||||
if not (is_public or was_public):
|
||||
return
|
||||
|
||||
db = get_db(current_app)
|
||||
bump_content_version(db)
|
||||
db.commit()
|
||||
|
||||
if not enabled():
|
||||
return
|
||||
|
||||
from starpunk.tags import get_note_tags
|
||||
try:
|
||||
tags = {t["name"] for t in get_note_tags(note.id)}
|
||||
except Exception:
|
||||
tags = set()
|
||||
tags |= set(old_tags)
|
||||
|
||||
topics = [topic_url(p) for p in ("/", "/feed.atom", "/feed.rss", "/feed.json")]
|
||||
if getattr(note, "like_of", None):
|
||||
topics.append(topic_url("/likes"))
|
||||
topics += [tag_topic(name) for name in sorted(tags)]
|
||||
queue_topics(db, topics)
|
||||
|
||||
|
||||
def queue_topics(db, topics: Iterable[str]) -> int:
|
||||
"""Queue delivery of topics: to their subscribers, or a ping to an external hub"""
|
||||
topics = list(dict.fromkeys(topics))
|
||||
if not topics:
|
||||
return 0
|
||||
if external_hub():
|
||||
for topic in topics:
|
||||
db.execute("DELETE FROM websub_deliveries WHERE subscription_id IS NULL AND topic = ?", (topic,))
|
||||
db.execute("INSERT INTO websub_deliveries (topic) VALUES (?)", (topic,))
|
||||
db.commit()
|
||||
return len(topics)
|
||||
|
||||
placeholders = ",".join("?" * len(topics))
|
||||
rows = db.execute(
|
||||
f"SELECT id, topic FROM websub_subscriptions WHERE topic IN ({placeholders}) "
|
||||
"AND verified_at IS NOT NULL AND expires_at > datetime('now')",
|
||||
topics,
|
||||
).fetchall()
|
||||
for sub_id, topic in rows:
|
||||
# The newest content replaces anything still waiting (ADR-071 6)
|
||||
db.execute("DELETE FROM websub_deliveries WHERE subscription_id = ?", (sub_id,))
|
||||
db.execute("INSERT INTO websub_deliveries (subscription_id, topic) VALUES (?, ?)", (sub_id, topic))
|
||||
db.commit()
|
||||
return len(rows)
|
||||
|
||||
|
||||
# --- Hub: subscription requests (spec 5.1) -------------------------------------
|
||||
|
||||
def accept_subscription_request(form, db) -> None:
|
||||
"""
|
||||
Validate a subscription request and queue it for verification
|
||||
|
||||
Raises:
|
||||
SubscriptionError: with the status and plain-text reason to return
|
||||
"""
|
||||
from starpunk.webmention_ssrf import SSRFBlockedError, validate_url_for_fetch
|
||||
|
||||
if external_hub():
|
||||
raise SubscriptionError("This site uses another hub: " + external_hub(), 400)
|
||||
|
||||
mode = (form.get("hub.mode") or "").strip()
|
||||
if mode not in ("subscribe", "unsubscribe"):
|
||||
raise SubscriptionError("hub.mode must be subscribe or unsubscribe")
|
||||
|
||||
callback = (form.get("hub.callback") or "").strip()
|
||||
parsed = urlparse(callback)
|
||||
if parsed.scheme not in ("http", "https") or not parsed.hostname:
|
||||
raise SubscriptionError("hub.callback must be an absolute http(s) URL")
|
||||
try:
|
||||
validate_url_for_fetch(callback)
|
||||
except (SSRFBlockedError, ValueError) as e:
|
||||
raise SubscriptionError(f"hub.callback is not reachable: {e}")
|
||||
|
||||
topic = canonical_topic(form.get("hub.topic") or "")
|
||||
if not topic:
|
||||
raise SubscriptionError("hub.topic is not a topic of this site; use the rel=self URL")
|
||||
|
||||
secret = form.get("hub.secret")
|
||||
if mode == "unsubscribe":
|
||||
secret = None # MUST be ignored for unsubscriptions
|
||||
if secret is not None and len(secret.encode("utf-8")) >= MAX_SECRET_BYTES:
|
||||
raise SubscriptionError("hub.secret must be less than 200 bytes")
|
||||
|
||||
lease = LEASE_DEFAULT
|
||||
if form.get("hub.lease_seconds"):
|
||||
try:
|
||||
lease = int(form["hub.lease_seconds"])
|
||||
except ValueError:
|
||||
raise SubscriptionError("hub.lease_seconds must be a number")
|
||||
lease = max(LEASE_MIN, min(lease, LEASE_MAX))
|
||||
|
||||
if mode == "subscribe":
|
||||
existing = db.execute(
|
||||
"SELECT 1 FROM websub_subscriptions WHERE topic = ? AND callback = ?", (topic, callback)
|
||||
).fetchone()
|
||||
if not existing:
|
||||
total = db.execute("SELECT COUNT(*) FROM websub_subscriptions").fetchone()[0]
|
||||
if total >= current_app.config.get("WEBSUB_MAX_SUBSCRIPTIONS", 1000):
|
||||
raise SubscriptionError("This hub has reached its subscription limit", 429)
|
||||
host = parsed.hostname.lower()
|
||||
per_host = sum(
|
||||
1 for (cb,) in db.execute("SELECT callback FROM websub_subscriptions")
|
||||
if (urlparse(cb).hostname or "").lower() == host
|
||||
)
|
||||
if per_host >= current_app.config.get("WEBSUB_MAX_PER_HOST", 25):
|
||||
raise SubscriptionError(f"Too many subscriptions for {host}", 429)
|
||||
|
||||
# The latest request for a topic and callback replaces any pending one
|
||||
db.execute(
|
||||
"""
|
||||
INSERT INTO websub_intents (topic, callback, mode, secret, lease_seconds)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT(topic, callback) DO UPDATE SET
|
||||
mode = excluded.mode, secret = excluded.secret,
|
||||
lease_seconds = excluded.lease_seconds, attempts = 0,
|
||||
next_attempt_at = CURRENT_TIMESTAMP, created_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(topic, callback, mode, secret, lease),
|
||||
)
|
||||
db.commit()
|
||||
|
||||
|
||||
def stats(db) -> dict:
|
||||
"""Active subscribers per topic and deliveries being retried (admin)"""
|
||||
by_topic = db.execute(
|
||||
"SELECT topic, COUNT(*) FROM websub_subscriptions WHERE expires_at > datetime('now') "
|
||||
"GROUP BY topic ORDER BY COUNT(*) DESC, topic"
|
||||
).fetchall()
|
||||
retrying = db.execute(
|
||||
"SELECT COUNT(*) FROM websub_deliveries WHERE attempts > 0"
|
||||
).fetchone()[0]
|
||||
return {"by_topic": [(t, n) for t, n in by_topic], "retrying": retrying,
|
||||
"external": external_hub()}
|
||||
|
||||
|
||||
def new_challenge() -> str:
|
||||
return secrets.token_urlsafe(32)
|
||||
@@ -0,0 +1,287 @@
|
||||
"""
|
||||
WebSub worker: verification of intent and content distribution (v2.10.0)
|
||||
|
||||
Runs like the Webmention worker: a daemon thread in every gunicorn process,
|
||||
but only the process holding the file lock works the queues.
|
||||
|
||||
- Intents (spec 5.3): GET the callback with a challenge; 2xx echoing it
|
||||
verifies. Anything else (including redirects) leaves the subscription as
|
||||
it was. Network errors and 5xx are retried a few times.
|
||||
- Deliveries (spec 6, 7): render the topic in-app and POST it with
|
||||
Link rel=hub/self and, if there is a secret, X-Hub-Signature (sha256).
|
||||
- External hub: a delivery without a subscription is a publish ping.
|
||||
|
||||
References:
|
||||
- https://www.w3.org/TR/websub/
|
||||
- ADR-071
|
||||
"""
|
||||
|
||||
import fcntl
|
||||
import hashlib
|
||||
import hmac
|
||||
import logging
|
||||
import threading
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from urllib.parse import urlencode, urlparse, urlsplit, urlunsplit
|
||||
|
||||
import httpx
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DELIVERY_BACKOFF = [60, 300, 1800, 7200, 21600] # then give up on this delivery
|
||||
INTENT_BACKOFF = [60, 300, 900]
|
||||
DROP_AFTER = timedelta(days=3) # failing this long without a success: unsubscribe
|
||||
PRUNE_EVERY = timedelta(hours=1)
|
||||
BATCH = 10
|
||||
|
||||
|
||||
def callback_with_params(callback: str, params: dict) -> str:
|
||||
"""Append parameters to the callback, keeping its query string (spec 5.3)"""
|
||||
parts = urlsplit(callback)
|
||||
query = parts.query + ("&" if parts.query else "") + urlencode(params)
|
||||
return urlunsplit((parts.scheme, parts.netloc, parts.path, query, parts.fragment))
|
||||
|
||||
|
||||
def signature(secret: str, body: bytes) -> str:
|
||||
return "sha256=" + hmac.new(secret.encode("utf-8"), body, hashlib.sha256).hexdigest()
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
|
||||
def _later(seconds: int) -> str:
|
||||
return (datetime.utcnow() + timedelta(seconds=seconds)).strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
|
||||
class WebSubWorker(threading.Thread):
|
||||
"""Daemon thread that verifies subscriptions and delivers content"""
|
||||
|
||||
def __init__(self, app, poll_interval: int = 5):
|
||||
super().__init__(daemon=True, name="websub-worker")
|
||||
self.app = app
|
||||
self.poll_interval = poll_interval
|
||||
self._stop_event = threading.Event()
|
||||
self._lock_file = None
|
||||
self._last_prune = None
|
||||
|
||||
# --- thread plumbing ----------------------------------------------------
|
||||
|
||||
def run(self):
|
||||
logger.info("WebSub worker started")
|
||||
while not self._stop_event.is_set():
|
||||
if not self._acquire_lock():
|
||||
self._stop_event.wait(self.poll_interval)
|
||||
continue
|
||||
try:
|
||||
with self.app.app_context():
|
||||
done = self.process_once()
|
||||
except Exception as e:
|
||||
logger.error(f"WebSub worker error: {e}", exc_info=True)
|
||||
done = 0
|
||||
if not done:
|
||||
self._stop_event.wait(self.poll_interval)
|
||||
self._release_lock()
|
||||
|
||||
def stop(self):
|
||||
self._stop_event.set()
|
||||
|
||||
def _acquire_lock(self) -> bool:
|
||||
if self._lock_file:
|
||||
return True
|
||||
path = Path(self.app.config["DATABASE_PATH"]).parent / "websub-worker.lock"
|
||||
lock_file = open(path, "w")
|
||||
try:
|
||||
fcntl.flock(lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except OSError:
|
||||
lock_file.close()
|
||||
return False
|
||||
self._lock_file = lock_file
|
||||
logger.info("WebSub worker holds the queue lock")
|
||||
return True
|
||||
|
||||
def _release_lock(self):
|
||||
if self._lock_file:
|
||||
self._lock_file.close()
|
||||
self._lock_file = None
|
||||
|
||||
# --- one pass (also called directly by tests) ---------------------------
|
||||
|
||||
def process_once(self) -> int:
|
||||
from starpunk.database import get_db
|
||||
|
||||
db = get_db(self.app)
|
||||
done = 0
|
||||
for row in db.execute(
|
||||
"SELECT * FROM websub_intents WHERE next_attempt_at <= ? ORDER BY id LIMIT ?",
|
||||
(_now(), BATCH),
|
||||
).fetchall():
|
||||
self.verify(db, row)
|
||||
done += 1
|
||||
for row in db.execute(
|
||||
"SELECT * FROM websub_deliveries WHERE next_attempt_at <= ? ORDER BY id LIMIT ?",
|
||||
(_now(), BATCH),
|
||||
).fetchall():
|
||||
self.deliver(db, row)
|
||||
done += 1
|
||||
if self._last_prune is None or datetime.utcnow() - self._last_prune > PRUNE_EVERY:
|
||||
self.prune(db)
|
||||
return done
|
||||
|
||||
# --- verification of intent (spec 5.3) ----------------------------------
|
||||
|
||||
def verify(self, db, intent) -> None:
|
||||
from starpunk.websub import new_challenge
|
||||
from starpunk.webmention_ssrf import SSRFBlockedError, safe_get, user_agent
|
||||
|
||||
challenge = new_challenge()
|
||||
params = {"hub.mode": intent["mode"], "hub.topic": intent["topic"], "hub.challenge": challenge}
|
||||
if intent["mode"] == "subscribe":
|
||||
params["hub.lease_seconds"] = intent["lease_seconds"]
|
||||
url = callback_with_params(intent["callback"], params)
|
||||
|
||||
try:
|
||||
result = safe_get(url, headers={"User-Agent": user_agent(self.app.config["SITE_URL"])},
|
||||
max_redirects=0, max_bytes=64 * 1024)
|
||||
verified = 200 <= result.status_code < 300 and result.text.strip() == challenge
|
||||
retry = result.status_code >= 500
|
||||
error = None if verified else f"HTTP {result.status_code}" + (
|
||||
"" if result.status_code >= 300 else ": challenge not echoed")
|
||||
except (SSRFBlockedError, ValueError) as e:
|
||||
verified, retry, error = False, False, f"Blocked: {e}"
|
||||
except httpx.TooManyRedirects:
|
||||
verified, retry, error = False, False, "Redirected (not allowed)"
|
||||
except httpx.HTTPError as e:
|
||||
verified, retry, error = False, True, str(e)[:200]
|
||||
|
||||
if verified:
|
||||
if intent["mode"] == "subscribe":
|
||||
db.execute(
|
||||
"""
|
||||
INSERT INTO websub_subscriptions
|
||||
(topic, callback, secret, lease_seconds, expires_at, verified_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(topic, callback) DO UPDATE SET
|
||||
secret = excluded.secret, lease_seconds = excluded.lease_seconds,
|
||||
expires_at = excluded.expires_at, verified_at = excluded.verified_at,
|
||||
last_error = NULL
|
||||
""",
|
||||
(intent["topic"], intent["callback"], intent["secret"], intent["lease_seconds"],
|
||||
_later(intent["lease_seconds"]), _now()),
|
||||
)
|
||||
else:
|
||||
db.execute("DELETE FROM websub_subscriptions WHERE topic = ? AND callback = ?",
|
||||
(intent["topic"], intent["callback"]))
|
||||
db.execute("DELETE FROM websub_intents WHERE id = ?", (intent["id"],))
|
||||
logger.info(f"WebSub {intent['mode']} verified: {intent['callback']} -> {intent['topic']}")
|
||||
elif retry and intent["attempts"] < len(INTENT_BACKOFF):
|
||||
db.execute("UPDATE websub_intents SET attempts = attempts + 1, next_attempt_at = ? WHERE id = ?",
|
||||
(_later(INTENT_BACKOFF[intent["attempts"]]), intent["id"]))
|
||||
else:
|
||||
# Verification failed: the subscription state stays as it was
|
||||
db.execute("DELETE FROM websub_intents WHERE id = ?", (intent["id"],))
|
||||
logger.info(f"WebSub {intent['mode']} not verified for {intent['callback']}: {error}")
|
||||
db.commit()
|
||||
|
||||
# --- content distribution (spec 6, 7) ------------------------------------
|
||||
|
||||
def render_topic(self, topic: str):
|
||||
"""(status, content_type, body) of a topic, rendered in-app"""
|
||||
path = urlparse(topic).path or "/"
|
||||
response = self.app.test_client().get(path, headers={"Accept": "*/*"})
|
||||
return response.status_code, response.headers.get("Content-Type", ""), response.get_data()
|
||||
|
||||
def deliver(self, db, delivery) -> None:
|
||||
from starpunk.websub import hub_url
|
||||
from starpunk.webmention_ssrf import SSRFBlockedError, safe_post, user_agent
|
||||
|
||||
site = self.app.config["SITE_URL"]
|
||||
sub = None
|
||||
try:
|
||||
if delivery["subscription_id"] is None:
|
||||
# External hub: ping it (hub.url is the de facto parameter)
|
||||
result = safe_post(hub_url(), data={"hub.mode": "publish", "hub.url": delivery["topic"],
|
||||
"hub.topic": delivery["topic"]},
|
||||
headers={"User-Agent": user_agent(site)})
|
||||
else:
|
||||
sub = db.execute(
|
||||
"SELECT * FROM websub_subscriptions WHERE id = ? AND expires_at > ?",
|
||||
(delivery["subscription_id"], _now()),
|
||||
).fetchone()
|
||||
if sub is None: # unsubscribed or expired meanwhile
|
||||
db.execute("DELETE FROM websub_deliveries WHERE id = ?", (delivery["id"],))
|
||||
db.commit()
|
||||
return
|
||||
status, content_type, body = self.render_topic(sub["topic"])
|
||||
if status != 200: # e.g. a tag page whose last note went
|
||||
db.execute("DELETE FROM websub_deliveries WHERE id = ?", (delivery["id"],))
|
||||
db.commit()
|
||||
return
|
||||
headers = {
|
||||
"Content-Type": content_type,
|
||||
"Link": f'<{hub_url()}>; rel="hub", <{sub["topic"]}>; rel="self"',
|
||||
"User-Agent": user_agent(site),
|
||||
}
|
||||
if sub["secret"]:
|
||||
headers["X-Hub-Signature"] = signature(sub["secret"], body)
|
||||
result = safe_post(sub["callback"], content=body, headers=headers)
|
||||
status, error = result.status_code, f"HTTP {result.status_code}"
|
||||
except (SSRFBlockedError, ValueError) as e:
|
||||
status, error = None, f"Blocked: {e}"
|
||||
except httpx.HTTPError as e:
|
||||
status, error = None, str(e)[:200] or type(e).__name__
|
||||
|
||||
if status is not None and 200 <= status < 300:
|
||||
db.execute("DELETE FROM websub_deliveries WHERE id = ?", (delivery["id"],))
|
||||
if sub is not None:
|
||||
db.execute("UPDATE websub_subscriptions SET last_success_at = ?, last_error = NULL WHERE id = ?",
|
||||
(_now(), sub["id"]))
|
||||
elif sub is not None and status == 410:
|
||||
db.execute("DELETE FROM websub_subscriptions WHERE id = ?", (sub["id"],))
|
||||
logger.info(f"WebSub subscriber gone (410), removed: {sub['callback']}")
|
||||
elif delivery["attempts"] < len(DELIVERY_BACKOFF) and not error.startswith("Blocked"):
|
||||
db.execute(
|
||||
"UPDATE websub_deliveries SET attempts = attempts + 1, next_attempt_at = ?, last_error = ? WHERE id = ?",
|
||||
(_later(DELIVERY_BACKOFF[delivery["attempts"]]), error, delivery["id"]),
|
||||
)
|
||||
if sub is not None:
|
||||
db.execute("UPDATE websub_subscriptions SET last_error = ? WHERE id = ?", (error, sub["id"]))
|
||||
else:
|
||||
db.execute("DELETE FROM websub_deliveries WHERE id = ?", (delivery["id"],))
|
||||
logger.warning(f"WebSub delivery of {delivery['topic']} gave up: {error}")
|
||||
if sub is not None:
|
||||
self._drop_if_failing(db, sub, error)
|
||||
db.commit()
|
||||
|
||||
def _drop_if_failing(self, db, sub, error: str) -> None:
|
||||
last_ok = sub["last_success_at"] or sub["verified_at"] or sub["created_at"]
|
||||
try:
|
||||
since = datetime.fromisoformat(str(last_ok))
|
||||
except ValueError:
|
||||
return
|
||||
if datetime.utcnow() - since > DROP_AFTER:
|
||||
db.execute("DELETE FROM websub_subscriptions WHERE id = ?", (sub["id"],))
|
||||
logger.info(f"WebSub subscription failing for {DROP_AFTER.days} days, removed: {sub['callback']}")
|
||||
else:
|
||||
db.execute("UPDATE websub_subscriptions SET last_error = ? WHERE id = ?", (error, sub["id"]))
|
||||
|
||||
# --- housekeeping -------------------------------------------------------
|
||||
|
||||
def prune(self, db) -> int:
|
||||
"""Remove subscriptions whose lease has run out (they weren't renewed)"""
|
||||
cursor = db.execute("DELETE FROM websub_subscriptions WHERE expires_at <= ?", (_now(),))
|
||||
db.commit()
|
||||
self._last_prune = datetime.utcnow()
|
||||
if cursor.rowcount:
|
||||
logger.info(f"WebSub: removed {cursor.rowcount} expired subscriptions")
|
||||
return cursor.rowcount
|
||||
|
||||
|
||||
def start_worker(app) -> Optional[WebSubWorker]:
|
||||
if not app.config.get("WEBSUB_ENABLED", True) or app.config.get("TESTING", False):
|
||||
return None
|
||||
worker = WebSubWorker(app, poll_interval=app.config.get("WEBSUB_POLL_INTERVAL", 5))
|
||||
worker.start()
|
||||
return worker
|
||||
+30
-2
@@ -92,6 +92,17 @@ main { flex: 1; margin-bottom: var(--spacing-xl); }
|
||||
.e-content ul, .e-content ol { margin: 0 0 var(--spacing-md) 1.25rem; }
|
||||
.e-content img { max-width: 100%; height: auto; }
|
||||
.e-content h1, .e-content h2, .e-content h3 { font-family: var(--font-body); }
|
||||
.reaction { margin-bottom: var(--spacing-sm); font-size: 0.9375rem; color: var(--color-text-light); overflow-wrap: anywhere; }
|
||||
.reaction-verb { font-weight: 600; margin-right: 0.25rem; }
|
||||
.reaction-like .reaction-verb { color: #c2410c; }
|
||||
.reaction-repost .reaction-verb { color: #15803d; }
|
||||
.reaction a { color: var(--color-text); font-weight: 500; }
|
||||
.reaction a:hover { color: var(--color-link); }
|
||||
.reaction-by { margin-left: 0.25rem; }
|
||||
@media (prefers-color-scheme: dark) {
|
||||
.reaction-like .reaction-verb { color: #fb923c; }
|
||||
.reaction-repost .reaction-verb { color: #4ade80; }
|
||||
}
|
||||
.read-more { margin: var(--spacing-sm) 0 0; font-size: 0.9375rem; }
|
||||
|
||||
.note-meta { display: flex; flex-wrap: wrap; align-items: baseline; gap: 0 0.4rem; margin-top: var(--spacing-sm); color: var(--color-text-light); font-size: 0.8125rem; }
|
||||
@@ -125,7 +136,8 @@ small { display: block; margin-top: var(--spacing-xs); color: var(--color-text-l
|
||||
.button-warning { background: var(--color-warning); color: #333; border-color: var(--color-warning); }
|
||||
.button-small { padding: 0.25rem 0.5rem; font-size: 0.875rem; }
|
||||
|
||||
.admin-container { max-width: 60rem; }
|
||||
.admin-page { --max-width: 60rem; }
|
||||
.table-scroll { overflow-x: auto; }
|
||||
.admin-nav { display: flex; gap: var(--spacing-md); padding: var(--spacing-md); background: var(--color-bg-alt); border-radius: var(--border-radius); margin-bottom: var(--spacing-lg); flex-wrap: wrap; align-items: center; }
|
||||
.admin-nav .logout-form { margin-left: auto; }
|
||||
.admin-nav .logout-form button { margin: 0; }
|
||||
@@ -136,7 +148,7 @@ small { display: block; margin-top: var(--spacing-xs); color: var(--color-text-l
|
||||
.note-table { width: 100%; border-collapse: collapse; margin-top: var(--spacing-md); }
|
||||
.note-table th, .note-table td { padding: var(--spacing-md); text-align: left; border-bottom: 1px solid var(--color-border); }
|
||||
.note-table th { background: var(--color-bg-alt); font-weight: 600; }
|
||||
.note-content-preview { max-width: 30rem; }
|
||||
.note-content-preview { min-width: 14rem; overflow-wrap: anywhere; }
|
||||
.note-excerpt { margin-bottom: var(--spacing-xs); }
|
||||
.note-slug { color: var(--color-text-light); font-family: var(--font-mono); font-size: 0.75rem; }
|
||||
.note-date { white-space: nowrap; color: var(--color-text-light); }
|
||||
@@ -320,6 +332,9 @@ small { display: block; margin-top: var(--spacing-xs); color: var(--color-text-l
|
||||
}
|
||||
|
||||
.webmention-author a {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: var(--spacing-sm);
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
@@ -349,6 +364,13 @@ small { display: block; margin-top: var(--spacing-xs); color: var(--color-text-l
|
||||
margin-left: 0;
|
||||
}
|
||||
|
||||
.facepile .face > .h-card,
|
||||
.facepile .face > .h-card > a {
|
||||
display: block;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
.facepile .face img {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
@@ -380,6 +402,12 @@ small { display: block; margin-top: var(--spacing-xs); color: var(--color-text-l
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
.webmention-mention .h-card img {
|
||||
border-radius: 50%;
|
||||
vertical-align: middle;
|
||||
margin-right: var(--spacing-xs);
|
||||
}
|
||||
|
||||
.webmention-mention p {
|
||||
margin: var(--spacing-xs) 0 0 0;
|
||||
font-size: 0.875rem;
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
{% extends "base.html" %}
|
||||
|
||||
{% block title %}Post Deleted - {{ config.SITE_NAME }}{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<article class="error-page">
|
||||
<h1>410 - Post Deleted</h1>
|
||||
<p>This post has been deleted.</p>
|
||||
<p><a href="/">Return to homepage</a></p>
|
||||
</article>
|
||||
{% endblock %}
|
||||
@@ -1,5 +1,8 @@
|
||||
{% extends "base.html" %}
|
||||
|
||||
{# Admin pages use a wider frame so tables fit under the header #}
|
||||
{% block body_class %} class="admin-page"{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div class="admin-container">
|
||||
<nav class="admin-nav">
|
||||
|
||||
@@ -20,7 +20,23 @@
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
{# WebSub subscribers (v2.10.0) #}
|
||||
{% if websub_stats %}
|
||||
<p class="note-meta websub-stats">
|
||||
{% if websub_stats.external %}
|
||||
WebSub: using {{ websub_stats.external }}
|
||||
{% elif websub_stats.by_topic %}
|
||||
WebSub subscribers:
|
||||
{% for topic, count in websub_stats.by_topic %}{{ topic|replace(config.SITE_URL.rstrip('/'), '') or '/' }} {{ count }}{% if not loop.last %}, {% endif %}{% endfor %}
|
||||
{% if websub_stats.retrying %} · {{ websub_stats.retrying }} deliveries being retried{% endif %}
|
||||
{% else %}
|
||||
WebSub: no subscribers yet
|
||||
{% endif %}
|
||||
</p>
|
||||
{% endif %}
|
||||
|
||||
{% if notes %}
|
||||
<div class="table-scroll">
|
||||
<table class="note-table">
|
||||
<thead>
|
||||
<tr>
|
||||
@@ -67,6 +83,7 @@
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
{# Pagination (v1.6.0) #}
|
||||
{% if total_pages > 1 %}
|
||||
|
||||
@@ -12,12 +12,15 @@
|
||||
|
||||
<form action="{{ url_for('admin.update_note_submit', note_id=note.id) }}" method="POST" class="note-form">
|
||||
<div class="form-group">
|
||||
<label for="content">Content (Markdown)</label>
|
||||
{% if note.reacts_to %}
|
||||
<p class="note-meta">{{ {'like': '★ Like of', 'repost': '🔁 Repost of', 'bookmark': '🔖 Bookmark of'}[note.kind] }} <a href="{{ note.reacts_to }}">{{ note.reacts_to }}</a></p>
|
||||
{% endif %}
|
||||
<label for="content">{% if note.reacts_to %}Comment (optional, Markdown){% else %}Content (Markdown){% endif %}</label>
|
||||
<textarea
|
||||
id="content"
|
||||
name="content"
|
||||
rows="20"
|
||||
required
|
||||
rows="{{ 6 if note.reacts_to else 20 }}"
|
||||
{% if not note.reacts_to %}required{% endif %}
|
||||
autofocus
|
||||
>{{ note.content }}</textarea>
|
||||
<small>Use Markdown syntax for formatting</small>
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
<p>Micropub apps you have approved. Revoke any you no longer use.</p>
|
||||
|
||||
{% if tokens %}
|
||||
<div class="table-scroll">
|
||||
<table class="note-table">
|
||||
<thead>
|
||||
<tr>
|
||||
@@ -37,6 +38,7 @@
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
{% else %}
|
||||
<p>No apps connected.</p>
|
||||
{% endif %}
|
||||
|
||||
@@ -43,6 +43,7 @@
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<div class="table-scroll">
|
||||
<table class="webmention-table">
|
||||
<thead>
|
||||
<tr>
|
||||
@@ -85,6 +86,7 @@
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</form>
|
||||
|
||||
{# Pagination #}
|
||||
|
||||
+20
-2
@@ -12,22 +12,39 @@
|
||||
{# Webmention endpoint (v1.6.0) #}
|
||||
{% if config.WEBMENTION_ENABLED %}
|
||||
<link rel="webmention" href="/webmention">
|
||||
{% endif %}
|
||||
|
||||
{# WebSub hub and topic on feed pages (v2.10.0) #}
|
||||
{% set websub = websub_links() %}
|
||||
{% if websub %}
|
||||
<link rel="hub" href="{{ websub.hub }}">
|
||||
<link rel="self" href="{{ websub.self }}">
|
||||
{% endif %}
|
||||
|
||||
{# Micropub and IndieAuth discovery (independent of Webmention, v2.8.0) #}
|
||||
<link rel="micropub" href="{{ absolute_url_for('micropub.micropub_endpoint') }}">
|
||||
<link rel="indieauth-metadata" href="{{ absolute_url_for('indieauth.metadata') }}">
|
||||
<link rel="authorization_endpoint" href="{{ absolute_url_for('indieauth.authorize') }}">
|
||||
<link rel="token_endpoint" href="{{ absolute_url_for('indieauth.token') }}">
|
||||
|
||||
{# Author's home page: rel=me makes the h-card linking to it this site's
|
||||
representative h-card; rel=author is the authorship fallback (v2.8.0) #}
|
||||
{% set author_home = (author.url or author.me) if author else config.ADMIN_ME %}
|
||||
{% if author_home %}
|
||||
<link rel="me" href="{{ author_home }}">
|
||||
<link rel="author" href="{{ author_home }}">
|
||||
{% endif %}
|
||||
|
||||
{# rel-me links from discovered author profile (v1.2.0 Phase 2) #}
|
||||
{% if author and author.rel_me_links %}
|
||||
{% for profile_url in author.rel_me_links %}
|
||||
{% for profile_url in author.rel_me_links if profile_url != author_home %}
|
||||
<link rel="me" href="{{ profile_url }}">
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
||||
{% block head %}{% endblock %}
|
||||
</head>
|
||||
<body>
|
||||
<body{% block body_class %}{% endblock %}>
|
||||
{% if config.DEV_MODE %}
|
||||
<div class="dev-mode-warning">
|
||||
WARNING: DEVELOPMENT MODE - Authentication bypassed
|
||||
@@ -38,6 +55,7 @@
|
||||
<a class="site-title" href="/">{{ config.SITE_NAME }}</a>
|
||||
<nav class="site-nav">
|
||||
<a href="/">Home</a>
|
||||
<a href="{{ url_for('public.likes') }}">Likes</a>
|
||||
<a href="{{ url_for('public.feed') }}">RSS</a>
|
||||
{% if g.me %}
|
||||
<a href="{{ url_for('admin.dashboard') }}">Admin</a>
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
{% extends "base.html" %}
|
||||
{% from "partials/note_preview.html" import note_preview %}
|
||||
|
||||
{% block title %}Likes · {{ config.SITE_NAME }}{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div class="h-feed">
|
||||
<h1 class="p-name page-title">Likes</h1>
|
||||
|
||||
{# Feed URL and author, as on the home page (v2.8.0) #}
|
||||
<a class="u-url" href="{{ absolute_url_for(request.endpoint, **request.view_args) }}" hidden>Feed URL</a>
|
||||
{% if author %}
|
||||
<span class="p-author h-card" hidden><a class="p-name u-url" href="{{ author.url or author.me }}">{{ author.name or author.url or author.me }}</a></span>
|
||||
{% endif %}
|
||||
|
||||
{% if notes %}
|
||||
{% for note in notes %}
|
||||
{{ note_preview(note, author) }}
|
||||
{% endfor %}
|
||||
{% else %}
|
||||
<p class="empty-state">Nothing liked yet.</p>
|
||||
{% endif %}
|
||||
|
||||
<nav class="note-nav">
|
||||
<a href="/">← All notes</a>
|
||||
</nav>
|
||||
</div>
|
||||
{% endblock %}
|
||||
+19
-85
@@ -1,12 +1,14 @@
|
||||
{% extends "base.html" %}
|
||||
{% from "partials/media.html" import display_media %}
|
||||
{% from "partials/reaction.html" import reaction %}
|
||||
{% import "partials/responses.html" as responses with context %}
|
||||
|
||||
{% block title %}{% if note.content.strip().startswith('#') %}{{ note.title }}{% else %}{{ note.excerpt|truncate(60) }}{% endif %} · {{ config.SITE_NAME }}{% endblock %}
|
||||
{% block title %}{% if note.has_title %}{{ note.title }}{% elif note.excerpt.strip() %}{{ note.excerpt|truncate(60) }}{% elif note.repost_of %}Repost{% elif note.like_of %}Like{% elif note.bookmark_of %}Bookmark{% elif note.in_reply_to %}Reply{% else %}Photo{% endif %} · {{ config.SITE_NAME }}{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<article class="h-entry">
|
||||
{# Detect if note has explicit title (starts with # heading) - per Q22 #}
|
||||
{% set has_explicit_title = note.content.strip().startswith('#') %}
|
||||
{# Notes starting with a '# Heading' have a title (Q22; Note.has_title) #}
|
||||
{% set has_explicit_title = note.has_title %}
|
||||
|
||||
{# p-name only if note has explicit title (per Q22) #}
|
||||
{% if has_explicit_title %}
|
||||
@@ -17,13 +19,14 @@
|
||||
{% if note.in_reply_to %}
|
||||
<div class="reply-context">
|
||||
<span class="reply-label">In reply to:</span>
|
||||
<blockquote class="h-cite">
|
||||
{# u-in-reply-to on the h-cite itself makes it the entry's property (Post Type Discovery) #}
|
||||
<blockquote class="u-in-reply-to h-cite">
|
||||
{% if reply_context and reply_context.fetch_status == 'fetched' %}
|
||||
{% if reply_context.content_snippet %}
|
||||
<p class="p-content">{{ reply_context.content_snippet }}</p>
|
||||
{% endif %}
|
||||
<cite>
|
||||
<a class="u-in-reply-to" href="{{ note.in_reply_to }}">
|
||||
<a class="u-url" href="{{ note.in_reply_to }}">
|
||||
{% if reply_context.author_name %}{{ reply_context.author_name }}{% endif %}
|
||||
{% if reply_context.site_name %} on {{ reply_context.site_name }}{% endif %}
|
||||
{% if not reply_context.author_name and not reply_context.site_name %}{{ note.in_reply_to }}{% endif %}
|
||||
@@ -31,16 +34,18 @@
|
||||
</cite>
|
||||
{% else %}
|
||||
<cite>
|
||||
<a class="u-in-reply-to" href="{{ note.in_reply_to }}">{{ note.in_reply_to }}</a>
|
||||
<a class="u-url" href="{{ note.in_reply_to }}">{{ note.in_reply_to }}</a>
|
||||
</cite>
|
||||
{% endif %}
|
||||
</blockquote>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{{ reaction(note) }}
|
||||
|
||||
{# u-photo placement: Per draft spec, u-photo must be direct child of h-entry, #}
|
||||
{# NOT inside e-content. Media is rendered ABOVE e-content to meet this requirement. #}
|
||||
{{ display_media(note.media) }}
|
||||
{{ display_media(note.media, note.excerpt|truncate(150)) }}
|
||||
|
||||
{# e-content: note content BELOW media (per ADR-057) #}
|
||||
<div class="e-content">
|
||||
@@ -49,9 +54,9 @@
|
||||
|
||||
<footer class="note-meta">
|
||||
{# u-url and u-uid same for notes (per Q23) #}
|
||||
<a class="u-url u-uid" href="{{ absolute_url_for('public.note', slug=note.slug) }}"><time class="dt-published" datetime="{{ note.created_at.isoformat() }}">{{ note.created_at.strftime('%B %-d, %Y · %-I:%M %p') }}</time></a>
|
||||
<a class="u-url u-uid" href="{{ absolute_url_for('public.note', slug=note.slug) }}"><time class="dt-published" datetime="{{ note.created_at|iso_utc }}">{{ note.created_at.strftime('%B %-d, %Y · %-I:%M %p') }}</time></a>
|
||||
{% if note.updated_at and note.updated_at != note.created_at %}
|
||||
<span class="updated">Updated <time class="dt-updated" datetime="{{ note.updated_at.isoformat() }}">{{ note.updated_at.strftime('%b %-d, %Y') }}</time></span>
|
||||
<span class="updated">Updated <time class="dt-updated" datetime="{{ note.updated_at|iso_utc }}">{{ note.updated_at.strftime('%b %-d, %Y') }}</time></span>
|
||||
{% endif %}
|
||||
{# rel="tag" per microformats2 p-category #}
|
||||
{% if note.tags %}
|
||||
@@ -85,106 +90,35 @@
|
||||
{% if replies %}
|
||||
<div class="webmention-group webmention-replies">
|
||||
<h3>{{ replies|length }} {{ 'Reply' if replies|length == 1 else 'Replies' }}</h3>
|
||||
{% for reply in replies %}
|
||||
<article class="webmention webmention-reply h-cite">
|
||||
<div class="webmention-author">
|
||||
{% if reply.author_photo %}
|
||||
<img src="{{ reply.author_photo }}" alt="" width="48" height="48" loading="lazy">
|
||||
{% endif %}
|
||||
<a class="p-author" href="{{ reply.author_url or reply.source_url }}">
|
||||
{{ reply.author_name or 'Anonymous' }}
|
||||
</a>
|
||||
</div>
|
||||
{% if reply.content_snippet %}
|
||||
<div class="p-content">{{ reply.content_snippet|safe }}</div>
|
||||
{% endif %}
|
||||
<a class="u-url webmention-date" href="{{ reply.source_url }}">
|
||||
<time class="dt-published" datetime="{{ reply.received_at.isoformat() }}">
|
||||
{{ reply.received_at.strftime('%B %d, %Y') }}
|
||||
</time>
|
||||
</a>
|
||||
</article>
|
||||
{% endfor %}
|
||||
{% for wm in replies %}{{ responses.reply(wm) }}{% endfor %}
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{% if likes %}
|
||||
<div class="webmention-group webmention-likes">
|
||||
<h3>{{ likes|length }} {{ 'Like' if likes|length == 1 else 'Likes' }}</h3>
|
||||
<div class="facepile">
|
||||
{% for wm in likes[:8] %}
|
||||
<a href="{{ wm.author_url or wm.source_url }}" class="face" title="{{ wm.author_name or 'Anonymous' }}">
|
||||
{% if wm.author_photo %}
|
||||
<img src="{{ wm.author_photo }}" alt="{{ wm.author_name or '' }}" width="32" height="32" loading="lazy">
|
||||
{% else %}
|
||||
<span class="face-initial" style="background-color: {{ wm.author_url|url_to_color }}">
|
||||
{{ (wm.author_name or '?')[0]|upper }}
|
||||
</span>
|
||||
{% endif %}
|
||||
</a>
|
||||
{% endfor %}
|
||||
{% if likes|length > 8 %}
|
||||
<span class="face-overflow">+{{ likes|length - 8 }} others</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
{{ responses.facepile(likes, 'u-like') }}
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{% if reposts %}
|
||||
<div class="webmention-group webmention-reposts">
|
||||
<h3>{{ reposts|length }} {{ 'Repost' if reposts|length == 1 else 'Reposts' }}</h3>
|
||||
<div class="facepile">
|
||||
{% for wm in reposts[:8] %}
|
||||
<a href="{{ wm.author_url or wm.source_url }}" class="face" title="{{ wm.author_name or 'Anonymous' }}">
|
||||
{% if wm.author_photo %}
|
||||
<img src="{{ wm.author_photo }}" alt="{{ wm.author_name or '' }}" width="32" height="32" loading="lazy">
|
||||
{% else %}
|
||||
<span class="face-initial" style="background-color: {{ wm.author_url|url_to_color }}">
|
||||
{{ (wm.author_name or '?')[0]|upper }}
|
||||
</span>
|
||||
{% endif %}
|
||||
</a>
|
||||
{% endfor %}
|
||||
{% if reposts|length > 8 %}
|
||||
<span class="face-overflow">+{{ reposts|length - 8 }} others</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
{{ responses.facepile(reposts, 'u-repost') }}
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{% if bookmarks %}
|
||||
<div class="webmention-group webmention-bookmarks">
|
||||
<h3>{{ bookmarks|length }} {{ 'Bookmark' if bookmarks|length == 1 else 'Bookmarks' }}</h3>
|
||||
<div class="facepile">
|
||||
{% for wm in bookmarks[:8] %}
|
||||
<a href="{{ wm.author_url or wm.source_url }}" class="face" title="{{ wm.author_name or 'Anonymous' }}">
|
||||
{% if wm.author_photo %}
|
||||
<img src="{{ wm.author_photo }}" alt="{{ wm.author_name or '' }}" width="32" height="32" loading="lazy">
|
||||
{% else %}
|
||||
<span class="face-initial" style="background-color: {{ wm.author_url|url_to_color }}">
|
||||
{{ (wm.author_name or '?')[0]|upper }}
|
||||
</span>
|
||||
{% endif %}
|
||||
</a>
|
||||
{% endfor %}
|
||||
{% if bookmarks|length > 8 %}
|
||||
<span class="face-overflow">+{{ bookmarks|length - 8 }} others</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
{{ responses.facepile(bookmarks, 'u-bookmark') }}
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{% if mentions %}
|
||||
<div class="webmention-group webmention-mentions">
|
||||
<h3>{{ mentions|length }} {{ 'Mention' if mentions|length == 1 else 'Mentions' }}</h3>
|
||||
{% for mention in mentions %}
|
||||
<div class="webmention webmention-mention">
|
||||
<a href="{{ mention.source_url }}">{{ mention.author_name or mention.source_url }}</a>
|
||||
{% if mention.content_snippet %}
|
||||
<p>{{ mention.content_snippet|truncate(200) }}</p>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% endfor %}
|
||||
{% for wm in mentions %}{{ responses.mention(wm) }}{% endfor %}
|
||||
</div>
|
||||
{% endif %}
|
||||
</section>
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
{# Reusable media display macro #}
|
||||
{% macro display_media(media_items) %}
|
||||
{# fallback_alt: for photos without a caption, usually the post's text (v2.6.5) #}
|
||||
{% macro display_media(media_items, fallback_alt='') %}
|
||||
{% if media_items %}
|
||||
<div class="note-media">
|
||||
{% for item in media_items %}
|
||||
<figure class="media-item">
|
||||
<img src="{{ url_for('public.media_file', path=item.path) }}"
|
||||
alt="{{ item.caption or 'Image' }}"
|
||||
alt="{{ item.caption or fallback_alt or 'Image' }}"
|
||||
class="u-photo"
|
||||
loading="lazy">
|
||||
{# No figcaption - caption is for alt text only #}
|
||||
|
||||
@@ -1,14 +1,21 @@
|
||||
{# One note in a list (home, tag pages). v2.3.0 #}
|
||||
{% from "partials/media.html" import display_media %}
|
||||
{% from "partials/reaction.html" import reaction %}
|
||||
{% macro note_preview(note, author) %}
|
||||
<article class="h-entry note-preview">
|
||||
{# Notes starting with a # heading have a title (Q22) #}
|
||||
{% set has_explicit_title = note.content.strip().startswith('#') %}
|
||||
{# Notes starting with a '# Heading' have a title (Q22; Note.has_title) #}
|
||||
{% set has_explicit_title = note.has_title %}
|
||||
{% if has_explicit_title %}
|
||||
<h2 class="p-name"><a href="{{ absolute_url_for('public.note', slug=note.slug) }}">{{ note.title }}</a></h2>
|
||||
{% endif %}
|
||||
|
||||
{{ display_media(note.media) }}
|
||||
{% if note.in_reply_to %}
|
||||
<p class="reaction reply-to"><span class="reaction-verb">↩ In reply to</span> <a class="u-in-reply-to" href="{{ note.in_reply_to }}">{{ note.in_reply_to }}</a></p>
|
||||
{% endif %}
|
||||
|
||||
{{ reaction(note) }}
|
||||
|
||||
{{ display_media(note.media, note.excerpt|truncate(150)) }}
|
||||
|
||||
{% if has_explicit_title %}
|
||||
{# Titled notes are articles: show a summary and link through #}
|
||||
@@ -17,13 +24,14 @@
|
||||
</div>
|
||||
<p class="read-more"><a href="{{ absolute_url_for('public.note', slug=note.slug) }}">Read more →</a></p>
|
||||
{% else %}
|
||||
{# Always present, even empty: without it a text-free photo post gets an implied name (mf2) #}
|
||||
<div class="e-content">
|
||||
{{ note.html|safe }}
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<footer class="note-meta">
|
||||
<a class="u-url" href="{{ absolute_url_for('public.note', slug=note.slug) }}"><time class="dt-published" datetime="{{ note.created_at.isoformat() }}">{{ note.created_at.strftime('%b %-d, %Y') }}</time></a>
|
||||
<a class="u-url u-uid" href="{{ absolute_url_for('public.note', slug=note.slug) }}"><time class="dt-published" datetime="{{ note.created_at|iso_utc }}">{{ note.created_at.strftime('%b %-d, %Y') }}</time></a>
|
||||
{% if note.tags %}
|
||||
<span class="note-tags">
|
||||
{% for tag in note.tags %}<a class="p-category" rel="tag" href="{{ url_for('public.tag', tag=tag.name) }}">{{ tag.display_name }}</a>{% endfor %}
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
{# "Liked" / "Reposted" line for likes and reposts (v2.5.0) #}
|
||||
{% macro reaction(note) %}
|
||||
{% if note.reacts_to %}
|
||||
{% set ctx = note.context if note.context is defined else None %}
|
||||
{% if ctx and ctx.title %}{% set label = ctx.title %}{% elif ctx and ctx.content_snippet %}{% set label = ctx.content_snippet|striptags|truncate(90) %}{% elif ctx and (ctx.author_name or ctx.site_name) %}{% set label = 'a post' %}{% else %}{% set label = note.reacts_to %}{% endif %}
|
||||
<p class="reaction reaction-{{ note.kind }}">
|
||||
<span class="reaction-verb">{{ {'like': '★ Liked', 'repost': '🔁 Reposted', 'bookmark': '🔖 Bookmarked'}[note.kind] }}</span>
|
||||
<a class="u-{{ note.kind }}-of" href="{{ note.reacts_to }}">{{ label }}</a>
|
||||
{% if ctx and (ctx.author_name or ctx.site_name) %}<span class="reaction-by">by {{ ctx.author_name or ctx.site_name }}</span>{% endif %}
|
||||
</p>
|
||||
{% endif %}
|
||||
{% endmacro %}
|
||||
@@ -0,0 +1,62 @@
|
||||
{# Received webmentions as microformats2 responses (v2.8.0)
|
||||
Each response is an h-cite in a property of the note's h-entry:
|
||||
u-comment (replies), u-like, u-repost, u-bookmark, u-mention.
|
||||
See https://indieweb.org/comments-presentation and microformats.org/wiki/h-entry #}
|
||||
|
||||
{# When the source said when it was posted, use that; else when we received it #}
|
||||
{% macro response_time(wm, fmt='%B %-d, %Y') %}
|
||||
{% set when = wm.source_published or wm.received_at %}
|
||||
<time class="dt-published" datetime="{{ when|iso_utc }}">{{ when.strftime(fmt) if when.strftime is defined else when }}</time>
|
||||
{%- endmacro %}
|
||||
|
||||
{# The response's own URL: the source's u-url if it gave one #}
|
||||
{% macro response_url(wm) %}{{ wm.source_post_url or wm.source_url }}{% endmacro %}
|
||||
|
||||
{# Author h-card: name, URL and photo. u-url is on an inner link: on the
|
||||
h-card element itself it would belong to the enclosing h-cite #}
|
||||
{% macro author_card(wm, class='', photo_size=48, show_name=True) %}
|
||||
<span class="p-author h-card {{ class }}"><a class="u-url" href="{{ wm.author_url or wm.source_url }}" title="{{ wm.author_name or 'Anonymous' }}">
|
||||
{%- if wm.author_photo %}<img class="u-photo" src="{{ wm.author_photo }}" alt="" width="{{ photo_size }}" height="{{ photo_size }}" loading="lazy">
|
||||
{%- elif not show_name %}<span class="face-initial" style="background-color: {{ wm.author_url|url_to_color }}" aria-hidden="true">{{ (wm.author_name or '?')[0]|upper }}</span>
|
||||
{%- endif %}
|
||||
<span class="p-name{{ '' if show_name else ' visually-hidden' }}">{{ wm.author_name or 'Anonymous' }}</span>
|
||||
</a></span>
|
||||
{%- endmacro %}
|
||||
|
||||
{# A reply, shown as a comment #}
|
||||
{% macro reply(wm) %}
|
||||
<article class="webmention webmention-reply u-comment h-cite">
|
||||
<div class="webmention-author">{{ author_card(wm) }}</div>
|
||||
{% if wm.content_snippet %}
|
||||
<div class="e-content">{{ wm.content_snippet|safe }}</div>
|
||||
{% endif %}
|
||||
<a class="u-url webmention-date" href="{{ response_url(wm) }}">{{ response_time(wm) }}</a>
|
||||
</article>
|
||||
{%- endmacro %}
|
||||
|
||||
{# Likes, reposts or bookmarks as a facepile; prop is u-like / u-repost / u-bookmark.
|
||||
Faces past `shown` are hidden on screen but kept for parsers. #}
|
||||
{% macro facepile(items, prop, shown=8) %}
|
||||
<div class="facepile">
|
||||
{% for wm in items %}
|
||||
<span class="face {{ prop }} h-cite"{% if loop.index > shown %} hidden{% endif %}>
|
||||
{{ author_card(wm, photo_size=32, show_name=False) }}
|
||||
<a class="u-url" href="{{ response_url(wm) }}" hidden>{{ response_time(wm) }}</a>
|
||||
</span>
|
||||
{% endfor %}
|
||||
{% if items|length > shown %}
|
||||
<span class="face-overflow">+{{ items|length - shown }} others</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
{%- endmacro %}
|
||||
|
||||
{# A post that mentions this note without replying #}
|
||||
{% macro mention(wm) %}
|
||||
<div class="webmention webmention-mention u-mention h-cite">
|
||||
{{ author_card(wm, photo_size=24) }}
|
||||
{% if wm.content_snippet %}
|
||||
<p class="p-content">{{ wm.content_snippet|striptags|truncate(200) }}</p>
|
||||
{% endif %}
|
||||
<a class="u-url webmention-date" href="{{ response_url(wm) }}">{{ response_time(wm) }}</a>
|
||||
</div>
|
||||
{%- endmacro %}
|
||||
@@ -7,6 +7,12 @@
|
||||
<div class="h-feed">
|
||||
<h1 class="p-name page-title">Notes tagged <span class="tag-name">{{ tag.display_name }}</span></h1>
|
||||
|
||||
{# Feed URL and author, as on the home page (v2.8.0) #}
|
||||
<a class="u-url" href="{{ absolute_url_for(request.endpoint, **request.view_args) }}" hidden>Feed URL</a>
|
||||
{% if author %}
|
||||
<span class="p-author h-card" hidden><a class="p-name u-url" href="{{ author.url or author.me }}">{{ author.name or author.url or author.me }}</a></span>
|
||||
{% endif %}
|
||||
|
||||
{% if notes %}
|
||||
{% for note in notes %}
|
||||
{{ note_preview(note, author) }}
|
||||
|
||||
+1
-1
@@ -230,7 +230,7 @@ class TestGenerateFeed:
|
||||
|
||||
# Check for Atom namespace and self-link
|
||||
assert "atom" in feed_xml
|
||||
assert "feed.xml" in feed_xml
|
||||
assert "feed.rss" in feed_xml # self link (v2.10.0: /feed.rss is the RSS topic)
|
||||
assert 'rel="self"' in feed_xml
|
||||
|
||||
def test_generate_feed_item_structure(self, app, sample_notes):
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
"""
|
||||
Likes and reposts (v2.5.0, ADR-068)
|
||||
|
||||
Fake GoToSocial and Bluesky servers check that likes/reposts of their own
|
||||
posts become native favourites/boosts/likes/reposts, and everything else
|
||||
falls back (reposts) or is skipped (likes).
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from starpunk import create_app, syndication
|
||||
from starpunk.indieauth import create_access_token
|
||||
from starpunk.notes import delete_note, get_note
|
||||
from starpunk.syndication import BlueskyClient, MastodonClient, get_note_syndications, process_due
|
||||
|
||||
SITE = "https://blog.example.com"
|
||||
GTS = "https://social.example.com"
|
||||
FEDI_POST = "https://mastodon.example/@alice/111"
|
||||
BSKY_POST = "https://bsky.app/profile/alice.example/post/3abc"
|
||||
WEB_PAGE = "https://someblog.example/2026/a-post"
|
||||
|
||||
|
||||
class FakeGTS:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
self.statuses = {}
|
||||
|
||||
def __call__(self, request):
|
||||
path = request.url.path
|
||||
self.calls.append((request.method, path))
|
||||
if path == "/api/v2/search":
|
||||
q = request.url.params["q"]
|
||||
return httpx.Response(200, json={"statuses": [{"id": "s1"}] if q == FEDI_POST else []})
|
||||
if path in ("/api/v1/statuses/s1/favourite", "/api/v1/statuses/s1/unfavourite",
|
||||
"/api/v1/statuses/s1/reblog", "/api/v1/statuses/s1/unreblog"):
|
||||
return httpx.Response(200, json={"id": "s1"})
|
||||
if path == "/api/v1/statuses" and request.method == "POST":
|
||||
sid = f"p{len(self.statuses) + 1}"
|
||||
self.statuses[sid] = json.loads(request.content)
|
||||
return httpx.Response(200, json={"id": sid, "url": f"{GTS}/@phil/statuses/{sid}"})
|
||||
if path.startswith("/api/v1/statuses/") and request.method == "DELETE":
|
||||
self.statuses.pop(path.rsplit("/", 1)[1], None)
|
||||
return httpx.Response(200, json={})
|
||||
return httpx.Response(500)
|
||||
|
||||
|
||||
class FakePDS:
|
||||
def __init__(self):
|
||||
self.records = {}
|
||||
self.calls = []
|
||||
|
||||
def __call__(self, request):
|
||||
nsid = request.url.path.removeprefix("/xrpc/")
|
||||
self.calls.append(nsid)
|
||||
if nsid == "com.atproto.server.createSession":
|
||||
return httpx.Response(200, json={"did": "did:plc:me", "handle": "me.example",
|
||||
"accessJwt": "a", "refreshJwt": "r"})
|
||||
if nsid == "com.atproto.identity.resolveHandle":
|
||||
return httpx.Response(200, json={"did": "did:plc:alice"})
|
||||
if nsid == "app.bsky.feed.getPosts":
|
||||
return httpx.Response(200, json={"posts": [{"uri": request.url.params["uris"], "cid": "cid1"}]})
|
||||
if nsid == "com.atproto.repo.createRecord":
|
||||
body = json.loads(request.content)
|
||||
uri = f"at://did:plc:me/{body['collection']}/r{len(self.records) + 1}"
|
||||
self.records[uri] = body["record"]
|
||||
return httpx.Response(200, json={"uri": uri, "cid": "c"})
|
||||
if nsid == "com.atproto.repo.deleteRecord":
|
||||
body = json.loads(request.content)
|
||||
self.records.pop(f"at://{body['repo']}/{body['collection']}/{body['rkey']}", None)
|
||||
return httpx.Response(200, json={})
|
||||
return httpx.Response(500, json={"error": nsid})
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def gts():
|
||||
return FakeGTS()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def pds():
|
||||
return FakePDS()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def app(tmp_path, gts, pds):
|
||||
data = tmp_path / "data"
|
||||
app = create_app({
|
||||
"TESTING": True, "SITE_URL": SITE, "ADMIN_ME": "https://example.com", "SESSION_SECRET": "x",
|
||||
"DATA_PATH": data, "NOTES_PATH": data / "notes", "DATABASE_PATH": data / "starpunk.db",
|
||||
"GOTOSOCIAL_URL": GTS, "GOTOSOCIAL_ACCESS_TOKEN": "t",
|
||||
"BLUESKY_HANDLE": "me.example", "BLUESKY_APP_PASSWORD": "p", "BLUESKY_SERVICE": "https://pds.example",
|
||||
"FEED_CACHE_ENABLED": False,
|
||||
})
|
||||
# The feed's note list is cached at module level across apps
|
||||
from starpunk.routes import public
|
||||
public._feed_cache["notes"] = None
|
||||
public._feed_cache["timestamp"] = None
|
||||
syndication._client_cache[GTS] = MastodonClient(GTS, "t", http=httpx.Client(transport=httpx.MockTransport(gts)))
|
||||
syndication._client_cache["bluesky:https://pds.example:me.example"] = BlueskyClient(
|
||||
"https://pds.example", "me.example", "p", http=httpx.Client(transport=httpx.MockTransport(pds)))
|
||||
yield app
|
||||
syndication._client_cache.clear()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def post(app, monkeypatch):
|
||||
# Don't fetch contexts from the network
|
||||
monkeypatch.setattr("starpunk.reply_context.fetch_reply_context", lambda *a, **k: None)
|
||||
with app.test_request_context():
|
||||
token = create_access_token("https://quill.example/", "create update delete")["access_token"]
|
||||
client = app.test_client()
|
||||
|
||||
def _post(**props):
|
||||
# Crosspost to every configured target; absent means none (v2.9.0).
|
||||
# Sent inside properties, as JSON clients like Quill do.
|
||||
with app.app_context():
|
||||
from starpunk.syndication import get_targets
|
||||
props["mp-syndicate-to"] = [t.uid for t in get_targets().values()]
|
||||
response = client.post("/micropub", json={"type": ["h-entry"], "properties": props},
|
||||
headers={"Authorization": f"Bearer {token}"})
|
||||
assert response.status_code == 201, response.get_json()
|
||||
location = response.headers["Location"]
|
||||
with app.app_context():
|
||||
note = get_note(slug=location.rsplit("/", 1)[1])
|
||||
process_due(limit=10)
|
||||
rows = {r["target"]: r for r in get_note_syndications(note.id)}
|
||||
return note, rows
|
||||
return _post
|
||||
|
||||
|
||||
def page(app, path):
|
||||
return app.test_client().get(path).data.decode()
|
||||
|
||||
|
||||
class TestSite:
|
||||
def test_like_without_comment(self, app, post):
|
||||
note, _ = post(**{"like-of": [WEB_PAGE]})
|
||||
assert note.kind == "like" and note.content == ""
|
||||
assert note.slug not in page(app, "/")
|
||||
likes = page(app, "/likes")
|
||||
assert f'class="u-like-of" href="{WEB_PAGE}"' in likes and "★ Liked" in likes
|
||||
assert note.slug not in page(app, "/feed.json")
|
||||
|
||||
def test_repost_in_stream_and_feed(self, app, post):
|
||||
note, _ = post(**{"repost-of": [WEB_PAGE], "content": ["Worth a read"]})
|
||||
home = page(app, "/")
|
||||
assert f'class="u-repost-of" href="{WEB_PAGE}"' in home and "Worth a read" in home
|
||||
feed = json.loads(page(app, "/feed.json"))
|
||||
item = next(i for i in feed["items"] if note.slug in i["url"])
|
||||
assert "Reposted" in item["content_html"] and WEB_PAGE in item["content_html"]
|
||||
|
||||
def test_webmention_goes_to_target_with_clean_url(self, app, post, monkeypatch):
|
||||
sent = {}
|
||||
monkeypatch.setattr("starpunk.webmention_sender.queue_webmentions_for_note",
|
||||
lambda **kw: sent.update(kw) or 0)
|
||||
app.config["WEBMENTION_ENABLED"] = True
|
||||
note, _ = post(**{"like-of": [WEB_PAGE]})
|
||||
assert sent["in_reply_to"] == WEB_PAGE
|
||||
assert sent["note_url"] == f"{SITE}/note/{note.slug}" # no double slash
|
||||
|
||||
def test_both_kinds_rejected(self, app, post):
|
||||
with pytest.raises(AssertionError):
|
||||
post(**{"like-of": [WEB_PAGE], "repost-of": [WEB_PAGE]})
|
||||
|
||||
|
||||
class TestGoToSocial:
|
||||
def test_like_of_fediverse_post_favourites(self, app, post, gts):
|
||||
note, rows = post(**{"like-of": [FEDI_POST]})
|
||||
assert rows["gotosocial"]["remote_id"] == "favourite:s1"
|
||||
assert rows["gotosocial"]["remote_url"] is None
|
||||
with app.test_request_context():
|
||||
delete_note(id=note.id)
|
||||
process_due(limit=10)
|
||||
assert ("POST", "/api/v1/statuses/s1/unfavourite") in gts.calls
|
||||
|
||||
def test_like_of_web_page_is_not_crossposted(self, app, post, gts):
|
||||
_, rows = post(**{"like-of": [WEB_PAGE]})
|
||||
assert rows["gotosocial"]["state"] == "synced" and rows["gotosocial"]["remote_id"] is None
|
||||
assert gts.statuses == {}
|
||||
|
||||
def test_repost_boosts(self, app, post, gts):
|
||||
_, rows = post(**{"repost-of": [FEDI_POST]})
|
||||
assert rows["gotosocial"]["remote_id"] == "reblog:s1"
|
||||
|
||||
def test_repost_with_comment_posts_status(self, app, post, gts):
|
||||
_, rows = post(**{"repost-of": [FEDI_POST], "content": ["So true"]})
|
||||
(status,) = gts.statuses.values()
|
||||
assert status["status"].startswith("So true") and FEDI_POST in status["status"]
|
||||
assert ("POST", "/api/v1/statuses/s1/reblog") not in gts.calls
|
||||
|
||||
def test_repost_of_web_page_posts_link(self, app, post, gts):
|
||||
post(**{"repost-of": [WEB_PAGE]})
|
||||
(status,) = gts.statuses.values()
|
||||
assert status["status"].startswith("🔁") and WEB_PAGE in status["status"]
|
||||
|
||||
|
||||
class TestBluesky:
|
||||
def test_like_of_bluesky_post(self, app, post, pds):
|
||||
note, rows = post(**{"like-of": [BSKY_POST]})
|
||||
uri = rows["bluesky"]["remote_id"]
|
||||
assert "/app.bsky.feed.like/" in uri
|
||||
assert pds.records[uri]["subject"] == {"uri": "at://did:plc:alice/app.bsky.feed.post/3abc", "cid": "cid1"}
|
||||
with app.test_request_context():
|
||||
delete_note(id=note.id)
|
||||
process_due(limit=10)
|
||||
assert uri not in pds.records
|
||||
|
||||
def test_like_of_web_page_skipped(self, app, post, pds):
|
||||
_, rows = post(**{"like-of": [WEB_PAGE]})
|
||||
assert rows["bluesky"]["remote_id"] is None and pds.records == {}
|
||||
|
||||
def test_repost_and_quote(self, app, post, pds):
|
||||
_, rows = post(**{"repost-of": [BSKY_POST]})
|
||||
assert "/app.bsky.feed.repost/" in rows["bluesky"]["remote_id"]
|
||||
_, rows = post(**{"repost-of": [BSKY_POST], "content": ["Great thread"]})
|
||||
record = pds.records[rows["bluesky"]["remote_id"]]
|
||||
assert record["text"] == "Great thread"
|
||||
assert record["embed"] == {"$type": "app.bsky.embed.record",
|
||||
"record": {"uri": "at://did:plc:alice/app.bsky.feed.post/3abc", "cid": "cid1"}}
|
||||
|
||||
def test_repost_of_web_page_posts_link_card(self, app, post, pds):
|
||||
_, rows = post(**{"repost-of": [WEB_PAGE]})
|
||||
record = pds.records[rows["bluesky"]["remote_id"]]
|
||||
assert record["embed"]["external"]["uri"] == WEB_PAGE
|
||||
assert record["text"].startswith("🔁")
|
||||
+10
-15
@@ -110,7 +110,7 @@ class TestImageValidation:
|
||||
assert mime_type == 'image/jpeg'
|
||||
assert width == 800
|
||||
assert height == 600
|
||||
assert file_data == image_data # No conversion
|
||||
assert isinstance(file_data, Image.Image) # validate_image returns a PIL Image (v1.5.0)
|
||||
|
||||
def test_valid_png(self):
|
||||
"""Test validation of valid PNG image"""
|
||||
@@ -120,7 +120,7 @@ class TestImageValidation:
|
||||
assert mime_type == 'image/png'
|
||||
assert width == 800
|
||||
assert height == 600
|
||||
assert file_data == image_data # No conversion
|
||||
assert isinstance(file_data, Image.Image) # validate_image returns a PIL Image (v1.5.0)
|
||||
|
||||
def test_valid_gif(self):
|
||||
"""Test validation of valid GIF image"""
|
||||
@@ -130,7 +130,7 @@ class TestImageValidation:
|
||||
assert mime_type == 'image/gif'
|
||||
assert width == 800
|
||||
assert height == 600
|
||||
assert file_data == image_data # No conversion
|
||||
assert isinstance(file_data, Image.Image) # validate_image returns a PIL Image (v1.5.0)
|
||||
|
||||
def test_valid_webp(self):
|
||||
"""Test validation of valid WebP image"""
|
||||
@@ -140,7 +140,7 @@ class TestImageValidation:
|
||||
assert mime_type == 'image/webp'
|
||||
assert width == 800
|
||||
assert height == 600
|
||||
assert file_data == image_data # No conversion
|
||||
assert isinstance(file_data, Image.Image) # validate_image returns a PIL Image (v1.5.0)
|
||||
|
||||
def test_file_too_large(self):
|
||||
"""Test rejection of >10MB file (per Q6)"""
|
||||
@@ -168,7 +168,7 @@ class TestImageValidation:
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
validate_image(corrupted_data, 'corrupt.jpg')
|
||||
|
||||
assert "Invalid or corrupted" in str(exc_info.value)
|
||||
assert "Not a supported image" in str(exc_info.value)
|
||||
|
||||
|
||||
class TestHEICSupport:
|
||||
@@ -183,11 +183,8 @@ class TestHEICSupport:
|
||||
assert mime_type == 'image/jpeg'
|
||||
assert width == 800
|
||||
assert height == 600
|
||||
# file_data should be different (converted to JPEG)
|
||||
assert file_data != heic_data
|
||||
# Verify it's actually JPEG by opening it
|
||||
img = Image.open(io.BytesIO(file_data))
|
||||
assert img.format == 'JPEG'
|
||||
# Converted to JPEG (validate_image returns a PIL Image)
|
||||
assert file_data.format == 'JPEG'
|
||||
|
||||
def test_heic_with_rgba_mode(self):
|
||||
"""Test HEIC with alpha channel is converted to RGB JPEG"""
|
||||
@@ -209,8 +206,7 @@ class TestHEICSupport:
|
||||
# Should be converted to JPEG (no alpha)
|
||||
assert mime_type == 'image/jpeg'
|
||||
# Verify it's RGB (no alpha)
|
||||
img = Image.open(io.BytesIO(file_data))
|
||||
assert img.mode == 'RGB'
|
||||
assert file_data.mode == 'RGB'
|
||||
|
||||
def test_heic_dimensions_preserved(self):
|
||||
"""Test HEIC conversion preserves dimensions"""
|
||||
@@ -273,9 +269,8 @@ class TestMPOSupport:
|
||||
assert mime_type == 'image/jpeg'
|
||||
assert width == 800
|
||||
assert height == 600
|
||||
# Verify it's actually JPEG by opening it
|
||||
img = Image.open(io.BytesIO(file_data))
|
||||
assert img.format == 'JPEG'
|
||||
# Converted to JPEG (validate_image returns a PIL Image)
|
||||
assert file_data.format == 'JPEG'
|
||||
# Note: MPO with single frame may have identical bytes to JPEG
|
||||
# The important part is the format detection and MIME type correction
|
||||
|
||||
|
||||
@@ -289,16 +289,20 @@ class TestRelMe:
|
||||
assert 'https://github.com/testuser' in rels['me']
|
||||
assert 'https://mastodon.social/@testuser' in rels['me']
|
||||
|
||||
def test_no_relme_without_author(self, client, app):
|
||||
"""No rel=me links if no author profile"""
|
||||
def test_relme_to_admin_me_without_author(self, client, app):
|
||||
"""Without a discovered profile, rel=me and rel=author still point at ADMIN_ME (v2.8.0)
|
||||
|
||||
The home page's p-author h-card links to the author's site; rel=me to
|
||||
the same URL makes it the representative h-card.
|
||||
"""
|
||||
with patch('starpunk.author_discovery.get_author_profile', return_value=None):
|
||||
response = client.get('/')
|
||||
|
||||
parsed = mf2py.parse(doc=response.data.decode(), url='http://localhost/')
|
||||
|
||||
rels = parsed.get('rels', {})
|
||||
# Should not have rel=me, or it should be empty
|
||||
assert len(rels.get('me', [])) == 0, "Should not have rel=me without author"
|
||||
assert rels.get('me') == [app.config['ADMIN_ME']]
|
||||
assert rels.get('author') == [app.config['ADMIN_ME']]
|
||||
|
||||
|
||||
class TestV130Microformats:
|
||||
|
||||
@@ -0,0 +1,268 @@
|
||||
"""
|
||||
Microformats2 fixes (v2.8.0)
|
||||
|
||||
Pages are rendered and parsed with mf2py; assertions are on the parsed data,
|
||||
which is what IndieWeb readers and Webmention receivers see.
|
||||
"""
|
||||
|
||||
import re
|
||||
import sqlite3
|
||||
from unittest.mock import patch
|
||||
|
||||
import mf2py
|
||||
import pytest
|
||||
|
||||
from starpunk import mf2_utils
|
||||
from starpunk.models import split_title
|
||||
from starpunk.webmention_processor import extract_author, extract_source_details
|
||||
from starpunk.webmention_worker import WebmentionWorker
|
||||
|
||||
BASE = 'http://localhost:5000'
|
||||
|
||||
|
||||
def parse(client, path):
|
||||
response = client.get(path)
|
||||
assert response.status_code == 200
|
||||
return mf2py.parse(doc=response.data.decode(), url=BASE + path)
|
||||
|
||||
|
||||
def entry_of(parsed):
|
||||
return next(i for i in parsed['items'] if 'h-entry' in i['type'])
|
||||
|
||||
|
||||
def make_note(app, content, **kw):
|
||||
from starpunk.notes import create_note
|
||||
with app.test_request_context():
|
||||
return create_note(content, published=True, **kw)
|
||||
|
||||
|
||||
def add_mention(app, note, kind, n, photo=True, **cols):
|
||||
db = sqlite3.connect(app.config['DATABASE_PATH'])
|
||||
fields = {
|
||||
'source_url': f'https://src.example/{kind}/{n}',
|
||||
'target_url': f'{BASE}/note/{note.slug}',
|
||||
'target_note_id': note.id,
|
||||
'mention_type': kind,
|
||||
'author_name': f'{kind.title()} Person {n}',
|
||||
'author_url': f'https://{kind}{n}.example/',
|
||||
'author_photo': f'https://{kind}{n}.example/me.jpg' if photo else None,
|
||||
'content_snippet': f'<p>{kind} text <a href="https://x.example/">link</a></p>',
|
||||
'validation_status': 'valid',
|
||||
'moderation_status': 'approved',
|
||||
**cols,
|
||||
}
|
||||
db.execute(f"INSERT INTO webmentions ({', '.join(fields)}) VALUES ({', '.join('?' * len(fields))})",
|
||||
tuple(fields.values()))
|
||||
db.commit()
|
||||
db.close()
|
||||
|
||||
|
||||
# A1/A2: received responses are properties of the entry
|
||||
|
||||
def test_responses_are_entry_properties(app, client):
|
||||
note = make_note(app, "A note")
|
||||
add_mention(app, note, 'reply', 1, source_post_url='https://src.example/reply/1#canonical',
|
||||
source_published='2026-11-01T10:00:00Z')
|
||||
for n in range(10):
|
||||
add_mention(app, note, 'like', n, photo=n % 2 == 0)
|
||||
add_mention(app, note, 'repost', 1)
|
||||
add_mention(app, note, 'bookmark', 1)
|
||||
add_mention(app, note, 'mention', 1)
|
||||
|
||||
props = entry_of(parse(client, f'/note/{note.slug}'))['properties']
|
||||
assert 'children' not in entry_of(parse(client, f'/note/{note.slug}'))
|
||||
|
||||
(comment,) = props['comment']
|
||||
assert comment['type'] == ['h-cite']
|
||||
author = comment['properties']['author'][0]
|
||||
assert author['type'] == ['h-card']
|
||||
assert author['properties']['name'] == ['Reply Person 1']
|
||||
assert author['properties']['url'] == ['https://reply1.example/']
|
||||
assert author['properties']['photo'][0] in ('https://reply1.example/me.jpg',
|
||||
{'value': 'https://reply1.example/me.jpg', 'alt': ''})
|
||||
assert '<a href="https://x.example/"' in comment['properties']['content'][0]['html']
|
||||
# the source's own URL and published time, not ours
|
||||
assert comment['properties']['url'] == ['https://src.example/reply/1#canonical']
|
||||
assert comment['properties']['published'] == ['2026-11-01T10:00:00Z']
|
||||
|
||||
# all ten likes, including the ones past the visible eight
|
||||
assert len(props['like']) == 10
|
||||
names = {like['properties']['author'][0]['properties']['name'][0] for like in props['like']}
|
||||
assert names == {f'Like Person {n}' for n in range(10)}
|
||||
assert props['like'][0]['properties']['url'] == ['https://src.example/like/0']
|
||||
|
||||
assert props['repost'][0]['properties']['author'][0]['properties']['name'] == ['Repost Person 1']
|
||||
assert props['bookmark'][0]['type'] == ['h-cite']
|
||||
assert props['mention'][0]['properties']['author'][0]['properties']['url'] == ['https://mention1.example/']
|
||||
|
||||
|
||||
# A3: dt-* are RFC 3339 UTC
|
||||
|
||||
UTC = re.compile(r'^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ$')
|
||||
|
||||
|
||||
def test_dates_are_utc(app, client):
|
||||
from starpunk.notes import update_note
|
||||
note = make_note(app, "Dated")
|
||||
with app.test_request_context():
|
||||
update_note(id=note.id, content="Dated, edited")
|
||||
add_mention(app, note, 'reply', 1)
|
||||
|
||||
props = entry_of(parse(client, f'/note/{note.slug}'))['properties']
|
||||
assert UTC.match(props['published'][0]), props['published']
|
||||
assert UTC.match(props['updated'][0]), props['updated']
|
||||
assert UTC.match(props['comment'][0]['properties']['published'][0])
|
||||
|
||||
home = parse(client, '/')['items'][0]['children'][0]['properties']
|
||||
assert UTC.match(home['published'][0])
|
||||
|
||||
|
||||
def test_micropub_source_published_is_utc():
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from starpunk.utils import iso_utc
|
||||
assert iso_utc(datetime(2026, 11, 1, 16, 25, 12, 921768)) == '2026-11-01T16:25:12Z'
|
||||
assert iso_utc(datetime(2026, 11, 1, 9, 0, tzinfo=timezone(timedelta(hours=-7)))) == '2026-11-01T16:00:00Z'
|
||||
assert iso_utc('2026-11-01T16:25:15+0000') == '2026-11-01T16:25:15Z'
|
||||
|
||||
|
||||
# A4: only a real Markdown heading is a title
|
||||
|
||||
@pytest.mark.parametrize("content,title", [
|
||||
("# Title\n\nBody", "Title"),
|
||||
("## Sub title ##\n\nBody", "Sub title"),
|
||||
("#indieweb is great", None),
|
||||
("#\n\nbody text", None),
|
||||
("####### seven", None),
|
||||
("Plain note", None),
|
||||
])
|
||||
def test_split_title(content, title):
|
||||
assert split_title(content)[0] == title
|
||||
|
||||
|
||||
@pytest.mark.parametrize("content,kind", [
|
||||
("#indieweb is great, posting from my phone", "note"),
|
||||
("#\n\nbody text", "note"),
|
||||
("# A Title\n\nThe body", "article"),
|
||||
])
|
||||
def test_post_type_of_hash_notes(app, client, content, kind):
|
||||
note = make_note(app, content)
|
||||
for path in (f'/note/{note.slug}', '/'):
|
||||
parsed = parse(client, path)
|
||||
entry = entry_of(parsed) if path != '/' else parsed['items'][0]['children'][0]
|
||||
props = entry['properties']
|
||||
if kind == 'note':
|
||||
assert 'name' not in props
|
||||
assert props['content'][0]['value'].strip()
|
||||
else:
|
||||
assert props['name'] == ['A Title']
|
||||
|
||||
|
||||
# A5: the note's own syndicated copy is not a response
|
||||
|
||||
def test_own_syndicated_copy_rejected(app):
|
||||
note = make_note(app, "Crossposted")
|
||||
db = sqlite3.connect(app.config['DATABASE_PATH'])
|
||||
db.execute("INSERT INTO syndications (note_id, target, remote_url) VALUES (?, 'gotosocial', ?)",
|
||||
(note.id, 'https://social.example/@me/statuses/1'))
|
||||
db.execute("INSERT INTO webmentions (source_url, target_url, target_note_id) VALUES (?, ?, ?)",
|
||||
('https://brid.gy/post/mastodon/@someone/1', f'{BASE}/note/{note.slug}', note.id))
|
||||
db.commit()
|
||||
|
||||
result = {'valid': True, 'mention_type': 'mention', 'author_name': 'Me', 'author_url': 'https://me.example/',
|
||||
'author_photo': None, 'content_snippet': 'x',
|
||||
'source_post_url': 'https://social.example/@me/statuses/1', 'source_uid': None,
|
||||
'source_published': None}
|
||||
worker = WebmentionWorker(app, db_path=app.config['DATABASE_PATH'], site_url=BASE)
|
||||
with patch("starpunk.webmention_worker.process_webmention", return_value=result):
|
||||
worker._validate_webmention(1, 'https://brid.gy/post/mastodon/@someone/1', 'x', note.id)
|
||||
|
||||
status, error = db.execute("SELECT validation_status, validation_error FROM webmentions").fetchone()
|
||||
assert status == 'invalid' and 'own syndicated copy' in error
|
||||
|
||||
|
||||
# B1/B3: discovery links
|
||||
|
||||
def test_head_links_without_webmention(app, client):
|
||||
app.config['WEBMENTION_ENABLED'] = False
|
||||
rels = parse(client, '/')['rels']
|
||||
assert 'webmention' not in rels
|
||||
for rel in ('micropub', 'indieauth-metadata', 'authorization_endpoint', 'token_endpoint'):
|
||||
assert rels.get(rel), rel
|
||||
|
||||
|
||||
def test_home_has_representative_hcard(app, client):
|
||||
# Fresh install: discovery fails, the fallback profile is used
|
||||
parsed = parse(client, '/')
|
||||
card = mf2_utils.representative_hcard(parsed, BASE + '/')
|
||||
assert card and card['properties']['url'] == [app.config['ADMIN_ME']]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ['/likes', '/tag/python'])
|
||||
def test_list_feeds_have_url_and_author(app, client, path):
|
||||
make_note(app, "Tagged", tags=["python"])
|
||||
feed = parse(client, path)['items'][0]
|
||||
assert feed['type'] == ['h-feed']
|
||||
assert feed['properties']['url'] == [BASE + path]
|
||||
assert feed['properties']['author'][0]['type'] == ['h-card']
|
||||
|
||||
|
||||
# B2: representative h-card
|
||||
|
||||
def _cards(html, url='https://me.example/'):
|
||||
return mf2py.parse(doc=html, url=url)
|
||||
|
||||
|
||||
def test_representative_hcard_rules():
|
||||
other = '<div class="h-card"><a class="p-name u-url" href="https://bob.example/">Bob</a></div>'
|
||||
# someone else's h-card only: none
|
||||
assert mf2_utils.representative_hcard(_cards(other), 'https://me.example/') is None
|
||||
# uid + url
|
||||
me = '<div class="h-card"><a class="p-name u-url u-uid" href="/">Me</a></div>'
|
||||
assert mf2_utils.first(mf2_utils.representative_hcard(_cards(other + me), 'https://me.example/'), 'name') == 'Me'
|
||||
# rel=me
|
||||
relme = ('<a rel="me" href="https://social.example/@me">x</a>'
|
||||
'<div class="h-card"><a class="p-name u-url" href="https://social.example/@me">Me</a></div>')
|
||||
assert mf2_utils.first(mf2_utils.representative_hcard(_cards(other + relme), 'https://me.example/'), 'name') == 'Me'
|
||||
# only one h-card, url = page
|
||||
single = '<div class="h-card"><a class="p-name u-url" href="/">Me</a></div>'
|
||||
assert mf2_utils.first(mf2_utils.representative_hcard(_cards(single), 'https://me.example/'), 'name') == 'Me'
|
||||
|
||||
|
||||
# B4: the responding entry and its author
|
||||
|
||||
FEED_PAGE = '''
|
||||
<div class="h-feed">
|
||||
<div class="p-author h-card"><a class="p-name u-url" href="https://feed.example/">Feed Owner</a></div>
|
||||
<div class="h-entry"><a class="u-url" href="https://feed.example/1">one</a>
|
||||
<div class="e-content">unrelated</div></div>
|
||||
<div class="h-entry"><a class="u-url" href="https://feed.example/2">two</a>
|
||||
<time class="dt-published" datetime="2026-11-01T09:00:00-07:00">x</time>
|
||||
<a class="u-in-reply-to" href="https://me.example/note/a">re</a>
|
||||
<div class="e-content">my reply</div></div>
|
||||
</div>'''
|
||||
|
||||
|
||||
def test_responding_entry_and_feed_author():
|
||||
parsed = mf2py.parse(doc=FEED_PAGE, url='https://feed.example/')
|
||||
entry = mf2_utils.responding_entry(parsed, 'https://me.example/note/a#frag')
|
||||
assert mf2_utils.first(entry, 'url') == 'https://feed.example/2'
|
||||
assert mf2_utils.response_type(entry, 'https://me.example/note/a') == 'reply'
|
||||
|
||||
author = extract_author(parsed, 'https://feed.example/', entry)
|
||||
assert author['author_name'] == 'Feed Owner'
|
||||
assert author['author_url'] == 'https://feed.example/'
|
||||
|
||||
details = extract_source_details(entry)
|
||||
assert details['source_post_url'] == 'https://feed.example/2'
|
||||
assert details['source_published'] == '2026-11-01T16:00:00Z'
|
||||
|
||||
|
||||
def test_author_as_url_or_name():
|
||||
url_author = '<div class="h-entry"><a class="u-author" href="https://al.example/">al</a></div>'
|
||||
parsed = mf2py.parse(doc=url_author, url='https://s.example/')
|
||||
assert extract_author(parsed, 'https://s.example/')['author_url'] == 'https://al.example/'
|
||||
|
||||
name_author = '<div class="h-entry"><span class="p-author">Al Smith</span></div>'
|
||||
parsed = mf2py.parse(doc=name_author, url='https://s.example/')
|
||||
author = extract_author(parsed, 'https://s.example/')
|
||||
assert author['author_name'] == 'Al Smith' and author['author_url'] == 'https://s.example/'
|
||||
+15
-27
@@ -372,33 +372,21 @@ def test_micropub_missing_content(client, mock_valid_token):
|
||||
assert data['error'] == 'invalid_request'
|
||||
|
||||
|
||||
def test_micropub_unsupported_action(client, mock_valid_token):
|
||||
"""Test unsupported actions (update, delete) return error"""
|
||||
def test_micropub_update_delete_need_scope(client, mock_valid_token):
|
||||
"""A create-only token can't update or delete (v2.4.0)"""
|
||||
with patch('starpunk.routes.micropub.verify_access_token', mock_valid_token):
|
||||
# Test update
|
||||
response = client.post(
|
||||
'/micropub',
|
||||
json={
|
||||
'action': 'update',
|
||||
'url': 'https://example.com/note/123'
|
||||
},
|
||||
headers={'Authorization': 'Bearer valid_token'}
|
||||
)
|
||||
for action in ('update', 'delete', 'undelete'):
|
||||
response = client.post(
|
||||
'/micropub',
|
||||
json={'action': action, 'url': 'https://example.com/note/123', 'replace': {}},
|
||||
headers={'Authorization': 'Bearer valid_token'}
|
||||
)
|
||||
assert response.status_code == 403, action
|
||||
assert response.get_json()['error'] == 'insufficient_scope'
|
||||
|
||||
|
||||
def test_micropub_unknown_action(client, mock_valid_token):
|
||||
with patch('starpunk.routes.micropub.verify_access_token', mock_valid_token):
|
||||
response = client.post('/micropub', json={'action': 'frobnicate'},
|
||||
headers={'Authorization': 'Bearer valid_token'})
|
||||
assert response.status_code == 400
|
||||
data = response.get_json()
|
||||
assert 'not supported' in data['error_description']
|
||||
|
||||
# Test delete
|
||||
response = client.post(
|
||||
'/micropub',
|
||||
json={
|
||||
'action': 'delete',
|
||||
'url': 'https://example.com/note/123'
|
||||
},
|
||||
headers={'Authorization': 'Bearer valid_token'}
|
||||
)
|
||||
|
||||
assert response.status_code == 400
|
||||
data = response.get_json()
|
||||
assert 'not supported' in data['error_description']
|
||||
@@ -0,0 +1,209 @@
|
||||
"""
|
||||
Micropub round 1 (v2.4.0): update, delete/undelete, drafts, replies,
|
||||
multipart and remote photos, q=source lists and filters, q=category.
|
||||
|
||||
Uses real tokens from StarPunk's own IndieAuth server.
|
||||
"""
|
||||
|
||||
import io
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from starpunk import create_app
|
||||
from starpunk.indieauth import create_access_token
|
||||
from starpunk.notes import get_note
|
||||
|
||||
SITE = "https://blog.example.com"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def app(tmp_path):
|
||||
data = tmp_path / "data"
|
||||
return create_app({
|
||||
"TESTING": True, "SITE_URL": SITE, "ADMIN_ME": "https://example.com",
|
||||
"SESSION_SECRET": "x", "DATA_PATH": data, "NOTES_PATH": data / "notes",
|
||||
"DATABASE_PATH": data / "starpunk.db",
|
||||
})
|
||||
|
||||
|
||||
def auth(app, scope="create update delete media"):
|
||||
with app.test_request_context():
|
||||
token = create_access_token("https://quill.example/", scope)["access_token"]
|
||||
return {"Authorization": f"Bearer {token}"}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(app):
|
||||
return app.test_client()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def h(app):
|
||||
return auth(app)
|
||||
|
||||
|
||||
def jpeg(color="red"):
|
||||
buf = io.BytesIO()
|
||||
Image.new("RGB", (64, 48), color).save(buf, format="JPEG")
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def create(client, h, **props):
|
||||
body = {"type": ["h-entry"], "properties": {"content": ["Hello"], **props}}
|
||||
response = client.post("/micropub", json=body, headers=h)
|
||||
assert response.status_code == 201, response.get_json()
|
||||
return response.headers["Location"]
|
||||
|
||||
|
||||
def note_at(app, url):
|
||||
with app.app_context():
|
||||
return get_note(slug=url.rstrip("/").rsplit("/", 1)[1])
|
||||
|
||||
|
||||
def source(client, h, url, *props):
|
||||
query = f"/micropub?q=source&url={url}" + "".join(f"&properties[]={p}" for p in props)
|
||||
return client.get(query, headers=h).get_json()
|
||||
|
||||
|
||||
class TestCreate:
|
||||
def test_draft(self, app, client, h):
|
||||
url = create(client, h, **{"post-status": ["draft"]})
|
||||
assert note_at(app, url).published is False
|
||||
assert client.get(url.replace(SITE, "")).status_code == 404
|
||||
|
||||
def test_reply(self, app, client, h):
|
||||
url = create(client, h, **{"in-reply-to": ["https://other.example/post/1"]})
|
||||
assert note_at(app, url).in_reply_to == "https://other.example/post/1"
|
||||
|
||||
def test_multipart_photo(self, app, client, h):
|
||||
response = client.post(
|
||||
"/micropub",
|
||||
data={"h": "entry", "content": "With a file", "photo": (io.BytesIO(jpeg()), "pic.jpg")},
|
||||
headers=h, content_type="multipart/form-data",
|
||||
)
|
||||
assert response.status_code == 201
|
||||
assert "photo" in source(client, h, response.headers["Location"])["properties"]
|
||||
|
||||
def test_remote_photo_is_downloaded(self, app, client, h, monkeypatch):
|
||||
# Fetched through the SSRF-checked, address-pinned fetcher (v2.9.0)
|
||||
monkeypatch.setattr("starpunk.webmention_ssrf.TRANSPORT",
|
||||
httpx.MockTransport(lambda req: httpx.Response(200, content=jpeg("blue"))))
|
||||
monkeypatch.setattr("starpunk.webmention_ssrf.socket.getaddrinfo",
|
||||
lambda *a, **k: [(2, 1, 6, "", ("93.184.216.34", 0))])
|
||||
url = create(client, h, photo=[{"value": "https://photos.example/cat.jpg", "alt": "A cat"}])
|
||||
(photo,) = source(client, h, url)["properties"]["photo"]
|
||||
assert photo["value"].startswith(f"{SITE}/media/") and photo["alt"] == "A cat"
|
||||
|
||||
def test_only_h_entry(self, client, h):
|
||||
response = client.post("/micropub", data={"h": "event", "content": "x"}, headers=h)
|
||||
assert response.status_code == 400
|
||||
|
||||
|
||||
class TestUpdate:
|
||||
def update(self, client, h, url, **ops):
|
||||
return client.post("/micropub", json={"action": "update", "url": url, **ops}, headers=h)
|
||||
|
||||
def test_replace_content_keeps_title(self, app, client, h):
|
||||
url = create(client, h, name=["My Title"])
|
||||
assert self.update(client, h, url, replace={"content": ["New body"]}).status_code == 204
|
||||
props = source(client, h, url)["properties"]
|
||||
assert props["name"] == ["My Title"] and props["content"] == ["New body"]
|
||||
|
||||
def test_replace_and_delete_name(self, app, client, h):
|
||||
url = create(client, h)
|
||||
self.update(client, h, url, replace={"name": ["Added"]})
|
||||
assert note_at(app, url).content.startswith("# Added")
|
||||
self.update(client, h, url, delete=["name"])
|
||||
assert "name" not in source(client, h, url)["properties"]
|
||||
|
||||
def test_categories(self, app, client, h):
|
||||
url = create(client, h, category=["one", "two"])
|
||||
self.update(client, h, url, add={"category": ["three", "One"]})
|
||||
assert source(client, h, url)["properties"]["category"] == ["one", "two", "three"] # entered order (v2.6.4)
|
||||
self.update(client, h, url, delete={"category": ["two"]})
|
||||
assert source(client, h, url)["properties"]["category"] == ["one", "three"]
|
||||
self.update(client, h, url, delete=["category"])
|
||||
assert "category" not in source(client, h, url)["properties"]
|
||||
|
||||
def test_post_status_and_reply(self, app, client, h):
|
||||
url = create(client, h)
|
||||
self.update(client, h, url, replace={"post-status": ["draft"], "in-reply-to": ["https://x.example/a"]})
|
||||
note = note_at(app, url)
|
||||
assert note.published is False and note.in_reply_to == "https://x.example/a"
|
||||
self.update(client, h, url, delete=["in-reply-to", "post-status"])
|
||||
note = note_at(app, url)
|
||||
assert note.published is True and not note.in_reply_to
|
||||
|
||||
def test_photos(self, app, client, h):
|
||||
uploaded = client.post("/micropub/media", data={"file": (io.BytesIO(jpeg()), "a.jpg")},
|
||||
headers=h, content_type="multipart/form-data").headers["Location"]
|
||||
url = create(client, h)
|
||||
self.update(client, h, url, add={"photo": [{"value": uploaded, "alt": "Red"}]})
|
||||
assert source(client, h, url)["properties"]["photo"] == [{"value": uploaded, "alt": "Red"}]
|
||||
self.update(client, h, url, delete={"photo": [uploaded]})
|
||||
assert "photo" not in source(client, h, url)["properties"]
|
||||
|
||||
def test_unsupported_property(self, client, h):
|
||||
url = create(client, h)
|
||||
response = self.update(client, h, url, replace={"published": ["2020-01-01T00:00:00Z"]})
|
||||
assert response.status_code == 400 and "published" in response.get_json()["error_description"]
|
||||
|
||||
def test_unknown_post(self, client, h):
|
||||
response = self.update(client, h, f"{SITE}/note/nope", replace={"content": ["x"]})
|
||||
assert response.status_code == 404
|
||||
|
||||
def test_form_update_rejected(self, client, h):
|
||||
url = create(client, h)
|
||||
assert client.post("/micropub", data={"action": "update", "url": url}, headers=h).status_code == 400
|
||||
|
||||
|
||||
class TestDelete:
|
||||
def test_delete_and_undelete(self, app, client, h):
|
||||
url = create(client, h, category=["kept"])
|
||||
path = url.replace(SITE, "")
|
||||
assert client.post("/micropub", data={"action": "delete", "url": url}, headers=h).status_code == 204
|
||||
assert client.get(path).status_code == 410 # Gone (Webmention 3.1.5)
|
||||
response = client.post("/micropub", json={"action": "undelete", "url": url}, headers=h)
|
||||
assert response.status_code == 204
|
||||
assert client.get(path).status_code == 200
|
||||
assert source(client, h, url)["properties"]["category"] == ["kept"]
|
||||
|
||||
def test_undelete_needs_a_deleted_post(self, client, h):
|
||||
url = create(client, h)
|
||||
assert client.post("/micropub", json={"action": "undelete", "url": url}, headers=h).status_code == 404
|
||||
|
||||
def test_delete_scope(self, app, client):
|
||||
h = auth(app, "create")
|
||||
url = create(client, h)
|
||||
assert client.post("/micropub", data={"action": "delete", "url": url}, headers=h).status_code == 403
|
||||
|
||||
|
||||
class TestQueries:
|
||||
def test_source_properties_filter(self, client, h):
|
||||
url = create(client, h, name=["T"], category=["x"])
|
||||
data = source(client, h, url, "name", "category")
|
||||
assert data == {"properties": {"name": ["T"], "category": ["x"]}}
|
||||
|
||||
def test_source_list_includes_drafts(self, client, h):
|
||||
create(client, h, content=["first"])
|
||||
create(client, h, content=["second"], **{"post-status": ["draft"]})
|
||||
items = client.get("/micropub?q=source&limit=5", headers=h).get_json()["items"]
|
||||
assert {i["properties"]["post-status"][0] for i in items} == {"published", "draft"}
|
||||
|
||||
def test_category(self, client, h):
|
||||
create(client, h, category=["IndieWeb", "python"])
|
||||
create(client, h, category=["IndieWeb"])
|
||||
assert client.get("/micropub?q=category", headers=h).get_json()["categories"] == ["IndieWeb", "python"]
|
||||
assert client.get("/micropub?q=category&filter=py", headers=h).get_json()["categories"] == ["python"]
|
||||
|
||||
def test_config_lists_queries(self, client, h):
|
||||
config = client.get("/micropub?q=config", headers=h).get_json()
|
||||
assert "category" in config["q"] and "media-endpoint" in config
|
||||
|
||||
|
||||
def test_media_scope_alone_can_upload(app, client):
|
||||
response = client.post("/micropub/media", data={"file": (io.BytesIO(jpeg()), "a.jpg")},
|
||||
headers=auth(app, "media"), content_type="multipart/form-data")
|
||||
assert response.status_code == 201
|
||||
@@ -0,0 +1,105 @@
|
||||
"""
|
||||
Micropub media fixes (v2.6.5): photo posts without text, the same photo
|
||||
twice, and failed creates that left a post (and a database lock) behind.
|
||||
"""
|
||||
|
||||
import io
|
||||
|
||||
from starpunk.database.pool import ConnectionPool
|
||||
from starpunk.notes import list_notes
|
||||
from starpunk.syndication import compose_text
|
||||
from tests.test_micropub_v240 import SITE, app, client, h, jpeg, note_at, source # noqa: F401
|
||||
|
||||
|
||||
def upload(client, h, color="red"):
|
||||
response = client.post("/micropub/media", data={"file": (io.BytesIO(jpeg(color)), "a.jpg")},
|
||||
headers=h, content_type="multipart/form-data")
|
||||
assert response.status_code == 201
|
||||
return response.headers["Location"]
|
||||
|
||||
|
||||
def post(client, h, props):
|
||||
return client.post("/micropub", json={"type": ["h-entry"], "properties": props}, headers=h)
|
||||
|
||||
|
||||
def update(client, h, url, **ops):
|
||||
return client.post("/micropub", json={"action": "update", "url": url, **ops}, headers=h)
|
||||
|
||||
|
||||
class TestPhotoWithoutText:
|
||||
def test_json_photo_only(self, app, client, h):
|
||||
photo = upload(client, h)
|
||||
response = post(client, h, {"photo": [{"value": photo, "alt": "Red"}]})
|
||||
assert response.status_code == 201, response.get_json()
|
||||
url = response.headers["Location"]
|
||||
assert note_at(app, url).content == ""
|
||||
assert source(client, h, url, "photo")["properties"]["photo"] == [{"value": photo, "alt": "Red"}]
|
||||
page = client.get(url.replace(SITE, ""))
|
||||
assert page.status_code == 200 and b"<title>Photo \xc2\xb7 " in page.data
|
||||
|
||||
def test_multipart_photo_only(self, client, h):
|
||||
response = client.post("/micropub", data={"h": "entry", "photo": (io.BytesIO(jpeg()), "p.jpg")},
|
||||
headers=h, content_type="multipart/form-data")
|
||||
assert response.status_code == 201, response.get_json()
|
||||
|
||||
def test_nothing_at_all_is_still_rejected(self, client, h):
|
||||
assert post(client, h, {}).status_code == 400
|
||||
# A photo that can't be found doesn't count
|
||||
assert post(client, h, {"photo": [f"{SITE}/media/nope.jpg"]}).status_code == 400
|
||||
|
||||
def test_text_can_be_cleared_while_photos_remain(self, app, client, h):
|
||||
photo = upload(client, h)
|
||||
url = post(client, h, {"content": ["Hi"], "photo": [photo]}).headers["Location"]
|
||||
assert update(client, h, url, replace={"content": [""]}).status_code == 204
|
||||
assert note_at(app, url).content == ""
|
||||
assert update(client, h, url, replace={"content": ["Back"]}).status_code == 204
|
||||
assert update(client, h, url, delete=["content"]).status_code == 204
|
||||
assert note_at(app, url).content == ""
|
||||
# Without photos the text has to stay
|
||||
response = update(client, h, url, delete=["content", "photo"])
|
||||
assert response.status_code == 400
|
||||
|
||||
def test_text_is_required_without_photos(self, client, h):
|
||||
url = post(client, h, {"content": ["Hi"]}).headers["Location"]
|
||||
assert update(client, h, url, replace={"content": [""]}).status_code == 400
|
||||
assert update(client, h, url, delete=["content"]).status_code == 400
|
||||
|
||||
def test_crosspost_text_starts_with_the_link(self, app, client, h):
|
||||
url = post(client, h, {"photo": [upload(client, h)]}).headers["Location"]
|
||||
with app.app_context():
|
||||
note = note_at(app, url)
|
||||
assert compose_text(note, url, ["cats"], 500) == f"{url}\n\n#cats"
|
||||
|
||||
|
||||
class TestFailures:
|
||||
def test_same_photo_twice_is_shown_once(self, client, h):
|
||||
photo = upload(client, h)
|
||||
response = post(client, h, {"content": ["Twice"], "photo": [photo, photo]})
|
||||
assert response.status_code == 201, response.get_json()
|
||||
assert source(client, h, response.headers["Location"], "photo")["properties"]["photo"] == [{"value": photo, "alt": ""}]
|
||||
|
||||
def test_failed_create_leaves_no_post_and_no_lock(self, app, client, h, monkeypatch):
|
||||
import starpunk.syndication as syndication
|
||||
|
||||
def boom(*args, **kwargs):
|
||||
raise RuntimeError("crosspost store down")
|
||||
|
||||
monkeypatch.setattr(syndication, "select_targets", boom)
|
||||
response = post(client, h, {"content": ["Doomed"], "photo": [upload(client, h)]})
|
||||
assert response.status_code == 500
|
||||
with app.app_context():
|
||||
assert list_notes(published_only=False) == []
|
||||
monkeypatch.undo()
|
||||
# The database is usable straight away
|
||||
assert post(client, h, {"content": ["Next"]}).status_code == 201
|
||||
|
||||
def test_pool_rolls_back_open_transactions(self, tmp_path):
|
||||
pool = ConnectionPool(str(tmp_path / "x.db"), pool_size=1, timeout=1)
|
||||
conn = pool.get_connection()
|
||||
conn.execute("CREATE TABLE t (x)")
|
||||
conn.commit()
|
||||
conn.execute("INSERT INTO t VALUES (1)")
|
||||
assert conn.in_transaction
|
||||
pool.return_connection(conn)
|
||||
assert not conn.in_transaction
|
||||
assert pool.get_connection().execute("SELECT count(*) FROM t").fetchone()[0] == 0
|
||||
@@ -0,0 +1,249 @@
|
||||
"""
|
||||
Micropub compliance fixes (v2.9.0)
|
||||
|
||||
Section numbers refer to the W3C Micropub Recommendation.
|
||||
"""
|
||||
|
||||
import io
|
||||
import sqlite3
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from tests.test_micropub_v240 import SITE, app, auth, client, create, h, jpeg, note_at, source # noqa: F401
|
||||
|
||||
|
||||
def syndications(app, url):
|
||||
note = note_at(app, url)
|
||||
db = sqlite3.connect(app.config["DATABASE_PATH"])
|
||||
return sorted(r[0] for r in db.execute("SELECT target FROM syndications WHERE note_id = ?", (note.id,)))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def targets(app):
|
||||
app.config.update(BLUESKY_HANDLE="me.example", BLUESKY_APP_PASSWORD="p")
|
||||
from starpunk import syndication
|
||||
syndication._client_cache.clear()
|
||||
with app.app_context():
|
||||
return {t.uid: name for name, t in syndication.get_targets().items()}
|
||||
|
||||
|
||||
# Crossposting: mp-* in JSON properties; absent means none
|
||||
|
||||
def test_mp_syndicate_to_inside_json_properties(app, client, h, targets):
|
||||
bsky_uid = next(uid for uid, name in targets.items() if name == "bluesky")
|
||||
url = create(client, h, **{"mp-syndicate-to": [bsky_uid]})
|
||||
assert syndications(app, url) == ["bluesky"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("extra", [{}, {"mp-syndicate-to": []}])
|
||||
def test_no_choice_means_no_crossposting(app, client, h, targets, extra):
|
||||
url = create(client, h, **extra)
|
||||
assert syndications(app, url) == []
|
||||
|
||||
|
||||
def test_form_mp_syndicate_to(app, client, h, targets):
|
||||
bsky_uid = next(uid for uid, name in targets.items() if name == "bluesky")
|
||||
response = client.post("/micropub", data={"content": "x", "mp-syndicate-to[]": bsky_uid}, headers=h)
|
||||
assert syndications(app, response.headers["Location"]) == ["bluesky"]
|
||||
|
||||
|
||||
def test_mp_slug_inside_json_properties(app, client, h):
|
||||
url = create(client, h, **{"mp-slug": ["my-slug"]})
|
||||
assert url.endswith("/note/my-slug")
|
||||
assert "mp-slug" not in source(client, h, url)["properties"]
|
||||
|
||||
|
||||
# Scopes
|
||||
|
||||
def test_source_needs_a_posting_scope(app, client, h):
|
||||
create(client, h, **{"post-status": ["draft"]})
|
||||
profile_only = auth(app, "profile")
|
||||
assert client.get("/micropub?q=source", headers=profile_only).status_code == 403
|
||||
assert client.get("/micropub?q=category", headers=profile_only).status_code == 403
|
||||
assert client.get("/micropub?q=config", headers=profile_only).status_code == 200
|
||||
|
||||
|
||||
def test_draft_scope_only_creates_drafts(app, client):
|
||||
url = create(client, auth(app, "draft"))
|
||||
assert note_at(app, url).published is False
|
||||
|
||||
|
||||
# Visibility
|
||||
|
||||
@pytest.mark.parametrize("visibility,published", [("public", True), ("unlisted", False), ("private", False)])
|
||||
def test_visibility(app, client, h, visibility, published):
|
||||
url = create(client, h, visibility=[visibility])
|
||||
assert note_at(app, url).published is published
|
||||
|
||||
|
||||
def test_unknown_visibility_rejected(client, h):
|
||||
body = {"type": ["h-entry"], "properties": {"content": ["x"], "visibility": ["friends"]}}
|
||||
assert client.post("/micropub", json=body, headers=h).status_code == 400
|
||||
|
||||
|
||||
# Photos that can't be used fail the request
|
||||
|
||||
def test_unreachable_photo_is_an_error(app, client, h, monkeypatch):
|
||||
monkeypatch.setattr("starpunk.webmention_ssrf.TRANSPORT", httpx.MockTransport(lambda r: httpx.Response(404)))
|
||||
monkeypatch.setattr("starpunk.webmention_ssrf.socket.getaddrinfo",
|
||||
lambda *a, **k: [(2, 1, 6, "", ("93.184.216.34", 0))])
|
||||
body = {"type": ["h-entry"], "properties": {"content": ["x"], "photo": ["https://p.example/404.jpg"]}}
|
||||
assert client.post("/micropub", json=body, headers=h).status_code == 400
|
||||
assert client.get("/micropub?q=source", headers=h).get_json()["items"] == []
|
||||
|
||||
|
||||
def test_unknown_local_media_is_an_error_on_update(app, client, h):
|
||||
url = create(client, h)
|
||||
response = client.post("/micropub", json={"action": "update", "url": url,
|
||||
"replace": {"photo": [f"{SITE}/media/2026/11/nope.jpg"]}}, headers=h)
|
||||
assert response.status_code == 400
|
||||
|
||||
|
||||
def test_redirect_to_private_address_is_not_fetched(app, client, h, monkeypatch):
|
||||
def resolve(host, *a, **k):
|
||||
return [(2, 1, 6, "", ("10.0.0.5" if host == "internal.example" else "93.184.216.34", 0))]
|
||||
|
||||
def site(request):
|
||||
return httpx.Response(302, headers={"Location": "http://internal.example/secret.jpg"})
|
||||
|
||||
monkeypatch.setattr("starpunk.webmention_ssrf.socket.getaddrinfo", resolve)
|
||||
monkeypatch.setattr("starpunk.webmention_ssrf.TRANSPORT", httpx.MockTransport(site))
|
||||
body = {"type": ["h-entry"], "properties": {"content": ["x"], "photo": ["https://p.example/a.jpg"]}}
|
||||
assert client.post("/micropub", json=body, headers=h).status_code == 400
|
||||
|
||||
|
||||
# JSON values must be arrays (3.3.2)
|
||||
|
||||
def test_scalar_json_values_are_not_split(app, client, h):
|
||||
body = {"type": ["h-entry"], "properties": {"content": "whole text", "category": "solo", "name": "Title"}}
|
||||
response = client.post("/micropub", json=body, headers=h)
|
||||
props = source(client, h, response.headers["Location"])["properties"]
|
||||
assert props["content"] == ["whole text"]
|
||||
assert props["category"] == ["solo"]
|
||||
assert props["name"] == ["Title"]
|
||||
|
||||
|
||||
def test_update_delete_values_must_be_arrays(app, client, h):
|
||||
url = create(client, h, category=["a", "b"])
|
||||
response = client.post("/micropub", json={"action": "update", "url": url, "delete": {"category": "a"}}, headers=h)
|
||||
assert response.status_code == 400
|
||||
|
||||
|
||||
def test_update_needs_an_operation(app, client, h):
|
||||
url = create(client, h)
|
||||
assert client.post("/micropub", json={"action": "update", "url": url}, headers=h).status_code == 400
|
||||
|
||||
|
||||
def test_person_tag_categories_are_ignored(app, client, h):
|
||||
url = create(client, h, category=["kept", {"type": ["h-card"], "properties": {"url": ["https://a.example/"]}}])
|
||||
assert source(client, h, url)["properties"]["category"] == ["kept"]
|
||||
|
||||
|
||||
# Properties that used to be dropped
|
||||
|
||||
def test_bookmark_of(app, client, h):
|
||||
body = {"type": ["h-entry"], "properties": {"bookmark-of": ["https://read.example/post"]}}
|
||||
url = client.post("/micropub", json=body, headers=h).headers["Location"]
|
||||
note = note_at(app, url)
|
||||
assert note.kind == "bookmark" and note.bookmark_of == "https://read.example/post"
|
||||
props = source(client, h, url)["properties"]
|
||||
assert props["bookmark-of"] == ["https://read.example/post"] and "content" not in props
|
||||
page = client.get(url.replace(SITE, "")).data.decode()
|
||||
assert 'class="u-bookmark-of" href="https://read.example/post"' in page
|
||||
|
||||
|
||||
def test_summary_or_name_without_content(app, client, h):
|
||||
response = client.post("/micropub", json={"type": ["h-entry"], "properties": {"summary": ["Just a summary"]}},
|
||||
headers=h)
|
||||
assert source(client, h, response.headers["Location"])["properties"]["content"] == ["Just a summary"]
|
||||
|
||||
response = client.post("/micropub", json={"type": ["h-entry"], "properties": {"name": ["Only a title"]}},
|
||||
headers=h)
|
||||
assert source(client, h, response.headers["Location"])["properties"]["name"] == ["Only a title"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("prop", ["video", "audio"])
|
||||
def test_unsupported_media_is_refused(client, h, prop):
|
||||
body = {"type": ["h-entry"], "properties": {"content": ["x"], prop: ["https://m.example/clip"]}}
|
||||
assert client.post("/micropub", json=body, headers=h).status_code == 400
|
||||
upload = {"content": "x", prop: (io.BytesIO(b"data"), "clip.mp4")}
|
||||
assert client.post("/micropub", data=upload, headers=h, content_type="multipart/form-data").status_code == 400
|
||||
|
||||
|
||||
# HTML content (3.7.2) and sanitizing
|
||||
|
||||
def test_html_content_comes_back_as_html(app, client, h):
|
||||
url = create(client, h, content=[{"html": "<p>Some <b>bold</b> text</p>"}])
|
||||
assert source(client, h, url)["properties"]["content"] == [{"html": "<p>Some <b>bold</b> text</p>"}]
|
||||
url = create(client, h, content=["plain *markdown*"])
|
||||
assert source(client, h, url)["properties"]["content"] == ["plain *markdown*"]
|
||||
|
||||
|
||||
def test_update_with_html_content(app, client, h):
|
||||
url = create(client, h)
|
||||
client.post("/micropub", json={"action": "update", "url": url,
|
||||
"replace": {"content": [{"html": "<p>new</p>"}]}}, headers=h)
|
||||
assert source(client, h, url)["properties"]["content"] == [{"html": "<p>new</p>"}]
|
||||
|
||||
|
||||
def test_html_is_sanitized_on_the_page(app, client, h):
|
||||
url = create(client, h, content=[{"html": '<p>hi<script>alert(1)</script><img src="/x.jpg" onerror="evil()">'
|
||||
'<a href="javascript:alert(2)">j</a><iframe src="https://e.example"></iframe></p>'}])
|
||||
page = client.get(url.replace(SITE, "")).data.decode()
|
||||
for bad in ("<script>alert(1)", "onerror", "javascript:alert", "<iframe"):
|
||||
assert bad not in page
|
||||
assert '<img src="/x.jpg">' in page
|
||||
|
||||
|
||||
# published (3.3)
|
||||
|
||||
def test_published_is_stored_as_utc(app, client, h):
|
||||
url = create(client, h, published=["2020-02-03T04:05:06-08:00"])
|
||||
assert source(client, h, url)["properties"]["published"] == ["2020-02-03T12:05:06Z"]
|
||||
created = note_at(app, url).created_at
|
||||
assert created.tzinfo is None and (created.hour, created.minute) == (12, 5)
|
||||
|
||||
|
||||
def test_unparseable_published_is_an_error(client, h):
|
||||
body = {"type": ["h-entry"], "properties": {"content": ["x"], "published": ["yesterday"]}}
|
||||
assert client.post("/micropub", json=body, headers=h).status_code == 400
|
||||
|
||||
|
||||
# Errors (3.8) and authentication (5)
|
||||
|
||||
@pytest.mark.parametrize("payload", ["{not json", "[1, 2]"])
|
||||
def test_malformed_json_is_400(client, h, payload):
|
||||
response = client.post("/micropub", data=payload, headers={**h, "Content-Type": "application/json"})
|
||||
assert response.status_code == 400
|
||||
assert response.get_json()["error"] == "invalid_request"
|
||||
|
||||
|
||||
def test_bearer_scheme_is_case_insensitive(client, h):
|
||||
lower = {"Authorization": h["Authorization"].replace("Bearer", "bearer")}
|
||||
assert client.get("/micropub?q=config", headers=lower).status_code == 200
|
||||
|
||||
|
||||
def test_token_in_header_and_body_is_400(client, h):
|
||||
response = client.post("/micropub", data={"content": "x", "access_token": "other"}, headers=h)
|
||||
assert response.status_code == 400
|
||||
|
||||
|
||||
def test_401_has_www_authenticate(client):
|
||||
response = client.get("/micropub?q=config")
|
||||
assert response.status_code == 401
|
||||
assert response.headers["WWW-Authenticate"].startswith("Bearer")
|
||||
response = client.get("/micropub?q=config", headers={"Authorization": "Bearer nope"})
|
||||
assert 'error="invalid_token"' in response.headers["WWW-Authenticate"]
|
||||
|
||||
|
||||
def test_media_error_does_not_leak_internals(client, h):
|
||||
response = client.post("/micropub/media", data={"file": (io.BytesIO(b"not an image"), "a.jpg")},
|
||||
headers=h, content_type="multipart/form-data")
|
||||
assert response.status_code == 400
|
||||
assert "BytesIO" not in response.get_json()["error_description"]
|
||||
|
||||
|
||||
def test_config_lists_bookmarks(client, h):
|
||||
types = {t["type"] for t in client.get("/micropub?q=config", headers=h).get_json()["post-types"]}
|
||||
assert "bookmark" in types
|
||||
@@ -0,0 +1,86 @@
|
||||
"""
|
||||
Migrations under concurrent gunicorn workers (v2.8.1)
|
||||
|
||||
The v2.8.0 deploy failed to boot: one worker applied migration 014 while
|
||||
another applied it again ("duplicate column name"), because
|
||||
conn.executescript() committed and released the migration lock early.
|
||||
"""
|
||||
|
||||
import multiprocessing
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
from starpunk.database.schema import INITIAL_SCHEMA_SQL
|
||||
from starpunk.migrations import execute_in_transaction, run_migrations, split_sql_statements
|
||||
|
||||
MIGRATIONS = Path(__file__).parent.parent / "migrations"
|
||||
|
||||
|
||||
def test_script_runs_inside_the_transaction(tmp_path):
|
||||
conn = sqlite3.connect(tmp_path / "t.db")
|
||||
conn.execute("CREATE TABLE t (a INTEGER)")
|
||||
conn.commit()
|
||||
|
||||
conn.execute("BEGIN IMMEDIATE")
|
||||
execute_in_transaction(conn, "ALTER TABLE t ADD COLUMN b TEXT;\nALTER TABLE t ADD COLUMN c TEXT;")
|
||||
conn.rollback()
|
||||
|
||||
assert [row[1] for row in conn.execute("PRAGMA table_info(t)")] == ["a"]
|
||||
|
||||
|
||||
def test_split_keeps_trigger_bodies_together():
|
||||
script = """
|
||||
-- comment; with a semicolon
|
||||
CREATE TABLE t (a TEXT DEFAULT 'x;y');
|
||||
CREATE TRIGGER tr AFTER DELETE ON t BEGIN
|
||||
DELETE FROM t WHERE a = old.a;
|
||||
SELECT 1;
|
||||
END;
|
||||
"""
|
||||
statements = split_sql_statements(script)
|
||||
assert len(statements) == 2
|
||||
assert statements[1].startswith("CREATE TRIGGER") and statements[1].endswith("END;")
|
||||
|
||||
|
||||
def _migrate(db_path, barrier, errors):
|
||||
try:
|
||||
barrier.wait()
|
||||
run_migrations(db_path)
|
||||
except Exception as e: # report to the parent
|
||||
errors.put(repr(e))
|
||||
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.parametrize("trial", range(5)) # the race hit ~1 in 5 runs
|
||||
def test_concurrent_workers_apply_each_migration_once(tmp_path, trial):
|
||||
# A database at migration 013, like production before v2.8.0
|
||||
db_path = tmp_path / "starpunk.db"
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.executescript(INITIAL_SCHEMA_SQL) # as init_db does
|
||||
conn.close()
|
||||
run_migrations(db_path)
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.execute("DELETE FROM schema_migrations WHERE migration_name = '014_webmention_source_details.sql'")
|
||||
conn.commit()
|
||||
columns = [row[1] for row in conn.execute("PRAGMA table_info(webmentions)")]
|
||||
if "source_post_url" in columns:
|
||||
conn.execute("ALTER TABLE webmentions DROP COLUMN source_post_url")
|
||||
conn.execute("ALTER TABLE webmentions DROP COLUMN source_published")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
ctx = multiprocessing.get_context("fork")
|
||||
barrier, errors = ctx.Barrier(4), ctx.Queue()
|
||||
workers = [ctx.Process(target=_migrate, args=(db_path, barrier, errors)) for _ in range(4)]
|
||||
for w in workers:
|
||||
w.start()
|
||||
for w in workers:
|
||||
w.join(timeout=60)
|
||||
|
||||
assert errors.empty(), errors.get()
|
||||
conn = sqlite3.connect(db_path)
|
||||
assert conn.execute(
|
||||
"SELECT COUNT(*) FROM schema_migrations WHERE migration_name = '014_webmention_source_details.sql'"
|
||||
).fetchone()[0] == 1
|
||||
@@ -43,7 +43,7 @@ class TestRetryLogic:
|
||||
from starpunk.database import init_db
|
||||
from starpunk import create_app
|
||||
|
||||
app = create_app({'DATABASE_PATH': str(temp_db)})
|
||||
app = create_app({'DATABASE_PATH': str(temp_db), 'TESTING': True})
|
||||
init_db(app)
|
||||
|
||||
# Verify migrations table exists and has records
|
||||
@@ -302,12 +302,13 @@ class TestErrorHandling:
|
||||
|
||||
def test_rollback_failure_causes_system_exit(self, temp_db):
|
||||
"""Test that rollback failure raises SystemExit"""
|
||||
with patch('sqlite3.connect') as mock_connect:
|
||||
with patch('sqlite3.connect') as mock_connect, \
|
||||
patch('starpunk.migrations.execute_in_transaction',
|
||||
side_effect=Exception("Migration failed")):
|
||||
mock_conn = MagicMock()
|
||||
mock_connect.return_value = mock_conn
|
||||
|
||||
# Make both migration and rollback fail
|
||||
mock_conn.executescript.side_effect = Exception("Migration failed")
|
||||
mock_conn.rollback.side_effect = Exception("Rollback failed")
|
||||
mock_conn.execute.return_value.fetchone.return_value = (0,)
|
||||
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
"""
|
||||
Rendered posts read as the right post type (v2.6.5).
|
||||
|
||||
Parses note pages and the home page with mf2py, then runs W3C Post Type
|
||||
Discovery (https://www.w3.org/TR/post-type-discovery/) on what a consumer
|
||||
sees. A photo post is any post with a photo, unless it's first a repost,
|
||||
like or reply.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import mf2py
|
||||
|
||||
import starpunk.reply_context as reply_context
|
||||
from tests.test_micropub_v240 import SITE, app, client, h, jpeg # noqa: F401
|
||||
from tests.test_micropub_v265 import upload
|
||||
|
||||
|
||||
def post_type(props):
|
||||
"""W3C Post Type Discovery, section 5 (no events or RSVPs here)."""
|
||||
for prop, kind in [("repost-of", "repost"), ("like-of", "like"), ("in-reply-to", "reply"),
|
||||
("video", "video"), ("photo", "photo")]:
|
||||
if props.get(prop):
|
||||
return kind
|
||||
|
||||
def first(prop):
|
||||
for v in props.get(prop, []):
|
||||
v = v.get("value", "") if isinstance(v, dict) else v
|
||||
if v.strip():
|
||||
return " ".join(v.split())
|
||||
return ""
|
||||
|
||||
content = first("content") or first("summary")
|
||||
name = first("name")
|
||||
if not content or not name or content.startswith(name):
|
||||
return "note"
|
||||
return "article"
|
||||
|
||||
|
||||
def entries(items):
|
||||
for item in items:
|
||||
if "h-entry" in item.get("type", []):
|
||||
yield item
|
||||
yield from entries(item.get("children", []))
|
||||
|
||||
|
||||
def value(v):
|
||||
"""A u-* value, or the url of a nested microformat (h-cite)."""
|
||||
return v.get("value") or v.get("properties", {}).get("url", [None])[0] if isinstance(v, dict) else v
|
||||
|
||||
|
||||
CASES = {
|
||||
"note": ({"content": ["Just a note"]}, "note"),
|
||||
"article": ({"name": ["A Title"], "content": ["Body text here"]}, "article"),
|
||||
"photo without text": ({"photo": ["PHOTO"]}, "photo"),
|
||||
"photo with caption": ({"content": ["Sunset"], "photo": [{"value": "PHOTO", "alt": "Orange sky"}]}, "photo"),
|
||||
"photo with title": ({"name": ["Trip"], "content": ["Day one"], "photo": ["PHOTO"]}, "photo"),
|
||||
"reply": ({"content": ["Agreed"], "in-reply-to": ["https://other.example/post/1"]}, "reply"),
|
||||
"reply with photo": ({"content": ["Mine"], "in-reply-to": ["https://other.example/post/2"],
|
||||
"photo": ["PHOTO"]}, "reply"),
|
||||
"repost": ({"repost-of": ["https://other.example/post/4"]}, "repost"),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", CASES)
|
||||
def test_post_type(app, client, h, monkeypatch, case):
|
||||
monkeypatch.setattr(reply_context, "fetch_reply_context", lambda *a, **k: None, raising=False)
|
||||
props, want = CASES[case]
|
||||
photo = upload(client, h) if "PHOTO" in str(props) else None
|
||||
props = {k: [photo if v == "PHOTO" else ({**v, "value": photo} if isinstance(v, dict) else v)
|
||||
for v in vs] for k, vs in props.items()}
|
||||
response = client.post("/micropub", json={"type": ["h-entry"], "properties": props}, headers=h)
|
||||
assert response.status_code == 201, response.get_json()
|
||||
url = response.headers["Location"]
|
||||
|
||||
(entry,) = entries(mf2py.parse(doc=client.get(url.replace(SITE, "")).data.decode(), url=url)["items"])
|
||||
home = mf2py.parse(doc=client.get("/").data.decode(), url=f"{SITE}/")
|
||||
(listed,) = [e for e in entries(home["items"]) if e["properties"].get("url") == [url]]
|
||||
|
||||
for e in (entry, listed):
|
||||
assert post_type(e["properties"]) == want
|
||||
if want == "reply":
|
||||
assert value(e["properties"]["in-reply-to"][0]) == props["in-reply-to"][0]
|
||||
if want == "photo" and "name" not in props:
|
||||
assert "name" not in e["properties"], "no implied name to show as a caption"
|
||||
|
||||
|
||||
def test_photo_alt_text(app, client, h):
|
||||
"""Alt text is the caption, else the post's text, never made up (v2.6.5)."""
|
||||
def alts(props):
|
||||
url = client.post("/micropub", json={"type": ["h-entry"], "properties": props},
|
||||
headers=h).headers["Location"]
|
||||
(entry,) = entries(mf2py.parse(doc=client.get(url.replace(SITE, "")).data.decode(), url=url)["items"])
|
||||
return [p["alt"] for p in entry["properties"]["photo"]]
|
||||
|
||||
assert alts({"content": ["Beach day"], "photo": [{"value": upload(client, h), "alt": "Waves"}]}) == ["Waves"]
|
||||
assert alts({"content": ["Beach day"], "photo": [upload(client, h, "blue")]}) == ["Beach day"]
|
||||
assert alts({"name": ["Trip"], "content": ["Day one"], "photo": [upload(client, h, "green")]}) == ["Day one"]
|
||||
assert alts({"photo": [upload(client, h, "gray")]}) == ["Image"]
|
||||
@@ -32,6 +32,7 @@ class FakeServer:
|
||||
self.statuses = {}
|
||||
self.calls = []
|
||||
self.media = 0
|
||||
self.uploads = [] # raw multipart bodies
|
||||
self.fail_next = None # status code to return once
|
||||
|
||||
def __call__(self, request: httpx.Request) -> httpx.Response:
|
||||
@@ -43,6 +44,7 @@ class FakeServer:
|
||||
path = request.url.path
|
||||
if path == "/api/v2/media":
|
||||
self.media += 1
|
||||
self.uploads.append(request.read())
|
||||
return httpx.Response(200, json={"id": f"m{self.media}"})
|
||||
if path == "/api/v1/statuses" and request.method == "POST":
|
||||
body = json.loads(request.content)
|
||||
@@ -271,6 +273,51 @@ class TestMedia:
|
||||
assert server.statuses["1"]["media_ids"] == ["m1"]
|
||||
|
||||
|
||||
class TestInlineImages:
|
||||
"""Clients like OutPost upload a photo and embed it in the content (v2.6.6)"""
|
||||
|
||||
def _upload(self, app):
|
||||
from starpunk.media import save_media
|
||||
|
||||
buf = io.BytesIO()
|
||||
Image.new("RGB", (800, 600), "orange").save(buf, format="JPEG")
|
||||
with app.test_request_context():
|
||||
return save_media(buf.getvalue(), "cat.jpg")
|
||||
|
||||
def test_embedded_own_image_is_attached(self, app, server):
|
||||
media = self._upload(app)
|
||||
url = f"https://blog.example.com/media/{media['path']}"
|
||||
publish(app, content=f"")
|
||||
status = server.statuses["1"]
|
||||
assert status["media_ids"] == ["m1"]
|
||||
assert b"Tortie cat on a chair" in server.uploads[0]
|
||||
assert "![" not in status["status"] and url not in status["status"]
|
||||
assert status["status"].startswith("https://blog.example.com/note/")
|
||||
|
||||
def test_text_around_image_is_kept(self, app, server):
|
||||
media = self._upload(app)
|
||||
publish(app, content=f"Look\n\n<img src=\"/media/{media['path']}\" alt=\"cat\">\n\nNice")
|
||||
status = server.statuses["1"]
|
||||
assert status["media_ids"] == ["m1"]
|
||||
assert status["status"].startswith("Look\n\nNice")
|
||||
|
||||
def test_image_both_attached_and_embedded_is_sent_once(self, app, server):
|
||||
from starpunk.media import attach_media_to_note
|
||||
|
||||
media = self._upload(app)
|
||||
with app.test_request_context():
|
||||
note = create_note(content=f"", published=True)
|
||||
attach_media_to_note(note.id, [media["id"]], ["caption"])
|
||||
select_targets(note.id, ["gotosocial"])
|
||||
process_due()
|
||||
assert server.statuses["1"]["media_ids"] == ["m1"]
|
||||
|
||||
def test_other_sites_images_are_ignored(self, app, server):
|
||||
publish(app, content=" hi")
|
||||
assert server.statuses["1"]["media_ids"] == []
|
||||
assert server.uploads == []
|
||||
|
||||
|
||||
class TestEntryPoints:
|
||||
@pytest.fixture
|
||||
def admin(self, app):
|
||||
@@ -309,7 +356,8 @@ class TestEntryPoints:
|
||||
).get_json()
|
||||
assert data["syndicate-to"] == [{"uid": f"{GTS}/", "name": "GoToSocial"}]
|
||||
|
||||
def test_micropub_default_is_all_targets(self, app):
|
||||
def test_micropub_default_is_no_targets(self, app):
|
||||
"""Absent mp-syndicate-to means no crossposting (v2.9.0)"""
|
||||
token = self._token(app)
|
||||
response = app.test_client().post(
|
||||
"/micropub", data={"h": "entry", "content": "via quill"},
|
||||
@@ -320,7 +368,7 @@ class TestEntryPoints:
|
||||
assert response.headers["Location"].startswith("https://blog.example.com/note/")
|
||||
with app.app_context():
|
||||
from starpunk.database import get_db
|
||||
assert get_db(app).execute("SELECT COUNT(*) FROM syndications").fetchone()[0] == 1
|
||||
assert get_db(app).execute("SELECT COUNT(*) FROM syndications").fetchone()[0] == 0
|
||||
|
||||
def test_micropub_explicit_empty_choice(self, app):
|
||||
token = self._token(app)
|
||||
|
||||
@@ -92,8 +92,31 @@ def pds():
|
||||
return FakePDS()
|
||||
|
||||
|
||||
class FakeWeb:
|
||||
"""Other people's pages, for link cards: url -> (status, headers, body)"""
|
||||
def __init__(self):
|
||||
self.pages = {}
|
||||
self.fetched = []
|
||||
|
||||
def __call__(self, request):
|
||||
url = str(request.url)
|
||||
self.fetched.append(url)
|
||||
status, headers, body = self.pages.get(url, (404, {}, b"nope"))
|
||||
return httpx.Response(status, headers=headers, content=body)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def app(tmp_path, pds):
|
||||
def web():
|
||||
return FakeWeb()
|
||||
|
||||
|
||||
def routed(pds, web):
|
||||
return httpx.MockTransport(
|
||||
lambda r: pds(r) if r.url.host in ("bsky.example", "pds.example") else web(r))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def app(tmp_path, pds, web):
|
||||
data = tmp_path / "data"
|
||||
app = create_app({
|
||||
"TESTING": True,
|
||||
@@ -108,7 +131,7 @@ def app(tmp_path, pds):
|
||||
"BLUESKY_SERVICE": SERVICE,
|
||||
})
|
||||
syndication._client_cache[f"bluesky:{SERVICE}:phil.example"] = BlueskyClient(
|
||||
SERVICE, "phil.example", "app-pass", http=httpx.Client(transport=httpx.MockTransport(pds))
|
||||
SERVICE, "phil.example", "app-pass", http=httpx.Client(transport=routed(pds, web))
|
||||
)
|
||||
yield app
|
||||
syndication._client_cache.clear()
|
||||
@@ -258,6 +281,101 @@ class TestMedia:
|
||||
assert image["aspectRatio"]["width"] > 0
|
||||
|
||||
|
||||
ARTICLE = "https://news.example/story"
|
||||
ARTICLE_HTML = b"""<html><head><title>Fallback title</title>
|
||||
<meta property="og:title" content="A Great Story">
|
||||
<meta property="og:description" content="What happened, in short.">
|
||||
<meta property="og:image" content="/img/story.png">
|
||||
</head><body>...</body></html>"""
|
||||
|
||||
|
||||
def png_bytes():
|
||||
buf = io.BytesIO()
|
||||
Image.new("RGB", (60, 40), (10, 120, 200)).save(buf, format="PNG")
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
class TestLinkCards:
|
||||
def test_note_link_gets_the_card(self, app, pds, web):
|
||||
web.pages[ARTICLE] = (200, {"content-type": "text/html"}, ARTICLE_HTML)
|
||||
web.pages["https://news.example/img/story.png"] = (200, {"content-type": "image/png"}, png_bytes())
|
||||
note = publish(app, content=f"Read [this story]({ARTICLE}) now")
|
||||
record = only_record(pds)
|
||||
external = record["embed"]["external"]
|
||||
assert external["uri"] == ARTICLE
|
||||
assert external["title"] == "A Great Story"
|
||||
assert external["description"] == "What happened, in short."
|
||||
assert external["thumb"]["$type"] == "blob"
|
||||
# The link text is clickable and the note's permalink is still in the text
|
||||
permalink = f"https://blog.example.com/note/{note.slug}"
|
||||
assert record["text"].startswith("Read this story now") and permalink in record["text"]
|
||||
raw = record["text"].encode()
|
||||
links = {raw[f["index"]["byteStart"]:f["index"]["byteEnd"]].decode(): f["features"][0]["uri"]
|
||||
for f in record["facets"]}
|
||||
assert links["this story"] == ARTICLE
|
||||
assert links[permalink] == permalink
|
||||
|
||||
def test_bare_url_gets_the_card(self, app, pds, web):
|
||||
web.pages[ARTICLE] = (200, {"content-type": "text/html"}, ARTICLE_HTML)
|
||||
publish(app, content=f"See {ARTICLE}.")
|
||||
assert only_record(pds)["embed"]["external"]["uri"] == ARTICLE
|
||||
|
||||
def test_unreachable_page_still_gets_a_card(self, app, pds, web):
|
||||
note = publish(app, content=f"See {ARTICLE}")
|
||||
external = only_record(pds)["embed"]["external"]
|
||||
assert external == {"uri": ARTICLE, "title": ARTICLE, "description": ""}
|
||||
assert row(app, note)["state"] == "synced"
|
||||
|
||||
def test_redirect_to_private_address_is_not_followed(self, app, pds, web, monkeypatch):
|
||||
web.pages[ARTICLE] = (302, {"location": "http://internal.example/admin"}, b"")
|
||||
monkeypatch.setattr("starpunk.micropub._public_address", lambda host: host != "internal.example")
|
||||
app.config["TESTING"] = False
|
||||
try:
|
||||
publish(app, content=f"See {ARTICLE}")
|
||||
finally:
|
||||
app.config["TESTING"] = True
|
||||
assert "http://internal.example/admin" not in web.fetched
|
||||
assert only_record(pds)["embed"]["external"]["title"] == ARTICLE
|
||||
|
||||
def test_images_win_over_link_card(self, app, pds, web):
|
||||
from starpunk.media import attach_media_to_note, save_media
|
||||
with app.test_request_context():
|
||||
note = create_note(content=f"Look {ARTICLE}", published=True)
|
||||
media = save_media(png_bytes(), "a.png")
|
||||
attach_media_to_note(note.id, [media["id"]], ["pic"])
|
||||
select_targets(note.id, ["bluesky"])
|
||||
process_due()
|
||||
assert only_record(pds)["embed"]["$type"] == "app.bsky.embed.images"
|
||||
assert web.fetched == []
|
||||
|
||||
|
||||
class TestLinkText:
|
||||
def test_note_links_in_reading_order(self):
|
||||
from starpunk.syndication import note_links
|
||||
md = "a [x](https://one.example) b https://two.example/p. c <https://three.example> [rel](/local)"
|
||||
assert note_links(md) == ["https://one.example", "https://two.example/p", "https://three.example"]
|
||||
|
||||
def test_link_modes(self):
|
||||
from starpunk.syndication import plain_text_links
|
||||
md = "[a story](https://s.example) and [https://u.example](https://u.example)"
|
||||
assert plain_text_links(md)[0] == "a story and https://u.example"
|
||||
assert plain_text_links(md, "inline")[0] == "a story (https://s.example) and https://u.example"
|
||||
assert plain_text_links(md, "anchor") == ("a story and https://u.example",
|
||||
[("a story", "https://s.example")])
|
||||
|
||||
def test_inline_url_budget(self):
|
||||
from starpunk.syndication import plain_text_links
|
||||
md = "https://bare.example then [one](https://1.example) [two](https://2.example) [one again](https://1.example)"
|
||||
text = plain_text_links(md, "inline", max_urls=2)[0]
|
||||
assert text == "https://bare.example then one (https://1.example) two one again (https://1.example)"
|
||||
|
||||
def test_anchor_facet_byte_offsets(self):
|
||||
text = "Café ☕ this story and more"
|
||||
(facet,) = bluesky_facets(text, [("this story", "https://s.example")])
|
||||
raw = text.encode()
|
||||
assert raw[facet["index"]["byteStart"]:facet["index"]["byteEnd"]] == b"this story"
|
||||
|
||||
|
||||
def test_both_targets_listed_for_micropub(tmp_path):
|
||||
data = tmp_path / "d"
|
||||
app = create_app({
|
||||
|
||||
@@ -0,0 +1,418 @@
|
||||
"""
|
||||
Tests for crossposting to Threads (starpunk/syndication.py, v2.6.0)
|
||||
|
||||
A fake Graph API (httpx.MockTransport) records what StarPunk sends.
|
||||
"""
|
||||
|
||||
import io
|
||||
import json
|
||||
import time
|
||||
from urllib.parse import parse_qsl
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from starpunk import create_app, syndication
|
||||
from starpunk.notes import create_note, delete_note, update_note
|
||||
from starpunk.syndication import (
|
||||
THREADS_REFRESH_EVERY,
|
||||
ThreadsClient,
|
||||
get_note_syndications,
|
||||
process_due,
|
||||
select_targets,
|
||||
threads_length,
|
||||
threads_topic,
|
||||
)
|
||||
|
||||
API = "https://graph.threads.example/v1.0"
|
||||
UID = "1789"
|
||||
|
||||
|
||||
class FakeThreads:
|
||||
def __init__(self):
|
||||
self.containers = {}
|
||||
self.posts = {}
|
||||
self.calls = []
|
||||
self.token = "tok1"
|
||||
self.refreshes = 0
|
||||
self.refresh_fails = False
|
||||
self.pending_polls = 0
|
||||
self.lose_next_publish = False
|
||||
self.fail_next = None
|
||||
self.fail_next_publish = False
|
||||
|
||||
def _json(self, code, body):
|
||||
return httpx.Response(code, json=body)
|
||||
|
||||
def _error(self, code, message, err_code, subcode=None):
|
||||
error = {"message": message, "type": "OAuthException", "code": err_code}
|
||||
if subcode:
|
||||
error["error_subcode"] = subcode
|
||||
return self._json(code, {"error": error})
|
||||
|
||||
def __call__(self, request: httpx.Request) -> httpx.Response:
|
||||
params = dict(request.url.params)
|
||||
if request.content:
|
||||
params.update(parse_qsl(request.content.decode()))
|
||||
path = request.url.path
|
||||
self.calls.append((request.method, path, params))
|
||||
|
||||
if path == "/refresh_access_token":
|
||||
assert params["grant_type"] == "th_refresh_token"
|
||||
if self.refresh_fails or params.get("access_token") != self.token:
|
||||
return self._error(400, "token too new", 190)
|
||||
self.refreshes += 1
|
||||
self.token = f"tok{self.refreshes + 1}"
|
||||
return self._json(200, {"access_token": self.token, "token_type": "bearer",
|
||||
"expires_in": 5184000})
|
||||
|
||||
if params.get("access_token") != self.token:
|
||||
return self._error(400, "Invalid OAuth access token", 190)
|
||||
if self.fail_next:
|
||||
code, err, self.fail_next = *self.fail_next, None
|
||||
return self._error(code, "nope", err)
|
||||
|
||||
path = path.removeprefix("/v1.0/")
|
||||
if path == "me":
|
||||
return self._json(200, {"id": UID, "username": "phil"})
|
||||
if path == f"{UID}/threads" and request.method == "POST":
|
||||
cid = f"c{len(self.containers) + 1}"
|
||||
self.containers[cid] = params
|
||||
return self._json(200, {"id": cid})
|
||||
if path == f"{UID}/threads" and request.method == "GET":
|
||||
return self._json(200, {"data": [
|
||||
{"id": pid, "text": p["text"], "permalink": f"https://www.threads.com/@phil/post/{pid}"}
|
||||
for pid, p in self.posts.items()]})
|
||||
if path == f"{UID}/threads_publish" and self.fail_next_publish:
|
||||
self.fail_next_publish = False
|
||||
return self._error(400, "Fatal", 24, 4279009)
|
||||
if path == f"{UID}/threads_publish":
|
||||
container = self.containers[params["creation_id"]]
|
||||
pid = f"p{len(self.posts) + 1}"
|
||||
self.posts[pid] = container
|
||||
if self.lose_next_publish:
|
||||
self.lose_next_publish = False
|
||||
raise httpx.ReadTimeout("lost reply")
|
||||
return self._json(200, {"id": pid})
|
||||
if path in self.containers and request.method == "GET":
|
||||
if self.pending_polls:
|
||||
self.pending_polls -= 1
|
||||
return self._json(200, {"status": "IN_PROGRESS"})
|
||||
return self._json(200, {"status": "FINISHED"})
|
||||
if path in self.posts and request.method == "GET":
|
||||
return self._json(200, {"id": path, "permalink": f"https://www.threads.com/@phil/post/{path}"})
|
||||
if request.method == "DELETE":
|
||||
if self.posts.pop(path, None) is None:
|
||||
return self._error(400, "Object does not exist", 100, 33)
|
||||
return self._json(200, {"success": True, "deleted_id": path})
|
||||
return self._json(500, {"error": {"message": "unknown", "code": 2}})
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def api():
|
||||
return FakeThreads()
|
||||
|
||||
|
||||
def make_client(app, api, token="tok1"):
|
||||
return ThreadsClient(token, app.config["DATA_PATH"] / "threads_token.json", api=API,
|
||||
http=httpx.Client(transport=httpx.MockTransport(api)), sleep=lambda s: None)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def app(tmp_path, api):
|
||||
data = tmp_path / "data"
|
||||
app = create_app({
|
||||
"TESTING": True,
|
||||
"SITE_URL": "https://blog.example.com",
|
||||
"ADMIN_ME": "https://example.com",
|
||||
"SESSION_SECRET": "test-secret",
|
||||
"DATA_PATH": data,
|
||||
"NOTES_PATH": data / "notes",
|
||||
"DATABASE_PATH": data / "starpunk.db",
|
||||
"THREADS_ACCESS_TOKEN": "tok1",
|
||||
"THREADS_USERNAME": "phil",
|
||||
})
|
||||
syndication._client_cache["threads"] = make_client(app, api)
|
||||
yield app
|
||||
syndication._client_cache.clear()
|
||||
|
||||
|
||||
def publish(app, content="Hello Threads", tags=None):
|
||||
with app.test_request_context():
|
||||
note = create_note(content=content, published=True, tags=tags)
|
||||
select_targets(note.id, ["threads"])
|
||||
process_due()
|
||||
return note
|
||||
|
||||
|
||||
def only_post(api):
|
||||
(post,) = api.posts.values()
|
||||
return post
|
||||
|
||||
|
||||
def row(app, note):
|
||||
with app.app_context():
|
||||
(r,) = get_note_syndications(note.id)
|
||||
return r
|
||||
|
||||
|
||||
def image_bytes(fmt="JPEG", size=(800, 600)):
|
||||
buf = io.BytesIO()
|
||||
Image.new("RGB", size, (200, 30, 30)).save(buf, format=fmt)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
class TestText:
|
||||
def test_emoji_count_as_bytes(self):
|
||||
assert threads_length("abc") == 3
|
||||
assert threads_length("☕") == 3
|
||||
|
||||
def test_topic_is_first_usable_tag(self):
|
||||
assert threads_topic(["...", "Indie.Web & Co", "x"]) == "IndieWeb Co"
|
||||
assert threads_topic([]) == ""
|
||||
|
||||
|
||||
class TestLifecycle:
|
||||
def test_publish_text_post(self, app, api):
|
||||
note = publish(app, content="Hello **Threads**", tags=["IndieWeb", "Second"])
|
||||
post = only_post(api)
|
||||
permalink = f"https://blog.example.com/note/{note.slug}"
|
||||
assert post["media_type"] == "TEXT"
|
||||
assert post["text"] == f"Hello Threads\n\n{permalink}"
|
||||
assert "link_attachment" not in post # Threads previews the first link itself
|
||||
assert post["topic_tag"] == "IndieWeb"
|
||||
r = row(app, note)
|
||||
assert r["state"] == "synced"
|
||||
assert r["remote_id"] == "p1"
|
||||
assert r["remote_url"] == "https://www.threads.com/@phil/post/p1"
|
||||
|
||||
def test_long_note_fits_500(self, app, api):
|
||||
note = publish(app, content="☕ word " * 300)
|
||||
text = only_post(api)["text"]
|
||||
assert threads_length(text) <= 500
|
||||
assert text.endswith(f"https://blog.example.com/note/{note.slug}")
|
||||
|
||||
def test_waits_for_container(self, app, api):
|
||||
api.pending_polls = 2
|
||||
note = publish(app)
|
||||
assert row(app, note)["state"] == "synced"
|
||||
|
||||
def test_edit_leaves_post_alone(self, app, api):
|
||||
note = publish(app, content="first")
|
||||
with app.test_request_context():
|
||||
update_note(id=note.id, content="second")
|
||||
process_due()
|
||||
assert only_post(api)["text"].startswith("first")
|
||||
assert row(app, note)["state"] == "synced"
|
||||
|
||||
def test_delete_and_unpublish_remove_post(self, app, api):
|
||||
note = publish(app)
|
||||
with app.test_request_context():
|
||||
update_note(id=note.id, published=False)
|
||||
process_due()
|
||||
assert api.posts == {}
|
||||
update_note(id=note.id, published=True)
|
||||
process_due()
|
||||
assert len(api.posts) == 1
|
||||
delete_note(id=note.id)
|
||||
process_due()
|
||||
assert api.posts == {}
|
||||
|
||||
def test_post_deleted_on_threads_is_fine(self, app, api):
|
||||
note = publish(app)
|
||||
api.posts.clear()
|
||||
with app.test_request_context():
|
||||
delete_note(id=note.id)
|
||||
process_due()
|
||||
assert row(app, note)["state"] == "synced"
|
||||
|
||||
|
||||
class TestErrors:
|
||||
def test_lost_reply_does_not_double_post(self, app, api):
|
||||
api.lose_next_publish = True
|
||||
note = publish(app)
|
||||
assert row(app, note)["state"] == "pending"
|
||||
with app.test_request_context():
|
||||
from starpunk.database import get_db
|
||||
get_db(app).execute("UPDATE syndications SET next_attempt_at = datetime('now')")
|
||||
get_db(app).commit()
|
||||
process_due()
|
||||
assert len(api.posts) == 1
|
||||
assert row(app, note)["state"] == "synced"
|
||||
|
||||
def test_publish_fatal_is_retried(self, app, api):
|
||||
api.fail_next_publish = True
|
||||
note = publish(app)
|
||||
r = row(app, note)
|
||||
assert r["state"] == "pending" and "Fatal" in r["last_error"] and "subcode 4279009" in r["last_error"]
|
||||
with app.test_request_context():
|
||||
from starpunk.database import get_db
|
||||
get_db(app).execute("UPDATE syndications SET next_attempt_at = datetime('now')")
|
||||
get_db(app).commit()
|
||||
process_due()
|
||||
assert len(api.posts) == 1
|
||||
assert row(app, note)["state"] == "synced"
|
||||
|
||||
def test_bad_token_fails_permanently(self, app, api):
|
||||
api.token = "other"
|
||||
note = publish(app)
|
||||
r = row(app, note)
|
||||
assert r["state"] == "failed" and "Invalid OAuth" in r["last_error"]
|
||||
assert "tok1" not in r["last_error"]
|
||||
|
||||
def test_rate_limit_retries(self, app, api):
|
||||
publish(app, content="warm up")
|
||||
api.fail_next = (400, 4) # Graph "application request limit reached"
|
||||
note = publish(app, content="later")
|
||||
assert row(app, note)["state"] == "pending"
|
||||
|
||||
|
||||
class TestLinks:
|
||||
def test_markdown_link_comes_first_for_the_preview(self, app, api):
|
||||
note = publish(app, content="Read [this story](https://news.example/story) now")
|
||||
text = only_post(api)["text"]
|
||||
assert text.startswith("Read this story (https://news.example/story) now")
|
||||
assert text.index("https://news.example/story") < text.index(f"/note/{note.slug}")
|
||||
|
||||
def test_stays_within_five_links(self, app, api):
|
||||
import re
|
||||
links = " ".join(f"[l{i}](https://l{i}.example)" for i in range(7))
|
||||
publish(app, content=f"Lots: {links}")
|
||||
text = only_post(api)["text"]
|
||||
assert len(set(re.findall(r"https?://[^\s)]+", text))) == 5
|
||||
assert "l6" in text # the extra links keep their text
|
||||
|
||||
|
||||
class TestMedia:
|
||||
def _note_with(self, app, images):
|
||||
from starpunk.media import attach_media_to_note, save_media
|
||||
with app.test_request_context():
|
||||
note = create_note(content="pics", published=True)
|
||||
ids = [save_media(data, name)["id"] for data, name in images]
|
||||
attach_media_to_note(note.id, ids, [f"alt {i}" for i in range(len(ids))])
|
||||
select_targets(note.id, ["threads"])
|
||||
process_due()
|
||||
return note
|
||||
|
||||
def test_single_image(self, app, api):
|
||||
self._note_with(app, [(image_bytes(), "a.jpg")])
|
||||
post = only_post(api)
|
||||
assert post["media_type"] == "IMAGE"
|
||||
assert post["image_url"].startswith("https://blog.example.com/media/")
|
||||
assert post["alt_text"] == "alt 0"
|
||||
assert "link_attachment" not in post
|
||||
|
||||
def test_carousel(self, app, api):
|
||||
self._note_with(app, [(image_bytes(), "a.jpg"), (image_bytes("PNG"), "b.png")])
|
||||
post = only_post(api)
|
||||
assert post["media_type"] == "CAROUSEL"
|
||||
children = [api.containers[c] for c in post["children"].split(",")]
|
||||
assert [c["is_carousel_item"] for c in children] == ["true", "true"]
|
||||
assert [c["alt_text"] for c in children] == ["alt 0", "alt 1"]
|
||||
|
||||
def test_unsupported_format_is_left_off(self, app, api):
|
||||
self._note_with(app, [(image_bytes("WEBP"), "a.webp")])
|
||||
assert only_post(api)["media_type"] == "TEXT"
|
||||
|
||||
|
||||
class TestToken:
|
||||
def test_refreshes_daily_and_saves(self, app, api):
|
||||
client = syndication._client_cache["threads"]
|
||||
with app.app_context():
|
||||
client.maintain()
|
||||
assert api.refreshes == 1
|
||||
client.maintain() # same day: nothing
|
||||
assert api.refreshes == 1
|
||||
saved = json.loads((app.config["DATA_PATH"] / "threads_token.json").read_text())
|
||||
assert saved["token"] == "tok2"
|
||||
# A restart picks up the saved token
|
||||
assert make_client(app, api)._token == "tok2"
|
||||
publish(app)
|
||||
assert len(api.posts) == 1
|
||||
|
||||
def test_other_worker_refresh_is_reused(self, app, api):
|
||||
first, second = make_client(app, api), make_client(app, api)
|
||||
with app.app_context():
|
||||
first.maintain()
|
||||
second.maintain()
|
||||
assert api.refreshes == 1
|
||||
assert second._token == "tok2"
|
||||
|
||||
def test_refresh_due_after_a_day(self, app, api, monkeypatch):
|
||||
client = syndication._client_cache["threads"]
|
||||
with app.app_context():
|
||||
client.maintain()
|
||||
later = time.time() + THREADS_REFRESH_EVERY + 1
|
||||
monkeypatch.setattr(syndication.time, "time", lambda: later)
|
||||
client.maintain()
|
||||
assert api.refreshes == 2
|
||||
|
||||
def test_failed_refresh_keeps_token(self, app, api):
|
||||
api.refresh_fails = True
|
||||
client = syndication._client_cache["threads"]
|
||||
with app.app_context():
|
||||
client.maintain()
|
||||
assert client._token == "tok1"
|
||||
publish(app)
|
||||
assert len(api.posts) == 1
|
||||
|
||||
def test_new_config_token_replaces_saved_one(self, app, api):
|
||||
with app.app_context():
|
||||
syndication._client_cache["threads"].maintain()
|
||||
api.token = "fresh"
|
||||
assert make_client(app, api, token="fresh")._token == "fresh"
|
||||
|
||||
def test_worker_runs_upkeep(self, app, api):
|
||||
with app.app_context():
|
||||
process_due()
|
||||
assert api.refreshes == 1
|
||||
|
||||
|
||||
def test_all_targets_listed_for_micropub(tmp_path):
|
||||
data = tmp_path / "d"
|
||||
app = create_app({
|
||||
"TESTING": True, "SITE_URL": "https://b.example", "ADMIN_ME": "https://e.example",
|
||||
"SESSION_SECRET": "x", "DATA_PATH": data, "NOTES_PATH": data / "notes",
|
||||
"DATABASE_PATH": data / "s.db",
|
||||
"GOTOSOCIAL_URL": "https://social.example", "GOTOSOCIAL_ACCESS_TOKEN": "t",
|
||||
"BLUESKY_HANDLE": "phil.example", "BLUESKY_APP_PASSWORD": "p",
|
||||
"THREADS_ACCESS_TOKEN": "x", "THREADS_USERNAME": "@phil",
|
||||
})
|
||||
with app.app_context():
|
||||
from starpunk.syndication import get_targets, resolve_targets
|
||||
assert set(get_targets()) == {"gotosocial", "bluesky", "threads"}
|
||||
assert resolve_targets(["https://www.threads.com/@phil"]) == ["threads"]
|
||||
syndication._client_cache.clear()
|
||||
|
||||
|
||||
class TestLikesReposts:
|
||||
PAGE = "https://other.example/post/1"
|
||||
|
||||
def _post(self, app, monkeypatch, content="", **kind):
|
||||
monkeypatch.setattr("starpunk.reply_context.fetch_reply_context", lambda *a, **k: None)
|
||||
with app.test_request_context():
|
||||
note = create_note(content=content, published=True, **kind)
|
||||
select_targets(note.id, ["threads"])
|
||||
process_due()
|
||||
return note
|
||||
|
||||
def test_like_is_not_crossposted(self, app, api, monkeypatch):
|
||||
note = self._post(app, monkeypatch, like_of=self.PAGE)
|
||||
assert api.posts == {}
|
||||
r = row(app, note)
|
||||
assert r["state"] == "synced" and r["remote_id"] is None
|
||||
|
||||
def test_repost_posts_link(self, app, api, monkeypatch):
|
||||
self._post(app, monkeypatch, repost_of=self.PAGE)
|
||||
post = only_post(api)
|
||||
assert post["text"].startswith("🔁") and self.PAGE in post["text"]
|
||||
assert "link_attachment" not in post
|
||||
|
||||
def test_repost_with_comment_keeps_comment(self, app, api, monkeypatch):
|
||||
note = self._post(app, monkeypatch, content="So **true**", repost_of=self.PAGE)
|
||||
assert only_post(api)["text"].startswith("So true")
|
||||
with app.test_request_context():
|
||||
delete_note(id=note.id)
|
||||
process_due()
|
||||
assert api.posts == {}
|
||||
@@ -0,0 +1,86 @@
|
||||
"""
|
||||
Tags keep the order they were entered in (v2.6.4)
|
||||
|
||||
Public pages list tags alphabetically; crossposts (the Threads topic, which
|
||||
tags fit on Bluesky), the editor and Micropub use the entered order, and
|
||||
saving from the editor or Micropub keeps it.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from starpunk import create_app
|
||||
from starpunk.auth import create_session
|
||||
from starpunk.indieauth import create_access_token
|
||||
from starpunk.notes import create_note, get_note
|
||||
from starpunk.tags import get_note_tags
|
||||
|
||||
ENTERED = ["Zebra", "apple", "Mango"]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def app(tmp_path):
|
||||
data = tmp_path / "data"
|
||||
data.mkdir()
|
||||
yield create_app(config={
|
||||
"TESTING": True, "DATABASE_PATH": data / "starpunk.db", "DATA_PATH": data,
|
||||
"NOTES_PATH": data / "notes", "SESSION_SECRET": "s", "ADMIN_ME": "https://test.example.com",
|
||||
"SITE_URL": "http://localhost:5000", "DEV_MODE": False,
|
||||
})
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def note(app):
|
||||
with app.test_request_context():
|
||||
return create_note(content="Tagged", published=True, tags=ENTERED)
|
||||
|
||||
|
||||
def names(app, note_id, **kw):
|
||||
with app.app_context():
|
||||
return [t["display_name"] for t in get_note_tags(note_id, **kw)]
|
||||
|
||||
|
||||
def test_alphabetical_by_default_entered_on_request(app, note):
|
||||
assert names(app, note.id) == ["apple", "Mango", "Zebra"]
|
||||
assert names(app, note.id, entered_order=True) == ENTERED
|
||||
|
||||
|
||||
def test_public_page_stays_alphabetical(app, note):
|
||||
html = app.test_client().get(f"/note/{note.slug}").data.decode()
|
||||
assert html.index(">apple<") < html.index(">Mango<") < html.index(">Zebra<")
|
||||
|
||||
|
||||
def test_editor_shows_and_keeps_entered_order(app, note):
|
||||
client = app.test_client()
|
||||
with app.test_request_context():
|
||||
client.set_cookie("starpunk_session", create_session("https://test.example.com"))
|
||||
html = client.get(f"/admin/edit/{note.id}").data.decode()
|
||||
assert 'value="Zebra, apple, Mango"' in html
|
||||
# Saving the form unchanged keeps the order
|
||||
response = client.post(f"/admin/edit/{note.id}", data={
|
||||
"content": "Tagged, edited", "published": "on", "tags": "Zebra, apple, Mango"})
|
||||
assert response.status_code in (200, 302)
|
||||
assert names(app, note.id, entered_order=True) == ENTERED
|
||||
|
||||
|
||||
def test_micropub_source_and_update_keep_entered_order(app, note):
|
||||
with app.test_request_context():
|
||||
token = create_access_token("https://quill.example/", "create update")["access_token"]
|
||||
client = app.test_client()
|
||||
auth = {"Authorization": f"Bearer {token}"}
|
||||
url = f"http://localhost:5000/note/{note.slug}"
|
||||
source = client.get(f"/micropub?q=source&url={url}", headers=auth).get_json()
|
||||
assert source["properties"]["category"] == ENTERED
|
||||
response = client.post("/micropub", headers=auth, json={
|
||||
"action": "update", "url": url, "add": {"category": ["Kiwi"]}})
|
||||
assert response.status_code in (200, 204), response.get_data(as_text=True)
|
||||
assert names(app, note.id, entered_order=True) == ENTERED + ["Kiwi"]
|
||||
|
||||
|
||||
def test_crossposts_use_entered_order(app, note):
|
||||
from starpunk.syndication import compose_text, threads_topic
|
||||
tags = names(app, note.id, entered_order=True)
|
||||
assert threads_topic(tags) == "Zebra"
|
||||
with app.app_context():
|
||||
text = compose_text(get_note(id=note.id, load_content=True),
|
||||
"https://x.example/note/1", tags, 25)
|
||||
assert "#Zebra" in text and "#Mango" not in text # room for the first tags only
|
||||
@@ -0,0 +1,364 @@
|
||||
"""
|
||||
W3C Webmention conformance tests (v2.7.0)
|
||||
|
||||
https://www.w3.org/TR/webmention/ — section numbers in test names.
|
||||
"""
|
||||
|
||||
import sqlite3
|
||||
from unittest.mock import patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from starpunk import webmention_ssrf
|
||||
from starpunk.webmention_processor import (
|
||||
TransientError, ValidationError, fetch_source, verify_link_presence,
|
||||
)
|
||||
from starpunk.webmention_worker import WebmentionWorker
|
||||
|
||||
PUBLIC = [(2, 1, 6, '', ('93.184.216.34', 0))]
|
||||
|
||||
|
||||
def original_url(request):
|
||||
"""The URL a pinned request was for (connections go to the checked IP)"""
|
||||
return f"{request.url.scheme}://{request.headers['host']}{request.url.raw_path.decode()}"
|
||||
BASE = 'https://t.example/test/1?x=1'
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def worker(app):
|
||||
return WebmentionWorker(app, db_path=app.config["DATABASE_PATH"],
|
||||
site_url=app.config["SITE_URL"])
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_web():
|
||||
"""Route safe_get through a MockTransport; all hosts resolve public"""
|
||||
routes = {}
|
||||
|
||||
def handler(request):
|
||||
return routes[original_url(request)](request)
|
||||
|
||||
with patch("starpunk.webmention_ssrf.socket.getaddrinfo", return_value=PUBLIC), \
|
||||
patch("starpunk.webmention_ssrf.TRANSPORT", httpx.MockTransport(handler)):
|
||||
yield routes
|
||||
|
||||
|
||||
# 3.1.2 Sender discovers receiver Webmention endpoint
|
||||
|
||||
@pytest.mark.parametrize("header,expected", [
|
||||
('<https://e.example/wm>; rel="webmention"', 'https://e.example/wm'),
|
||||
('</wm>; rel=webmention', 'https://t.example/wm'),
|
||||
('<wm>; rel="other webmention"', 'https://t.example/test/wm'),
|
||||
('<wm>; title="a;b,c"; rel="webmention"', 'https://t.example/test/wm'),
|
||||
('<wm>; REL="WebMention"', 'https://t.example/test/wm'),
|
||||
('<a>; rel="other", </wm?q=1>; rel="webmention"', 'https://t.example/wm?q=1'),
|
||||
('<x>; rel="webmention-not"', None),
|
||||
('<x>; rel="not-webmention"', None),
|
||||
])
|
||||
def test_3_1_2_link_header(worker, header, expected):
|
||||
assert worker._parse_link_header(header, BASE) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("html,expected", [
|
||||
# <a> before <link>: first in document order wins
|
||||
('<a rel="webmention" href="/a">x</a><link rel="webmention" href="/l">', 'https://t.example/a'),
|
||||
('<link rel="webmention somethingelse" href="/l">', 'https://t.example/l'),
|
||||
('<link rel="Webmention" href="/l">', 'https://t.example/l'),
|
||||
# empty href is the target itself
|
||||
('<link rel="webmention" href="">', BASE),
|
||||
# no href: skipped
|
||||
('<link rel="webmention"><a rel="webmention" href="/a">', 'https://t.example/a'),
|
||||
# comments, escaped markup and other rels are ignored
|
||||
('<!-- <link rel="webmention" href="/bad"> -->'
|
||||
'<a rel="not-webmention" href="/bad"></a>'
|
||||
'<code><a rel="webmention" href="/bad"></code>'
|
||||
'<link rel="webmention" href="/ok">', 'https://t.example/ok'),
|
||||
('<p>nothing</p>', None),
|
||||
])
|
||||
def test_3_1_2_html(worker, html, expected):
|
||||
assert worker._parse_html_for_endpoint(html, BASE) == expected
|
||||
|
||||
|
||||
def test_3_1_2_header_before_html_and_relative_to_final_url(app, worker, mock_web):
|
||||
mock_web['https://t.example/old'] = lambda r: httpx.Response(
|
||||
301, headers={'Location': 'https://t.example/new/page'})
|
||||
mock_web['https://t.example/new/page'] = lambda r: httpx.Response(
|
||||
200, headers={'Link': '<wm>; rel="webmention"', 'Content-Type': 'text/html'},
|
||||
text='<link rel="webmention" href="/html">')
|
||||
with app.app_context():
|
||||
assert worker._discover_endpoint('https://t.example/old') == 'https://t.example/new/wm'
|
||||
|
||||
|
||||
def test_3_1_2_multiple_link_headers(app, worker, mock_web):
|
||||
mock_web['https://t.example/p'] = lambda r: httpx.Response(
|
||||
200, headers=[('Link', '</other>; rel="other"'), ('Link', '</wm>; rel="webmention"')])
|
||||
with app.app_context():
|
||||
assert worker._discover_endpoint('https://t.example/p') == 'https://t.example/wm'
|
||||
|
||||
|
||||
def test_3_1_2_server_error_is_retried_not_final(app, worker, mock_web):
|
||||
mock_web['https://t.example/p'] = lambda r: httpx.Response(503)
|
||||
with app.app_context(), pytest.raises(httpx.HTTPError):
|
||||
worker._discover_endpoint('https://t.example/p')
|
||||
|
||||
|
||||
# 4.5 SSRF: every redirect hop is checked
|
||||
|
||||
def test_4_5_redirect_to_private_address_blocked(app):
|
||||
def resolve(host, *a, **k):
|
||||
ip = '169.254.169.254' if host == 'metadata.internal' else '93.184.216.34'
|
||||
return [(2, 1, 6, '', (ip, 0))]
|
||||
|
||||
def handler(request):
|
||||
assert request.url.host == '93.184.216.34' # never connects to the private one
|
||||
return httpx.Response(302, headers={'Location': 'http://metadata.internal/'})
|
||||
|
||||
with app.app_context(), \
|
||||
patch("starpunk.webmention_ssrf.socket.getaddrinfo", side_effect=resolve), \
|
||||
patch("starpunk.webmention_ssrf.TRANSPORT", httpx.MockTransport(handler)):
|
||||
with pytest.raises(ValidationError, match="Blocked"):
|
||||
fetch_source('https://evil.example/', 'http://localhost:5000')
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ip", [
|
||||
'127.0.0.1', '10.1.2.3', '169.254.169.254', '0.0.0.0', '100.64.0.1',
|
||||
'::1', '::ffff:127.0.0.1', 'fe80::1', '224.0.0.1',
|
||||
])
|
||||
def test_4_5_blocked_addresses(ip):
|
||||
assert webmention_ssrf.is_private_ip(ip)
|
||||
|
||||
|
||||
def test_4_5_any_blocked_address_blocks(app):
|
||||
mixed = PUBLIC + [(10, 1, 6, '', ('::1', 0, 0, 0))]
|
||||
with app.app_context(), patch("starpunk.webmention_ssrf.socket.getaddrinfo", return_value=mixed):
|
||||
with pytest.raises(webmention_ssrf.SSRFBlockedError):
|
||||
webmention_ssrf.validate_url_for_fetch('https://x.example/')
|
||||
|
||||
|
||||
# 3.2.2 Webmention verification
|
||||
|
||||
TARGET = 'https://me.example/note/abc'
|
||||
|
||||
|
||||
@pytest.mark.parametrize("content,content_type,expected", [
|
||||
('<a href="/note/abc#c">x</a>', 'text/html', True),
|
||||
('<img src="https://me.example/note/abc">', 'text/html; charset=utf-8', True),
|
||||
('<a href="https://me.example/note/abcdef">x</a>', 'text/html', False),
|
||||
('<p>https://me.example/note/abc</p>', 'text/html', False),
|
||||
('<!-- <a href="https://me.example/note/abc"> -->', 'text/html', False),
|
||||
('{"a": ["https://me.example/note/abc"]}', 'application/json', True),
|
||||
('{"a": "https://me.example/note/abcdef"}', 'application/json', False),
|
||||
('see https://me.example/note/abc', 'text/plain', True),
|
||||
])
|
||||
def test_3_2_2_exact_match(content, content_type, expected):
|
||||
assert verify_link_presence(content, TARGET, content_type, 'https://me.example/x') is expected
|
||||
|
||||
|
||||
def test_3_2_2_accept_header_and_status(app, mock_web):
|
||||
seen = {}
|
||||
|
||||
def page(request):
|
||||
seen['accept'] = request.headers.get('Accept')
|
||||
return httpx.Response(410)
|
||||
|
||||
mock_web['https://s.example/p'] = page
|
||||
with app.app_context():
|
||||
with pytest.raises(ValidationError, match="410"):
|
||||
fetch_source('https://s.example/p', 'http://localhost:5000')
|
||||
assert 'text/html' in seen['accept']
|
||||
|
||||
mock_web['https://s.example/p'] = lambda r: httpx.Response(500)
|
||||
with app.app_context(), pytest.raises(TransientError):
|
||||
fetch_source('https://s.example/p', 'http://localhost:5000')
|
||||
|
||||
|
||||
# 3.2.1 Request verification
|
||||
|
||||
def _post(client, source, target):
|
||||
return client.post('/webmention', data={'source': source, 'target': target})
|
||||
|
||||
|
||||
def test_3_2_1_target_must_be_a_note(app, client, sample_note):
|
||||
note_url = f"http://localhost:5000/note/{sample_note.slug}"
|
||||
assert _post(client, 'https://s.example/a', note_url).status_code == 202
|
||||
assert _post(client, 'https://s.example/a', 'http://localhost:5000/nope').status_code == 400
|
||||
assert _post(client, 'https://s.example/a', 'http://localhost:5000/').status_code == 400
|
||||
assert _post(client, 'https://s.example/a', 'https://other.example/note/x').status_code == 400
|
||||
assert _post(client, note_url, note_url).status_code == 400
|
||||
assert _post(client, 'ftp://s.example/a', note_url).status_code == 400
|
||||
|
||||
|
||||
def test_3_2_1_host_case_and_fragment(app, client, sample_note):
|
||||
note_url = f"http://LOCALHOST:5000/note/{sample_note.slug}"
|
||||
assert _post(client, 'https://s.example/a', note_url + '#one').status_code == 202
|
||||
assert _post(client, 'https://s.example/a', note_url + '#two').status_code == 202
|
||||
db = sqlite3.connect(app.config['DATABASE_PATH'])
|
||||
rows = db.execute("SELECT target_url FROM webmentions").fetchall()
|
||||
assert rows == [(note_url,)]
|
||||
|
||||
|
||||
# 3.2.4 Updating existing Webmentions
|
||||
|
||||
def _insert_mention(app, note_id, **cols):
|
||||
db = sqlite3.connect(app.config['DATABASE_PATH'])
|
||||
fields = {'source_url': 'https://s.example/a', 'target_url': 'http://localhost:5000/n',
|
||||
'target_note_id': note_id, **cols}
|
||||
db.execute(f"INSERT INTO webmentions ({', '.join(fields)}) VALUES ({', '.join('?' * len(fields))})",
|
||||
tuple(fields.values()))
|
||||
db.commit()
|
||||
return db
|
||||
|
||||
|
||||
VALID = {'valid': True, 'mention_type': 'reply', 'author_name': 'A', 'author_url': 'https://a.example/',
|
||||
'author_photo': None, 'content_snippet': 'hi'}
|
||||
|
||||
|
||||
def test_3_2_4_update_keeps_moderation(app, worker, sample_note):
|
||||
db = _insert_mention(app, sample_note.id, validation_status='valid', moderation_status='hidden',
|
||||
validated_at='2026-01-01')
|
||||
with patch("starpunk.webmention_worker.process_webmention", return_value=VALID):
|
||||
worker._validate_webmention(1, 'https://s.example/a', 'http://localhost:5000/n', sample_note.id)
|
||||
assert db.execute("SELECT validation_status, moderation_status FROM webmentions").fetchone() == ('valid', 'hidden')
|
||||
|
||||
|
||||
def test_3_2_4_new_mention_gets_default_moderation(app, worker, sample_note):
|
||||
db = _insert_mention(app, sample_note.id)
|
||||
with patch("starpunk.webmention_worker.process_webmention", return_value=VALID):
|
||||
worker._validate_webmention(1, 'https://s.example/a', 'http://localhost:5000/n', sample_note.id)
|
||||
assert db.execute("SELECT validation_status, moderation_status FROM webmentions").fetchone() == ('valid', 'approved')
|
||||
|
||||
|
||||
def test_3_2_4_transient_error_keeps_valid_mention(app, worker, sample_note):
|
||||
db = _insert_mention(app, sample_note.id, validation_status='pending', moderation_status='approved',
|
||||
validated_at='2026-01-01', content_snippet='kept')
|
||||
with patch("starpunk.webmention_worker.process_webmention", side_effect=TransientError("timeout")):
|
||||
worker._validate_webmention(1, 'https://s.example/a', 'http://localhost:5000/n', sample_note.id)
|
||||
assert db.execute("SELECT validation_status, content_snippet FROM webmentions").fetchone() == ('valid', 'kept')
|
||||
|
||||
|
||||
def test_3_2_4_gone_source_removes_mention(app, worker, sample_note):
|
||||
db = _insert_mention(app, sample_note.id, validation_status='pending', moderation_status='approved',
|
||||
validated_at='2026-01-01')
|
||||
with patch("starpunk.webmention_worker.process_webmention",
|
||||
side_effect=ValidationError("Source returned 410")):
|
||||
worker._validate_webmention(1, 'https://s.example/a', 'http://localhost:5000/n', sample_note.id)
|
||||
assert db.execute("SELECT validation_status FROM webmentions").fetchone() == ('invalid',)
|
||||
|
||||
|
||||
# 3.1.4 Updated posts / 3.1.5 Deleted posts
|
||||
|
||||
def _queue(app):
|
||||
db = sqlite3.connect(app.config['DATABASE_PATH'])
|
||||
return db.execute(
|
||||
"SELECT target_url, status, source_note_id FROM webmention_queue ORDER BY target_url"
|
||||
).fetchall()
|
||||
|
||||
|
||||
def _sent(app):
|
||||
db = sqlite3.connect(app.config['DATABASE_PATH'])
|
||||
db.execute("UPDATE webmention_queue SET status = 'sent'")
|
||||
db.commit()
|
||||
|
||||
|
||||
def test_3_1_4_update_resends_previous_targets(app):
|
||||
from starpunk.notes import create_note, update_note
|
||||
|
||||
with app.test_request_context():
|
||||
note = create_note("Reply", published=True, in_reply_to="https://a.example/post")
|
||||
_sent(app)
|
||||
update_note(id=note.id, content="Now about [b](https://b.example/x)", in_reply_to="")
|
||||
targets = {t: s for t, s, _ in _queue(app)}
|
||||
assert targets == {'https://a.example/post': 'pending', 'https://b.example/x': 'pending'}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("soft", [True, False])
|
||||
def test_3_1_5_delete_resends_and_returns_410(app, client, soft):
|
||||
from starpunk.notes import create_note, delete_note
|
||||
|
||||
with app.test_request_context():
|
||||
note = create_note("See [a](https://a.example/x)", published=True)
|
||||
_sent(app)
|
||||
delete_note(id=note.id, soft=soft)
|
||||
|
||||
rows = _queue(app)
|
||||
assert [(t, s) for t, s, _ in rows] == [('https://a.example/x', 'pending')]
|
||||
if not soft:
|
||||
assert rows[0][2] is None # detached, so it survives the delete
|
||||
assert client.get(f"/note/{note.slug}").status_code == 410
|
||||
assert client.get("/note/never-existed").status_code == 404
|
||||
|
||||
|
||||
# 4.5 Connections go to the address that was checked (no DNS rebinding)
|
||||
|
||||
def test_4_5_connection_pinned_to_checked_address(app):
|
||||
seen = []
|
||||
|
||||
def handler(request):
|
||||
seen.append((str(request.url), request.headers['host'],
|
||||
request.extensions.get('sni_hostname')))
|
||||
return httpx.Response(200, text='ok')
|
||||
|
||||
with app.app_context(), \
|
||||
patch("starpunk.webmention_ssrf.socket.getaddrinfo", return_value=PUBLIC) as resolve, \
|
||||
patch("starpunk.webmention_ssrf.TRANSPORT", httpx.MockTransport(handler)):
|
||||
webmention_ssrf.safe_get('https://site.example:8443/p?q=1', {})
|
||||
webmention_ssrf.safe_post('http://site.example/wm', {'source': 's'}, {})
|
||||
|
||||
assert seen == [
|
||||
('https://93.184.216.34:8443/p?q=1', 'site.example:8443', 'site.example'),
|
||||
('http://93.184.216.34/wm', 'site.example', None),
|
||||
]
|
||||
assert resolve.call_count == 2 # one lookup per request, then pinned
|
||||
|
||||
|
||||
def test_4_5_ipv4_preferred(app):
|
||||
with app.app_context(), patch("starpunk.webmention_ssrf.socket.getaddrinfo",
|
||||
return_value=[(10, 1, 6, '', ('2606:2800::1', 0, 0, 0))] + PUBLIC):
|
||||
assert webmention_ssrf.safe_resolve('x.example') == '93.184.216.34'
|
||||
|
||||
|
||||
# 3.1.4 A change in the responses a note shows is an update of the note
|
||||
|
||||
def _note_with_sent_mention(app):
|
||||
from starpunk.notes import create_note
|
||||
with app.test_request_context():
|
||||
note = create_note("See [a](https://a.example/x)", published=True)
|
||||
_sent(app)
|
||||
db = sqlite3.connect(app.config['DATABASE_PATH'])
|
||||
db.execute("INSERT INTO webmentions (source_url, target_url, target_note_id) VALUES (?, ?, ?)",
|
||||
('https://s.example/reply', f'http://localhost:5000/note/{note.slug}', note.id))
|
||||
db.commit()
|
||||
return note, db
|
||||
|
||||
|
||||
def test_3_1_4_new_response_resends(app, worker):
|
||||
note, db = _note_with_sent_mention(app)
|
||||
with patch("starpunk.webmention_worker.process_webmention", return_value=VALID):
|
||||
worker._validate_webmention(1, 'https://s.example/reply', 'x', note.id)
|
||||
assert [(t, s) for t, s, _ in _queue(app)] == [('https://a.example/x', 'pending')]
|
||||
|
||||
|
||||
def test_3_1_4_unchanged_response_does_not_resend(app, worker):
|
||||
note, db = _note_with_sent_mention(app)
|
||||
with patch("starpunk.webmention_worker.process_webmention", return_value=VALID):
|
||||
worker._validate_webmention(1, 'https://s.example/reply', 'x', note.id)
|
||||
_sent(app)
|
||||
db.execute("UPDATE webmentions SET validation_status = 'pending'") # re-sent by source
|
||||
db.commit()
|
||||
worker._validate_webmention(1, 'https://s.example/reply', 'x', note.id)
|
||||
assert [(t, s) for t, s, _ in _queue(app)] == [('https://a.example/x', 'sent')]
|
||||
|
||||
|
||||
def test_3_1_4_moderation_change_resends(app, worker):
|
||||
from starpunk.webmention_queries import update_webmention_status
|
||||
note, db = _note_with_sent_mention(app)
|
||||
with patch("starpunk.webmention_worker.process_webmention", return_value=VALID):
|
||||
worker._validate_webmention(1, 'https://s.example/reply', 'x', note.id)
|
||||
_sent(app)
|
||||
with app.app_context():
|
||||
update_webmention_status([1], 'approved') # already approved: no change
|
||||
assert _queue(app)[0][1] == 'sent'
|
||||
with app.app_context():
|
||||
update_webmention_status([1], 'hidden')
|
||||
assert _queue(app)[0][1] == 'pending'
|
||||
@@ -0,0 +1,133 @@
|
||||
"""
|
||||
Webmention worker regression tests (v2.6.8)
|
||||
|
||||
The worker thread used Flask's current_app (via the SSRF checks) without an
|
||||
app context, so every send and validation failed. It was also stopped
|
||||
whenever any app context ended.
|
||||
"""
|
||||
|
||||
import sqlite3
|
||||
|
||||
import httpx
|
||||
import time
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from starpunk.webmention_worker import WebmentionWorker
|
||||
|
||||
PUBLIC = [(2, 1, 6, '', ('93.184.216.34', 0))]
|
||||
|
||||
|
||||
def _wait_for(predicate, timeout=5):
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
if predicate():
|
||||
return True
|
||||
time.sleep(0.05)
|
||||
return False
|
||||
|
||||
|
||||
def test_worker_sends_queued_webmention(app):
|
||||
db_path = app.config["DATABASE_PATH"]
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.execute(
|
||||
"INSERT INTO webmention_queue (source_url, target_url, status) "
|
||||
"VALUES (?, ?, 'pending')",
|
||||
("http://localhost:5000/note/1", "https://example.org/post"),
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
posts = []
|
||||
|
||||
def target_site(request):
|
||||
if request.method == "POST":
|
||||
posts.append((request.headers["host"], request.url.path, request.content))
|
||||
return httpx.Response(202)
|
||||
return httpx.Response(200, headers={"Link": '</wm>; rel="webmention"'})
|
||||
|
||||
worker = WebmentionWorker(app, db_path=db_path, site_url=app.config["SITE_URL"],
|
||||
poll_interval=0.05)
|
||||
with patch("starpunk.webmention_ssrf.socket.getaddrinfo", return_value=PUBLIC), \
|
||||
patch("starpunk.webmention_ssrf.TRANSPORT", httpx.MockTransport(target_site)):
|
||||
worker.start()
|
||||
try:
|
||||
def status():
|
||||
c = sqlite3.connect(db_path)
|
||||
row = c.execute("SELECT status, last_error FROM webmention_queue").fetchone()
|
||||
c.close()
|
||||
return row
|
||||
assert _wait_for(lambda: status()[0] != "pending"), status()
|
||||
assert status() == ("sent", None)
|
||||
assert [(h, p) for h, p, _ in posts] == [("example.org", "/wm")]
|
||||
assert b"source=" in posts[0][2] and b"target=" in posts[0][2]
|
||||
finally:
|
||||
worker.stop()
|
||||
worker.join(timeout=2)
|
||||
|
||||
|
||||
def test_worker_survives_app_context_teardown(app):
|
||||
worker = WebmentionWorker(app, db_path=app.config["DATABASE_PATH"],
|
||||
site_url=app.config["SITE_URL"], poll_interval=0.05)
|
||||
worker.start()
|
||||
try:
|
||||
with app.app_context():
|
||||
pass
|
||||
app.test_client().get("/")
|
||||
time.sleep(0.2)
|
||||
assert worker.is_alive()
|
||||
finally:
|
||||
worker.stop()
|
||||
worker.join(timeout=2)
|
||||
|
||||
|
||||
def test_only_one_worker_processes_queues(app):
|
||||
"""Each gunicorn process starts a worker; only one may hold the queue lock"""
|
||||
kwargs = dict(db_path=app.config["DATABASE_PATH"], site_url=app.config["SITE_URL"])
|
||||
first = WebmentionWorker(app, **kwargs)
|
||||
second = WebmentionWorker(app, **kwargs)
|
||||
try:
|
||||
assert first._acquire_lock()
|
||||
assert not second._acquire_lock()
|
||||
first._release_lock()
|
||||
assert second._acquire_lock()
|
||||
finally:
|
||||
first._release_lock()
|
||||
second._release_lock()
|
||||
|
||||
|
||||
def test_author_photo_with_alt_text_is_a_string():
|
||||
"""mf2py returns u-photo with alt as a dict; the database needs a URL string"""
|
||||
import mf2py
|
||||
from starpunk.webmention_processor import extract_author
|
||||
|
||||
html = """
|
||||
<div class="h-entry">
|
||||
<span class="p-author h-card">
|
||||
<a class="u-url p-name" href="https://a.example/">A Person</a>
|
||||
<img class="u-photo" src="https://a.example/me.jpg" alt=""></span>
|
||||
<a href="https://site.example/note/1">target</a>
|
||||
</div>"""
|
||||
author = extract_author(mf2py.parse(doc=html, url="https://src.example/"),
|
||||
"https://src.example/")
|
||||
assert author == {
|
||||
"author_name": "A Person",
|
||||
"author_url": "https://a.example/",
|
||||
"author_photo": "https://a.example/me.jpg",
|
||||
}
|
||||
|
||||
|
||||
def test_author_urls_must_be_http():
|
||||
"""A javascript: author URL would be rendered as a clickable link (XSS)"""
|
||||
import mf2py
|
||||
from starpunk.webmention_processor import extract_author
|
||||
|
||||
html = """
|
||||
<div class="h-entry">
|
||||
<span class="p-author h-card">
|
||||
<a class="u-url p-name" href="javascript:alert(1)">Evil</a>
|
||||
<img class="u-photo" src="data:image/svg+xml,x" alt=""></span>
|
||||
</div>"""
|
||||
author = extract_author(mf2py.parse(doc=html, url="https://src.example/"),
|
||||
"https://src.example/")
|
||||
assert author["author_url"] == "https://src.example/"
|
||||
assert author["author_photo"] is None
|
||||
@@ -0,0 +1,315 @@
|
||||
"""
|
||||
WebSub publisher and built-in hub (v2.10.0, ADR-071)
|
||||
|
||||
Section numbers refer to the W3C WebSub Recommendation. Subscribers are
|
||||
simulated with an httpx MockTransport behind the SSRF-safe fetcher.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import sqlite3
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
import httpx
|
||||
import mf2py
|
||||
import pytest
|
||||
|
||||
from starpunk.websub_worker import WebSubWorker
|
||||
|
||||
SITE = "http://localhost:5000"
|
||||
HUB = f"{SITE}/websub"
|
||||
PUBLIC = [(2, 1, 6, "", ("93.184.216.34", 0))]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def web(monkeypatch):
|
||||
"""Subscriber callbacks: handler(request) -> response, by host"""
|
||||
hosts, seen = {}, []
|
||||
|
||||
def handler(request):
|
||||
host = request.headers["host"]
|
||||
seen.append(request)
|
||||
return hosts[host](request)
|
||||
|
||||
monkeypatch.setattr("starpunk.webmention_ssrf.socket.getaddrinfo", lambda *a, **k: PUBLIC)
|
||||
monkeypatch.setattr("starpunk.webmention_ssrf.TRANSPORT", httpx.MockTransport(handler))
|
||||
return hosts, seen
|
||||
|
||||
|
||||
def echo(request):
|
||||
"""A subscriber that confirms every verification and accepts content"""
|
||||
if request.method == "GET":
|
||||
return httpx.Response(200, text=parse_qs(urlparse(str(request.url)).query)["hub.challenge"][0])
|
||||
return httpx.Response(202)
|
||||
|
||||
|
||||
def db(app):
|
||||
conn = sqlite3.connect(app.config["DATABASE_PATH"])
|
||||
conn.row_factory = sqlite3.Row
|
||||
return conn
|
||||
|
||||
|
||||
def run_worker(app, times=1):
|
||||
worker = WebSubWorker(app)
|
||||
with app.app_context():
|
||||
for _ in range(times):
|
||||
worker.process_once()
|
||||
return worker
|
||||
|
||||
|
||||
def subscribe(client, topic, callback="https://sub.example/cb?id=1", **extra):
|
||||
form = {"hub.mode": "subscribe", "hub.topic": topic, "hub.callback": callback, **extra}
|
||||
return client.post("/websub", data=form)
|
||||
|
||||
|
||||
def make_note(app, content="Hello", **kw):
|
||||
from starpunk.notes import create_note
|
||||
with app.test_request_context():
|
||||
return create_note(content, published=kw.pop("published", True), **kw)
|
||||
|
||||
|
||||
def links(response):
|
||||
return response.headers.getlist("Link")
|
||||
|
||||
|
||||
# 4 Discovery -------------------------------------------------------------------
|
||||
|
||||
@pytest.mark.parametrize("path,self_url", [
|
||||
("/", f"{SITE}/"),
|
||||
("/feed.atom", f"{SITE}/feed.atom"),
|
||||
("/feed.rss", f"{SITE}/feed.rss"),
|
||||
("/feed.xml", f"{SITE}/feed.rss"),
|
||||
("/feed.json", f"{SITE}/feed.json"),
|
||||
("/likes", f"{SITE}/likes"),
|
||||
])
|
||||
def test_topics_advertise_hub_and_self(client, path, self_url):
|
||||
response = client.get(path)
|
||||
values = links(response)
|
||||
assert f'<{HUB}>; rel="hub"' in values
|
||||
assert [v for v in values if 'rel="self"' in v] == [f'<{self_url}>; rel="self"']
|
||||
|
||||
|
||||
def test_negotiated_feed_self_is_the_format_served(client):
|
||||
response = client.get("/feed", headers={"Accept": "application/atom+xml"})
|
||||
assert f'<{SITE}/feed.atom>; rel="self"' in links(response)
|
||||
|
||||
|
||||
def test_tag_page_is_a_topic(app, client):
|
||||
make_note(app, tags=["Python"])
|
||||
assert f'<{SITE}/tag/python>; rel="self"' in links(client.get("/tag/Python"))
|
||||
|
||||
|
||||
def test_note_page_is_not_a_topic(app, client):
|
||||
note = make_note(app)
|
||||
assert not [v for v in links(client.get(f"/note/{note.slug}")) if 'rel="hub"' in v]
|
||||
|
||||
|
||||
def test_embedded_links(client):
|
||||
rels = mf2py.parse(doc=client.get("/").data.decode(), url=SITE + "/")["rels"]
|
||||
assert rels["hub"] == [HUB] and rels["self"] == [f"{SITE}/"]
|
||||
assert f'<link rel="hub" href="{HUB}"/>' in client.get("/feed.atom").data.decode()
|
||||
assert f'<atom:link href="{HUB}" rel="hub"/>' in client.get("/feed.rss").data.decode()
|
||||
assert json.loads(client.get("/feed.json").data)["hubs"] == [{"type": "WebSub", "url": HUB}]
|
||||
|
||||
|
||||
def test_external_hub(app, client, web):
|
||||
app.config["WEBSUB_HUB_URL"] = "https://hub.example/"
|
||||
hosts, seen = web
|
||||
hosts["hub.example"] = lambda r: httpx.Response(204)
|
||||
assert '<https://hub.example/>; rel="hub"' in links(client.get("/"))
|
||||
assert subscribe(client, f"{SITE}/").status_code == 400 # our hub is off
|
||||
|
||||
make_note(app)
|
||||
run_worker(app)
|
||||
pings = {parse_qs(r.content.decode())["hub.url"][0] for r in seen}
|
||||
assert pings == {f"{SITE}/", f"{SITE}/feed.atom", f"{SITE}/feed.rss", f"{SITE}/feed.json"}
|
||||
|
||||
|
||||
# 5.1 Subscription requests ----------------------------------------------------------
|
||||
|
||||
def test_subscription_request_is_accepted(app, client, web):
|
||||
response = subscribe(client, f"{SITE}/feed.atom", **{"hub.secret": "s3cret", "extra": "ignored"})
|
||||
assert response.status_code == 202
|
||||
row = db(app).execute("SELECT * FROM websub_intents").fetchone()
|
||||
assert (row["mode"], row["topic"], row["secret"]) == ("subscribe", f"{SITE}/feed.atom", "s3cret")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("form,reason", [
|
||||
({"hub.mode": "watch"}, "hub.mode"),
|
||||
({"hub.topic": "https://elsewhere.example/feed"}, "hub.topic"),
|
||||
({"hub.topic": f"{SITE}/note/x"}, "hub.topic"),
|
||||
({"hub.topic": f"{SITE}/tag/missing"}, "hub.topic"),
|
||||
({"hub.callback": "ftp://sub.example/cb"}, "hub.callback"),
|
||||
({"hub.secret": "x" * 200}, "hub.secret"),
|
||||
])
|
||||
def test_bad_requests(client, web, form, reason):
|
||||
base = {"hub.mode": "subscribe", "hub.topic": f"{SITE}/", "hub.callback": "https://sub.example/cb"}
|
||||
response = client.post("/websub", data={**base, **form})
|
||||
assert response.status_code == 400 and reason in response.data.decode()
|
||||
|
||||
|
||||
def test_private_callback_refused(client, monkeypatch):
|
||||
monkeypatch.setattr("starpunk.webmention_ssrf.socket.getaddrinfo",
|
||||
lambda *a, **k: [(2, 1, 6, "", ("10.0.0.2", 0))])
|
||||
assert subscribe(client, f"{SITE}/").status_code == 400
|
||||
|
||||
|
||||
def test_per_host_limit(app, client, web):
|
||||
app.config["WEBSUB_MAX_PER_HOST"] = 1
|
||||
hosts, _ = web
|
||||
hosts["sub.example"] = echo
|
||||
subscribe(client, f"{SITE}/")
|
||||
run_worker(app)
|
||||
assert subscribe(client, f"{SITE}/feed.atom").status_code == 429
|
||||
|
||||
|
||||
# 5.3 Verification of intent --------------------------------------------------------
|
||||
|
||||
def test_verification_activates_subscription(app, client, web):
|
||||
hosts, seen = web
|
||||
hosts["sub.example"] = echo
|
||||
subscribe(client, f"{SITE}/", **{"hub.lease_seconds": "86400"})
|
||||
run_worker(app)
|
||||
|
||||
query = parse_qs(urlparse(str(seen[0].url)).query)
|
||||
assert query["id"] == ["1"] # the callback's own query string is kept
|
||||
assert query["hub.mode"] == ["subscribe"] and query["hub.topic"] == [f"{SITE}/"]
|
||||
assert query["hub.lease_seconds"] == ["86400"]
|
||||
sub = db(app).execute("SELECT * FROM websub_subscriptions").fetchone()
|
||||
assert sub["topic"] == f"{SITE}/" and sub["verified_at"]
|
||||
assert db(app).execute("SELECT COUNT(*) FROM websub_intents").fetchone()[0] == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("response", [
|
||||
httpx.Response(200, text="wrong"),
|
||||
httpx.Response(404),
|
||||
httpx.Response(302, headers={"Location": "https://sub.example/other"}),
|
||||
])
|
||||
def test_failed_verification_changes_nothing(app, client, web, response):
|
||||
hosts, _ = web
|
||||
hosts["sub.example"] = lambda r: response
|
||||
subscribe(client, f"{SITE}/")
|
||||
run_worker(app)
|
||||
assert db(app).execute("SELECT COUNT(*) FROM websub_subscriptions").fetchone()[0] == 0
|
||||
assert db(app).execute("SELECT COUNT(*) FROM websub_intents").fetchone()[0] == 0
|
||||
|
||||
|
||||
def test_unsubscribe(app, client, web):
|
||||
hosts, _ = web
|
||||
hosts["sub.example"] = echo
|
||||
subscribe(client, f"{SITE}/")
|
||||
run_worker(app)
|
||||
client.post("/websub", data={"hub.mode": "unsubscribe", "hub.topic": f"{SITE}/",
|
||||
"hub.callback": "https://sub.example/cb?id=1"})
|
||||
run_worker(app)
|
||||
assert db(app).execute("SELECT COUNT(*) FROM websub_subscriptions").fetchone()[0] == 0
|
||||
|
||||
|
||||
def test_resubscribe_changes_secret_only_once_verified(app, client, web):
|
||||
hosts, _ = web
|
||||
hosts["sub.example"] = echo
|
||||
subscribe(client, f"{SITE}/", **{"hub.secret": "one"})
|
||||
run_worker(app)
|
||||
hosts["sub.example"] = lambda r: httpx.Response(404)
|
||||
subscribe(client, f"{SITE}/", **{"hub.secret": "two"})
|
||||
run_worker(app)
|
||||
assert db(app).execute("SELECT secret FROM websub_subscriptions").fetchone()[0] == "one"
|
||||
|
||||
|
||||
# 6, 7 Content distribution ---------------------------------------------------------
|
||||
|
||||
def _subscribed(app, client, web, topic=f"{SITE}/feed.atom", secret=None):
|
||||
hosts, seen = web
|
||||
hosts["sub.example"] = echo
|
||||
extra = {"hub.secret": secret} if secret else {}
|
||||
subscribe(client, topic, **extra)
|
||||
run_worker(app)
|
||||
seen.clear()
|
||||
return hosts, seen
|
||||
|
||||
|
||||
def test_new_note_is_delivered_signed(app, client, web):
|
||||
hosts, seen = _subscribed(app, client, web, secret="s3cret")
|
||||
make_note(app, "Fresh post")
|
||||
run_worker(app)
|
||||
|
||||
(post,) = [r for r in seen if r.method == "POST"]
|
||||
assert post.headers["content-type"].startswith("application/atom+xml")
|
||||
assert b"Fresh post" in post.content
|
||||
assert f'<{HUB}>; rel="hub"' in post.headers["link"] and f'<{SITE}/feed.atom>; rel="self"' in post.headers["link"]
|
||||
expected = "sha256=" + hmac.new(b"s3cret", post.content, hashlib.sha256).hexdigest()
|
||||
assert post.headers["x-hub-signature"] == expected
|
||||
assert db(app).execute("SELECT COUNT(*) FROM websub_deliveries").fetchone()[0] == 0
|
||||
|
||||
|
||||
def test_no_signature_without_secret(app, client, web):
|
||||
hosts, seen = _subscribed(app, client, web)
|
||||
make_note(app)
|
||||
run_worker(app)
|
||||
assert "x-hub-signature" not in [r for r in seen if r.method == "POST"][0].headers
|
||||
|
||||
|
||||
def test_drafts_are_not_delivered(app, client, web):
|
||||
_subscribed(app, client, web)
|
||||
make_note(app, published=False)
|
||||
assert db(app).execute("SELECT COUNT(*) FROM websub_deliveries").fetchone()[0] == 0
|
||||
|
||||
|
||||
def test_tag_topics_include_removed_tags(app, client, web):
|
||||
from starpunk.notes import update_note
|
||||
note = make_note(app, tags=["old"])
|
||||
_subscribed(app, client, web, topic=f"{SITE}/tag/old")
|
||||
with app.test_request_context():
|
||||
update_note(id=note.id, content="Hello", tags=["new"])
|
||||
assert db(app).execute("SELECT topic FROM websub_deliveries").fetchone()[0] == f"{SITE}/tag/old"
|
||||
|
||||
|
||||
def test_delete_is_delivered(app, client, web):
|
||||
from starpunk.notes import delete_note
|
||||
note = make_note(app, "Going away")
|
||||
hosts, seen = _subscribed(app, client, web)
|
||||
with app.test_request_context():
|
||||
delete_note(id=note.id, soft=False)
|
||||
run_worker(app)
|
||||
(post,) = [r for r in seen if r.method == "POST"]
|
||||
assert b"Going away" not in post.content
|
||||
|
||||
|
||||
def test_410_removes_subscription(app, client, web):
|
||||
hosts, _ = _subscribed(app, client, web)
|
||||
hosts["sub.example"] = lambda r: httpx.Response(410)
|
||||
make_note(app)
|
||||
run_worker(app)
|
||||
assert db(app).execute("SELECT COUNT(*) FROM websub_subscriptions").fetchone()[0] == 0
|
||||
|
||||
|
||||
def test_failures_are_retried(app, client, web):
|
||||
hosts, _ = _subscribed(app, client, web)
|
||||
hosts["sub.example"] = lambda r: httpx.Response(500)
|
||||
make_note(app)
|
||||
run_worker(app)
|
||||
row = db(app).execute("SELECT attempts, next_attempt_at, last_error FROM websub_deliveries").fetchone()
|
||||
assert row["attempts"] == 1 and row["last_error"] == "HTTP 500"
|
||||
|
||||
|
||||
def test_expired_subscriptions_get_nothing(app, client, web):
|
||||
_subscribed(app, client, web)
|
||||
conn = db(app)
|
||||
conn.execute("UPDATE websub_subscriptions SET expires_at = datetime('now', '-1 minute')")
|
||||
conn.commit()
|
||||
make_note(app)
|
||||
assert conn.execute("SELECT COUNT(*) FROM websub_deliveries").fetchone()[0] == 0
|
||||
run_worker(app) # prunes
|
||||
assert conn.execute("SELECT COUNT(*) FROM websub_subscriptions").fetchone()[0] == 0
|
||||
|
||||
|
||||
# Feeds are fresh right after a change (ADR-071 3) ------------------------------------
|
||||
|
||||
def test_feed_shows_new_note_immediately(app, client):
|
||||
from starpunk.routes import public
|
||||
make_note(app, "First")
|
||||
assert b"First" in client.get("/feed.atom").data
|
||||
make_note(app, "Second") # cached note list is stale now
|
||||
assert b"Second" in client.get("/feed.atom").data
|
||||
assert public._feed_cache["notes"] is not None
|
||||
Reference in new issue
Block a user