Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4ed2033cf9 | ||
|
|
d354dc9733 | ||
|
|
88aea09e04 | ||
|
|
7e2cdef150 | ||
|
|
058b1959f9 | ||
|
|
ceef048003 | ||
|
|
caa66f7bb1 | ||
|
|
0465ff924b | ||
|
|
815aff5443 | ||
|
|
4306633a42 | ||
|
|
c5c2d9832e | ||
|
|
483c24b565 | ||
|
|
d312fa0074 | ||
|
|
647f6c723b | ||
|
|
81c730813c | ||
|
|
113f28aff1 | ||
|
|
76b81d6ad9 | ||
|
|
44f77a3adf | ||
|
|
b247de011d | ||
|
|
65fc246d55 | ||
|
|
f3ed1b3488 | ||
|
|
d400e4d838 | ||
|
|
55ce6a1fdd | ||
|
|
6aaac982d5 | ||
|
|
2654692d15 | ||
|
|
f8a429cdc1 | ||
|
|
afef1fc9a5 | ||
|
|
b450c0c648 | ||
|
|
13936749de | ||
|
|
742e2e3ec3 | ||
|
|
d6a916ba7d | ||
|
|
e654dd84de | ||
|
|
abb75f080e | ||
|
|
d5157a6059 | ||
|
|
393228002b | ||
|
|
874465e103 | ||
|
|
3f5e7dc735 | ||
|
|
75946be7b2 |
No files matched your search
+41
-20
@@ -1,4 +1,4 @@
|
||||
# StarPunk configuration template (v2.10.1)
|
||||
# StarPunk configuration template (v2.14.0)
|
||||
#
|
||||
# Copy to .env and fill in your values. DO NOT commit .env.
|
||||
# Every variable here is read by starpunk/config.py. Uncommented lines are
|
||||
@@ -73,10 +73,10 @@ DATABASE_PATH=./data/starpunk.db
|
||||
# FLASK / LOGGING
|
||||
# =============================================================================
|
||||
|
||||
# development or production (default: development)
|
||||
FLASK_ENV=development
|
||||
# 1 = debug on, 0 = off (default: 1). Set 0 in production.
|
||||
FLASK_DEBUG=1
|
||||
# development or production (default: production)
|
||||
FLASK_ENV=production
|
||||
# 1 = debug on, 0 = off (default: 0). `flask run --debug` turns it on locally.
|
||||
FLASK_DEBUG=0
|
||||
# DEBUG, INFO, WARNING, ERROR or CRITICAL (default: INFO)
|
||||
LOG_LEVEL=INFO
|
||||
# Overrides the version shown in the footer and /health (default: the
|
||||
@@ -104,6 +104,10 @@ LOG_LEVEL=INFO
|
||||
# (default: false)
|
||||
TRUSTED_PROXY=false
|
||||
|
||||
# CSRF tokens on admin forms, logout and the IndieAuth consent (default: true).
|
||||
# Leave on; tests turn it off unless they test it.
|
||||
CSRF_ENABLED=true
|
||||
|
||||
# =============================================================================
|
||||
# FEEDS
|
||||
# =============================================================================
|
||||
@@ -120,10 +124,11 @@ TRUSTED_PROXY=false
|
||||
# WEBMENTION
|
||||
# =============================================================================
|
||||
|
||||
# Receive webmentions at /webmention (default: true)
|
||||
# Webmention endpoint at /webmention, its Link header, and showing responses
|
||||
# (webmentions and backfeed) on notes; sending is not affected (default: true)
|
||||
# WEBMENTION_ENABLED=true
|
||||
# Background worker that verifies received webmentions and sends outgoing
|
||||
# ones (default: true)
|
||||
# Background worker that verifies received webmentions, sends outgoing ones,
|
||||
# and sends response notifications (v2.14.0) (default: true)
|
||||
# WEBMENTION_WORKER_ENABLED=true
|
||||
# Seconds between queue checks (default: 10) and after an error (default: 60)
|
||||
# WEBMENTION_POLL_INTERVAL=10
|
||||
@@ -161,7 +166,8 @@ WEBSUB_ENABLED=true
|
||||
|
||||
# GoToSocial or any Mastodon-API server (v2.1.0, ADR-065). Create an app
|
||||
# token with scopes: write:statuses write:media, plus read:search
|
||||
# write:favourites for native likes and boosts (v2.5.0)
|
||||
# write:favourites for native likes and boosts (v2.5.0), plus read:statuses
|
||||
# read:accounts (or read) for backfeed (v2.13.0)
|
||||
GOTOSOCIAL_URL=
|
||||
GOTOSOCIAL_ACCESS_TOKEN=
|
||||
# Name shown in the editor and Micropub apps (default: GoToSocial)
|
||||
@@ -181,7 +187,8 @@ BLUESKY_APP_PASSWORD=
|
||||
# BLUESKY_MAX_CHARS=300
|
||||
|
||||
# Threads (v2.6.0, ADR-069). A long-lived user token with threads_basic,
|
||||
# threads_content_publish and threads_delete; StarPunk refreshes it daily and
|
||||
# threads_content_publish and threads_delete (plus threads_read_replies and
|
||||
# threads_manage_insights for backfeed, v2.13.0); StarPunk refreshes it daily and
|
||||
# keeps the current one in DATA_PATH/threads_token.json.
|
||||
# How to get one: docs/design/v2.6.0/2026-10-31-threads-crossposting.md
|
||||
THREADS_ACCESS_TOKEN=
|
||||
@@ -194,19 +201,33 @@ THREADS_USERNAME=
|
||||
# Seconds between syndication worker passes (default: 10)
|
||||
# SYNDICATION_POLL_INTERVAL=10
|
||||
|
||||
# Backfeed (v2.13.0, ADR-073): bring likes, reposts, quotes and replies to the
|
||||
# crossposts back to the note as responses. Needs the read scopes above, and
|
||||
# runs in the syndication worker, so at least one target must be configured.
|
||||
# BACKFEED_ENABLED=false
|
||||
# Stop checking a note's responses after this many days (default: 90)
|
||||
# BACKFEED_MAX_AGE_DAYS=90
|
||||
|
||||
# =============================================================================
|
||||
# MONITORING
|
||||
# NOTIFICATIONS (v2.14.0, ADR-074; optional)
|
||||
# =============================================================================
|
||||
|
||||
# Metrics shown on /admin/metrics-dashboard (default: true)
|
||||
# METRICS_ENABLED=true
|
||||
# Queries slower than this many seconds are logged (default: 1.0)
|
||||
# METRICS_SLOW_QUERY_THRESHOLD=1.0
|
||||
# Fraction of operations recorded, 0.0-1.0 (default: 1.0)
|
||||
# METRICS_SAMPLING_RATE=1.0
|
||||
# METRICS_BUFFER_SIZE=1000
|
||||
# Seconds between memory samples (default: 30)
|
||||
# METRICS_MEMORY_INTERVAL=30
|
||||
# Apprise URLs to notify, space- or comma-separated
|
||||
# (https://github.com/caronc/apprise/wiki). Empty: notifications off.
|
||||
# Examples: gotifys://gotify.example.com/APPTOKEN ntfys://my-topic
|
||||
# apprises://apprise.example.com/KEY (an apprise-api server)
|
||||
# NOTIFY_URLS=
|
||||
# What to notify: reply, mention, quote, like, repost, bookmark, problems
|
||||
# (a crosspost that failed for good, a silo refusing backfeed). Names are
|
||||
# case-insensitive; unknown ones are ignored.
|
||||
# NOTIFY_EVENTS=reply,mention,quote,problems
|
||||
# Priority of every message, added to URLs that don't set one:
|
||||
# low, moderate, normal, high, emergency (Gotify also takes 0-10)
|
||||
# NOTIFY_PRIORITY=normal
|
||||
|
||||
# =============================================================================
|
||||
# DEBUGGING
|
||||
# =============================================================================
|
||||
|
||||
# Keep uploads that fail processing in DATA_PATH/debug for diagnosis
|
||||
# (default: false); cleaned up by age (default: 7 days) and total size
|
||||
|
||||
+136
-5
@@ -7,6 +7,140 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [2.14.2] - 2026-11-10
|
||||
|
||||
### Added
|
||||
|
||||
- Quotes have a count in the line under each note (❝) and in the counts on the home, tag and Likes pages. It includes quotes a silo reports without names (Threads, and people who opted out of backfeed), which were stored but shown nowhere
|
||||
|
||||
### Removed
|
||||
|
||||
- The editor pages no longer load `marked` from a CDN and an empty `static/js/preview.js`; nothing used them
|
||||
- The empty `templates/feed.xml`, the v1 `static/indielogin-test.html`, and the `THREADS_API` setting that crossposting read but nothing could set
|
||||
|
||||
## [2.14.1] - 2026-11-10
|
||||
|
||||
### Fixed
|
||||
|
||||
- Clearing the Tags field in the note editor removes all of the note's tags. It used to leave them unchanged, so a note's last tags couldn't be removed from the editor
|
||||
- Image captions are escaped in the HTML of RSS, Atom and JSON Feed items. A caption containing `"`, `<` or `&` broke the item's image markup (the Atom enclosure's title was already escaped)
|
||||
|
||||
## [2.14.0] - 2026-11-10
|
||||
|
||||
Notifications (ADR-074).
|
||||
|
||||
### Added
|
||||
|
||||
- StarPunk can notify you of new responses through [Apprise](https://github.com/caronc/apprise): Gotify, ntfy, Pushover, email, Matrix, Discord and about 100 more, or an apprise-api server. Set `NOTIFY_URLS` to one or more Apprise URLs (e.g. `gotifys://gotify.example.com/APPTOKEN`)
|
||||
- `NOTIFY_EVENTS` chooses what you hear about: `reply`, `mention`, `quote`, `like`, `repost`, `bookmark`, and `problems` (a crosspost that failed for good, or a silo refusing backfeed). Default `reply,mention,quote,problems`. Replies, mentions and quotes come one at a time with their text; likes, reposts and bookmarks are grouped per note
|
||||
- `NOTIFY_PRIORITY` (default `normal`) sets the priority of every message
|
||||
- Received webmentions and backfed responses are notified alike, each once. Responses waiting for moderation say so and link to the admin
|
||||
- Admin → Webmentions shows the notification settings and the last send error, with **Send test notification**
|
||||
|
||||
### Changed
|
||||
|
||||
- New dependency: `apprise` 2.2. Migration 021 marks every existing response as already notified, and responses found by a backfill (or on a crosspost's first check of an older note) are never notified
|
||||
|
||||
## [2.13.4] - 2026-11-10
|
||||
|
||||
### Added
|
||||
|
||||
- The home page, tag pages and the Likes page show each note's like, repost and reply counts next to its date (♥ 3 🔁 1 💬 2), linking to the note's responses. Notes with none show nothing
|
||||
|
||||
### Changed
|
||||
|
||||
- Clicking anywhere on a reply opens it where it was posted (the reply on Bluesky, GoToSocial or Threads, or the page that sent the webmention). The author link and links in the reply still go to their own pages
|
||||
|
||||
### Fixed
|
||||
|
||||
- The "Liked", "Reposted" and "In reply to" lines on notes picked up styles meant for v2.13.2's counts line (both used the class `reaction`). The counts line's classes are now `response-*`
|
||||
|
||||
## [2.13.3] - 2026-11-10
|
||||
|
||||
### Changed
|
||||
|
||||
- Likes and reposts by people who opted out of backfeed (`#nobridge` or `#nobot` in their profile, or Bluesky's "don't show me to logged-out users" label) are counted in a note's totals, without their name or avatar ("+1 on Bluesky"). Before, they were left out entirely, so totals were lower than on the silo
|
||||
|
||||
## [2.13.2] - 2026-11-10
|
||||
|
||||
### Changed
|
||||
|
||||
- Likes, reposts and bookmarks are no longer rows of faces under "Responses". A note shows one line of counts under the post (♥ likes, 🔁 reposts, 💬 replies, bookmarks), and hovering or tapping a count lists who, with where each came from. Threads' anonymous counts are part of the totals ("+1 on Threads"), so the separate "On Threads" line is gone. The Responses section now holds replies, quotes and mentions
|
||||
- Web actions (https://indieweb.org/webactions): each like, repost and reply count is an `<indie-action>` for the note, with links to do the same on the note's crossposts ("Like on Bluesky · GoToSocial · Threads"). The icons show even before anyone has responded, when the note has crossposts
|
||||
|
||||
### Fixed
|
||||
|
||||
- Avatars that GoToSocial hasn't cached (it redirects to the original server) are saved; the redirect is followed through the safe fetcher
|
||||
|
||||
## [2.13.1] - 2026-11-10
|
||||
|
||||
### Fixed
|
||||
|
||||
- Avatars from your own GoToSocial server are saved again. When StarPunk reaches it through the same proxy, its name resolves to an internal address, which the safe fetcher refused. That one server (`GOTOSOCIAL_URL`) is now fetched directly, as crossposting already does
|
||||
- Bluesky avatars are saved again. Bluesky's image CDN refuses some servers' IPv4 addresses (HTTP 403), so avatars are now downloaded from the account's own server (`com.atproto.sync.getBlob`)
|
||||
|
||||
## [2.13.0] - 2026-11-10
|
||||
|
||||
Backfeed: responses to crossposts come back to the note (ADR-073).
|
||||
|
||||
### Added
|
||||
|
||||
- Likes, reposts, quotes and replies to a note's crossposts on GoToSocial and Bluesky, and replies on Threads, are shown on the note as responses, labelled "via Bluesky" (or GoToSocial, Threads). StarPunk checks each crosspost itself: every 15 minutes on a note's first day, hourly in its first week, daily until 30 days, weekly until `BACKFEED_MAX_AGE_DAYS` (90). Withdrawn likes and deleted replies drop off at the next check
|
||||
- Threads like, repost and quote counts, shown as a line such as "On Threads: 12 likes · 3 reposts" (the Threads API doesn't say who)
|
||||
- A Quotes group in a note's responses
|
||||
- **Backfill responses** on the admin Webmentions page checks every crosspost once, however old the note; each note's edit page shows when its responses were last checked, with **Check now**
|
||||
- The admin Webmentions list can be filtered by where a response came from, and shows deleted ones under **Deleted**
|
||||
- Settings `BACKFEED_ENABLED` (default off) and `BACKFEED_MAX_AGE_DAYS`. Backfeed needs read access: a GoToSocial token with `read:statuses` and `read:accounts` (or `read`), and a Threads token with `threads_read_replies` and `threads_manage_insights`. Bluesky app passwords already work
|
||||
|
||||
### Changed
|
||||
|
||||
- Response authors' avatars are saved on this site (96 px WebP in `DATA_PATH/avatars`) and shown from there, instead of being loaded from other sites. Until a copy is saved the initial is shown
|
||||
- Deleting a webmention in the admin keeps it as a deleted record, so a re-sent webmention or the next check doesn't bring it back
|
||||
- People with `#nobridge` or `#nobot` in their profile, and Bluesky accounts that ask not to be shown to logged-out visitors, are not backfed; only public responses are
|
||||
- A webmention for a response that was already backfed (for example a reply also sent by Bridgy) is ignored, so it isn't shown twice
|
||||
- The Threads client uses `graph.threads.com`, the host in Meta's current documentation
|
||||
|
||||
## [2.12.1] - 2026-11-10
|
||||
|
||||
### Fixed
|
||||
|
||||
- Micropub `q=category` (the tag list apps use for autocomplete) only lists tags on posts that still exist. It used to include tags that an edit had removed from every post, and tags only used on deleted posts. Drafts still count; a deleted post's tags come back if it is undeleted
|
||||
|
||||
## [2.12.0] - 2026-11-03
|
||||
|
||||
The metrics dashboard is gone (ADR-072).
|
||||
|
||||
### Removed
|
||||
|
||||
- The admin metrics dashboard (`/admin/metrics-dashboard`, the "Metrics" admin link) and its JSON endpoint `/admin/metrics`. Its numbers were kept in memory per server process, so they covered a quarter of the traffic and vanished on every restart
|
||||
- Everything that only fed it: request and database-query timing, the memory monitor thread, feed-generation counters (`starpunk/monitoring/`), and the `X-Request-ID` response header. Slow queries are no longer logged
|
||||
- The `METRICS_ENABLED`, `METRICS_SLOW_QUERY_THRESHOLD`, `METRICS_SAMPLING_RATE`, `METRICS_BUFFER_SIZE` and `METRICS_MEMORY_INTERVAL` settings (ignored if still set), and the `psutil` dependency
|
||||
- The `metrics` section of `/admin/health`; its database, pool, filesystem and disk checks are unchanged, as is `/health`
|
||||
|
||||
## [2.11.0] - 2026-11-03
|
||||
|
||||
Maintenance release: dependency upgrades and the High-priority backlog items.
|
||||
|
||||
### Security
|
||||
|
||||
- Every dependency is upgraded to its current release. The previous set had 97 published vulnerabilities (Pillow 64, PyJWT 22, gunicorn 4, and Flask, Markdown, python-dotenv, pytest and pillow-heif); the new set has none known. Notable versions: Flask 3.1, gunicorn 26, Pillow 13, PyJWT 2.15, httpx 0.28, nh3 0.3, Markdown 3.11, pillow-heif 1.8, psutil 7
|
||||
- Admin forms, logout and the IndieAuth consent screen now carry a CSRF token (an HMAC of your login session, no storage needed); requests without it are refused. Endpoints that apps call with tokens (Micropub, Webmention, WebSub, the token endpoint) are unaffected
|
||||
- Fetching an IndieAuth app's page and fetching pages and images for crosspost link cards now use the same protected fetcher as webmentions: every redirect is checked against private addresses, the connection goes to the checked address, and the whole fetch has a time limit
|
||||
- Debug mode is off and the environment is production unless set otherwise (`flask run --debug` still turns debug on locally); the container image sets both explicitly
|
||||
|
||||
### Changed
|
||||
|
||||
- Runs on Python 3.14 (was 3.11, which reaches end of life in 2027)
|
||||
- Rate limits on `/webmention` and `/websub` are counted in the database and shared by all server processes. Before, each of the four processes counted separately, so the real limit was four times the setting
|
||||
- Test dependencies are in `requirements-test.txt` (used by CI); `pytest` is no longer installed in the production image, and `lxml` and `python-dateutil`, which StarPunk uses directly, are now pinned
|
||||
- `uv` is pinned in the Containerfile instead of `latest`
|
||||
|
||||
### Fixed
|
||||
|
||||
- Deleting a note or a photo left its tag, photo and image-size rows behind, because SQLite only applies the schema's cascades when foreign keys are switched on. They now are; migration 018 removes rows left by earlier deletes (production had three)
|
||||
- Deprecated `datetime.utcnow()` and sqlite3's default date adapter (both removed in a future Python) replaced; the stored formats are unchanged
|
||||
- `compose.yaml` pinned version 0.6.0 (which would override the real version shown) and set an unused variable; a stray duplicate `dev_auth.py` and a `test.ini` with a private host name were removed
|
||||
|
||||
## [2.10.2] - 2026-11-03
|
||||
|
||||
### Fixed
|
||||
@@ -518,7 +652,7 @@ Webmention now follows the [W3C Recommendation](https://www.w3.org/TR/webmention
|
||||
- Successful uploads logged at INFO level with file details
|
||||
- Removed duplicate logging in Micropub media endpoint
|
||||
|
||||
## [1.4.0rc1] - 2025-12-16
|
||||
## [1.4.0-rc.1] - 2025-12-16
|
||||
|
||||
### Added
|
||||
|
||||
@@ -2061,7 +2195,4 @@ This is a release candidate for testing. Stable 1.0.0 will be released after tes
|
||||
- ADR-007: Slug generation algorithm
|
||||
- ADR-008: Versioning strategy
|
||||
|
||||
[Unreleased]: https://github.com/YOUR_USERNAME/starpunk/compare/v0.4.0...HEAD
|
||||
[0.4.0]: https://github.com/YOUR_USERNAME/starpunk/compare/v0.3.0...v0.4.0
|
||||
[0.3.0]: https://github.com/YOUR_USERNAME/starpunk/compare/v0.1.0...v0.3.0
|
||||
[0.1.0]: https://github.com/YOUR_USERNAME/starpunk/releases/tag/v0.1.0
|
||||
Releases are tagged `vX.Y.Z` in https://git.thesatelliteoflove.com/phil/StarPunk
|
||||
@@ -7,8 +7,8 @@ This file contains operational instructions for Claude agents working on this pr
|
||||
- We use **uv** for Python virtual environment management
|
||||
- All Python commands must be run with `uv run` prefix
|
||||
- Example: `uv run pytest`, `uv run flask --app app.py run --debug`
|
||||
- There is no `pyproject.toml`: create the environment with `uv venv --python 3.11 && uv pip install -r requirements-dev.txt`
|
||||
- Tests need `SESSION_SECRET` set: `SESSION_SECRET=test uv run pytest`. The full suite takes about 6 minutes, 5 of them in `tests/test_migration_race_condition.py`; add `--ignore=tests/test_migration_race_condition.py` while iterating, but run everything before a release
|
||||
- There is no `pyproject.toml`: create the environment with `uv venv --python 3.14 && uv pip install -r requirements-dev.txt`
|
||||
- Tests need `SESSION_SECRET` set. Locally, run only the tests for the area you changed (`SESSION_SECRET=test uv run pytest tests/test_<area>.py`); CI runs the full suite (about 6 minutes) on every `vX.Y.Z` tag, so don't run the whole suite yourself unless asked
|
||||
- Local login without OIDC: `DEV_MODE=true` and `DEV_ADMIN_ME` in `.env`, then `/dev/login`
|
||||
- Setup details: `docs/standards/development-setup.md`
|
||||
|
||||
@@ -50,12 +50,12 @@ When invoking the agent-developer, always remind it to:
|
||||
|
||||
5. **Follow git protocol**
|
||||
- Adhere to git branching strategy in `docs/standards/git-branching-strategy.md`
|
||||
- Create feature branches for non-trivial changes, in their own git worktree (`<prefix>/vX.Y.Z-description`, prefixes `feat/`, `fix/`, `ci/`, `docs/`)
|
||||
- Create feature branches for non-trivial changes, in their own git worktree (`<prefix>/vX.Y.Z-description`, prefixes `feat/`, `fix/`, `ci/`, `chore/`, `docs/`)
|
||||
- Write clear commit messages (`feat: ... (vX.Y.Z)`)
|
||||
|
||||
## Release Process
|
||||
|
||||
Follow "Release process" in `docs/standards/git-branching-strategy.md`. In short: work in a worktree on a `feat/`/`fix/` branch → bump `__version__` and `__version_info__` → `CHANGELOG.md` entry → design note in `docs/design/vX.Y.Z/` (or ADR amendment) → full test suite → commit → `git merge --no-ff` into `main` ("Merge branch '<branch>' for vX.Y.Z release") → annotated tag `vX.Y.Z` → push `main` and the tag. CI (`.gitea/workflows/build-container.yml`) runs the tests and publishes the image; production is then updated from the separate Ansible deployment repository. Back up the database before deploying a release with a migration.
|
||||
Follow "Release process" in `docs/standards/git-branching-strategy.md`. In short: work in a worktree on a `feat/`/`fix/`/`chore/` branch → bump `__version__` and `__version_info__` → `CHANGELOG.md` entry → design note in `docs/design/vX.Y.Z/` (or ADR amendment) → tests for the changed area → commit → `git merge --no-ff` into `main` ("Merge branch '<branch>' for vX.Y.Z release") → annotated tag `vX.Y.Z` → push `main` and the tag. CI (`.gitea/workflows/build-container.yml`) runs the full test suite and publishes the image; production is then updated from the separate Ansible deployment repository. Back up the database before deploying a release with a migration.
|
||||
|
||||
Merging to `main`, tagging, pushing and deploying happen only when the project owner asks for a release.
|
||||
|
||||
|
||||
+9
-6
@@ -6,10 +6,10 @@
|
||||
# ============================================================================
|
||||
# Build Stage - Install dependencies in virtual environment
|
||||
# ============================================================================
|
||||
FROM python:3.11-slim AS builder
|
||||
FROM python:3.14-slim AS builder
|
||||
|
||||
# Install uv for fast dependency installation
|
||||
COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.9.28 /uv /usr/local/bin/uv
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
@@ -25,11 +25,12 @@ RUN uv venv /opt/venv && \
|
||||
# ============================================================================
|
||||
# Test Stage - Runs the test suite (CI builds it with --target test)
|
||||
# ============================================================================
|
||||
FROM python:3.11-slim AS test
|
||||
FROM python:3.14-slim AS test
|
||||
|
||||
COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.9.28 /uv /usr/local/bin/uv
|
||||
COPY --from=builder /opt/venv /opt/venv
|
||||
RUN uv pip install --no-cache --python /opt/venv/bin/python pytest pytest-mock
|
||||
COPY requirements-test.txt .
|
||||
RUN uv pip install --no-cache --python /opt/venv/bin/python -r requirements-test.txt
|
||||
|
||||
ENV PATH="/opt/venv/bin:$PATH" \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
@@ -45,7 +46,7 @@ RUN python -m pytest -q -p no:cacheprovider tests
|
||||
# ============================================================================
|
||||
# Runtime Stage - Minimal production image (default target: keep it last)
|
||||
# ============================================================================
|
||||
FROM python:3.11-slim
|
||||
FROM python:3.14-slim
|
||||
|
||||
# Create non-root user for security
|
||||
# UID/GID 1000 is standard for first user on most systems
|
||||
@@ -60,6 +61,8 @@ COPY --from=builder /opt/venv /opt/venv
|
||||
ENV PATH="/opt/venv/bin:$PATH" \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
FLASK_ENV=production \
|
||||
FLASK_DEBUG=0 \
|
||||
FLASK_APP=app.py \
|
||||
DATA_PATH=/data \
|
||||
NOTES_PATH=/data/notes \
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2024 [Your Name]
|
||||
Copyright (c) 2025-2026 Phil Skentelbery
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
||||
@@ -24,8 +24,9 @@ what changed in each release is in [CHANGELOG.md](CHANGELOG.md).
|
||||
Micropub apps (consent screen, tokens, revocation, Connected Apps page)
|
||||
- Admin login through any OpenID Connect provider (e.g. Pocket ID)
|
||||
- [Webmention](https://www.w3.org/TR/webmention/): receiving (verified,
|
||||
moderated, rate-limited) and sending; replies, likes and reposts shown on
|
||||
each note
|
||||
moderated, rate-limited) and sending; replies, quotes and mentions shown
|
||||
on each note, with likes, reposts and replies as a line of counts with
|
||||
hover cards and `<indie-action>` web actions
|
||||
- [WebSub](https://www.w3.org/TR/websub/) with a built-in hub, so readers get
|
||||
new posts within seconds
|
||||
- [Microformats2](https://microformats.org/wiki/h-entry): h-entry, h-feed and
|
||||
@@ -38,10 +39,15 @@ what changed in each release is in [CHANGELOG.md](CHANGELOG.md).
|
||||
- POSSE crossposting to GoToSocial/Mastodon, Bluesky and Threads, with
|
||||
`u-syndication` links back; edits and deletes follow, native likes and
|
||||
boosts where the target supports them
|
||||
- Backfeed: likes, reposts and replies to crossposts on GoToSocial and
|
||||
Bluesky (and Bluesky quotes), and replies on Threads (with its like and
|
||||
repost counts), come back as responses (`BACKFEED_ENABLED`, ADR-073)
|
||||
|
||||
**Operations**
|
||||
- Migrations applied automatically at startup
|
||||
- `/health` endpoint, admin metrics dashboard, structured logs
|
||||
- `/health` endpoint, `/admin/health` diagnostics, structured logs
|
||||
- Notifications of new responses and crossposting problems through
|
||||
Apprise (Gotify, ntfy, email…; `NOTIFY_URLS`, ADR-074)
|
||||
- Fetches of user-supplied URLs are limited to public addresses, size and
|
||||
time
|
||||
|
||||
@@ -54,7 +60,6 @@ git clone https://git.thesatelliteoflove.com/phil/StarPunk.git && cd StarPunk
|
||||
cp .env.example .env # set SITE_URL, ADMIN_ME, SESSION_SECRET, OIDC_*
|
||||
docker build -t starpunk -f Containerfile .
|
||||
docker run -d --name starpunk -p 127.0.0.1:8000:8000 --env-file .env \
|
||||
-e FLASK_ENV=production -e FLASK_DEBUG=0 \
|
||||
-v ./starpunk-data:/data starpunk
|
||||
```
|
||||
|
||||
@@ -69,11 +74,11 @@ redirect URI with your OIDC provider. Details:
|
||||
### Local development
|
||||
|
||||
```bash
|
||||
uv venv --python 3.11
|
||||
uv venv --python 3.14
|
||||
uv pip install -r requirements-dev.txt
|
||||
cp .env.example .env # set SESSION_SECRET; add DEV_MODE=true and DEV_ADMIN_ME
|
||||
uv run flask --app app.py run --debug # http://localhost:5000
|
||||
SESSION_SECRET=test uv run pytest # tests (about 6 minutes)
|
||||
uv run flask --app app.py run --debug # http://localhost:5000
|
||||
SESSION_SECRET=test uv run pytest tests/test_<area>.py # tests for what you changed; CI runs the full suite
|
||||
```
|
||||
|
||||
With `DEV_MODE=true`, `/dev/login` logs you in without OIDC. Never enable
|
||||
@@ -86,13 +91,15 @@ All settings are environment variables (or a `.env` file), each listed with
|
||||
its default in [`.env.example`](.env.example). Required in production:
|
||||
`SESSION_SECRET`, `SITE_URL`, `ADMIN_ME`, `OIDC_ISSUER`, `OIDC_CLIENT_ID`,
|
||||
`OIDC_CLIENT_SECRET` and `OIDC_ADMIN_SUB`. Crossposting is enabled by
|
||||
setting a target's credentials.
|
||||
setting a target's credentials. Backfeed is off unless
|
||||
`BACKFEED_ENABLED=true`; notifications start when `NOTIFY_URLS` is set.
|
||||
|
||||
## Data and backups
|
||||
|
||||
Everything lives in the data directory (`/data` in the container): the
|
||||
SQLite database, note files in `notes/`, uploaded media and logs. Back it
|
||||
up regularly, and always before upgrading to a release with a migration.
|
||||
SQLite database, note files in `notes/`, uploaded media, saved response
|
||||
avatars in `avatars/`, the Threads token (`threads_token.json`) and logs.
|
||||
Back it up regularly, and always before upgrading to a release with a migration.
|
||||
|
||||
## Documentation
|
||||
|
||||
@@ -113,4 +120,5 @@ up regularly, and always before upgrading to a release with a migration.
|
||||
|
||||
## License
|
||||
|
||||
MIT, see [LICENSE](LICENSE).
|
||||
[MIT](LICENSE): use, change and share it for anything, including
|
||||
commercially, as long as the copyright and license notice stay with it.
|
||||
+2
-7
@@ -11,12 +11,11 @@
|
||||
# docker compose logs -f
|
||||
# docker compose down
|
||||
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
starpunk:
|
||||
# Container configuration
|
||||
image: starpunk:0.6.0
|
||||
# Built locally from this checkout (release images: see README)
|
||||
image: starpunk:local
|
||||
container_name: starpunk
|
||||
|
||||
# Build configuration
|
||||
@@ -49,10 +48,6 @@ services:
|
||||
- NOTES_PATH=/data/notes
|
||||
- DATABASE_PATH=/data/starpunk.db
|
||||
|
||||
# Application metadata
|
||||
- VERSION=0.6.0
|
||||
- ENVIRONMENT=production
|
||||
|
||||
# Volume mounts for persistent data
|
||||
# All application data stored in ./container-data on host
|
||||
volumes:
|
||||
|
||||
-103
@@ -1,103 +0,0 @@
|
||||
"""
|
||||
Development authentication module for StarPunk
|
||||
|
||||
WARNING: This module provides a development-only authentication mechanism
|
||||
that bypasses IndieLogin. It should NEVER be enabled in production.
|
||||
|
||||
This module is separate from production auth (starpunk/auth.py) to maintain
|
||||
clear architectural boundaries and enable easy security audits.
|
||||
|
||||
Security measures:
|
||||
- Only active when DEV_MODE=true
|
||||
- Returns 404 if DEV_MODE=false
|
||||
- Requires DEV_ADMIN_ME configuration
|
||||
- Logs prominent warnings
|
||||
- Cannot coexist with production SITE_URL
|
||||
- Visual warnings in UI
|
||||
|
||||
Functions:
|
||||
is_dev_mode: Check if development mode is enabled
|
||||
validate_dev_config: Validate development configuration
|
||||
create_dev_session: Create session without authentication
|
||||
"""
|
||||
|
||||
from flask import current_app
|
||||
|
||||
from starpunk.auth import create_session
|
||||
|
||||
|
||||
def is_dev_mode() -> bool:
|
||||
"""
|
||||
Check if development mode is enabled
|
||||
|
||||
Returns:
|
||||
True if DEV_MODE is enabled, False otherwise
|
||||
|
||||
Example:
|
||||
>>> from starpunk.dev_auth import is_dev_mode
|
||||
>>> if is_dev_mode():
|
||||
... print("Development mode active")
|
||||
"""
|
||||
return current_app.config.get("DEV_MODE", False)
|
||||
|
||||
|
||||
def validate_dev_config() -> None:
|
||||
"""
|
||||
Validate development mode configuration
|
||||
|
||||
Checks that DEV_MODE configuration is valid and safe:
|
||||
- DEV_ADMIN_ME must be set if DEV_MODE is true
|
||||
- Warns if DEV_MODE is enabled with production-like SITE_URL
|
||||
|
||||
Raises:
|
||||
ValueError: If DEV_MODE is true but DEV_ADMIN_ME is not set
|
||||
|
||||
Logs:
|
||||
WARNING: If DEV_MODE is enabled with HTTPS SITE_URL
|
||||
"""
|
||||
dev_mode = current_app.config.get("DEV_MODE", False)
|
||||
|
||||
if dev_mode:
|
||||
# Require DEV_ADMIN_ME
|
||||
dev_admin_me = current_app.config.get("DEV_ADMIN_ME")
|
||||
if not dev_admin_me:
|
||||
raise ValueError("DEV_MODE=true requires DEV_ADMIN_ME to be set")
|
||||
|
||||
# Warn if production-like configuration detected
|
||||
site_url = current_app.config.get("SITE_URL", "")
|
||||
if site_url.startswith("https://"):
|
||||
current_app.logger.warning(
|
||||
"WARNING: DEV_MODE is enabled with production SITE_URL. "
|
||||
"This is likely a misconfiguration. "
|
||||
"DEV_MODE should only be used in local development."
|
||||
)
|
||||
|
||||
|
||||
def create_dev_session(me: str) -> str:
|
||||
"""
|
||||
Create development session without authentication
|
||||
|
||||
WARNING: This bypasses IndieLogin authentication entirely.
|
||||
Only use in development environments.
|
||||
|
||||
Args:
|
||||
me: User identity URL (from DEV_ADMIN_ME config)
|
||||
|
||||
Returns:
|
||||
Session token (same format as production sessions)
|
||||
|
||||
Logs:
|
||||
WARNING: Logs that dev session was created without authentication
|
||||
|
||||
Example:
|
||||
>>> token = create_dev_session("https://example.com")
|
||||
>>> # Session created without IndieLogin verification
|
||||
"""
|
||||
current_app.logger.warning(
|
||||
f"DEV MODE: Creating session for {me} without authentication. "
|
||||
f"This should NEVER happen in production!"
|
||||
)
|
||||
|
||||
# Use production session creation (same session format)
|
||||
# This ensures dev sessions work identically to production
|
||||
return create_session(me)
|
||||
@@ -1,9 +1,11 @@
|
||||
# Architecture Documentation Index
|
||||
|
||||
This directory describes the **current** design of StarPunk (v2.10.1). Per
|
||||
This directory describes the **current** design of StarPunk (v2.14.0). Per
|
||||
[../DOCUMENTATION.md](../DOCUMENTATION.md) it is updated at the end of each
|
||||
release. Decisions and their history are in [../decisions/](../decisions/);
|
||||
per-release designs and reports are in [../design/](../design/).
|
||||
per-release designs and reports are in [../design/](../design/). The
|
||||
original v0.x technology choices are kept in
|
||||
[../design/v0.x/technology-stack-legacy.md](../design/v0.x/technology-stack-legacy.md).
|
||||
|
||||
## Documents
|
||||
|
||||
@@ -11,12 +13,11 @@ per-release designs and reports are in [../design/](../design/).
|
||||
|----------|-------------|
|
||||
| [overview.md](overview.md) | System overview: purpose, request flow, modules and routes, background workers, storage, auth model, URL surface, caching, security posture |
|
||||
| [technology-stack.md](technology-stack.md) | Current runtime, Python dependencies, standards implemented, testing and CI |
|
||||
| [database.md](database.md) | SQLite schema after migrations 001–017, migration runner and its lock, connection pool |
|
||||
| [database.md](database.md) | SQLite schema after migrations 001–021, migration runner and its lock, connection pool |
|
||||
| [deployment.md](deployment.md) | Container image stages, CI workflow, release process, `/data` layout, reverse proxy, all environment variables |
|
||||
| [simplified-auth-architecture.md](simplified-auth-architecture.md) | Admin login via OpenID Connect and the built-in IndieAuth server that issues Micropub tokens (ADR-064) |
|
||||
| [indieweb-protocols.md](indieweb-protocols.md) | How Micropub, IndieAuth, Webmention, microformats2, WebSub and the feeds are implemented, with compliance status and known gaps |
|
||||
| [syndication-architecture.md](syndication-architecture.md) | Crossposting (POSSE) to GoToSocial, Bluesky and Threads: syndications table, worker, reconcile rules |
|
||||
| [technology-stack-legacy.md](technology-stack-legacy.md) | Historical: the original v0.x technology choices (IndieLogin era); not current |
|
||||
| [indieweb-protocols.md](indieweb-protocols.md) | How Micropub, IndieAuth, Webmention, microformats2, WebSub and the feeds are implemented (including backfeed responses, web actions and notifications), with compliance status and known gaps |
|
||||
| [syndication-architecture.md](syndication-architecture.md) | Crossposting (POSSE) to GoToSocial, Bluesky and Threads: syndications table, worker, reconcile rules, backfeed of responses (ADR-073) |
|
||||
|
||||
## Where to start
|
||||
|
||||
@@ -37,4 +38,4 @@ per-release designs and reports are in [../design/](../design/).
|
||||
|
||||
---
|
||||
|
||||
**Last Updated**: 2026-11-03 (v2.10.1)
|
||||
**Last Updated**: 2026-11-10 (v2.14.0)
|
||||
@@ -1,8 +1,8 @@
|
||||
# Database Architecture
|
||||
|
||||
**Version**: 2.10.1 (2026-11-03)
|
||||
**Version**: 2.14.0 (2026-11-10)
|
||||
**Status**: Current. Verified against a database created by `create_app()`
|
||||
with migrations 001–017.
|
||||
with migrations 001–021.
|
||||
|
||||
StarPunk uses one SQLite file, `starpunk.db` under `DATA_PATH` (`/data` in
|
||||
the container), in WAL mode. Note *content* is not in the database: it lives
|
||||
@@ -71,6 +71,10 @@ Background: [ADR-031](../decisions/ADR-031-database-migration-system-redesign.md
|
||||
| `015_bookmarks_and_content_format.sql` | 2.9 | `notes.bookmark_of`, `notes.content_format` |
|
||||
| `016_websub.sql` | 2.10.0 | `websub_subscriptions`, `websub_intents`, `websub_deliveries`, `site_state` |
|
||||
| `017_websub_hardening.sql` | 2.10.1 | `websub_intents.request_id`, `requested_topic`, `callback_ip`; `websub_subscriptions.callback_ip` |
|
||||
| `018_foreign_key_cleanup.sql` | 2.11.0 | Removes rows orphaned while foreign keys weren't enforced |
|
||||
| `019_shared_rate_limits.sql` | 2.11.0 | `rate_limits` (counters shared by all processes) |
|
||||
| `020_backfeed.sql` | 2.13.0 | Rebuilds `webmentions` adding `origin`, `remote_id`, the `quote` type and an index on (`target_note_id`, `origin`); `syndications.backfeed_next_at`, `backfeed_checked_at`, `backfeed_claimed_until`, `backfeed_attempts`, `backfeed_error`, `backfeed_backfill`, `likes_count`, `reposts_count`, `quotes_count` |
|
||||
| `021_notifications.sql` | 2.14.0 | `webmentions.notified_at`; existing rows are marked as notified |
|
||||
|
||||
New migrations are numbered `NNN_description.sql`, must be safe to run inside
|
||||
one transaction, and should use `IF NOT EXISTS` where SQLite allows it.
|
||||
@@ -141,17 +145,20 @@ above.
|
||||
|
||||
### IndieWeb interactions
|
||||
|
||||
**`webmentions`**: received webmentions, one per (`source_url`,
|
||||
`target_url`).
|
||||
**`webmentions`**: received webmentions and backfed silo responses
|
||||
(v2.13.0), one per (`source_url`, `target_url`).
|
||||
|
||||
| Column | Purpose |
|
||||
|--------|---------|
|
||||
| `target_note_id` | The note mentioned (SET NULL if the note is deleted) |
|
||||
| `mention_type` | `reply`, `like`, `repost`, `bookmark`, `mention` |
|
||||
| `mention_type` | `reply`, `like`, `repost`, `bookmark`, `mention`, `quote` |
|
||||
| `author_name`, `author_url`, `author_photo`, `content_snippet` | From the source's mf2 (content sanitised) |
|
||||
| `source_post_url`, `source_published` | The source entry's own `u-url` and `dt-published` |
|
||||
| `validation_status` | `pending`, `valid`, `invalid`, `error` (set by the worker) |
|
||||
| `moderation_status` | `pending`, `approved`, `hidden`, `deleted` (auto-approved when `WEBMENTION_AUTO_APPROVE`) |
|
||||
| `moderation_status` | `pending`, `approved`, `hidden`, `deleted` (auto-approved when `WEBMENTION_AUTO_APPROVE`). `deleted` is a tombstone: deleting in the admin keeps the row, so a re-sent webmention or the next backfeed check doesn't bring it back; re-validation keeps `hidden` and `deleted` |
|
||||
| `origin` | `webmention`, or the silo it was backfed from (`gotosocial`, `bluesky`, `threads`) |
|
||||
| `remote_id` | Silo id of a backfed response (or of the person, for likes and reposts) |
|
||||
| `notified_at` | When the owner was notified of it (v2.14.0); NULL means not yet |
|
||||
| `validation_error`, `validated_at`, `received_at`, `updated_at` | Bookkeeping |
|
||||
|
||||
**`webmention_queue`**: outgoing webmentions, one per (`source_url`,
|
||||
@@ -176,7 +183,12 @@ the note should be on that target; `remote_id` and `remote_url` identify the
|
||||
remote post. `state` (`pending`, `synced`, `failed`), `version` (bumped on
|
||||
every note change, so an edit made during a send is not lost), `attempts`,
|
||||
`last_error`, `next_attempt_at` and `claimed_until` (a 5-minute lease that
|
||||
lets any gunicorn process take the row exclusively) drive the worker. See
|
||||
lets any gunicorn process take the row exclusively) drive the worker.
|
||||
Backfeed (v2.13.0) adds `backfeed_next_at`, `backfeed_checked_at`,
|
||||
`backfeed_claimed_until` (its own lease), `backfeed_attempts`,
|
||||
`backfeed_error` and `backfeed_backfill` (set while a backfill is due), and
|
||||
`likes_count`, `reposts_count`, `quotes_count`: silo-only counts (all of
|
||||
Threads; elsewhere people who opted out). See
|
||||
[syndication-architecture.md](syndication-architecture.md).
|
||||
|
||||
### WebSub
|
||||
@@ -199,25 +211,33 @@ pending row of each kind. `attempts`, `next_attempt_at`, `last_error`.
|
||||
|
||||
**`site_state`**: key/value pairs. `content_version` is bumped whenever a
|
||||
public note changes; every gunicorn process compares it to decide whether
|
||||
its cached feed data is stale.
|
||||
its cached feed data is stale. Backfeed keeps `backfeed_backfill_last` (spacing
|
||||
backfill checks across processes), `backfeed_backfill_started` and
|
||||
`backfeed_backfill_finished`; notifications keep `notify_last_error` (shown
|
||||
in the admin, cleared by a successful send).
|
||||
|
||||
### Bookkeeping
|
||||
|
||||
**`schema_migrations`**: applied migrations (see above).
|
||||
|
||||
**`rate_limits`** (v2.11.0): request counts for `/webmention` and `/websub`,
|
||||
one row per key (`<limiter>:<ip or domain>`) and time window, so every
|
||||
server process enforces one shared limit. Old windows are pruned as it goes.
|
||||
|
||||
## Connections
|
||||
|
||||
- **Request connections**: `starpunk/database/pool.py`. `init_pool()` creates
|
||||
one `ConnectionPool` per process (up to 5 idle connections, 10-second busy
|
||||
timeout; `DB_POOL_SIZE` and `DB_TIMEOUT` can be set only through the
|
||||
`create_app()` config dict). Connections use `sqlite3.Row` rows,
|
||||
`check_same_thread=False` and `PRAGMA journal_mode=WAL`, and are wrapped
|
||||
in `MonitoredConnection` for query timing when metrics are on. `get_db()`
|
||||
`check_same_thread=False`, `PRAGMA journal_mode=WAL` and (since v2.11.0)
|
||||
`PRAGMA foreign_keys=ON`, so `ON DELETE CASCADE`/`SET NULL` run. `get_db()`
|
||||
stores one connection in `flask.g` per request or app context; on teardown
|
||||
it goes back to the pool, rolled back first if a transaction was left
|
||||
open.
|
||||
- **Direct connections**: the migration runner, the FTS start-up check, the
|
||||
webmention worker and sender, reply-context fetching and search helpers
|
||||
webmention worker and sender, the notification sweep (`notifications.py`),
|
||||
avatar upkeep (`avatars.py`), reply-context fetching and search helpers
|
||||
open their own `sqlite3.connect(db_path)` connections.
|
||||
- WAL mode lets readers carry on while one writer writes. The four gunicorn
|
||||
processes and the worker threads share the one file; writes are short, so
|
||||
@@ -227,9 +247,9 @@ its cached feed data is stale.
|
||||
|
||||
The application does not back itself up. A consistent copy needs the
|
||||
database (use SQLite's online backup, e.g. `sqlite3 starpunk.db ".backup …"`,
|
||||
because of WAL) together with `notes/` and `media/`; see
|
||||
[deployment.md](deployment.md).
|
||||
because of WAL) together with `notes/` and `media/`. `avatars/` is optional:
|
||||
missing avatars are fetched again. See [deployment.md](deployment.md).
|
||||
|
||||
---
|
||||
|
||||
**Last Updated**: 2026-11-03 (v2.10.1)
|
||||
**Last Updated**: 2026-11-10 (v2.14.0)
|
||||
@@ -1,6 +1,6 @@
|
||||
# Deployment Architecture
|
||||
|
||||
**Version**: 2.10.1 (2026-11-03)
|
||||
**Version**: 2.14.0 (2026-11-10)
|
||||
**Status**: Current
|
||||
|
||||
How StarPunk is built, configured and run in production. The production site
|
||||
@@ -24,12 +24,12 @@ deployment repository, not here.
|
||||
|
||||
## Container image (`Containerfile`)
|
||||
|
||||
A multi-stage build on `python:3.11-slim`; it works with Podman and Docker.
|
||||
A multi-stage build on `python:3.14-slim`; it works with Podman and Docker.
|
||||
|
||||
| Stage | Purpose |
|
||||
|-------|---------|
|
||||
| `builder` | Copies `uv` from `ghcr.io/astral-sh/uv`, creates `/opt/venv` and installs `requirements.txt` into it |
|
||||
| `test` | Copies the venv, adds `pytest` and `pytest-mock`, copies the source and runs `python -m pytest -q -p no:cacheprovider tests` with test values for `SESSION_SECRET`, `SITE_URL` and `ADMIN_ME`. The stage only builds if the tests pass. |
|
||||
| `test` | Copies the venv, installs `requirements-test.txt`, copies the source and runs `python -m pytest -q -p no:cacheprovider tests` with test values for `SESSION_SECRET`, `SITE_URL` and `ADMIN_ME`. The stage only builds if the tests pass. |
|
||||
| runtime (last, the default target) | Creates user `starpunk` (UID/GID 1000) and `/data/notes`, copies the venv and the application to `/app`, sets `DATA_PATH=/data`, `NOTES_PATH=/data/notes`, `DATABASE_PATH=/data/starpunk.db`, runs as `starpunk`, exposes port 8000 |
|
||||
|
||||
The runtime image's command is gunicorn:
|
||||
@@ -70,21 +70,18 @@ authentication.
|
||||
lower case.
|
||||
4. Remove the builder and log out, even after a failure.
|
||||
|
||||
There is no CI on branches or pull requests; run the tests locally before
|
||||
merging (see [technology-stack.md](technology-stack.md#testing)).
|
||||
There is no CI on branches or pull requests. Locally, run the tests for the
|
||||
area you changed; CI runs the full suite on the tag (see
|
||||
[technology-stack.md](technology-stack.md#testing)).
|
||||
|
||||
## Release process
|
||||
|
||||
1. Finish the change on a branch, including the version bump in
|
||||
`starpunk/__init__.py` (`__version__`, `__version_info__`) and a
|
||||
`CHANGELOG.md` entry.
|
||||
2. Merge to `main`.
|
||||
3. Tag the merge commit `vX.Y.Z` and push the tag. CI tests, builds and
|
||||
pushes the image.
|
||||
4. In the Ansible-managed deployment, change the StarPunk image tag to the new
|
||||
version and apply it. The container is recreated from the new image; new
|
||||
migrations run automatically at start-up.
|
||||
5. Check `/health` reports the new version.
|
||||
Branching, version bump, changelog, merge and tagging are in
|
||||
[git-branching-strategy.md](../standards/git-branching-strategy.md#release-process).
|
||||
Pushing a `vX.Y.Z` tag makes CI test, build and push the image. Then, in the
|
||||
Ansible-managed deployment, set `starpunk_version` to the new tag and deploy
|
||||
the `starpunk` service: the container is recreated from the new image and new
|
||||
migrations run at start-up. Check that `/health` reports the new version.
|
||||
|
||||
Rolling back means deploying the previous tag. Migrations are not reversed
|
||||
automatically, so a release with a migration needs a database backup first
|
||||
@@ -105,6 +102,7 @@ persistent is there; the image itself holds no state.
|
||||
| `media/YYYY/MM/` | Uploaded images and their variants; `media/.tmp/` while processing |
|
||||
| `logs/starpunk.log` | Application log, rotated at 10 MB, 10 files kept |
|
||||
| `webmention-worker.lock`, `websub-worker.lock` | `flock` files choosing the process that runs each worker |
|
||||
| `avatars/<sha256>.webp` | Saved copies of response authors' avatars (v2.13.0), served at `/avatars/`; unused ones removed weekly |
|
||||
| `threads_token.json` (+ `.lock`) | Current Threads access token, refreshed daily (only with Threads configured) |
|
||||
| `debug/` | Failed uploads, only with `DEBUG_SAVE_FAILED_UPLOADS=true`; cleaned up after `DEBUG_FILE_MAX_AGE_DAYS` |
|
||||
|
||||
@@ -161,6 +159,7 @@ unless noted.
|
||||
| `INDIEAUTH_TOKEN_LIFETIME_DAYS` | `90` | Lifetime of Micropub access tokens |
|
||||
| `DEV_MODE` | `false` | Enables `/dev/login`; never in production |
|
||||
| `DEV_ADMIN_ME` | empty | Required when `DEV_MODE=true` |
|
||||
| `CSRF_ENABLED` | `true` | CSRF tokens on admin forms, logout and IndieAuth consent; leave on |
|
||||
|
||||
`TOKEN_ENDPOINT` and `INDIELOGIN_URL` are v1 settings; if present they are
|
||||
ignored with a warning.
|
||||
@@ -172,8 +171,8 @@ ignored with a warning.
|
||||
| `DATA_PATH` | `./data` (image: `/data`) | Root of all stored data |
|
||||
| `DATABASE_PATH` | `./data/starpunk.db` (image: `/data/starpunk.db`) | |
|
||||
| `NOTES_PATH` | `./data/notes` (image: `/data/notes`) | Created at start-up, but note files are always written to `DATA_PATH/notes` |
|
||||
| `FLASK_ENV` | `development` | Set `production` in production |
|
||||
| `FLASK_DEBUG` | `1` | Set `0` in production (the image does not set it) |
|
||||
| `FLASK_ENV` | `production` | Reported in `/health` and admin diagnostics; the image sets `production` |
|
||||
| `FLASK_DEBUG` | `0` | Debug off unless `1` (`flask run --debug` sets it locally); the image sets `0` |
|
||||
| `LOG_LEVEL` | `INFO` | `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL` |
|
||||
| `VERSION` | package `__version__` | Overrides the reported version; normally unset |
|
||||
|
||||
@@ -190,8 +189,8 @@ ignored with a warning.
|
||||
|
||||
| Variable | Default | Notes |
|
||||
|----------|---------|-------|
|
||||
| `WEBMENTION_ENABLED` | `true` | Endpoint, `Link` header and sending |
|
||||
| `WEBMENTION_WORKER_ENABLED` | `true` | |
|
||||
| `WEBMENTION_ENABLED` | `true` | Endpoint, `Link` header, showing responses on notes (not sending) |
|
||||
| `WEBMENTION_WORKER_ENABLED` | `true` | Verifies received and sends outgoing webmentions; saves avatars; sends response notifications |
|
||||
| `WEBMENTION_POLL_INTERVAL` | `10` | Seconds |
|
||||
| `WEBMENTION_ERROR_BACKOFF` | `60` | Seconds after a worker error |
|
||||
| `WEBMENTION_BATCH_SIZE` | `5` | Items per pass |
|
||||
@@ -219,7 +218,7 @@ starts only if at least one is.
|
||||
|
||||
| Variable | Default | Notes |
|
||||
|----------|---------|-------|
|
||||
| `GOTOSOCIAL_URL`, `GOTOSOCIAL_ACCESS_TOKEN` | empty | GoToSocial or any Mastodon-API server |
|
||||
| `GOTOSOCIAL_URL`, `GOTOSOCIAL_ACCESS_TOKEN` | empty | GoToSocial or any Mastodon-API server. Token scopes: `write:statuses write:media read:search write:favourites`, plus `read:statuses read:accounts` (or `read`) for backfeed |
|
||||
| `GOTOSOCIAL_NAME` | `GoToSocial` | Label in the UI and `syndicate-to` |
|
||||
| `GOTOSOCIAL_VISIBILITY` | `public` | |
|
||||
| `GOTOSOCIAL_MAX_CHARS` | `5000` | |
|
||||
@@ -228,22 +227,29 @@ starts only if at least one is.
|
||||
| `BLUESKY_SERVICE` | `https://bsky.social` | |
|
||||
| `BLUESKY_NAME` | `Bluesky` | |
|
||||
| `BLUESKY_MAX_CHARS` | `300` | |
|
||||
| `THREADS_ACCESS_TOKEN` | empty | Long-lived token; refreshed automatically into `threads_token.json` |
|
||||
| `THREADS_ACCESS_TOKEN` | empty | Long-lived token; refreshed automatically into `threads_token.json`. Backfeed also needs `threads_read_replies` and `threads_manage_insights` |
|
||||
| `THREADS_USERNAME` | empty | |
|
||||
| `THREADS_NAME` | `Threads` | |
|
||||
| `THREADS_MAX_CHARS` | `500` | |
|
||||
| `THREADS_MAX_MEDIA` | `4` | |
|
||||
| `SYNDICATION_POLL_INTERVAL` | `10` | Seconds |
|
||||
| `BACKFEED_ENABLED` | `false` | Bring responses to crossposts back to notes (v2.13.0); needs read scopes on the GoToSocial and Threads tokens |
|
||||
| `BACKFEED_MAX_AGE_DAYS` | `90` | Days after posting to keep checking a crosspost's responses |
|
||||
|
||||
### Metrics and debugging
|
||||
### Notifications
|
||||
|
||||
| Variable | Default | Notes |
|
||||
|----------|---------|-------|
|
||||
| `NOTIFY_URLS` | empty (off) | Apprise URLs, space- or comma-separated (v2.14.0) |
|
||||
| `NOTIFY_EVENTS` | `reply,mention,quote,problems` | Also `like`, `repost`, `bookmark` |
|
||||
| `NOTIFY_PRIORITY` | `normal` | Added to URLs that don't set a priority |
|
||||
|
||||
### Debugging
|
||||
|
||||
The `METRICS_*` variables were removed in v2.12.0 and are ignored if set.
|
||||
|
||||
| Variable | Default | Notes |
|
||||
|----------|---------|-------|
|
||||
| `METRICS_ENABLED` | `true` | HTTP/database metrics and the memory monitor |
|
||||
| `METRICS_SLOW_QUERY_THRESHOLD` | `1.0` | Seconds |
|
||||
| `METRICS_SAMPLING_RATE` | `1.0` | |
|
||||
| `METRICS_BUFFER_SIZE` | `1000` | |
|
||||
| `METRICS_MEMORY_INTERVAL` | `30` | Seconds |
|
||||
| `DEBUG_SAVE_FAILED_UPLOADS` | `false` | Keep failed uploads in `debug/` |
|
||||
| `DEBUG_FILE_MAX_AGE_DAYS` | `7` | |
|
||||
| `DEBUG_FILE_MAX_SIZE_MB` | `100` | |
|
||||
@@ -255,15 +261,14 @@ from the environment.
|
||||
## Running locally
|
||||
|
||||
```bash
|
||||
uv venv && uv pip install -r requirements-dev.txt
|
||||
uv venv --python 3.14 && uv pip install -r requirements-dev.txt
|
||||
cp .env.example .env # set SESSION_SECRET; DEV_MODE=true and DEV_ADMIN_ME for local login
|
||||
flask --app app.py run # or: gunicorn app:app
|
||||
uv run flask --app app.py run --debug # or: uv run gunicorn app:app
|
||||
```
|
||||
|
||||
`compose.yaml` builds and runs the image with `./container-data` mounted at
|
||||
`/data`; its image tag and `VERSION` values are out of date, so override them
|
||||
or build with `podman build -t starpunk .`.
|
||||
`compose.yaml` builds the image from this checkout (`starpunk:local`) and runs
|
||||
it with `./container-data` mounted at `/data`.
|
||||
|
||||
---
|
||||
|
||||
**Last Updated**: 2026-11-03 (v2.10.1)
|
||||
**Last Updated**: 2026-11-10 (v2.14.0)
|
||||
@@ -1,6 +1,6 @@
|
||||
# IndieWeb and W3C Protocols
|
||||
|
||||
How StarPunk implements each protocol it speaks, as of v2.10.1, with its
|
||||
How StarPunk implements each protocol it speaks, as of v2.14.0, with its
|
||||
compliance status and the gaps that remain. Each section links to the
|
||||
decision records and review documents behind it.
|
||||
|
||||
@@ -12,6 +12,8 @@ decision records and review documents behind it.
|
||||
| [Microformats2](#microformats2) | Publisher and consumer | Reviewed in v2.8.0 | `templates/`, `starpunk/mf2_utils.py`, `starpunk/author_discovery.py` |
|
||||
| [WebSub](#websub) | Publisher and built-in hub (own topics only) | Adversarial review in v2.10.1 | `starpunk/websub.py`, `starpunk/websub_worker.py`, `starpunk/routes/websub.py` |
|
||||
| [Feeds](#feeds) | RSS 2.0, Atom 1.0, JSON Feed 1.1, OPML 2.0 | Stable | `starpunk/feeds/`, `starpunk/routes/public.py` |
|
||||
| Backfeed | Responses to crossposts read from the silo APIs and stored as webmentions (v2.13.0); see [syndication-architecture.md](syndication-architecture.md#backfeed-v2130-adr-073) | ADR-073 | `starpunk/backfeed.py` |
|
||||
| [Web actions](#moderation-and-display) | `<indie-action>` on the like, repost and reply counts (v2.13.2) | Markup only | `templates/partials/responses.html` |
|
||||
|
||||
Crossposting to other networks (POSSE) is covered in
|
||||
[syndication-architecture.md](syndication-architecture.md).
|
||||
@@ -19,8 +21,10 @@ Crossposting to other networks (POSSE) is covered in
|
||||
## Shared infrastructure: the SSRF-safe fetcher
|
||||
|
||||
Webmention (fetching sources, discovering endpoints, sending), WebSub
|
||||
(verification and delivery) and Micropub (downloading remote photos) all
|
||||
make requests to URLs chosen by someone else. They use
|
||||
(verification and delivery), Micropub (downloading remote photos), IndieAuth
|
||||
(client pages), reply contexts, crosspost link cards (v2.11.0) and response
|
||||
avatars (v2.13.0; except those on StarPunk's own GoToSocial) all make
|
||||
requests to URLs chosen by someone else. They use
|
||||
`starpunk/webmention_ssrf.py`:
|
||||
|
||||
- `validate_url_for_fetch()`: http(s) only. Every address from
|
||||
@@ -128,7 +132,7 @@ repost, bookmark.
|
||||
| `q=syndicate-to` | any token | `{"syndicate-to": [{"uid", "name"}]}` |
|
||||
| `q=source&url=...[&properties[]=...]` | create/update/delete/draft | One post as mf2 JSON. Content comes back as `{"html": ...}` if it was authored as HTML. Includes `syndication` URLs and `post-status` |
|
||||
| `q=source` without `url` | same | `{"items": [...]}`, drafts included, `limit` (default 10, max 50) and `offset` |
|
||||
| `q=category[&filter=...]` | same | Tags in use, most used first |
|
||||
| `q=category[&filter=...]` | same | Tags on notes that aren't deleted (drafts count), most used first (v2.12.1) |
|
||||
|
||||
Unknown or missing `q` gets 400.
|
||||
|
||||
@@ -171,8 +175,7 @@ Full description, scopes and gaps:
|
||||
Decision: [ADR-064](../decisions/ADR-064-oidc-admin-login-and-builtin-indieauth-server.md).
|
||||
|
||||
Known gaps: `scopes_supported` doesn't list `draft`. There are no
|
||||
introspection, userinfo or refresh tokens. The client metadata fetch
|
||||
doesn't use the pinned SSRF fetcher.
|
||||
introspection, userinfo or refresh tokens.
|
||||
|
||||
---
|
||||
|
||||
@@ -197,7 +200,8 @@ route isn't registered at all.
|
||||
|
||||
1. Rate limits: 60 requests per minute per client IP and 10 per minute per
|
||||
source domain (`WEBMENTION_RATE_LIMIT_*`). Over the limit gets 429 with
|
||||
`Retry-After`. The limiter is in memory, per process. The client IP
|
||||
`Retry-After`. Counts are shared by all server processes (`rate_limits`
|
||||
table, v2.11.0). The client IP
|
||||
comes from `X-Forwarded-For` only when `TRUSTED_PROXY=true`.
|
||||
2. Form-encoded (JSON also accepted). `source` and `target` are required,
|
||||
http(s), at most 2048 characters, and different from each other.
|
||||
@@ -238,25 +242,46 @@ Verification is asynchronous (`WebmentionWorker`, poll
|
||||
- Valid: `validation_status = 'valid'`. A first-time mention gets
|
||||
`moderation_status = 'approved'` if `WEBMENTION_AUTO_APPROVE` (default
|
||||
true), otherwise `pending`. An update keeps the admin's earlier
|
||||
moderation decision.
|
||||
moderation decision, including `hidden` and `deleted`.
|
||||
- Source returns 4xx (including 410), or no longer links: `invalid`,
|
||||
which hides it (3.2.4 deletion).
|
||||
- Network error, timeout or 5xx: a mention that was valid before stays
|
||||
valid with its old data. A new one becomes `error`.
|
||||
- A source that is one of the note's own syndicated copies
|
||||
(`syndications.remote_url`) is `invalid` (v2.8.0).
|
||||
- A source that backfeed has already stored for the note is `invalid`
|
||||
("Already backfed from the silo", v2.13.0).
|
||||
- **Re-send on display change** (3.1.4, v2.7.2): if what the note shows
|
||||
for this webmention changed, the note's own outgoing webmentions are
|
||||
re-queued. That covers a new response, a changed author or snippet, or a
|
||||
change of shown/hidden. Unchanged re-validations send nothing, so two
|
||||
sites doing the same don't ping-pong.
|
||||
|
||||
**Moderation and display.** `/admin/webmentions` lists, filters, and
|
||||
approves, hides or deletes mentions, one at a time or in bulk. Changes that
|
||||
alter what is shown also re-send the note's webmentions. Note pages show
|
||||
approved, valid mentions grouped as replies (`u-comment h-cite`), likes,
|
||||
reposts and bookmarks (facepiles), and mentions. See
|
||||
[Microformats2](#microformats2).
|
||||
### Moderation and display
|
||||
|
||||
`/admin/webmentions` lists, filters (by status, including Deleted, and by
|
||||
origin), and approves, hides or deletes responses, one at a time or in bulk;
|
||||
it also has the Backfill button. Deleting keeps the row as a `deleted`
|
||||
tombstone, so a re-sent webmention or backfeed doesn't bring it back.
|
||||
Changes that alter what is shown also re-send the note's webmentions.
|
||||
|
||||
Note pages show approved, valid responses. Under the note, a line of counts
|
||||
(♥ likes, 🔁 reposts, 💬 replies, ❝ quotes (v2.14.2), bookmarks) opens a card on hover or focus
|
||||
listing who responded and where from, including anonymous silo counts. Each
|
||||
like, repost and reply count holds an `<indie-action do with>` web action
|
||||
whose fallback links to the note's crossposts (v2.13.2). The Responses
|
||||
section below holds replies (`u-comment h-cite`), quotes and mentions;
|
||||
backfed ones say "via <silo>". See [Microformats2](#microformats2).
|
||||
|
||||
### Notifications (v2.14.0, ADR-074)
|
||||
|
||||
With `NOTIFY_URLS` set, new valid responses (received and backfed) are sent
|
||||
to the owner once through Apprise by a sweep in the webmention worker, which
|
||||
sets `webmentions.notified_at` (migration 021). Likes, reposts and bookmarks
|
||||
are grouped per note; which kinds are sent is set by `NOTIFY_EVENTS`
|
||||
(default replies, mentions, quotes and problems). Items still unsent after
|
||||
24 hours are dropped. Responses found by a backfill, or by a copy's first
|
||||
check on a note older than a day, are never notified.
|
||||
|
||||
### Sending
|
||||
|
||||
@@ -338,8 +363,9 @@ representative h-card, authorship, Post Type Discovery. Decisions:
|
||||
| `/note/<slug>` | `h-entry`: `p-name` only when the note starts with a `# Heading` (so plain notes have no implied name); `u-photo` as direct children above `e-content`; `u-url u-uid` with `dt-published` (RFC 3339 UTC `Z`); `dt-updated`; `p-category` with `rel=tag`; `u-syndication` links; hidden `p-author h-card` |
|
||||
| Replies | `u-in-reply-to h-cite` holding the cached reply context (`p-content`, `u-url`) |
|
||||
| Likes / reposts / bookmarks | `u-like-of` / `u-repost-of` / `u-bookmark-of` link with "★ Liked" / "🔁 Reposted" / "🔖 Bookmarked" |
|
||||
| Received responses | `u-comment h-cite` (replies: `p-author h-card` with `u-url` on an inner link, `u-photo`, `p-name`; `e-content`; `u-url`; `dt-published`), and `u-like`, `u-repost`, `u-bookmark`, `u-mention` h-cites. Faces beyond 8 are `hidden`, not omitted |
|
||||
| Received responses | `u-comment h-cite` (replies: `p-author h-card` with `u-url` on an inner link, `u-photo`, `p-name`; `e-content`; `u-url`; `dt-published`), likes, reposts and bookmarks as `u-like`, `u-repost`, `u-bookmark` h-cites in the counts line card; mentions and quotes as `u-mention` h-cites |
|
||||
| `/likes`, `/tag/<tag>` | `h-feed` with `p-name`, `u-url`, `p-author h-card` |
|
||||
| Response counts | `/`, `/tag/<tag>` and `/likes` show each note's like, repost and reply counts, linking to its `#responses` (v2.13.4) |
|
||||
|
||||
Post Type Discovery gives note, article, reply, like, repost, bookmark and
|
||||
photo correctly.
|
||||
@@ -445,7 +471,7 @@ Drafts that stay drafts change nothing.
|
||||
|
||||
Limits (429 when hit):
|
||||
|
||||
- 30 requests per minute per client IP (per process; needs
|
||||
- 30 requests per minute per client IP (shared by all processes; needs
|
||||
`TRUSTED_PROXY=true` behind a proxy);
|
||||
- at most 200 pending verifications;
|
||||
- `WEBSUB_MAX_SUBSCRIPTIONS` (1000) in total;
|
||||
@@ -488,7 +514,6 @@ retried.
|
||||
|
||||
- The hub serves only this site's topics, by design. websub.rocks' hub
|
||||
tests use their own topics, so they can't be run against it.
|
||||
- Rate limiting is per process and in memory.
|
||||
- Deliveries always carry the full topic content (no diffs). Fat pings
|
||||
aren't sent to external hubs.
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# StarPunk Architecture Overview
|
||||
|
||||
**Version**: 2.10.1 (2026-11-03)
|
||||
**Version**: 2.14.0 (2026-11-10)
|
||||
**Status**: Current. Describes the system as it is in the code today.
|
||||
|
||||
## Purpose
|
||||
@@ -10,7 +10,9 @@ owner) writes short notes, which are published as microformats2-marked HTML
|
||||
pages and as RSS, Atom and JSON feeds. The site takes part in the IndieWeb:
|
||||
it accepts posts from Micropub clients, issues its own IndieAuth tokens for
|
||||
them, sends and receives Webmentions, pushes feed updates with WebSub, and
|
||||
crossposts (POSSE) to GoToSocial, Bluesky and Threads.
|
||||
crossposts (POSSE) to GoToSocial, Bluesky and Threads. Responses to those
|
||||
crossposts are pulled back and shown on the note (backfeed), and the owner
|
||||
can be notified of new responses.
|
||||
|
||||
Guiding principle: "every line of code must justify its existence". It is a
|
||||
single Flask application with server-side rendering, SQLite and plain files.
|
||||
@@ -39,12 +41,13 @@ Key decisions:
|
||||
[ADR-067](../decisions/ADR-067-micropub-update-delete.md) (Micropub update/delete),
|
||||
[ADR-068](../decisions/ADR-068-likes-and-reposts.md) (likes and reposts),
|
||||
[ADR-071](../decisions/ADR-071-websub.md) (WebSub),
|
||||
[ADR-072](../decisions/ADR-072-remove-metrics-dashboard.md) (metrics dashboard removed),
|
||||
[ADR-073](../decisions/ADR-073-backfeed.md) (backfeed),
|
||||
[ADR-074](../decisions/ADR-074-notifications.md) (notifications),
|
||||
[ADR-057](../decisions/ADR-057-media-attachment-model.md) and
|
||||
[ADR-058](../decisions/ADR-058-image-optimization-strategy.md) (media),
|
||||
[ADR-034](../decisions/ADR-034-full-text-search.md) (search),
|
||||
[ADR-052](../decisions/ADR-052-configuration-system-architecture.md) (configuration),
|
||||
[ADR-053](../decisions/ADR-053-performance-monitoring-strategy.md) and
|
||||
[ADR-060](../decisions/ADR-060-production-hotfix-metrics-dashboard.md) (metrics),
|
||||
[ADR-054](../decisions/ADR-054-structured-logging-architecture.md) (logging),
|
||||
[ADR-055](../decisions/ADR-055-error-handling-philosophy.md) (errors).
|
||||
|
||||
@@ -65,7 +68,6 @@ Key decisions:
|
||||
│ webmention worker works its queues only while holding a lock │
|
||||
│ websub worker works its queues only while holding a lock │
|
||||
│ syndication worker claims rows with a lease in SQLite │
|
||||
│ memory monitor per-process metrics │
|
||||
└───────────────┬──────────────────────────────┬─────────────────────┘
|
||||
│ │
|
||||
┌───────▼────────┐ ┌──────────▼──────────────────┐
|
||||
@@ -73,8 +75,9 @@ Key decisions:
|
||||
│ starpunk.db │ │ OIDC provider, IndieAuth │
|
||||
│ notes/*.md │ │ client pages, webmention │
|
||||
│ media/ │ │ sources/targets, WebSub │
|
||||
│ logs/, locks │ │ callbacks, GoToSocial, │
|
||||
└────────────────┘ │ Bluesky, Threads │
|
||||
│ avatars/ │ │ callbacks, GoToSocial, │
|
||||
│ logs/, locks │ │ Bluesky, Threads, avatars, │
|
||||
└────────────────┘ │ notifications (Apprise) │
|
||||
└─────────────────────────────┘
|
||||
```
|
||||
|
||||
@@ -86,7 +89,7 @@ Key decisions:
|
||||
validates configuration, configures logging, cleans up old debug and
|
||||
temporary upload files, creates the schema and runs migrations (under a
|
||||
lock, see [database.md](database.md)), creates the connection pool,
|
||||
installs the HTTP metrics middleware, configures the feed cache and rate
|
||||
configures the feed cache and rate
|
||||
limiters, fills the FTS index if it is empty, registers blueprints, error
|
||||
handlers and template helpers, and starts the background threads.
|
||||
3. `before_request` assigns a correlation ID (included in every log line) and,
|
||||
@@ -94,8 +97,7 @@ Key decisions:
|
||||
4. The blueprint handler runs. Database access goes through `get_db()`, which
|
||||
hands out a pooled, request-scoped SQLite connection.
|
||||
5. `after_request` hooks add `Link: </webmention>; rel="webmention"` to HTML
|
||||
responses, WebSub `Link rel=hub` / `rel=self` headers to topic responses,
|
||||
and request metrics with an `X-Request-ID` header.
|
||||
responses, WebSub `Link rel=hub` / `rel=self` headers to topic responses.
|
||||
6. Errors are rendered by the central handlers in `starpunk/errors.py`
|
||||
(JSON for Micropub, HTML pages from `templates/<code>.html` elsewhere).
|
||||
|
||||
@@ -131,17 +133,20 @@ wait for outbound network calls; the workers make them later.
|
||||
| `reply_context.py` | Fetches and caches the post a reply responds to |
|
||||
| `webmention_sender.py` | Extracts links from rendered notes and queues outgoing webmentions |
|
||||
| `webmention_processor.py` | Validates incoming webmentions: fetch source, parse mf2, classify, sanitise content |
|
||||
| `webmention_worker.py` | Background thread working the incoming and outgoing webmention queues |
|
||||
| `webmention_worker.py` | Background thread working the incoming and outgoing webmention queues; also runs avatar upkeep and the notification sweep |
|
||||
| `webmention_queries.py` | Webmention queries for display and moderation |
|
||||
| `webmention_ssrf.py` | SSRF-safe fetcher: public-address checks, pinned connections, manual redirects, size and time limits |
|
||||
| `websub.py` | WebSub publisher (discovery headers and links, `content_version`, queueing) and hub request handling |
|
||||
| `websub_worker.py` | Background thread: verification of intent, content delivery, publish pings to an external hub |
|
||||
| `syndication.py` | POSSE clients (Mastodon API/GoToSocial, Bluesky, Threads), reconcile logic, syndication worker |
|
||||
| `syndication.py` | POSSE clients (Mastodon API/GoToSocial, Bluesky, Threads), reconcile logic, syndication worker (its loop also calls `backfeed.process_due()`) |
|
||||
| `backfeed.py` | Backfeed: reads responses to crossposts from the silos, stores them as webmentions, schedules checks and backfills |
|
||||
| `avatars.py` | Saves response authors' avatars as 96 px WebP files in `avatars/`; fills missing ones and removes unused ones |
|
||||
| `notifications.py` | Notifications through Apprise: the sweep of new responses and immediate problem messages |
|
||||
| `csrf.py` | CSRF tokens (HMAC of the session cookie) for admin forms, logout and the IndieAuth consent |
|
||||
| `rate_limiter.py` | In-memory sliding-window rate limiter |
|
||||
| `errors.py` | Central error handlers |
|
||||
| `feeds/` | `rss.py` (feedgen), `atom.py`, `json_feed.py`, `opml.py`, `negotiation.py` (Accept header), `cache.py` (LRU + TTL feed cache with ETags) |
|
||||
| `feed.py` | Backward-compatibility re-export of `feeds.rss` |
|
||||
| `monitoring/` | Metrics buffer, HTTP middleware, database timing wrapper, memory monitor, business metrics |
|
||||
| `database/` | `schema.py` (`INITIAL_SCHEMA_SQL`), `init.py` (`init_db`), `pool.py` (connection pool) |
|
||||
| `migrations.py` | Migration runner for `migrations/NNN_*.sql` |
|
||||
|
||||
@@ -149,7 +154,7 @@ wait for outbound network calls; the workers make them later.
|
||||
|
||||
| Blueprint | File | Paths |
|
||||
|-----------|------|-------|
|
||||
| `public` | `public.py` | `/`, `/note/<slug>`, `/tag/<tag>`, `/likes`, `/media/<path>`, feeds, `/opml.xml` |
|
||||
| `public` | `public.py` | `/`, `/note/<slug>`, `/tag/<tag>`, `/likes`, `/media/<path>`, `/avatars/<name>`, feeds, `/opml.xml` |
|
||||
| `auth` | `auth.py` | `/auth/login` (GET, POST), `/auth/callback`, `/auth/logout` (POST) |
|
||||
| `indieauth` | `indieauth.py` | `/.well-known/oauth-authorization-server`, `/auth/authorize` (GET, POST), `/auth/authorize/approve`, `/auth/token` (GET, POST), `/auth/revoke` |
|
||||
| `micropub` | `micropub.py` | `/micropub` (GET, POST), `/micropub/media` (POST), `/media` (POST, legacy alias) |
|
||||
@@ -168,20 +173,20 @@ Jinja2 templates live in `templates/`: public pages, `admin/`, `partials/`
|
||||
reposts and bookmarks) and one page per HTTP error. `templates/base.html`
|
||||
carries the discovery links (`rel=micropub`, `indieauth-metadata`,
|
||||
`authorization_endpoint`, `token_endpoint`, `webmention`, `hub`/`self`, `me`,
|
||||
feeds). Static assets are one stylesheet (`static/css/style.css`) and one
|
||||
script (`static/js/preview.js`, the Markdown preview in the admin editor).
|
||||
Everything works without JavaScript.
|
||||
feeds). Static assets are one stylesheet (`static/css/style.css`); admin
|
||||
pages have small inline scripts. Everything works without JavaScript.
|
||||
|
||||
## Public URL surface
|
||||
|
||||
| Path | Purpose |
|
||||
|------|---------|
|
||||
| `/` | Home page: recent published notes (h-feed); likes are kept off it |
|
||||
| `/note/<slug>` | Note permalink (h-entry), with approved webmentions and reply context |
|
||||
| `/note/<slug>` | Note permalink (h-entry), with approved responses (webmentions and backfed silo responses), a counts line with web actions, and reply context |
|
||||
| `/tag/<tag>` | Notes with a tag |
|
||||
| `/likes` | Likes |
|
||||
| `/search`, `/api/search` | Full-text search page and JSON API |
|
||||
| `/media/<path>` | Uploaded images and their variants |
|
||||
| `/avatars/<name>` | Saved copies of response authors' avatars |
|
||||
| `/feed` | Feed chosen by the `Accept` header (RSS by default; `Vary: Accept`) |
|
||||
| `/feed.rss`, `/feed.atom`, `/feed.json` | RSS 2.0, Atom 1.0, JSON Feed 1.1 |
|
||||
| `/feed.xml` | Legacy alias of `/feed.rss` |
|
||||
@@ -196,8 +201,15 @@ Everything works without JavaScript.
|
||||
| `/admin/webmentions`, `/admin/webmentions/<id>`, `…/action`, `…/bulk-action` | Webmention moderation |
|
||||
| `/admin/tokens`, `/admin/tokens/<id>/revoke` | Issued IndieAuth tokens |
|
||||
| `/admin/syndication/<id>/retry` | Retry a failed crosspost |
|
||||
| `/admin/metrics-dashboard`, `/admin/metrics`, `/admin/health` | Monitoring (HTML dashboard, JSON metrics, full diagnostics) |
|
||||
| `/admin/syndication/<id>/check` (POST) | Check one crosspost for responses now |
|
||||
| `/admin/backfeed/backfill` (POST) | Check every crosspost for responses once (backfill) |
|
||||
| `/admin/notifications/test` (POST) | Send a test notification |
|
||||
| `/admin/health` | Full diagnostics (JSON): database, pool statistics, filesystem, disk |
|
||||
| `/health` | Public liveness check (`{"status": "ok", "version": …}`); `?detailed=true` requires login |
|
||||
| `/dev/login` | Development login; only when `DEV_MODE=true` |
|
||||
|
||||
`/`, `/tag/<tag>` and `/likes` show each note's response counts, linking to
|
||||
its responses (v2.13.4).
|
||||
|
||||
## Background workers
|
||||
|
||||
@@ -207,10 +219,9 @@ coordinated through the data volume:
|
||||
|
||||
| Thread | Started when | Coordination | Work |
|
||||
|--------|--------------|--------------|------|
|
||||
| Webmention worker (`webmention_worker.py`) | `WEBMENTION_WORKER_ENABLED` and not testing | Non-blocking `fcntl.flock` on `webmention-worker.lock` next to the database. Only the holder works the queues; the others try again each poll interval. The OS releases the lock if the holder dies, and the new holder resets rows stuck in `sending`. | Validates received webmentions; discovers endpoints and sends queued outgoing webmentions, with retries |
|
||||
| Webmention worker (`webmention_worker.py`) | `WEBMENTION_WORKER_ENABLED` and not testing | Non-blocking `fcntl.flock` on `webmention-worker.lock` next to the database. Only the holder works the queues; the others try again each poll interval. The OS releases the lock if the holder dies, and the new holder resets rows stuck in `sending`. | Validates received webmentions; discovers endpoints and sends queued outgoing webmentions, with retries; saves missing response avatars every 10 minutes and removes unused ones weekly (v2.13.0); sweeps new responses into notifications and sets `webmentions.notified_at` (v2.14.0, `notifications.py`, ADR-074) |
|
||||
| WebSub worker (`websub_worker.py`) | `WEBSUB_ENABLED` and not testing | Same pattern with `websub-worker.lock` | Verifies subscription intents, delivers topic content (signed with `X-Hub-Signature` when the subscriber gave a secret), retries, expires leases; pings an external hub instead if `WEBSUB_HUB_URL` is set |
|
||||
| Syndication worker (`syndication.py`) | At least one crosspost target configured, and not testing | No file lock. Rows in `syndications` are claimed with a 5-minute `claimed_until` lease by an atomic `UPDATE`, so any process may work a row but only one at a time; a `version` column keeps edits made during a send. Threads token refreshes are serialised with a lock file next to `threads_token.json`. | Creates, updates or deletes the remote post for each (note, target) pair; up to 8 attempts |
|
||||
| Memory monitor (`monitoring/memory.py`) | `METRICS_ENABLED` and not testing | None (per-process metrics) | Samples process memory every `METRICS_MEMORY_INTERVAL` seconds |
|
||||
| Syndication worker (`syndication.py`) | At least one crosspost target configured, and not testing | No file lock. Rows in `syndications` are claimed with a 5-minute `claimed_until` lease by an atomic `UPDATE`, so any process may work a row but only one at a time; a `version` column keeps edits made during a send. Threads token refreshes are serialised with a lock file next to `threads_token.json`. | Creates, updates or deletes the remote post for each (note, target) pair; up to 8 attempts. Then, only when `BACKFEED_ENABLED`, backfeed: checks copies for responses, each claimed with a `backfeed_claimed_until` lease (v2.13.0, `backfeed.py`, ADR-073) |
|
||||
|
||||
All workers run inside an app context, are daemon threads (they end with the
|
||||
process) and sleep for their poll interval when there is nothing to do. Under
|
||||
@@ -231,9 +242,12 @@ Content is kept in files and everything else in SQLite, both under
|
||||
- **Media**: `media/YYYY/MM/<uuid>.<ext>` plus `<uuid>_<variant>.<ext>`
|
||||
variants, staged in `media/.tmp/` while being processed.
|
||||
- **Database**: `starpunk.db` (WAL mode): note metadata, tags, media records,
|
||||
sessions, IndieAuth codes and tokens, webmentions and the outgoing queue,
|
||||
syndications, WebSub subscriptions and queues, caches (author h-card, reply
|
||||
sessions, IndieAuth codes and tokens, webmentions (including backfed silo
|
||||
responses, marked by `origin` and `remote_id`) and the outgoing queue,
|
||||
syndications (with backfeed state and silo-only counts), WebSub subscriptions and queues, caches (author h-card, reply
|
||||
contexts), the FTS5 index and `site_state`. See [database.md](database.md).
|
||||
- **Avatars**: `avatars/<sha256>.webp`, saved copies of response authors'
|
||||
avatars (v2.13.0); they can be re-fetched, so losing them is harmless.
|
||||
- **Other files**: `logs/starpunk.log` (rotating), worker lock files,
|
||||
`threads_token.json`, and `debug/` for failed uploads when
|
||||
`DEBUG_SAVE_FAILED_UPLOADS=true`.
|
||||
@@ -285,21 +299,28 @@ their credentials are set. The full list with defaults is in
|
||||
- **Author h-card** (`author_profile`, 24 hours) and **reply contexts**
|
||||
(`reply_contexts`) are cached in the database.
|
||||
- The webmention worker caches endpoint discovery results in memory.
|
||||
- **Response avatars** are saved once as files named by the SHA-256 of the
|
||||
source URL and served from `/avatars/<name>` with
|
||||
`Cache-Control: public, max-age=604800` (one week).
|
||||
|
||||
## Security posture
|
||||
|
||||
- **Outbound requests (SSRF)**: fetches of URLs that come from outside
|
||||
(webmention sources and targets, endpoint discovery, reply contexts, WebSub
|
||||
callbacks, remote Micropub photos, IndieAuth client pages, crosspost link
|
||||
cards) are checked first: every address the host resolves to must be public
|
||||
(private, loopback, link-local, CGNAT, multicast and reserved ranges are
|
||||
refused). The fetcher in `webmention_ssrf.py`, used for webmentions, reply
|
||||
contexts, WebSub and remote Micropub photos, follows redirects by hand and
|
||||
re-checks each hop, connects to the address it checked (no second DNS
|
||||
- **Outbound requests (SSRF)**: every fetch of a URL that comes from
|
||||
outside (webmention sources and targets, endpoint discovery, reply
|
||||
contexts, WebSub callbacks, remote Micropub photos, IndieAuth client pages,
|
||||
crosspost link cards since v2.11.0, response avatars since v2.13.0) goes
|
||||
through the fetcher in `webmention_ssrf.py`. Every address the host
|
||||
resolves to must be public (private, loopback, link-local, CGNAT,
|
||||
multicast and reserved ranges are refused); it follows redirects by hand
|
||||
and re-checks each hop, connects to the address it checked (no second DNS
|
||||
lookup) and limits response size (1 MB by default) and total time.
|
||||
Crosspost link cards use their own check that also re-checks each redirect.
|
||||
The IndieAuth client-page fetch checks only the initial URL. `DEV_MODE`
|
||||
relaxes the checks for local testing.
|
||||
`DEV_MODE` relaxes the checks for local testing. Exceptions: avatars on
|
||||
the site's own `GOTOSOCIAL_URL` host are fetched directly, as the
|
||||
crossposting client does, since behind the same proxy it may resolve to an
|
||||
internal address (v2.13.1); Bluesky CDN avatars are fetched from the
|
||||
account's PDS with `com.atproto.sync.getBlob`, itself through the safe
|
||||
fetcher. Apprise notifications go to the owner-configured `NOTIFY_URLS`,
|
||||
outside the safe fetcher.
|
||||
- **HTML sanitising**: rendered notes (Markdown or HTML sent by a Micropub
|
||||
client) and webmention content are cleaned with nh3 allow-lists; search
|
||||
highlighting escapes everything except `<mark>`.
|
||||
@@ -307,15 +328,17 @@ their credentials are set. The full list with defaults is in
|
||||
(defaults 60 and 10 per 60 seconds); `/websub` 30 requests per IP per
|
||||
minute, plus limits on subscriptions in total, per callback host and per
|
||||
callback address, and on requests waiting for verification. Request-rate
|
||||
counters are in memory and per process.
|
||||
counters live in the `rate_limits` table, shared by all server processes
|
||||
(v2.11.0).
|
||||
- **Client IP behind a proxy**: with `TRUSTED_PROXY=true` the first address in
|
||||
`X-Forwarded-For` is used as the client IP for rate limiting. Set it only
|
||||
when the app is reachable solely through a proxy that sets that header
|
||||
(Caddy does); without it, every client shares the proxy's address.
|
||||
- **Auth**: hashed session and token storage, single-use login state, PKCE
|
||||
for both OIDC and IndieAuth, redirect URIs checked against the client,
|
||||
post-login redirects limited to this site. Admin form POSTs rely on the
|
||||
SameSite=Lax session cookie (there are no per-form CSRF tokens).
|
||||
post-login redirects limited to this site. Admin form POSTs, logout and
|
||||
the IndieAuth consent carry a CSRF token: an HMAC of the session cookie,
|
||||
checked by `starpunk/csrf.py` (v2.11.0), on top of the SameSite=Lax cookie.
|
||||
- **Uploads**: request bodies are capped at 50 MB; images are processed by
|
||||
Pillow and stored under random (UUID) names.
|
||||
- **Container**: runs as a non-root user. HTTPS and security headers (HSTS,
|
||||
@@ -325,14 +348,11 @@ their credentials are set. The full list with defaults is in
|
||||
|
||||
- Logs go to stdout and `logs/starpunk.log` (10 MB × 10 files); each line
|
||||
carries the request's correlation ID.
|
||||
- Metrics (`monitoring/`): HTTP request timings, database query timings
|
||||
(with a slow-query threshold), memory, pool statistics and business
|
||||
counters, in an in-memory buffer per process, shown at
|
||||
`/admin/metrics-dashboard` and `/admin/metrics`.
|
||||
- `/health` reports status, version and webmention-worker statistics;
|
||||
`/health?detailed=true` checks database, filesystem and disk space;
|
||||
`/admin/health` adds metrics, pool statistics and system information.
|
||||
`/admin/health` adds pool statistics and filesystem details. (The metrics
|
||||
dashboard was removed in v2.12.0, see ADR-072.)
|
||||
|
||||
---
|
||||
|
||||
**Last Updated**: 2026-11-03 (v2.10.1)
|
||||
**Last Updated**: 2026-11-10 (v2.14.0)
|
||||
@@ -1,6 +1,6 @@
|
||||
# Authentication and Authorization Architecture
|
||||
|
||||
Current as of v2.10.1. Decision record: [ADR-064](../decisions/ADR-064-oidc-admin-login-and-builtin-indieauth-server.md),
|
||||
Current as of v2.14.0 (no auth changes since v2.11.0). Decision record: [ADR-064](../decisions/ADR-064-oidc-admin-login-and-builtin-indieauth-server.md),
|
||||
which supersedes the "no self-hosted IndieAuth" decisions
|
||||
([ADR-050](../decisions/ADR-050-remove-custom-indieauth-server.md),
|
||||
[ADR-056](../decisions/ADR-056-no-selfhosted-indieauth.md)) and the external
|
||||
@@ -328,14 +328,16 @@ In place:
|
||||
untrusted redirect URIs never followed.
|
||||
- Post-login redirects are limited to this site.
|
||||
|
||||
CSRF (v2.11.0): admin form POSTs, `/auth/logout` and the consent form's
|
||||
approve button need a `csrf_token` field (or `X-CSRF-Token` header) equal to
|
||||
an HMAC-SHA256 of the `starpunk_session` cookie under `SECRET_KEY`; anything
|
||||
else gets 400. The token changes with every login and needs no storage.
|
||||
Endpoints apps call with bearer tokens are not affected. The client metadata
|
||||
fetch uses `webmention_ssrf.safe_get` (every redirect hop checked, the
|
||||
connection pinned, a 5-second limit).
|
||||
|
||||
Known gaps:
|
||||
|
||||
- Admin forms and the consent form have no CSRF token. They rely on the
|
||||
`SameSite=Lax` session cookie, which cross-site POSTs don't carry.
|
||||
- The client metadata fetch checks the `client_id` URL against private
|
||||
addresses, but then makes a plain `httpx.get` that follows redirects.
|
||||
Redirect hops are not re-checked and the connection is not pinned, unlike
|
||||
`webmention_ssrf.safe_get`.
|
||||
- `scopes_supported` leaves out `draft`, although Micropub honours it and
|
||||
the consent screen grants it when requested. Unknown scopes are also shown
|
||||
and granted as requested; they just have no effect.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Syndication (POSSE) Architecture
|
||||
|
||||
Current as of v2.10.1. Code: `starpunk/syndication.py`.
|
||||
Current as of v2.14.0. Code: `starpunk/syndication.py`, `starpunk/backfeed.py`.
|
||||
|
||||
StarPunk publishes on its own site first and then copies each note to other
|
||||
networks (POSSE: Publish on your Own Site, Syndicate Elsewhere). Later
|
||||
@@ -16,7 +16,9 @@ Decisions: [ADR-065](../decisions/ADR-065-direct-syndication-posse.md)
|
||||
(Bluesky), [ADR-068](../decisions/ADR-068-likes-and-reposts.md) (likes and
|
||||
reposts), [ADR-069](../decisions/ADR-069-threads-syndication.md) (Threads),
|
||||
[ADR-070](../decisions/ADR-070-crosspost-links-and-cards.md) (links and link
|
||||
cards), and the [v2.9.0 Micropub review](../design/v2.9.0/2026-11-03-micropub-review.md)
|
||||
cards), [ADR-073](../decisions/ADR-073-backfeed.md) (backfeed),
|
||||
[ADR-074](../decisions/ADR-074-notifications.md) (notifications, including
|
||||
crossposting problems), and the [v2.9.0 Micropub review](../design/v2.9.0/2026-11-03-micropub-review.md)
|
||||
(bookmarks, `mp-syndicate-to` default).
|
||||
|
||||
## Targets
|
||||
@@ -34,7 +36,10 @@ in the process.
|
||||
Display names come from `GOTOSOCIAL_NAME`, `BLUESKY_NAME` and
|
||||
`THREADS_NAME`. GoToSocial statuses use `GOTOSOCIAL_VISIBILITY` (default
|
||||
`public`). The GoToSocial token needs `write:statuses` and `write:media`,
|
||||
plus `read:search` and `write:favourites` for likes and boosts.
|
||||
plus `read:search` and `write:favourites` for likes and boosts, and
|
||||
`read:statuses` and `read:accounts` (or `read`) for backfeed. Backfeed on
|
||||
Threads needs `threads_read_replies` and `threads_manage_insights`. The
|
||||
Threads client uses the API host `graph.threads.com` (v2.13.0).
|
||||
|
||||
Bluesky and Threads can't edit posts through their APIs. Once a copy exists
|
||||
there, edits to the note don't change it, but unpublish and delete still
|
||||
@@ -59,6 +64,9 @@ is sent until the note is published.
|
||||
|
||||
## Data: the `syndications` table (migration 012)
|
||||
|
||||
Migration 020 adds the backfeed columns; see [Backfeed](#backfeed-v2130-adr-073)
|
||||
and [database.md](database.md#crossposting).
|
||||
|
||||
One row per (note, target), unique on `(note_id, target)`:
|
||||
|
||||
| Column | Meaning |
|
||||
@@ -83,6 +91,8 @@ delete and the worker can still remove the copy.
|
||||
| Micropub photo changes | `note_changed(id)` | Same |
|
||||
| `delete_note()` (soft or hard), `undelete_note()` | `note_changed(id)` | Same |
|
||||
| Retry button on the edit page | `retry(id)` | That row set to `pending` now, attempts reset |
|
||||
| Check now (`POST /admin/syndication/<id>/check`) | `backfeed.check_now(id)` | That copy's responses checked at the next worker pass |
|
||||
| Backfill responses (`POST /admin/backfeed/backfill`) | `backfeed.start_backfill()` | Every eligible copy flagged for one check |
|
||||
|
||||
## Worker
|
||||
|
||||
@@ -130,7 +140,8 @@ again with the new content. `_fail()` does the same.
|
||||
errors (codes 10, 190, 200).
|
||||
- Other errors retry after `min(30 s × 2^attempts, 6 h)`. After 8 attempts
|
||||
the row becomes `failed`. The error is shown on the edit page with a
|
||||
Retry button.
|
||||
Retry button. When a row finally fails, the owner also gets a `problems`
|
||||
notification (ADR-074).
|
||||
- `RemoteGone` (404, or Graph error 100/33) during an update means the copy
|
||||
was deleted remotely, so a new one is created. During a delete it counts as
|
||||
success.
|
||||
@@ -223,6 +234,54 @@ Webmention worker rejects a webmention whose source is one of the note's
|
||||
own copies (e.g. backfed by Bridgy), so a copy is never shown as a response
|
||||
to itself (v2.8.0).
|
||||
|
||||
## Backfeed (v2.13.0, ADR-073)
|
||||
|
||||
`starpunk/backfeed.py` brings responses to the copies back to the note.
|
||||
After each crosspost pass, the worker loop calls `backfeed.process_due()`,
|
||||
which does nothing unless `BACKFEED_ENABLED`; otherwise it checks up to three synced copies of published notes (not likes or
|
||||
reposts) that are due, each under a lease (`backfeed_claimed_until`):
|
||||
|
||||
- **Reading.** Each client has `responses(remote_id, remote_url)`, which
|
||||
returns a `Harvest`: GoToSocial `favourited_by`, `reblogged_by` and
|
||||
public direct replies from `context`; Bluesky `getLikes`,
|
||||
`getRepostedBy`, `getQuotes` and `getPostThread` (depth 1), plus
|
||||
`getProfiles` for reply and quote authors' bios; Threads `replies` and
|
||||
`insights` (like, repost and quote counts). Lists are paged up to 1,000
|
||||
items; a list cut short makes the harvest incomplete.
|
||||
- **Storing.** Responses are upserted into `webmentions` with `origin` (the
|
||||
target id) and `remote_id`, `validation_status = 'valid'`, keyed by
|
||||
`(source_url, target_url)`. Likes and reposts use the copy's URL plus
|
||||
`#liked-by-<actor>` / `#reposted-by-<actor>`. A new row gets the
|
||||
`WEBMENTION_AUTO_APPROVE` default; an existing row keeps its moderation.
|
||||
A complete harvest marks that silo's rows for the note that weren't
|
||||
found `invalid` ("No longer on <silo>"). Authors with `#nobridge` or
|
||||
`#nobot` in their bio, or Bluesky's `!no-unauthenticated` label, are not
|
||||
stored and their rows are deleted; since v2.13.3 their likes, reposts and
|
||||
quotes are counted anonymously instead (`hidden_counts()`). If the note's
|
||||
shown responses changed, its webmentions are re-sent (spec 3.1.4).
|
||||
`syndications.likes_count`, `reposts_count` and `quotes_count` hold the
|
||||
silo-only counts: all of Threads; elsewhere people who opted out. New
|
||||
responses found by a backfill, or by a copy's first check on a note older
|
||||
than a day, are stored as already notified (v2.14.0).
|
||||
- **Schedule.** `backfeed_next_at` follows the note's age: 15 minutes (day
|
||||
1), 1 hour (week 1), 1 day (to 30 days), 1 week (to
|
||||
`BACKFEED_MAX_AGE_DAYS`), with up to 10% jitter; none after. Failures back
|
||||
off like crossposts; after 8 they wait a month.
|
||||
- **Backfill.** `start_backfill()` flags every eligible copy
|
||||
(`backfeed_backfill = 1`). Flagged copies are checked only when no
|
||||
routine check is due, at most one every 5 seconds across workers
|
||||
(`site_state.backfeed_backfill_last`). Start and finish times are in
|
||||
`site_state`.
|
||||
- **Refused credentials.** A 401/403 (or Graph auth error) puts the target
|
||||
in `backfeed.denied` until restart; the admin Webmentions page says which
|
||||
scopes are needed, and a `problems` notification is sent. Crossposting
|
||||
is unaffected.
|
||||
|
||||
Avatars of all responses (received and backfed) are saved by
|
||||
`starpunk/avatars.py` and served from `/avatars/<sha256>.webp`; the
|
||||
webmention worker fills in missing ones every 10 minutes and removes unused
|
||||
ones weekly.
|
||||
|
||||
## Client sessions and tokens
|
||||
|
||||
- **Bluesky**: `createSession` with the app password, then
|
||||
@@ -246,10 +305,6 @@ to itself (v2.8.0).
|
||||
|
||||
- Bluesky and Threads copies don't follow edits (platform limitation).
|
||||
Unticking and re-ticking a target makes a new copy.
|
||||
- Link-card and Threads image fetches check every redirect hop against
|
||||
private addresses but don't pin DNS, and use a per-request rather than an
|
||||
overall time limit. Webmention and WebSub fetching use the stricter
|
||||
`webmention_ssrf` fetcher.
|
||||
- No per-target choice for Micropub updates: `mp-syndicate-to` is read only
|
||||
on create.
|
||||
- Likes of ordinary web pages, and all likes on Threads, stay on the site.
|
||||
@@ -1,8 +1,8 @@
|
||||
# StarPunk Technology Stack
|
||||
|
||||
**Version**: 2.10.1 (2026-11-03)
|
||||
**Version**: 2.14.0 (2026-11-10)
|
||||
**Status**: Current. The original v0.x stack write-up is kept as
|
||||
[technology-stack-legacy.md](technology-stack-legacy.md) for history.
|
||||
[technology-stack-legacy.md](../design/v0.x/technology-stack-legacy.md) for history.
|
||||
|
||||
StarPunk deliberately uses few dependencies: a WSGI web framework, SQLite
|
||||
from the standard library, plain files, server-rendered templates and a
|
||||
@@ -14,46 +14,50 @@ the pieces fit together.
|
||||
|
||||
| Component | Choice |
|
||||
|-----------|--------|
|
||||
| Language | Python 3.11 (`python:3.11-slim` base image) |
|
||||
| Web framework | Flask 3.0 (Werkzeug, Jinja2, itsdangerous, click come with it) |
|
||||
| WSGI server | gunicorn 21.2, 4 sync workers (see [deployment.md](deployment.md)) |
|
||||
| Language | Python 3.14 (`python:3.14-slim` base image; since v2.11.0) |
|
||||
| Web framework | Flask 3.1 (Werkzeug, Jinja2, itsdangerous, click come with it) |
|
||||
| WSGI server | gunicorn 26.2, 4 sync workers (see [deployment.md](deployment.md)) |
|
||||
| Database | SQLite (standard-library `sqlite3`), WAL mode, FTS5 for search |
|
||||
| Content storage | Markdown files on disk |
|
||||
| Background work | `threading` daemon threads; `fcntl.flock` file locks to pick one process |
|
||||
| Templates | Jinja2, server-side rendered |
|
||||
| Front end | One hand-written stylesheet; one small vanilla JS file for the editor's Markdown preview |
|
||||
| Front end | One hand-written stylesheet; small inline scripts on admin pages; no JavaScript on public pages |
|
||||
| Packaging | `uv` creates the virtual environment and installs requirements (locally and in the image) |
|
||||
| Container | Multi-stage `Containerfile` (Podman/Docker), non-root user |
|
||||
| Reverse proxy | Caddy in production (examples for Caddy and nginx in the repo) |
|
||||
|
||||
## Python dependencies (`requirements.txt`)
|
||||
|
||||
Pinned to a minor series. Upgraded in v2.11.0, when the set had no known
|
||||
vulnerabilities (`pip-audit`); check again before each release.
|
||||
|
||||
| Package | Pin | Used for |
|
||||
|---------|-----|----------|
|
||||
| Flask | `3.0.*` | Web framework |
|
||||
| gunicorn | `21.2.*` | Production WSGI server |
|
||||
| markdown | `3.5.*` | Rendering notes (`extra`, `codehilite`, `nl2br` and a bare-URL linking extension) |
|
||||
| feedgen | `1.0.*` | RSS 2.0 feed (`starpunk/feeds/rss.py`); Atom and JSON Feed are generated by StarPunk's own code |
|
||||
| httpx | `0.27.*` | All outbound HTTP: OIDC, IndieAuth client discovery, webmention and WebSub requests (through the SSRF-safe wrapper in `webmention_ssrf.py`), crossposting APIs, health check |
|
||||
| python-dotenv | `1.0.*` | Loading `.env` |
|
||||
| beautifulsoup4 | `4.12.*` | HTML parsing: IndieAuth client pages, crosspost text and link cards |
|
||||
| Flask | `3.1.*` | Web framework |
|
||||
| gunicorn | `26.2.*` | Production WSGI server |
|
||||
| markdown | `3.11.*` | Rendering notes (`extra`, `codehilite`, `nl2br` and a bare-URL linking extension) |
|
||||
| nh3 | `0.3.*` | HTML sanitising (rendered notes, webmention content) |
|
||||
| beautifulsoup4 | `4.15.*` | HTML parsing: IndieAuth client pages, crosspost text and link cards |
|
||||
| lxml | `6.1.*` | Outgoing link extraction, webmention endpoint discovery and source checks |
|
||||
| mf2py | `2.0.*` | Parsing microformats2 (webmention sources, reply contexts, author h-card) |
|
||||
| pytest | `8.0.*` | Test runner (listed here as well as in the image's test stage) |
|
||||
| psutil | `5.9.*` | Memory monitor and system metrics |
|
||||
| Pillow | `10.1.*` | Image validation, resizing and variants |
|
||||
| pillow-heif | `0.18.*` | HEIC/HEIF uploads (iPhone photos) |
|
||||
| nh3 | `0.2.*` | HTML sanitising (rendered notes, webmention content) |
|
||||
| PyJWT[crypto] | `2.10.*` | Verifying OIDC ID tokens (JWKS, RS/ES signatures via `cryptography`) |
|
||||
| python-dateutil | `4.0.*` | Dates in reply contexts |
|
||||
| feedgen | `1.0.*` | RSS 2.0 feed (`starpunk/feeds/rss.py`); Atom and JSON Feed are generated by StarPunk's own code |
|
||||
| httpx | `0.28.*` | All outbound HTTP; requests to user-supplied URLs go through `webmention_ssrf.safe_get`/`safe_post` |
|
||||
| apprise | `2.2.*` | Notifications to the owner's services (`starpunk/notifications.py`, ADR-074); sends to the URLs the owner configures, outside the SSRF-safe fetcher |
|
||||
| PyJWT[crypto] | `2.15.*` | Verifying OIDC ID tokens (JWKS, RS/ES signatures via `cryptography`) |
|
||||
| python-dotenv | `1.2.*` | Loading `.env` |
|
||||
| Pillow | `13.0.*` | Image validation, resizing and variants; converting response avatars to 96 px WebP (`avatars.py`) |
|
||||
| pillow-heif | `1.8.*` | HEIC/HEIF uploads (iPhone photos) |
|
||||
|
||||
Packages imported directly but installed only as dependencies of the above:
|
||||
`lxml` (outgoing link extraction, webmention endpoint discovery and source checks; from feedgen) and `python-dateutil` (dates in
|
||||
reply contexts; from feedgen). `markupsafe` comes with Flask/Jinja2.
|
||||
`markupsafe` comes with Flask/Jinja2.
|
||||
|
||||
### Development dependencies (`requirements-dev.txt`)
|
||||
### Test and development dependencies
|
||||
|
||||
Includes `requirements.txt`, plus `pytest-cov`, `pytest-mock`, `black`,
|
||||
`flake8`, `mypy` and `gunicorn`. Coding standards are in
|
||||
[../standards/](../standards/).
|
||||
- `requirements-test.txt`: `pytest` 9.2 and `pytest-mock` 3.15, used by the
|
||||
Containerfile test stage (CI).
|
||||
- `requirements-dev.txt`: runtime and test requirements plus `pytest-cov`,
|
||||
`black`, `flake8`, `mypy` (not enforced) and `pip-audit`. Coding standards
|
||||
are in [../standards/](../standards/).
|
||||
|
||||
### Deliberately not used
|
||||
|
||||
@@ -65,10 +69,10 @@ front-end framework or bundler, no separate search server.
|
||||
|
||||
| Area | Standards |
|
||||
|------|-----------|
|
||||
| IndieWeb | Micropub (create, update, delete, undelete, queries, media endpoint), IndieAuth (authorization server with PKCE, metadata endpoint, token revocation), Webmention (send and receive), WebSub (publisher and built-in hub), microformats2 (h-entry, h-feed, h-card markup and parsing), rel=me |
|
||||
| IndieWeb | Micropub (create, update, delete, undelete, queries, media endpoint), IndieAuth (authorization server with PKCE, metadata endpoint, token revocation), Webmention (send and receive), WebSub (publisher and built-in hub), microformats2 (h-entry, h-feed, h-card markup and parsing), rel=me, web actions (`<indie-action>`), backfeed of responses from the silos |
|
||||
| Login | OpenID Connect authorization code flow with PKCE (provider: Pocket ID) |
|
||||
| Feeds | RSS 2.0, Atom 1.0, JSON Feed 1.1, OPML 2.0, HTTP content negotiation |
|
||||
| Crossposting | Mastodon client API (GoToSocial), AT Protocol (Bluesky XRPC), Threads Graph API |
|
||||
| Crossposting | Mastodon client API (GoToSocial), AT Protocol (Bluesky XRPC), Threads Graph API; the same APIs are read for backfeed |
|
||||
|
||||
Protocol details: [indieweb-protocols.md](indieweb-protocols.md),
|
||||
[simplified-auth-architecture.md](simplified-auth-architecture.md),
|
||||
@@ -76,7 +80,7 @@ Protocol details: [indieweb-protocols.md](indieweb-protocols.md),
|
||||
|
||||
## Testing
|
||||
|
||||
- **Framework**: pytest with pytest-mock; about 1,250 tests in `tests/`
|
||||
- **Framework**: pytest with pytest-mock; about 1,260 tests in `tests/`
|
||||
(`test_*.py` per module or feature, shared fixtures in `tests/conftest.py`,
|
||||
helpers in `tests/helpers/`).
|
||||
- **Isolation**: the `app` fixture calls `create_app()` with `TESTING=True`
|
||||
@@ -89,12 +93,14 @@ Protocol details: [indieweb-protocols.md](indieweb-protocols.md),
|
||||
- **Running locally**:
|
||||
|
||||
```bash
|
||||
uv venv && uv pip install -r requirements-dev.txt
|
||||
.venv/bin/python -m pytest
|
||||
uv venv --python 3.14 && uv pip install -r requirements-dev.txt
|
||||
SESSION_SECRET=test uv run pytest tests/test_<area>.py
|
||||
```
|
||||
|
||||
Locally, run the tests for the area you changed; CI runs the full suite.
|
||||
|
||||
- **In CI**: the `test` stage of the `Containerfile` installs the production
|
||||
requirements plus pytest and pytest-mock and runs the whole suite. The
|
||||
requirements plus `requirements-test.txt` and runs the whole suite. The
|
||||
image is only built and pushed if it passes. Checklist:
|
||||
[../standards/testing-checklist.md](../standards/testing-checklist.md).
|
||||
|
||||
@@ -116,4 +122,4 @@ standards.
|
||||
|
||||
---
|
||||
|
||||
**Last Updated**: 2026-11-03 (v2.10.1)
|
||||
**Last Updated**: 2026-11-10 (v2.14.0)
|
||||
@@ -1,7 +1,9 @@
|
||||
# ADR-053: Performance Monitoring Strategy
|
||||
|
||||
## Status
|
||||
Accepted
|
||||
Superseded by ADR-072 (was: Accepted, v1.1.1)
|
||||
|
||||
> The metrics system was removed in v2.12.0. (status updated 2026-11-03)
|
||||
|
||||
## Context
|
||||
StarPunk v1.1.1 introduces performance monitoring to help operators understand system behavior in production. Currently, we have no visibility into:
|
||||
|
||||
@@ -2,7 +2,9 @@
|
||||
|
||||
## Status
|
||||
|
||||
Accepted (v1.1.1-rc.2, 2025-11-25)
|
||||
Superseded by ADR-072 (was: Accepted, v1.1.1-rc.2, 2025-11-25)
|
||||
|
||||
> The metrics dashboard was removed in v2.12.0. (status updated 2026-11-03)
|
||||
|
||||
> Reconstructed on 2026-11-03 from code, changelog and related documents;
|
||||
> the original was lost. The original
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
# ADR-072: Remove the metrics dashboard
|
||||
|
||||
## Status
|
||||
|
||||
Accepted (v2.12.0, 2026-11-03). Supersedes ADR-053 and ADR-060.
|
||||
|
||||
## Context
|
||||
|
||||
v1.1.1 and v1.1.2 added built-in performance monitoring (ADR-053): HTTP
|
||||
request timing middleware, a wrapper on every database connection that
|
||||
timed queries, a memory-sampling thread, feed-generation counters, an
|
||||
in-memory buffer of the results, and an HTML dashboard at
|
||||
`/admin/metrics-dashboard` with a JSON twin at `/admin/metrics` (ADR-060
|
||||
covered adapting the data for the dashboard template).
|
||||
|
||||
In practice it told us little:
|
||||
|
||||
- The buffer lives in memory in each of the four gunicorn processes, so the
|
||||
dashboard showed whichever process served the request, and everything was
|
||||
lost on restart or deploy.
|
||||
- Nothing in it was ever used to find or fix a problem. Logs (with
|
||||
correlation IDs), `/health` and `/admin/health` were.
|
||||
- It cost something on every request and every query, kept a thread per
|
||||
process, and needed `psutil`, about 1,300 lines of code and three test
|
||||
files.
|
||||
|
||||
## Decision
|
||||
|
||||
Remove the dashboard and everything that only fed it: the `starpunk/monitoring`
|
||||
package, the HTTP metrics middleware (including its `X-Request-ID`
|
||||
response header), `MonitoredConnection`, the memory monitor, the
|
||||
feed-generation tracking calls, `/admin/metrics-dashboard`,
|
||||
`/admin/metrics`, the `METRICS_*` settings and the `psutil` dependency.
|
||||
|
||||
Keep:
|
||||
|
||||
- `/health` and `/health?detailed=true`;
|
||||
- `/admin/health`, without its `metrics` section (database, pool
|
||||
statistics, filesystem and disk checks stay);
|
||||
- connection pool statistics (`get_pool_stats`) and the feed cache, which
|
||||
do work of their own;
|
||||
- structured logs with correlation IDs (ADR-054).
|
||||
|
||||
## Consequences
|
||||
|
||||
- Less code and one fewer dependency; no timing work on each request or
|
||||
query; one fewer thread per process.
|
||||
- Slow database queries are no longer logged. If that is ever needed, a
|
||||
query timer can be added at the pool level, or an external tool used.
|
||||
- `METRICS_*` environment variables are ignored if still set.
|
||||
- Bookmarks to `/admin/metrics-dashboard` now get a 404.
|
||||
@@ -0,0 +1,281 @@
|
||||
# ADR-073: Backfeed of responses to crossposts
|
||||
|
||||
## Status
|
||||
|
||||
Accepted (v2.13.0, 2026-11-10; amended v2.13.2, v2.13.3)
|
||||
|
||||
Implementation notes (v2.13.0):
|
||||
|
||||
- Until an avatar has a local copy, pages show the initial and keep the
|
||||
original URL only as `<data class="u-photo">` for microformats parsers,
|
||||
so browsers don't load it.
|
||||
- Bluesky post views carry no bio, so reply and quote authors' profiles
|
||||
are fetched (`getProfiles`, 25 at a time) to honour `#nobridge`.
|
||||
- A Threads token without `threads_manage_insights` stops Threads
|
||||
backfeed as a whole, as for any refused credential.
|
||||
- Backfill checks take a shared slot in `site_state`
|
||||
(`backfeed_backfill_last`), so all gunicorn workers together do at most
|
||||
one every 5 seconds.
|
||||
|
||||
## Context
|
||||
|
||||
StarPunk crossposts notes to GoToSocial, Bluesky and Threads (ADR-065,
|
||||
ADR-066, ADR-069). People like, repost, quote and reply to those copies,
|
||||
and none of that reaches the note. [Backfeed](https://indieweb.org/backfeed)
|
||||
brings those responses back to the original post.
|
||||
|
||||
The usual tool is [Bridgy](https://brid.gy/about), which polls the silos and
|
||||
sends webmentions. It is a poor fit here:
|
||||
|
||||
- It supports Mastodon and Bluesky, not Threads.
|
||||
- It only checks the 30 most recent posts, about every 30 minutes, and
|
||||
says nothing when a like is withdrawn or a reply deleted.
|
||||
- Since v2.8.0 StarPunk rejects webmentions from a note's own crossposts
|
||||
(`_is_own_syndicated_copy`, `webmention_worker.py`). Bridgy's like and
|
||||
repost pages most likely carry the crosspost's URL plus a fragment as
|
||||
`u-url`, so they would be rejected. Replies have their own URL and get
|
||||
through (production webmention #1 was a Bridgy reply from Mastodon).
|
||||
- Every Bridgy source is on `brid.gy`, which shares one source-domain rate
|
||||
limit of 10 webmentions a minute (ADR-063, Decision 10).
|
||||
- Every response passes through a third party.
|
||||
|
||||
StarPunk already holds a credential for each silo and knows each copy
|
||||
(`syndications.remote_id`, `remote_url`). What the silos let it read:
|
||||
|
||||
| Silo | Likes | Reposts | Quotes | Replies | Credential change |
|
||||
|---|---|---|---|---|---|
|
||||
| Bluesky | `getLikes` | `getRepostedBy` | `getQuotes` | `getPostThread` | none (app password) |
|
||||
| GoToSocial | `favourited_by` | `reblogged_by` | not supported | `context` | read scopes |
|
||||
| Threads | count only | count only | count only | `replies` | `threads_read_replies`, `threads_manage_insights` |
|
||||
|
||||
The Threads API never says who liked, reposted or quoted a post; the
|
||||
Insights API gives lifetime counts only.
|
||||
|
||||
The production GoToSocial prunes remote posts older than 90 days, but never
|
||||
a thread that a local account started or took part in (GoToSocial's "Post
|
||||
Caching and Pruning" docs), so replies to StarPunk's copies are kept. On
|
||||
2026-11-10 the production database still had 106 remote replies to the
|
||||
account older than 90 days, back to August 2024.
|
||||
|
||||
## Decision
|
||||
|
||||
### 1. Native polling, no Bridgy
|
||||
|
||||
StarPunk polls each silo itself for responses to its own copies. Bridgy is
|
||||
switched off for the Mastodon and Bluesky accounts once this ships. The
|
||||
own-copy rule stays as it is.
|
||||
|
||||
### 2. Backfed responses are webmentions
|
||||
|
||||
Responses are stored in the `webmentions` table, so moderation, the
|
||||
Responses section, and re-sending webmentions when a note's responses
|
||||
change (spec 3.1.4, `note_responses_changed`) work as they do now. A
|
||||
backfed row:
|
||||
|
||||
- `source_url`: the response's own URL where it has one (a reply or quote
|
||||
post). Likes and reposts have no page, so they get the copy's URL plus
|
||||
`#liked-by-<actor id>` or `#reposted-by-<actor id>`. The
|
||||
`(source_url, target_url)` key makes repeated checks update the same row.
|
||||
- `target_url`: the note's permalink. `target_note_id`: the note.
|
||||
- `origin`: new column. `webmention` for received webmentions (the
|
||||
default), else `gotosocial`, `bluesky` or `threads`.
|
||||
- `remote_id`: new column. The silo's id for the response (status id,
|
||||
AT-URI, Threads media id), or the actor's id for a like or repost.
|
||||
- `mention_type`: `like`, `repost`, `reply` or the new `quote`.
|
||||
- Author name, profile URL and avatar from the silo's account data;
|
||||
`content_snippet` from the reply or quote text, sanitised the same way as
|
||||
received webmentions (Mastodon HTML through nh3, Bluesky and Threads text
|
||||
escaped with links kept); `source_published` from the silo's timestamp.
|
||||
- `validation_status = 'valid'`: StarPunk read it from the silo's API
|
||||
itself, so there's nothing to fetch and verify. Moderation follows
|
||||
`WEBMENTION_AUTO_APPROVE` like any first-time webmention.
|
||||
|
||||
Only replies directly to the copy are taken, not deeper replies in the
|
||||
thread. Responses from the accounts StarPunk posts as are skipped.
|
||||
|
||||
### 3. Schedule
|
||||
|
||||
Each synced copy of a published note is checked on a schedule that slows
|
||||
as the note ages:
|
||||
|
||||
| Note age | Check every |
|
||||
|---|---|
|
||||
| under 1 day | 15 minutes |
|
||||
| 1 to 7 days | 1 hour |
|
||||
| 7 to 30 days | 1 day |
|
||||
| 30 to 90 days | 1 week |
|
||||
| over 90 days | not checked |
|
||||
|
||||
A random jitter of up to 10% spreads the checks. The cutoff is
|
||||
`BACKFEED_MAX_AGE_DAYS` (default 90). Copies of likes and reposts
|
||||
(ADR-068) aren't checked: they have no responses of their own. A note that
|
||||
is unpublished or deleted is not checked, and its rows stay as they are.
|
||||
|
||||
### 4. Withdrawn likes and deleted replies
|
||||
|
||||
A check that read the whole list (every page, under a cap of 1,000 items
|
||||
per list) marks that note's rows from that silo that are no longer there
|
||||
as `validation_status = 'invalid'` with the error "No longer on
|
||||
<silo>". They drop off the page but keep their moderation decision, and
|
||||
come back if they reappear. A check that stopped early (an error, or the
|
||||
cap) removes nothing.
|
||||
|
||||
### 5. Privacy
|
||||
|
||||
- Only public responses: GoToSocial `visibility` `public` (not unlisted,
|
||||
followers-only or direct); on Bluesky, nothing from accounts with the
|
||||
`!no-unauthenticated` label.
|
||||
- People who opt out are skipped: `#nobridge` or `#nobot` in their bio
|
||||
(Mastodon `note`, Bluesky `description`), the same signal Bridgy honours.
|
||||
A row already stored for them is removed at the next check.
|
||||
- Admin **Delete** stops being a real `DELETE` and sets
|
||||
`moderation_status = 'deleted'` (the value exists but is unused), so a
|
||||
later check or re-sent webmention can't bring the response back. This
|
||||
applies to all webmentions.
|
||||
|
||||
### 6. Avatars are cached locally
|
||||
|
||||
Received webmentions hotlink author photos today. That leaks visitors'
|
||||
addresses to other sites, and Threads and Bluesky avatar URLs change or
|
||||
expire. Avatars are fetched once with `safe_get` (1 MB limit), resized to
|
||||
96 px WebP with Pillow and saved as `DATA_PATH/avatars/<sha256 of URL>.webp`,
|
||||
served from `/avatars/`. They're refetched when the URL changes, and files
|
||||
no row uses are removed weekly. Received webmentions use the same cache
|
||||
from their validation step. A failed fetch falls back to the coloured
|
||||
initial.
|
||||
|
||||
### 7. Threads counts
|
||||
|
||||
Threads likes, reposts and quotes are shown as anonymous counts, not as
|
||||
responses. The worker reads
|
||||
`/{media-id}/insights?metric=likes,reposts,quotes` with each Threads check
|
||||
and stores the numbers on the copy's `syndications` row. The note shows a
|
||||
plain line such as "On Threads: 12 likes · 3 reposts" after the
|
||||
Responses, with no `u-like` or `h-cite` markup, because these aren't
|
||||
people. Threads `replies` counts aren't used: the replies themselves are
|
||||
shown. `views` and `shares` are left out (Meta marks them "in
|
||||
development").
|
||||
|
||||
### 8. Backfill
|
||||
|
||||
An admin action, **Backfill responses**, checks every synced copy once,
|
||||
whatever the note's age. Nothing runs by itself on deploy; the owner runs
|
||||
it after the first deploy (and may run it again later). It:
|
||||
|
||||
- marks every synced copy due now, flagged as a backfill;
|
||||
- is worked through by the same worker, at most one copy every 5 seconds,
|
||||
ahead of the routine schedule only when nothing routine is due;
|
||||
- records when it started and finished; the admin page shows this, and
|
||||
the button is disabled while one is running.
|
||||
|
||||
Upserts make it safe to repeat.
|
||||
|
||||
### 9. Worker
|
||||
|
||||
The checks run in the syndication worker's loop (ADR-065) after crossposts,
|
||||
reusing its clients, token refresh (`maintain()`) and the lease pattern:
|
||||
a copy is claimed with `backfeed_claimed_until` so only one gunicorn worker
|
||||
checks it. A failed check backs off like a failed crosspost
|
||||
(`min(30 s × 2^attempts, 6 h)`) without changing the copy's crosspost
|
||||
state. If a silo answers with a permission error, backfeed for that silo
|
||||
stops until restart and the admin page says which credential needs which
|
||||
scope; crossposting is unaffected.
|
||||
|
||||
Per check: Bluesky 4 calls (likes, reposts, quotes, thread), GoToSocial 3,
|
||||
Threads 2 (replies, insights), plus extra pages.
|
||||
|
||||
### 10. Data (migration 020)
|
||||
|
||||
- `webmentions`: rebuilt to add `quote` to the `mention_type` CHECK, with
|
||||
new columns `origin TEXT NOT NULL DEFAULT 'webmention'` and
|
||||
`remote_id TEXT`, and an index on `(target_note_id, origin)`.
|
||||
- `syndications`: new columns `backfeed_next_at`, `backfeed_checked_at`,
|
||||
`backfeed_claimed_until`, `backfeed_attempts`, `backfeed_error`,
|
||||
`backfeed_backfill` (flag), and `likes_count`, `reposts_count`,
|
||||
`quotes_count` (Threads only).
|
||||
- The backfill's start and finish times go in the existing `site_state`
|
||||
key/value table (migration 016): `backfeed_backfill_started`,
|
||||
`backfeed_backfill_finished`.
|
||||
|
||||
### 11. Display and admin
|
||||
|
||||
- Each backfed response shows a small "via Bluesky" (or GoToSocial,
|
||||
Threads) label. Quotes get their own group, rendered like mentions with
|
||||
the quote text, as `u-mention h-cite`.
|
||||
- The note editor shows when responses were last checked, with a
|
||||
**Check now** button for each copy.
|
||||
- The webmentions list can be filtered by origin.
|
||||
- The dashboard shows backfeed errors next to crosspost errors.
|
||||
|
||||
### 12. Configuration and credentials
|
||||
|
||||
- `BACKFEED_ENABLED` (default `false`; production sets `true`) and
|
||||
`BACKFEED_MAX_AGE_DAYS` (default 90).
|
||||
- GoToSocial: a new token with read access added to the current scopes
|
||||
(`read:statuses` and `read:accounts`, or plain `read`; checked against
|
||||
GoToSocial's scope list when the token is issued).
|
||||
- Threads: the Meta app is re-authorised with `threads_read_replies` and
|
||||
`threads_manage_insights` added. The new `THREADS_ACCESS_TOKEN` replaces
|
||||
the saved token file automatically (ADR-069, Decision 2).
|
||||
- Bluesky: no change.
|
||||
- Production tokens go in the Ansible vault (`bin/vault-run`).
|
||||
|
||||
While at it, the Threads client moves from `graph.threads.net` to
|
||||
`graph.threads.com`, the host Meta's current documentation uses.
|
||||
|
||||
## Amendments (v2.13.2, 2026-11-10)
|
||||
|
||||
- **Display.** Likes, reposts and bookmarks are a line of counts under the
|
||||
note instead of facepiles in the Responses section. Hovering or focusing
|
||||
a count lists the people (as `u-like` / `u-repost` / `u-bookmark`
|
||||
h-cites) and the silo each came from. Threads' anonymous counts are added
|
||||
to the totals ("+N on Threads") instead of a separate line (Decision 7).
|
||||
- **Web actions.** Each like, repost and reply count is an
|
||||
`<indie-action do="…" with="<permalink>">` whose fallback is a link to
|
||||
the same action on each crosspost. No web-action polyfill is loaded;
|
||||
handlers such as browser extensions can use the markup.
|
||||
|
||||
## Amendments (v2.13.3, 2026-11-10)
|
||||
|
||||
- **Opted-out people are counted.** Their likes, reposts and quotes are
|
||||
still not stored or named (Decision 5), but each check records how many
|
||||
there were on the copy's `syndications` row (`likes_count`,
|
||||
`reposts_count`, `quotes_count`, the columns Threads' counts use), and
|
||||
the counts line adds them to the totals as "N on <silo>". Totals then
|
||||
match the silo.
|
||||
|
||||
## Consequences
|
||||
|
||||
- Responses on all three silos show on the note within 15 minutes for new
|
||||
notes, with no third party involved.
|
||||
- Withdrawn likes and deleted replies disappear by themselves for 90 days;
|
||||
after that a note's responses are frozen.
|
||||
- Threads shows replies in full but likes, reposts and quotes only as
|
||||
numbers.
|
||||
- Backfeed stores other people's public posts and avatars. The privacy
|
||||
rules (Decision 5) and moderation are the controls; there is no consent
|
||||
step.
|
||||
- More API calls. At about 50 new notes a month the routine schedule is
|
||||
roughly 5,000 calls a month across the three silos, well inside their
|
||||
limits. The backfill makes one burst of a few calls per copy.
|
||||
- Deleting a webmention in the admin now leaves a tombstone row.
|
||||
- A crosspost removed and posted again gets a new copy; responses to the
|
||||
old copy are marked gone at the next check.
|
||||
|
||||
## Alternatives considered
|
||||
|
||||
- **Bridgy.** No Threads, 30-post window, no withdrawals, a third party in
|
||||
the path, and it needs the own-copy rule and rate limit loosened (see
|
||||
Context). Rejected.
|
||||
- **A separate responses table.** Would duplicate the display, moderation
|
||||
and 3.1.4 re-sending code that `webmentions` already has. Rejected.
|
||||
- **Notification feeds** (Mastodon `/api/v1/notifications`, Bluesky
|
||||
`listNotifications`): one call covers every post, but they only report
|
||||
new events, so withdrawals are never seen and a backfill can't use them.
|
||||
Could be added later as a fast path in front of per-post checks.
|
||||
- **Streaming** (Bluesky Jetstream, Mastodon streaming API): real time, but
|
||||
needs a long-lived connection per gunicorn worker. Not worth it for a
|
||||
personal site.
|
||||
- **Keep the Threads counts out.** They aren't IndieWeb responses, but they
|
||||
are the only signal Threads gives, and showing them anonymously costs one
|
||||
call and identifies no one. Kept.
|
||||
@@ -0,0 +1,155 @@
|
||||
# ADR-074: Notifications through Apprise
|
||||
|
||||
## Status
|
||||
|
||||
Accepted (v2.14.0, 2026-11-10)
|
||||
|
||||
## Context
|
||||
|
||||
Responses now reach a note from two places: received webmentions
|
||||
(ADR-063) and backfeed from the crossposts (ADR-073). The owner only finds
|
||||
out about them by opening the site or the admin. Crossposts that finally
|
||||
fail, and silos that refuse a backfeed credential, are just as invisible.
|
||||
|
||||
Owners use different notification services. The production site uses
|
||||
Gotify (read on iOS through iGotify). Others use ntfy, Pushover, email,
|
||||
Matrix, Discord and so on. [Apprise](https://github.com/caronc/apprise)
|
||||
covers about 100 of them behind one URL syntax (`gotifys://host/token`,
|
||||
`ntfys://topic`, `mailtos://…`), both as a Python library and as a
|
||||
separate HTTP server (apprise-api).
|
||||
|
||||
Options considered:
|
||||
|
||||
1. **Gotify only.** Simplest for this site, no use to anyone else.
|
||||
2. **Post to an apprise-api server.** Standard, but every owner needs one
|
||||
running, and messages take an extra hop.
|
||||
3. **Embed the Apprise library.** Sends straight to each service; an owner
|
||||
who prefers a central apprise-api can still use its `apprise://` /
|
||||
`apprises://` URL.
|
||||
4. **Gotify plus a generic JSON webhook, hand-written.** No dependency, but
|
||||
every other service needs glue the owner has to write.
|
||||
|
||||
## Decision
|
||||
|
||||
### 1. Embed Apprise
|
||||
|
||||
StarPunk depends on the `apprise` package and sends with it directly.
|
||||
`NOTIFY_URLS` holds one or more Apprise URLs (space- or comma-separated);
|
||||
each message goes to all of them. With no URLs, notifications are off and
|
||||
nothing else changes.
|
||||
|
||||
### 2. What can be notified
|
||||
|
||||
`NOTIFY_EVENTS` is a comma-separated list. Default:
|
||||
`reply,mention,quote,problems`.
|
||||
|
||||
| Event | Sent when |
|
||||
|---|---|
|
||||
| `reply`, `mention`, `quote`, `like`, `repost`, `bookmark` | a response of that type is first shown on a note, or first awaits moderation (received webmention or backfeed alike) |
|
||||
| `problems` | a crosspost fails for good (permanent error or last attempt), or a silo refuses the backfeed credential (ADR-073 Decision 9) |
|
||||
|
||||
### 3. Messages
|
||||
|
||||
Markdown, so services that render it (Gotify, iGotify, ntfy, Matrix…)
|
||||
show links; others get the text.
|
||||
|
||||
- **Replies, mentions, quotes:** one message each. Title "↩ Reply from Ana
|
||||
Lima via Bluesky" (or "💬 Mention from …", "❝ Quote from …"); body the
|
||||
response text (plain, up to 300 characters), then a link to the note's
|
||||
responses (`<permalink>#responses`).
|
||||
- **Likes, reposts, bookmarks:** grouped per note and type in each pass:
|
||||
"♥ 3 new likes on 'Starpunk now supports…'", with up to five names and a
|
||||
link to the note.
|
||||
- **Awaiting moderation:** when `WEBMENTION_AUTO_APPROVE` is off, the title
|
||||
says so and the link goes to the admin webmention page instead.
|
||||
- **Problems:** "⚠ Crosspost to Bluesky failed" / "⚠ Backfeed for Threads
|
||||
stopped" with the error and a link to the note's edit page or the
|
||||
Webmentions page.
|
||||
|
||||
Apprise's Gotify plugin has no click-through link, so the link is in the
|
||||
text.
|
||||
|
||||
### 3a. Priority
|
||||
|
||||
Every message is sent with the same priority, `NOTIFY_PRIORITY` (default
|
||||
`normal`). Apprise takes priority as a `priority` argument in each service
|
||||
URL, with values that differ by service, so StarPunk adds
|
||||
`priority=<NOTIFY_PRIORITY>` to each URL in `NOTIFY_URLS` that doesn't set
|
||||
one itself; a URL can still set its own. Apprise's names (`low`,
|
||||
`moderate`, `normal`, `high`, `emergency`) work for most services; Gotify
|
||||
also takes 0–10 (it maps the names to 0, 3, 5, 8, 10). Different
|
||||
priorities per event can come later.
|
||||
|
||||
### 4. Delivery
|
||||
|
||||
- New column `webmentions.notified_at`. A sweep finds responses with
|
||||
`notified_at IS NULL`, `validation_status = 'valid'`, a moderation status
|
||||
of `approved` or `pending`, and a type in `NOTIFY_EVENTS`, sends them,
|
||||
and sets `notified_at`. Types not selected are marked without sending.
|
||||
- The sweep runs in the webmention worker loop, which already runs in one
|
||||
process at a time (its file lock), so nothing is sent twice by several
|
||||
gunicorn workers. It handles received and backfed responses alike.
|
||||
- A failed send leaves `notified_at` empty, so the next pass (about 10
|
||||
seconds later) retries, with backoff up to 15 minutes. Responses older
|
||||
than 24 hours that still haven't gone out are marked and dropped, with a
|
||||
warning, so an outage doesn't end in a flood. The last error is shown on
|
||||
the admin Webmentions page.
|
||||
- Problems are sent at the moment they happen (from the syndication and
|
||||
backfeed workers) and not retried: they are also in the log and admin.
|
||||
- Sending never blocks or breaks validation, crossposting or backfeed.
|
||||
|
||||
### 5. No floods from history
|
||||
|
||||
- Migration 021 sets `notified_at` on every existing response.
|
||||
- Responses stored by a backfill check (ADR-073 Decision 8) are stored with
|
||||
`notified_at` set.
|
||||
- So are responses found on a crosspost's first check when its note is more
|
||||
than a day old (copies linked after the fact).
|
||||
- Grouping (Decision 3) keeps a busy note to one like message per pass.
|
||||
|
||||
### 6. Admin
|
||||
|
||||
- **Send test notification** on the admin Webmentions page sends a test
|
||||
message to every URL and reports each result.
|
||||
- The page shows whether notifications are on, which events are selected,
|
||||
and the last send error.
|
||||
|
||||
### 7. Configuration
|
||||
|
||||
| Setting | Default | Meaning |
|
||||
|---|---|---|
|
||||
| `NOTIFY_URLS` | empty (off) | Apprise URLs, space- or comma-separated |
|
||||
| `NOTIFY_EVENTS` | `reply,mention,quote,problems` | which events are sent |
|
||||
| `NOTIFY_PRIORITY` | `normal` | priority of every message, added to URLs that don't set one |
|
||||
|
||||
Settings stay in the environment, like the rest of StarPunk's (ADR-052).
|
||||
URLs contain tokens and belong with the other secrets.
|
||||
|
||||
## Consequences
|
||||
|
||||
- One new dependency, `apprise`, with its own (requests, PyYAML, click,
|
||||
Markdown, certifi). It must install on Python 3.14 in the container
|
||||
image; this is checked before the release.
|
||||
- Owners can send to almost any service without writing glue, directly or
|
||||
through apprise-api.
|
||||
- Each response is notified at most once, a few seconds after it appears,
|
||||
and only if its type is selected.
|
||||
- Apprise opens outbound connections to whatever URLs the owner configures,
|
||||
outside the SSRF-safe fetcher. That is intended: the owner chooses these
|
||||
URLs, as with the crosspost targets. A service behind the same reverse
|
||||
proxy as StarPunk may need the deployment's internal-address entry
|
||||
(`extra_hosts`), as GoToSocial does.
|
||||
- Notification text includes other people's names and words, sent to the
|
||||
owner's own services only.
|
||||
|
||||
## Alternatives considered
|
||||
|
||||
- **Gotify only / hand-written webhook.** See Context, options 1 and 4: no
|
||||
dependency, but each service beyond Gotify needs glue the owner writes.
|
||||
- **apprise-api only.** See option 2. Still possible through an
|
||||
`apprises://` URL.
|
||||
- **Sending from where responses are stored** (webmention worker,
|
||||
backfeed) instead of a sweep. Two code paths, no retry, and backfeed
|
||||
runs in every gunicorn worker. Rejected.
|
||||
- **Choosing events in the admin UI.** Possible later; the environment is
|
||||
where StarPunk's settings live today.
|
||||
+10
-7
@@ -5,8 +5,8 @@ records one decision: its context, what was decided, and the consequences.
|
||||
Superseded ADRs are kept as history; their Status line names the ADR that
|
||||
replaced them.
|
||||
|
||||
**Total ADR files**: 67 (66 numbers: ADR-001 to ADR-044 and ADR-050 to
|
||||
ADR-071, with two files numbered ADR-012). See [Numbering](#numbering).
|
||||
**Total ADR files**: 70 (69 numbers: ADR-001 to ADR-044 and ADR-050 to
|
||||
ADR-074, with two files numbered ADR-012). See [Numbering](#numbering).
|
||||
|
||||
Status values: **Accepted** (in force), **Superseded** (replaced by a later
|
||||
ADR), **Deprecated** (withdrawn, nothing replaces it; none at present),
|
||||
@@ -85,6 +85,7 @@ See also [IndieWeb protocols](../architecture/indieweb-protocols.md).
|
||||
| [066](ADR-066-bluesky-syndication.md) | Crossposting to Bluesky | Accepted | Bluesky as a second crosspost target (v2.2.0). |
|
||||
| [069](ADR-069-threads-syndication.md) | Crossposting to Threads | Accepted | Threads as a third target, with automatic token refresh (v2.6.0). |
|
||||
| [070](ADR-070-crosspost-links-and-cards.md) | Links and link cards in crossposts | Accepted | Keep note links in crossposts and build link cards for the linked page (v2.6.3). |
|
||||
| [073](ADR-073-backfeed.md) | Backfeed of responses to crossposts | Accepted | Poll GoToSocial, Bluesky and Threads for responses to crossposts and store them as webmentions; admin backfill (v2.13.0, amended v2.13.2–v2.13.3). |
|
||||
|
||||
### Authentication (current)
|
||||
|
||||
@@ -101,10 +102,10 @@ See also [IndieWeb protocols](../architecture/indieweb-protocols.md).
|
||||
|---|---|---|---|
|
||||
| [012](ADR-012-http-error-handling-policy.md) | HTTP Error Handling Policy | Accepted | Correct HTTP status codes (e.g. 404 for missing notes) across routes. |
|
||||
| [052](ADR-052-configuration-system-architecture.md) | Configuration System Architecture | Accepted | Environment variables with defaults, loaded in one module. |
|
||||
| [053](ADR-053-performance-monitoring-strategy.md) | Performance Monitoring Strategy | Accepted | Built-in, standard-library performance metrics (v1.1.1). |
|
||||
| [054](ADR-054-structured-logging-architecture.md) | Structured Logging Architecture | Accepted | Rotating log files with correlation IDs (v1.1.1). |
|
||||
| [055](ADR-055-error-handling-philosophy.md) | Error Handling Philosophy | Accepted | Layered handling: graceful degradation, clear user messages, detailed logs. |
|
||||
| [060](ADR-060-production-hotfix-metrics-dashboard.md) | Route Adapter for the Metrics Dashboard | Accepted | Adapt monitoring data to the dashboard template in the route (v1.1.1-rc.2). *Reconstructed.* |
|
||||
| [072](ADR-072-remove-metrics-dashboard.md) | Remove the metrics dashboard | Accepted | Drop the in-memory metrics system and dashboard; keep `/health`, `/admin/health` and logs (v2.12.0). |
|
||||
| [074](ADR-074-notifications.md) | Notifications through Apprise | Accepted | Notify the owner of new responses and crosspost/backfeed problems via the embedded Apprise library (v2.14.0). |
|
||||
|
||||
See also [deployment](../architecture/deployment.md).
|
||||
|
||||
@@ -117,6 +118,8 @@ designs of v0.x–v1.6.0, all replaced by ADR-064 in v2.0.0.
|
||||
|---|---|---|---|
|
||||
| [005](ADR-005-indielogin-authentication.md) | IndieLogin Authentication Integration | Superseded by 064 | Admin login through indielogin.com. |
|
||||
| [007](ADR-007-slug-generation-algorithm.md) | Slug Generation Algorithm | Superseded by 062 | Slugs from the first words of the note. |
|
||||
| [053](ADR-053-performance-monitoring-strategy.md) | Performance Monitoring Strategy | Superseded by 072 | Built-in, standard-library performance metrics (v1.1.1). |
|
||||
| [060](ADR-060-production-hotfix-metrics-dashboard.md) | Route Adapter for the Metrics Dashboard | Superseded by 072 | Adapt monitoring data to the dashboard template in the route (v1.1.1-rc.2). *Reconstructed.* |
|
||||
| [016](ADR-016-indieauth-client-discovery.md) | IndieAuth Client Discovery Mechanism | Superseded by 019 | h-app markup for client discovery. |
|
||||
| [017](ADR-017-oauth-client-metadata-document.md) | OAuth Client ID Metadata Document Implementation | Superseded by 019 | OAuth client metadata endpoint. |
|
||||
| [018](ADR-018-indieauth-detailed-logging.md) | IndieAuth Detailed Logging Strategy | Superseded by 064 | Redacted HTTP logging of the IndieAuth login flow. |
|
||||
@@ -166,7 +169,7 @@ was carried out and is not itself superseded.
|
||||
|
||||
## How to create a new ADR
|
||||
|
||||
1. Use the next number after the highest existing one (currently ADR-072).
|
||||
1. Use the next number after the highest existing one (currently ADR-075).
|
||||
2. Name it `ADR-NNN-brief-descriptive-title.md`.
|
||||
3. Follow the format of recent ADRs (e.g. ADR-070, ADR-071):
|
||||
|
||||
@@ -199,5 +202,5 @@ was carried out and is not itself superseded.
|
||||
|
||||
---
|
||||
|
||||
**Last Updated**: 2026-11-03
|
||||
**Total ADR files**: 67
|
||||
**Last Updated**: 2026-11-10
|
||||
**Total ADR files**: 70
|
||||
+53
-3
@@ -21,7 +21,13 @@ is in `../architecture/`; decisions are in `../decisions/` (ADRs).
|
||||
|
||||
## Contents
|
||||
|
||||
[v1.0.0](#v100) · [v1.1.1](#v111) · [v1.1.2](#v112) · [v1.2.0](#v120) · [v1.3.0](#v130) · [v1.3.1](#v131) · [v1.4.0](#v140) · [v1.4.1](#v141) · [v1.4.2](#v142) · [v1.5.0](#v150) · [v1.6.0](#v160) · [v2.0.0](#v200) · [v2.0.1](#v201) · [v2.0.2](#v202) · [v2.1.0](#v210) · [v2.2.0](#v220) · [v2.3.0](#v230) · [v2.3.1](#v231) · [v2.3.2](#v232) · [v2.3.3](#v233) · [v2.4.0](#v240) · [v2.5.0](#v250) · [v2.6.0](#v260) · [v2.6.1](#v261) · [v2.6.2](#v262) · [v2.6.3](#v263) · [v2.6.4](#v264) · [v2.6.5](#v265) · [v2.6.6](#v266) · [v2.6.7](#v267) · [v2.6.8](#v268) · [v2.6.9](#v269) · [v2.7.0](#v270) · [v2.7.1](#v271) · [v2.7.2](#v272) · [v2.8.0](#v280) · [v2.8.1](#v281) · [v2.9.0](#v290) · [v2.10.0](#v2100) · [v2.10.1](#v2101) · [v2.10.2](#v2102) · [hotfix](#hotfix)
|
||||
[v0.x](#v0x) · [v1.0.0](#v100) · [v1.1.1](#v111) · [v1.1.2](#v112) · [v1.2.0](#v120) · [v1.3.0](#v130) · [v1.3.1](#v131) · [v1.4.0](#v140) · [v1.4.1](#v141) · [v1.4.2](#v142) · [v1.5.0](#v150) · [v1.6.0](#v160) · [v2.0.0](#v200) · [v2.0.1](#v201) · [v2.0.2](#v202) · [v2.1.0](#v210) · [v2.2.0](#v220) · [v2.3.0](#v230) · [v2.3.1](#v231) · [v2.3.2](#v232) · [v2.3.3](#v233) · [v2.4.0](#v240) · [v2.5.0](#v250) · [v2.6.0](#v260) · [v2.6.1](#v261) · [v2.6.2](#v262) · [v2.6.3](#v263) · [v2.6.4](#v264) · [v2.6.5](#v265) · [v2.6.6](#v266) · [v2.6.7](#v267) · [v2.6.8](#v268) · [v2.6.9](#v269) · [v2.7.0](#v270) · [v2.7.1](#v271) · [v2.7.2](#v272) · [v2.8.0](#v280) · [v2.8.1](#v281) · [v2.9.0](#v290) · [v2.10.0](#v2100) · [v2.10.1](#v2101) · [v2.10.2](#v2102) · [v2.11.0](#v2110) · [v2.12.0](#v2120) · [v2.12.1](#v2121) · [v2.13.0](#v2130) · [v2.13.1](#v2131) · [v2.13.2](#v2132) · [v2.13.3](#v2133) · [v2.13.4](#v2134) · [v2.14.0](#v2140) · [v2.14.1](#v2141) · [v2.14.2](#v2142) · [hotfix](#hotfix)
|
||||
|
||||
## v0.x
|
||||
|
||||
Not a release folder: documents from before v1.0.0 kept for history.
|
||||
|
||||
- [technology-stack-legacy.md](v0.x/technology-stack-legacy.md) — StarPunk Technology Stack - Quick Reference
|
||||
|
||||
## v1.0.0
|
||||
|
||||
@@ -408,7 +414,51 @@ OIDC admin login and built-in IndieAuth server.
|
||||
|
||||
## v2.10.2
|
||||
|
||||
- [2026-11-03-reply-context-fetch.md](v2.10.2/2026-11-03-reply-context-fetch.md) - Reply contexts fetched again (fetch_source returns three values since v2.7.0)
|
||||
- [2026-11-03-reply-context-fetch.md](v2.10.2/2026-11-03-reply-context-fetch.md) — Reply context fetch
|
||||
|
||||
## v2.11.0
|
||||
|
||||
- [2026-11-03-maintenance-release.md](v2.11.0/2026-11-03-maintenance-release.md) — Maintenance release
|
||||
|
||||
## v2.12.0
|
||||
|
||||
- [2026-11-03-remove-metrics.md](v2.12.0/2026-11-03-remove-metrics.md) — Remove the metrics dashboard
|
||||
|
||||
## v2.12.1
|
||||
|
||||
- [2026-11-10-category-unused-tags.md](v2.12.1/2026-11-10-category-unused-tags.md) — q=category lists only tags in use
|
||||
|
||||
## v2.13.0
|
||||
|
||||
- [2026-11-10-backfeed.md](v2.13.0/2026-11-10-backfeed.md) — Backfeed (implementation report)
|
||||
|
||||
## v2.13.1
|
||||
|
||||
- [2026-11-10-avatar-fetch.md](v2.13.1/2026-11-10-avatar-fetch.md) — Avatar fetching in production
|
||||
|
||||
## v2.13.2
|
||||
|
||||
- [2026-11-10-reaction-counts.md](v2.13.2/2026-11-10-reaction-counts.md) — Reaction counts and web actions
|
||||
|
||||
## v2.13.3
|
||||
|
||||
- [2026-11-10-count-opted-out.md](v2.13.3/2026-11-10-count-opted-out.md) — Count opted-out people without naming them
|
||||
|
||||
## v2.13.4
|
||||
|
||||
- [2026-11-10-list-counts.md](v2.13.4/2026-11-10-list-counts.md) — Counts on lists of notes, reply cards as links
|
||||
|
||||
## v2.14.0
|
||||
|
||||
- [2026-11-10-notifications.md](v2.14.0/2026-11-10-notifications.md) — Notifications (implementation report)
|
||||
|
||||
## v2.14.1
|
||||
|
||||
- [2026-11-10-tags-and-captions.md](v2.14.1/2026-11-10-tags-and-captions.md) — Clearing tags in the editor; captions escaped in feeds
|
||||
|
||||
## v2.14.2
|
||||
|
||||
- [2026-11-10-quotes-cleanup.md](v2.14.2/2026-11-10-quotes-cleanup.md) — Quote counts; dead files removed
|
||||
|
||||
## hotfix
|
||||
|
||||
@@ -429,4 +479,4 @@ IndieAuth PKCE endpoint discovery hotfix (2025-12-17, between v1.5.0 and v1.6.0;
|
||||
|
||||
---
|
||||
|
||||
**Last Updated**: 2026-11-03
|
||||
**Last Updated**: 2026-11-10
|
||||
+1
-1
@@ -3,7 +3,7 @@
|
||||
> **Historical document.** This describes StarPunk's original v0.x technology
|
||||
> choices (for example IndieLogin.com for admin login) and is kept for
|
||||
> context only. It does not describe the current system; see
|
||||
> [technology-stack.md](technology-stack.md) and [overview.md](overview.md).
|
||||
> [technology-stack.md](../../architecture/technology-stack.md) and [overview.md](../../architecture/overview.md).
|
||||
|
||||
## Project Understanding
|
||||
|
||||
@@ -1901,7 +1901,7 @@ class Webmention:
|
||||
## References
|
||||
|
||||
- [ADR-004: File-Based Note Storage](../../decisions/ADR-004-file-based-note-storage.md)
|
||||
- [Database Schema](../../../starpunk/monitoring/database.py)
|
||||
- [Database Schema](../../../starpunk/database/schema.py)
|
||||
- [Phase 1.1: Core Utilities](phase-1.1-core-utilities.md)
|
||||
- [Python Coding Standards](../../standards/python-coding-standards.md)
|
||||
- [Python Dataclasses Documentation](https://docs.python.org/3/library/dataclasses.html)
|
||||
|
||||
@@ -2061,7 +2061,7 @@ Phase 2.1 is complete when:
|
||||
- [ADR-007: Slug Generation Algorithm](../../decisions/ADR-007-slug-generation-algorithm.md)
|
||||
- [Phase 1.1: Core Utilities](phase-1.1-core-utilities.md)
|
||||
- [Phase 1.2: Data Models](phase-1.2-data-models.md)
|
||||
- [Database Schema](../../../starpunk/monitoring/database.py)
|
||||
- [Database Schema](../../../starpunk/database/schema.py)
|
||||
- [Python Coding Standards](../../standards/python-coding-standards.md)
|
||||
- [Architecture Overview](../../architecture/overview.md)
|
||||
- Security Architecture (`security.md`, not preserved)
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
# v2.11.0: Maintenance release
|
||||
|
||||
Dependency upgrades and the five High items in the backlog (all re-checked
|
||||
against the code first and still present).
|
||||
|
||||
## Dependencies
|
||||
|
||||
`pip-audit` on the v2.10.2 set found 97 published vulnerabilities in eight
|
||||
packages (Pillow 64, PyJWT 22, gunicorn 4, Flask 2, Markdown 2,
|
||||
python-dotenv 2, pytest 2, pillow-heif 1). All direct dependencies were moved
|
||||
to their current minor series (`requirements.txt`); the new set has no known
|
||||
vulnerabilities. Python moved from 3.11 to 3.14 (`python:3.14-slim`); the
|
||||
whole suite passes with `-W error::DeprecationWarning`.
|
||||
|
||||
- `requirements-test.txt` (pytest, pytest-mock) is installed by the
|
||||
Containerfile test stage; pytest left the runtime requirements.
|
||||
- `lxml` and `python-dateutil` are pinned (used directly, previously only
|
||||
transitive). `uv` is pinned to 0.9.28 in the Containerfile.
|
||||
- `datetime.utcnow()` (34 calls in `starpunk/`, more in tests) became
|
||||
`datetime.now(timezone.utc).replace(tzinfo=None)`: same naive-UTC values.
|
||||
sqlite3's default datetime adapters, deprecated in 3.12, are registered
|
||||
explicitly in `starpunk/database/__init__.py` with the same output.
|
||||
|
||||
## High backlog items
|
||||
|
||||
1. **Foreign keys.** `PRAGMA foreign_keys=ON` on every pooled connection
|
||||
(`database/pool.py`); all deletes of notes, media and WebSub subscriptions
|
||||
go through the pool. Every foreign key is `CASCADE` or `SET NULL`, so no
|
||||
delete can be blocked. Migration 018 removes orphans (production: three
|
||||
`note_tags` rows from hard-deleted notes). Crosspost rows still have no
|
||||
foreign key on purpose.
|
||||
2. **CSRF.** `starpunk/csrf.py`: token = HMAC-SHA256(SECRET_KEY,
|
||||
"csrf:" + starpunk_session cookie); `before_request` checks the
|
||||
`csrf_token` field or `X-CSRF-Token` header on every admin POST,
|
||||
`auth.logout` and `indieauth.approve`; 400 otherwise. All 12 POST forms
|
||||
carry it. `CSRF_ENABLED` (default true); test apps turn it off unless they
|
||||
set it.
|
||||
3. **Fetchers.** `indieauth.fetch_client_redirect_uris` and
|
||||
`syndication._fetch_public` use `webmention_ssrf.safe_get` (per-hop
|
||||
checks, DNS pinning, hard deadline). `safe_get` gained `truncate` for
|
||||
reads that stop at the size limit.
|
||||
4. **Debug default.** `FLASK_DEBUG` defaults to 0 and `FLASK_ENV` to
|
||||
production; the runtime image sets both.
|
||||
5. **Shared rate limits.** `SharedRateLimiter` keeps sliding-window counters
|
||||
in `rate_limits` (migration 019): current window plus the previous one
|
||||
weighted by overlap. Used by `/webmention` (per IP and per source domain)
|
||||
and `/websub`; old windows are pruned every 200 calls.
|
||||
|
||||
## Housekeeping
|
||||
|
||||
`compose.yaml` no longer pins 0.6.0 or sets `VERSION`/`ENVIRONMENT`; the
|
||||
unused root `dev_auth.py` and the committed `test.ini` were removed.
|
||||
|
||||
## Tests
|
||||
|
||||
New: `tests/test_csrf.py`, `tests/test_foreign_keys.py`,
|
||||
`tests/test_shared_rate_limit.py`, IndieAuth client-metadata tests through the
|
||||
fetcher (including a redirect to a private address). Crosspost tests route
|
||||
link-card fetches through the fetcher's transport. Full suite: 1264 passed
|
||||
on Python 3.14.
|
||||
@@ -0,0 +1,28 @@
|
||||
# v2.12.0: Remove the metrics dashboard
|
||||
|
||||
Decision: [ADR-072](../../decisions/ADR-072-remove-metrics-dashboard.md).
|
||||
|
||||
## Removed
|
||||
|
||||
| What | Where |
|
||||
|---|---|
|
||||
| Metrics buffer, HTTP middleware, database timing wrapper, memory monitor, business metrics | `starpunk/monitoring/` (deleted) |
|
||||
| HTTP middleware and memory monitor start-up | `starpunk/__init__.py` |
|
||||
| `MonitoredConnection` wrapping, `slow_query_threshold` and `metrics_enabled` arguments | `starpunk/database/pool.py` |
|
||||
| `track_feed_generated` calls and their timing code | `starpunk/feeds/atom.py`, `json_feed.py`, `rss.py` |
|
||||
| `/admin/metrics-dashboard`, `/admin/metrics`, `transform_metrics_for_template`, the `metrics` block of `/admin/health` | `starpunk/routes/admin.py` |
|
||||
| Dashboard template and the admin nav link | `templates/admin/metrics_dashboard.html`, `templates/admin/base.html` |
|
||||
| `METRICS_ENABLED`, `METRICS_SLOW_QUERY_THRESHOLD`, `METRICS_SAMPLING_RATE`, `METRICS_BUFFER_SIZE`, `METRICS_MEMORY_INTERVAL` | `starpunk/config.py`, `.env.example` |
|
||||
| `psutil` | `requirements.txt` |
|
||||
| Tests | `tests/test_monitoring.py`, `tests/test_monitoring_feed_statistics.py`, `tests/test_admin_feed_statistics.py` |
|
||||
|
||||
## Kept
|
||||
|
||||
`/health`, `/health?detailed=true`, `/admin/health` (database, pool,
|
||||
filesystem, disk), pool statistics, the feed cache and its own hit/miss
|
||||
counters, and correlation IDs in logs.
|
||||
|
||||
## Upgrade
|
||||
|
||||
No migration. `METRICS_*` variables can be deleted from the environment;
|
||||
they are ignored if left.
|
||||
@@ -0,0 +1,28 @@
|
||||
# v2.12.1: q=category lists only tags in use
|
||||
|
||||
## Problem
|
||||
|
||||
`q=category` (added in v2.4.0) is what Micropub clients use for tag
|
||||
autocomplete. It read every row of `tags` with a `LEFT JOIN` on
|
||||
`note_tags`, so it also returned:
|
||||
|
||||
- tags no post uses any more (a `tags` row stays after an edit removes
|
||||
the tag from its last post), and
|
||||
- tags only used on soft-deleted posts (`note_tags` rows stay so that
|
||||
undelete can restore them).
|
||||
|
||||
Clients then offered tags that appear nowhere on the site.
|
||||
|
||||
## Fix
|
||||
|
||||
`_all_categories()` in `starpunk/micropub.py` inner-joins `note_tags` and
|
||||
`notes` with `deleted_at IS NULL`. Order is unchanged: most used first,
|
||||
then by name. Drafts count, since they are the owner's posts. Unused
|
||||
`tags` rows are left in place; nothing else reads them as a list.
|
||||
|
||||
## Verification
|
||||
|
||||
`tests/test_micropub_v240.py::TestQueries::test_category_skips_unused_tags`
|
||||
retags one post and deletes another, checks only the live tag is listed,
|
||||
then undeletes and checks the deleted post's tag returns. It fails on
|
||||
v2.12.0.
|
||||
@@ -0,0 +1,53 @@
|
||||
# v2.13.0: Backfeed (implementation report)
|
||||
|
||||
Implements ADR-073.
|
||||
|
||||
## What changed
|
||||
|
||||
- `migrations/020_backfeed.sql`: `webmentions` rebuilt with `origin`,
|
||||
`remote_id` and the `quote` type (rows copied, indexes recreated, plus
|
||||
`(target_note_id, origin)`); `syndications` gains the `backfeed_*`
|
||||
columns and Threads counts.
|
||||
- `starpunk/backfeed.py` (new): `Response`/`Harvest`, schedule, lease,
|
||||
storing and pruning, opt-outs, backfill, `check_now`, `status`.
|
||||
- `starpunk/syndication.py`: `responses()` on `MastodonClient`,
|
||||
`BlueskyClient` and `ThreadsClient`; HTTP status kept on errors
|
||||
(`exc.status`) so refused credentials can be told apart; Mastodon `_call`
|
||||
accepts absolute URLs (pagination links); the worker loop runs backfeed
|
||||
after crossposts; Threads host is `graph.threads.com`.
|
||||
- `starpunk/avatars.py` (new) and `/avatars/<name>`: local avatar copies
|
||||
for every response; `local_avatar` template filter.
|
||||
- Webmentions: Quotes group and "via" label; admin delete is a tombstone
|
||||
(`moderation_status = 'deleted'`, hidden from the list unless filtered);
|
||||
origin filter; a webmention can't reset a backfed row, and one whose
|
||||
source is an already backfed response is rejected ("Already backfed").
|
||||
- Admin: Backfeed panel with **Backfill responses** on the Webmentions
|
||||
page; **Check now** and last-checked time per crosspost on the edit page.
|
||||
- Settings `BACKFEED_ENABLED` (default false), `BACKFEED_MAX_AGE_DAYS` (90).
|
||||
|
||||
## Verification
|
||||
|
||||
- `tests/test_backfeed.py` (22 tests): schedule; text/facet HTML; storing,
|
||||
updating and moderation kept; pruning only on complete harvests and
|
||||
restoring; opt-outs; tombstones; backfill and its pacing; likes/reposts
|
||||
and drafts skipped; refused credentials; backoff; note page (via label,
|
||||
quotes, Threads counts, local avatars); avatar save/serve/cleanup;
|
||||
webmention can't overwrite a backfed row; the three clients against fake
|
||||
APIs; migration 020 on a pre-020 database keeps rows and foreign keys;
|
||||
the admin Backfeed panel, backfill and Check now; Threads avatar URLs
|
||||
(signed, changing per request) aren't refetched while a copy exists.
|
||||
- Tests for the touched areas (webmentions, syndication, microformats,
|
||||
migrations, templates, routes): all pass. The full suite runs in CI.
|
||||
- `test_microformats_v280` expects response authors' `u-photo`: kept via
|
||||
`<data class="u-photo">` until the avatar is saved.
|
||||
|
||||
## Deploying
|
||||
|
||||
1. Issue a new GoToSocial token with `read:statuses read:accounts` added
|
||||
to the current scopes, and re-authorise the Threads app with
|
||||
`threads_read_replies` and `threads_manage_insights`. Put both in the
|
||||
Ansible vault.
|
||||
2. Set `BACKFEED_ENABLED=true` in the StarPunk environment and deploy
|
||||
(migration 020 runs at start: back up the database first).
|
||||
3. Run **Backfill responses** once from Admin → Webmentions. Turn Bridgy
|
||||
off for the Mastodon and Bluesky accounts.
|
||||
@@ -0,0 +1,33 @@
|
||||
# v2.13.1: Avatar fetching in production
|
||||
|
||||
## Problem
|
||||
|
||||
After deploying v2.13.0, backfeed stored responses but no avatar was saved:
|
||||
|
||||
- GoToSocial avatars (`https://social.../fileserver/...`):
|
||||
`SSRFBlockedError: Blocked request to private IP: 172.20.0.5`. Inside the
|
||||
container the GoToSocial host resolves to the reverse proxy's internal
|
||||
address (the deployment's hairpin `extra_hosts` entry).
|
||||
- Bluesky avatars (`https://cdn.bsky.app/img/avatar/...`): HTTP 403 from
|
||||
BunnyCDN. From the server, the CDN answers 403 over IPv4 and 200 over
|
||||
IPv6; the container has no IPv6 route. From elsewhere IPv4 works.
|
||||
|
||||
## Fix (`starpunk/avatars.py`, `_download`)
|
||||
|
||||
- The host of `GOTOSOCIAL_URL` is fetched with a plain HTTP client (no
|
||||
redirects, 1 MB limit), as `MastodonClient` already talks to it. Every
|
||||
other host still goes through `safe_get`, so private addresses stay
|
||||
blocked.
|
||||
- `cdn.bsky.app/img/<kind>/plain/<did>/<cid>` URLs are fetched from the
|
||||
account's PDS with `com.atproto.sync.getBlob`; the PDS comes from the DID
|
||||
document (`plc.directory` for `did:plc`, `/.well-known/did.json` for
|
||||
`did:web`). Both requests use `safe_get`. The cache file is still named
|
||||
after the original URL.
|
||||
|
||||
## Verification
|
||||
|
||||
- `tests/test_backfeed.py::test_avatars_from_own_gotosocial_and_bluesky_pds`:
|
||||
own GoToSocial on a private address is fetched, a Bluesky CDN URL is
|
||||
fetched from its PDS, another private address is still refused.
|
||||
- Checked against the real PLC directory and a bsky.network PDS.
|
||||
- From the production server: getBlob over IPv4 returns the image (200).
|
||||
@@ -0,0 +1,32 @@
|
||||
# v2.13.2: Reaction counts and web actions
|
||||
|
||||
## Why
|
||||
|
||||
With backfeed live, notes showed "5 Likes" and "1 Repost" headings over
|
||||
rows of faces (blank until avatars were saved), plus a separate
|
||||
"On Threads: 1 like" line. The owner asked for the usual presentation: a
|
||||
flat count under the post, with who on hover. Web actions
|
||||
(https://indieweb.org/webactions) fit the same line.
|
||||
|
||||
## What changed
|
||||
|
||||
- `templates/partials/responses.html`: `facepile` and `silo_counts` are
|
||||
replaced by `reactions` / `reaction`. Each count is a focusable
|
||||
`.reaction` with an SVG icon; its `.reaction-who` card lists the people
|
||||
(avatar, name, silo) inside a scrollable `.reaction-people`, Threads'
|
||||
anonymous counts, and an `<indie-action>` with links to the crossposts.
|
||||
The reply count has no people list (replies are listed below).
|
||||
- `templates/note.html`: the counts line follows the note's footer; the
|
||||
Responses section shows only when there are replies, quotes or mentions.
|
||||
- `static/css/style.css`: facepile styles replaced by `.reactions`; the card
|
||||
opens on `:hover` and `:focus-within` (tap on touch screens).
|
||||
- `starpunk/avatars.py`: a redirect from the own GoToSocial server (media it
|
||||
hasn't cached) is followed through `safe_get`.
|
||||
|
||||
## Verification
|
||||
|
||||
- `tests/test_backfeed.py` (24 tests): totals include Threads' anonymous
|
||||
count, `<indie-action>` and crosspost links present; redirected avatar
|
||||
saved. Microformats, template and route tests pass.
|
||||
- Rendered locally in headless Chromium in dark and light themes with the
|
||||
like and reply cards open.
|
||||
@@ -0,0 +1,22 @@
|
||||
# v2.13.3: Count opted-out people without naming them
|
||||
|
||||
## Why
|
||||
|
||||
A note's Bluesky copy had 1 like on Bluesky and 0 on the note. The liker's
|
||||
account has the `!no-unauthenticated` label, so backfeed skipped them
|
||||
(ADR-073 Decision 5). Every other Bluesky copy's like and repost counts
|
||||
matched what StarPunk had stored.
|
||||
|
||||
## Change
|
||||
|
||||
`backfeed.hidden_counts(harvest)`: the silo's own counts when it gives no
|
||||
names (Threads), else the number of opted-out likes, reposts and quotes in
|
||||
the harvest. `check()` stores it on the copy's `syndications` row, and the
|
||||
counts line (v2.13.2) already adds those columns to the totals as
|
||||
"N on <silo>". Opted-out people are still not stored.
|
||||
|
||||
## Verification
|
||||
|
||||
`tests/test_backfeed.py::test_people_who_opt_out_are_counted_not_named`:
|
||||
label, `#nobot` and `#nobridge` opt-outs counted (2 likes, 1 repost), not
|
||||
stored, not named on the page; totals include them.
|
||||
@@ -0,0 +1,24 @@
|
||||
# v2.13.4: Counts on lists of notes, reply cards as links
|
||||
|
||||
## What changed
|
||||
|
||||
- `webmention_queries.response_counts(note_ids)`: likes, reposts and
|
||||
replies per note in two grouped queries (shown webmentions, plus the
|
||||
anonymous counts on synced `syndications` rows). `routes/public.py`
|
||||
`attach_response_counts()` runs after `attach_contexts()` on the home,
|
||||
tag and Likes pages.
|
||||
- `partials/responses.html` `list_counts`: icons and numbers in the note's
|
||||
meta line, linking to `<permalink>#responses` (an anchor on the note page).
|
||||
- Reply cards: the date link (already the reply's own URL) stretches over
|
||||
the card with `::after`; author and content links sit above it.
|
||||
- v2.13.2's `.reaction*` classes clashed with the older `.reaction` lines
|
||||
(Liked / Reposted / In reply to, `partials/reaction.html`): the counts
|
||||
line now uses `.response-count*` / `.response-card*` / `.response-*`.
|
||||
|
||||
## Verification
|
||||
|
||||
- `tests/test_backfeed.py::test_home_page_shows_counts`; template, route,
|
||||
microformats and likes/reposts tests pass.
|
||||
- Rendered the home page, a note page and the Likes page in headless
|
||||
Chromium: counts beside the date, quiet notes unchanged, the Liked line
|
||||
laid out as before v2.13.2.
|
||||
@@ -0,0 +1,38 @@
|
||||
# v2.14.0: Notifications (implementation report)
|
||||
|
||||
Implements ADR-074.
|
||||
|
||||
## What changed
|
||||
|
||||
- `starpunk/notifications.py` (new): settings (`urls`, `events`,
|
||||
`with_priority`), `send` (Apprise, Markdown, 15 s timeout, never raises,
|
||||
errors captured with `apprise.LogCapture`), `problem`, and `Sweeper`,
|
||||
which sends due responses, groups likes/reposts/bookmarks per note,
|
||||
marks `notified_at`, backs off after a failure (10 s doubling to 15 min),
|
||||
drops responses older than 24 hours, and records the last error in
|
||||
`site_state.notify_last_error`.
|
||||
- `migrations/021_notifications.sql`: `webmentions.notified_at`, set on all
|
||||
existing rows.
|
||||
- `webmention_worker.py`: runs the sweep each pass (one process holds the
|
||||
worker lock).
|
||||
- `backfeed.py`: `store(..., quiet=True)` inserts new rows as notified;
|
||||
`check()` is quiet for backfill checks and a copy's first check of a note
|
||||
older than a day. A refused credential sends a `problems` message.
|
||||
- `syndication.py`: `_fail` sends a `problems` message when it gives up.
|
||||
- Admin Webmentions page: status (events, services, priority, last error)
|
||||
and **Send test notification** (`POST /admin/notifications/test`).
|
||||
- Settings `NOTIFY_URLS`, `NOTIFY_EVENTS`, `NOTIFY_PRIORITY`; dependency
|
||||
`apprise==2.2.*`.
|
||||
|
||||
## Verification
|
||||
|
||||
- `tests/test_notifications.py` (11 tests): settings parsing and priority
|
||||
injection; one message per reply/mention, likes grouped, unselected types
|
||||
marked, pending moderation linked to the admin; nothing sent twice;
|
||||
failure backoff and retry, last error set and cleared; 24-hour drop; off
|
||||
without URLs; quiet backfill and late first checks; problems (selected or
|
||||
not) and the final crosspost failure; admin status and test button; a
|
||||
real Apprise send (`json://`) to a local server; migration 021.
|
||||
- Webmention, syndication, backfeed, admin and migration tests pass.
|
||||
- Apprise 2.2.0 installs and imports on Python 3.14.
|
||||
- Production Gotify (3.1.1) accepted a message with the StarPunk app token.
|
||||
@@ -0,0 +1,27 @@
|
||||
# v2.14.1: Clearing tags in the editor; captions escaped in feeds
|
||||
|
||||
Two bugs from the backlog (High), confirmed against the code at v2.14.0.
|
||||
|
||||
## Clearing tags
|
||||
|
||||
`routes/admin.py` (edit POST) passed `tags=tags if tags else None`, and
|
||||
`notes.update_note` treats `None` as "unchanged", so an empty Tags field
|
||||
kept the old tags although the form says "Leave blank to remove all tags".
|
||||
It now passes the parsed list (possibly empty) whenever the form has a
|
||||
`tags` field, and `None` only when it doesn't.
|
||||
|
||||
## Captions in feed HTML
|
||||
|
||||
The media `<img>` built for item HTML put the caption into `alt="…"`
|
||||
unescaped in four places: `feeds/rss.py` (two), `feeds/atom.py` and
|
||||
`feeds/json_feed.py`. A caption with `"`, `<` or `&` broke the markup. The
|
||||
caption and the media URL are now `html.escape(..., quote=True)`d. The
|
||||
Atom enclosure's `title` attribute was already escaped with `_escape_xml`
|
||||
and is unchanged (escaping it again would double it).
|
||||
|
||||
## Verification
|
||||
|
||||
`tests/test_tags_and_captions.py` (4 tests, all failing before the fix):
|
||||
caption escaped once in RSS, Atom (and the enclosure title not doubled) and
|
||||
JSON Feed; the editor leaves tags alone without the field and clears them
|
||||
with an empty one. Feed, admin, tag, public-route and media tests pass.
|
||||
@@ -0,0 +1,31 @@
|
||||
# v2.14.2: Quote counts; dead files removed
|
||||
|
||||
The two remaining items from the v2.14.0 documentation review.
|
||||
|
||||
## Quote counts
|
||||
|
||||
Backfeed stored anonymous quote counts (`syndications.quotes_count`: all
|
||||
Threads quotes, and quotes by people who opted out elsewhere) that nothing
|
||||
displayed. The counts line (`partials/responses.html` `reactions`) now has a
|
||||
quote count after replies: named quotes plus those counts. Its card shows
|
||||
only the anonymous part ("2 on Threads"); named quotes stay listed under
|
||||
Responses. Quotes have no web action (there is no standard verb).
|
||||
`webmention_queries.response_counts` adds `quotes`, so lists of notes show
|
||||
them too.
|
||||
|
||||
## Removed
|
||||
|
||||
- `static/js/preview.js` (empty since the first commit) and the jsDelivr
|
||||
`marked` script the editor pages loaded with it.
|
||||
- `templates/feed.xml` (empty, never rendered) and `static/indielogin-test.html`
|
||||
(a v1 IndieLogin test page).
|
||||
- `syndication.get_targets()` read `THREADS_API` from the config, which
|
||||
`config.py` never loads; the client now always uses the `THREADS_API`
|
||||
constant.
|
||||
|
||||
## Verification
|
||||
|
||||
`tests/test_backfeed.py::test_quote_counts_include_anonymous_ones`; backfeed,
|
||||
syndication, admin, template, public-route, microformats, notification,
|
||||
webmention and feed tests pass. Rendered the counts line with the quote card
|
||||
open in headless Chromium.
|
||||
+18
-54
@@ -1,9 +1,9 @@
|
||||
# StarPunk Backlog
|
||||
|
||||
**Last Updated**: 2026-11-03 (current release: v2.10.1)
|
||||
**Last Updated**: 2026-11-10 (current release: v2.14.0)
|
||||
|
||||
These are the open features and bugs, sorted by priority. Each item was checked against the code
|
||||
at v2.10.1. Work that has shipped is listed per release in `vX.Y.Z/RELEASE.md` and in
|
||||
at v2.14.0. Work that has shipped is listed per release in `vX.Y.Z/RELEASE.md` and in
|
||||
`CHANGELOG.md`.
|
||||
|
||||
## Priority Levels
|
||||
@@ -23,45 +23,12 @@ at v2.10.1. Work that has shipped is listed per release in `vX.Y.Z/RELEASE.md` a
|
||||
|
||||
## High
|
||||
|
||||
### SQLite foreign keys are not enforced (ON DELETE CASCADE is inert)
|
||||
- **Description**: No connection runs `PRAGMA foreign_keys = ON`, so the `ON DELETE CASCADE` clauses in the schema do nothing. When a note is deleted from the admin (a hard delete), its `note_tags`, `note_media`, `syndications` and webmention rows stay behind. `delete_media()` relies on the cascade to remove `media_variants` and `note_media` rows, so those are left orphaned too. Before turning the pragma on, check for existing orphan rows and add a migration that cleans them up. Each cascade also needs tests.
|
||||
- **Where**: `starpunk/database/` (connection setup/pool), `starpunk/notes.py` (`delete_note`, hard delete), `starpunk/media.py` (`delete_media`), `migrations/007`–`010`, `016`
|
||||
- **Source**: v2.10.x review; confirmed in code (there is no `foreign_keys` pragma anywhere in `starpunk/`)
|
||||
|
||||
### No CSRF tokens on admin and consent forms
|
||||
- **Description**: Admin forms (create/edit/delete notes, webmention moderation, token revocation) and the IndieAuth consent form rely only on the `SameSite=Lax` session cookie. Lax blocks cross-site POSTs in current browsers, but a per-session CSRF token is the standard defence and covers older clients and same-site subdomains.
|
||||
- **Where**: `starpunk/routes/admin.py`, `starpunk/routes/indieauth.py`, `templates/admin/*`
|
||||
- **Source**: Documentation refresh 2026-11-03 (architecture review)
|
||||
|
||||
### Two outbound fetches don't use the hardened fetcher
|
||||
- **Description**: IndieAuth client-metadata discovery (`indieauth.fetch_client_redirect_uris`) checks only the first URL, then uses `httpx.get(follow_redirects=True)`: redirect hops aren't checked and DNS isn't pinned. Crosspost link cards and Threads image fetches (`syndication._fetch_public`) check each hop but don't pin DNS, and their 10 s timeout is per read. Move both to `webmention_ssrf.safe_get`, as Webmention, WebSub and Micropub photo downloads already do.
|
||||
- **Where**: `starpunk/indieauth.py`, `starpunk/syndication.py`
|
||||
- **Source**: Documentation refresh 2026-11-03
|
||||
|
||||
### `FLASK_DEBUG` defaults to on
|
||||
- **Description**: `config.py` reads `FLASK_DEBUG` with default `"1"` (and `FLASK_ENV` defaults to development); the runtime image sets neither. Production is safe only because the deployment sets `FLASK_DEBUG=0`. Default to off, or set it in the `Containerfile` runtime stage.
|
||||
- **Where**: `starpunk/config.py`, `Containerfile`
|
||||
- **Source**: Documentation refresh 2026-11-03 (production verified: DEBUG is False there)
|
||||
|
||||
### Rate limiters are per process
|
||||
- **Description**: `RateLimiter` keeps its counters in memory, and gunicorn runs 4 workers. The WebSub hub limit of 30/min is therefore really 4×30/min, and the webmention endpoint limit is multiplied the same way. Consider a counter stored in SQLite that all workers share.
|
||||
- **Where**: `starpunk/rate_limiter.py`, `starpunk/routes/websub.py`, `starpunk/routes/webmention.py`, `Containerfile` (`--workers 4`)
|
||||
- **Source**: ADR-071 (Consequences: rate limiting bounds subscriber-chosen requests); v2.10.1 review
|
||||
None open.
|
||||
|
||||
---
|
||||
|
||||
## Medium
|
||||
|
||||
### Admin editor cannot clear all tags
|
||||
- **Description**: When a note is saved with an empty tag field, the route passes `tags=None`. `update_note` treats that as "unchanged", so the last tags can't be removed from the editor.
|
||||
- **Where**: `starpunk/routes/admin.py` (edit POST: `tags=tags if tags else None`), `starpunk/notes.py` (`update_note`)
|
||||
- **Source**: Old backlog (v1.3.0 follow-up); still present
|
||||
|
||||
### Media captions not escaped in RSS HTML
|
||||
- **Description**: The RSS description embeds `<img alt="{caption}">` without escaping the caption. A caption containing `"` or `<` breaks the item's HTML.
|
||||
- **Where**: `starpunk/feeds/rss.py` (two places, ~lines 136 and 277)
|
||||
- **Source**: Old backlog (Architect Review 2.2.10); still present
|
||||
|
||||
### No rate limiting on upload endpoints
|
||||
- **Description**: Media uploads (`/admin/new` and the Micropub media endpoint) have no rate limit. Both require authentication, so the risk comes from a leaked token or session.
|
||||
- **Where**: `starpunk/routes/admin.py`, `starpunk/routes/micropub.py`
|
||||
@@ -92,28 +59,13 @@ at v2.10.1. Work that has shipped is listed per release in `vX.Y.Z/RELEASE.md` a
|
||||
- **Where**: `starpunk/webmention_processor.py`, `starpunk/webmention_worker.py`
|
||||
- **Source**: v1.6.0 design reconstruction (docs/design/v1.6.0/background-worker.md)
|
||||
|
||||
### Webmention rate limiter never cleaned up
|
||||
- **Description**: `cleanup_rate_limiters()` is never called, so per-IP and per-domain keys accumulate for the life of each process.
|
||||
- **Where**: `starpunk/rate_limiter.py`, `starpunk/routes/webmention.py`
|
||||
- **Source**: v1.6.0 design reconstruction
|
||||
|
||||
### Atom and JSON Feed not linked from pages
|
||||
- **Description**: `base.html` advertises only `/feed` (as RSS) and OPML with `rel=alternate`; add `<link rel="alternate">` for `/feed.atom` and `/feed.json` so readers can pick a format.
|
||||
- **Where**: `templates/base.html`
|
||||
- **Source**: Documentation refresh 2026-11-03
|
||||
|
||||
### Housekeeping in the repository
|
||||
- **Description**: `compose.yaml` is stale (`image: starpunk:0.6.0`, `VERSION=0.6.0` would override the real version, unused `ENVIRONMENT`); the root `dev_auth.py` duplicates `starpunk/dev_auth.py` and is unused; `LICENSE` says "Copyright (c) 2024 [Your Name]"; `test.ini` contains a private host name; `lxml` and `python-dateutil` are used directly but only installed as dependencies of other packages, while `pytest` is pinned in the production `requirements.txt`.
|
||||
- **Where**: `compose.yaml`, `dev_auth.py`, `LICENSE`, `test.ini`, `requirements*.txt`
|
||||
- **Source**: Documentation refresh 2026-11-03
|
||||
|
||||
### Backfeed responses to crossposts
|
||||
- **Description**: Idea: bring likes, boosts and replies to StarPunk's own crossposts back to the note as responses. Two options: poll GoToSocial (the token would need a `read` scope), or rely on Bridgy, which sends webmentions. Since v2.8.0, StarPunk ignores webmentions *from* a note's own crossposts, so that rule would need to change to work with Bridgy.
|
||||
- **Where**: `starpunk/syndication.py`, `starpunk/webmention_worker.py`
|
||||
- **Source**: v2.8.0 microformats2 review discussion
|
||||
|
||||
### Tag-filtered feeds
|
||||
- **Description**: Tag pages are h-feeds and WebSub topics, but there are no RSS, Atom or JSON feeds per tag (for example `/tag/python/feed.atom`). Tag pages also have no pagination, and the editor has no tag autocomplete.
|
||||
- **Description**: Tag pages are h-feeds and WebSub topics, but there are no RSS, Atom or JSON feeds per tag (for example `/tag/python/feed.atom`). Tag pages also have no pagination, and the editor has no tag autocomplete (Micropub `q=category`, v2.12.1, already gives the list a `<datalist>` could use).
|
||||
- **Where**: `starpunk/routes/public.py`, `starpunk/feeds/`, `templates/admin/`
|
||||
- **Source**: Old backlog (Tag enhancements)
|
||||
|
||||
@@ -192,7 +144,7 @@ at v2.10.1. Work that has shipped is listed per release in `vX.Y.Z/RELEASE.md` a
|
||||
- **Source**: Old backlog (v1.5.0 scoping)
|
||||
|
||||
### Unused settings and legacy tables
|
||||
- **Description**: `NOTES_PATH` is read and its folder created, but notes are always written under `DATA_PATH/notes`. `SITE_AUTHOR` is required but shown nowhere. `INITIAL_SCHEMA_SQL` recreates the unused `tokens` and `authorization_codes` tables (dropped by migration 004) on every start, and `auth_state` is unused. Version fallbacks are stale (`author_discovery.py` "1.2.0", `base.html` "0.5.0").
|
||||
- **Description**: `NOTES_PATH` is read and its folder created, but notes are always written under `DATA_PATH/notes`. `SITE_AUTHOR` is checked as required but defaults to "Unknown" and is shown nowhere. `INITIAL_SCHEMA_SQL` recreates the unused `tokens` and `authorization_codes` tables (dropped by migration 004) on every start, and `auth_state` is unused. Version fallbacks are stale (`author_discovery.py` "1.2.0", `base.html` "0.5.0").
|
||||
- **Where**: `starpunk/config.py`, `starpunk/utils.py`, `starpunk/database/schema.py`, `starpunk/author_discovery.py`, `templates/base.html`
|
||||
- **Source**: Documentation refresh 2026-11-03 (docs/architecture/database.md, deployment.md)
|
||||
|
||||
@@ -209,10 +161,22 @@ at v2.10.1. Work that has shipped is listed per release in `vX.Y.Z/RELEASE.md` a
|
||||
|
||||
---
|
||||
|
||||
## Done since the last update (2025-12-17)
|
||||
## Done (recent releases)
|
||||
|
||||
These items were removed from the backlog because they have shipped:
|
||||
|
||||
- v2.14.2: quote counts (with anonymous ones) shown; dead editor scripts, files and the `THREADS_API` read removed
|
||||
- v2.14.1: the editor can clear all tags; image captions escaped in RSS, Atom and JSON Feed HTML
|
||||
|
||||
- v2.14.0: notifications of new responses and crosspost/backfeed problems through Apprise (ADR-074)
|
||||
- v2.13.0–v2.13.4: backfeed of likes, reposts, quotes and replies from GoToSocial, Bluesky and Threads (ADR-073); counts line with web actions; counts on lists of notes; local avatar copies
|
||||
- v2.12.1: Micropub `q=category` lists only tags in use
|
||||
- v2.12.0: metrics dashboard and monitoring package removed (ADR-072)
|
||||
|
||||
- LICENSE: MIT, copyright Phil Skentelbery 2025-2026 (2026-11-03)
|
||||
|
||||
- v2.11.0: SQLite foreign keys enforced (orphans cleaned up by migration 018); CSRF tokens on admin, logout and consent forms; IndieAuth client and crosspost link-card fetches moved onto the SSRF-safe fetcher; `FLASK_DEBUG`/`FLASK_ENV` default to off/production; rate limits shared by all processes (and old windows pruned, so the webmention limiter no longer grows); dependencies upgraded (Python 3.14, no known vulnerabilities); stale `compose.yaml`, duplicate `dev_auth.py` and `test.ini` removed or fixed; `lxml`/`python-dateutil` pinned, `pytest` out of the runtime requirements
|
||||
|
||||
- POSSE: GoToSocial (v2.1.0), Bluesky (v2.2.0), Threads (v2.6.0)
|
||||
- Webmentions: receive, send, display and moderation (v1.6.0); fixed and made spec-compliant in v2.6.8–v2.7.2
|
||||
- Reply contexts, replies, likes, reposts and bookmarks (v2.4.0, v2.5.0, v2.9.0)
|
||||
|
||||
@@ -4,19 +4,19 @@ Future work and per-release plans. See `../DOCUMENTATION.md` (PROJECTPLAN).
|
||||
|
||||
## Backlog
|
||||
|
||||
- **[BACKLOG.md](BACKLOG.md)** - Open features and bugs by priority (updated for v2.10.1)
|
||||
- **[BACKLOG.md](BACKLOG.md)** - Open features and bugs by priority (updated for v2.14.0)
|
||||
|
||||
## Releases
|
||||
|
||||
- **[v1.0.0](v1.0.0/RELEASE.md)** (Released 2025-11-24)
|
||||
- **[v1.1.0](v1.1.0/RELEASE.md)** (Released 2025-11-25)
|
||||
- **[v1.1.1](v1.1.1/RELEASE.md)** (Released 2025-11-26)
|
||||
- **[v1.1.2](v1.1.2/RELEASE.md)** (Released 2025-11-27)
|
||||
- **[v1.1.1](v1.1.1/RELEASE.md)** (Released 2025-11-25)
|
||||
- **[v1.1.2](v1.1.2/RELEASE.md)** (Released 2025-11-28)
|
||||
- **[v1.2.0](v1.2.0/RELEASE.md)** (Released 2025-12-09)
|
||||
- **[v1.3.0](v1.3.0/RELEASE.md)** (Complete)
|
||||
- **[v1.3.1](v1.3.1/RELEASE.md)** - Feed Categories/Tags Support (Planning)
|
||||
- **[v1.4.0](v1.4.0/RELEASE.md)** - Micropub Media Endpoint, Large Image Support, Enhanced Feed Media (Planning)
|
||||
- **[v1.5.0](v1.5.0/RELEASE.md)** - Stability, Test Coverage, and Technical Debt Reduction (Approved)
|
||||
- **[v1.3.0](v1.3.0/RELEASE.md)** (Released 2025-12-10)
|
||||
- **[v1.3.1](v1.3.1/RELEASE.md)** - Feed Categories/Tags Support (Released 2025-12-10)
|
||||
- **[v1.4.0](v1.4.0/RELEASE.md)** - Micropub Media Endpoint, Large Image Support, Enhanced Feed Media (Released 2025-12-16 as 1.4.0rc1, then 1.4.1 and 1.4.2)
|
||||
- **[v1.5.0](v1.5.0/RELEASE.md)** - Stability, Test Coverage, and Technical Debt Reduction (Released 2025-12-17)
|
||||
- **[v2.0.0](v2.0.0/RELEASE.md)** - OIDC admin login and built-in IndieAuth server (Released 2026-10-31)
|
||||
- **[v2.0.1](v2.0.1/RELEASE.md)** - Absolute URLs behind HTTPS proxy (Released 2026-10-31)
|
||||
- **[v2.0.2](v2.0.2/RELEASE.md)** - CI registry login (Released 2026-10-31)
|
||||
@@ -47,3 +47,14 @@ Future work and per-release plans. See `../DOCUMENTATION.md` (PROJECTPLAN).
|
||||
- **[v2.10.0](v2.10.0/RELEASE.md)** - WebSub (Released 2026-11-03)
|
||||
- **[v2.10.1](v2.10.1/RELEASE.md)** - WebSub hardening (Released 2026-11-03)
|
||||
- **[v2.10.2](v2.10.2/RELEASE.md)** - Reply context fetch fix (Released 2026-11-03)
|
||||
- **[v2.11.0](v2.11.0/RELEASE.md)** - Maintenance: dependency upgrades, High backlog items (Released 2026-11-03)
|
||||
- **[v2.12.0](v2.12.0/RELEASE.md)** - Metrics dashboard removed (Released 2026-11-03)
|
||||
- **[v2.12.1](v2.12.1/RELEASE.md)** - `q=category` lists only tags in use (Released 2026-11-10)
|
||||
- **[v2.13.0](v2.13.0/RELEASE.md)** - Backfeed of responses to crossposts (Released 2026-11-10)
|
||||
- **[v2.13.1](v2.13.1/RELEASE.md)** - Avatar fetching fixes (Released 2026-11-10)
|
||||
- **[v2.13.2](v2.13.2/RELEASE.md)** - Counts line and web actions (Released 2026-11-10)
|
||||
- **[v2.13.3](v2.13.3/RELEASE.md)** - Opted-out people counted without names (Released 2026-11-10)
|
||||
- **[v2.13.4](v2.13.4/RELEASE.md)** - Counts on lists of notes, reply cards link out (Released 2026-11-10)
|
||||
- **[v2.14.0](v2.14.0/RELEASE.md)** - Notifications through Apprise (Released 2026-11-10)
|
||||
- **[v2.14.1](v2.14.1/RELEASE.md)** - Clearing tags in the editor; captions escaped in feeds (Released 2026-11-10)
|
||||
- **[v2.14.2](v2.14.2/RELEASE.md)** - Quote counts; dead files removed (Released 2026-11-10)
|
||||
@@ -1,6 +1,6 @@
|
||||
# StarPunk v1.1.1 Release
|
||||
|
||||
**Status**: Released 2025-11-26
|
||||
**Status**: Released 2025-11-25
|
||||
|
||||
## Features
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# StarPunk v1.1.2 Release
|
||||
|
||||
**Status**: Released 2025-11-27
|
||||
**Status**: Released 2025-11-28
|
||||
**Codename**: Syndicate
|
||||
|
||||
## Features
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# StarPunk v1.3.0 Release
|
||||
|
||||
**Status**: Complete
|
||||
**Status**: Released 2025-12-10
|
||||
**Codename**: "Categories"
|
||||
**Branch**: `feature/v1.3.0-tags-microformats`
|
||||
**Approved**: 2025-12-10
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# StarPunk v1.3.1 Release
|
||||
|
||||
**Status**: Planning
|
||||
**Status**: Released 2025-12-10
|
||||
**Codename**: "Syndicate Tags"
|
||||
**Focus**: Feed Categories/Tags Support
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# StarPunk v1.4.0 Release
|
||||
|
||||
**Status**: Planning
|
||||
**Status**: Released 2025-12-16 (as 1.4.0rc1; then 1.4.1 and 1.4.2)
|
||||
**Codename**: "Media"
|
||||
**Focus**: Micropub Media Endpoint, Large Image Support, Enhanced Feed Media
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# StarPunk v1.5.0 Release Plan
|
||||
|
||||
**Status**: Approved
|
||||
**Status**: Released 2025-12-17
|
||||
**Codename**: "Trigger"
|
||||
**Focus**: Stability, Test Coverage, and Technical Debt Reduction
|
||||
**Last Updated**: 2025-12-17
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# v2.10.2
|
||||
|
||||
- **Date**: 2026-11-03
|
||||
- **Status**: Released
|
||||
- **Status**: Released 2026-11-03
|
||||
|
||||
## Bugs fixed
|
||||
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
# v2.11.0
|
||||
|
||||
- **Date**: 2026-11-03
|
||||
- **Status**: Released 2026-11-03
|
||||
- **Focus**: Maintenance: dependency upgrades and the High backlog items
|
||||
|
||||
## Security
|
||||
|
||||
- All dependencies upgraded (97 known vulnerabilities before, none after); Python 3.14
|
||||
- CSRF tokens on admin forms, logout and the IndieAuth consent
|
||||
- IndieAuth client and crosspost link-card fetches use the SSRF-safe fetcher
|
||||
- Debug off and production environment by default
|
||||
|
||||
## Changes
|
||||
|
||||
- Rate limits shared by all server processes
|
||||
- `requirements-test.txt`; pytest out of the production image; `lxml` and `python-dateutil` pinned; `uv` pinned
|
||||
|
||||
## Bugs fixed
|
||||
|
||||
- Foreign keys enforced, so deletes cascade; orphan rows removed (migration 018)
|
||||
- Deprecated `datetime.utcnow()` and sqlite3 adapters replaced
|
||||
- Stale `compose.yaml`, duplicate `dev_auth.py` and `test.ini` cleaned up
|
||||
|
||||
## Related
|
||||
|
||||
- [CHANGELOG](../../../CHANGELOG.md)
|
||||
- [Design note](../../design/v2.11.0/2026-11-03-maintenance-release.md)
|
||||
@@ -0,0 +1,21 @@
|
||||
# v2.12.0
|
||||
|
||||
- **Date**: 2026-11-03
|
||||
- **Status**: Released 2026-11-03
|
||||
- **Focus**: Remove the metrics dashboard
|
||||
|
||||
## Removed
|
||||
|
||||
- Admin metrics dashboard (`/admin/metrics-dashboard`) and `/admin/metrics`
|
||||
- The `starpunk/monitoring` package: request timing, query timing, memory monitor thread, feed counters
|
||||
- `METRICS_*` settings and the `psutil` dependency
|
||||
|
||||
## Kept
|
||||
|
||||
- `/health`, `/admin/health` (without the metrics section), logs with correlation IDs
|
||||
|
||||
## Related
|
||||
|
||||
- [CHANGELOG](../../../CHANGELOG.md)
|
||||
- [ADR-072](../../decisions/ADR-072-remove-metrics-dashboard.md)
|
||||
- [Design note](../../design/v2.12.0/2026-11-03-remove-metrics.md)
|
||||
@@ -0,0 +1,14 @@
|
||||
# v2.12.1
|
||||
|
||||
- **Date**: 2026-11-10
|
||||
- **Status**: Released
|
||||
- **Focus**: Micropub `q=category` lists only tags in use
|
||||
|
||||
## Fixed
|
||||
|
||||
- `q=category` returns only tags on posts that exist (not ones an edit removed, nor ones only on deleted posts)
|
||||
|
||||
## Related
|
||||
|
||||
- [CHANGELOG](../../../CHANGELOG.md)
|
||||
- [Design note](../../design/v2.12.1/2026-11-10-category-unused-tags.md)
|
||||
@@ -0,0 +1,25 @@
|
||||
# v2.13.0
|
||||
|
||||
- **Date**: 2026-11-10
|
||||
- **Status**: Released
|
||||
- **Focus**: Backfeed of responses to crossposts
|
||||
|
||||
## Added
|
||||
|
||||
- Likes, reposts, quotes and replies to GoToSocial and Bluesky crossposts, and replies on Threads, shown on the note
|
||||
- Age-based checking schedule up to `BACKFEED_MAX_AGE_DAYS` (90); admin backfill and per-crosspost Check now
|
||||
- Threads like/repost/quote counts; a Quotes group; origin filter and Deleted view in the admin
|
||||
- Settings `BACKFEED_ENABLED`, `BACKFEED_MAX_AGE_DAYS`; migration 020
|
||||
|
||||
## Changed
|
||||
|
||||
- Response avatars saved locally (`/avatars/`)
|
||||
- Admin delete keeps a tombstone
|
||||
- `#nobridge` / `#nobot` and Bluesky's logged-out label honoured; already backfed responses not shown twice from Bridgy
|
||||
- Threads client on `graph.threads.com`
|
||||
|
||||
## Related
|
||||
|
||||
- [CHANGELOG](../../../CHANGELOG.md)
|
||||
- [ADR-073](../../decisions/ADR-073-backfeed.md)
|
||||
- [Design note](../../design/v2.13.0/2026-11-10-backfeed.md)
|
||||
@@ -0,0 +1,15 @@
|
||||
# v2.13.1
|
||||
|
||||
- **Date**: 2026-11-10
|
||||
- **Status**: Released
|
||||
- **Focus**: Avatar fetching in production
|
||||
|
||||
## Fixed
|
||||
|
||||
- Avatars from the site's own GoToSocial (resolves to an internal address) and from Bluesky (CDN refuses some server IPv4 addresses; fetched from the PDS instead)
|
||||
|
||||
## Related
|
||||
|
||||
- [CHANGELOG](../../../CHANGELOG.md)
|
||||
- [ADR-073](../../decisions/ADR-073-backfeed.md)
|
||||
- [Design note](../../design/v2.13.1/2026-11-10-avatar-fetch.md)
|
||||
@@ -0,0 +1,20 @@
|
||||
# v2.13.2
|
||||
|
||||
- **Date**: 2026-11-10
|
||||
- **Status**: Released
|
||||
- **Focus**: Counts line for likes and reposts; web actions
|
||||
|
||||
## Changed
|
||||
|
||||
- Likes, reposts and bookmarks shown as one line of counts under the note, with who on hover; Threads counts folded in
|
||||
- `<indie-action>` web actions with links to the note's crossposts
|
||||
|
||||
## Fixed
|
||||
|
||||
- Avatars GoToSocial hasn't cached (redirects to the original server)
|
||||
|
||||
## Related
|
||||
|
||||
- [CHANGELOG](../../../CHANGELOG.md)
|
||||
- [ADR-073](../../decisions/ADR-073-backfeed.md)
|
||||
- [Design note](../../design/v2.13.2/2026-11-10-reaction-counts.md)
|
||||
@@ -0,0 +1,15 @@
|
||||
# v2.13.3
|
||||
|
||||
- **Date**: 2026-11-10
|
||||
- **Status**: Released
|
||||
- **Focus**: Opted-out people counted without names
|
||||
|
||||
## Changed
|
||||
|
||||
- Likes and reposts by people who opted out of backfeed count in the totals, without name or avatar
|
||||
|
||||
## Related
|
||||
|
||||
- [CHANGELOG](../../../CHANGELOG.md)
|
||||
- [ADR-073](../../decisions/ADR-073-backfeed.md)
|
||||
- [Design note](../../design/v2.13.3/2026-11-10-count-opted-out.md)
|
||||
@@ -0,0 +1,22 @@
|
||||
# v2.13.4
|
||||
|
||||
- **Date**: 2026-11-10
|
||||
- **Status**: Released
|
||||
- **Focus**: Counts on lists of notes; reply cards
|
||||
|
||||
## Added
|
||||
|
||||
- Like, repost and reply counts on the home, tag and Likes pages
|
||||
|
||||
## Changed
|
||||
|
||||
- A whole reply card opens the reply where it was posted
|
||||
|
||||
## Fixed
|
||||
|
||||
- The counts line's CSS clashed with the Liked / Reposted / In reply to lines (`.reaction`)
|
||||
|
||||
## Related
|
||||
|
||||
- [CHANGELOG](../../../CHANGELOG.md)
|
||||
- [Design note](../../design/v2.13.4/2026-11-10-list-counts.md)
|
||||
@@ -0,0 +1,16 @@
|
||||
# v2.14.0
|
||||
|
||||
- **Date**: 2026-11-10
|
||||
- **Status**: Released
|
||||
- **Focus**: Notifications through Apprise
|
||||
|
||||
## Added
|
||||
|
||||
- Notifications of new responses and crosspost/backfeed problems via the embedded Apprise library
|
||||
- Settings `NOTIFY_URLS`, `NOTIFY_EVENTS`, `NOTIFY_PRIORITY`; admin status and Send test notification; migration 021
|
||||
|
||||
## Related
|
||||
|
||||
- [CHANGELOG](../../../CHANGELOG.md)
|
||||
- [ADR-074](../../decisions/ADR-074-notifications.md)
|
||||
- [Design note](../../design/v2.14.0/2026-11-10-notifications.md)
|
||||
@@ -0,0 +1,15 @@
|
||||
# v2.14.1
|
||||
|
||||
- **Date**: 2026-11-10
|
||||
- **Status**: Released
|
||||
- **Focus**: The two High backlog bugs
|
||||
|
||||
## Fixed
|
||||
|
||||
- The editor can clear all of a note's tags
|
||||
- Image captions escaped in RSS, Atom and JSON Feed item HTML
|
||||
|
||||
## Related
|
||||
|
||||
- [CHANGELOG](../../../CHANGELOG.md)
|
||||
- [Design note](../../design/v2.14.1/2026-11-10-tags-and-captions.md)
|
||||
@@ -0,0 +1,18 @@
|
||||
# v2.14.2
|
||||
|
||||
- **Date**: 2026-11-10
|
||||
- **Status**: Released
|
||||
- **Focus**: Quote counts; dead files removed
|
||||
|
||||
## Added
|
||||
|
||||
- Quote counts (with anonymous ones) in the counts line and on lists of notes
|
||||
|
||||
## Removed
|
||||
|
||||
- Unused editor scripts (`marked` from a CDN, empty `preview.js`), `templates/feed.xml`, `static/indielogin-test.html`, the `THREADS_API` config read
|
||||
|
||||
## Related
|
||||
|
||||
- [CHANGELOG](../../../CHANGELOG.md)
|
||||
- [Design note](../../design/v2.14.2/2026-11-10-quotes-cleanup.md)
|
||||
+4
-4
@@ -1,11 +1,11 @@
|
||||
# Utility Function Patterns
|
||||
|
||||
**Status**: Active
|
||||
**Last Updated**: 2026-11-03 (v2.10.1; reviewed, content from 2025-11-18)
|
||||
**Status**: Deprecated (2026-11-10). Superseded by [python-coding-standards.md](../python-coding-standards.md), which this document contradicts on type hints (`X | None`, built-in generics, hints on new and changed public functions).
|
||||
**Last Updated**: 2026-11-03 (v2.10.1; content from 2025-11-18)
|
||||
|
||||
## Purpose
|
||||
|
||||
This document establishes coding patterns and conventions specifically for utility functions in StarPunk (`starpunk/utils.py`, `starpunk/slug_utils.py`, `starpunk/mf2_utils.py` and helpers in other modules). These patterns ensure utilities are consistent, testable, and maintainable. General rules, including project-wide patterns for configuration, database access and outbound HTTP, are in [python-coding-standards.md](python-coding-standards.md); where the two differ, that document wins.
|
||||
This document establishes coding patterns and conventions specifically for utility functions in StarPunk (`starpunk/utils.py`, `starpunk/slug_utils.py`, `starpunk/mf2_utils.py` and helpers in other modules). These patterns ensure utilities are consistent, testable, and maintainable. General rules, including project-wide patterns for configuration, database access and outbound HTTP, are in [python-coding-standards.md](../python-coding-standards.md); where the two differ, that document wins.
|
||||
|
||||
The examples are illustrative. Several mirror real functions in `starpunk/utils.py` (`calculate_content_hash`, `write_note_file`, `validate_slug`, `generate_slug`, `validate_note_path`); others (`require_valid_slug`, `iter_note_files`) only show the pattern.
|
||||
|
||||
@@ -731,7 +731,7 @@ def generate_slug(content: str) -> str:
|
||||
|
||||
## References
|
||||
|
||||
- [Python Coding Standards](python-coding-standards.md)
|
||||
- [Python Coding Standards](../python-coding-standards.md)
|
||||
- [PEP 8 - Style Guide](https://peps.python.org/pep-0008/)
|
||||
- [PEP 257 - Docstring Conventions](https://peps.python.org/pep-0257/)
|
||||
- [PEP 484 - Type Hints](https://peps.python.org/pep-0484/)
|
||||
@@ -1,10 +1,10 @@
|
||||
# Standards Index
|
||||
|
||||
**Last Updated**: 2026-11-03 (v2.10.1)
|
||||
**Last Updated**: 2026-11-10 (v2.14.0)
|
||||
|
||||
Standards for developing and releasing StarPunk. Folder rules are in
|
||||
[../DOCUMENTATION.md](../DOCUMENTATION.md). All standards below were checked
|
||||
against the code at v2.10.1.
|
||||
against the code at v2.14.0.
|
||||
|
||||
## Active standards
|
||||
|
||||
@@ -16,7 +16,6 @@ against the code at v2.10.1.
|
||||
| [versioning-strategy.md](versioning-strategy.md) | Choosing MAJOR / MINOR / PATCH |
|
||||
| [version-implementation-guide.md](version-implementation-guide.md) | Where the version lives, changelog entry and design note format |
|
||||
| [python-coding-standards.md](python-coding-standards.md) | Style, naming, and project patterns (config, database, outbound HTTP, errors, routes, workers) |
|
||||
| [utility-function-patterns.md](utility-function-patterns.md) | Patterns for small helper functions |
|
||||
| [cookie-naming-convention.md](cookie-naming-convention.md) | `starpunk_` cookie prefix and the cookies in use |
|
||||
| [documentation-organization.md](documentation-organization.md) | Templates and conventions for docs (folder rules: DOCUMENTATION.md) |
|
||||
|
||||
@@ -31,8 +30,8 @@ against the code at v2.10.1.
|
||||
## Key rules in brief
|
||||
|
||||
- `uv` for environments, `uv run` for every command.
|
||||
- Tests need `SESSION_SECRET`; the full suite takes about 6 minutes, 5 of
|
||||
them in `tests/test_migration_race_condition.py`.
|
||||
- Tests need `SESSION_SECRET`. Locally, run only the tests for the area you
|
||||
changed; CI runs the full suite (about 6 minutes) on every `vX.Y.Z` tag.
|
||||
- Every release: branch in a worktree, bump `__version__` and
|
||||
`__version_info__`, changelog entry, design note (or ADR amendment),
|
||||
`git merge --no-ff`, annotated `vX.Y.Z` tag; CI tests and publishes the
|
||||
@@ -43,8 +42,9 @@ against the code at v2.10.1.
|
||||
|
||||
## Deprecated standards
|
||||
|
||||
Standards that no longer apply move to `DEPRECATED/` (none at
|
||||
present). Earlier versions of the standards above are in git history.
|
||||
Standards that no longer apply move to `DEPRECATED/`:
|
||||
[utility-function-patterns.md](DEPRECATED/utility-function-patterns.md)
|
||||
(superseded by python-coding-standards.md, 2026-11-10). Earlier versions of the standards above are in git history.
|
||||
|
||||
## Related
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
**Status**: Active
|
||||
**Date**: 2025-11-18
|
||||
**Last Updated**: 2026-11-03 (v2.10.1; cookie list checked against the code)
|
||||
**Last Updated**: 2026-11-10 (v2.14.0; cookie list checked against the code)
|
||||
**Version**: 1.1
|
||||
|
||||
## Purpose
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# Development Setup
|
||||
|
||||
**Status**: Active
|
||||
**Last Updated**: 2026-11-03 (v2.10.1)
|
||||
**Last Updated**: 2026-11-10 (v2.14.0)
|
||||
|
||||
How to set up a local StarPunk environment, run the app and run the tests.
|
||||
Each configuration variable is described in [`.env.example`](../../.env.example);
|
||||
@@ -9,7 +9,7 @@ deployment is in [architecture/deployment.md](../architecture/deployment.md).
|
||||
|
||||
## Requirements
|
||||
|
||||
- Python 3.11 (the container image uses `python:3.11-slim`)
|
||||
- Python 3.14 (the container image uses `python:3.14-slim`)
|
||||
- [uv](https://docs.astral.sh/uv/) for the virtual environment and packages ([ADR-006](../decisions/ADR-006-python-virtual-environment-uv.md))
|
||||
- git
|
||||
- Optional: Docker or Podman, to build the image and run the CI test stage locally
|
||||
@@ -17,13 +17,14 @@ deployment is in [architecture/deployment.md](../architecture/deployment.md).
|
||||
## Environment
|
||||
|
||||
The repository has no `pyproject.toml`; dependencies are in
|
||||
`requirements.txt` (runtime) and `requirements-dev.txt` (adds pytest-cov,
|
||||
pytest-mock, black, flake8, mypy).
|
||||
`requirements.txt` (runtime) and `requirements-dev.txt` (adds
|
||||
`requirements-test.txt` (pytest, pytest-mock), plus pytest-cov, black, flake8,
|
||||
mypy and pip-audit).
|
||||
|
||||
```bash
|
||||
git clone https://git.thesatelliteoflove.com/phil/StarPunk.git
|
||||
cd StarPunk
|
||||
uv venv --python 3.11 # creates .venv/ (gitignored)
|
||||
uv venv --python 3.14 # creates .venv/ (gitignored)
|
||||
uv pip install -r requirements-dev.txt
|
||||
```
|
||||
|
||||
@@ -84,10 +85,9 @@ banner to every page. It sets the `starpunk_session` cookie for
|
||||
server.
|
||||
|
||||
Each app process also starts background threads: the webmention worker,
|
||||
the WebSub hub worker, the memory monitor and, when a crossposting target
|
||||
is configured, the syndication worker. Turn off what you don't need locally
|
||||
with `WEBMENTION_WORKER_ENABLED=false`, `WEBSUB_ENABLED=false` and
|
||||
`METRICS_ENABLED=false`. Leave `GOTOSOCIAL_*`, `BLUESKY_*` and `THREADS_*`
|
||||
the WebSub hub worker and, when a crossposting target is configured, the
|
||||
syndication worker. Turn off what you don't need locally with
|
||||
`WEBMENTION_WORKER_ENABLED=false` and `WEBSUB_ENABLED=false`. Leave `GOTOSOCIAL_*`, `BLUESKY_*` and `THREADS_*`
|
||||
empty unless you mean to post to those accounts.
|
||||
|
||||
Production runs the same app under gunicorn (`gunicorn app:app`, see `CMD`
|
||||
@@ -101,9 +101,9 @@ docker run --rm -p 8000:8000 --env-file .env -v ./container-data:/data starpunk:
|
||||
## Running the tests
|
||||
|
||||
```bash
|
||||
SESSION_SECRET=test uv run pytest # full suite
|
||||
SESSION_SECRET=test uv run pytest tests/test_websub.py # one file
|
||||
SESSION_SECRET=test uv run pytest tests/test_websub.py # the area you changed
|
||||
SESSION_SECRET=test uv run pytest tests/test_micropub.py -k delete -x
|
||||
SESSION_SECRET=test uv run pytest # full suite (CI runs this; rarely needed locally)
|
||||
```
|
||||
|
||||
- `SESSION_SECRET` must be set (in `.env` or the environment) because
|
||||
@@ -112,11 +112,11 @@ SESSION_SECRET=test uv run pytest tests/test_micropub.py -k delete -x
|
||||
`ADMIN_ME=https://test.example.com`; set them too to reproduce CI exactly.
|
||||
- `tests/conftest.py` builds each app with `TESTING=True` and a temporary
|
||||
`DATA_PATH` and database, so tests never touch `./data`. Under
|
||||
`TESTING=True` the background workers and memory monitor don't start and
|
||||
`TESTING=True` the background workers don't start and
|
||||
the production OIDC check is skipped.
|
||||
- `tests/test_migration_race_condition.py` is slow (about 5 minutes). While
|
||||
iterating, add `--ignore=tests/test_migration_race_condition.py`; run the
|
||||
full suite before releasing.
|
||||
- Locally, run the tests for the area you changed; the full suite, including
|
||||
the slow `tests/test_migration_race_condition.py` (about 5 minutes), runs in
|
||||
CI on the release tag.
|
||||
- Coverage: `SESSION_SECRET=test uv run pytest --cov=starpunk --cov-report=term-missing`.
|
||||
- The CI test stage can be reproduced exactly with
|
||||
`docker build --target test -f Containerfile .` (see
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# Documentation Organization
|
||||
|
||||
**Status**: Active
|
||||
**Last Updated**: 2026-11-03 (v2.10.1)
|
||||
**Last Updated**: 2026-11-10 (v2.14.0)
|
||||
|
||||
**[docs/DOCUMENTATION.md](../DOCUMENTATION.md) is authoritative** for what
|
||||
goes in which folder and supersedes this document. This standard only adds
|
||||
@@ -94,5 +94,5 @@ top saying what replaced it, and removed from the active list in
|
||||
|
||||
- Architecture documents describing a design that no longer exists are
|
||||
either rewritten or renamed with a `-legacy` suffix and a note at the top
|
||||
(e.g. `architecture/technology-stack-legacy.md`).
|
||||
(e.g. `design/v0.x/technology-stack-legacy.md`).
|
||||
- Design notes and ADRs are historical records and stay where they are.
|
||||
@@ -1,7 +1,7 @@
|
||||
# Git Branching and Release Process
|
||||
|
||||
**Status**: Active
|
||||
**Last Updated**: 2026-11-03 (v2.10.1)
|
||||
**Last Updated**: 2026-11-10 (v2.14.0)
|
||||
|
||||
How changes reach `main`, how releases are cut, and what happens after a
|
||||
tag is pushed. This is the flow used for the v2 releases (see
|
||||
@@ -27,6 +27,7 @@ differ, this document describes current practice.
|
||||
| `feat/` | new behaviour (MINOR) | `feat/v2.10.0-websub` |
|
||||
| `fix/` | bug and security fixes (PATCH) | `fix/v2.10.1-websub-hardening` |
|
||||
| `ci/` | build and CI changes | `ci/v2.7.1-pytest` |
|
||||
| `chore/` | maintenance, removals, licensing | `chore/remove-metrics` |
|
||||
| `docs/` | documentation only, no release | `docs/refresh-v2.10.1` |
|
||||
|
||||
Release branches are named `<prefix>/vX.Y.Z-<short-description>`, so the
|
||||
@@ -63,7 +64,7 @@ cd ~/Documents/projects/StarPunk # main checkout, on main
|
||||
git pull
|
||||
git worktree add ../StarPunk-worktrees/v2.10.1 -b fix/v2.10.1-websub-hardening main
|
||||
cd ../StarPunk-worktrees/v2.10.1
|
||||
uv venv --python 3.11 && uv pip install -r requirements-dev.txt
|
||||
uv venv --python 3.14 && uv pip install -r requirements-dev.txt
|
||||
```
|
||||
|
||||
### 2. Make the change
|
||||
@@ -83,8 +84,9 @@ Code, tests and docs together:
|
||||
- [ ] Add a design note `docs/design/vX.Y.Z/YYYY-MM-DD-<topic>.md` (what
|
||||
changed and why, how it was verified). Small releases fully covered by
|
||||
an ADR amendment and the changelog may skip it
|
||||
- [ ] Full test suite passes:
|
||||
`SESSION_SECRET=test uv run pytest`
|
||||
- [ ] Tests for the changed area pass locally, e.g.
|
||||
`SESSION_SECRET=test uv run pytest tests/test_notifications.py`;
|
||||
CI runs the full suite on the tag (step 4)
|
||||
|
||||
Commit (usually one commit):
|
||||
|
||||
|
||||
@@ -1,16 +1,15 @@
|
||||
# Python Coding Standards
|
||||
|
||||
**Status**: Active
|
||||
**Last Updated**: 2026-11-03 (v2.10.1)
|
||||
**Last Updated**: 2026-11-10 (v2.14.0)
|
||||
|
||||
How Python code in StarPunk is written. The project rule comes first:
|
||||
"Every line of code must justify its existence. When in doubt, leave it
|
||||
out." Patterns specific to small helper functions are in
|
||||
[utility-function-patterns.md](utility-function-patterns.md).
|
||||
out."
|
||||
|
||||
## Base style
|
||||
|
||||
- [PEP 8](https://peps.python.org/pep-0008/) naming and layout, Python 3.11.
|
||||
- [PEP 8](https://peps.python.org/pep-0008/) naming and layout, Python 3.14. Use `datetime.now(timezone.utc)`, not the deprecated `datetime.utcnow()`.
|
||||
- **Match the file you are editing.** 4-space indents everywhere; quote
|
||||
style varies between modules (newer code uses double quotes); keep lines
|
||||
to roughly 100 characters.
|
||||
@@ -147,5 +146,4 @@ functions or `Test...` classes with descriptive names
|
||||
## References
|
||||
|
||||
- [PEP 8](https://peps.python.org/pep-0008/), [PEP 257](https://peps.python.org/pep-0257/), [PEP 484](https://peps.python.org/pep-0484/)
|
||||
- [utility-function-patterns.md](utility-function-patterns.md)
|
||||
- [ADR-055: Error handling philosophy](../decisions/ADR-055-error-handling-philosophy.md)
|
||||
@@ -1,7 +1,7 @@
|
||||
# Testing Checklist
|
||||
|
||||
**Status**: Active
|
||||
**Last Updated**: 2026-11-03 (v2.10.1)
|
||||
**Last Updated**: 2026-11-10 (v2.14.0)
|
||||
|
||||
What to test, how to run the tests, and what CI does with them. Setup and
|
||||
the environment variables the tests need are in
|
||||
@@ -12,12 +12,14 @@ the environment variables the tests need are in
|
||||
**Locally**
|
||||
|
||||
```bash
|
||||
SESSION_SECRET=test uv run pytest # everything
|
||||
SESSION_SECRET=test uv run pytest --ignore=tests/test_migration_race_condition.py # fast loop
|
||||
SESSION_SECRET=test uv run pytest tests/test_websub.py tests/test_websub_hardening.py # the area you changed
|
||||
SESSION_SECRET=test uv run pytest # everything (CI does this)
|
||||
```
|
||||
|
||||
`tests/test_migration_race_condition.py` takes about 5 minutes; skip it
|
||||
while iterating, never before a release.
|
||||
Locally, run the tests for the area you changed. The full suite, including
|
||||
the 5-minute `tests/test_migration_race_condition.py`, runs in CI. For wide
|
||||
changes (migrations, `config.py`, `create_app`, `tests/conftest.py`) you can
|
||||
build the CI test stage first: `docker build --target test -f Containerfile .`
|
||||
|
||||
**In CI** (`.gitea/workflows/build-container.yml`), on every pushed
|
||||
`vX.Y.Z` tag (or a manual run):
|
||||
@@ -30,16 +32,17 @@ while iterating, never before a release.
|
||||
2. Only if that succeeds is the runtime image built and pushed as
|
||||
`:vX.Y.Z` and `:latest`.
|
||||
|
||||
So a failing test blocks the release image; it does not block the merge or
|
||||
the tag. Run the full suite before merging (see
|
||||
[git-branching-strategy.md](git-branching-strategy.md#release-process)).
|
||||
So a failing test blocks the release image, not the merge or the tag; CI is
|
||||
where the full suite runs. If it fails, fix it in the next PATCH (see
|
||||
[git-branching-strategy.md](git-branching-strategy.md#release-process),
|
||||
step 4).
|
||||
To reproduce CI locally: `docker build --target test -f Containerfile .`
|
||||
(pytest-cov is not installed in that stage, so don't depend on it in
|
||||
tests).
|
||||
|
||||
## Every change
|
||||
|
||||
- [ ] Full suite passes locally
|
||||
- [ ] Tests for the changed area pass locally (CI runs the full suite)
|
||||
- [ ] New behaviour has tests; a bug fix has a test that failed before the fix
|
||||
- [ ] Tests use the `app` / `client` fixtures from `tests/conftest.py`
|
||||
(temporary data directory, `TESTING=True`); nothing touches `./data`
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# Version Implementation Guide
|
||||
|
||||
**Status**: Active
|
||||
**Last Updated**: 2026-11-03 (v2.10.1)
|
||||
**Last Updated**: 2026-11-10 (v2.14.0)
|
||||
|
||||
Where the version number lives and what to write for each release. Which
|
||||
number to pick: [versioning-strategy.md](versioning-strategy.md). Git steps:
|
||||
@@ -14,8 +14,8 @@ There is one source: `starpunk/__init__.py`, at the end of the file.
|
||||
```python
|
||||
# Package version (Semantic Versioning 2.0.0)
|
||||
# See docs/standards/versioning-strategy.md for details
|
||||
__version__ = "2.10.1"
|
||||
__version_info__ = (2, 10, 1)
|
||||
__version__ = "X.Y.Z"
|
||||
__version_info__ = (X, Y, Z)
|
||||
```
|
||||
|
||||
Update both lines. Nothing else holds the number: there is no
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
# Versioning Strategy
|
||||
|
||||
**Status**: Active
|
||||
**Last Updated**: 2026-11-03 (v2.10.1)
|
||||
**Last Updated**: 2026-11-10 (v2.14.0)
|
||||
|
||||
StarPunk uses [Semantic Versioning 2.0.0](https://semver.org/):
|
||||
`MAJOR.MINOR.PATCH`, currently **2.10.1**. Every release is a git tag
|
||||
`MAJOR.MINOR.PATCH` (the current version is in `starpunk/__init__.py`). Every release is a git tag
|
||||
`vX.Y.Z` and a `CHANGELOG.md` entry. Where the number is stored and how to
|
||||
write the changelog entry is in
|
||||
[version-implementation-guide.md](version-implementation-guide.md); the
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
-- Migration 018: Remove rows orphaned while foreign keys weren't enforced (v2.11.0)
|
||||
-- From v2.11.0 every pooled connection runs PRAGMA foreign_keys=ON, so these
|
||||
-- cascades happen on delete. This removes what earlier deletes left behind.
|
||||
|
||||
DELETE FROM note_tags WHERE note_id NOT IN (SELECT id FROM notes);
|
||||
DELETE FROM note_tags WHERE tag_id NOT IN (SELECT id FROM tags);
|
||||
DELETE FROM note_media WHERE note_id NOT IN (SELECT id FROM notes);
|
||||
DELETE FROM note_media WHERE media_id NOT IN (SELECT id FROM media);
|
||||
DELETE FROM media_variants WHERE media_id NOT IN (SELECT id FROM media);
|
||||
DELETE FROM webmention_queue WHERE source_note_id IS NOT NULL AND source_note_id NOT IN (SELECT id FROM notes);
|
||||
UPDATE webmentions SET target_note_id = NULL WHERE target_note_id IS NOT NULL AND target_note_id NOT IN (SELECT id FROM notes);
|
||||
DELETE FROM websub_deliveries WHERE subscription_id IS NOT NULL AND subscription_id NOT IN (SELECT id FROM websub_subscriptions);
|
||||
@@ -0,0 +1,9 @@
|
||||
-- Migration 019: Rate limit counters shared by all server processes (v2.11.0)
|
||||
-- One row per key and time window; see starpunk/rate_limiter.py SharedRateLimiter.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS rate_limits (
|
||||
key TEXT NOT NULL,
|
||||
window INTEGER NOT NULL,
|
||||
count INTEGER NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (key, window)
|
||||
);
|
||||
@@ -0,0 +1,66 @@
|
||||
-- Migration 020: Backfeed of responses to crossposts (v2.13.0, ADR-073)
|
||||
--
|
||||
-- webmentions: where a response came from (origin, remote_id) and a new
|
||||
-- 'quote' type. SQLite can't change a CHECK constraint, so the table is
|
||||
-- rebuilt (as in migration 003). Nothing references webmentions.
|
||||
--
|
||||
-- syndications: when each copy's responses were last checked and are next
|
||||
-- due, a lease so one gunicorn worker checks a copy, and Threads counts.
|
||||
|
||||
CREATE TABLE webmentions_new (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
source_url TEXT NOT NULL,
|
||||
target_url TEXT NOT NULL,
|
||||
target_note_id INTEGER,
|
||||
mention_type TEXT NOT NULL DEFAULT 'mention'
|
||||
CHECK (mention_type IN ('reply', 'like', 'repost', 'bookmark', 'mention', 'quote')),
|
||||
author_name TEXT,
|
||||
author_url TEXT,
|
||||
author_photo TEXT,
|
||||
content_snippet TEXT,
|
||||
validation_status TEXT NOT NULL DEFAULT 'pending'
|
||||
CHECK (validation_status IN ('pending', 'valid', 'invalid', 'error')),
|
||||
moderation_status TEXT NOT NULL DEFAULT 'pending'
|
||||
CHECK (moderation_status IN ('pending', 'approved', 'hidden', 'deleted')),
|
||||
validation_error TEXT,
|
||||
validated_at TIMESTAMP,
|
||||
received_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
source_post_url TEXT,
|
||||
source_published TIMESTAMP,
|
||||
origin TEXT NOT NULL DEFAULT 'webmention',
|
||||
remote_id TEXT,
|
||||
FOREIGN KEY (target_note_id) REFERENCES notes(id) ON DELETE SET NULL,
|
||||
UNIQUE(source_url, target_url)
|
||||
);
|
||||
|
||||
INSERT INTO webmentions_new (
|
||||
id, source_url, target_url, target_note_id, mention_type, author_name, author_url,
|
||||
author_photo, content_snippet, validation_status, moderation_status, validation_error,
|
||||
validated_at, received_at, updated_at, source_post_url, source_published
|
||||
)
|
||||
SELECT
|
||||
id, source_url, target_url, target_note_id, mention_type, author_name, author_url,
|
||||
author_photo, content_snippet, validation_status, moderation_status, validation_error,
|
||||
validated_at, received_at, updated_at, source_post_url, source_published
|
||||
FROM webmentions;
|
||||
|
||||
DROP TABLE webmentions;
|
||||
|
||||
ALTER TABLE webmentions_new RENAME TO webmentions;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_webmentions_target_note ON webmentions(target_note_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_webmentions_moderation ON webmentions(moderation_status);
|
||||
CREATE INDEX IF NOT EXISTS idx_webmentions_validation ON webmentions(validation_status);
|
||||
CREATE INDEX IF NOT EXISTS idx_webmentions_received ON webmentions(received_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_webmentions_note_origin ON webmentions(target_note_id, origin);
|
||||
|
||||
ALTER TABLE syndications ADD COLUMN backfeed_next_at TIMESTAMP;
|
||||
ALTER TABLE syndications ADD COLUMN backfeed_checked_at TIMESTAMP;
|
||||
ALTER TABLE syndications ADD COLUMN backfeed_claimed_until TIMESTAMP;
|
||||
ALTER TABLE syndications ADD COLUMN backfeed_attempts INTEGER NOT NULL DEFAULT 0;
|
||||
ALTER TABLE syndications ADD COLUMN backfeed_error TEXT;
|
||||
ALTER TABLE syndications ADD COLUMN backfeed_backfill INTEGER NOT NULL DEFAULT 0;
|
||||
ALTER TABLE syndications ADD COLUMN likes_count INTEGER;
|
||||
ALTER TABLE syndications ADD COLUMN reposts_count INTEGER;
|
||||
ALTER TABLE syndications ADD COLUMN quotes_count INTEGER;
|
||||
@@ -0,0 +1,6 @@
|
||||
-- Migration 021: Notifications (v2.14.0, ADR-074)
|
||||
-- When the owner was notified of a response. Everything already stored
|
||||
-- counts as notified, so turning notifications on doesn't send history.
|
||||
|
||||
ALTER TABLE webmentions ADD COLUMN notified_at TIMESTAMP;
|
||||
UPDATE webmentions SET notified_at = CURRENT_TIMESTAMP;
|
||||
+7
-15
@@ -1,17 +1,9 @@
|
||||
# StarPunk Development Dependencies
|
||||
# Includes code quality and testing tools
|
||||
|
||||
# Include production dependencies
|
||||
# StarPunk development dependencies
|
||||
-r requirements.txt
|
||||
-r requirements-test.txt
|
||||
|
||||
# Testing
|
||||
pytest-cov>=5.0.0 # Test coverage reporting
|
||||
pytest-mock>=3.12.0 # Mocking for tests
|
||||
|
||||
# Code Quality
|
||||
black>=24.0.0 # Code formatting
|
||||
flake8>=7.0.0 # Linting
|
||||
mypy>=1.8.0 # Type checking
|
||||
|
||||
# WSGI Server (for production-like testing)
|
||||
gunicorn>=21.2.0
|
||||
pytest-cov==7.* # Coverage reports
|
||||
black==26.* # Formatting (not enforced, see docs/standards)
|
||||
flake8==7.* # Linting (not enforced)
|
||||
mypy==1.* # Type checking (not enforced)
|
||||
pip-audit==2.* # Known-vulnerability check of the dependency set
|
||||
@@ -0,0 +1,3 @@
|
||||
# Test dependencies: used by the Containerfile test stage and CI (v2.11.0)
|
||||
pytest==9.2.*
|
||||
pytest-mock==3.15.*
|
||||
+21
-36
@@ -1,42 +1,27 @@
|
||||
# StarPunk Production Dependencies
|
||||
# Python 3.11+ required
|
||||
# StarPunk runtime dependencies (v2.11.0)
|
||||
# Pinned to a minor series; see docs/architecture/technology-stack.md.
|
||||
|
||||
# Web Framework
|
||||
Flask==3.0.*
|
||||
# Web framework and server
|
||||
Flask==3.1.*
|
||||
gunicorn==26.2.*
|
||||
|
||||
# WSGI Server (Production)
|
||||
gunicorn==21.2.*
|
||||
|
||||
# Content Processing
|
||||
markdown==3.5.*
|
||||
|
||||
# Feed Generation
|
||||
# Content
|
||||
markdown==3.11.*
|
||||
nh3==0.3.*
|
||||
beautifulsoup4==4.15.*
|
||||
lxml==6.1.*
|
||||
mf2py==2.0.*
|
||||
python-dateutil==4.0.*
|
||||
feedgen==1.0.*
|
||||
|
||||
# HTTP Client (for IndieAuth)
|
||||
httpx==0.27.*
|
||||
# HTTP and auth
|
||||
httpx==0.28.*
|
||||
PyJWT[crypto]==2.15.*
|
||||
python-dotenv==1.2.*
|
||||
|
||||
# Configuration Management
|
||||
python-dotenv==1.0.*
|
||||
# Notifications (v2.14.0, ADR-074)
|
||||
apprise==2.2.*
|
||||
|
||||
# HTML Parsing (for IndieAuth endpoint discovery)
|
||||
beautifulsoup4==4.12.*
|
||||
|
||||
# Microformats2 Parsing (v1.2.0)
|
||||
mf2py==2.0.*
|
||||
|
||||
# Testing Framework
|
||||
pytest==8.0.*
|
||||
|
||||
# System Monitoring (v1.1.2)
|
||||
psutil==5.9.*
|
||||
|
||||
# Image Processing (v1.2.0, updated v1.4.2 for HEIC support)
|
||||
Pillow==10.1.*
|
||||
|
||||
# HEIC/HEIF Support (v1.4.2 - iPhone photos)
|
||||
pillow-heif==0.18.*
|
||||
|
||||
# HTML Sanitization (v1.6.0 - Webmention content)
|
||||
nh3==0.2.*
|
||||
PyJWT[crypto]==2.10.*
|
||||
# Images
|
||||
Pillow==13.0.*
|
||||
pillow-heif==1.8.*
|
||||
+14
-19
@@ -142,12 +142,6 @@ def create_app(config=None):
|
||||
# Initialize connection pool
|
||||
init_pool(app)
|
||||
|
||||
# Setup HTTP metrics middleware (v1.1.2 Phase 1)
|
||||
if app.config.get('METRICS_ENABLED', True):
|
||||
from starpunk.monitoring import setup_http_metrics
|
||||
setup_http_metrics(app)
|
||||
app.logger.info("HTTP metrics middleware enabled")
|
||||
|
||||
# Initialize feed cache (v1.1.2 Phase 3)
|
||||
if app.config.get('FEED_CACHE_ENABLED', True):
|
||||
from starpunk.feeds import configure_cache
|
||||
@@ -223,6 +217,14 @@ def create_app(config=None):
|
||||
from starpunk.utils import iso_utc
|
||||
app.add_template_filter(iso_utc, 'iso_utc')
|
||||
|
||||
# Template filter - local copy of a response author's avatar, or None (v2.13.0)
|
||||
@app.template_filter('local_avatar')
|
||||
def local_avatar_filter(url):
|
||||
from flask import url_for
|
||||
from starpunk.avatars import cached_name
|
||||
name = cached_name(app.config.get('DATA_PATH', 'data'), url)
|
||||
return url_for('public.avatar', name=name) if name else None
|
||||
|
||||
# Template filter - URL to color for facepile (v1.6.0)
|
||||
@app.template_filter('url_to_color')
|
||||
def url_to_color_filter(url):
|
||||
@@ -309,6 +311,10 @@ def create_app(config=None):
|
||||
response.headers.add('Link', '</webmention>; rel="webmention"')
|
||||
return response
|
||||
|
||||
# CSRF tokens on admin and consent forms (v2.11.0)
|
||||
from starpunk import csrf
|
||||
csrf.init_app(app)
|
||||
|
||||
# WebSub discovery: Link rel=hub and rel=self on topics (v2.10.0)
|
||||
from starpunk.websub import add_discovery_headers, template_links
|
||||
app.after_request(add_discovery_headers)
|
||||
@@ -319,17 +325,6 @@ def create_app(config=None):
|
||||
|
||||
register_error_handlers(app)
|
||||
|
||||
# Start memory monitor thread (v1.1.2 Phase 1)
|
||||
# Per CQ5: Skip in test mode
|
||||
if app.config.get('METRICS_ENABLED', True) and not app.config.get('TESTING', False):
|
||||
from starpunk.monitoring import MemoryMonitor
|
||||
memory_monitor = MemoryMonitor(interval=app.config.get('METRICS_MEMORY_INTERVAL', 30))
|
||||
memory_monitor.start()
|
||||
app.memory_monitor = memory_monitor
|
||||
app.logger.info(f"Memory monitor started (interval={memory_monitor.interval}s)")
|
||||
# No teardown_appcontext stop: that ran after every request and stopped
|
||||
# the monitor straight away (v2.10.0). It is a daemon thread.
|
||||
|
||||
# Start webmention worker thread (v1.6.0 Phase 2)
|
||||
# Per background-worker.md: Skip in test mode
|
||||
if app.config.get('WEBMENTION_WORKER_ENABLED', True) and not app.config.get('TESTING', False):
|
||||
@@ -470,5 +465,5 @@ def create_app(config=None):
|
||||
|
||||
# Package version (Semantic Versioning 2.0.0)
|
||||
# See docs/standards/versioning-strategy.md for details
|
||||
__version__ = "2.10.2"
|
||||
__version_info__ = (2, 10, 2)
|
||||
__version__ = "2.14.2"
|
||||
__version_info__ = (2, 14, 2)
|
||||
+3
-3
@@ -32,7 +32,7 @@ import base64
|
||||
import hashlib
|
||||
import secrets
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from functools import wraps
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
from urllib.parse import urlencode
|
||||
@@ -223,7 +223,7 @@ def initiate_login(next_url: Optional[str] = None) -> str:
|
||||
nonce,
|
||||
code_verifier,
|
||||
_safe_next_url(next_url),
|
||||
datetime.utcnow() + timedelta(minutes=LOGIN_STATE_MINUTES),
|
||||
datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(minutes=LOGIN_STATE_MINUTES),
|
||||
),
|
||||
)
|
||||
db.commit()
|
||||
@@ -361,7 +361,7 @@ def create_session(me: str) -> str:
|
||||
|
||||
# Calculate expiry (use configured session lifetime or default to 30 days)
|
||||
session_lifetime = current_app.config.get("SESSION_LIFETIME", 30)
|
||||
expires_at = datetime.utcnow() + timedelta(days=session_lifetime)
|
||||
expires_at = datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(days=session_lifetime)
|
||||
|
||||
current_app.logger.debug(f"Auth: Session expiry: {expires_at} ({session_lifetime} days)")
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@ Fallback Behavior (per Q14):
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Dict, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
@@ -236,7 +236,7 @@ def get_author_profile(me_url: str, refresh: bool = False) -> Dict:
|
||||
if cached:
|
||||
# Check if cache is still valid
|
||||
cached_until = datetime.fromisoformat(cached['cached_until'])
|
||||
if datetime.utcnow() < cached_until:
|
||||
if datetime.now(timezone.utc).replace(tzinfo=None) < cached_until:
|
||||
current_app.logger.debug(f"Using cached author profile for {me_url}")
|
||||
|
||||
# Parse rel_me_links from JSON
|
||||
@@ -326,7 +326,7 @@ def save_author_profile(me_url: str, profile: Dict) -> None:
|
||||
db = get_db(current_app)
|
||||
|
||||
# Calculate cache expiry (24 hours from now)
|
||||
cached_until = datetime.utcnow() + timedelta(hours=CACHE_TTL_HOURS)
|
||||
cached_until = datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(hours=CACHE_TTL_HOURS)
|
||||
|
||||
# Convert rel_me_links to JSON (per Q17)
|
||||
rel_me_json = json.dumps(profile.get('rel_me_links', []))
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
"""
|
||||
Local copies of response authors' avatars (v2.13.0, ADR-073 Decision 6)
|
||||
|
||||
Pages used to show other sites' avatar URLs directly, which tells those
|
||||
sites who is reading, and Bluesky and Threads avatar URLs change or expire.
|
||||
Each avatar is fetched once (SSRF-safe), shrunk to a 96 px WebP and saved
|
||||
as DATA_PATH/avatars/<sha256 of its URL>.webp. Pages show the saved copy,
|
||||
or the coloured initial until there is one.
|
||||
|
||||
Paths and URLs are passed in, so the webmention worker can use it. Fetching
|
||||
needs an app context (the SSRF-safe fetcher reads DEV_MODE).
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import logging
|
||||
import re
|
||||
import sqlite3
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SIZE = 96
|
||||
MAX_BYTES = 1024 * 1024
|
||||
# Bluesky avatar on its CDN: .../img/<kind>/plain/<did>/<cid>[@jpeg]
|
||||
BSKY_CDN_RE = re.compile(
|
||||
r"^https://cdn\.bsky\.app/img/[a-z_]+/plain/(did:(?:plc|web):[A-Za-z0-9._:%-]+)/([a-z2-7]+)(?:@\w+)?$")
|
||||
NAME_RE = re.compile(r"^[0-9a-f]{64}\.webp$")
|
||||
CLEANUP_EVERY = 7 * 24 * 3600
|
||||
|
||||
|
||||
def name_for(url: str) -> str:
|
||||
return hashlib.sha256(url.encode()).hexdigest() + ".webp"
|
||||
|
||||
|
||||
def folder(data_path) -> Path:
|
||||
return Path(data_path) / "avatars"
|
||||
|
||||
|
||||
def cached_name(data_path, url: Optional[str]) -> Optional[str]:
|
||||
"""File name of the saved copy of this avatar, if there is one"""
|
||||
if not url or not url.startswith(("http://", "https://")):
|
||||
return None
|
||||
name = name_for(url)
|
||||
return name if (folder(data_path) / name).is_file() else None
|
||||
|
||||
|
||||
def fetch(data_path, url: Optional[str], site_url: str) -> bool:
|
||||
"""
|
||||
Save a copy of an avatar unless there is one. Returns True if one exists
|
||||
afterwards. Failures are logged and the page keeps showing the initial.
|
||||
"""
|
||||
if not url or not url.startswith(("http://", "https://")):
|
||||
return False
|
||||
path = folder(data_path) / name_for(url)
|
||||
if path.is_file():
|
||||
return True
|
||||
from PIL import Image
|
||||
|
||||
try:
|
||||
with Image.open(io.BytesIO(_download(url, site_url))) as image:
|
||||
image.load()
|
||||
image = image.convert("RGBA" if "A" in image.getbands() else "RGB")
|
||||
side = min(image.size)
|
||||
left, top = (image.width - side) // 2, (image.height - side) // 2
|
||||
image = image.crop((left, top, left + side, top + side))
|
||||
image = image.resize((SIZE, SIZE), Image.LANCZOS)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_suffix(".tmp")
|
||||
image.save(tmp, "WEBP", quality=80)
|
||||
tmp.replace(path)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.info(f"Could not save avatar {url}: {type(e).__name__}: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def _own_instance_host() -> Optional[str]:
|
||||
"""Host of the GoToSocial server StarPunk posts to (it may resolve to an internal address)"""
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from flask import current_app
|
||||
return urlparse(current_app.config.get("GOTOSOCIAL_URL") or "").hostname
|
||||
|
||||
|
||||
def _get(url: str, headers: dict) -> bytes:
|
||||
"""GET a public URL through the SSRF-safe fetcher"""
|
||||
from starpunk.webmention_ssrf import safe_get
|
||||
|
||||
result = safe_get(url, headers=headers, max_bytes=MAX_BYTES, max_redirects=5,
|
||||
timeout=10.0, truncate=False)
|
||||
if result.status_code >= 400:
|
||||
raise ValueError(f"HTTP {result.status_code}")
|
||||
return result.content
|
||||
|
||||
|
||||
def _pds(did: str, headers: dict) -> str:
|
||||
"""The PDS that hosts a Bluesky account, from its DID document"""
|
||||
if did.startswith("did:plc:"):
|
||||
doc_url = f"https://plc.directory/{did}"
|
||||
else: # did:web:<host>
|
||||
doc_url = f"https://{did[len('did:web:'):]}/.well-known/did.json"
|
||||
import json
|
||||
|
||||
for service in json.loads(_get(doc_url, headers)).get("service") or []:
|
||||
if service.get("id", "").endswith("#atproto_pds") and service.get("serviceEndpoint"):
|
||||
return service["serviceEndpoint"].rstrip("/")
|
||||
raise ValueError(f"no PDS for {did}")
|
||||
|
||||
|
||||
def _download(url: str, site_url: str) -> bytes:
|
||||
"""
|
||||
The avatar's bytes (v2.13.1):
|
||||
- from StarPunk's own GoToSocial directly, as the crossposting client
|
||||
does: behind the same proxy it resolves to an internal address, which
|
||||
the SSRF-safe fetcher refuses;
|
||||
- Bluesky CDN avatars from the account's PDS (com.atproto.sync.getBlob),
|
||||
because the CDN refuses some servers' IPv4 addresses;
|
||||
- anything else through the SSRF-safe fetcher.
|
||||
"""
|
||||
from urllib.parse import quote, urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
from starpunk import webmention_ssrf
|
||||
from starpunk.webmention_ssrf import user_agent
|
||||
|
||||
headers = {"User-Agent": user_agent(site_url)}
|
||||
own = _own_instance_host()
|
||||
if own and urlparse(url).hostname == own:
|
||||
with httpx.Client(timeout=10.0, transport=webmention_ssrf.TRANSPORT) as http:
|
||||
with http.stream("GET", url, headers=headers) as response:
|
||||
if response.is_redirect:
|
||||
# GoToSocial sends media it hasn't cached to the original
|
||||
# server: that one is fetched like any outside URL (v2.13.2)
|
||||
from urllib.parse import urljoin
|
||||
return _get(urljoin(url, response.headers["location"]), headers)
|
||||
if response.status_code >= 400:
|
||||
raise ValueError(f"HTTP {response.status_code}")
|
||||
data = b""
|
||||
for chunk in response.iter_bytes():
|
||||
data += chunk
|
||||
if len(data) > MAX_BYTES:
|
||||
raise ValueError("too large")
|
||||
return data
|
||||
match = BSKY_CDN_RE.match(url)
|
||||
if match:
|
||||
did, cid = match.groups()
|
||||
url = (f"{_pds(did, headers)}/xrpc/com.atproto.sync.getBlob"
|
||||
f"?did={quote(did, safe=':')}&cid={cid}")
|
||||
return _get(url, headers)
|
||||
|
||||
|
||||
def fill_missing(db_path, data_path, site_url: str, limit: int = 20) -> int:
|
||||
"""Save avatars of shown responses that have no copy yet. Returns how many were tried."""
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
urls = [row[0] for row in conn.execute(
|
||||
"""
|
||||
SELECT DISTINCT author_photo FROM webmentions
|
||||
WHERE author_photo IS NOT NULL AND author_photo != ''
|
||||
AND validation_status = 'valid' AND moderation_status = 'approved'
|
||||
ORDER BY id DESC
|
||||
"""
|
||||
)]
|
||||
finally:
|
||||
conn.close()
|
||||
tried = 0
|
||||
for url in urls:
|
||||
if tried >= limit:
|
||||
break
|
||||
if cached_name(data_path, url) or not url.startswith(("http://", "https://")):
|
||||
continue
|
||||
fetch(data_path, url, site_url)
|
||||
tried += 1
|
||||
return tried
|
||||
|
||||
|
||||
def remove_unused(db_path, data_path) -> int:
|
||||
"""Delete saved avatars no stored response uses. Returns how many were removed."""
|
||||
directory = folder(data_path)
|
||||
if not directory.is_dir():
|
||||
return 0
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
used = {name_for(row[0]) for row in conn.execute(
|
||||
"SELECT DISTINCT author_photo FROM webmentions WHERE author_photo IS NOT NULL")}
|
||||
finally:
|
||||
conn.close()
|
||||
removed = 0
|
||||
for path in directory.iterdir():
|
||||
if NAME_RE.match(path.name) and path.name not in used:
|
||||
path.unlink(missing_ok=True)
|
||||
removed += 1
|
||||
return removed
|
||||
|
||||
|
||||
class Upkeep:
|
||||
"""Runs fill_missing every `fill_every` seconds and remove_unused weekly"""
|
||||
|
||||
def __init__(self, fill_every: int = 600):
|
||||
self.fill_every = fill_every
|
||||
self._filled_at = 0.0
|
||||
self._cleaned_at = 0.0
|
||||
|
||||
def run(self, db_path, data_path, site_url: str) -> None:
|
||||
now = time.time()
|
||||
if now - self._filled_at >= self.fill_every:
|
||||
self._filled_at = now
|
||||
fill_missing(db_path, data_path, site_url)
|
||||
if now - self._cleaned_at >= CLEANUP_EVERY:
|
||||
self._cleaned_at = now
|
||||
removed = remove_unused(db_path, data_path)
|
||||
if removed:
|
||||
logger.info(f"Removed {removed} unused avatars")
|
||||
@@ -0,0 +1,485 @@
|
||||
"""
|
||||
Backfeed of responses to crossposts (v2.13.0, ADR-073)
|
||||
|
||||
StarPunk asks each silo for the responses to its own copies of a note
|
||||
(likes, reposts, quotes and replies) and stores them in `webmentions`, so
|
||||
moderation, the Responses section and re-sending webmentions when a note's
|
||||
responses change (spec 3.1.4) work as they do for received webmentions.
|
||||
|
||||
Each synced copy of a published note is checked on a schedule that slows as
|
||||
the note ages, and not at all after BACKFEED_MAX_AGE_DAYS. A backfill
|
||||
checks every copy once, whatever its age. Checks run in the syndication
|
||||
worker's loop (`process_due`), one copy at a time under a lease.
|
||||
|
||||
A check that read the whole list marks responses that are no longer there
|
||||
as invalid, so withdrawn likes and deleted replies drop off the page.
|
||||
"""
|
||||
|
||||
import html
|
||||
import random
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from flask import current_app
|
||||
|
||||
from starpunk.database import get_db
|
||||
|
||||
DAY = 24 * 3600
|
||||
SCHEDULE = [ # (note younger than, check every), in seconds
|
||||
(DAY, 15 * 60),
|
||||
(7 * DAY, 3600),
|
||||
(30 * DAY, DAY),
|
||||
]
|
||||
LATE_INTERVAL = 7 * DAY # from 30 days until the cutoff
|
||||
CLAIM_MINUTES = 5
|
||||
MAX_ATTEMPTS = 8
|
||||
BACKFILL_GAP = 5 # seconds between backfill checks, across workers
|
||||
SILO_NAMES = {"gotosocial": "GoToSocial", "bluesky": "Bluesky", "threads": "Threads"}
|
||||
OPT_OUT_RE = re.compile(r"#no(bridge|bot)\b", re.IGNORECASE)
|
||||
|
||||
# Silos whose credential was refused this run: target id -> message
|
||||
denied: Dict[str, str] = {}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Response:
|
||||
"""One like, repost, quote or reply on a silo"""
|
||||
kind: str # like, repost, quote, reply
|
||||
source_url: str # its URL (likes/reposts: the copy + #liked-by-<id>)
|
||||
remote_id: str # silo id of the response, or of the actor
|
||||
author_id: str = ""
|
||||
author_name: str = ""
|
||||
author_url: str = ""
|
||||
author_photo: Optional[str] = None
|
||||
author_bio: str = ""
|
||||
opted_out: bool = False # set by the client from silo-specific signals
|
||||
content_html: Optional[str] = None
|
||||
published: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Harvest:
|
||||
"""What one check found. complete=False: some list was cut short."""
|
||||
responses: List[Response]
|
||||
complete: bool = True
|
||||
# Likes/reposts/quotes shown only as numbers: all of them on Threads
|
||||
# (its API doesn't say who); elsewhere, those by people who opted out
|
||||
counts: Optional[Dict[str, int]] = None
|
||||
|
||||
|
||||
# --- Text -----------------------------------------------------------------
|
||||
|
||||
def text_html(text: str) -> str:
|
||||
"""Plain text as HTML: escaped, URLs linked, line breaks kept"""
|
||||
escaped = html.escape(text)
|
||||
linked = re.sub(r"https?://[^\s<>&\"]+",
|
||||
lambda m: f'<a href="{m.group(0)}">{m.group(0)}</a>', escaped)
|
||||
return "<p>" + linked.replace("\n", "<br>") + "</p>" if text.strip() else ""
|
||||
|
||||
|
||||
def facets_html(text: str, facets: list) -> str:
|
||||
"""Bluesky post text as HTML, with its link facets (byte ranges) as links"""
|
||||
data = text.encode()
|
||||
links = []
|
||||
for facet in facets:
|
||||
index = facet.get("index") or {}
|
||||
for feature in facet.get("features") or []:
|
||||
uri = feature.get("uri") or ""
|
||||
if feature.get("$type") == "app.bsky.richtext.facet#link" and uri.startswith(("http://", "https://")):
|
||||
links.append((index.get("byteStart", 0), index.get("byteEnd", 0), uri))
|
||||
links.sort()
|
||||
parts, pos = [], 0
|
||||
for start, end, uri in links:
|
||||
if start < pos or end > len(data) or start >= end:
|
||||
continue
|
||||
parts.append(html.escape(data[pos:start].decode(errors="ignore")))
|
||||
label = html.escape(data[start:end].decode(errors="ignore"))
|
||||
parts.append(f'<a href="{html.escape(uri, quote=True)}">{label}</a>')
|
||||
pos = end
|
||||
parts.append(html.escape(data[pos:].decode(errors="ignore")))
|
||||
body = "".join(parts)
|
||||
return "<p>" + body.replace("\n", "<br>") + "</p>" if text.strip() else ""
|
||||
|
||||
|
||||
# --- Scheduling --------------------------------------------------------------
|
||||
|
||||
def interval_for(age_seconds: float, max_age_days: int) -> Optional[int]:
|
||||
"""Seconds until a note of this age is checked again; None past the cutoff"""
|
||||
if age_seconds >= max_age_days * DAY:
|
||||
return None
|
||||
for younger_than, every in SCHEDULE:
|
||||
if age_seconds < younger_than:
|
||||
return every
|
||||
return LATE_INTERVAL
|
||||
|
||||
|
||||
def _age_seconds(created_at) -> float:
|
||||
if isinstance(created_at, str):
|
||||
created_at = datetime.fromisoformat(created_at.replace("Z", "+00:00"))
|
||||
if created_at.tzinfo is None:
|
||||
created_at = created_at.replace(tzinfo=timezone.utc)
|
||||
return (datetime.now(timezone.utc) - created_at).total_seconds()
|
||||
|
||||
|
||||
_ELIGIBLE = """
|
||||
FROM syndications s JOIN notes n ON n.id = s.note_id
|
||||
WHERE s.state = 'synced' AND s.enabled = 1 AND s.remote_id IS NOT NULL
|
||||
AND n.published = 1 AND n.deleted_at IS NULL
|
||||
AND n.like_of IS NULL AND n.repost_of IS NULL
|
||||
"""
|
||||
|
||||
|
||||
def _claim(db, row_id: int) -> bool:
|
||||
cursor = db.execute(
|
||||
f"""
|
||||
UPDATE syndications SET backfeed_claimed_until = datetime('now', '+{CLAIM_MINUTES} minutes')
|
||||
WHERE id = ? AND (backfeed_claimed_until IS NULL OR backfeed_claimed_until < datetime('now'))
|
||||
""",
|
||||
(row_id,),
|
||||
)
|
||||
db.commit()
|
||||
return cursor.rowcount == 1
|
||||
|
||||
|
||||
def _backfill_turn(db) -> bool:
|
||||
"""Take the next backfill slot: at most one check every BACKFILL_GAP seconds"""
|
||||
cursor = db.execute(
|
||||
f"""
|
||||
UPDATE site_state SET value = datetime('now')
|
||||
WHERE key = 'backfeed_backfill_last' AND value <= datetime('now', '-{BACKFILL_GAP} seconds')
|
||||
"""
|
||||
)
|
||||
if cursor.rowcount == 0:
|
||||
cursor = db.execute(
|
||||
"INSERT OR IGNORE INTO site_state (key, value) VALUES ('backfeed_backfill_last', datetime('now'))")
|
||||
db.commit()
|
||||
return cursor.rowcount == 1
|
||||
|
||||
|
||||
def _due(db, limit: int) -> list:
|
||||
"""Routine checks due now; a backfill check only when none are"""
|
||||
max_age = int(current_app.config.get("BACKFEED_MAX_AGE_DAYS", 90))
|
||||
lease = "(s.backfeed_claimed_until IS NULL OR s.backfeed_claimed_until < datetime('now'))"
|
||||
rows = db.execute(
|
||||
f"""
|
||||
SELECT s.*, n.created_at AS note_created, n.slug AS note_slug {_ELIGIBLE}
|
||||
AND {lease}
|
||||
AND n.created_at >= datetime('now', ?)
|
||||
AND (s.backfeed_next_at IS NULL OR s.backfeed_next_at <= datetime('now'))
|
||||
ORDER BY s.backfeed_next_at IS NOT NULL, s.backfeed_next_at LIMIT ?
|
||||
""",
|
||||
(f"-{max_age} days", limit),
|
||||
).fetchall()
|
||||
if rows:
|
||||
return rows
|
||||
backfill = db.execute(
|
||||
f"""
|
||||
SELECT s.*, n.created_at AS note_created, n.slug AS note_slug {_ELIGIBLE}
|
||||
AND {lease} AND s.backfeed_backfill = 1
|
||||
AND (s.backfeed_next_at IS NULL OR s.backfeed_next_at <= datetime('now'))
|
||||
ORDER BY s.id LIMIT 1
|
||||
"""
|
||||
).fetchall()
|
||||
if backfill and _backfill_turn(db):
|
||||
return backfill
|
||||
return []
|
||||
|
||||
|
||||
def _finish(db, row) -> None:
|
||||
max_age = int(current_app.config.get("BACKFEED_MAX_AGE_DAYS", 90))
|
||||
every = interval_for(_age_seconds(row["note_created"]), max_age)
|
||||
if every is None:
|
||||
next_at = None
|
||||
else:
|
||||
next_at = f"+{int(every * random.uniform(1.0, 1.1))} seconds"
|
||||
db.execute(
|
||||
"""
|
||||
UPDATE syndications
|
||||
SET backfeed_checked_at = datetime('now'), backfeed_claimed_until = NULL,
|
||||
backfeed_attempts = 0, backfeed_error = NULL, backfeed_backfill = 0,
|
||||
backfeed_next_at = CASE WHEN ? IS NULL THEN NULL ELSE datetime('now', ?) END
|
||||
WHERE id = ?
|
||||
""",
|
||||
(next_at, next_at, row["id"]),
|
||||
)
|
||||
db.commit()
|
||||
|
||||
|
||||
def _fail(db, row, message: str) -> None:
|
||||
"""Back off; after MAX_ATTEMPTS drop any backfill and try again in a month"""
|
||||
attempts = row["backfeed_attempts"] + 1
|
||||
stop = attempts >= MAX_ATTEMPTS
|
||||
delay = 30 * DAY if stop else min(30 * (2 ** attempts), 6 * 3600)
|
||||
db.execute(
|
||||
f"""
|
||||
UPDATE syndications
|
||||
SET backfeed_attempts = ?, backfeed_error = ?, backfeed_claimed_until = NULL,
|
||||
backfeed_backfill = CASE WHEN ? THEN 0 ELSE backfeed_backfill END,
|
||||
backfeed_next_at = datetime('now', '+{delay} seconds')
|
||||
WHERE id = ?
|
||||
""",
|
||||
(attempts, message[:500], 1 if stop else 0, row["id"]),
|
||||
)
|
||||
db.commit()
|
||||
current_app.logger.warning(
|
||||
f"Backfeed {row['target']} for note {row['note_id']} failed (attempt {attempts}): {message}")
|
||||
|
||||
|
||||
# --- Storing -----------------------------------------------------------------
|
||||
|
||||
def _opted_out(response: Response) -> bool:
|
||||
return response.opted_out or bool(OPT_OUT_RE.search(html.unescape(response.author_bio or "")))
|
||||
|
||||
|
||||
def _display_state(db, note_id: int) -> list:
|
||||
return [tuple(r) for r in db.execute(
|
||||
"""
|
||||
SELECT id, mention_type, author_name, author_url, author_photo, content_snippet
|
||||
FROM webmentions
|
||||
WHERE target_note_id = ? AND validation_status = 'valid' AND moderation_status = 'approved'
|
||||
ORDER BY id
|
||||
""",
|
||||
(note_id,),
|
||||
)]
|
||||
|
||||
|
||||
def hidden_counts(harvest: Harvest) -> Dict[str, int]:
|
||||
"""
|
||||
Likes, reposts and quotes to show only as numbers: the silo's own counts
|
||||
when it gives no names (Threads), else those by people who opted out
|
||||
(v2.13.3: they're counted, not named)
|
||||
"""
|
||||
if harvest.counts is not None:
|
||||
return harvest.counts
|
||||
hidden = [r.kind for r in harvest.responses if _opted_out(r)]
|
||||
return {k: hidden.count(kind) for k, kind in
|
||||
(("likes", "like"), ("reposts", "repost"), ("quotes", "quote"))}
|
||||
|
||||
|
||||
def store(db, note_id: int, target_url: str, origin: str, harvest: Harvest,
|
||||
quiet: bool = False) -> None:
|
||||
"""
|
||||
Upsert a check's responses for one note and silo. With a complete
|
||||
harvest, that silo's responses for the note that weren't found are
|
||||
marked invalid. Responses from people who opted out are removed (they
|
||||
are counted instead, see `hidden_counts`). With `quiet`, new responses
|
||||
are stored as already notified (ADR-074 Decision 5).
|
||||
"""
|
||||
from starpunk.webmention_processor import _snippet
|
||||
|
||||
moderation = "approved" if current_app.config.get("WEBMENTION_AUTO_APPROVE", True) else "pending"
|
||||
silo = SILO_NAMES.get(origin, origin)
|
||||
keep = [r for r in harvest.responses if not _opted_out(r)]
|
||||
dropped = [r.source_url for r in harvest.responses if _opted_out(r)]
|
||||
|
||||
from starpunk.avatars import cached_name
|
||||
|
||||
before = _display_state(db, note_id)
|
||||
for r in keep:
|
||||
snippet = _snippet(r.content_html) if r.content_html else None
|
||||
# Threads avatar URLs are signed and change on every request: keep
|
||||
# the stored URL while its saved copy exists, so it isn't refetched
|
||||
old = db.execute("SELECT author_photo FROM webmentions WHERE source_url = ? AND target_url = ?",
|
||||
(r.source_url, target_url)).fetchone()
|
||||
if old and old["author_photo"] and origin == "threads" \
|
||||
and cached_name(current_app.config["DATA_PATH"], old["author_photo"]):
|
||||
r.author_photo = old["author_photo"]
|
||||
db.execute(
|
||||
"""
|
||||
INSERT INTO webmentions (
|
||||
source_url, target_url, target_note_id, mention_type, author_name, author_url,
|
||||
author_photo, content_snippet, validation_status, moderation_status,
|
||||
validated_at, source_published, origin, remote_id, notified_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'valid', ?, CURRENT_TIMESTAMP, ?, ?, ?,
|
||||
CASE WHEN ? THEN CURRENT_TIMESTAMP END)
|
||||
ON CONFLICT(source_url, target_url) DO UPDATE SET
|
||||
target_note_id = excluded.target_note_id,
|
||||
mention_type = excluded.mention_type,
|
||||
author_name = excluded.author_name,
|
||||
author_url = excluded.author_url,
|
||||
author_photo = excluded.author_photo,
|
||||
content_snippet = excluded.content_snippet,
|
||||
source_published = COALESCE(excluded.source_published, webmentions.source_published),
|
||||
origin = excluded.origin,
|
||||
remote_id = excluded.remote_id,
|
||||
validation_status = 'valid',
|
||||
validation_error = NULL,
|
||||
validated_at = CURRENT_TIMESTAMP,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(r.source_url, target_url, note_id, r.kind, r.author_name or None, r.author_url or None,
|
||||
r.author_photo or None, snippet, moderation, r.published, origin, r.remote_id,
|
||||
1 if quiet else 0),
|
||||
)
|
||||
if dropped:
|
||||
marks = ",".join("?" * len(dropped))
|
||||
db.execute(f"DELETE FROM webmentions WHERE origin = ? AND target_note_id = ? AND source_url IN ({marks})",
|
||||
[origin, note_id] + dropped)
|
||||
if harvest.complete:
|
||||
found = [r.source_url for r in keep]
|
||||
marks = ",".join("?" * len(found)) or "''"
|
||||
db.execute(
|
||||
f"""
|
||||
UPDATE webmentions
|
||||
SET validation_status = 'invalid', validation_error = ?, updated_at = CURRENT_TIMESTAMP
|
||||
WHERE target_note_id = ? AND origin = ? AND validation_status = 'valid'
|
||||
AND source_url NOT IN ({marks})
|
||||
""",
|
||||
[f"No longer on {silo}", note_id, origin] + found,
|
||||
)
|
||||
db.commit()
|
||||
|
||||
if _display_state(db, note_id) != before:
|
||||
try:
|
||||
from pathlib import Path
|
||||
|
||||
from starpunk.webmention_sender import note_responses_changed
|
||||
note_responses_changed(Path(current_app.config["DATABASE_PATH"]), [note_id])
|
||||
except Exception as e:
|
||||
current_app.logger.warning(f"Failed to re-send webmentions for note {note_id}: {e}")
|
||||
|
||||
from starpunk import avatars
|
||||
for r in keep:
|
||||
avatars.fetch(current_app.config["DATA_PATH"], r.author_photo,
|
||||
current_app.config.get("SITE_URL", ""))
|
||||
|
||||
|
||||
def _store_counts(db, row_id: int, counts: Optional[dict]) -> None:
|
||||
if counts is None:
|
||||
return
|
||||
db.execute(
|
||||
"UPDATE syndications SET likes_count = ?, reposts_count = ?, quotes_count = ? WHERE id = ?",
|
||||
(counts.get("likes"), counts.get("reposts"), counts.get("quotes"), row_id),
|
||||
)
|
||||
db.commit()
|
||||
|
||||
|
||||
# --- Checking ---------------------------------------------------------------
|
||||
|
||||
def _is_denied(error) -> bool:
|
||||
"""The silo refused the credential (missing scope or bad token)"""
|
||||
from starpunk.syndication import _GRAPH_AUTH
|
||||
|
||||
return getattr(error, "status", None) in (401, 403) or getattr(error, "code", None) in _GRAPH_AUTH
|
||||
|
||||
|
||||
SCOPES = {
|
||||
"gotosocial": "a GoToSocial token with read:statuses and read:accounts (or read)",
|
||||
"bluesky": "a working Bluesky app password",
|
||||
"threads": "a Threads token with threads_read_replies and threads_manage_insights",
|
||||
}
|
||||
|
||||
|
||||
def check(row, target) -> None:
|
||||
"""Check one copy now and store what it found (raises SyndicationError)"""
|
||||
db = get_db(current_app)
|
||||
site = current_app.config["SITE_URL"].rstrip("/")
|
||||
harvest = target.client.responses(row["remote_id"], row["remote_url"] or "")
|
||||
# No notifications for history: a backfill, or a copy's first check on an
|
||||
# older note (e.g. one linked after the fact). ADR-074 Decision 5.
|
||||
quiet = bool(row["backfeed_backfill"]) or (
|
||||
row["backfeed_checked_at"] is None and _age_seconds(row["note_created"]) > DAY)
|
||||
store(db, row["note_id"], f"{site}/note/{row['note_slug']}", row["target"], harvest, quiet=quiet)
|
||||
_store_counts(db, row["id"], hidden_counts(harvest))
|
||||
|
||||
|
||||
def process_due(limit: int = 3) -> int:
|
||||
"""Check up to `limit` copies that are due. Returns how many were handled."""
|
||||
if not current_app.config.get("BACKFEED_ENABLED", False):
|
||||
return 0
|
||||
from starpunk.syndication import SyndicationError, get_targets
|
||||
|
||||
targets = get_targets()
|
||||
db = get_db(current_app)
|
||||
handled = 0
|
||||
for row in _due(db, limit):
|
||||
if row["target"] in denied or row["target"] not in targets:
|
||||
continue
|
||||
if not _claim(db, row["id"]):
|
||||
continue
|
||||
handled += 1
|
||||
try:
|
||||
check(row, targets[row["target"]])
|
||||
except SyndicationError as e:
|
||||
if _is_denied(e):
|
||||
denied[row["target"]] = f"Needs {SCOPES.get(row['target'], 'read access')}: {e}"
|
||||
current_app.logger.error(f"Backfeed for {row['target']} stopped: {denied[row['target']]}")
|
||||
from starpunk import notifications
|
||||
site = current_app.config["SITE_URL"].rstrip("/")
|
||||
notifications.problem(current_app.config, current_app.config["DATABASE_PATH"],
|
||||
f"Backfeed for {SILO_NAMES.get(row['target'], row['target'])} stopped",
|
||||
denied[row["target"]], f"{site}/admin/webmentions")
|
||||
db.execute("UPDATE syndications SET backfeed_claimed_until = NULL WHERE id = ?", (row["id"],))
|
||||
db.commit()
|
||||
continue
|
||||
_fail(db, row, str(e))
|
||||
continue
|
||||
except Exception as e: # one bad copy mustn't stop the worker
|
||||
_fail(db, row, f"unexpected error: {e}")
|
||||
continue
|
||||
_finish(db, row)
|
||||
_note_backfill_done(db)
|
||||
return handled
|
||||
|
||||
|
||||
# --- Backfill and admin ---------------------------------------------------------
|
||||
|
||||
def _state(db, key: str) -> Optional[str]:
|
||||
row = db.execute("SELECT value FROM site_state WHERE key = ?", (key,)).fetchone()
|
||||
return row["value"] if row and row["value"] else None
|
||||
|
||||
|
||||
def _set_state(db, key: str, value: str) -> None:
|
||||
db.execute("INSERT INTO site_state (key, value) VALUES (?, ?) "
|
||||
"ON CONFLICT(key) DO UPDATE SET value = excluded.value", (key, value))
|
||||
|
||||
|
||||
def start_backfill() -> int:
|
||||
"""Mark every eligible copy for one check, whatever its age. Returns how many."""
|
||||
db = get_db(current_app)
|
||||
cursor = db.execute(
|
||||
f"""
|
||||
UPDATE syndications
|
||||
SET backfeed_backfill = 1, backfeed_attempts = 0, backfeed_next_at = datetime('now')
|
||||
WHERE id IN (SELECT s.id {_ELIGIBLE})
|
||||
"""
|
||||
)
|
||||
now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
|
||||
_set_state(db, "backfeed_backfill_started", now)
|
||||
_set_state(db, "backfeed_backfill_finished", "")
|
||||
db.commit()
|
||||
return cursor.rowcount
|
||||
|
||||
|
||||
def _note_backfill_done(db) -> None:
|
||||
if _state(db, "backfeed_backfill_started") and not _state(db, "backfeed_backfill_finished"):
|
||||
left = db.execute("SELECT COUNT(*) FROM syndications WHERE backfeed_backfill = 1").fetchone()[0]
|
||||
if left == 0:
|
||||
_set_state(db, "backfeed_backfill_finished",
|
||||
datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S"))
|
||||
db.commit()
|
||||
|
||||
|
||||
def check_now(syndication_id: int) -> None:
|
||||
"""Check this copy's responses at the next worker pass"""
|
||||
db = get_db(current_app)
|
||||
db.execute("UPDATE syndications SET backfeed_next_at = datetime('now'), backfeed_attempts = 0 "
|
||||
"WHERE id = ?", (syndication_id,))
|
||||
db.commit()
|
||||
|
||||
|
||||
def status() -> dict:
|
||||
"""For the admin pages: backfill progress and silos whose credential was refused"""
|
||||
db = get_db(current_app)
|
||||
return {
|
||||
"enabled": bool(current_app.config.get("BACKFEED_ENABLED", False)),
|
||||
"started": _state(db, "backfeed_backfill_started"),
|
||||
"finished": _state(db, "backfeed_backfill_finished"),
|
||||
"remaining": db.execute(
|
||||
"SELECT COUNT(*) FROM syndications WHERE backfeed_backfill = 1").fetchone()[0],
|
||||
"denied": dict(denied),
|
||||
"errors": db.execute(
|
||||
"SELECT COUNT(*) FROM syndications WHERE backfeed_error IS NOT NULL").fetchone()[0],
|
||||
}
|
||||
+17
-9
@@ -71,8 +71,10 @@ def load_config(app, config_override=None):
|
||||
app.config["DATABASE_PATH"] = Path(os.getenv("DATABASE_PATH", "./data/starpunk.db"))
|
||||
|
||||
# Flask environment
|
||||
app.config["ENV"] = os.getenv("FLASK_ENV", "development")
|
||||
app.config["DEBUG"] = os.getenv("FLASK_DEBUG", "1") == "1"
|
||||
# Safe defaults (v2.11.0): production, debug off. `flask run --debug`
|
||||
# sets FLASK_DEBUG=1 for local development.
|
||||
app.config["ENV"] = os.getenv("FLASK_ENV", "production")
|
||||
app.config["DEBUG"] = os.getenv("FLASK_DEBUG", "0") == "1"
|
||||
|
||||
# Logging
|
||||
app.config["LOG_LEVEL"] = os.getenv("LOG_LEVEL", "INFO")
|
||||
@@ -97,13 +99,6 @@ def load_config(app, config_override=None):
|
||||
# Flask MAX_CONTENT_LENGTH limits request body size (matches media.py MAX_FILE_SIZE)
|
||||
app.config["MAX_CONTENT_LENGTH"] = 50 * 1024 * 1024 # 50MB
|
||||
|
||||
# Metrics configuration (v1.1.2 Phase 1)
|
||||
app.config["METRICS_ENABLED"] = os.getenv("METRICS_ENABLED", "true").lower() == "true"
|
||||
app.config["METRICS_SLOW_QUERY_THRESHOLD"] = float(os.getenv("METRICS_SLOW_QUERY_THRESHOLD", "1.0"))
|
||||
app.config["METRICS_SAMPLING_RATE"] = float(os.getenv("METRICS_SAMPLING_RATE", "1.0"))
|
||||
app.config["METRICS_BUFFER_SIZE"] = int(os.getenv("METRICS_BUFFER_SIZE", "1000"))
|
||||
app.config["METRICS_MEMORY_INTERVAL"] = int(os.getenv("METRICS_MEMORY_INTERVAL", "30"))
|
||||
|
||||
# Debug file configuration (v1.5.0 Phase 2)
|
||||
app.config["DEBUG_SAVE_FAILED_UPLOADS"] = os.getenv("DEBUG_SAVE_FAILED_UPLOADS", "false").lower() == "true"
|
||||
app.config["DEBUG_FILE_MAX_AGE_DAYS"] = int(os.getenv("DEBUG_FILE_MAX_AGE_DAYS", "7"))
|
||||
@@ -151,10 +146,23 @@ def load_config(app, config_override=None):
|
||||
app.config["THREADS_MAX_CHARS"] = int(os.getenv("THREADS_MAX_CHARS", "500"))
|
||||
app.config["THREADS_MAX_MEDIA"] = int(os.getenv("THREADS_MAX_MEDIA", "4"))
|
||||
app.config["SYNDICATION_POLL_INTERVAL"] = int(os.getenv("SYNDICATION_POLL_INTERVAL", "10"))
|
||||
# Backfeed of responses to crossposts (v2.13.0, ADR-073)
|
||||
app.config["BACKFEED_ENABLED"] = os.getenv("BACKFEED_ENABLED", "false").lower() == "true"
|
||||
app.config["BACKFEED_MAX_AGE_DAYS"] = int(os.getenv("BACKFEED_MAX_AGE_DAYS", "90"))
|
||||
# Notifications through Apprise (v2.14.0, ADR-074)
|
||||
app.config["NOTIFY_URLS"] = os.getenv("NOTIFY_URLS", "")
|
||||
app.config["NOTIFY_EVENTS"] = os.getenv("NOTIFY_EVENTS", "reply,mention,quote,problems")
|
||||
app.config["NOTIFY_PRIORITY"] = os.getenv("NOTIFY_PRIORITY", "normal")
|
||||
|
||||
# Apply overrides if provided
|
||||
# CSRF protection for admin and consent forms (v2.11.0); on by default.
|
||||
# Tests turn it off unless they set CSRF_ENABLED themselves.
|
||||
app.config["CSRF_ENABLED"] = os.getenv("CSRF_ENABLED", "true").lower() == "true"
|
||||
|
||||
if config_override:
|
||||
app.config.update(config_override)
|
||||
if app.config.get("TESTING") and "CSRF_ENABLED" not in config_override:
|
||||
app.config["CSRF_ENABLED"] = False
|
||||
|
||||
# Normalize SITE_URL trailing slash (in case override provided URL without slash)
|
||||
if "SITE_URL" in app.config:
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
"""
|
||||
CSRF protection for the admin and consent forms (v2.11.0)
|
||||
|
||||
The token is an HMAC of the login session cookie under the app secret: no
|
||||
storage, tied to the session, and different after every login. Forms carry
|
||||
it as a hidden `csrf_token` field (template global `csrf_token()`); POSTs to
|
||||
protected endpoints without the right token get 400.
|
||||
|
||||
Protected: every admin POST, logout, and the IndieAuth consent ("approve").
|
||||
Endpoints used by apps with bearer tokens (Micropub, token, revoke,
|
||||
Webmention, WebSub) are not: they have no cookie-based authority.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
|
||||
from flask import abort, current_app, request
|
||||
|
||||
SESSION_COOKIE = "starpunk_session"
|
||||
FIELD = "csrf_token"
|
||||
HEADER = "X-CSRF-Token"
|
||||
PROTECTED_ENDPOINTS = {"auth.logout", "indieauth.approve"}
|
||||
|
||||
|
||||
def csrf_token() -> str:
|
||||
"""The CSRF token for the current login session ('' when not logged in)"""
|
||||
session = request.cookies.get(SESSION_COOKIE)
|
||||
if not session:
|
||||
return ""
|
||||
key = str(current_app.config["SECRET_KEY"]).encode("utf-8")
|
||||
return hmac.new(key, b"csrf:" + session.encode("utf-8"), hashlib.sha256).hexdigest()
|
||||
|
||||
|
||||
def _protected() -> bool:
|
||||
return request.blueprint == "admin" or request.endpoint in PROTECTED_ENDPOINTS
|
||||
|
||||
|
||||
def check_csrf():
|
||||
"""before_request: refuse state-changing requests without a valid token"""
|
||||
if request.method in ("GET", "HEAD", "OPTIONS") or not current_app.config.get("CSRF_ENABLED", True):
|
||||
return None
|
||||
if not _protected():
|
||||
return None
|
||||
expected = csrf_token()
|
||||
if not expected:
|
||||
return None # not logged in: the route's own auth check refuses it
|
||||
sent = request.form.get(FIELD) or request.headers.get(HEADER) or ""
|
||||
if not hmac.compare_digest(sent, expected):
|
||||
current_app.logger.warning(f"CSRF token missing or wrong for {request.endpoint}")
|
||||
abort(400, description="The form expired or came from another site. Reload the page and try again.")
|
||||
return None
|
||||
|
||||
|
||||
def init_app(app) -> None:
|
||||
app.before_request(check_csrf)
|
||||
app.add_template_global(csrf_token, "csrf_token")
|
||||
@@ -6,11 +6,19 @@ Provides database initialization and connection pooling
|
||||
Per v1.1.1 Phase 1:
|
||||
- Connection pooling for improved performance (ADR-053)
|
||||
- Request-scoped connections via Flask's g object
|
||||
- Pool statistics for monitoring
|
||||
- Pool statistics
|
||||
"""
|
||||
|
||||
from starpunk.database.init import init_db
|
||||
from starpunk.database.pool import init_pool, get_db, get_pool_stats
|
||||
from starpunk.database.schema import INITIAL_SCHEMA_SQL
|
||||
|
||||
# Explicit datetime adapters (v2.11.0). Python 3.12 deprecated sqlite3's
|
||||
# built-in ones; these store exactly what they did ("YYYY-MM-DD HH:MM:SS[.ffffff]").
|
||||
import datetime as _datetime
|
||||
import sqlite3 as _sqlite3
|
||||
|
||||
_sqlite3.register_adapter(_datetime.datetime, lambda value: value.isoformat(" "))
|
||||
_sqlite3.register_adapter(_datetime.date, lambda value: value.isoformat())
|
||||
|
||||
__all__ = ['init_db', 'init_pool', 'get_db', 'get_pool_stats', 'INITIAL_SCHEMA_SQL']
|
||||
@@ -5,7 +5,7 @@ Per ADR-053 and developer Q&A Q2, CQ1:
|
||||
- Provides connection pooling for improved performance
|
||||
- Integrates with Flask's g object for request-scoped connections
|
||||
- Maintains same interface as get_db() for transparency
|
||||
- Pool statistics available for metrics
|
||||
- Pool statistics (shown in /admin/health)
|
||||
- Wraps connections with MonitoredConnection for timing (v1.1.2 Phase 1)
|
||||
|
||||
Note: Migrations use direct connections (not pooled) for isolation
|
||||
@@ -27,7 +27,7 @@ class ConnectionPool:
|
||||
but this provides connection reuse and request-scoped connection management.
|
||||
"""
|
||||
|
||||
def __init__(self, db_path, pool_size=5, timeout=10.0, slow_query_threshold=1.0, metrics_enabled=True):
|
||||
def __init__(self, db_path, pool_size=5, timeout=10.0):
|
||||
"""
|
||||
Initialize connection pool
|
||||
|
||||
@@ -35,14 +35,10 @@ class ConnectionPool:
|
||||
db_path: Path to SQLite database file
|
||||
pool_size: Maximum number of connections in pool
|
||||
timeout: Timeout for getting connection (seconds)
|
||||
slow_query_threshold: Threshold in seconds for slow query detection (v1.1.2)
|
||||
metrics_enabled: Whether to enable metrics collection (v1.1.2)
|
||||
"""
|
||||
self.db_path = Path(db_path)
|
||||
self.pool_size = pool_size
|
||||
self.timeout = timeout
|
||||
self.slow_query_threshold = slow_query_threshold
|
||||
self.metrics_enabled = metrics_enabled
|
||||
self._pool = deque(maxlen=pool_size)
|
||||
self._lock = Lock()
|
||||
self._stats = {
|
||||
@@ -56,8 +52,6 @@ class ConnectionPool:
|
||||
def _create_connection(self):
|
||||
"""
|
||||
Create a new database connection
|
||||
|
||||
Per CQ1: Wraps connection with MonitoredConnection if metrics enabled
|
||||
"""
|
||||
conn = sqlite3.connect(
|
||||
self.db_path,
|
||||
@@ -68,14 +62,11 @@ class ConnectionPool:
|
||||
|
||||
# Enable WAL mode for better concurrency
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
# Enforce foreign keys, so ON DELETE CASCADE / SET NULL actually run
|
||||
# (SQLite leaves them off per connection; v2.11.0)
|
||||
conn.execute("PRAGMA foreign_keys=ON")
|
||||
|
||||
self._stats['connections_created'] += 1
|
||||
|
||||
# Wrap with monitoring if enabled (v1.1.2 Phase 1)
|
||||
if self.metrics_enabled:
|
||||
from starpunk.monitoring import MonitoredConnection
|
||||
return MonitoredConnection(conn, self.slow_query_threshold)
|
||||
|
||||
return conn
|
||||
|
||||
def get_connection(self):
|
||||
@@ -168,8 +159,6 @@ def init_pool(app):
|
||||
"""
|
||||
Initialize the connection pool
|
||||
|
||||
Per CQ2: Passes metrics configuration from app config
|
||||
|
||||
Args:
|
||||
app: Flask application instance
|
||||
"""
|
||||
@@ -178,20 +167,8 @@ def init_pool(app):
|
||||
db_path = app.config['DATABASE_PATH']
|
||||
pool_size = app.config.get('DB_POOL_SIZE', 5)
|
||||
timeout = app.config.get('DB_TIMEOUT', 10.0)
|
||||
slow_query_threshold = app.config.get('METRICS_SLOW_QUERY_THRESHOLD', 1.0)
|
||||
metrics_enabled = app.config.get('METRICS_ENABLED', True)
|
||||
|
||||
_pool = ConnectionPool(
|
||||
db_path,
|
||||
pool_size,
|
||||
timeout,
|
||||
slow_query_threshold,
|
||||
metrics_enabled
|
||||
)
|
||||
app.logger.info(
|
||||
f"Database connection pool initialized "
|
||||
f"(size={pool_size}, metrics={'enabled' if metrics_enabled else 'disabled'})"
|
||||
)
|
||||
_pool = ConnectionPool(db_path, pool_size, timeout)
|
||||
app.logger.info(f"Database connection pool initialized (size={pool_size})")
|
||||
|
||||
# Register teardown handler
|
||||
@app.teardown_appcontext
|
||||
|
||||
+3
-19
@@ -16,14 +16,13 @@ Standards:
|
||||
"""
|
||||
|
||||
# Standard library imports
|
||||
from html import escape
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
import time
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
# Local imports
|
||||
from starpunk.models import Note
|
||||
from starpunk.monitoring.business import track_feed_generated
|
||||
|
||||
|
||||
# ATOM namespace
|
||||
@@ -127,10 +126,6 @@ def generate_atom_streaming(
|
||||
# Remove trailing slash from site_url for consistency
|
||||
site_url = site_url.rstrip("/")
|
||||
|
||||
# Track feed generation timing
|
||||
start_time = time.time()
|
||||
item_count = 0
|
||||
|
||||
# Current timestamp for updated
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
@@ -159,8 +154,6 @@ def generate_atom_streaming(
|
||||
# Yield entries (newest first)
|
||||
# Notes from database are already in DESC order (newest first)
|
||||
for note in notes[:limit]:
|
||||
item_count += 1
|
||||
|
||||
# Build permalink URL
|
||||
permalink = f"{site_url}{note.permalink}"
|
||||
|
||||
@@ -207,8 +200,8 @@ def generate_atom_streaming(
|
||||
html_content += '<div class="media">'
|
||||
for item in note.media:
|
||||
media_url = f"{site_url}/media/{item['path']}"
|
||||
caption = item.get('caption', '')
|
||||
html_content += f'<img src="{media_url}" alt="{caption}" />'
|
||||
caption = escape(item.get('caption') or '', quote=True) # v2.14.1
|
||||
html_content += f'<img src="{escape(media_url, quote=True)}" alt="{caption}" />'
|
||||
html_content += '</div>'
|
||||
|
||||
# Add text content below media
|
||||
@@ -229,15 +222,6 @@ def generate_atom_streaming(
|
||||
# Yield closing tag
|
||||
yield '</feed>\n'
|
||||
|
||||
# Track feed generation metrics
|
||||
duration_ms = (time.time() - start_time) * 1000
|
||||
track_feed_generated(
|
||||
format='atom',
|
||||
item_count=item_count,
|
||||
duration_ms=duration_ms,
|
||||
cached=False
|
||||
)
|
||||
|
||||
|
||||
def _escape_xml(text: str) -> str:
|
||||
"""
|
||||
|
||||
@@ -16,14 +16,13 @@ Standards:
|
||||
"""
|
||||
|
||||
# Standard library imports
|
||||
from html import escape
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional, Dict, Any
|
||||
import time
|
||||
import json
|
||||
|
||||
# Local imports
|
||||
from starpunk.models import Note
|
||||
from starpunk.monitoring.business import track_feed_generated
|
||||
|
||||
|
||||
def generate_json_feed(
|
||||
@@ -74,9 +73,6 @@ def generate_json_feed(
|
||||
# Remove trailing slash from site_url for consistency
|
||||
site_url = site_url.rstrip("/")
|
||||
|
||||
# Track feed generation timing
|
||||
start_time = time.time()
|
||||
|
||||
# Build feed object
|
||||
feed = _build_feed_object(
|
||||
site_url=site_url,
|
||||
@@ -88,14 +84,6 @@ def generate_json_feed(
|
||||
# Serialize to JSON (pretty-printed)
|
||||
feed_json = json.dumps(feed, ensure_ascii=False, indent=2)
|
||||
|
||||
# Track feed generation metrics
|
||||
duration_ms = (time.time() - start_time) * 1000
|
||||
track_feed_generated(
|
||||
format='json',
|
||||
item_count=min(len(notes), limit),
|
||||
duration_ms=duration_ms,
|
||||
cached=False
|
||||
)
|
||||
|
||||
return feed_json
|
||||
|
||||
@@ -147,10 +135,6 @@ def generate_json_feed_streaming(
|
||||
# Remove trailing slash from site_url for consistency
|
||||
site_url = site_url.rstrip("/")
|
||||
|
||||
# Track feed generation timing
|
||||
start_time = time.time()
|
||||
item_count = 0
|
||||
|
||||
# Start feed object
|
||||
yield '{\n'
|
||||
yield f' "version": "https://jsonfeed.org/version/1.1",\n'
|
||||
@@ -170,8 +154,6 @@ def generate_json_feed_streaming(
|
||||
# Notes from database are already in DESC order (newest first)
|
||||
items = notes[:limit]
|
||||
for i, note in enumerate(items):
|
||||
item_count += 1
|
||||
|
||||
# Build item object
|
||||
item = _build_item_object(site_url, note)
|
||||
|
||||
@@ -193,15 +175,6 @@ def generate_json_feed_streaming(
|
||||
yield ' ]\n'
|
||||
yield '}\n'
|
||||
|
||||
# Track feed generation metrics
|
||||
duration_ms = (time.time() - start_time) * 1000
|
||||
track_feed_generated(
|
||||
format='json',
|
||||
item_count=item_count,
|
||||
duration_ms=duration_ms,
|
||||
cached=False
|
||||
)
|
||||
|
||||
|
||||
def _build_feed_object(
|
||||
site_url: str,
|
||||
@@ -275,8 +248,8 @@ def _build_item_object(site_url: str, note: Note) -> Dict[str, Any]:
|
||||
content_html += '<div class="media">'
|
||||
for media_item in note.media:
|
||||
media_url = f"{site_url}/media/{media_item['path']}"
|
||||
caption = media_item.get('caption', '')
|
||||
content_html += f'<img src="{media_url}" alt="{caption}" />'
|
||||
caption = escape(media_item.get('caption') or '', quote=True) # v2.14.1
|
||||
content_html += f'<img src="{escape(media_url, quote=True)}" alt="{caption}" />'
|
||||
content_html += '</div>'
|
||||
|
||||
# Add text content below media
|
||||
|
||||
@@ -14,7 +14,7 @@ Per v1.1.2 Phase 3:
|
||||
Specification: http://opml.org/spec2.opml
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timezone
|
||||
from xml.sax.saxutils import escape
|
||||
|
||||
|
||||
@@ -57,7 +57,7 @@ def generate_opml(site_url: str, site_name: str) -> str:
|
||||
safe_site_name = escape(site_name)
|
||||
|
||||
# RFC 822 date format (required by OPML spec)
|
||||
creation_date = datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT')
|
||||
creation_date = datetime.now(timezone.utc).replace(tzinfo=None).strftime('%a, %d %b %Y %H:%M:%S GMT')
|
||||
|
||||
# Build OPML document
|
||||
opml_lines = [
|
||||
|
||||
+5
-32
@@ -20,16 +20,15 @@ Standards:
|
||||
"""
|
||||
|
||||
# Standard library imports
|
||||
from html import escape
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
import time
|
||||
|
||||
# Third-party imports
|
||||
from feedgen.feed import FeedGenerator
|
||||
|
||||
# Local imports
|
||||
from starpunk.models import Note
|
||||
from starpunk.monitoring.business import track_feed_generated
|
||||
|
||||
|
||||
def generate_rss(
|
||||
@@ -98,9 +97,6 @@ def generate_rss(
|
||||
# Set last build date to now
|
||||
fg.lastBuildDate(datetime.now(timezone.utc))
|
||||
|
||||
# Track feed generation timing
|
||||
start_time = time.time()
|
||||
|
||||
# Add items (limit to configured maximum, newest first)
|
||||
# Notes from database are DESC but feedgen reverses them, so we reverse back
|
||||
for note in reversed(notes[:limit]):
|
||||
@@ -132,8 +128,8 @@ def generate_rss(
|
||||
html_content += '<div class="media">'
|
||||
for item in note.media:
|
||||
media_url = f"{site_url}/media/{item['path']}"
|
||||
caption = item.get('caption', '')
|
||||
html_content += f'<img src="{media_url}" alt="{caption}" />'
|
||||
caption = escape(item.get('caption') or '', quote=True) # v2.14.1
|
||||
html_content += f'<img src="{escape(media_url, quote=True)}" alt="{caption}" />'
|
||||
html_content += '</div>'
|
||||
|
||||
# Add text content below media
|
||||
@@ -165,14 +161,6 @@ def generate_rss(
|
||||
# We need to inject media:content and media:thumbnail elements
|
||||
feed_xml = _inject_media_rss_elements(feed_xml, site_url, notes[:limit])
|
||||
|
||||
# Track feed generation metrics
|
||||
duration_ms = (time.time() - start_time) * 1000
|
||||
track_feed_generated(
|
||||
format='rss',
|
||||
item_count=min(len(notes), limit),
|
||||
duration_ms=duration_ms,
|
||||
cached=False
|
||||
)
|
||||
|
||||
return feed_xml
|
||||
|
||||
@@ -227,10 +215,6 @@ def generate_rss_streaming(
|
||||
# Remove trailing slash from site_url for consistency
|
||||
site_url = site_url.rstrip("/")
|
||||
|
||||
# Track feed generation timing
|
||||
start_time = time.time()
|
||||
item_count = 0
|
||||
|
||||
# Current timestamp for lastBuildDate
|
||||
now = datetime.now(timezone.utc)
|
||||
last_build = format_rfc822_date(now)
|
||||
@@ -251,8 +235,6 @@ def generate_rss_streaming(
|
||||
# Yield items (newest first)
|
||||
# Notes from database are already in DESC order (newest first)
|
||||
for note in notes[:limit]:
|
||||
item_count += 1
|
||||
|
||||
# Build permalink URL
|
||||
permalink = f"{site_url}{note.permalink}"
|
||||
|
||||
@@ -273,8 +255,8 @@ def generate_rss_streaming(
|
||||
html_content += '<div class="media">'
|
||||
for item in note.media:
|
||||
media_url = f"{site_url}/media/{item['path']}"
|
||||
caption = item.get('caption', '')
|
||||
html_content += f'<img src="{media_url}" alt="{caption}" />'
|
||||
caption = escape(item.get('caption') or '', quote=True) # v2.14.1
|
||||
html_content += f'<img src="{escape(media_url, quote=True)}" alt="{caption}" />'
|
||||
html_content += '</div>'
|
||||
|
||||
# Add text content below media
|
||||
@@ -371,15 +353,6 @@ def generate_rss_streaming(
|
||||
yield " </channel>\n"
|
||||
yield "</rss>\n"
|
||||
|
||||
# Track feed generation metrics
|
||||
duration_ms = (time.time() - start_time) * 1000
|
||||
track_feed_generated(
|
||||
format='rss',
|
||||
item_count=item_count,
|
||||
duration_ms=duration_ms,
|
||||
cached=False
|
||||
)
|
||||
|
||||
|
||||
def _inject_media_rss_elements(feed_xml: str, site_url: str, notes: list[Note]) -> str:
|
||||
"""
|
||||
|
||||
+18
-17
@@ -13,7 +13,7 @@ current IndieAuth spec: https://indieauth.spec.indieweb.org/
|
||||
import base64
|
||||
import hashlib
|
||||
import secrets
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Dict, List, Optional
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
@@ -83,41 +83,42 @@ def fetch_client_redirect_uris(client_id: str) -> List[str]:
|
||||
Supports a JSON client metadata document ("redirect_uris") and the older
|
||||
HTML/Link header form (rel="redirect_uri"). Returns [] on any failure.
|
||||
"""
|
||||
from starpunk.webmention_ssrf import SSRFBlockedError, validate_url_for_fetch
|
||||
import json
|
||||
from starpunk.webmention_ssrf import FetchTooLargeError, SSRFBlockedError, safe_get, user_agent
|
||||
|
||||
# Every redirect hop is checked and the connection pinned to the checked
|
||||
# address, with a hard time limit (v2.11.0; DEV_MODE allows localhost)
|
||||
try:
|
||||
if not current_app.config.get("DEV_MODE"):
|
||||
validate_url_for_fetch(client_id)
|
||||
response = httpx.get(
|
||||
result = safe_get(
|
||||
client_id,
|
||||
timeout=5.0,
|
||||
follow_redirects=True,
|
||||
headers={"Accept": "application/json, text/html;q=0.9"},
|
||||
headers={"Accept": "application/json, text/html;q=0.9",
|
||||
"User-Agent": user_agent(current_app.config.get("SITE_URL", ""))},
|
||||
max_bytes=500_000, timeout=5.0, truncate=True,
|
||||
)
|
||||
except (httpx.HTTPError, SSRFBlockedError, ValueError) as e:
|
||||
except (httpx.HTTPError, SSRFBlockedError, ValueError, FetchTooLargeError) as e:
|
||||
current_app.logger.info(f"IndieAuth: could not fetch client {client_id}: {e}")
|
||||
return []
|
||||
|
||||
if response.status_code != 200:
|
||||
if result.status_code != 200:
|
||||
return []
|
||||
|
||||
base = str(response.url)
|
||||
base = result.url
|
||||
uris: List[str] = []
|
||||
|
||||
if "json" in response.headers.get("Content-Type", ""):
|
||||
if "json" in result.headers.get("Content-Type", ""):
|
||||
try:
|
||||
data = response.json()
|
||||
data = json.loads(result.text)
|
||||
except ValueError:
|
||||
return []
|
||||
if isinstance(data, dict) and isinstance(data.get("redirect_uris"), list):
|
||||
uris.extend(u for u in data["redirect_uris"] if isinstance(u, str))
|
||||
return uris
|
||||
|
||||
link = response.links.get("redirect_uri")
|
||||
link = httpx.Response(200, headers=result.headers).links.get("redirect_uri")
|
||||
if link and link.get("url"):
|
||||
uris.append(urljoin(base, link["url"]))
|
||||
|
||||
soup = BeautifulSoup(response.text[:500_000], "html.parser")
|
||||
soup = BeautifulSoup(result.text, "html.parser")
|
||||
for tag in soup.find_all(["link", "a"], href=True):
|
||||
if "redirect_uri" in (tag.get("rel") or []):
|
||||
uris.append(urljoin(base, tag["href"]))
|
||||
@@ -203,7 +204,7 @@ def create_authorization_code(
|
||||
redirect_uri,
|
||||
scope,
|
||||
code_challenge,
|
||||
datetime.utcnow() + timedelta(minutes=CODE_LIFETIME_MINUTES),
|
||||
datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(minutes=CODE_LIFETIME_MINUTES),
|
||||
),
|
||||
)
|
||||
db.commit()
|
||||
@@ -258,7 +259,7 @@ def create_access_token(client_id: str, scope: str) -> Dict[str, Any]:
|
||||
"""Issue an access token (stored hashed); returns the token response"""
|
||||
token = secrets.token_urlsafe(32)
|
||||
lifetime_days = current_app.config.get("INDIEAUTH_TOKEN_LIFETIME_DAYS", 90)
|
||||
expires_at = datetime.utcnow() + timedelta(days=lifetime_days)
|
||||
expires_at = datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(days=lifetime_days)
|
||||
|
||||
db = get_db(current_app)
|
||||
db.execute(
|
||||
|
||||
@@ -842,13 +842,16 @@ def note_to_mf2(note, wanted: Optional[list] = None) -> dict:
|
||||
|
||||
|
||||
def _all_categories(prefix: str = "") -> list:
|
||||
"""Tags in use, most used first (q=category)"""
|
||||
"""Tags on posts that exist (drafts included), most used first (q=category)"""
|
||||
from starpunk.database import get_db
|
||||
|
||||
# Inner joins: tags left behind by edits or soft-deleted posts don't count
|
||||
rows = get_db(current_app).execute(
|
||||
"""
|
||||
SELECT t.display_name, COUNT(nt.note_id) AS uses
|
||||
FROM tags t LEFT JOIN note_tags nt ON nt.tag_id = t.id
|
||||
SELECT t.display_name, COUNT(*) AS uses
|
||||
FROM tags t
|
||||
JOIN note_tags nt ON nt.tag_id = t.id
|
||||
JOIN notes n ON n.id = nt.note_id AND n.deleted_at IS NULL
|
||||
GROUP BY t.id ORDER BY uses DESC, t.display_name
|
||||
"""
|
||||
).fetchall()
|
||||
|
||||
+9
-9
@@ -8,7 +8,7 @@ state. All models are immutable and use dataclasses for clean structure.
|
||||
|
||||
# Standard library imports
|
||||
from dataclasses import dataclass, field, replace
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
import re
|
||||
@@ -698,7 +698,7 @@ class Session:
|
||||
>>> session.is_expired
|
||||
False
|
||||
"""
|
||||
return datetime.utcnow() >= self.expires_at
|
||||
return datetime.now(timezone.utc).replace(tzinfo=None) >= self.expires_at
|
||||
|
||||
@property
|
||||
def is_active(self) -> bool:
|
||||
@@ -726,7 +726,7 @@ class Session:
|
||||
>>> session.age
|
||||
datetime.timedelta(days=2, seconds=3600)
|
||||
"""
|
||||
return datetime.utcnow() - self.created_at
|
||||
return datetime.now(timezone.utc).replace(tzinfo=None) - self.created_at
|
||||
|
||||
@property
|
||||
def time_until_expiry(self) -> timedelta:
|
||||
@@ -740,7 +740,7 @@ class Session:
|
||||
>>> session.time_until_expiry
|
||||
datetime.timedelta(days=28)
|
||||
"""
|
||||
return self.expires_at - datetime.utcnow()
|
||||
return self.expires_at - datetime.now(timezone.utc).replace(tzinfo=None)
|
||||
|
||||
def is_valid(self) -> bool:
|
||||
"""
|
||||
@@ -787,7 +787,7 @@ class Session:
|
||||
>>> updated.last_used_at
|
||||
datetime.datetime(2024, 11, 18, 15, 30, 0)
|
||||
"""
|
||||
return replace(self, last_used_at=datetime.utcnow())
|
||||
return replace(self, last_used_at=datetime.now(timezone.utc).replace(tzinfo=None))
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
@@ -863,7 +863,7 @@ class Token:
|
||||
me: str
|
||||
client_id: Optional[str] = None
|
||||
scope: Optional[str] = None
|
||||
created_at: datetime = field(default_factory=lambda: datetime.utcnow())
|
||||
created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc).replace(tzinfo=None))
|
||||
expires_at: Optional[datetime] = None
|
||||
|
||||
@classmethod
|
||||
@@ -926,7 +926,7 @@ class Token:
|
||||
# No expiry set, token never expires
|
||||
return False
|
||||
|
||||
return datetime.utcnow() >= self.expires_at
|
||||
return datetime.now(timezone.utc).replace(tzinfo=None) >= self.expires_at
|
||||
|
||||
@property
|
||||
def is_active(self) -> bool:
|
||||
@@ -1145,7 +1145,7 @@ class AuthState:
|
||||
>>> auth_state.is_expired
|
||||
False
|
||||
"""
|
||||
return datetime.utcnow() >= self.expires_at
|
||||
return datetime.now(timezone.utc).replace(tzinfo=None) >= self.expires_at
|
||||
|
||||
@property
|
||||
def is_active(self) -> bool:
|
||||
@@ -1169,7 +1169,7 @@ class AuthState:
|
||||
>>> auth_state.age
|
||||
datetime.timedelta(seconds=120)
|
||||
"""
|
||||
return datetime.utcnow() - self.created_at
|
||||
return datetime.now(timezone.utc).replace(tzinfo=None) - self.created_at
|
||||
|
||||
def is_valid(self) -> bool:
|
||||
"""
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
"""
|
||||
Performance monitoring for StarPunk
|
||||
|
||||
This package provides performance monitoring capabilities including:
|
||||
- Metrics collection with circular buffers
|
||||
- Operation timing (database, HTTP, rendering)
|
||||
- Per-process metrics with aggregation
|
||||
- Configurable sampling rates
|
||||
- Database query monitoring (v1.1.2 Phase 1)
|
||||
- HTTP request/response metrics (v1.1.2 Phase 1)
|
||||
- Memory monitoring (v1.1.2 Phase 1)
|
||||
|
||||
Per ADR-053 and developer Q&A Q6, Q12:
|
||||
- Each process maintains its own circular buffer
|
||||
- Buffers store recent metrics (default 1000 entries)
|
||||
- Metrics include process ID for multi-process deployment
|
||||
- Sampling rates are configurable per operation type
|
||||
"""
|
||||
|
||||
from starpunk.monitoring.metrics import MetricsBuffer, record_metric, get_metrics, get_metrics_stats
|
||||
from starpunk.monitoring.database import MonitoredConnection
|
||||
from starpunk.monitoring.http import setup_http_metrics
|
||||
from starpunk.monitoring.memory import MemoryMonitor
|
||||
from starpunk.monitoring import business
|
||||
|
||||
__all__ = [
|
||||
"MetricsBuffer",
|
||||
"record_metric",
|
||||
"get_metrics",
|
||||
"get_metrics_stats",
|
||||
"MonitoredConnection",
|
||||
"setup_http_metrics",
|
||||
"MemoryMonitor",
|
||||
"business",
|
||||
]
|
||||
@@ -1,298 +0,0 @@
|
||||
"""
|
||||
Business metrics for StarPunk operations
|
||||
|
||||
Per v1.1.2 Phase 1:
|
||||
- Track note operations (create, update, delete)
|
||||
- Track feed generation and cache hits/misses
|
||||
- Track content statistics
|
||||
|
||||
Per v1.1.2 Phase 3:
|
||||
- Track feed statistics by format
|
||||
- Track feed cache hit/miss rates
|
||||
- Provide feed statistics dashboard
|
||||
|
||||
Example usage:
|
||||
>>> from starpunk.monitoring.business import track_note_created
|
||||
>>> track_note_created(note_id=123, content_length=500)
|
||||
"""
|
||||
|
||||
from typing import Optional, Dict, Any
|
||||
|
||||
from starpunk.monitoring.metrics import record_metric, get_metrics_stats
|
||||
|
||||
|
||||
def track_note_created(note_id: int, content_length: int, has_media: bool = False) -> None:
|
||||
"""
|
||||
Track note creation event
|
||||
|
||||
Args:
|
||||
note_id: ID of created note
|
||||
content_length: Length of note content in characters
|
||||
has_media: Whether note has media attachments
|
||||
"""
|
||||
metadata = {
|
||||
'note_id': note_id,
|
||||
'content_length': content_length,
|
||||
'has_media': has_media,
|
||||
}
|
||||
|
||||
record_metric(
|
||||
'render', # Use 'render' for business metrics
|
||||
'note_created',
|
||||
content_length,
|
||||
metadata,
|
||||
force=True # Always track business events
|
||||
)
|
||||
|
||||
|
||||
def track_note_updated(note_id: int, content_length: int, fields_changed: Optional[list] = None) -> None:
|
||||
"""
|
||||
Track note update event
|
||||
|
||||
Args:
|
||||
note_id: ID of updated note
|
||||
content_length: New length of note content
|
||||
fields_changed: List of fields that were changed
|
||||
"""
|
||||
metadata = {
|
||||
'note_id': note_id,
|
||||
'content_length': content_length,
|
||||
}
|
||||
|
||||
if fields_changed:
|
||||
metadata['fields_changed'] = ','.join(fields_changed)
|
||||
|
||||
record_metric(
|
||||
'render',
|
||||
'note_updated',
|
||||
content_length,
|
||||
metadata,
|
||||
force=True
|
||||
)
|
||||
|
||||
|
||||
def track_note_deleted(note_id: int) -> None:
|
||||
"""
|
||||
Track note deletion event
|
||||
|
||||
Args:
|
||||
note_id: ID of deleted note
|
||||
"""
|
||||
metadata = {
|
||||
'note_id': note_id,
|
||||
}
|
||||
|
||||
record_metric(
|
||||
'render',
|
||||
'note_deleted',
|
||||
0, # No meaningful duration for deletion
|
||||
metadata,
|
||||
force=True
|
||||
)
|
||||
|
||||
|
||||
def track_feed_generated(format: str, item_count: int, duration_ms: float, cached: bool = False) -> None:
|
||||
"""
|
||||
Track feed generation event
|
||||
|
||||
Args:
|
||||
format: Feed format (rss, atom, json)
|
||||
item_count: Number of items in feed
|
||||
duration_ms: Time taken to generate feed
|
||||
cached: Whether feed was served from cache
|
||||
"""
|
||||
metadata = {
|
||||
'format': format,
|
||||
'item_count': item_count,
|
||||
'cached': cached,
|
||||
}
|
||||
|
||||
operation = f'feed_{format}{"_cached" if cached else "_generated"}'
|
||||
|
||||
record_metric(
|
||||
'render',
|
||||
operation,
|
||||
duration_ms,
|
||||
metadata,
|
||||
force=True # Always track feed operations
|
||||
)
|
||||
|
||||
|
||||
def track_cache_hit(cache_type: str, key: str) -> None:
|
||||
"""
|
||||
Track cache hit event
|
||||
|
||||
Args:
|
||||
cache_type: Type of cache (feed, etc.)
|
||||
key: Cache key that was hit
|
||||
"""
|
||||
metadata = {
|
||||
'cache_type': cache_type,
|
||||
'key': key,
|
||||
}
|
||||
|
||||
record_metric(
|
||||
'render',
|
||||
f'{cache_type}_cache_hit',
|
||||
0,
|
||||
metadata,
|
||||
force=True
|
||||
)
|
||||
|
||||
|
||||
def track_cache_miss(cache_type: str, key: str) -> None:
|
||||
"""
|
||||
Track cache miss event
|
||||
|
||||
Args:
|
||||
cache_type: Type of cache (feed, etc.)
|
||||
key: Cache key that was missed
|
||||
"""
|
||||
metadata = {
|
||||
'cache_type': cache_type,
|
||||
'key': key,
|
||||
}
|
||||
|
||||
record_metric(
|
||||
'render',
|
||||
f'{cache_type}_cache_miss',
|
||||
0,
|
||||
metadata,
|
||||
force=True
|
||||
)
|
||||
|
||||
|
||||
def get_feed_statistics() -> Dict[str, Any]:
|
||||
"""
|
||||
Get aggregated feed statistics from metrics buffer and feed cache.
|
||||
|
||||
Analyzes metrics to provide feed-specific statistics including:
|
||||
- Total requests by format (RSS, ATOM, JSON)
|
||||
- Cache hit/miss rates by format
|
||||
- Feed generation times by format
|
||||
- Format popularity (percentage breakdown)
|
||||
- Feed cache internal statistics
|
||||
|
||||
Returns:
|
||||
Dictionary with feed statistics:
|
||||
{
|
||||
'by_format': {
|
||||
'rss': {'generated': int, 'cached': int, 'total': int, 'avg_duration_ms': float},
|
||||
'atom': {...},
|
||||
'json': {...}
|
||||
},
|
||||
'cache': {
|
||||
'hits': int,
|
||||
'misses': int,
|
||||
'hit_rate': float (0.0-1.0),
|
||||
'entries': int,
|
||||
'evictions': int
|
||||
},
|
||||
'total_requests': int,
|
||||
'format_percentages': {
|
||||
'rss': float,
|
||||
'atom': float,
|
||||
'json': float
|
||||
}
|
||||
}
|
||||
|
||||
Example:
|
||||
>>> stats = get_feed_statistics()
|
||||
>>> print(f"RSS requests: {stats['by_format']['rss']['total']}")
|
||||
>>> print(f"Cache hit rate: {stats['cache']['hit_rate']:.2%}")
|
||||
"""
|
||||
# Get all metrics
|
||||
all_metrics = get_metrics_stats()
|
||||
|
||||
# Initialize result structure
|
||||
result = {
|
||||
'by_format': {
|
||||
'rss': {'generated': 0, 'cached': 0, 'total': 0, 'avg_duration_ms': 0.0},
|
||||
'atom': {'generated': 0, 'cached': 0, 'total': 0, 'avg_duration_ms': 0.0},
|
||||
'json': {'generated': 0, 'cached': 0, 'total': 0, 'avg_duration_ms': 0.0},
|
||||
},
|
||||
'cache': {
|
||||
'hits': 0,
|
||||
'misses': 0,
|
||||
'hit_rate': 0.0,
|
||||
},
|
||||
'total_requests': 0,
|
||||
'format_percentages': {
|
||||
'rss': 0.0,
|
||||
'atom': 0.0,
|
||||
'json': 0.0,
|
||||
},
|
||||
}
|
||||
|
||||
# Get by_operation metrics if available
|
||||
by_operation = all_metrics.get('by_operation', {})
|
||||
|
||||
# Count feed operations by format
|
||||
for operation_name, op_stats in by_operation.items():
|
||||
# Feed operations are named: feed_rss_generated, feed_rss_cached, etc.
|
||||
if operation_name.startswith('feed_'):
|
||||
parts = operation_name.split('_')
|
||||
if len(parts) >= 3:
|
||||
format_name = parts[1] # rss, atom, or json
|
||||
operation_type = parts[2] # generated or cached
|
||||
|
||||
if format_name in result['by_format']:
|
||||
count = op_stats.get('count', 0)
|
||||
|
||||
if operation_type == 'generated':
|
||||
result['by_format'][format_name]['generated'] = count
|
||||
# Track average duration for generated feeds
|
||||
result['by_format'][format_name]['avg_duration_ms'] = op_stats.get('avg_duration_ms', 0.0)
|
||||
elif operation_type == 'cached':
|
||||
result['by_format'][format_name]['cached'] = count
|
||||
|
||||
# Update total for this format
|
||||
result['by_format'][format_name]['total'] = (
|
||||
result['by_format'][format_name]['generated'] +
|
||||
result['by_format'][format_name]['cached']
|
||||
)
|
||||
|
||||
# Track cache hits/misses
|
||||
elif operation_name == 'feed_cache_hit':
|
||||
result['cache']['hits'] = op_stats.get('count', 0)
|
||||
elif operation_name == 'feed_cache_miss':
|
||||
result['cache']['misses'] = op_stats.get('count', 0)
|
||||
|
||||
# Calculate total requests across all formats
|
||||
result['total_requests'] = sum(
|
||||
fmt['total'] for fmt in result['by_format'].values()
|
||||
)
|
||||
|
||||
# Calculate cache hit rate
|
||||
total_cache_requests = result['cache']['hits'] + result['cache']['misses']
|
||||
if total_cache_requests > 0:
|
||||
result['cache']['hit_rate'] = result['cache']['hits'] / total_cache_requests
|
||||
|
||||
# Calculate format percentages
|
||||
if result['total_requests'] > 0:
|
||||
for format_name, fmt_stats in result['by_format'].items():
|
||||
result['format_percentages'][format_name] = (
|
||||
fmt_stats['total'] / result['total_requests']
|
||||
)
|
||||
|
||||
# Get feed cache statistics if available
|
||||
try:
|
||||
from starpunk.feeds import get_cache
|
||||
feed_cache = get_cache()
|
||||
cache_stats = feed_cache.get_stats()
|
||||
|
||||
# Merge cache stats (prefer FeedCache internal stats over metrics)
|
||||
result['cache']['entries'] = cache_stats.get('entries', 0)
|
||||
result['cache']['evictions'] = cache_stats.get('evictions', 0)
|
||||
|
||||
# Use FeedCache hit rate if available and more accurate
|
||||
if cache_stats.get('hits', 0) + cache_stats.get('misses', 0) > 0:
|
||||
result['cache']['hits'] = cache_stats.get('hits', 0)
|
||||
result['cache']['misses'] = cache_stats.get('misses', 0)
|
||||
result['cache']['hit_rate'] = cache_stats.get('hit_rate', 0.0)
|
||||
|
||||
except ImportError:
|
||||
# Feed cache not available, use defaults
|
||||
pass
|
||||
|
||||
return result
|
||||
@@ -1,236 +0,0 @@
|
||||
"""
|
||||
Database operation monitoring wrapper
|
||||
|
||||
Per ADR-053, v1.1.2 Phase 1, and developer Q&A CQ1, IQ1, IQ3:
|
||||
- Wraps SQLite connections at the pool level
|
||||
- Times all database operations
|
||||
- Extracts query type and table name (best effort)
|
||||
- Detects slow queries based on configurable threshold
|
||||
- Records metrics to the metrics collector
|
||||
|
||||
Example usage:
|
||||
>>> from starpunk.monitoring.database import MonitoredConnection
|
||||
>>> conn = sqlite3.connect(':memory:')
|
||||
>>> monitored = MonitoredConnection(conn, metrics_collector)
|
||||
>>> cursor = monitored.execute('SELECT * FROM notes')
|
||||
"""
|
||||
|
||||
import re
|
||||
import sqlite3
|
||||
import time
|
||||
from typing import Optional, Any, Tuple
|
||||
|
||||
from starpunk.monitoring.metrics import record_metric
|
||||
|
||||
|
||||
class MonitoredConnection:
|
||||
"""
|
||||
Wrapper for SQLite connections that monitors performance
|
||||
|
||||
Per CQ1: Wraps connections at the pool level
|
||||
Per IQ1: Uses simple regex for table name extraction
|
||||
Per IQ3: Single configurable slow query threshold
|
||||
"""
|
||||
|
||||
def __init__(self, connection: sqlite3.Connection, slow_query_threshold: float = 1.0):
|
||||
"""
|
||||
Initialize monitored connection wrapper
|
||||
|
||||
Args:
|
||||
connection: SQLite connection to wrap
|
||||
slow_query_threshold: Threshold in seconds for slow query detection
|
||||
"""
|
||||
self._connection = connection
|
||||
self._slow_query_threshold = slow_query_threshold
|
||||
|
||||
def execute(self, query: str, parameters: Optional[Tuple] = None) -> sqlite3.Cursor:
|
||||
"""
|
||||
Execute a query with performance monitoring
|
||||
|
||||
Args:
|
||||
query: SQL query to execute
|
||||
parameters: Optional query parameters
|
||||
|
||||
Returns:
|
||||
sqlite3.Cursor: Query cursor
|
||||
"""
|
||||
start_time = time.perf_counter()
|
||||
query_type = self._get_query_type(query)
|
||||
table_name = self._extract_table_name(query)
|
||||
|
||||
try:
|
||||
if parameters:
|
||||
cursor = self._connection.execute(query, parameters)
|
||||
else:
|
||||
cursor = self._connection.execute(query)
|
||||
|
||||
duration_sec = time.perf_counter() - start_time
|
||||
duration_ms = duration_sec * 1000
|
||||
|
||||
# Record metric (forced if slow query)
|
||||
is_slow = duration_sec >= self._slow_query_threshold
|
||||
metadata = {
|
||||
'query_type': query_type,
|
||||
'table': table_name,
|
||||
'is_slow': is_slow,
|
||||
}
|
||||
|
||||
# Add query text for slow queries (for debugging)
|
||||
if is_slow:
|
||||
# Truncate query to avoid storing huge queries
|
||||
metadata['query'] = query[:200] if len(query) > 200 else query
|
||||
|
||||
record_metric(
|
||||
'database',
|
||||
f'{query_type} {table_name}',
|
||||
duration_ms,
|
||||
metadata,
|
||||
force=is_slow # Always record slow queries
|
||||
)
|
||||
|
||||
return cursor
|
||||
|
||||
except Exception as e:
|
||||
duration_sec = time.perf_counter() - start_time
|
||||
duration_ms = duration_sec * 1000
|
||||
|
||||
# Record error metric
|
||||
metadata = {
|
||||
'query_type': query_type,
|
||||
'table': table_name,
|
||||
'error': str(e),
|
||||
'query': query[:200] if len(query) > 200 else query
|
||||
}
|
||||
|
||||
record_metric(
|
||||
'database',
|
||||
f'{query_type} {table_name} ERROR',
|
||||
duration_ms,
|
||||
metadata,
|
||||
force=True # Always record errors
|
||||
)
|
||||
|
||||
raise
|
||||
|
||||
def executemany(self, query: str, parameters) -> sqlite3.Cursor:
|
||||
"""
|
||||
Execute a query with multiple parameter sets
|
||||
|
||||
Args:
|
||||
query: SQL query to execute
|
||||
parameters: Sequence of parameter tuples
|
||||
|
||||
Returns:
|
||||
sqlite3.Cursor: Query cursor
|
||||
"""
|
||||
start_time = time.perf_counter()
|
||||
query_type = self._get_query_type(query)
|
||||
table_name = self._extract_table_name(query)
|
||||
|
||||
try:
|
||||
cursor = self._connection.executemany(query, parameters)
|
||||
duration_ms = (time.perf_counter() - start_time) * 1000
|
||||
|
||||
# Record metric
|
||||
metadata = {
|
||||
'query_type': query_type,
|
||||
'table': table_name,
|
||||
'batch': True,
|
||||
}
|
||||
|
||||
record_metric(
|
||||
'database',
|
||||
f'{query_type} {table_name} BATCH',
|
||||
duration_ms,
|
||||
metadata
|
||||
)
|
||||
|
||||
return cursor
|
||||
|
||||
except Exception as e:
|
||||
duration_ms = (time.perf_counter() - start_time) * 1000
|
||||
|
||||
metadata = {
|
||||
'query_type': query_type,
|
||||
'table': table_name,
|
||||
'error': str(e),
|
||||
'batch': True
|
||||
}
|
||||
|
||||
record_metric(
|
||||
'database',
|
||||
f'{query_type} {table_name} BATCH ERROR',
|
||||
duration_ms,
|
||||
metadata,
|
||||
force=True
|
||||
)
|
||||
|
||||
raise
|
||||
|
||||
def _get_query_type(self, query: str) -> str:
|
||||
"""
|
||||
Extract query type from SQL statement
|
||||
|
||||
Args:
|
||||
query: SQL query
|
||||
|
||||
Returns:
|
||||
Query type (SELECT, INSERT, UPDATE, DELETE, etc.)
|
||||
"""
|
||||
query_upper = query.strip().upper()
|
||||
|
||||
for query_type in ['SELECT', 'INSERT', 'UPDATE', 'DELETE', 'CREATE', 'DROP', 'ALTER', 'PRAGMA']:
|
||||
if query_upper.startswith(query_type):
|
||||
return query_type
|
||||
|
||||
return 'OTHER'
|
||||
|
||||
def _extract_table_name(self, query: str) -> str:
|
||||
"""
|
||||
Extract table name from query (best effort)
|
||||
|
||||
Per IQ1: Keep it simple with basic regex patterns.
|
||||
Returns "unknown" for complex queries.
|
||||
|
||||
Note: Complex queries (JOINs, subqueries, CTEs) return "unknown".
|
||||
This covers 90% of queries accurately.
|
||||
|
||||
Args:
|
||||
query: SQL query
|
||||
|
||||
Returns:
|
||||
Table name or "unknown"
|
||||
"""
|
||||
query_lower = query.lower().strip()
|
||||
|
||||
# Simple patterns that cover 90% of cases
|
||||
patterns = [
|
||||
r'from\s+(\w+)',
|
||||
r'update\s+(\w+)',
|
||||
r'insert\s+into\s+(\w+)',
|
||||
r'delete\s+from\s+(\w+)',
|
||||
r'create\s+table\s+(?:if\s+not\s+exists\s+)?(\w+)',
|
||||
r'drop\s+table\s+(?:if\s+exists\s+)?(\w+)',
|
||||
r'alter\s+table\s+(\w+)',
|
||||
]
|
||||
|
||||
for pattern in patterns:
|
||||
match = re.search(pattern, query_lower)
|
||||
if match:
|
||||
return match.group(1)
|
||||
|
||||
# Complex queries (JOINs, subqueries, CTEs)
|
||||
return "unknown"
|
||||
|
||||
# Delegate all other connection methods to the wrapped connection
|
||||
def __getattr__(self, name: str) -> Any:
|
||||
"""Delegate all other methods to the wrapped connection"""
|
||||
return getattr(self._connection, name)
|
||||
|
||||
def __enter__(self):
|
||||
"""Support context manager protocol"""
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
"""Support context manager protocol"""
|
||||
return self._connection.__exit__(exc_type, exc_val, exc_tb)
|
||||
@@ -1,133 +0,0 @@
|
||||
"""
|
||||
HTTP request/response metrics middleware
|
||||
|
||||
Per v1.1.2 Phase 1 and developer Q&A IQ2:
|
||||
- Times all HTTP requests
|
||||
- Generates request IDs for tracking (IQ2)
|
||||
- Records status codes, methods, routes
|
||||
- Tracks request and response sizes
|
||||
- Adds X-Request-ID header to all responses (not just debug mode)
|
||||
|
||||
Example usage:
|
||||
>>> from starpunk.monitoring.http import setup_http_metrics
|
||||
>>> app = Flask(__name__)
|
||||
>>> setup_http_metrics(app)
|
||||
"""
|
||||
|
||||
import time
|
||||
import uuid
|
||||
from flask import g, request, Flask
|
||||
from typing import Any
|
||||
|
||||
from starpunk.monitoring.metrics import record_metric
|
||||
|
||||
|
||||
def setup_http_metrics(app: Flask) -> None:
|
||||
"""
|
||||
Setup HTTP metrics collection for Flask app
|
||||
|
||||
Per IQ2: Generates request IDs and adds X-Request-ID header in all modes
|
||||
|
||||
Args:
|
||||
app: Flask application instance
|
||||
"""
|
||||
|
||||
@app.before_request
|
||||
def start_request_metrics():
|
||||
"""
|
||||
Initialize request metrics tracking
|
||||
|
||||
Per IQ2: Generate UUID request ID and store in g
|
||||
"""
|
||||
# Generate request ID (IQ2: in all modes, not just debug)
|
||||
g.request_id = str(uuid.uuid4())
|
||||
|
||||
# Store request start time and metadata
|
||||
g.request_start_time = time.perf_counter()
|
||||
g.request_metadata = {
|
||||
'method': request.method,
|
||||
'endpoint': request.endpoint or 'unknown',
|
||||
'path': request.path,
|
||||
'content_length': request.content_length or 0,
|
||||
}
|
||||
|
||||
@app.after_request
|
||||
def record_response_metrics(response):
|
||||
"""
|
||||
Record HTTP response metrics
|
||||
|
||||
Args:
|
||||
response: Flask response object
|
||||
|
||||
Returns:
|
||||
Modified response with X-Request-ID header
|
||||
"""
|
||||
# Skip if metrics not initialized (shouldn't happen in normal flow)
|
||||
if not hasattr(g, 'request_start_time'):
|
||||
return response
|
||||
|
||||
# Calculate request duration
|
||||
duration_sec = time.perf_counter() - g.request_start_time
|
||||
duration_ms = duration_sec * 1000
|
||||
|
||||
# Get response size
|
||||
response_size = 0
|
||||
|
||||
# Check if response is in direct passthrough mode (streaming)
|
||||
if hasattr(response, 'direct_passthrough') and response.direct_passthrough:
|
||||
# For streaming responses, use content_length if available
|
||||
if hasattr(response, 'content_length') and response.content_length:
|
||||
response_size = response.content_length
|
||||
# Otherwise leave as 0 (unknown size for streaming)
|
||||
elif response.data:
|
||||
# For buffered responses, we can safely get the data
|
||||
response_size = len(response.data)
|
||||
elif hasattr(response, 'content_length') and response.content_length:
|
||||
response_size = response.content_length
|
||||
|
||||
# Build metadata
|
||||
metadata = {
|
||||
**g.request_metadata,
|
||||
'status_code': response.status_code,
|
||||
'response_size': response_size,
|
||||
}
|
||||
|
||||
# Record metric
|
||||
operation_name = f"{g.request_metadata['method']} {g.request_metadata['endpoint']}"
|
||||
record_metric(
|
||||
'http',
|
||||
operation_name,
|
||||
duration_ms,
|
||||
metadata
|
||||
)
|
||||
|
||||
# Add request ID header (IQ2: in all modes)
|
||||
response.headers['X-Request-ID'] = g.request_id
|
||||
|
||||
return response
|
||||
|
||||
@app.teardown_request
|
||||
def record_error_metrics(error=None):
|
||||
"""
|
||||
Record metrics for requests that result in errors
|
||||
|
||||
Args:
|
||||
error: Exception if request failed
|
||||
"""
|
||||
if error and hasattr(g, 'request_start_time'):
|
||||
duration_ms = (time.perf_counter() - g.request_start_time) * 1000
|
||||
|
||||
metadata = {
|
||||
**g.request_metadata,
|
||||
'error': str(error),
|
||||
'error_type': type(error).__name__,
|
||||
}
|
||||
|
||||
operation_name = f"{g.request_metadata['method']} {g.request_metadata['endpoint']} ERROR"
|
||||
record_metric(
|
||||
'http',
|
||||
operation_name,
|
||||
duration_ms,
|
||||
metadata,
|
||||
force=True # Always record errors
|
||||
)
|
||||
@@ -1,191 +0,0 @@
|
||||
"""
|
||||
Memory monitoring background thread
|
||||
|
||||
Per v1.1.2 Phase 1 and developer Q&A CQ5, IQ8:
|
||||
- Background daemon thread for continuous memory monitoring
|
||||
- Tracks RSS and VMS memory usage
|
||||
- Detects memory growth and potential leaks
|
||||
- 5-second baseline period after startup (IQ8)
|
||||
- Skipped in test mode (CQ5)
|
||||
|
||||
Example usage:
|
||||
>>> from starpunk.monitoring.memory import MemoryMonitor
|
||||
>>> monitor = MemoryMonitor(interval=30)
|
||||
>>> monitor.start() # Runs as daemon thread
|
||||
>>> # ... application runs ...
|
||||
>>> monitor.stop()
|
||||
"""
|
||||
|
||||
import gc
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from typing import Dict, Any
|
||||
|
||||
import psutil
|
||||
|
||||
from starpunk.monitoring.metrics import record_metric
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MemoryMonitor(threading.Thread):
|
||||
"""
|
||||
Background thread for memory monitoring
|
||||
|
||||
Per CQ5: Daemon thread that auto-terminates with main process
|
||||
Per IQ8: 5-second baseline period after startup
|
||||
"""
|
||||
|
||||
def __init__(self, interval: int = 30):
|
||||
"""
|
||||
Initialize memory monitor thread
|
||||
|
||||
Args:
|
||||
interval: Monitoring interval in seconds (default: 30)
|
||||
"""
|
||||
super().__init__(daemon=True) # CQ5: daemon thread
|
||||
self.interval = interval
|
||||
self._stop_event = threading.Event()
|
||||
self._process = psutil.Process()
|
||||
self._baseline_memory = None
|
||||
self._high_water_mark = 0
|
||||
|
||||
def run(self):
|
||||
"""
|
||||
Main monitoring loop
|
||||
|
||||
Per IQ8: Wait 5 seconds for app initialization before setting baseline
|
||||
"""
|
||||
try:
|
||||
# Wait for app initialization (IQ8: 5 seconds)
|
||||
time.sleep(5)
|
||||
|
||||
# Set baseline memory
|
||||
memory_info = self._get_memory_info()
|
||||
self._baseline_memory = memory_info['rss_mb']
|
||||
logger.info(f"Memory monitor baseline set: {self._baseline_memory:.2f} MB RSS")
|
||||
|
||||
# Start monitoring loop
|
||||
while not self._stop_event.is_set():
|
||||
try:
|
||||
self._collect_metrics()
|
||||
except Exception as e:
|
||||
logger.error(f"Memory monitoring error: {e}", exc_info=True)
|
||||
|
||||
# Wait for interval or until stop event
|
||||
self._stop_event.wait(self.interval)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Memory monitor thread failed: {e}", exc_info=True)
|
||||
|
||||
def _collect_metrics(self):
|
||||
"""Collect and record memory metrics"""
|
||||
memory_info = self._get_memory_info()
|
||||
gc_stats = self._get_gc_stats()
|
||||
|
||||
# Update high water mark
|
||||
if memory_info['rss_mb'] > self._high_water_mark:
|
||||
self._high_water_mark = memory_info['rss_mb']
|
||||
|
||||
# Calculate growth rate (MB/hour) if baseline is set
|
||||
growth_rate = 0.0
|
||||
if self._baseline_memory:
|
||||
growth_rate = memory_info['rss_mb'] - self._baseline_memory
|
||||
|
||||
# Record metrics
|
||||
metadata = {
|
||||
'rss_mb': memory_info['rss_mb'],
|
||||
'vms_mb': memory_info['vms_mb'],
|
||||
'percent': memory_info['percent'],
|
||||
'high_water_mb': self._high_water_mark,
|
||||
'growth_mb': growth_rate,
|
||||
'gc_collections': gc_stats['collections'],
|
||||
'gc_collected': gc_stats['collected'],
|
||||
}
|
||||
|
||||
record_metric(
|
||||
'render', # Use 'render' operation type for memory metrics
|
||||
'memory_usage',
|
||||
memory_info['rss_mb'],
|
||||
metadata,
|
||||
force=True # Always record memory metrics
|
||||
)
|
||||
|
||||
# Warn if significant growth detected (>10MB growth from baseline)
|
||||
if growth_rate > 10.0:
|
||||
logger.warning(
|
||||
f"Memory growth detected: +{growth_rate:.2f} MB from baseline "
|
||||
f"(current: {memory_info['rss_mb']:.2f} MB, baseline: {self._baseline_memory:.2f} MB)"
|
||||
)
|
||||
|
||||
def _get_memory_info(self) -> Dict[str, float]:
|
||||
"""
|
||||
Get current process memory usage
|
||||
|
||||
Returns:
|
||||
Dict with memory info in MB
|
||||
"""
|
||||
memory = self._process.memory_info()
|
||||
|
||||
return {
|
||||
'rss_mb': memory.rss / (1024 * 1024), # Resident Set Size
|
||||
'vms_mb': memory.vms / (1024 * 1024), # Virtual Memory Size
|
||||
'percent': self._process.memory_percent(),
|
||||
}
|
||||
|
||||
def _get_gc_stats(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Get garbage collection statistics
|
||||
|
||||
Returns:
|
||||
Dict with GC stats
|
||||
"""
|
||||
# Get collection counts per generation
|
||||
counts = gc.get_count()
|
||||
|
||||
# Perform a quick gen 0 collection and count collected objects
|
||||
collected = gc.collect(0)
|
||||
|
||||
return {
|
||||
'collections': {
|
||||
'gen0': counts[0],
|
||||
'gen1': counts[1],
|
||||
'gen2': counts[2],
|
||||
},
|
||||
'collected': collected,
|
||||
'uncollectable': len(gc.garbage),
|
||||
}
|
||||
|
||||
def stop(self):
|
||||
"""
|
||||
Stop the monitoring thread gracefully
|
||||
|
||||
Sets the stop event to signal the thread to exit
|
||||
"""
|
||||
logger.info("Stopping memory monitor")
|
||||
self._stop_event.set()
|
||||
|
||||
def get_stats(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Get current memory statistics
|
||||
|
||||
Returns:
|
||||
Dict with current memory stats
|
||||
"""
|
||||
if not self._baseline_memory:
|
||||
return {'status': 'initializing'}
|
||||
|
||||
memory_info = self._get_memory_info()
|
||||
|
||||
return {
|
||||
'status': 'running',
|
||||
'current_rss_mb': memory_info['rss_mb'],
|
||||
'baseline_rss_mb': self._baseline_memory,
|
||||
'growth_mb': memory_info['rss_mb'] - self._baseline_memory,
|
||||
'high_water_mb': self._high_water_mark,
|
||||
'percent': memory_info['percent'],
|
||||
}
|
||||
@@ -1,425 +0,0 @@
|
||||
"""
|
||||
Metrics collection and buffering for performance monitoring
|
||||
|
||||
Per ADR-053 and developer Q&A Q6, Q12:
|
||||
- Per-process circular buffers using deque
|
||||
- Configurable buffer size (default 1000 entries)
|
||||
- Include process ID in all metrics
|
||||
- Configuration-based sampling rates
|
||||
- Operation types: database, http, render
|
||||
|
||||
Example usage:
|
||||
>>> from starpunk.monitoring import record_metric, get_metrics
|
||||
>>>
|
||||
>>> # Record a database operation
|
||||
>>> record_metric('database', 'query', duration_ms=45.2, query='SELECT * FROM notes')
|
||||
>>>
|
||||
>>> # Get all metrics
|
||||
>>> metrics = get_metrics()
|
||||
>>> print(f"Collected {len(metrics)} metrics")
|
||||
"""
|
||||
|
||||
import os
|
||||
import random
|
||||
import time
|
||||
from collections import deque
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from datetime import datetime
|
||||
from threading import Lock
|
||||
from typing import Any, Deque, Dict, List, Literal, Optional, Union
|
||||
|
||||
# Operation types for categorizing metrics
|
||||
OperationType = Literal["database", "http", "render"]
|
||||
|
||||
# Module-level circular buffer (per-process)
|
||||
# Each process in a multi-process deployment maintains its own buffer
|
||||
_metrics_buffer: Optional["MetricsBuffer"] = None
|
||||
_buffer_lock = Lock()
|
||||
|
||||
|
||||
@dataclass
|
||||
class Metric:
|
||||
"""
|
||||
Represents a single performance metric
|
||||
|
||||
Attributes:
|
||||
operation_type: Type of operation (database/http/render)
|
||||
operation_name: Name/description of operation
|
||||
timestamp: When the metric was recorded (ISO format)
|
||||
duration_ms: Duration in milliseconds
|
||||
process_id: Process ID that recorded the metric
|
||||
metadata: Additional operation-specific data
|
||||
"""
|
||||
operation_type: OperationType
|
||||
operation_name: str
|
||||
timestamp: str
|
||||
duration_ms: float
|
||||
process_id: int
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert metric to dictionary for serialization"""
|
||||
return asdict(self)
|
||||
|
||||
|
||||
class MetricsBuffer:
|
||||
"""
|
||||
Circular buffer for storing performance metrics
|
||||
|
||||
Per developer Q&A Q6:
|
||||
- Uses deque for efficient circular buffer
|
||||
- Per-process storage (not shared across workers)
|
||||
- Thread-safe with locking
|
||||
- Configurable max size (default 1000)
|
||||
- Automatic eviction of oldest entries when full
|
||||
|
||||
Per developer Q&A Q12:
|
||||
- Configurable sampling rates per operation type
|
||||
- Default 100% sampling (suitable for low-traffic sites)
|
||||
- Slow queries always logged regardless of sampling
|
||||
|
||||
Example:
|
||||
>>> buffer = MetricsBuffer(max_size=1000)
|
||||
>>> buffer.record('database', 'query', 45.2, {'query': 'SELECT ...'})
|
||||
>>> metrics = buffer.get_all()
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
max_size: int = 1000,
|
||||
sampling_rates: Optional[Union[Dict[OperationType, float], float]] = None
|
||||
):
|
||||
"""
|
||||
Initialize metrics buffer
|
||||
|
||||
Args:
|
||||
max_size: Maximum number of metrics to store
|
||||
sampling_rates: Either:
|
||||
- float: Global sampling rate for all operation types (0.0-1.0)
|
||||
- dict: Mapping operation type to sampling rate
|
||||
Default: 1.0 (100% sampling)
|
||||
"""
|
||||
self.max_size = max_size
|
||||
self._buffer: Deque[Metric] = deque(maxlen=max_size)
|
||||
self._lock = Lock()
|
||||
self._process_id = os.getpid()
|
||||
|
||||
# Handle different sampling_rates types
|
||||
if sampling_rates is None:
|
||||
# Default to 100% sampling for all types
|
||||
self._sampling_rates = {
|
||||
"database": 1.0,
|
||||
"http": 1.0,
|
||||
"render": 1.0,
|
||||
}
|
||||
elif isinstance(sampling_rates, (int, float)):
|
||||
# Global rate for all types
|
||||
rate = float(sampling_rates)
|
||||
self._sampling_rates = {
|
||||
"database": rate,
|
||||
"http": rate,
|
||||
"render": rate,
|
||||
}
|
||||
else:
|
||||
# Dict with per-type rates
|
||||
self._sampling_rates = sampling_rates
|
||||
|
||||
def record(
|
||||
self,
|
||||
operation_type: OperationType,
|
||||
operation_name: str,
|
||||
duration_ms: float,
|
||||
metadata: Optional[Dict[str, Any]] = None,
|
||||
force: bool = False
|
||||
) -> bool:
|
||||
"""
|
||||
Record a performance metric
|
||||
|
||||
Args:
|
||||
operation_type: Type of operation (database/http/render)
|
||||
operation_name: Name/description of operation
|
||||
duration_ms: Duration in milliseconds
|
||||
metadata: Additional operation-specific data
|
||||
force: If True, bypass sampling (for slow query logging)
|
||||
|
||||
Returns:
|
||||
True if metric was recorded, False if skipped due to sampling
|
||||
|
||||
Example:
|
||||
>>> buffer.record('database', 'SELECT notes', 45.2,
|
||||
... {'query': 'SELECT * FROM notes LIMIT 10'})
|
||||
True
|
||||
"""
|
||||
# Apply sampling (unless forced)
|
||||
if not force:
|
||||
sampling_rate = self._sampling_rates.get(operation_type, 0.1)
|
||||
if random.random() > sampling_rate:
|
||||
return False
|
||||
|
||||
metric = Metric(
|
||||
operation_type=operation_type,
|
||||
operation_name=operation_name,
|
||||
timestamp=datetime.utcnow().isoformat() + "Z",
|
||||
duration_ms=duration_ms,
|
||||
process_id=self._process_id,
|
||||
metadata=metadata or {}
|
||||
)
|
||||
|
||||
with self._lock:
|
||||
self._buffer.append(metric)
|
||||
|
||||
return True
|
||||
|
||||
def get_all(self) -> List[Metric]:
|
||||
"""
|
||||
Get all metrics from buffer
|
||||
|
||||
Returns:
|
||||
List of metrics (oldest to newest)
|
||||
|
||||
Example:
|
||||
>>> metrics = buffer.get_all()
|
||||
>>> len(metrics)
|
||||
1000
|
||||
"""
|
||||
with self._lock:
|
||||
return list(self._buffer)
|
||||
|
||||
def get_recent(self, count: int) -> List[Metric]:
|
||||
"""
|
||||
Get most recent N metrics
|
||||
|
||||
Args:
|
||||
count: Number of recent metrics to return
|
||||
|
||||
Returns:
|
||||
List of most recent metrics (newest first)
|
||||
|
||||
Example:
|
||||
>>> recent = buffer.get_recent(10)
|
||||
>>> len(recent)
|
||||
10
|
||||
"""
|
||||
with self._lock:
|
||||
# Convert to list, reverse to get newest first, then slice
|
||||
all_metrics = list(self._buffer)
|
||||
all_metrics.reverse()
|
||||
return all_metrics[:count]
|
||||
|
||||
def get_by_type(self, operation_type: OperationType) -> List[Metric]:
|
||||
"""
|
||||
Get all metrics of a specific type
|
||||
|
||||
Args:
|
||||
operation_type: Type to filter by (database/http/render)
|
||||
|
||||
Returns:
|
||||
List of metrics matching the type
|
||||
|
||||
Example:
|
||||
>>> db_metrics = buffer.get_by_type('database')
|
||||
"""
|
||||
with self._lock:
|
||||
return [m for m in self._buffer if m.operation_type == operation_type]
|
||||
|
||||
def get_slow_operations(
|
||||
self,
|
||||
threshold_ms: float = 1000.0,
|
||||
operation_type: Optional[OperationType] = None
|
||||
) -> List[Metric]:
|
||||
"""
|
||||
Get operations that exceeded a duration threshold
|
||||
|
||||
Args:
|
||||
threshold_ms: Duration threshold in milliseconds
|
||||
operation_type: Optional type filter
|
||||
|
||||
Returns:
|
||||
List of slow operations
|
||||
|
||||
Example:
|
||||
>>> slow_queries = buffer.get_slow_operations(1000, 'database')
|
||||
"""
|
||||
with self._lock:
|
||||
metrics = list(self._buffer)
|
||||
|
||||
# Filter by type if specified
|
||||
if operation_type:
|
||||
metrics = [m for m in metrics if m.operation_type == operation_type]
|
||||
|
||||
# Filter by duration threshold
|
||||
return [m for m in metrics if m.duration_ms >= threshold_ms]
|
||||
|
||||
def get_stats(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Get statistics about the buffer
|
||||
|
||||
Returns:
|
||||
Dict with buffer statistics
|
||||
|
||||
Example:
|
||||
>>> stats = buffer.get_stats()
|
||||
>>> stats['total_count']
|
||||
1000
|
||||
"""
|
||||
with self._lock:
|
||||
metrics = list(self._buffer)
|
||||
|
||||
# Calculate stats per operation type
|
||||
type_stats = {}
|
||||
for op_type in ["database", "http", "render"]:
|
||||
type_metrics = [m for m in metrics if m.operation_type == op_type]
|
||||
if type_metrics:
|
||||
durations = [m.duration_ms for m in type_metrics]
|
||||
type_stats[op_type] = {
|
||||
"count": len(type_metrics),
|
||||
"avg_duration_ms": sum(durations) / len(durations),
|
||||
"min_duration_ms": min(durations),
|
||||
"max_duration_ms": max(durations),
|
||||
}
|
||||
else:
|
||||
type_stats[op_type] = {
|
||||
"count": 0,
|
||||
"avg_duration_ms": 0.0,
|
||||
"min_duration_ms": 0.0,
|
||||
"max_duration_ms": 0.0,
|
||||
}
|
||||
|
||||
return {
|
||||
"total_count": len(metrics),
|
||||
"max_size": self.max_size,
|
||||
"process_id": self._process_id,
|
||||
"sampling_rates": self._sampling_rates,
|
||||
"by_type": type_stats,
|
||||
}
|
||||
|
||||
def clear(self) -> None:
|
||||
"""
|
||||
Clear all metrics from buffer
|
||||
|
||||
Example:
|
||||
>>> buffer.clear()
|
||||
"""
|
||||
with self._lock:
|
||||
self._buffer.clear()
|
||||
|
||||
def set_sampling_rate(
|
||||
self,
|
||||
operation_type: OperationType,
|
||||
rate: float
|
||||
) -> None:
|
||||
"""
|
||||
Update sampling rate for an operation type
|
||||
|
||||
Args:
|
||||
operation_type: Type to update
|
||||
rate: New sampling rate (0.0-1.0)
|
||||
|
||||
Example:
|
||||
>>> buffer.set_sampling_rate('database', 0.5) # 50% sampling
|
||||
"""
|
||||
if not 0.0 <= rate <= 1.0:
|
||||
raise ValueError("Sampling rate must be between 0.0 and 1.0")
|
||||
|
||||
with self._lock:
|
||||
self._sampling_rates[operation_type] = rate
|
||||
|
||||
|
||||
def get_buffer() -> MetricsBuffer:
|
||||
"""
|
||||
Get or create the module-level metrics buffer
|
||||
|
||||
This ensures a single buffer per process. In multi-process deployments
|
||||
(e.g., gunicorn), each worker process will have its own buffer.
|
||||
|
||||
Returns:
|
||||
MetricsBuffer instance for this process
|
||||
|
||||
Example:
|
||||
>>> buffer = get_buffer()
|
||||
>>> buffer.record('database', 'query', 45.2)
|
||||
"""
|
||||
global _metrics_buffer
|
||||
|
||||
if _metrics_buffer is None:
|
||||
with _buffer_lock:
|
||||
# Double-check locking pattern
|
||||
if _metrics_buffer is None:
|
||||
# Get configuration from Flask app if available
|
||||
try:
|
||||
from flask import current_app
|
||||
max_size = current_app.config.get('METRICS_BUFFER_SIZE', 1000)
|
||||
sampling_rate = current_app.config.get('METRICS_SAMPLING_RATE', 1.0)
|
||||
except (ImportError, RuntimeError):
|
||||
# Flask not available or no app context
|
||||
max_size = 1000
|
||||
sampling_rate = 1.0 # Default to 100%
|
||||
|
||||
_metrics_buffer = MetricsBuffer(
|
||||
max_size=max_size,
|
||||
sampling_rates=sampling_rate
|
||||
)
|
||||
|
||||
return _metrics_buffer
|
||||
|
||||
|
||||
def record_metric(
|
||||
operation_type: OperationType,
|
||||
operation_name: str,
|
||||
duration_ms: float,
|
||||
metadata: Optional[Dict[str, Any]] = None,
|
||||
force: bool = False
|
||||
) -> bool:
|
||||
"""
|
||||
Record a metric using the module-level buffer
|
||||
|
||||
Convenience function that uses get_buffer() internally.
|
||||
|
||||
Args:
|
||||
operation_type: Type of operation (database/http/render)
|
||||
operation_name: Name/description of operation
|
||||
duration_ms: Duration in milliseconds
|
||||
metadata: Additional operation-specific data
|
||||
force: If True, bypass sampling (for slow query logging)
|
||||
|
||||
Returns:
|
||||
True if metric was recorded, False if skipped due to sampling
|
||||
|
||||
Example:
|
||||
>>> record_metric('database', 'SELECT notes', 45.2,
|
||||
... {'query': 'SELECT * FROM notes LIMIT 10'})
|
||||
True
|
||||
"""
|
||||
buffer = get_buffer()
|
||||
return buffer.record(operation_type, operation_name, duration_ms, metadata, force)
|
||||
|
||||
|
||||
def get_metrics() -> List[Metric]:
|
||||
"""
|
||||
Get all metrics from the module-level buffer
|
||||
|
||||
Returns:
|
||||
List of metrics (oldest to newest)
|
||||
|
||||
Example:
|
||||
>>> metrics = get_metrics()
|
||||
>>> len(metrics)
|
||||
1000
|
||||
"""
|
||||
buffer = get_buffer()
|
||||
return buffer.get_all()
|
||||
|
||||
|
||||
def get_metrics_stats() -> Dict[str, Any]:
|
||||
"""
|
||||
Get statistics from the module-level buffer
|
||||
|
||||
Returns:
|
||||
Dict with buffer statistics
|
||||
|
||||
Example:
|
||||
>>> stats = get_metrics_stats()
|
||||
>>> print(f"Total metrics: {stats['total_count']}")
|
||||
"""
|
||||
buffer = get_buffer()
|
||||
return buffer.get_stats()
|
||||
+4
-4
@@ -20,7 +20,7 @@ Exceptions:
|
||||
"""
|
||||
|
||||
# Standard library imports
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
@@ -226,7 +226,7 @@ def create_note(
|
||||
|
||||
# 2. SETUP
|
||||
if created_at is None:
|
||||
created_at = datetime.utcnow()
|
||||
created_at = datetime.now(timezone.utc).replace(tzinfo=None)
|
||||
|
||||
updated_at = created_at # Same as created_at for new notes
|
||||
|
||||
@@ -735,7 +735,7 @@ def update_note(
|
||||
)
|
||||
|
||||
# 3. SETUP
|
||||
updated_at = datetime.utcnow()
|
||||
updated_at = datetime.now(timezone.utc).replace(tzinfo=None)
|
||||
data_dir = Path(current_app.config["DATA_PATH"])
|
||||
note_path = data_dir / existing_note.file_path
|
||||
|
||||
@@ -1052,7 +1052,7 @@ def delete_note(
|
||||
|
||||
if soft:
|
||||
# SOFT DELETE: Mark as deleted in database
|
||||
deleted_at = datetime.utcnow()
|
||||
deleted_at = datetime.now(timezone.utc).replace(tzinfo=None)
|
||||
|
||||
try:
|
||||
db.execute(
|
||||
|
||||
@@ -0,0 +1,236 @@
|
||||
"""
|
||||
Notifications through Apprise (v2.14.0, ADR-074)
|
||||
|
||||
The owner hears about new responses (received webmentions and backfeed
|
||||
alike) and about crossposting problems, on whatever service the Apprise
|
||||
URLs in NOTIFY_URLS point at (Gotify, ntfy, Pushover, email, an
|
||||
apprise-api server…).
|
||||
|
||||
Responses are sent by a sweep (`sweep`) in the webmention worker, which
|
||||
runs in one process at a time: each response with `notified_at` empty is
|
||||
sent once and marked. Likes, reposts and bookmarks are grouped per note.
|
||||
Failed sends are retried with backoff; responses still unsent after 24
|
||||
hours are dropped. Problems (`problem`) are sent at once, without retry.
|
||||
|
||||
Nothing here may break validation, crossposting or backfeed: every send is
|
||||
wrapped, and failures are only logged and shown in the admin.
|
||||
"""
|
||||
|
||||
import html
|
||||
import logging
|
||||
import re
|
||||
import sqlite3
|
||||
import time
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from urllib.parse import parse_qsl, urlsplit
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
RESPONSE_EVENTS = ("reply", "mention", "quote", "like", "repost", "bookmark")
|
||||
EVENTS = RESPONSE_EVENTS + ("problems",)
|
||||
DEFAULT_EVENTS = "reply,mention,quote,problems"
|
||||
GROUPED = ("like", "repost", "bookmark")
|
||||
DROP_AFTER_HOURS = 24
|
||||
MAX_BACKOFF = 15 * 60
|
||||
SEND_TIMEOUT = 15
|
||||
|
||||
HEADINGS = {
|
||||
"reply": "↩ Reply from {who}",
|
||||
"mention": "💬 Mention from {who}",
|
||||
"quote": "❝ Quote from {who}",
|
||||
}
|
||||
GROUP_HEADINGS = {
|
||||
"like": ("♥", "like", "likes"),
|
||||
"repost": ("🔁", "repost", "reposts"),
|
||||
"bookmark": ("🔖", "bookmark", "bookmarks"),
|
||||
}
|
||||
|
||||
|
||||
# --- Settings ---------------------------------------------------------------------
|
||||
|
||||
def urls(config) -> List[str]:
|
||||
"""NOTIFY_URLS as a list (space- or comma-separated)"""
|
||||
return [u for u in re.split(r"[\s,]+", config.get("NOTIFY_URLS") or "") if u]
|
||||
|
||||
|
||||
def events(config) -> set:
|
||||
"""NOTIFY_EVENTS as a set of known event names"""
|
||||
chosen = {e.strip().lower() for e in (config.get("NOTIFY_EVENTS") or DEFAULT_EVENTS).split(",")}
|
||||
return chosen & set(EVENTS)
|
||||
|
||||
|
||||
def enabled(config) -> bool:
|
||||
return bool(urls(config))
|
||||
|
||||
|
||||
def with_priority(url: str, priority: str) -> str:
|
||||
"""The URL with `priority=` added, unless it sets one itself"""
|
||||
if not priority or any(k == "priority" for k, _ in parse_qsl(urlsplit(url).query)):
|
||||
return url
|
||||
return url + ("&" if "?" in url else "?") + "priority=" + priority
|
||||
|
||||
|
||||
# --- Sending ----------------------------------------------------------------------------
|
||||
|
||||
def _deliver(targets: List[str], title: str, body: str) -> Tuple[bool, str]:
|
||||
"""Send one message to every URL. Returns (all sent, error text)."""
|
||||
import apprise
|
||||
|
||||
sender = apprise.Apprise()
|
||||
bad = [u.split("://", 1)[0] + "://…" for u in targets if not sender.add(u)]
|
||||
if bad:
|
||||
return False, f"Not a usable Apprise URL: {', '.join(bad)}"
|
||||
with apprise.LogCapture(level=logging.WARNING, fmt="%(message)s") as log:
|
||||
result = sender.notify(body, title=title, body_format=apprise.NotifyFormat.MARKDOWN,
|
||||
timeout=SEND_TIMEOUT)
|
||||
return bool(result), log.getvalue().strip()[:500]
|
||||
|
||||
|
||||
def send(config, title: str, body: str) -> Tuple[bool, str]:
|
||||
"""Send to all NOTIFY_URLS at NOTIFY_PRIORITY. Never raises."""
|
||||
targets = [with_priority(u, config.get("NOTIFY_PRIORITY") or "normal") for u in urls(config)]
|
||||
if not targets:
|
||||
return False, "No NOTIFY_URLS set"
|
||||
try:
|
||||
ok, error = _deliver(targets, title, body)
|
||||
except Exception as e: # a broken service mustn't break the caller
|
||||
ok, error = False, f"{type(e).__name__}: {e}"
|
||||
if not ok:
|
||||
logger.warning(f"Notification '{title}' not sent: {error or 'unknown error'}")
|
||||
return ok, error
|
||||
|
||||
|
||||
def problem(config, db_path, title: str, detail: str, link: str) -> None:
|
||||
"""A crosspost or backfeed problem, sent at once if 'problems' is selected"""
|
||||
if not enabled(config) or "problems" not in events(config):
|
||||
return
|
||||
ok, error = send(config, f"⚠ {title}", f"{detail}\n\n[Open in StarPunk]({link})")
|
||||
_record(db_path, ok, error)
|
||||
|
||||
|
||||
def _record(db_path, ok: bool, error: str) -> None:
|
||||
"""Remember the last failure for the admin page (cleared by a success)"""
|
||||
try:
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
conn.execute("INSERT INTO site_state (key, value) VALUES ('notify_last_error', ?) "
|
||||
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||
("" if ok else time.strftime("%Y-%m-%d %H:%M:%S UTC ", time.gmtime())
|
||||
+ (error or "unknown error"),))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
except sqlite3.Error as e:
|
||||
logger.warning(f"Could not record notification result: {e}")
|
||||
|
||||
|
||||
def last_error(db) -> str:
|
||||
row = db.execute("SELECT value FROM site_state WHERE key = 'notify_last_error'").fetchone()
|
||||
return row[0] if row else ""
|
||||
|
||||
|
||||
# --- Messages -----------------------------------------------------------------------------
|
||||
|
||||
def _plain(snippet: Optional[str], limit: int = 300) -> str:
|
||||
text = html.unescape(re.sub(r"<[^>]+>", " ", snippet or ""))
|
||||
text = re.sub(r"\s+", " ", text).strip()
|
||||
return text if len(text) <= limit else text[:limit - 1].rsplit(" ", 1)[0] + "…"
|
||||
|
||||
|
||||
def _note_label(conn, note_id: int) -> str:
|
||||
from starpunk.notes import get_note
|
||||
|
||||
try:
|
||||
note = get_note(id=note_id, load_content=True)
|
||||
label = " ".join((note.title or "").split()) if note else ""
|
||||
except Exception:
|
||||
label = ""
|
||||
return (label[:57] + "…") if len(label) > 60 else (label or "a note")
|
||||
|
||||
|
||||
def _messages(rows, site_url: str, conn) -> List[Tuple[str, str, List[int]]]:
|
||||
"""(title, body, webmention ids) for a batch of responses"""
|
||||
site = site_url.rstrip("/")
|
||||
names = {"gotosocial": "GoToSocial", "bluesky": "Bluesky", "threads": "Threads"}
|
||||
messages, groups = [], {}
|
||||
for row in rows:
|
||||
via = names.get(row["origin"]) if row["origin"] != "webmention" else None
|
||||
who = (row["author_name"] or "someone") + (f" via {via}" if via else "")
|
||||
pending = row["moderation_status"] == "pending"
|
||||
link = (f"{site}/admin/webmentions/{row['id']}" if pending
|
||||
else f"{site}/note/{row['slug']}#responses")
|
||||
if row["mention_type"] in GROUPED:
|
||||
groups.setdefault((row["target_note_id"], row["mention_type"], row["slug"], pending), []).append(
|
||||
(row["id"], row["author_name"] or "someone", link))
|
||||
continue
|
||||
title = HEADINGS.get(row["mention_type"], "Response from {who}").format(who=who)
|
||||
if pending:
|
||||
title += " (awaiting moderation)"
|
||||
text = _plain(row["content_snippet"])
|
||||
body = (f"{text}\n\n" if text else "") + f"On “{_note_label(conn, row['target_note_id'])}”: [open]({link})"
|
||||
messages.append((title, body, [row["id"]]))
|
||||
for (note_id, kind, slug, pending), items in groups.items():
|
||||
icon, one, many = GROUP_HEADINGS[kind]
|
||||
n = len(items)
|
||||
title = f"{icon} {n} new {one if n == 1 else many} on “{_note_label(conn, note_id)}”"
|
||||
if pending:
|
||||
title += " (awaiting moderation)"
|
||||
people = ", ".join(name for _, name, _ in items[:5]) + (f" and {n - 5} more" if n > 5 else "")
|
||||
messages.append((title, f"{people}\n\n[open]({items[0][2]})", [i for i, _, _ in items]))
|
||||
return messages
|
||||
|
||||
|
||||
# --- Sweep ----------------------------------------------------------------------------------
|
||||
|
||||
class Sweeper:
|
||||
"""Sends new responses; one instance per webmention worker"""
|
||||
|
||||
def __init__(self):
|
||||
self._failures = 0
|
||||
self._next_try = 0.0
|
||||
|
||||
def run(self, config, db_path, site_url: str) -> int:
|
||||
"""Send what's due. Returns how many responses were sent."""
|
||||
if not enabled(config) or time.time() < self._next_try:
|
||||
return 0
|
||||
chosen = events(config)
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
try:
|
||||
# Too old to be news (an outage, say): drop quietly
|
||||
dropped = conn.execute(
|
||||
f"""UPDATE webmentions SET notified_at = CURRENT_TIMESTAMP
|
||||
WHERE notified_at IS NULL
|
||||
AND received_at < datetime('now', '-{DROP_AFTER_HOURS} hours')""").rowcount
|
||||
if dropped:
|
||||
logger.warning(f"Dropped {dropped} notification(s) older than {DROP_AFTER_HOURS} hours")
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT w.*, n.slug FROM webmentions w JOIN notes n ON n.id = w.target_note_id
|
||||
WHERE w.notified_at IS NULL AND w.validation_status = 'valid'
|
||||
AND w.moderation_status IN ('approved', 'pending')
|
||||
ORDER BY w.id LIMIT 100
|
||||
"""
|
||||
).fetchall()
|
||||
skip = [r["id"] for r in rows if r["mention_type"] not in chosen]
|
||||
if skip:
|
||||
conn.execute(f"UPDATE webmentions SET notified_at = CURRENT_TIMESTAMP "
|
||||
f"WHERE id IN ({','.join('?' * len(skip))})", skip)
|
||||
conn.commit()
|
||||
due = [r for r in rows if r["mention_type"] in chosen]
|
||||
sent = 0
|
||||
for title, body, ids in _messages(due, site_url, conn):
|
||||
ok, error = send(config, title, body)
|
||||
_record(db_path, ok, error)
|
||||
if not ok:
|
||||
self._failures += 1
|
||||
self._next_try = time.time() + min(10 * 2 ** self._failures, MAX_BACKOFF)
|
||||
break
|
||||
self._failures = 0
|
||||
conn.execute(f"UPDATE webmentions SET notified_at = CURRENT_TIMESTAMP "
|
||||
f"WHERE id IN ({','.join('?' * len(ids))})", ids)
|
||||
conn.commit()
|
||||
sent += len(ids)
|
||||
return sent
|
||||
finally:
|
||||
conn.close()
|
||||
@@ -138,6 +138,79 @@ _ip_limiter: RateLimiter | None = None
|
||||
_domain_limiter: RateLimiter | None = None
|
||||
|
||||
|
||||
class SharedRateLimiter:
|
||||
"""
|
||||
Rate limiter whose counts live in SQLite, shared by every gunicorn
|
||||
process (v2.11.0). In-memory counters let each of the 4 workers allow the
|
||||
full limit, so the real limit was four times the setting.
|
||||
|
||||
Sliding-window counter: the current window's count plus the previous
|
||||
window's, weighted by how much of it still overlaps the last
|
||||
`window_seconds`. Same interface as RateLimiter; needs an app context.
|
||||
"""
|
||||
|
||||
CLEANUP_EVERY = 200 # calls between removals of old windows
|
||||
|
||||
def __init__(self, limit: int, window_seconds: int, prefix: str):
|
||||
self.limit = limit
|
||||
self.window_seconds = window_seconds
|
||||
self.prefix = prefix
|
||||
self._calls = 0
|
||||
|
||||
def _db(self):
|
||||
from flask import current_app
|
||||
from starpunk.database import get_db
|
||||
return get_db(current_app)
|
||||
|
||||
def _windows(self):
|
||||
now = time.time()
|
||||
current = int(now // self.window_seconds)
|
||||
overlap = 1 - (now % self.window_seconds) / self.window_seconds
|
||||
return now, current, overlap
|
||||
|
||||
def _estimate(self, db, key: str) -> float:
|
||||
_, current, overlap = self._windows()
|
||||
counts = dict(db.execute(
|
||||
"SELECT window, count FROM rate_limits WHERE key = ? AND window IN (?, ?)",
|
||||
(f"{self.prefix}:{key}", current - 1, current),
|
||||
).fetchall())
|
||||
return counts.get(current - 1, 0) * overlap + counts.get(current, 0)
|
||||
|
||||
def is_allowed(self, key: str) -> bool:
|
||||
db = self._db()
|
||||
_, current, _ = self._windows()
|
||||
if self._estimate(db, key) >= self.limit:
|
||||
return False
|
||||
db.execute(
|
||||
"INSERT INTO rate_limits (key, window, count) VALUES (?, ?, 1) "
|
||||
"ON CONFLICT(key, window) DO UPDATE SET count = count + 1",
|
||||
(f"{self.prefix}:{key}", current),
|
||||
)
|
||||
db.commit()
|
||||
self._calls += 1
|
||||
if self._calls % self.CLEANUP_EVERY == 0:
|
||||
self.cleanup()
|
||||
return True
|
||||
|
||||
def get_retry_after(self, key: str) -> int:
|
||||
now, current, _ = self._windows()
|
||||
return max(1, int((current + 1) * self.window_seconds - now) + 1)
|
||||
|
||||
def get_request_count(self, key: str) -> int:
|
||||
return int(self._estimate(self._db(), key))
|
||||
|
||||
def cleanup(self, max_idle_seconds: int = 300) -> int:
|
||||
"""Remove windows too old to matter (this limiter's keys only)"""
|
||||
_, current, _ = self._windows()
|
||||
db = self._db()
|
||||
cursor = db.execute(
|
||||
"DELETE FROM rate_limits WHERE key LIKE ? AND window < ?",
|
||||
(f"{self.prefix}:%", current - 1),
|
||||
)
|
||||
db.commit()
|
||||
return cursor.rowcount
|
||||
|
||||
|
||||
def init_rate_limiters(app) -> None:
|
||||
"""
|
||||
Initialize rate limiters with Flask app configuration.
|
||||
@@ -157,8 +230,9 @@ def init_rate_limiters(app) -> None:
|
||||
domain_limit = app.config.get('WEBMENTION_RATE_LIMIT_DOMAIN', 10)
|
||||
window = app.config.get('WEBMENTION_RATE_LIMIT_WINDOW', 60)
|
||||
|
||||
_ip_limiter = RateLimiter(ip_limit, window)
|
||||
_domain_limiter = RateLimiter(domain_limit, window)
|
||||
# Shared by all server processes (v2.11.0)
|
||||
_ip_limiter = SharedRateLimiter(ip_limit, window, "webmention-ip")
|
||||
_domain_limiter = SharedRateLimiter(domain_limit, window, "webmention-domain")
|
||||
|
||||
app.logger.info(
|
||||
f"Webmention rate limiter initialized: {ip_limit}/min IP, "
|
||||
|
||||
+64
-210
@@ -5,11 +5,10 @@ Handles authenticated admin functionality including dashboard, note creation,
|
||||
editing, and deletion. All routes require authentication.
|
||||
"""
|
||||
|
||||
from flask import Blueprint, flash, g, jsonify, redirect, render_template, request, url_for
|
||||
from flask import Blueprint, current_app, flash, g, jsonify, redirect, render_template, request, url_for
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
from datetime import datetime
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -260,6 +259,7 @@ def edit_note_form(note_id: int):
|
||||
syndication_targets=get_targets().values(),
|
||||
syndications=syndications,
|
||||
syndication_enabled=enabled,
|
||||
backfeed_enabled=current_app.config.get("BACKFEED_ENABLED", False),
|
||||
)
|
||||
|
||||
|
||||
@@ -309,7 +309,9 @@ def update_note_submit(note_id: int):
|
||||
id=note_id,
|
||||
content=content,
|
||||
published=published,
|
||||
tags=tags if tags else None
|
||||
# An empty Tags field removes all tags (v2.14.1); a form without
|
||||
# the field leaves them alone
|
||||
tags=tags if "tags" in request.form else None
|
||||
)
|
||||
if "syndication_form" in request.form:
|
||||
from starpunk.syndication import select_targets
|
||||
@@ -366,202 +368,6 @@ def delete_note_submit(note_id: int):
|
||||
return redirect(url_for("admin.dashboard"))
|
||||
|
||||
|
||||
def transform_metrics_for_template(metrics_stats):
|
||||
"""
|
||||
Transform metrics stats to match template structure
|
||||
|
||||
The template expects direct access to metrics.database.count, but
|
||||
get_metrics_stats() returns metrics.by_type.database.count.
|
||||
This function adapts the data structure to match template expectations.
|
||||
|
||||
Args:
|
||||
metrics_stats: Dict from get_metrics_stats() with nested by_type structure
|
||||
|
||||
Returns:
|
||||
Dict with flattened structure matching template expectations
|
||||
|
||||
Per ADR-060: Route Adapter Pattern for template compatibility
|
||||
"""
|
||||
transformed = {}
|
||||
|
||||
# Map by_type to direct access
|
||||
for op_type in ['database', 'http', 'render']:
|
||||
if 'by_type' in metrics_stats and op_type in metrics_stats['by_type']:
|
||||
type_data = metrics_stats['by_type'][op_type]
|
||||
transformed[op_type] = {
|
||||
'count': type_data.get('count', 0),
|
||||
'avg': type_data.get('avg_duration_ms', 0),
|
||||
'min': type_data.get('min_duration_ms', 0),
|
||||
'max': type_data.get('max_duration_ms', 0)
|
||||
}
|
||||
else:
|
||||
# Provide defaults for missing types or when by_type doesn't exist
|
||||
transformed[op_type] = {
|
||||
'count': 0,
|
||||
'avg': 0,
|
||||
'min': 0,
|
||||
'max': 0
|
||||
}
|
||||
|
||||
# Keep other top-level stats
|
||||
transformed['total_count'] = metrics_stats.get('total_count', 0)
|
||||
transformed['max_size'] = metrics_stats.get('max_size', 1000)
|
||||
transformed['process_id'] = metrics_stats.get('process_id', 0)
|
||||
|
||||
return transformed
|
||||
|
||||
|
||||
@bp.route("/metrics-dashboard")
|
||||
@require_auth
|
||||
def metrics_dashboard():
|
||||
"""
|
||||
Metrics visualization dashboard (Phase 3)
|
||||
|
||||
Displays performance metrics, database statistics, feed statistics,
|
||||
and system health with visual charts and auto-refresh capability.
|
||||
|
||||
Per Q19 requirements:
|
||||
- Server-side rendering with Jinja2
|
||||
- htmx for auto-refresh
|
||||
- Chart.js from CDN for graphs
|
||||
- Progressive enhancement (works without JS)
|
||||
|
||||
Per v1.1.2 Phase 3:
|
||||
- Feed statistics by format
|
||||
- Cache hit/miss rates
|
||||
- Format popularity breakdown
|
||||
|
||||
Returns:
|
||||
Rendered dashboard template with metrics
|
||||
|
||||
Decorator: @require_auth
|
||||
Template: templates/admin/metrics_dashboard.html
|
||||
"""
|
||||
# Defensive imports with graceful degradation for missing modules
|
||||
try:
|
||||
from starpunk.database.pool import get_pool_stats
|
||||
from starpunk.monitoring import get_metrics_stats
|
||||
from starpunk.monitoring.business import get_feed_statistics
|
||||
monitoring_available = True
|
||||
except ImportError:
|
||||
monitoring_available = False
|
||||
# Provide fallback functions that return error messages
|
||||
def get_pool_stats():
|
||||
return {"error": "Database pool monitoring not available"}
|
||||
def get_metrics_stats():
|
||||
return {"error": "Monitoring module not implemented"}
|
||||
def get_feed_statistics():
|
||||
return {"error": "Feed statistics not available"}
|
||||
|
||||
# Get current metrics for initial page load
|
||||
metrics_data = {}
|
||||
pool_stats = {}
|
||||
feed_stats = {}
|
||||
|
||||
try:
|
||||
raw_metrics = get_metrics_stats()
|
||||
metrics_data = transform_metrics_for_template(raw_metrics)
|
||||
except Exception as e:
|
||||
flash(f"Error loading metrics: {e}", "warning")
|
||||
# Provide safe defaults matching template expectations
|
||||
metrics_data = {
|
||||
'database': {'count': 0, 'avg': 0, 'min': 0, 'max': 0},
|
||||
'http': {'count': 0, 'avg': 0, 'min': 0, 'max': 0},
|
||||
'render': {'count': 0, 'avg': 0, 'min': 0, 'max': 0},
|
||||
'total_count': 0,
|
||||
'max_size': 1000,
|
||||
'process_id': 0
|
||||
}
|
||||
|
||||
try:
|
||||
pool_stats = get_pool_stats()
|
||||
except Exception as e:
|
||||
flash(f"Error loading pool stats: {e}", "warning")
|
||||
|
||||
try:
|
||||
feed_stats = get_feed_statistics()
|
||||
except Exception as e:
|
||||
flash(f"Error loading feed stats: {e}", "warning")
|
||||
# Provide safe defaults
|
||||
feed_stats = {
|
||||
'by_format': {
|
||||
'rss': {'generated': 0, 'cached': 0, 'total': 0, 'avg_duration_ms': 0.0},
|
||||
'atom': {'generated': 0, 'cached': 0, 'total': 0, 'avg_duration_ms': 0.0},
|
||||
'json': {'generated': 0, 'cached': 0, 'total': 0, 'avg_duration_ms': 0.0},
|
||||
},
|
||||
'cache': {'hits': 0, 'misses': 0, 'hit_rate': 0.0, 'entries': 0, 'evictions': 0},
|
||||
'total_requests': 0,
|
||||
'format_percentages': {'rss': 0.0, 'atom': 0.0, 'json': 0.0},
|
||||
}
|
||||
|
||||
return render_template(
|
||||
"admin/metrics_dashboard.html",
|
||||
metrics=metrics_data,
|
||||
pool=pool_stats,
|
||||
feeds=feed_stats,
|
||||
user_me=g.me
|
||||
)
|
||||
|
||||
|
||||
@bp.route("/metrics")
|
||||
@require_auth
|
||||
def metrics():
|
||||
"""
|
||||
Performance metrics and database pool statistics endpoint
|
||||
|
||||
Per Phase 2 requirements:
|
||||
- Expose database pool statistics
|
||||
- Show performance metrics from MetricsBuffer
|
||||
- Requires authentication
|
||||
|
||||
Per v1.1.2 Phase 3:
|
||||
- Include feed statistics
|
||||
|
||||
Returns:
|
||||
JSON with metrics, pool statistics, and feed statistics
|
||||
|
||||
Response codes:
|
||||
200: Metrics retrieved successfully
|
||||
|
||||
Decorator: @require_auth
|
||||
"""
|
||||
from flask import current_app
|
||||
from starpunk.database.pool import get_pool_stats
|
||||
from starpunk.monitoring import get_metrics_stats
|
||||
from starpunk.monitoring.business import get_feed_statistics
|
||||
|
||||
response = {
|
||||
"timestamp": datetime.utcnow().isoformat() + "Z",
|
||||
"process_id": os.getpid(),
|
||||
"database": {},
|
||||
"performance": {},
|
||||
"feeds": {}
|
||||
}
|
||||
|
||||
# Get database pool statistics
|
||||
try:
|
||||
pool_stats = get_pool_stats()
|
||||
response["database"]["pool"] = pool_stats
|
||||
except Exception as e:
|
||||
response["database"]["pool"] = {"error": str(e)}
|
||||
|
||||
# Get performance metrics
|
||||
try:
|
||||
metrics_stats = get_metrics_stats()
|
||||
response["performance"] = metrics_stats
|
||||
except Exception as e:
|
||||
response["performance"] = {"error": str(e)}
|
||||
|
||||
# Get feed statistics
|
||||
try:
|
||||
feed_stats = get_feed_statistics()
|
||||
response["feeds"] = feed_stats
|
||||
except Exception as e:
|
||||
response["feeds"] = {"error": str(e)}
|
||||
|
||||
return jsonify(response), 200
|
||||
|
||||
|
||||
@bp.route("/health")
|
||||
@require_auth
|
||||
def health_diagnostics():
|
||||
@@ -571,7 +377,7 @@ def health_diagnostics():
|
||||
Per developer Q&A Q10:
|
||||
- Always requires authentication
|
||||
- Provides comprehensive diagnostics
|
||||
- Includes metrics, database pool statistics, and system info
|
||||
- Includes database pool statistics and system info
|
||||
|
||||
Returns:
|
||||
JSON with complete system diagnostics
|
||||
@@ -591,7 +397,6 @@ def health_diagnostics():
|
||||
"environment": current_app.config.get("ENV", "unknown"),
|
||||
"process_id": os.getpid(),
|
||||
"checks": {},
|
||||
"metrics": {},
|
||||
"database": {}
|
||||
}
|
||||
|
||||
@@ -665,14 +470,6 @@ def health_diagnostics():
|
||||
}
|
||||
overall_healthy = False
|
||||
|
||||
# Performance metrics
|
||||
try:
|
||||
from starpunk.monitoring import get_metrics_stats
|
||||
metrics_stats = get_metrics_stats()
|
||||
diagnostics["metrics"] = metrics_stats
|
||||
except Exception as e:
|
||||
diagnostics["metrics"] = {"error": str(e)}
|
||||
|
||||
# Update overall status
|
||||
diagnostics["status"] = "healthy" if overall_healthy else "unhealthy"
|
||||
|
||||
@@ -702,18 +499,25 @@ def webmentions():
|
||||
from starpunk.webmention_queries import list_webmentions
|
||||
|
||||
status_filter = request.args.get('status')
|
||||
origin_filter = request.args.get('origin') or None
|
||||
page = int(request.args.get('page', 1))
|
||||
|
||||
webmentions, total_pages = list_webmentions(
|
||||
status_filter=status_filter,
|
||||
page=page,
|
||||
per_page=25
|
||||
per_page=25,
|
||||
origin_filter=origin_filter,
|
||||
)
|
||||
|
||||
from starpunk import backfeed
|
||||
|
||||
return render_template(
|
||||
"admin/webmentions.html",
|
||||
webmentions=webmentions,
|
||||
status_filter=status_filter,
|
||||
origin_filter=origin_filter,
|
||||
backfeed=backfeed.status(),
|
||||
notify=_notify_status(),
|
||||
page=page,
|
||||
total_pages=total_pages,
|
||||
user_me=g.me
|
||||
@@ -880,6 +684,56 @@ def revoke_token(token_id):
|
||||
return redirect(url_for("admin.tokens"))
|
||||
|
||||
|
||||
def _notify_status() -> dict:
|
||||
"""Notification settings and last error for the Webmentions page (v2.14.0)"""
|
||||
from starpunk import notifications
|
||||
from starpunk.database import get_db
|
||||
|
||||
return {
|
||||
"enabled": notifications.enabled(current_app.config),
|
||||
"events": sorted(notifications.events(current_app.config)),
|
||||
"priority": current_app.config.get("NOTIFY_PRIORITY") or "normal",
|
||||
"services": [u.split("://", 1)[0] for u in notifications.urls(current_app.config)],
|
||||
"last_error": notifications.last_error(get_db(current_app)),
|
||||
}
|
||||
|
||||
|
||||
@bp.route("/notifications/test", methods=["POST"])
|
||||
@require_auth
|
||||
def notifications_test():
|
||||
"""Send a test notification to every NOTIFY_URLS service (v2.14.0)"""
|
||||
from starpunk import notifications
|
||||
|
||||
ok, error = notifications.send(current_app.config, "StarPunk test notification",
|
||||
"Notifications from StarPunk are working.")
|
||||
notifications._record(current_app.config["DATABASE_PATH"], ok, error)
|
||||
flash("Test notification sent" if ok else f"Test notification failed: {error or 'unknown error'}",
|
||||
"success" if ok else "error")
|
||||
return redirect(url_for("admin.webmentions"))
|
||||
|
||||
|
||||
@bp.route("/backfeed/backfill", methods=["POST"])
|
||||
@require_auth
|
||||
def backfeed_backfill():
|
||||
"""Check every crosspost once for responses, whatever its age (v2.13.0)"""
|
||||
from starpunk import backfeed
|
||||
|
||||
count = backfeed.start_backfill()
|
||||
flash(f"Backfill started: {count} crosspost(s) to check", "success")
|
||||
return redirect(url_for("admin.webmentions"))
|
||||
|
||||
|
||||
@bp.route("/syndication/<int:syndication_id>/check", methods=["POST"])
|
||||
@require_auth
|
||||
def check_syndication_responses(syndication_id):
|
||||
"""Check one crosspost's responses at the next worker pass (v2.13.0)"""
|
||||
from starpunk import backfeed
|
||||
|
||||
backfeed.check_now(syndication_id)
|
||||
flash("Responses will be checked shortly", "success")
|
||||
return redirect(request.referrer or url_for("admin.dashboard"))
|
||||
|
||||
|
||||
@bp.route("/syndication/<int:syndication_id>/retry", methods=["POST"])
|
||||
@require_auth
|
||||
def retry_syndication(syndication_id):
|
||||
|
||||
Loaded 100 of 142 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user