diff --git a/.env.example b/.env.example
index c693ccd..be1e920 100644
--- a/.env.example
+++ b/.env.example
@@ -73,10 +73,10 @@ DATABASE_PATH=./data/starpunk.db
# FLASK / LOGGING
# =============================================================================
-# development or production (default: development)
-FLASK_ENV=development
-# 1 = debug on, 0 = off (default: 1). Set 0 in production.
-FLASK_DEBUG=1
+# development or production (default: production)
+FLASK_ENV=production
+# 1 = debug on, 0 = off (default: 0). `flask run --debug` turns it on locally.
+FLASK_DEBUG=0
# DEBUG, INFO, WARNING, ERROR or CRITICAL (default: INFO)
LOG_LEVEL=INFO
# Overrides the version shown in the footer and /health (default: the
@@ -104,6 +104,10 @@ LOG_LEVEL=INFO
# (default: false)
TRUSTED_PROXY=false
+# CSRF tokens on admin forms, logout and the IndieAuth consent (default: true).
+# Leave on; tests turn it off unless they test it.
+CSRF_ENABLED=true
+
# =============================================================================
# FEEDS
# =============================================================================
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2df7f16..0111781 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
+## [2.11.0] - 2026-11-03
+
+Maintenance release: dependency upgrades and the High-priority backlog items.
+
+### Security
+
+- Every dependency is upgraded to its current release. The previous set had 97 published vulnerabilities (Pillow 64, PyJWT 22, gunicorn 4, and Flask, Markdown, python-dotenv, pytest and pillow-heif); the new set has none known. Notable versions: Flask 3.1, gunicorn 26, Pillow 13, PyJWT 2.15, httpx 0.28, nh3 0.3, Markdown 3.11, pillow-heif 1.8, psutil 7
+- Admin forms, logout and the IndieAuth consent screen now carry a CSRF token (an HMAC of your login session, no storage needed); requests without it are refused. Endpoints that apps call with tokens (Micropub, Webmention, WebSub, the token endpoint) are unaffected
+- Fetching an IndieAuth app's page and fetching pages and images for crosspost link cards now use the same protected fetcher as webmentions: every redirect is checked against private addresses, the connection goes to the checked address, and the whole fetch has a time limit
+- Debug mode is off and the environment is production unless set otherwise (`flask run --debug` still turns debug on locally); the container image sets both explicitly
+
+### Changed
+
+- Runs on Python 3.14 (was 3.11, which reaches end of life in 2027)
+- Rate limits on `/webmention` and `/websub` are counted in the database and shared by all server processes. Before, each of the four processes counted separately, so the real limit was four times the setting
+- Test dependencies are in `requirements-test.txt` (used by CI); `pytest` is no longer installed in the production image, and `lxml` and `python-dateutil`, which StarPunk uses directly, are now pinned
+- `uv` is pinned in the Containerfile instead of `latest`
+
+### Fixed
+
+- Deleting a note or a photo left its tag, photo and image-size rows behind, because SQLite only applies the schema's cascades when foreign keys are switched on. They now are; migration 018 removes rows left by earlier deletes (production had three)
+- Deprecated `datetime.utcnow()` and sqlite3's default date adapter (both removed in a future Python) replaced; the stored formats are unchanged
+- `compose.yaml` pinned version 0.6.0 (which would override the real version shown) and set an unused variable; a stray duplicate `dev_auth.py` and a `test.ini` with a private host name were removed
+
## [2.10.2] - 2026-11-03
### Fixed
diff --git a/CLAUDE.md b/CLAUDE.md
index 61d2c33..f5936c3 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -7,7 +7,7 @@ This file contains operational instructions for Claude agents working on this pr
- We use **uv** for Python virtual environment management
- All Python commands must be run with `uv run` prefix
- Example: `uv run pytest`, `uv run flask --app app.py run --debug`
-- There is no `pyproject.toml`: create the environment with `uv venv --python 3.11 && uv pip install -r requirements-dev.txt`
+- There is no `pyproject.toml`: create the environment with `uv venv --python 3.14 && uv pip install -r requirements-dev.txt`
- Tests need `SESSION_SECRET` set: `SESSION_SECRET=test uv run pytest`. The full suite takes about 6 minutes, 5 of them in `tests/test_migration_race_condition.py`; add `--ignore=tests/test_migration_race_condition.py` while iterating, but run everything before a release
- Local login without OIDC: `DEV_MODE=true` and `DEV_ADMIN_ME` in `.env`, then `/dev/login`
- Setup details: `docs/standards/development-setup.md`
diff --git a/Containerfile b/Containerfile
index 27d06cb..64787e6 100644
--- a/Containerfile
+++ b/Containerfile
@@ -6,10 +6,10 @@
# ============================================================================
# Build Stage - Install dependencies in virtual environment
# ============================================================================
-FROM python:3.11-slim AS builder
+FROM python:3.14-slim AS builder
# Install uv for fast dependency installation
-COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
+COPY --from=ghcr.io/astral-sh/uv:0.9.28 /uv /usr/local/bin/uv
WORKDIR /build
@@ -25,11 +25,12 @@ RUN uv venv /opt/venv && \
# ============================================================================
# Test Stage - Runs the test suite (CI builds it with --target test)
# ============================================================================
-FROM python:3.11-slim AS test
+FROM python:3.14-slim AS test
-COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
+COPY --from=ghcr.io/astral-sh/uv:0.9.28 /uv /usr/local/bin/uv
COPY --from=builder /opt/venv /opt/venv
-RUN uv pip install --no-cache --python /opt/venv/bin/python pytest pytest-mock
+COPY requirements-test.txt .
+RUN uv pip install --no-cache --python /opt/venv/bin/python -r requirements-test.txt
ENV PATH="/opt/venv/bin:$PATH" \
PYTHONUNBUFFERED=1 \
@@ -45,7 +46,7 @@ RUN python -m pytest -q -p no:cacheprovider tests
# ============================================================================
# Runtime Stage - Minimal production image (default target: keep it last)
# ============================================================================
-FROM python:3.11-slim
+FROM python:3.14-slim
# Create non-root user for security
# UID/GID 1000 is standard for first user on most systems
@@ -60,6 +61,8 @@ COPY --from=builder /opt/venv /opt/venv
ENV PATH="/opt/venv/bin:$PATH" \
PYTHONUNBUFFERED=1 \
PYTHONDONTWRITEBYTECODE=1 \
+ FLASK_ENV=production \
+ FLASK_DEBUG=0 \
FLASK_APP=app.py \
DATA_PATH=/data \
NOTES_PATH=/data/notes \
diff --git a/README.md b/README.md
index 152450c..71dd988 100644
--- a/README.md
+++ b/README.md
@@ -69,7 +69,7 @@ redirect URI with your OIDC provider. Details:
### Local development
```bash
-uv venv --python 3.11
+uv venv --python 3.14
uv pip install -r requirements-dev.txt
cp .env.example .env # set SESSION_SECRET; add DEV_MODE=true and DEV_ADMIN_ME
uv run flask --app app.py run --debug # http://localhost:5000
diff --git a/compose.yaml b/compose.yaml
index 615f718..bf737c7 100644
--- a/compose.yaml
+++ b/compose.yaml
@@ -11,12 +11,11 @@
# docker compose logs -f
# docker compose down
-version: '3.8'
-
services:
starpunk:
# Container configuration
- image: starpunk:0.6.0
+ # Built locally from this checkout (release images: see README)
+ image: starpunk:local
container_name: starpunk
# Build configuration
@@ -49,10 +48,6 @@ services:
- NOTES_PATH=/data/notes
- DATABASE_PATH=/data/starpunk.db
- # Application metadata
- - VERSION=0.6.0
- - ENVIRONMENT=production
-
# Volume mounts for persistent data
# All application data stored in ./container-data on host
volumes:
diff --git a/dev_auth.py b/dev_auth.py
deleted file mode 100644
index ee9aa9b..0000000
--- a/dev_auth.py
+++ /dev/null
@@ -1,103 +0,0 @@
-"""
-Development authentication module for StarPunk
-
-WARNING: This module provides a development-only authentication mechanism
-that bypasses IndieLogin. It should NEVER be enabled in production.
-
-This module is separate from production auth (starpunk/auth.py) to maintain
-clear architectural boundaries and enable easy security audits.
-
-Security measures:
-- Only active when DEV_MODE=true
-- Returns 404 if DEV_MODE=false
-- Requires DEV_ADMIN_ME configuration
-- Logs prominent warnings
-- Cannot coexist with production SITE_URL
-- Visual warnings in UI
-
-Functions:
- is_dev_mode: Check if development mode is enabled
- validate_dev_config: Validate development configuration
- create_dev_session: Create session without authentication
-"""
-
-from flask import current_app
-
-from starpunk.auth import create_session
-
-
-def is_dev_mode() -> bool:
- """
- Check if development mode is enabled
-
- Returns:
- True if DEV_MODE is enabled, False otherwise
-
- Example:
- >>> from starpunk.dev_auth import is_dev_mode
- >>> if is_dev_mode():
- ... print("Development mode active")
- """
- return current_app.config.get("DEV_MODE", False)
-
-
-def validate_dev_config() -> None:
- """
- Validate development mode configuration
-
- Checks that DEV_MODE configuration is valid and safe:
- - DEV_ADMIN_ME must be set if DEV_MODE is true
- - Warns if DEV_MODE is enabled with production-like SITE_URL
-
- Raises:
- ValueError: If DEV_MODE is true but DEV_ADMIN_ME is not set
-
- Logs:
- WARNING: If DEV_MODE is enabled with HTTPS SITE_URL
- """
- dev_mode = current_app.config.get("DEV_MODE", False)
-
- if dev_mode:
- # Require DEV_ADMIN_ME
- dev_admin_me = current_app.config.get("DEV_ADMIN_ME")
- if not dev_admin_me:
- raise ValueError("DEV_MODE=true requires DEV_ADMIN_ME to be set")
-
- # Warn if production-like configuration detected
- site_url = current_app.config.get("SITE_URL", "")
- if site_url.startswith("https://"):
- current_app.logger.warning(
- "WARNING: DEV_MODE is enabled with production SITE_URL. "
- "This is likely a misconfiguration. "
- "DEV_MODE should only be used in local development."
- )
-
-
-def create_dev_session(me: str) -> str:
- """
- Create development session without authentication
-
- WARNING: This bypasses IndieLogin authentication entirely.
- Only use in development environments.
-
- Args:
- me: User identity URL (from DEV_ADMIN_ME config)
-
- Returns:
- Session token (same format as production sessions)
-
- Logs:
- WARNING: Logs that dev session was created without authentication
-
- Example:
- >>> token = create_dev_session("https://example.com")
- >>> # Session created without IndieLogin verification
- """
- current_app.logger.warning(
- f"DEV MODE: Creating session for {me} without authentication. "
- f"This should NEVER happen in production!"
- )
-
- # Use production session creation (same session format)
- # This ensures dev sessions work identically to production
- return create_session(me)
diff --git a/docs/architecture/INDEX.md b/docs/architecture/INDEX.md
index cd02ffb..6f22d4a 100644
--- a/docs/architecture/INDEX.md
+++ b/docs/architecture/INDEX.md
@@ -11,7 +11,7 @@ per-release designs and reports are in [../design/](../design/).
|----------|-------------|
| [overview.md](overview.md) | System overview: purpose, request flow, modules and routes, background workers, storage, auth model, URL surface, caching, security posture |
| [technology-stack.md](technology-stack.md) | Current runtime, Python dependencies, standards implemented, testing and CI |
-| [database.md](database.md) | SQLite schema after migrations 001–017, migration runner and its lock, connection pool |
+| [database.md](database.md) | SQLite schema after migrations 001–019, migration runner and its lock, connection pool |
| [deployment.md](deployment.md) | Container image stages, CI workflow, release process, `/data` layout, reverse proxy, all environment variables |
| [simplified-auth-architecture.md](simplified-auth-architecture.md) | Admin login via OpenID Connect and the built-in IndieAuth server that issues Micropub tokens (ADR-064) |
| [indieweb-protocols.md](indieweb-protocols.md) | How Micropub, IndieAuth, Webmention, microformats2, WebSub and the feeds are implemented, with compliance status and known gaps |
diff --git a/docs/architecture/database.md b/docs/architecture/database.md
index ecf2639..c05e596 100644
--- a/docs/architecture/database.md
+++ b/docs/architecture/database.md
@@ -1,8 +1,8 @@
# Database Architecture
-**Version**: 2.10.1 (2026-11-03)
+**Version**: 2.11.0 (2026-11-03)
**Status**: Current. Verified against a database created by `create_app()`
-with migrations 001–017.
+with migrations 001–019.
StarPunk uses one SQLite file, `starpunk.db` under `DATA_PATH` (`/data` in
the container), in WAL mode. Note *content* is not in the database: it lives
@@ -71,6 +71,8 @@ Background: [ADR-031](../decisions/ADR-031-database-migration-system-redesign.md
| `015_bookmarks_and_content_format.sql` | 2.9 | `notes.bookmark_of`, `notes.content_format` |
| `016_websub.sql` | 2.10.0 | `websub_subscriptions`, `websub_intents`, `websub_deliveries`, `site_state` |
| `017_websub_hardening.sql` | 2.10.1 | `websub_intents.request_id`, `requested_topic`, `callback_ip`; `websub_subscriptions.callback_ip` |
+| `018_foreign_key_cleanup.sql` | 2.11.0 | Removes rows orphaned while foreign keys weren't enforced |
+| `019_shared_rate_limits.sql` | 2.11.0 | `rate_limits` (counters shared by all processes) |
New migrations are numbered `NNN_description.sql`, must be safe to run inside
one transaction, and should use `IF NOT EXISTS` where SQLite allows it.
@@ -205,13 +207,18 @@ its cached feed data is stale.
**`schema_migrations`**: applied migrations (see above).
+**`rate_limits`** (v2.11.0): request counts for `/webmention` and `/websub`,
+one row per key (`:`) and time window, so every
+server process enforces one shared limit. Old windows are pruned as it goes.
+
## Connections
- **Request connections**: `starpunk/database/pool.py`. `init_pool()` creates
one `ConnectionPool` per process (up to 5 idle connections, 10-second busy
timeout; `DB_POOL_SIZE` and `DB_TIMEOUT` can be set only through the
`create_app()` config dict). Connections use `sqlite3.Row` rows,
- `check_same_thread=False` and `PRAGMA journal_mode=WAL`, and are wrapped
+ `check_same_thread=False`, `PRAGMA journal_mode=WAL` and (since v2.11.0)
+ `PRAGMA foreign_keys=ON`, so `ON DELETE CASCADE`/`SET NULL` run, and are wrapped
in `MonitoredConnection` for query timing when metrics are on. `get_db()`
stores one connection in `flask.g` per request or app context; on teardown
it goes back to the pool, rolled back first if a transaction was left
diff --git a/docs/architecture/deployment.md b/docs/architecture/deployment.md
index 1bd1efa..eded392 100644
--- a/docs/architecture/deployment.md
+++ b/docs/architecture/deployment.md
@@ -24,12 +24,12 @@ deployment repository, not here.
## Container image (`Containerfile`)
-A multi-stage build on `python:3.11-slim`; it works with Podman and Docker.
+A multi-stage build on `python:3.14-slim`; it works with Podman and Docker.
| Stage | Purpose |
|-------|---------|
| `builder` | Copies `uv` from `ghcr.io/astral-sh/uv`, creates `/opt/venv` and installs `requirements.txt` into it |
-| `test` | Copies the venv, adds `pytest` and `pytest-mock`, copies the source and runs `python -m pytest -q -p no:cacheprovider tests` with test values for `SESSION_SECRET`, `SITE_URL` and `ADMIN_ME`. The stage only builds if the tests pass. |
+| `test` | Copies the venv, installs `requirements-test.txt`, copies the source and runs `python -m pytest -q -p no:cacheprovider tests` with test values for `SESSION_SECRET`, `SITE_URL` and `ADMIN_ME`. The stage only builds if the tests pass. |
| runtime (last, the default target) | Creates user `starpunk` (UID/GID 1000) and `/data/notes`, copies the venv and the application to `/app`, sets `DATA_PATH=/data`, `NOTES_PATH=/data/notes`, `DATABASE_PATH=/data/starpunk.db`, runs as `starpunk`, exposes port 8000 |
The runtime image's command is gunicorn:
@@ -172,8 +172,8 @@ ignored with a warning.
| `DATA_PATH` | `./data` (image: `/data`) | Root of all stored data |
| `DATABASE_PATH` | `./data/starpunk.db` (image: `/data/starpunk.db`) | |
| `NOTES_PATH` | `./data/notes` (image: `/data/notes`) | Created at start-up, but note files are always written to `DATA_PATH/notes` |
-| `FLASK_ENV` | `development` | Set `production` in production |
-| `FLASK_DEBUG` | `1` | Set `0` in production (the image does not set it) |
+| `FLASK_ENV` | `production` | Reported in `/health` and admin diagnostics; the image sets `production` |
+| `FLASK_DEBUG` | `0` | Debug off unless `1` (`flask run --debug` sets it locally); the image sets `0` |
| `LOG_LEVEL` | `INFO` | `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL` |
| `VERSION` | package `__version__` | Overrides the reported version; normally unset |
diff --git a/docs/architecture/indieweb-protocols.md b/docs/architecture/indieweb-protocols.md
index 0bb4558..a55eebc 100644
--- a/docs/architecture/indieweb-protocols.md
+++ b/docs/architecture/indieweb-protocols.md
@@ -197,7 +197,8 @@ route isn't registered at all.
1. Rate limits: 60 requests per minute per client IP and 10 per minute per
source domain (`WEBMENTION_RATE_LIMIT_*`). Over the limit gets 429 with
- `Retry-After`. The limiter is in memory, per process. The client IP
+ `Retry-After`. Counts are shared by all server processes (`rate_limits`
+ table, v2.11.0). The client IP
comes from `X-Forwarded-For` only when `TRUSTED_PROXY=true`.
2. Form-encoded (JSON also accepted). `source` and `target` are required,
http(s), at most 2048 characters, and different from each other.
@@ -445,7 +446,7 @@ Drafts that stay drafts change nothing.
Limits (429 when hit):
-- 30 requests per minute per client IP (per process; needs
+- 30 requests per minute per client IP (shared by all processes; needs
`TRUSTED_PROXY=true` behind a proxy);
- at most 200 pending verifications;
- `WEBSUB_MAX_SUBSCRIPTIONS` (1000) in total;
@@ -488,7 +489,6 @@ retried.
- The hub serves only this site's topics, by design. websub.rocks' hub
tests use their own topics, so they can't be run against it.
-- Rate limiting is per process and in memory.
- Deliveries always carry the full topic content (no diffs). Fat pings
aren't sent to external hubs.
diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md
index c79b0ac..75e6efd 100644
--- a/docs/architecture/overview.md
+++ b/docs/architecture/overview.md
@@ -293,13 +293,12 @@ their credentials are set. The full list with defaults is in
callbacks, remote Micropub photos, IndieAuth client pages, crosspost link
cards) are checked first: every address the host resolves to must be public
(private, loopback, link-local, CGNAT, multicast and reserved ranges are
- refused). The fetcher in `webmention_ssrf.py`, used for webmentions, reply
- contexts, WebSub and remote Micropub photos, follows redirects by hand and
- re-checks each hop, connects to the address it checked (no second DNS
- lookup) and limits response size (1 MB by default) and total time.
- Crosspost link cards use their own check that also re-checks each redirect.
- The IndieAuth client-page fetch checks only the initial URL. `DEV_MODE`
- relaxes the checks for local testing.
+ refused). Every fetch of a user-supplied URL (webmentions, reply contexts,
+ WebSub, remote Micropub photos, IndieAuth client pages and, since v2.11.0,
+ crosspost link cards) uses the fetcher in `webmention_ssrf.py`: it follows
+ redirects by hand and re-checks each hop, connects to the address it
+ checked (no second DNS lookup) and limits response size (1 MB by default)
+ and total time. `DEV_MODE` relaxes the checks for local testing.
- **HTML sanitising**: rendered notes (Markdown or HTML sent by a Micropub
client) and webmention content are cleaned with nh3 allow-lists; search
highlighting escapes everything except ``.
@@ -307,15 +306,17 @@ their credentials are set. The full list with defaults is in
(defaults 60 and 10 per 60 seconds); `/websub` 30 requests per IP per
minute, plus limits on subscriptions in total, per callback host and per
callback address, and on requests waiting for verification. Request-rate
- counters are in memory and per process.
+ counters live in the `rate_limits` table, shared by all server processes
+ (v2.11.0).
- **Client IP behind a proxy**: with `TRUSTED_PROXY=true` the first address in
`X-Forwarded-For` is used as the client IP for rate limiting. Set it only
when the app is reachable solely through a proxy that sets that header
(Caddy does); without it, every client shares the proxy's address.
- **Auth**: hashed session and token storage, single-use login state, PKCE
for both OIDC and IndieAuth, redirect URIs checked against the client,
- post-login redirects limited to this site. Admin form POSTs rely on the
- SameSite=Lax session cookie (there are no per-form CSRF tokens).
+ post-login redirects limited to this site. Admin form POSTs, logout and
+ the IndieAuth consent carry a CSRF token: an HMAC of the session cookie,
+ checked by `starpunk/csrf.py` (v2.11.0), on top of the SameSite=Lax cookie.
- **Uploads**: request bodies are capped at 50 MB; images are processed by
Pillow and stored under random (UUID) names.
- **Container**: runs as a non-root user. HTTPS and security headers (HSTS,
diff --git a/docs/architecture/simplified-auth-architecture.md b/docs/architecture/simplified-auth-architecture.md
index 0bac38f..f20bf58 100644
--- a/docs/architecture/simplified-auth-architecture.md
+++ b/docs/architecture/simplified-auth-architecture.md
@@ -328,14 +328,16 @@ In place:
untrusted redirect URIs never followed.
- Post-login redirects are limited to this site.
+CSRF (v2.11.0): admin form POSTs, `/auth/logout` and the consent form's
+approve button need a `csrf_token` field (or `X-CSRF-Token` header) equal to
+an HMAC-SHA256 of the `starpunk_session` cookie under `SECRET_KEY`; anything
+else gets 400. The token changes with every login and needs no storage.
+Endpoints apps call with bearer tokens are not affected. The client metadata
+fetch uses `webmention_ssrf.safe_get` (every redirect hop checked, the
+connection pinned, a 5-second limit).
+
Known gaps:
-- Admin forms and the consent form have no CSRF token. They rely on the
- `SameSite=Lax` session cookie, which cross-site POSTs don't carry.
-- The client metadata fetch checks the `client_id` URL against private
- addresses, but then makes a plain `httpx.get` that follows redirects.
- Redirect hops are not re-checked and the connection is not pinned, unlike
- `webmention_ssrf.safe_get`.
- `scopes_supported` leaves out `draft`, although Micropub honours it and
the consent screen grants it when requested. Unknown scopes are also shown
and granted as requested; they just have no effect.
diff --git a/docs/architecture/syndication-architecture.md b/docs/architecture/syndication-architecture.md
index 6c0f17f..c18c069 100644
--- a/docs/architecture/syndication-architecture.md
+++ b/docs/architecture/syndication-architecture.md
@@ -246,10 +246,6 @@ to itself (v2.8.0).
- Bluesky and Threads copies don't follow edits (platform limitation).
Unticking and re-ticking a target makes a new copy.
-- Link-card and Threads image fetches check every redirect hop against
- private addresses but don't pin DNS, and use a per-request rather than an
- overall time limit. Webmention and WebSub fetching use the stricter
- `webmention_ssrf` fetcher.
- No per-target choice for Micropub updates: `mp-syndicate-to` is read only
on create.
- Likes of ordinary web pages, and all likes on Threads, stay on the site.
diff --git a/docs/architecture/technology-stack.md b/docs/architecture/technology-stack.md
index 41b0f42..6705a54 100644
--- a/docs/architecture/technology-stack.md
+++ b/docs/architecture/technology-stack.md
@@ -14,9 +14,9 @@ the pieces fit together.
| Component | Choice |
|-----------|--------|
-| Language | Python 3.11 (`python:3.11-slim` base image) |
-| Web framework | Flask 3.0 (Werkzeug, Jinja2, itsdangerous, click come with it) |
-| WSGI server | gunicorn 21.2, 4 sync workers (see [deployment.md](deployment.md)) |
+| Language | Python 3.14 (`python:3.14-slim` base image; since v2.11.0) |
+| Web framework | Flask 3.1 (Werkzeug, Jinja2, itsdangerous, click come with it) |
+| WSGI server | gunicorn 26.2, 4 sync workers (see [deployment.md](deployment.md)) |
| Database | SQLite (standard-library `sqlite3`), WAL mode, FTS5 for search |
| Content storage | Markdown files on disk |
| Background work | `threading` daemon threads; `fcntl.flock` file locks to pick one process |
@@ -28,32 +28,36 @@ the pieces fit together.
## Python dependencies (`requirements.txt`)
+Pinned to a minor series. Upgraded in v2.11.0, when the set had no known
+vulnerabilities (`pip-audit`); check again before each release.
+
| Package | Pin | Used for |
|---------|-----|----------|
-| Flask | `3.0.*` | Web framework |
-| gunicorn | `21.2.*` | Production WSGI server |
-| markdown | `3.5.*` | Rendering notes (`extra`, `codehilite`, `nl2br` and a bare-URL linking extension) |
-| feedgen | `1.0.*` | RSS 2.0 feed (`starpunk/feeds/rss.py`); Atom and JSON Feed are generated by StarPunk's own code |
-| httpx | `0.27.*` | All outbound HTTP: OIDC, IndieAuth client discovery, webmention and WebSub requests (through the SSRF-safe wrapper in `webmention_ssrf.py`), crossposting APIs, health check |
-| python-dotenv | `1.0.*` | Loading `.env` |
-| beautifulsoup4 | `4.12.*` | HTML parsing: IndieAuth client pages, crosspost text and link cards |
+| Flask | `3.1.*` | Web framework |
+| gunicorn | `26.2.*` | Production WSGI server |
+| markdown | `3.11.*` | Rendering notes (`extra`, `codehilite`, `nl2br` and a bare-URL linking extension) |
+| nh3 | `0.3.*` | HTML sanitising (rendered notes, webmention content) |
+| beautifulsoup4 | `4.15.*` | HTML parsing: IndieAuth client pages, crosspost text and link cards |
+| lxml | `6.1.*` | Outgoing link extraction, webmention endpoint discovery and source checks |
| mf2py | `2.0.*` | Parsing microformats2 (webmention sources, reply contexts, author h-card) |
-| pytest | `8.0.*` | Test runner (listed here as well as in the image's test stage) |
-| psutil | `5.9.*` | Memory monitor and system metrics |
-| Pillow | `10.1.*` | Image validation, resizing and variants |
-| pillow-heif | `0.18.*` | HEIC/HEIF uploads (iPhone photos) |
-| nh3 | `0.2.*` | HTML sanitising (rendered notes, webmention content) |
-| PyJWT[crypto] | `2.10.*` | Verifying OIDC ID tokens (JWKS, RS/ES signatures via `cryptography`) |
+| python-dateutil | `4.0.*` | Dates in reply contexts |
+| feedgen | `1.0.*` | RSS 2.0 feed (`starpunk/feeds/rss.py`); Atom and JSON Feed are generated by StarPunk's own code |
+| httpx | `0.28.*` | All outbound HTTP; requests to user-supplied URLs go through `webmention_ssrf.safe_get`/`safe_post` |
+| PyJWT[crypto] | `2.15.*` | Verifying OIDC ID tokens (JWKS, RS/ES signatures via `cryptography`) |
+| python-dotenv | `1.2.*` | Loading `.env` |
+| Pillow | `13.0.*` | Image validation, resizing and variants |
+| pillow-heif | `1.8.*` | HEIC/HEIF uploads (iPhone photos) |
+| psutil | `7.2.*` | Memory monitor and system metrics |
-Packages imported directly but installed only as dependencies of the above:
-`lxml` (outgoing link extraction, webmention endpoint discovery and source checks; from feedgen) and `python-dateutil` (dates in
-reply contexts; from feedgen). `markupsafe` comes with Flask/Jinja2.
+`markupsafe` comes with Flask/Jinja2.
-### Development dependencies (`requirements-dev.txt`)
+### Test and development dependencies
-Includes `requirements.txt`, plus `pytest-cov`, `pytest-mock`, `black`,
-`flake8`, `mypy` and `gunicorn`. Coding standards are in
-[../standards/](../standards/).
+- `requirements-test.txt`: `pytest` 9.2 and `pytest-mock` 3.15, used by the
+ Containerfile test stage (CI).
+- `requirements-dev.txt`: runtime and test requirements plus `pytest-cov`,
+ `black`, `flake8`, `mypy` (not enforced) and `pip-audit`. Coding standards
+ are in [../standards/](../standards/).
### Deliberately not used
@@ -94,7 +98,7 @@ Protocol details: [indieweb-protocols.md](indieweb-protocols.md),
```
- **In CI**: the `test` stage of the `Containerfile` installs the production
- requirements plus pytest and pytest-mock and runs the whole suite. The
+ requirements plus `requirements-test.txt` and runs the whole suite. The
image is only built and pushed if it passes. Checklist:
[../standards/testing-checklist.md](../standards/testing-checklist.md).
diff --git a/docs/design/INDEX.md b/docs/design/INDEX.md
index 33d9c55..c702b8d 100644
--- a/docs/design/INDEX.md
+++ b/docs/design/INDEX.md
@@ -21,7 +21,7 @@ is in `../architecture/`; decisions are in `../decisions/` (ADRs).
## Contents
-[v1.0.0](#v100) · [v1.1.1](#v111) · [v1.1.2](#v112) · [v1.2.0](#v120) · [v1.3.0](#v130) · [v1.3.1](#v131) · [v1.4.0](#v140) · [v1.4.1](#v141) · [v1.4.2](#v142) · [v1.5.0](#v150) · [v1.6.0](#v160) · [v2.0.0](#v200) · [v2.0.1](#v201) · [v2.0.2](#v202) · [v2.1.0](#v210) · [v2.2.0](#v220) · [v2.3.0](#v230) · [v2.3.1](#v231) · [v2.3.2](#v232) · [v2.3.3](#v233) · [v2.4.0](#v240) · [v2.5.0](#v250) · [v2.6.0](#v260) · [v2.6.1](#v261) · [v2.6.2](#v262) · [v2.6.3](#v263) · [v2.6.4](#v264) · [v2.6.5](#v265) · [v2.6.6](#v266) · [v2.6.7](#v267) · [v2.6.8](#v268) · [v2.6.9](#v269) · [v2.7.0](#v270) · [v2.7.1](#v271) · [v2.7.2](#v272) · [v2.8.0](#v280) · [v2.8.1](#v281) · [v2.9.0](#v290) · [v2.10.0](#v2100) · [v2.10.1](#v2101) · [v2.10.2](#v2102) · [hotfix](#hotfix)
+[v1.0.0](#v100) · [v1.1.1](#v111) · [v1.1.2](#v112) · [v1.2.0](#v120) · [v1.3.0](#v130) · [v1.3.1](#v131) · [v1.4.0](#v140) · [v1.4.1](#v141) · [v1.4.2](#v142) · [v1.5.0](#v150) · [v1.6.0](#v160) · [v2.0.0](#v200) · [v2.0.1](#v201) · [v2.0.2](#v202) · [v2.1.0](#v210) · [v2.2.0](#v220) · [v2.3.0](#v230) · [v2.3.1](#v231) · [v2.3.2](#v232) · [v2.3.3](#v233) · [v2.4.0](#v240) · [v2.5.0](#v250) · [v2.6.0](#v260) · [v2.6.1](#v261) · [v2.6.2](#v262) · [v2.6.3](#v263) · [v2.6.4](#v264) · [v2.6.5](#v265) · [v2.6.6](#v266) · [v2.6.7](#v267) · [v2.6.8](#v268) · [v2.6.9](#v269) · [v2.7.0](#v270) · [v2.7.1](#v271) · [v2.7.2](#v272) · [v2.8.0](#v280) · [v2.8.1](#v281) · [v2.9.0](#v290) · [v2.10.0](#v2100) · [v2.10.1](#v2101) · [v2.10.2](#v2102) · [v2.11.0](#v2110) · [hotfix](#hotfix)
## v1.0.0
@@ -410,6 +410,10 @@ OIDC admin login and built-in IndieAuth server.
- [2026-11-03-reply-context-fetch.md](v2.10.2/2026-11-03-reply-context-fetch.md) - Reply contexts fetched again (fetch_source returns three values since v2.7.0)
+## v2.11.0
+
+- [2026-11-03-maintenance-release.md](v2.11.0/2026-11-03-maintenance-release.md) - Dependency upgrades (Python 3.14), foreign keys, CSRF, fetchers, debug default, shared rate limits
+
## hotfix
Not a release folder.
diff --git a/docs/design/v2.11.0/2026-11-03-maintenance-release.md b/docs/design/v2.11.0/2026-11-03-maintenance-release.md
new file mode 100644
index 0000000..becf735
--- /dev/null
+++ b/docs/design/v2.11.0/2026-11-03-maintenance-release.md
@@ -0,0 +1,60 @@
+# v2.11.0: Maintenance release
+
+Dependency upgrades and the five High items in the backlog (all re-checked
+against the code first and still present).
+
+## Dependencies
+
+`pip-audit` on the v2.10.2 set found 97 published vulnerabilities in eight
+packages (Pillow 64, PyJWT 22, gunicorn 4, Flask 2, Markdown 2,
+python-dotenv 2, pytest 2, pillow-heif 1). All direct dependencies were moved
+to their current minor series (`requirements.txt`); the new set has no known
+vulnerabilities. Python moved from 3.11 to 3.14 (`python:3.14-slim`); the
+whole suite passes with `-W error::DeprecationWarning`.
+
+- `requirements-test.txt` (pytest, pytest-mock) is installed by the
+ Containerfile test stage; pytest left the runtime requirements.
+- `lxml` and `python-dateutil` are pinned (used directly, previously only
+ transitive). `uv` is pinned to 0.9.28 in the Containerfile.
+- `datetime.utcnow()` (34 calls in `starpunk/`, more in tests) became
+ `datetime.now(timezone.utc).replace(tzinfo=None)`: same naive-UTC values.
+ sqlite3's default datetime adapters, deprecated in 3.12, are registered
+ explicitly in `starpunk/database/__init__.py` with the same output.
+
+## High backlog items
+
+1. **Foreign keys.** `PRAGMA foreign_keys=ON` on every pooled connection
+ (`database/pool.py`); all deletes of notes, media and WebSub subscriptions
+ go through the pool. Every foreign key is `CASCADE` or `SET NULL`, so no
+ delete can be blocked. Migration 018 removes orphans (production: three
+ `note_tags` rows from hard-deleted notes). Crosspost rows still have no
+ foreign key on purpose.
+2. **CSRF.** `starpunk/csrf.py`: token = HMAC-SHA256(SECRET_KEY,
+ "csrf:" + starpunk_session cookie); `before_request` checks the
+ `csrf_token` field or `X-CSRF-Token` header on every admin POST,
+ `auth.logout` and `indieauth.approve`; 400 otherwise. All 12 POST forms
+ carry it. `CSRF_ENABLED` (default true); test apps turn it off unless they
+ set it.
+3. **Fetchers.** `indieauth.fetch_client_redirect_uris` and
+ `syndication._fetch_public` use `webmention_ssrf.safe_get` (per-hop
+ checks, DNS pinning, hard deadline). `safe_get` gained `truncate` for
+ reads that stop at the size limit.
+4. **Debug default.** `FLASK_DEBUG` defaults to 0 and `FLASK_ENV` to
+ production; the runtime image sets both.
+5. **Shared rate limits.** `SharedRateLimiter` keeps sliding-window counters
+ in `rate_limits` (migration 019): current window plus the previous one
+ weighted by overlap. Used by `/webmention` (per IP and per source domain)
+ and `/websub`; old windows are pruned every 200 calls.
+
+## Housekeeping
+
+`compose.yaml` no longer pins 0.6.0 or sets `VERSION`/`ENVIRONMENT`; the
+unused root `dev_auth.py` and the committed `test.ini` were removed.
+
+## Tests
+
+New: `tests/test_csrf.py`, `tests/test_foreign_keys.py`,
+`tests/test_shared_rate_limit.py`, IndieAuth client-metadata tests through the
+fetcher (including a redirect to a private address). Crosspost tests route
+link-card fetches through the fetcher's transport. Full suite: 1264 passed
+on Python 3.14.
diff --git a/docs/projectplan/BACKLOG.md b/docs/projectplan/BACKLOG.md
index 9e28b30..31bda58 100644
--- a/docs/projectplan/BACKLOG.md
+++ b/docs/projectplan/BACKLOG.md
@@ -1,9 +1,9 @@
# StarPunk Backlog
-**Last Updated**: 2026-11-03 (current release: v2.10.1)
+**Last Updated**: 2026-11-03 (current release: v2.11.0)
These are the open features and bugs, sorted by priority. Each item was checked against the code
-at v2.10.1. Work that has shipped is listed per release in `vX.Y.Z/RELEASE.md` and in
+at v2.11.0. Work that has shipped is listed per release in `vX.Y.Z/RELEASE.md` and in
`CHANGELOG.md`.
## Priority Levels
@@ -23,30 +23,7 @@ at v2.10.1. Work that has shipped is listed per release in `vX.Y.Z/RELEASE.md` a
## High
-### SQLite foreign keys are not enforced (ON DELETE CASCADE is inert)
-- **Description**: No connection runs `PRAGMA foreign_keys = ON`, so the `ON DELETE CASCADE` clauses in the schema do nothing. When a note is deleted from the admin (a hard delete), its `note_tags`, `note_media`, `syndications` and webmention rows stay behind. `delete_media()` relies on the cascade to remove `media_variants` and `note_media` rows, so those are left orphaned too. Before turning the pragma on, check for existing orphan rows and add a migration that cleans them up. Each cascade also needs tests.
-- **Where**: `starpunk/database/` (connection setup/pool), `starpunk/notes.py` (`delete_note`, hard delete), `starpunk/media.py` (`delete_media`), `migrations/007`–`010`, `016`
-- **Source**: v2.10.x review; confirmed in code (there is no `foreign_keys` pragma anywhere in `starpunk/`)
-
-### No CSRF tokens on admin and consent forms
-- **Description**: Admin forms (create/edit/delete notes, webmention moderation, token revocation) and the IndieAuth consent form rely only on the `SameSite=Lax` session cookie. Lax blocks cross-site POSTs in current browsers, but a per-session CSRF token is the standard defence and covers older clients and same-site subdomains.
-- **Where**: `starpunk/routes/admin.py`, `starpunk/routes/indieauth.py`, `templates/admin/*`
-- **Source**: Documentation refresh 2026-11-03 (architecture review)
-
-### Two outbound fetches don't use the hardened fetcher
-- **Description**: IndieAuth client-metadata discovery (`indieauth.fetch_client_redirect_uris`) checks only the first URL, then uses `httpx.get(follow_redirects=True)`: redirect hops aren't checked and DNS isn't pinned. Crosspost link cards and Threads image fetches (`syndication._fetch_public`) check each hop but don't pin DNS, and their 10 s timeout is per read. Move both to `webmention_ssrf.safe_get`, as Webmention, WebSub and Micropub photo downloads already do.
-- **Where**: `starpunk/indieauth.py`, `starpunk/syndication.py`
-- **Source**: Documentation refresh 2026-11-03
-
-### `FLASK_DEBUG` defaults to on
-- **Description**: `config.py` reads `FLASK_DEBUG` with default `"1"` (and `FLASK_ENV` defaults to development); the runtime image sets neither. Production is safe only because the deployment sets `FLASK_DEBUG=0`. Default to off, or set it in the `Containerfile` runtime stage.
-- **Where**: `starpunk/config.py`, `Containerfile`
-- **Source**: Documentation refresh 2026-11-03 (production verified: DEBUG is False there)
-
-### Rate limiters are per process
-- **Description**: `RateLimiter` keeps its counters in memory, and gunicorn runs 4 workers. The WebSub hub limit of 30/min is therefore really 4×30/min, and the webmention endpoint limit is multiplied the same way. Consider a counter stored in SQLite that all workers share.
-- **Where**: `starpunk/rate_limiter.py`, `starpunk/routes/websub.py`, `starpunk/routes/webmention.py`, `Containerfile` (`--workers 4`)
-- **Source**: ADR-071 (Consequences: rate limiting bounds subscriber-chosen requests); v2.10.1 review
+None open.
---
@@ -92,20 +69,15 @@ at v2.10.1. Work that has shipped is listed per release in `vX.Y.Z/RELEASE.md` a
- **Where**: `starpunk/webmention_processor.py`, `starpunk/webmention_worker.py`
- **Source**: v1.6.0 design reconstruction (docs/design/v1.6.0/background-worker.md)
-### Webmention rate limiter never cleaned up
-- **Description**: `cleanup_rate_limiters()` is never called, so per-IP and per-domain keys accumulate for the life of each process.
-- **Where**: `starpunk/rate_limiter.py`, `starpunk/routes/webmention.py`
-- **Source**: v1.6.0 design reconstruction
-
### Atom and JSON Feed not linked from pages
- **Description**: `base.html` advertises only `/feed` (as RSS) and OPML with `rel=alternate`; add `` for `/feed.atom` and `/feed.json` so readers can pick a format.
- **Where**: `templates/base.html`
- **Source**: Documentation refresh 2026-11-03
-### Housekeeping in the repository
-- **Description**: `compose.yaml` is stale (`image: starpunk:0.6.0`, `VERSION=0.6.0` would override the real version, unused `ENVIRONMENT`); the root `dev_auth.py` duplicates `starpunk/dev_auth.py` and is unused; `LICENSE` says "Copyright (c) 2024 [Your Name]"; `test.ini` contains a private host name; `lxml` and `python-dateutil` are used directly but only installed as dependencies of other packages, while `pytest` is pinned in the production `requirements.txt`.
-- **Where**: `compose.yaml`, `dev_auth.py`, `LICENSE`, `test.ini`, `requirements*.txt`
-- **Source**: Documentation refresh 2026-11-03
+### LICENSE placeholder
+- **Description**: `LICENSE` still reads "Copyright (c) 2024 [Your Name]". Fill in the holder and year.
+- **Where**: `LICENSE`
+- **Source**: Documentation refresh 2026-11-03 (rest of that housekeeping item done in v2.11.0)
### Backfeed responses to crossposts
- **Description**: Idea: bring likes, boosts and replies to StarPunk's own crossposts back to the note as responses. Two options: poll GoToSocial (the token would need a `read` scope), or rely on Bridgy, which sends webmentions. Since v2.8.0, StarPunk ignores webmentions *from* a note's own crossposts, so that rule would need to change to work with Bridgy.
@@ -213,6 +185,8 @@ at v2.10.1. Work that has shipped is listed per release in `vX.Y.Z/RELEASE.md` a
These items were removed from the backlog because they have shipped:
+- v2.11.0: SQLite foreign keys enforced (orphans cleaned up by migration 018); CSRF tokens on admin, logout and consent forms; IndieAuth client and crosspost link-card fetches moved onto the SSRF-safe fetcher; `FLASK_DEBUG`/`FLASK_ENV` default to off/production; rate limits shared by all processes (and old windows pruned, so the webmention limiter no longer grows); dependencies upgraded (Python 3.14, no known vulnerabilities); stale `compose.yaml`, duplicate `dev_auth.py` and `test.ini` removed or fixed; `lxml`/`python-dateutil` pinned, `pytest` out of the runtime requirements
+
- POSSE: GoToSocial (v2.1.0), Bluesky (v2.2.0), Threads (v2.6.0)
- Webmentions: receive, send, display and moderation (v1.6.0); fixed and made spec-compliant in v2.6.8–v2.7.2
- Reply contexts, replies, likes, reposts and bookmarks (v2.4.0, v2.5.0, v2.9.0)
diff --git a/docs/projectplan/INDEX.md b/docs/projectplan/INDEX.md
index 40abd19..5d41228 100644
--- a/docs/projectplan/INDEX.md
+++ b/docs/projectplan/INDEX.md
@@ -4,7 +4,7 @@ Future work and per-release plans. See `../DOCUMENTATION.md` (PROJECTPLAN).
## Backlog
-- **[BACKLOG.md](BACKLOG.md)** - Open features and bugs by priority (updated for v2.10.1)
+- **[BACKLOG.md](BACKLOG.md)** - Open features and bugs by priority (updated for v2.11.0)
## Releases
@@ -47,3 +47,4 @@ Future work and per-release plans. See `../DOCUMENTATION.md` (PROJECTPLAN).
- **[v2.10.0](v2.10.0/RELEASE.md)** - WebSub (Released 2026-11-03)
- **[v2.10.1](v2.10.1/RELEASE.md)** - WebSub hardening (Released 2026-11-03)
- **[v2.10.2](v2.10.2/RELEASE.md)** - Reply context fetch fix (Released 2026-11-03)
+- **[v2.11.0](v2.11.0/RELEASE.md)** - Maintenance: dependency upgrades, High backlog items (Released 2026-11-03)
diff --git a/docs/projectplan/v2.11.0/RELEASE.md b/docs/projectplan/v2.11.0/RELEASE.md
new file mode 100644
index 0000000..b8fc913
--- /dev/null
+++ b/docs/projectplan/v2.11.0/RELEASE.md
@@ -0,0 +1,28 @@
+# v2.11.0
+
+- **Date**: 2026-11-03
+- **Status**: Released
+- **Focus**: Maintenance: dependency upgrades and the High backlog items
+
+## Security
+
+- All dependencies upgraded (97 known vulnerabilities before, none after); Python 3.14
+- CSRF tokens on admin forms, logout and the IndieAuth consent
+- IndieAuth client and crosspost link-card fetches use the SSRF-safe fetcher
+- Debug off and production environment by default
+
+## Changes
+
+- Rate limits shared by all server processes
+- `requirements-test.txt`; pytest out of the production image; `lxml` and `python-dateutil` pinned; `uv` pinned
+
+## Bugs fixed
+
+- Foreign keys enforced, so deletes cascade; orphan rows removed (migration 018)
+- Deprecated `datetime.utcnow()` and sqlite3 adapters replaced
+- Stale `compose.yaml`, duplicate `dev_auth.py` and `test.ini` cleaned up
+
+## Related
+
+- [CHANGELOG](../../../CHANGELOG.md)
+- [Design note](../../design/v2.11.0/2026-11-03-maintenance-release.md)
diff --git a/docs/standards/development-setup.md b/docs/standards/development-setup.md
index 42f3df3..c49a324 100644
--- a/docs/standards/development-setup.md
+++ b/docs/standards/development-setup.md
@@ -9,7 +9,7 @@ deployment is in [architecture/deployment.md](../architecture/deployment.md).
## Requirements
-- Python 3.11 (the container image uses `python:3.11-slim`)
+- Python 3.14 (the container image uses `python:3.14-slim`)
- [uv](https://docs.astral.sh/uv/) for the virtual environment and packages ([ADR-006](../decisions/ADR-006-python-virtual-environment-uv.md))
- git
- Optional: Docker or Podman, to build the image and run the CI test stage locally
@@ -23,7 +23,7 @@ pytest-mock, black, flake8, mypy).
```bash
git clone https://git.thesatelliteoflove.com/phil/StarPunk.git
cd StarPunk
-uv venv --python 3.11 # creates .venv/ (gitignored)
+uv venv --python 3.14 # creates .venv/ (gitignored)
uv pip install -r requirements-dev.txt
```
diff --git a/docs/standards/git-branching-strategy.md b/docs/standards/git-branching-strategy.md
index 0c6fb8e..48378ac 100644
--- a/docs/standards/git-branching-strategy.md
+++ b/docs/standards/git-branching-strategy.md
@@ -63,7 +63,7 @@ cd ~/Documents/projects/StarPunk # main checkout, on main
git pull
git worktree add ../StarPunk-worktrees/v2.10.1 -b fix/v2.10.1-websub-hardening main
cd ../StarPunk-worktrees/v2.10.1
-uv venv --python 3.11 && uv pip install -r requirements-dev.txt
+uv venv --python 3.14 && uv pip install -r requirements-dev.txt
```
### 2. Make the change
diff --git a/docs/standards/python-coding-standards.md b/docs/standards/python-coding-standards.md
index c94a9e2..b9fc33a 100644
--- a/docs/standards/python-coding-standards.md
+++ b/docs/standards/python-coding-standards.md
@@ -10,7 +10,7 @@ out." Patterns specific to small helper functions are in
## Base style
-- [PEP 8](https://peps.python.org/pep-0008/) naming and layout, Python 3.11.
+- [PEP 8](https://peps.python.org/pep-0008/) naming and layout, Python 3.14. Use `datetime.now(timezone.utc)`, not the deprecated `datetime.utcnow()`.
- **Match the file you are editing.** 4-space indents everywhere; quote
style varies between modules (newer code uses double quotes); keep lines
to roughly 100 characters.
diff --git a/migrations/018_foreign_key_cleanup.sql b/migrations/018_foreign_key_cleanup.sql
new file mode 100644
index 0000000..6bf448c
--- /dev/null
+++ b/migrations/018_foreign_key_cleanup.sql
@@ -0,0 +1,12 @@
+-- Migration 018: Remove rows orphaned while foreign keys weren't enforced (v2.11.0)
+-- From v2.11.0 every pooled connection runs PRAGMA foreign_keys=ON, so these
+-- cascades happen on delete. This removes what earlier deletes left behind.
+
+DELETE FROM note_tags WHERE note_id NOT IN (SELECT id FROM notes);
+DELETE FROM note_tags WHERE tag_id NOT IN (SELECT id FROM tags);
+DELETE FROM note_media WHERE note_id NOT IN (SELECT id FROM notes);
+DELETE FROM note_media WHERE media_id NOT IN (SELECT id FROM media);
+DELETE FROM media_variants WHERE media_id NOT IN (SELECT id FROM media);
+DELETE FROM webmention_queue WHERE source_note_id IS NOT NULL AND source_note_id NOT IN (SELECT id FROM notes);
+UPDATE webmentions SET target_note_id = NULL WHERE target_note_id IS NOT NULL AND target_note_id NOT IN (SELECT id FROM notes);
+DELETE FROM websub_deliveries WHERE subscription_id IS NOT NULL AND subscription_id NOT IN (SELECT id FROM websub_subscriptions);
diff --git a/migrations/019_shared_rate_limits.sql b/migrations/019_shared_rate_limits.sql
new file mode 100644
index 0000000..7457a09
--- /dev/null
+++ b/migrations/019_shared_rate_limits.sql
@@ -0,0 +1,9 @@
+-- Migration 019: Rate limit counters shared by all server processes (v2.11.0)
+-- One row per key and time window; see starpunk/rate_limiter.py SharedRateLimiter.
+
+CREATE TABLE IF NOT EXISTS rate_limits (
+ key TEXT NOT NULL,
+ window INTEGER NOT NULL,
+ count INTEGER NOT NULL DEFAULT 0,
+ PRIMARY KEY (key, window)
+);
diff --git a/requirements-dev.txt b/requirements-dev.txt
index a8a6246..a832530 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -1,17 +1,9 @@
-# StarPunk Development Dependencies
-# Includes code quality and testing tools
-
-# Include production dependencies
+# StarPunk development dependencies
-r requirements.txt
+-r requirements-test.txt
-# Testing
-pytest-cov>=5.0.0 # Test coverage reporting
-pytest-mock>=3.12.0 # Mocking for tests
-
-# Code Quality
-black>=24.0.0 # Code formatting
-flake8>=7.0.0 # Linting
-mypy>=1.8.0 # Type checking
-
-# WSGI Server (for production-like testing)
-gunicorn>=21.2.0
+pytest-cov==7.* # Coverage reports
+black==26.* # Formatting (not enforced, see docs/standards)
+flake8==7.* # Linting (not enforced)
+mypy==1.* # Type checking (not enforced)
+pip-audit==2.* # Known-vulnerability check of the dependency set
diff --git a/requirements-test.txt b/requirements-test.txt
new file mode 100644
index 0000000..6f61a92
--- /dev/null
+++ b/requirements-test.txt
@@ -0,0 +1,3 @@
+# Test dependencies: used by the Containerfile test stage and CI (v2.11.0)
+pytest==9.2.*
+pytest-mock==3.15.*
diff --git a/requirements.txt b/requirements.txt
index a97e85d..b7650ad 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,42 +1,27 @@
-# StarPunk Production Dependencies
-# Python 3.11+ required
+# StarPunk runtime dependencies (v2.11.0)
+# Pinned to a minor series; see docs/architecture/technology-stack.md.
-# Web Framework
-Flask==3.0.*
+# Web framework and server
+Flask==3.1.*
+gunicorn==26.2.*
-# WSGI Server (Production)
-gunicorn==21.2.*
-
-# Content Processing
-markdown==3.5.*
-
-# Feed Generation
+# Content
+markdown==3.11.*
+nh3==0.3.*
+beautifulsoup4==4.15.*
+lxml==6.1.*
+mf2py==2.0.*
+python-dateutil==4.0.*
feedgen==1.0.*
-# HTTP Client (for IndieAuth)
-httpx==0.27.*
+# HTTP and auth
+httpx==0.28.*
+PyJWT[crypto]==2.15.*
+python-dotenv==1.2.*
-# Configuration Management
-python-dotenv==1.0.*
+# Images
+Pillow==13.0.*
+pillow-heif==1.8.*
-# HTML Parsing (for IndieAuth endpoint discovery)
-beautifulsoup4==4.12.*
-
-# Microformats2 Parsing (v1.2.0)
-mf2py==2.0.*
-
-# Testing Framework
-pytest==8.0.*
-
-# System Monitoring (v1.1.2)
-psutil==5.9.*
-
-# Image Processing (v1.2.0, updated v1.4.2 for HEIC support)
-Pillow==10.1.*
-
-# HEIC/HEIF Support (v1.4.2 - iPhone photos)
-pillow-heif==0.18.*
-
-# HTML Sanitization (v1.6.0 - Webmention content)
-nh3==0.2.*
-PyJWT[crypto]==2.10.*
+# Monitoring
+psutil==7.2.*
diff --git a/starpunk/__init__.py b/starpunk/__init__.py
index 622a7df..e1d59aa 100644
--- a/starpunk/__init__.py
+++ b/starpunk/__init__.py
@@ -309,6 +309,10 @@ def create_app(config=None):
response.headers.add('Link', '; rel="webmention"')
return response
+ # CSRF tokens on admin and consent forms (v2.11.0)
+ from starpunk import csrf
+ csrf.init_app(app)
+
# WebSub discovery: Link rel=hub and rel=self on topics (v2.10.0)
from starpunk.websub import add_discovery_headers, template_links
app.after_request(add_discovery_headers)
@@ -470,5 +474,5 @@ def create_app(config=None):
# Package version (Semantic Versioning 2.0.0)
# See docs/standards/versioning-strategy.md for details
-__version__ = "2.10.2"
-__version_info__ = (2, 10, 2)
+__version__ = "2.11.0"
+__version_info__ = (2, 11, 0)
diff --git a/starpunk/auth.py b/starpunk/auth.py
index 8093311..6c2afc1 100644
--- a/starpunk/auth.py
+++ b/starpunk/auth.py
@@ -32,7 +32,7 @@ import base64
import hashlib
import secrets
import time
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
from functools import wraps
from typing import Any, Dict, Optional, Tuple
from urllib.parse import urlencode
@@ -223,7 +223,7 @@ def initiate_login(next_url: Optional[str] = None) -> str:
nonce,
code_verifier,
_safe_next_url(next_url),
- datetime.utcnow() + timedelta(minutes=LOGIN_STATE_MINUTES),
+ datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(minutes=LOGIN_STATE_MINUTES),
),
)
db.commit()
@@ -361,7 +361,7 @@ def create_session(me: str) -> str:
# Calculate expiry (use configured session lifetime or default to 30 days)
session_lifetime = current_app.config.get("SESSION_LIFETIME", 30)
- expires_at = datetime.utcnow() + timedelta(days=session_lifetime)
+ expires_at = datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(days=session_lifetime)
current_app.logger.debug(f"Auth: Session expiry: {expires_at} ({session_lifetime} days)")
diff --git a/starpunk/author_discovery.py b/starpunk/author_discovery.py
index b3e1c64..c489012 100644
--- a/starpunk/author_discovery.py
+++ b/starpunk/author_discovery.py
@@ -22,7 +22,7 @@ Fallback Behavior (per Q14):
import json
import logging
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
from typing import Dict, Optional
from urllib.parse import urlparse
@@ -236,7 +236,7 @@ def get_author_profile(me_url: str, refresh: bool = False) -> Dict:
if cached:
# Check if cache is still valid
cached_until = datetime.fromisoformat(cached['cached_until'])
- if datetime.utcnow() < cached_until:
+ if datetime.now(timezone.utc).replace(tzinfo=None) < cached_until:
current_app.logger.debug(f"Using cached author profile for {me_url}")
# Parse rel_me_links from JSON
@@ -326,7 +326,7 @@ def save_author_profile(me_url: str, profile: Dict) -> None:
db = get_db(current_app)
# Calculate cache expiry (24 hours from now)
- cached_until = datetime.utcnow() + timedelta(hours=CACHE_TTL_HOURS)
+ cached_until = datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(hours=CACHE_TTL_HOURS)
# Convert rel_me_links to JSON (per Q17)
rel_me_json = json.dumps(profile.get('rel_me_links', []))
diff --git a/starpunk/config.py b/starpunk/config.py
index bee665f..33f6c1d 100644
--- a/starpunk/config.py
+++ b/starpunk/config.py
@@ -71,8 +71,10 @@ def load_config(app, config_override=None):
app.config["DATABASE_PATH"] = Path(os.getenv("DATABASE_PATH", "./data/starpunk.db"))
# Flask environment
- app.config["ENV"] = os.getenv("FLASK_ENV", "development")
- app.config["DEBUG"] = os.getenv("FLASK_DEBUG", "1") == "1"
+ # Safe defaults (v2.11.0): production, debug off. `flask run --debug`
+ # sets FLASK_DEBUG=1 for local development.
+ app.config["ENV"] = os.getenv("FLASK_ENV", "production")
+ app.config["DEBUG"] = os.getenv("FLASK_DEBUG", "0") == "1"
# Logging
app.config["LOG_LEVEL"] = os.getenv("LOG_LEVEL", "INFO")
@@ -153,8 +155,14 @@ def load_config(app, config_override=None):
app.config["SYNDICATION_POLL_INTERVAL"] = int(os.getenv("SYNDICATION_POLL_INTERVAL", "10"))
# Apply overrides if provided
+ # CSRF protection for admin and consent forms (v2.11.0); on by default.
+ # Tests turn it off unless they set CSRF_ENABLED themselves.
+ app.config["CSRF_ENABLED"] = os.getenv("CSRF_ENABLED", "true").lower() == "true"
+
if config_override:
app.config.update(config_override)
+ if app.config.get("TESTING") and "CSRF_ENABLED" not in config_override:
+ app.config["CSRF_ENABLED"] = False
# Normalize SITE_URL trailing slash (in case override provided URL without slash)
if "SITE_URL" in app.config:
diff --git a/starpunk/csrf.py b/starpunk/csrf.py
new file mode 100644
index 0000000..ee92b87
--- /dev/null
+++ b/starpunk/csrf.py
@@ -0,0 +1,56 @@
+"""
+CSRF protection for the admin and consent forms (v2.11.0)
+
+The token is an HMAC of the login session cookie under the app secret: no
+storage, tied to the session, and different after every login. Forms carry
+it as a hidden `csrf_token` field (template global `csrf_token()`); POSTs to
+protected endpoints without the right token get 400.
+
+Protected: every admin POST, logout, and the IndieAuth consent ("approve").
+Endpoints used by apps with bearer tokens (Micropub, token, revoke,
+Webmention, WebSub) are not: they have no cookie-based authority.
+"""
+
+import hashlib
+import hmac
+
+from flask import abort, current_app, request
+
+SESSION_COOKIE = "starpunk_session"
+FIELD = "csrf_token"
+HEADER = "X-CSRF-Token"
+PROTECTED_ENDPOINTS = {"auth.logout", "indieauth.approve"}
+
+
+def csrf_token() -> str:
+ """The CSRF token for the current login session ('' when not logged in)"""
+ session = request.cookies.get(SESSION_COOKIE)
+ if not session:
+ return ""
+ key = str(current_app.config["SECRET_KEY"]).encode("utf-8")
+ return hmac.new(key, b"csrf:" + session.encode("utf-8"), hashlib.sha256).hexdigest()
+
+
+def _protected() -> bool:
+ return request.blueprint == "admin" or request.endpoint in PROTECTED_ENDPOINTS
+
+
+def check_csrf():
+ """before_request: refuse state-changing requests without a valid token"""
+ if request.method in ("GET", "HEAD", "OPTIONS") or not current_app.config.get("CSRF_ENABLED", True):
+ return None
+ if not _protected():
+ return None
+ expected = csrf_token()
+ if not expected:
+ return None # not logged in: the route's own auth check refuses it
+ sent = request.form.get(FIELD) or request.headers.get(HEADER) or ""
+ if not hmac.compare_digest(sent, expected):
+ current_app.logger.warning(f"CSRF token missing or wrong for {request.endpoint}")
+ abort(400, description="The form expired or came from another site. Reload the page and try again.")
+ return None
+
+
+def init_app(app) -> None:
+ app.before_request(check_csrf)
+ app.add_template_global(csrf_token, "csrf_token")
diff --git a/starpunk/database/__init__.py b/starpunk/database/__init__.py
index 29f48bd..0eb8835 100644
--- a/starpunk/database/__init__.py
+++ b/starpunk/database/__init__.py
@@ -13,4 +13,12 @@ from starpunk.database.init import init_db
from starpunk.database.pool import init_pool, get_db, get_pool_stats
from starpunk.database.schema import INITIAL_SCHEMA_SQL
+# Explicit datetime adapters (v2.11.0). Python 3.12 deprecated sqlite3's
+# built-in ones; these store exactly what they did ("YYYY-MM-DD HH:MM:SS[.ffffff]").
+import datetime as _datetime
+import sqlite3 as _sqlite3
+
+_sqlite3.register_adapter(_datetime.datetime, lambda value: value.isoformat(" "))
+_sqlite3.register_adapter(_datetime.date, lambda value: value.isoformat())
+
__all__ = ['init_db', 'init_pool', 'get_db', 'get_pool_stats', 'INITIAL_SCHEMA_SQL']
diff --git a/starpunk/database/pool.py b/starpunk/database/pool.py
index d998fdc..47e865e 100644
--- a/starpunk/database/pool.py
+++ b/starpunk/database/pool.py
@@ -68,6 +68,9 @@ class ConnectionPool:
# Enable WAL mode for better concurrency
conn.execute("PRAGMA journal_mode=WAL")
+ # Enforce foreign keys, so ON DELETE CASCADE / SET NULL actually run
+ # (SQLite leaves them off per connection; v2.11.0)
+ conn.execute("PRAGMA foreign_keys=ON")
self._stats['connections_created'] += 1
diff --git a/starpunk/feeds/opml.py b/starpunk/feeds/opml.py
index a41a7a2..0af0476 100644
--- a/starpunk/feeds/opml.py
+++ b/starpunk/feeds/opml.py
@@ -14,7 +14,7 @@ Per v1.1.2 Phase 3:
Specification: http://opml.org/spec2.opml
"""
-from datetime import datetime
+from datetime import datetime, timezone
from xml.sax.saxutils import escape
@@ -57,7 +57,7 @@ def generate_opml(site_url: str, site_name: str) -> str:
safe_site_name = escape(site_name)
# RFC 822 date format (required by OPML spec)
- creation_date = datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT')
+ creation_date = datetime.now(timezone.utc).replace(tzinfo=None).strftime('%a, %d %b %Y %H:%M:%S GMT')
# Build OPML document
opml_lines = [
diff --git a/starpunk/indieauth.py b/starpunk/indieauth.py
index b8f10b9..ed1fa83 100644
--- a/starpunk/indieauth.py
+++ b/starpunk/indieauth.py
@@ -13,7 +13,7 @@ current IndieAuth spec: https://indieauth.spec.indieweb.org/
import base64
import hashlib
import secrets
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
from typing import Any, Dict, List, Optional
from urllib.parse import urljoin, urlparse
@@ -83,41 +83,42 @@ def fetch_client_redirect_uris(client_id: str) -> List[str]:
Supports a JSON client metadata document ("redirect_uris") and the older
HTML/Link header form (rel="redirect_uri"). Returns [] on any failure.
"""
- from starpunk.webmention_ssrf import SSRFBlockedError, validate_url_for_fetch
+ import json
+ from starpunk.webmention_ssrf import FetchTooLargeError, SSRFBlockedError, safe_get, user_agent
+ # Every redirect hop is checked and the connection pinned to the checked
+ # address, with a hard time limit (v2.11.0; DEV_MODE allows localhost)
try:
- if not current_app.config.get("DEV_MODE"):
- validate_url_for_fetch(client_id)
- response = httpx.get(
+ result = safe_get(
client_id,
- timeout=5.0,
- follow_redirects=True,
- headers={"Accept": "application/json, text/html;q=0.9"},
+ headers={"Accept": "application/json, text/html;q=0.9",
+ "User-Agent": user_agent(current_app.config.get("SITE_URL", ""))},
+ max_bytes=500_000, timeout=5.0, truncate=True,
)
- except (httpx.HTTPError, SSRFBlockedError, ValueError) as e:
+ except (httpx.HTTPError, SSRFBlockedError, ValueError, FetchTooLargeError) as e:
current_app.logger.info(f"IndieAuth: could not fetch client {client_id}: {e}")
return []
- if response.status_code != 200:
+ if result.status_code != 200:
return []
- base = str(response.url)
+ base = result.url
uris: List[str] = []
- if "json" in response.headers.get("Content-Type", ""):
+ if "json" in result.headers.get("Content-Type", ""):
try:
- data = response.json()
+ data = json.loads(result.text)
except ValueError:
return []
if isinstance(data, dict) and isinstance(data.get("redirect_uris"), list):
uris.extend(u for u in data["redirect_uris"] if isinstance(u, str))
return uris
- link = response.links.get("redirect_uri")
+ link = httpx.Response(200, headers=result.headers).links.get("redirect_uri")
if link and link.get("url"):
uris.append(urljoin(base, link["url"]))
- soup = BeautifulSoup(response.text[:500_000], "html.parser")
+ soup = BeautifulSoup(result.text, "html.parser")
for tag in soup.find_all(["link", "a"], href=True):
if "redirect_uri" in (tag.get("rel") or []):
uris.append(urljoin(base, tag["href"]))
@@ -203,7 +204,7 @@ def create_authorization_code(
redirect_uri,
scope,
code_challenge,
- datetime.utcnow() + timedelta(minutes=CODE_LIFETIME_MINUTES),
+ datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(minutes=CODE_LIFETIME_MINUTES),
),
)
db.commit()
@@ -258,7 +259,7 @@ def create_access_token(client_id: str, scope: str) -> Dict[str, Any]:
"""Issue an access token (stored hashed); returns the token response"""
token = secrets.token_urlsafe(32)
lifetime_days = current_app.config.get("INDIEAUTH_TOKEN_LIFETIME_DAYS", 90)
- expires_at = datetime.utcnow() + timedelta(days=lifetime_days)
+ expires_at = datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(days=lifetime_days)
db = get_db(current_app)
db.execute(
diff --git a/starpunk/models.py b/starpunk/models.py
index ef10df2..84a5c06 100644
--- a/starpunk/models.py
+++ b/starpunk/models.py
@@ -8,7 +8,7 @@ state. All models are immutable and use dataclasses for clean structure.
# Standard library imports
from dataclasses import dataclass, field, replace
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any, Optional
import re
@@ -698,7 +698,7 @@ class Session:
>>> session.is_expired
False
"""
- return datetime.utcnow() >= self.expires_at
+ return datetime.now(timezone.utc).replace(tzinfo=None) >= self.expires_at
@property
def is_active(self) -> bool:
@@ -726,7 +726,7 @@ class Session:
>>> session.age
datetime.timedelta(days=2, seconds=3600)
"""
- return datetime.utcnow() - self.created_at
+ return datetime.now(timezone.utc).replace(tzinfo=None) - self.created_at
@property
def time_until_expiry(self) -> timedelta:
@@ -740,7 +740,7 @@ class Session:
>>> session.time_until_expiry
datetime.timedelta(days=28)
"""
- return self.expires_at - datetime.utcnow()
+ return self.expires_at - datetime.now(timezone.utc).replace(tzinfo=None)
def is_valid(self) -> bool:
"""
@@ -787,7 +787,7 @@ class Session:
>>> updated.last_used_at
datetime.datetime(2024, 11, 18, 15, 30, 0)
"""
- return replace(self, last_used_at=datetime.utcnow())
+ return replace(self, last_used_at=datetime.now(timezone.utc).replace(tzinfo=None))
def to_dict(self) -> dict[str, Any]:
"""
@@ -863,7 +863,7 @@ class Token:
me: str
client_id: Optional[str] = None
scope: Optional[str] = None
- created_at: datetime = field(default_factory=lambda: datetime.utcnow())
+ created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc).replace(tzinfo=None))
expires_at: Optional[datetime] = None
@classmethod
@@ -926,7 +926,7 @@ class Token:
# No expiry set, token never expires
return False
- return datetime.utcnow() >= self.expires_at
+ return datetime.now(timezone.utc).replace(tzinfo=None) >= self.expires_at
@property
def is_active(self) -> bool:
@@ -1145,7 +1145,7 @@ class AuthState:
>>> auth_state.is_expired
False
"""
- return datetime.utcnow() >= self.expires_at
+ return datetime.now(timezone.utc).replace(tzinfo=None) >= self.expires_at
@property
def is_active(self) -> bool:
@@ -1169,7 +1169,7 @@ class AuthState:
>>> auth_state.age
datetime.timedelta(seconds=120)
"""
- return datetime.utcnow() - self.created_at
+ return datetime.now(timezone.utc).replace(tzinfo=None) - self.created_at
def is_valid(self) -> bool:
"""
diff --git a/starpunk/monitoring/metrics.py b/starpunk/monitoring/metrics.py
index 62c5d69..7351d9c 100644
--- a/starpunk/monitoring/metrics.py
+++ b/starpunk/monitoring/metrics.py
@@ -24,7 +24,7 @@ import random
import time
from collections import deque
from dataclasses import dataclass, field, asdict
-from datetime import datetime
+from datetime import datetime, timezone
from threading import Lock
from typing import Any, Deque, Dict, List, Literal, Optional, Union
@@ -159,7 +159,7 @@ class MetricsBuffer:
metric = Metric(
operation_type=operation_type,
operation_name=operation_name,
- timestamp=datetime.utcnow().isoformat() + "Z",
+ timestamp=datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + "Z",
duration_ms=duration_ms,
process_id=self._process_id,
metadata=metadata or {}
diff --git a/starpunk/notes.py b/starpunk/notes.py
index daf4e42..b91532d 100644
--- a/starpunk/notes.py
+++ b/starpunk/notes.py
@@ -20,7 +20,7 @@ Exceptions:
"""
# Standard library imports
-from datetime import datetime
+from datetime import datetime, timezone
from pathlib import Path
from typing import Optional
@@ -226,7 +226,7 @@ def create_note(
# 2. SETUP
if created_at is None:
- created_at = datetime.utcnow()
+ created_at = datetime.now(timezone.utc).replace(tzinfo=None)
updated_at = created_at # Same as created_at for new notes
@@ -735,7 +735,7 @@ def update_note(
)
# 3. SETUP
- updated_at = datetime.utcnow()
+ updated_at = datetime.now(timezone.utc).replace(tzinfo=None)
data_dir = Path(current_app.config["DATA_PATH"])
note_path = data_dir / existing_note.file_path
@@ -1052,7 +1052,7 @@ def delete_note(
if soft:
# SOFT DELETE: Mark as deleted in database
- deleted_at = datetime.utcnow()
+ deleted_at = datetime.now(timezone.utc).replace(tzinfo=None)
try:
db.execute(
diff --git a/starpunk/rate_limiter.py b/starpunk/rate_limiter.py
index 5a69f74..08cb05c 100644
--- a/starpunk/rate_limiter.py
+++ b/starpunk/rate_limiter.py
@@ -138,6 +138,79 @@ _ip_limiter: RateLimiter | None = None
_domain_limiter: RateLimiter | None = None
+class SharedRateLimiter:
+ """
+ Rate limiter whose counts live in SQLite, shared by every gunicorn
+ process (v2.11.0). In-memory counters let each of the 4 workers allow the
+ full limit, so the real limit was four times the setting.
+
+ Sliding-window counter: the current window's count plus the previous
+ window's, weighted by how much of it still overlaps the last
+ `window_seconds`. Same interface as RateLimiter; needs an app context.
+ """
+
+ CLEANUP_EVERY = 200 # calls between removals of old windows
+
+ def __init__(self, limit: int, window_seconds: int, prefix: str):
+ self.limit = limit
+ self.window_seconds = window_seconds
+ self.prefix = prefix
+ self._calls = 0
+
+ def _db(self):
+ from flask import current_app
+ from starpunk.database import get_db
+ return get_db(current_app)
+
+ def _windows(self):
+ now = time.time()
+ current = int(now // self.window_seconds)
+ overlap = 1 - (now % self.window_seconds) / self.window_seconds
+ return now, current, overlap
+
+ def _estimate(self, db, key: str) -> float:
+ _, current, overlap = self._windows()
+ counts = dict(db.execute(
+ "SELECT window, count FROM rate_limits WHERE key = ? AND window IN (?, ?)",
+ (f"{self.prefix}:{key}", current - 1, current),
+ ).fetchall())
+ return counts.get(current - 1, 0) * overlap + counts.get(current, 0)
+
+ def is_allowed(self, key: str) -> bool:
+ db = self._db()
+ _, current, _ = self._windows()
+ if self._estimate(db, key) >= self.limit:
+ return False
+ db.execute(
+ "INSERT INTO rate_limits (key, window, count) VALUES (?, ?, 1) "
+ "ON CONFLICT(key, window) DO UPDATE SET count = count + 1",
+ (f"{self.prefix}:{key}", current),
+ )
+ db.commit()
+ self._calls += 1
+ if self._calls % self.CLEANUP_EVERY == 0:
+ self.cleanup()
+ return True
+
+ def get_retry_after(self, key: str) -> int:
+ now, current, _ = self._windows()
+ return max(1, int((current + 1) * self.window_seconds - now) + 1)
+
+ def get_request_count(self, key: str) -> int:
+ return int(self._estimate(self._db(), key))
+
+ def cleanup(self, max_idle_seconds: int = 300) -> int:
+ """Remove windows too old to matter (this limiter's keys only)"""
+ _, current, _ = self._windows()
+ db = self._db()
+ cursor = db.execute(
+ "DELETE FROM rate_limits WHERE key LIKE ? AND window < ?",
+ (f"{self.prefix}:%", current - 1),
+ )
+ db.commit()
+ return cursor.rowcount
+
+
def init_rate_limiters(app) -> None:
"""
Initialize rate limiters with Flask app configuration.
@@ -157,8 +230,9 @@ def init_rate_limiters(app) -> None:
domain_limit = app.config.get('WEBMENTION_RATE_LIMIT_DOMAIN', 10)
window = app.config.get('WEBMENTION_RATE_LIMIT_WINDOW', 60)
- _ip_limiter = RateLimiter(ip_limit, window)
- _domain_limiter = RateLimiter(domain_limit, window)
+ # Shared by all server processes (v2.11.0)
+ _ip_limiter = SharedRateLimiter(ip_limit, window, "webmention-ip")
+ _domain_limiter = SharedRateLimiter(domain_limit, window, "webmention-domain")
app.logger.info(
f"Webmention rate limiter initialized: {ip_limit}/min IP, "
diff --git a/starpunk/routes/admin.py b/starpunk/routes/admin.py
index 00dc8ca..739d0ad 100644
--- a/starpunk/routes/admin.py
+++ b/starpunk/routes/admin.py
@@ -9,7 +9,7 @@ from flask import Blueprint, flash, g, jsonify, redirect, render_template, reque
import logging
import os
import shutil
-from datetime import datetime
+from datetime import datetime, timezone
logger = logging.getLogger(__name__)
@@ -531,7 +531,7 @@ def metrics():
from starpunk.monitoring.business import get_feed_statistics
response = {
- "timestamp": datetime.utcnow().isoformat() + "Z",
+ "timestamp": datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + "Z",
"process_id": os.getpid(),
"database": {},
"performance": {},
diff --git a/starpunk/routes/public.py b/starpunk/routes/public.py
index cd3c51c..5cf4726 100644
--- a/starpunk/routes/public.py
+++ b/starpunk/routes/public.py
@@ -6,7 +6,7 @@ No authentication required for these routes.
"""
import hashlib
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
from flask import Blueprint, abort, render_template, Response, current_app, request, send_from_directory
@@ -54,7 +54,7 @@ def _get_cached_notes():
# Get cache duration from config (in seconds)
cache_seconds = current_app.config.get("FEED_CACHE_SECONDS", 300)
cache_duration = timedelta(seconds=cache_seconds)
- now = datetime.utcnow()
+ now = datetime.now(timezone.utc).replace(tzinfo=None)
# The cached list is stale once any public note changed, in any process
# (v2.10.0): every change bumps content_version in the database
diff --git a/starpunk/routes/websub.py b/starpunk/routes/websub.py
index 4f1019a..02472a3 100644
--- a/starpunk/routes/websub.py
+++ b/starpunk/routes/websub.py
@@ -9,16 +9,15 @@ optional hub.secret and hub.lease_seconds. Answers 202 and verifies later
from flask import Blueprint, current_app, request
from starpunk.database import get_db
-from starpunk.rate_limiter import RateLimiter
+from starpunk.rate_limiter import SharedRateLimiter
from starpunk import websub
bp = Blueprint("websub", __name__)
# Subscription requests per client IP per minute
-# (per gunicorn process; needs TRUSTED_PROXY=true behind a reverse proxy,
-# or every client shares the proxy's address)
-_limiter = RateLimiter(limit=30, window_seconds=60)
-_requests = 0
+# Shared by all server processes (v2.11.0). Needs TRUSTED_PROXY=true behind a
+# reverse proxy, or every client shares the proxy's address.
+_limiter = SharedRateLimiter(limit=30, window_seconds=60, prefix="websub")
def _text(message: str, status: int):
@@ -31,10 +30,6 @@ def hub():
return _text("WebSub is not enabled on this site", 404)
from starpunk.routes.webmention import get_client_ip
- global _requests
- _requests += 1
- if _requests % 500 == 0:
- _limiter.cleanup() # forget idle clients so memory doesn't grow
if not _limiter.is_allowed(get_client_ip()):
return _text("Too many requests", 429)
diff --git a/starpunk/slug_utils.py b/starpunk/slug_utils.py
index 5bbc95f..d89c9bb 100644
--- a/starpunk/slug_utils.py
+++ b/starpunk/slug_utils.py
@@ -14,7 +14,7 @@ Per developer Q&A Q8:
import re
import unicodedata
import logging
-from datetime import datetime
+from datetime import datetime, timezone
from typing import Optional, Set
logger = logging.getLogger(__name__)
@@ -138,7 +138,7 @@ def sanitize_slug(slug: str, allow_timestamp_fallback: bool = False) -> str:
# Check if normalization resulted in empty slug
if not slug and allow_timestamp_fallback:
# Per Q8: Use timestamp-based fallback
- timestamp = datetime.utcnow().strftime('%Y%m%d-%H%M%S')
+ timestamp = datetime.now(timezone.utc).replace(tzinfo=None).strftime('%Y%m%d-%H%M%S')
slug = timestamp
logger.warning(
f"Slug normalization failed for input '{original_slug}' "
@@ -268,7 +268,7 @@ def generate_timestamp_slug(
'20251216143052-2'
"""
if created_at is None:
- created_at = datetime.utcnow()
+ created_at = datetime.now(timezone.utc).replace(tzinfo=None)
if existing_slugs is None:
existing_slugs = set()
@@ -340,7 +340,7 @@ def validate_and_sanitize_custom_slug(custom_slug: str, existing_slugs: Set[str]
if not sanitized:
# This should never happen with allow_timestamp_fallback=True
# but handle it just in case
- timestamp = datetime.utcnow().strftime('%Y%m%d-%H%M%S')
+ timestamp = datetime.now(timezone.utc).replace(tzinfo=None).strftime('%Y%m%d-%H%M%S')
sanitized = timestamp
logger.error(
f"Unexpected empty slug after sanitization with fallback. "
@@ -361,7 +361,7 @@ def validate_and_sanitize_custom_slug(custom_slug: str, existing_slugs: Set[str]
if not validate_slug(sanitized):
# This should rarely happen after sanitization
# but if it does, use timestamp fallback
- timestamp = datetime.utcnow().strftime('%Y%m%d-%H%M%S')
+ timestamp = datetime.now(timezone.utc).replace(tzinfo=None).strftime('%Y%m%d-%H%M%S')
logger.warning(
f"Slug '{sanitized}' (from '{custom_slug}') failed validation. "
f"Using timestamp fallback: {timestamp}"
@@ -375,7 +375,7 @@ def validate_and_sanitize_custom_slug(custom_slug: str, existing_slugs: Set[str]
except ValueError as e:
# This should rarely happen, but if it does, use timestamp
# Per Q8: Never fail Micropub request
- timestamp = datetime.utcnow().strftime('%Y%m%d-%H%M%S')
+ timestamp = datetime.now(timezone.utc).replace(tzinfo=None).strftime('%Y%m%d-%H%M%S')
logger.error(
f"Could not create unique slug from '{custom_slug}'. "
f"Using timestamp: {timestamp}. Error: {e}"
diff --git a/starpunk/syndication.py b/starpunk/syndication.py
index 8f0afd8..5ab9743 100644
--- a/starpunk/syndication.py
+++ b/starpunk/syndication.py
@@ -444,39 +444,37 @@ CARD_IMAGE_LIMIT = 5 * 1024 * 1024
def _fetch_public(http: httpx.Client, url: str, limit: int, truncate: bool) -> tuple:
"""
- GET a public web URL, checking every redirect hop (no requests into the
- LAN). Reads at most `limit` bytes: stops there if `truncate`, else fails.
- Returns (data, content_type, final_url).
- """
- from urllib.parse import urljoin, urlparse
- from starpunk.micropub import _public_address
+ GET a public web URL (link card pages, images to re-upload).
- site = current_app.config.get("SITE_URL", "")
- for _ in range(5):
- parsed = urlparse(url)
- if parsed.scheme not in ("http", "https") or not parsed.hostname:
- raise SyndicationError(f"not a web URL: {url}", permanent=True)
- if not current_app.config.get("TESTING") and not _public_address(parsed.hostname):
- raise SyndicationError(f"not a public address: {url}", permanent=True)
- try:
- with http.stream("GET", url, follow_redirects=False, timeout=10.0,
- headers={"User-Agent": f"StarPunk (+{site})"}) as response:
- if response.is_redirect and "location" in response.headers:
- url = urljoin(url, response.headers["location"])
- continue
- if response.status_code >= 400:
- raise SyndicationError(f"{url}: HTTP {response.status_code}")
- data = bytearray()
- for chunk in response.iter_bytes():
- data += chunk
- if len(data) > limit:
- if truncate:
- break
- raise SyndicationError(f"{url}: too large")
- return bytes(data), response.headers.get("content-type", ""), url
- except httpx.HTTPError as e:
- raise SyndicationError(f"{url}: {type(e).__name__}")
- raise SyndicationError(f"{url}: too many redirects")
+ Uses the shared SSRF-safe fetcher (v2.11.0): every redirect hop checked,
+ the connection pinned to the checked address, a hard 10 s limit. Reads at
+ most `limit` bytes: stops there if `truncate`, else fails.
+ Returns (data, content_type, final_url). `http` is no longer used.
+ """
+ from urllib.parse import urlparse
+ from starpunk.webmention_ssrf import (
+ FetchTooLargeError, ResolutionError, SSRFBlockedError, safe_get, user_agent,
+ )
+
+ parsed = urlparse(url)
+ if parsed.scheme not in ("http", "https") or not parsed.hostname:
+ raise SyndicationError(f"not a web URL: {url}", permanent=True)
+ try:
+ result = safe_get(url, headers={"User-Agent": user_agent(current_app.config.get("SITE_URL", ""))},
+ max_bytes=limit, max_redirects=5, timeout=10.0, truncate=truncate)
+ except ResolutionError as e:
+ raise SyndicationError(f"{url}: {e}")
+ except (SSRFBlockedError, ValueError) as e:
+ raise SyndicationError(f"not a public address: {url} ({e})", permanent=True)
+ except FetchTooLargeError:
+ raise SyndicationError(f"{url}: too large")
+ except httpx.TooManyRedirects:
+ raise SyndicationError(f"{url}: too many redirects")
+ except httpx.HTTPError as e:
+ raise SyndicationError(f"{url}: {type(e).__name__}")
+ if result.status_code >= 400:
+ raise SyndicationError(f"{url}: HTTP {result.status_code}")
+ return result.content, result.headers.get("content-type", ""), result.url
def page_card(http: httpx.Client, url: str) -> dict:
diff --git a/starpunk/utils.py b/starpunk/utils.py
index aa18f09..4438ddf 100644
--- a/starpunk/utils.py
+++ b/starpunk/utils.py
@@ -217,7 +217,7 @@ def generate_slug(content: str, created_at: Optional[datetime] = None) -> str:
# If slug is empty or too short, use timestamp fallback
if len(slug) < MIN_SLUG_LENGTH:
if created_at is None:
- created_at = datetime.utcnow()
+ created_at = datetime.now(timezone.utc).replace(tzinfo=None)
slug = created_at.strftime("%Y%m%d-%H%M%S")
# Truncate to maximum length
@@ -592,7 +592,7 @@ def delete_note_file(
month = parts[notes_idx + 2]
except (ValueError, IndexError):
# If path doesn't follow expected structure, use current date
- now = datetime.utcnow()
+ now = datetime.now(timezone.utc).replace(tzinfo=None)
year = now.strftime("%Y")
month = now.strftime("%m")
diff --git a/starpunk/webmention_ssrf.py b/starpunk/webmention_ssrf.py
index 1a4fd6c..fabc47c 100644
--- a/starpunk/webmention_ssrf.py
+++ b/starpunk/webmention_ssrf.py
@@ -250,7 +250,8 @@ def safe_post(url: str, data: Optional[dict] = None, headers: Optional[dict] = N
def safe_get(url: str, headers: dict, max_bytes: int = MAX_BYTES,
- max_redirects: int = MAX_REDIRECTS, timeout: float = TIMEOUT) -> FetchResult:
+ max_redirects: int = MAX_REDIRECTS, timeout: float = TIMEOUT,
+ truncate: bool = False) -> FetchResult:
"""
GET a URL, following redirects by hand and checking each hop.
@@ -258,6 +259,8 @@ def safe_get(url: str, headers: dict, max_bytes: int = MAX_BYTES,
url: URL to fetch
headers: Request headers
max_bytes: Response body limit
+ truncate: Stop at max_bytes and return what was read, instead of
+ raising FetchTooLargeError (v2.11.0)
max_redirects: Redirect limit
timeout: Total time limit in seconds
@@ -287,13 +290,16 @@ def safe_get(url: str, headers: dict, max_bytes: int = MAX_BYTES,
continue
content_length = response.headers.get('Content-Length', '')
- if content_length.isdigit() and int(content_length) > max_bytes:
+ if not truncate and content_length.isdigit() and int(content_length) > max_bytes:
raise FetchTooLargeError(f"Response exceeds size limit ({max_bytes} bytes)")
content = b''
for chunk in response.iter_bytes(chunk_size=8192):
content += chunk
if len(content) > max_bytes:
+ if truncate:
+ content = content[:max_bytes]
+ break
raise FetchTooLargeError(f"Response exceeds size limit ({max_bytes} bytes)")
if time.monotonic() > deadline:
raise httpx.TimeoutException("Fetch exceeded time limit")
diff --git a/starpunk/webmention_worker.py b/starpunk/webmention_worker.py
index 4af1f01..1042447 100644
--- a/starpunk/webmention_worker.py
+++ b/starpunk/webmention_worker.py
@@ -19,7 +19,7 @@ import threading
import time
import sqlite3
import httpx
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Dict, Any, Optional
from urllib.parse import urljoin
@@ -116,7 +116,7 @@ class WebmentionWorker(threading.Thread):
# Update last processed time
if processed > 0:
- self._last_processed = datetime.utcnow()
+ self._last_processed = datetime.now(timezone.utc).replace(tzinfo=None)
# Sleep if nothing processed
if processed == 0:
@@ -790,7 +790,7 @@ class WebmentionWorker(threading.Thread):
}
delay = delays.get(new_retry_count)
if delay:
- next_retry_at = datetime.utcnow() + delay
+ next_retry_at = datetime.now(timezone.utc).replace(tzinfo=None) + delay
try:
conn = sqlite3.connect(self.db_path)
diff --git a/starpunk/websub_worker.py b/starpunk/websub_worker.py
index b4410fd..2d2b38e 100644
--- a/starpunk/websub_worker.py
+++ b/starpunk/websub_worker.py
@@ -25,7 +25,7 @@ import hashlib
import hmac
import logging
import threading
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Optional
from urllib.parse import urlencode, urlparse, urlsplit, urlunsplit
@@ -53,11 +53,11 @@ def signature(secret: str, body: bytes) -> str:
def _now() -> str:
- return datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
+ return datetime.now(timezone.utc).replace(tzinfo=None).strftime("%Y-%m-%d %H:%M:%S")
def _later(seconds: int) -> str:
- return (datetime.utcnow() + timedelta(seconds=seconds)).strftime("%Y-%m-%d %H:%M:%S")
+ return (datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(seconds=seconds)).strftime("%Y-%m-%d %H:%M:%S")
class WebSubWorker(threading.Thread):
@@ -132,7 +132,7 @@ class WebSubWorker(threading.Thread):
).fetchall():
self._guarded(db, row, "delivery", lambda d, r: self.deliver(d, r, rendered), self._retry_or_drop)
done += 1
- if self._last_prune is None or datetime.utcnow() - self._last_prune > PRUNE_EVERY:
+ if self._last_prune is None or datetime.now(timezone.utc).replace(tzinfo=None) - self._last_prune > PRUNE_EVERY:
self.prune(db)
return done
@@ -348,7 +348,7 @@ class WebSubWorker(threading.Thread):
db.execute("DELETE FROM websub_deliveries WHERE subscription_id IS NOT NULL AND "
"subscription_id NOT IN (SELECT id FROM websub_subscriptions)")
db.commit()
- self._last_prune = datetime.utcnow()
+ self._last_prune = datetime.now(timezone.utc).replace(tzinfo=None)
if cursor.rowcount:
logger.info(f"WebSub: removed {cursor.rowcount} expired subscriptions")
return cursor.rowcount
diff --git a/templates/admin/authorize.html b/templates/admin/authorize.html
index f4300d0..786986a 100644
--- a/templates/admin/authorize.html
+++ b/templates/admin/authorize.html
@@ -16,7 +16,7 @@
After approval you will be sent to {{ auth_request.redirect_uri }}