diff --git a/.env.example b/.env.example index c693ccd..be1e920 100644 --- a/.env.example +++ b/.env.example @@ -73,10 +73,10 @@ DATABASE_PATH=./data/starpunk.db # FLASK / LOGGING # ============================================================================= -# development or production (default: development) -FLASK_ENV=development -# 1 = debug on, 0 = off (default: 1). Set 0 in production. -FLASK_DEBUG=1 +# development or production (default: production) +FLASK_ENV=production +# 1 = debug on, 0 = off (default: 0). `flask run --debug` turns it on locally. +FLASK_DEBUG=0 # DEBUG, INFO, WARNING, ERROR or CRITICAL (default: INFO) LOG_LEVEL=INFO # Overrides the version shown in the footer and /health (default: the @@ -104,6 +104,10 @@ LOG_LEVEL=INFO # (default: false) TRUSTED_PROXY=false +# CSRF tokens on admin forms, logout and the IndieAuth consent (default: true). +# Leave on; tests turn it off unless they test it. +CSRF_ENABLED=true + # ============================================================================= # FEEDS # ============================================================================= diff --git a/CHANGELOG.md b/CHANGELOG.md index 2df7f16..0111781 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [2.11.0] - 2026-11-03 + +Maintenance release: dependency upgrades and the High-priority backlog items. + +### Security + +- Every dependency is upgraded to its current release. The previous set had 97 published vulnerabilities (Pillow 64, PyJWT 22, gunicorn 4, and Flask, Markdown, python-dotenv, pytest and pillow-heif); the new set has none known. Notable versions: Flask 3.1, gunicorn 26, Pillow 13, PyJWT 2.15, httpx 0.28, nh3 0.3, Markdown 3.11, pillow-heif 1.8, psutil 7 +- Admin forms, logout and the IndieAuth consent screen now carry a CSRF token (an HMAC of your login session, no storage needed); requests without it are refused. Endpoints that apps call with tokens (Micropub, Webmention, WebSub, the token endpoint) are unaffected +- Fetching an IndieAuth app's page and fetching pages and images for crosspost link cards now use the same protected fetcher as webmentions: every redirect is checked against private addresses, the connection goes to the checked address, and the whole fetch has a time limit +- Debug mode is off and the environment is production unless set otherwise (`flask run --debug` still turns debug on locally); the container image sets both explicitly + +### Changed + +- Runs on Python 3.14 (was 3.11, which reaches end of life in 2027) +- Rate limits on `/webmention` and `/websub` are counted in the database and shared by all server processes. Before, each of the four processes counted separately, so the real limit was four times the setting +- Test dependencies are in `requirements-test.txt` (used by CI); `pytest` is no longer installed in the production image, and `lxml` and `python-dateutil`, which StarPunk uses directly, are now pinned +- `uv` is pinned in the Containerfile instead of `latest` + +### Fixed + +- Deleting a note or a photo left its tag, photo and image-size rows behind, because SQLite only applies the schema's cascades when foreign keys are switched on. They now are; migration 018 removes rows left by earlier deletes (production had three) +- Deprecated `datetime.utcnow()` and sqlite3's default date adapter (both removed in a future Python) replaced; the stored formats are unchanged +- `compose.yaml` pinned version 0.6.0 (which would override the real version shown) and set an unused variable; a stray duplicate `dev_auth.py` and a `test.ini` with a private host name were removed + ## [2.10.2] - 2026-11-03 ### Fixed diff --git a/CLAUDE.md b/CLAUDE.md index 61d2c33..f5936c3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -7,7 +7,7 @@ This file contains operational instructions for Claude agents working on this pr - We use **uv** for Python virtual environment management - All Python commands must be run with `uv run` prefix - Example: `uv run pytest`, `uv run flask --app app.py run --debug` -- There is no `pyproject.toml`: create the environment with `uv venv --python 3.11 && uv pip install -r requirements-dev.txt` +- There is no `pyproject.toml`: create the environment with `uv venv --python 3.14 && uv pip install -r requirements-dev.txt` - Tests need `SESSION_SECRET` set: `SESSION_SECRET=test uv run pytest`. The full suite takes about 6 minutes, 5 of them in `tests/test_migration_race_condition.py`; add `--ignore=tests/test_migration_race_condition.py` while iterating, but run everything before a release - Local login without OIDC: `DEV_MODE=true` and `DEV_ADMIN_ME` in `.env`, then `/dev/login` - Setup details: `docs/standards/development-setup.md` diff --git a/Containerfile b/Containerfile index 27d06cb..64787e6 100644 --- a/Containerfile +++ b/Containerfile @@ -6,10 +6,10 @@ # ============================================================================ # Build Stage - Install dependencies in virtual environment # ============================================================================ -FROM python:3.11-slim AS builder +FROM python:3.14-slim AS builder # Install uv for fast dependency installation -COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv +COPY --from=ghcr.io/astral-sh/uv:0.9.28 /uv /usr/local/bin/uv WORKDIR /build @@ -25,11 +25,12 @@ RUN uv venv /opt/venv && \ # ============================================================================ # Test Stage - Runs the test suite (CI builds it with --target test) # ============================================================================ -FROM python:3.11-slim AS test +FROM python:3.14-slim AS test -COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv +COPY --from=ghcr.io/astral-sh/uv:0.9.28 /uv /usr/local/bin/uv COPY --from=builder /opt/venv /opt/venv -RUN uv pip install --no-cache --python /opt/venv/bin/python pytest pytest-mock +COPY requirements-test.txt . +RUN uv pip install --no-cache --python /opt/venv/bin/python -r requirements-test.txt ENV PATH="/opt/venv/bin:$PATH" \ PYTHONUNBUFFERED=1 \ @@ -45,7 +46,7 @@ RUN python -m pytest -q -p no:cacheprovider tests # ============================================================================ # Runtime Stage - Minimal production image (default target: keep it last) # ============================================================================ -FROM python:3.11-slim +FROM python:3.14-slim # Create non-root user for security # UID/GID 1000 is standard for first user on most systems @@ -60,6 +61,8 @@ COPY --from=builder /opt/venv /opt/venv ENV PATH="/opt/venv/bin:$PATH" \ PYTHONUNBUFFERED=1 \ PYTHONDONTWRITEBYTECODE=1 \ + FLASK_ENV=production \ + FLASK_DEBUG=0 \ FLASK_APP=app.py \ DATA_PATH=/data \ NOTES_PATH=/data/notes \ diff --git a/README.md b/README.md index 152450c..71dd988 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ redirect URI with your OIDC provider. Details: ### Local development ```bash -uv venv --python 3.11 +uv venv --python 3.14 uv pip install -r requirements-dev.txt cp .env.example .env # set SESSION_SECRET; add DEV_MODE=true and DEV_ADMIN_ME uv run flask --app app.py run --debug # http://localhost:5000 diff --git a/compose.yaml b/compose.yaml index 615f718..bf737c7 100644 --- a/compose.yaml +++ b/compose.yaml @@ -11,12 +11,11 @@ # docker compose logs -f # docker compose down -version: '3.8' - services: starpunk: # Container configuration - image: starpunk:0.6.0 + # Built locally from this checkout (release images: see README) + image: starpunk:local container_name: starpunk # Build configuration @@ -49,10 +48,6 @@ services: - NOTES_PATH=/data/notes - DATABASE_PATH=/data/starpunk.db - # Application metadata - - VERSION=0.6.0 - - ENVIRONMENT=production - # Volume mounts for persistent data # All application data stored in ./container-data on host volumes: diff --git a/dev_auth.py b/dev_auth.py deleted file mode 100644 index ee9aa9b..0000000 --- a/dev_auth.py +++ /dev/null @@ -1,103 +0,0 @@ -""" -Development authentication module for StarPunk - -WARNING: This module provides a development-only authentication mechanism -that bypasses IndieLogin. It should NEVER be enabled in production. - -This module is separate from production auth (starpunk/auth.py) to maintain -clear architectural boundaries and enable easy security audits. - -Security measures: -- Only active when DEV_MODE=true -- Returns 404 if DEV_MODE=false -- Requires DEV_ADMIN_ME configuration -- Logs prominent warnings -- Cannot coexist with production SITE_URL -- Visual warnings in UI - -Functions: - is_dev_mode: Check if development mode is enabled - validate_dev_config: Validate development configuration - create_dev_session: Create session without authentication -""" - -from flask import current_app - -from starpunk.auth import create_session - - -def is_dev_mode() -> bool: - """ - Check if development mode is enabled - - Returns: - True if DEV_MODE is enabled, False otherwise - - Example: - >>> from starpunk.dev_auth import is_dev_mode - >>> if is_dev_mode(): - ... print("Development mode active") - """ - return current_app.config.get("DEV_MODE", False) - - -def validate_dev_config() -> None: - """ - Validate development mode configuration - - Checks that DEV_MODE configuration is valid and safe: - - DEV_ADMIN_ME must be set if DEV_MODE is true - - Warns if DEV_MODE is enabled with production-like SITE_URL - - Raises: - ValueError: If DEV_MODE is true but DEV_ADMIN_ME is not set - - Logs: - WARNING: If DEV_MODE is enabled with HTTPS SITE_URL - """ - dev_mode = current_app.config.get("DEV_MODE", False) - - if dev_mode: - # Require DEV_ADMIN_ME - dev_admin_me = current_app.config.get("DEV_ADMIN_ME") - if not dev_admin_me: - raise ValueError("DEV_MODE=true requires DEV_ADMIN_ME to be set") - - # Warn if production-like configuration detected - site_url = current_app.config.get("SITE_URL", "") - if site_url.startswith("https://"): - current_app.logger.warning( - "WARNING: DEV_MODE is enabled with production SITE_URL. " - "This is likely a misconfiguration. " - "DEV_MODE should only be used in local development." - ) - - -def create_dev_session(me: str) -> str: - """ - Create development session without authentication - - WARNING: This bypasses IndieLogin authentication entirely. - Only use in development environments. - - Args: - me: User identity URL (from DEV_ADMIN_ME config) - - Returns: - Session token (same format as production sessions) - - Logs: - WARNING: Logs that dev session was created without authentication - - Example: - >>> token = create_dev_session("https://example.com") - >>> # Session created without IndieLogin verification - """ - current_app.logger.warning( - f"DEV MODE: Creating session for {me} without authentication. " - f"This should NEVER happen in production!" - ) - - # Use production session creation (same session format) - # This ensures dev sessions work identically to production - return create_session(me) diff --git a/docs/architecture/INDEX.md b/docs/architecture/INDEX.md index cd02ffb..6f22d4a 100644 --- a/docs/architecture/INDEX.md +++ b/docs/architecture/INDEX.md @@ -11,7 +11,7 @@ per-release designs and reports are in [../design/](../design/). |----------|-------------| | [overview.md](overview.md) | System overview: purpose, request flow, modules and routes, background workers, storage, auth model, URL surface, caching, security posture | | [technology-stack.md](technology-stack.md) | Current runtime, Python dependencies, standards implemented, testing and CI | -| [database.md](database.md) | SQLite schema after migrations 001–017, migration runner and its lock, connection pool | +| [database.md](database.md) | SQLite schema after migrations 001–019, migration runner and its lock, connection pool | | [deployment.md](deployment.md) | Container image stages, CI workflow, release process, `/data` layout, reverse proxy, all environment variables | | [simplified-auth-architecture.md](simplified-auth-architecture.md) | Admin login via OpenID Connect and the built-in IndieAuth server that issues Micropub tokens (ADR-064) | | [indieweb-protocols.md](indieweb-protocols.md) | How Micropub, IndieAuth, Webmention, microformats2, WebSub and the feeds are implemented, with compliance status and known gaps | diff --git a/docs/architecture/database.md b/docs/architecture/database.md index ecf2639..c05e596 100644 --- a/docs/architecture/database.md +++ b/docs/architecture/database.md @@ -1,8 +1,8 @@ # Database Architecture -**Version**: 2.10.1 (2026-11-03) +**Version**: 2.11.0 (2026-11-03) **Status**: Current. Verified against a database created by `create_app()` -with migrations 001–017. +with migrations 001–019. StarPunk uses one SQLite file, `starpunk.db` under `DATA_PATH` (`/data` in the container), in WAL mode. Note *content* is not in the database: it lives @@ -71,6 +71,8 @@ Background: [ADR-031](../decisions/ADR-031-database-migration-system-redesign.md | `015_bookmarks_and_content_format.sql` | 2.9 | `notes.bookmark_of`, `notes.content_format` | | `016_websub.sql` | 2.10.0 | `websub_subscriptions`, `websub_intents`, `websub_deliveries`, `site_state` | | `017_websub_hardening.sql` | 2.10.1 | `websub_intents.request_id`, `requested_topic`, `callback_ip`; `websub_subscriptions.callback_ip` | +| `018_foreign_key_cleanup.sql` | 2.11.0 | Removes rows orphaned while foreign keys weren't enforced | +| `019_shared_rate_limits.sql` | 2.11.0 | `rate_limits` (counters shared by all processes) | New migrations are numbered `NNN_description.sql`, must be safe to run inside one transaction, and should use `IF NOT EXISTS` where SQLite allows it. @@ -205,13 +207,18 @@ its cached feed data is stale. **`schema_migrations`**: applied migrations (see above). +**`rate_limits`** (v2.11.0): request counts for `/webmention` and `/websub`, +one row per key (`:`) and time window, so every +server process enforces one shared limit. Old windows are pruned as it goes. + ## Connections - **Request connections**: `starpunk/database/pool.py`. `init_pool()` creates one `ConnectionPool` per process (up to 5 idle connections, 10-second busy timeout; `DB_POOL_SIZE` and `DB_TIMEOUT` can be set only through the `create_app()` config dict). Connections use `sqlite3.Row` rows, - `check_same_thread=False` and `PRAGMA journal_mode=WAL`, and are wrapped + `check_same_thread=False`, `PRAGMA journal_mode=WAL` and (since v2.11.0) + `PRAGMA foreign_keys=ON`, so `ON DELETE CASCADE`/`SET NULL` run, and are wrapped in `MonitoredConnection` for query timing when metrics are on. `get_db()` stores one connection in `flask.g` per request or app context; on teardown it goes back to the pool, rolled back first if a transaction was left diff --git a/docs/architecture/deployment.md b/docs/architecture/deployment.md index 1bd1efa..eded392 100644 --- a/docs/architecture/deployment.md +++ b/docs/architecture/deployment.md @@ -24,12 +24,12 @@ deployment repository, not here. ## Container image (`Containerfile`) -A multi-stage build on `python:3.11-slim`; it works with Podman and Docker. +A multi-stage build on `python:3.14-slim`; it works with Podman and Docker. | Stage | Purpose | |-------|---------| | `builder` | Copies `uv` from `ghcr.io/astral-sh/uv`, creates `/opt/venv` and installs `requirements.txt` into it | -| `test` | Copies the venv, adds `pytest` and `pytest-mock`, copies the source and runs `python -m pytest -q -p no:cacheprovider tests` with test values for `SESSION_SECRET`, `SITE_URL` and `ADMIN_ME`. The stage only builds if the tests pass. | +| `test` | Copies the venv, installs `requirements-test.txt`, copies the source and runs `python -m pytest -q -p no:cacheprovider tests` with test values for `SESSION_SECRET`, `SITE_URL` and `ADMIN_ME`. The stage only builds if the tests pass. | | runtime (last, the default target) | Creates user `starpunk` (UID/GID 1000) and `/data/notes`, copies the venv and the application to `/app`, sets `DATA_PATH=/data`, `NOTES_PATH=/data/notes`, `DATABASE_PATH=/data/starpunk.db`, runs as `starpunk`, exposes port 8000 | The runtime image's command is gunicorn: @@ -172,8 +172,8 @@ ignored with a warning. | `DATA_PATH` | `./data` (image: `/data`) | Root of all stored data | | `DATABASE_PATH` | `./data/starpunk.db` (image: `/data/starpunk.db`) | | | `NOTES_PATH` | `./data/notes` (image: `/data/notes`) | Created at start-up, but note files are always written to `DATA_PATH/notes` | -| `FLASK_ENV` | `development` | Set `production` in production | -| `FLASK_DEBUG` | `1` | Set `0` in production (the image does not set it) | +| `FLASK_ENV` | `production` | Reported in `/health` and admin diagnostics; the image sets `production` | +| `FLASK_DEBUG` | `0` | Debug off unless `1` (`flask run --debug` sets it locally); the image sets `0` | | `LOG_LEVEL` | `INFO` | `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL` | | `VERSION` | package `__version__` | Overrides the reported version; normally unset | diff --git a/docs/architecture/indieweb-protocols.md b/docs/architecture/indieweb-protocols.md index 0bb4558..a55eebc 100644 --- a/docs/architecture/indieweb-protocols.md +++ b/docs/architecture/indieweb-protocols.md @@ -197,7 +197,8 @@ route isn't registered at all. 1. Rate limits: 60 requests per minute per client IP and 10 per minute per source domain (`WEBMENTION_RATE_LIMIT_*`). Over the limit gets 429 with - `Retry-After`. The limiter is in memory, per process. The client IP + `Retry-After`. Counts are shared by all server processes (`rate_limits` + table, v2.11.0). The client IP comes from `X-Forwarded-For` only when `TRUSTED_PROXY=true`. 2. Form-encoded (JSON also accepted). `source` and `target` are required, http(s), at most 2048 characters, and different from each other. @@ -445,7 +446,7 @@ Drafts that stay drafts change nothing. Limits (429 when hit): -- 30 requests per minute per client IP (per process; needs +- 30 requests per minute per client IP (shared by all processes; needs `TRUSTED_PROXY=true` behind a proxy); - at most 200 pending verifications; - `WEBSUB_MAX_SUBSCRIPTIONS` (1000) in total; @@ -488,7 +489,6 @@ retried. - The hub serves only this site's topics, by design. websub.rocks' hub tests use their own topics, so they can't be run against it. -- Rate limiting is per process and in memory. - Deliveries always carry the full topic content (no diffs). Fat pings aren't sent to external hubs. diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index c79b0ac..75e6efd 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -293,13 +293,12 @@ their credentials are set. The full list with defaults is in callbacks, remote Micropub photos, IndieAuth client pages, crosspost link cards) are checked first: every address the host resolves to must be public (private, loopback, link-local, CGNAT, multicast and reserved ranges are - refused). The fetcher in `webmention_ssrf.py`, used for webmentions, reply - contexts, WebSub and remote Micropub photos, follows redirects by hand and - re-checks each hop, connects to the address it checked (no second DNS - lookup) and limits response size (1 MB by default) and total time. - Crosspost link cards use their own check that also re-checks each redirect. - The IndieAuth client-page fetch checks only the initial URL. `DEV_MODE` - relaxes the checks for local testing. + refused). Every fetch of a user-supplied URL (webmentions, reply contexts, + WebSub, remote Micropub photos, IndieAuth client pages and, since v2.11.0, + crosspost link cards) uses the fetcher in `webmention_ssrf.py`: it follows + redirects by hand and re-checks each hop, connects to the address it + checked (no second DNS lookup) and limits response size (1 MB by default) + and total time. `DEV_MODE` relaxes the checks for local testing. - **HTML sanitising**: rendered notes (Markdown or HTML sent by a Micropub client) and webmention content are cleaned with nh3 allow-lists; search highlighting escapes everything except ``. @@ -307,15 +306,17 @@ their credentials are set. The full list with defaults is in (defaults 60 and 10 per 60 seconds); `/websub` 30 requests per IP per minute, plus limits on subscriptions in total, per callback host and per callback address, and on requests waiting for verification. Request-rate - counters are in memory and per process. + counters live in the `rate_limits` table, shared by all server processes + (v2.11.0). - **Client IP behind a proxy**: with `TRUSTED_PROXY=true` the first address in `X-Forwarded-For` is used as the client IP for rate limiting. Set it only when the app is reachable solely through a proxy that sets that header (Caddy does); without it, every client shares the proxy's address. - **Auth**: hashed session and token storage, single-use login state, PKCE for both OIDC and IndieAuth, redirect URIs checked against the client, - post-login redirects limited to this site. Admin form POSTs rely on the - SameSite=Lax session cookie (there are no per-form CSRF tokens). + post-login redirects limited to this site. Admin form POSTs, logout and + the IndieAuth consent carry a CSRF token: an HMAC of the session cookie, + checked by `starpunk/csrf.py` (v2.11.0), on top of the SameSite=Lax cookie. - **Uploads**: request bodies are capped at 50 MB; images are processed by Pillow and stored under random (UUID) names. - **Container**: runs as a non-root user. HTTPS and security headers (HSTS, diff --git a/docs/architecture/simplified-auth-architecture.md b/docs/architecture/simplified-auth-architecture.md index 0bac38f..f20bf58 100644 --- a/docs/architecture/simplified-auth-architecture.md +++ b/docs/architecture/simplified-auth-architecture.md @@ -328,14 +328,16 @@ In place: untrusted redirect URIs never followed. - Post-login redirects are limited to this site. +CSRF (v2.11.0): admin form POSTs, `/auth/logout` and the consent form's +approve button need a `csrf_token` field (or `X-CSRF-Token` header) equal to +an HMAC-SHA256 of the `starpunk_session` cookie under `SECRET_KEY`; anything +else gets 400. The token changes with every login and needs no storage. +Endpoints apps call with bearer tokens are not affected. The client metadata +fetch uses `webmention_ssrf.safe_get` (every redirect hop checked, the +connection pinned, a 5-second limit). + Known gaps: -- Admin forms and the consent form have no CSRF token. They rely on the - `SameSite=Lax` session cookie, which cross-site POSTs don't carry. -- The client metadata fetch checks the `client_id` URL against private - addresses, but then makes a plain `httpx.get` that follows redirects. - Redirect hops are not re-checked and the connection is not pinned, unlike - `webmention_ssrf.safe_get`. - `scopes_supported` leaves out `draft`, although Micropub honours it and the consent screen grants it when requested. Unknown scopes are also shown and granted as requested; they just have no effect. diff --git a/docs/architecture/syndication-architecture.md b/docs/architecture/syndication-architecture.md index 6c0f17f..c18c069 100644 --- a/docs/architecture/syndication-architecture.md +++ b/docs/architecture/syndication-architecture.md @@ -246,10 +246,6 @@ to itself (v2.8.0). - Bluesky and Threads copies don't follow edits (platform limitation). Unticking and re-ticking a target makes a new copy. -- Link-card and Threads image fetches check every redirect hop against - private addresses but don't pin DNS, and use a per-request rather than an - overall time limit. Webmention and WebSub fetching use the stricter - `webmention_ssrf` fetcher. - No per-target choice for Micropub updates: `mp-syndicate-to` is read only on create. - Likes of ordinary web pages, and all likes on Threads, stay on the site. diff --git a/docs/architecture/technology-stack.md b/docs/architecture/technology-stack.md index 41b0f42..6705a54 100644 --- a/docs/architecture/technology-stack.md +++ b/docs/architecture/technology-stack.md @@ -14,9 +14,9 @@ the pieces fit together. | Component | Choice | |-----------|--------| -| Language | Python 3.11 (`python:3.11-slim` base image) | -| Web framework | Flask 3.0 (Werkzeug, Jinja2, itsdangerous, click come with it) | -| WSGI server | gunicorn 21.2, 4 sync workers (see [deployment.md](deployment.md)) | +| Language | Python 3.14 (`python:3.14-slim` base image; since v2.11.0) | +| Web framework | Flask 3.1 (Werkzeug, Jinja2, itsdangerous, click come with it) | +| WSGI server | gunicorn 26.2, 4 sync workers (see [deployment.md](deployment.md)) | | Database | SQLite (standard-library `sqlite3`), WAL mode, FTS5 for search | | Content storage | Markdown files on disk | | Background work | `threading` daemon threads; `fcntl.flock` file locks to pick one process | @@ -28,32 +28,36 @@ the pieces fit together. ## Python dependencies (`requirements.txt`) +Pinned to a minor series. Upgraded in v2.11.0, when the set had no known +vulnerabilities (`pip-audit`); check again before each release. + | Package | Pin | Used for | |---------|-----|----------| -| Flask | `3.0.*` | Web framework | -| gunicorn | `21.2.*` | Production WSGI server | -| markdown | `3.5.*` | Rendering notes (`extra`, `codehilite`, `nl2br` and a bare-URL linking extension) | -| feedgen | `1.0.*` | RSS 2.0 feed (`starpunk/feeds/rss.py`); Atom and JSON Feed are generated by StarPunk's own code | -| httpx | `0.27.*` | All outbound HTTP: OIDC, IndieAuth client discovery, webmention and WebSub requests (through the SSRF-safe wrapper in `webmention_ssrf.py`), crossposting APIs, health check | -| python-dotenv | `1.0.*` | Loading `.env` | -| beautifulsoup4 | `4.12.*` | HTML parsing: IndieAuth client pages, crosspost text and link cards | +| Flask | `3.1.*` | Web framework | +| gunicorn | `26.2.*` | Production WSGI server | +| markdown | `3.11.*` | Rendering notes (`extra`, `codehilite`, `nl2br` and a bare-URL linking extension) | +| nh3 | `0.3.*` | HTML sanitising (rendered notes, webmention content) | +| beautifulsoup4 | `4.15.*` | HTML parsing: IndieAuth client pages, crosspost text and link cards | +| lxml | `6.1.*` | Outgoing link extraction, webmention endpoint discovery and source checks | | mf2py | `2.0.*` | Parsing microformats2 (webmention sources, reply contexts, author h-card) | -| pytest | `8.0.*` | Test runner (listed here as well as in the image's test stage) | -| psutil | `5.9.*` | Memory monitor and system metrics | -| Pillow | `10.1.*` | Image validation, resizing and variants | -| pillow-heif | `0.18.*` | HEIC/HEIF uploads (iPhone photos) | -| nh3 | `0.2.*` | HTML sanitising (rendered notes, webmention content) | -| PyJWT[crypto] | `2.10.*` | Verifying OIDC ID tokens (JWKS, RS/ES signatures via `cryptography`) | +| python-dateutil | `4.0.*` | Dates in reply contexts | +| feedgen | `1.0.*` | RSS 2.0 feed (`starpunk/feeds/rss.py`); Atom and JSON Feed are generated by StarPunk's own code | +| httpx | `0.28.*` | All outbound HTTP; requests to user-supplied URLs go through `webmention_ssrf.safe_get`/`safe_post` | +| PyJWT[crypto] | `2.15.*` | Verifying OIDC ID tokens (JWKS, RS/ES signatures via `cryptography`) | +| python-dotenv | `1.2.*` | Loading `.env` | +| Pillow | `13.0.*` | Image validation, resizing and variants | +| pillow-heif | `1.8.*` | HEIC/HEIF uploads (iPhone photos) | +| psutil | `7.2.*` | Memory monitor and system metrics | -Packages imported directly but installed only as dependencies of the above: -`lxml` (outgoing link extraction, webmention endpoint discovery and source checks; from feedgen) and `python-dateutil` (dates in -reply contexts; from feedgen). `markupsafe` comes with Flask/Jinja2. +`markupsafe` comes with Flask/Jinja2. -### Development dependencies (`requirements-dev.txt`) +### Test and development dependencies -Includes `requirements.txt`, plus `pytest-cov`, `pytest-mock`, `black`, -`flake8`, `mypy` and `gunicorn`. Coding standards are in -[../standards/](../standards/). +- `requirements-test.txt`: `pytest` 9.2 and `pytest-mock` 3.15, used by the + Containerfile test stage (CI). +- `requirements-dev.txt`: runtime and test requirements plus `pytest-cov`, + `black`, `flake8`, `mypy` (not enforced) and `pip-audit`. Coding standards + are in [../standards/](../standards/). ### Deliberately not used @@ -94,7 +98,7 @@ Protocol details: [indieweb-protocols.md](indieweb-protocols.md), ``` - **In CI**: the `test` stage of the `Containerfile` installs the production - requirements plus pytest and pytest-mock and runs the whole suite. The + requirements plus `requirements-test.txt` and runs the whole suite. The image is only built and pushed if it passes. Checklist: [../standards/testing-checklist.md](../standards/testing-checklist.md). diff --git a/docs/design/INDEX.md b/docs/design/INDEX.md index 33d9c55..c702b8d 100644 --- a/docs/design/INDEX.md +++ b/docs/design/INDEX.md @@ -21,7 +21,7 @@ is in `../architecture/`; decisions are in `../decisions/` (ADRs). ## Contents -[v1.0.0](#v100) · [v1.1.1](#v111) · [v1.1.2](#v112) · [v1.2.0](#v120) · [v1.3.0](#v130) · [v1.3.1](#v131) · [v1.4.0](#v140) · [v1.4.1](#v141) · [v1.4.2](#v142) · [v1.5.0](#v150) · [v1.6.0](#v160) · [v2.0.0](#v200) · [v2.0.1](#v201) · [v2.0.2](#v202) · [v2.1.0](#v210) · [v2.2.0](#v220) · [v2.3.0](#v230) · [v2.3.1](#v231) · [v2.3.2](#v232) · [v2.3.3](#v233) · [v2.4.0](#v240) · [v2.5.0](#v250) · [v2.6.0](#v260) · [v2.6.1](#v261) · [v2.6.2](#v262) · [v2.6.3](#v263) · [v2.6.4](#v264) · [v2.6.5](#v265) · [v2.6.6](#v266) · [v2.6.7](#v267) · [v2.6.8](#v268) · [v2.6.9](#v269) · [v2.7.0](#v270) · [v2.7.1](#v271) · [v2.7.2](#v272) · [v2.8.0](#v280) · [v2.8.1](#v281) · [v2.9.0](#v290) · [v2.10.0](#v2100) · [v2.10.1](#v2101) · [v2.10.2](#v2102) · [hotfix](#hotfix) +[v1.0.0](#v100) · [v1.1.1](#v111) · [v1.1.2](#v112) · [v1.2.0](#v120) · [v1.3.0](#v130) · [v1.3.1](#v131) · [v1.4.0](#v140) · [v1.4.1](#v141) · [v1.4.2](#v142) · [v1.5.0](#v150) · [v1.6.0](#v160) · [v2.0.0](#v200) · [v2.0.1](#v201) · [v2.0.2](#v202) · [v2.1.0](#v210) · [v2.2.0](#v220) · [v2.3.0](#v230) · [v2.3.1](#v231) · [v2.3.2](#v232) · [v2.3.3](#v233) · [v2.4.0](#v240) · [v2.5.0](#v250) · [v2.6.0](#v260) · [v2.6.1](#v261) · [v2.6.2](#v262) · [v2.6.3](#v263) · [v2.6.4](#v264) · [v2.6.5](#v265) · [v2.6.6](#v266) · [v2.6.7](#v267) · [v2.6.8](#v268) · [v2.6.9](#v269) · [v2.7.0](#v270) · [v2.7.1](#v271) · [v2.7.2](#v272) · [v2.8.0](#v280) · [v2.8.1](#v281) · [v2.9.0](#v290) · [v2.10.0](#v2100) · [v2.10.1](#v2101) · [v2.10.2](#v2102) · [v2.11.0](#v2110) · [hotfix](#hotfix) ## v1.0.0 @@ -410,6 +410,10 @@ OIDC admin login and built-in IndieAuth server. - [2026-11-03-reply-context-fetch.md](v2.10.2/2026-11-03-reply-context-fetch.md) - Reply contexts fetched again (fetch_source returns three values since v2.7.0) +## v2.11.0 + +- [2026-11-03-maintenance-release.md](v2.11.0/2026-11-03-maintenance-release.md) - Dependency upgrades (Python 3.14), foreign keys, CSRF, fetchers, debug default, shared rate limits + ## hotfix Not a release folder. diff --git a/docs/design/v2.11.0/2026-11-03-maintenance-release.md b/docs/design/v2.11.0/2026-11-03-maintenance-release.md new file mode 100644 index 0000000..becf735 --- /dev/null +++ b/docs/design/v2.11.0/2026-11-03-maintenance-release.md @@ -0,0 +1,60 @@ +# v2.11.0: Maintenance release + +Dependency upgrades and the five High items in the backlog (all re-checked +against the code first and still present). + +## Dependencies + +`pip-audit` on the v2.10.2 set found 97 published vulnerabilities in eight +packages (Pillow 64, PyJWT 22, gunicorn 4, Flask 2, Markdown 2, +python-dotenv 2, pytest 2, pillow-heif 1). All direct dependencies were moved +to their current minor series (`requirements.txt`); the new set has no known +vulnerabilities. Python moved from 3.11 to 3.14 (`python:3.14-slim`); the +whole suite passes with `-W error::DeprecationWarning`. + +- `requirements-test.txt` (pytest, pytest-mock) is installed by the + Containerfile test stage; pytest left the runtime requirements. +- `lxml` and `python-dateutil` are pinned (used directly, previously only + transitive). `uv` is pinned to 0.9.28 in the Containerfile. +- `datetime.utcnow()` (34 calls in `starpunk/`, more in tests) became + `datetime.now(timezone.utc).replace(tzinfo=None)`: same naive-UTC values. + sqlite3's default datetime adapters, deprecated in 3.12, are registered + explicitly in `starpunk/database/__init__.py` with the same output. + +## High backlog items + +1. **Foreign keys.** `PRAGMA foreign_keys=ON` on every pooled connection + (`database/pool.py`); all deletes of notes, media and WebSub subscriptions + go through the pool. Every foreign key is `CASCADE` or `SET NULL`, so no + delete can be blocked. Migration 018 removes orphans (production: three + `note_tags` rows from hard-deleted notes). Crosspost rows still have no + foreign key on purpose. +2. **CSRF.** `starpunk/csrf.py`: token = HMAC-SHA256(SECRET_KEY, + "csrf:" + starpunk_session cookie); `before_request` checks the + `csrf_token` field or `X-CSRF-Token` header on every admin POST, + `auth.logout` and `indieauth.approve`; 400 otherwise. All 12 POST forms + carry it. `CSRF_ENABLED` (default true); test apps turn it off unless they + set it. +3. **Fetchers.** `indieauth.fetch_client_redirect_uris` and + `syndication._fetch_public` use `webmention_ssrf.safe_get` (per-hop + checks, DNS pinning, hard deadline). `safe_get` gained `truncate` for + reads that stop at the size limit. +4. **Debug default.** `FLASK_DEBUG` defaults to 0 and `FLASK_ENV` to + production; the runtime image sets both. +5. **Shared rate limits.** `SharedRateLimiter` keeps sliding-window counters + in `rate_limits` (migration 019): current window plus the previous one + weighted by overlap. Used by `/webmention` (per IP and per source domain) + and `/websub`; old windows are pruned every 200 calls. + +## Housekeeping + +`compose.yaml` no longer pins 0.6.0 or sets `VERSION`/`ENVIRONMENT`; the +unused root `dev_auth.py` and the committed `test.ini` were removed. + +## Tests + +New: `tests/test_csrf.py`, `tests/test_foreign_keys.py`, +`tests/test_shared_rate_limit.py`, IndieAuth client-metadata tests through the +fetcher (including a redirect to a private address). Crosspost tests route +link-card fetches through the fetcher's transport. Full suite: 1264 passed +on Python 3.14. diff --git a/docs/projectplan/BACKLOG.md b/docs/projectplan/BACKLOG.md index 9e28b30..31bda58 100644 --- a/docs/projectplan/BACKLOG.md +++ b/docs/projectplan/BACKLOG.md @@ -1,9 +1,9 @@ # StarPunk Backlog -**Last Updated**: 2026-11-03 (current release: v2.10.1) +**Last Updated**: 2026-11-03 (current release: v2.11.0) These are the open features and bugs, sorted by priority. Each item was checked against the code -at v2.10.1. Work that has shipped is listed per release in `vX.Y.Z/RELEASE.md` and in +at v2.11.0. Work that has shipped is listed per release in `vX.Y.Z/RELEASE.md` and in `CHANGELOG.md`. ## Priority Levels @@ -23,30 +23,7 @@ at v2.10.1. Work that has shipped is listed per release in `vX.Y.Z/RELEASE.md` a ## High -### SQLite foreign keys are not enforced (ON DELETE CASCADE is inert) -- **Description**: No connection runs `PRAGMA foreign_keys = ON`, so the `ON DELETE CASCADE` clauses in the schema do nothing. When a note is deleted from the admin (a hard delete), its `note_tags`, `note_media`, `syndications` and webmention rows stay behind. `delete_media()` relies on the cascade to remove `media_variants` and `note_media` rows, so those are left orphaned too. Before turning the pragma on, check for existing orphan rows and add a migration that cleans them up. Each cascade also needs tests. -- **Where**: `starpunk/database/` (connection setup/pool), `starpunk/notes.py` (`delete_note`, hard delete), `starpunk/media.py` (`delete_media`), `migrations/007`–`010`, `016` -- **Source**: v2.10.x review; confirmed in code (there is no `foreign_keys` pragma anywhere in `starpunk/`) - -### No CSRF tokens on admin and consent forms -- **Description**: Admin forms (create/edit/delete notes, webmention moderation, token revocation) and the IndieAuth consent form rely only on the `SameSite=Lax` session cookie. Lax blocks cross-site POSTs in current browsers, but a per-session CSRF token is the standard defence and covers older clients and same-site subdomains. -- **Where**: `starpunk/routes/admin.py`, `starpunk/routes/indieauth.py`, `templates/admin/*` -- **Source**: Documentation refresh 2026-11-03 (architecture review) - -### Two outbound fetches don't use the hardened fetcher -- **Description**: IndieAuth client-metadata discovery (`indieauth.fetch_client_redirect_uris`) checks only the first URL, then uses `httpx.get(follow_redirects=True)`: redirect hops aren't checked and DNS isn't pinned. Crosspost link cards and Threads image fetches (`syndication._fetch_public`) check each hop but don't pin DNS, and their 10 s timeout is per read. Move both to `webmention_ssrf.safe_get`, as Webmention, WebSub and Micropub photo downloads already do. -- **Where**: `starpunk/indieauth.py`, `starpunk/syndication.py` -- **Source**: Documentation refresh 2026-11-03 - -### `FLASK_DEBUG` defaults to on -- **Description**: `config.py` reads `FLASK_DEBUG` with default `"1"` (and `FLASK_ENV` defaults to development); the runtime image sets neither. Production is safe only because the deployment sets `FLASK_DEBUG=0`. Default to off, or set it in the `Containerfile` runtime stage. -- **Where**: `starpunk/config.py`, `Containerfile` -- **Source**: Documentation refresh 2026-11-03 (production verified: DEBUG is False there) - -### Rate limiters are per process -- **Description**: `RateLimiter` keeps its counters in memory, and gunicorn runs 4 workers. The WebSub hub limit of 30/min is therefore really 4×30/min, and the webmention endpoint limit is multiplied the same way. Consider a counter stored in SQLite that all workers share. -- **Where**: `starpunk/rate_limiter.py`, `starpunk/routes/websub.py`, `starpunk/routes/webmention.py`, `Containerfile` (`--workers 4`) -- **Source**: ADR-071 (Consequences: rate limiting bounds subscriber-chosen requests); v2.10.1 review +None open. --- @@ -92,20 +69,15 @@ at v2.10.1. Work that has shipped is listed per release in `vX.Y.Z/RELEASE.md` a - **Where**: `starpunk/webmention_processor.py`, `starpunk/webmention_worker.py` - **Source**: v1.6.0 design reconstruction (docs/design/v1.6.0/background-worker.md) -### Webmention rate limiter never cleaned up -- **Description**: `cleanup_rate_limiters()` is never called, so per-IP and per-domain keys accumulate for the life of each process. -- **Where**: `starpunk/rate_limiter.py`, `starpunk/routes/webmention.py` -- **Source**: v1.6.0 design reconstruction - ### Atom and JSON Feed not linked from pages - **Description**: `base.html` advertises only `/feed` (as RSS) and OPML with `rel=alternate`; add `` for `/feed.atom` and `/feed.json` so readers can pick a format. - **Where**: `templates/base.html` - **Source**: Documentation refresh 2026-11-03 -### Housekeeping in the repository -- **Description**: `compose.yaml` is stale (`image: starpunk:0.6.0`, `VERSION=0.6.0` would override the real version, unused `ENVIRONMENT`); the root `dev_auth.py` duplicates `starpunk/dev_auth.py` and is unused; `LICENSE` says "Copyright (c) 2024 [Your Name]"; `test.ini` contains a private host name; `lxml` and `python-dateutil` are used directly but only installed as dependencies of other packages, while `pytest` is pinned in the production `requirements.txt`. -- **Where**: `compose.yaml`, `dev_auth.py`, `LICENSE`, `test.ini`, `requirements*.txt` -- **Source**: Documentation refresh 2026-11-03 +### LICENSE placeholder +- **Description**: `LICENSE` still reads "Copyright (c) 2024 [Your Name]". Fill in the holder and year. +- **Where**: `LICENSE` +- **Source**: Documentation refresh 2026-11-03 (rest of that housekeeping item done in v2.11.0) ### Backfeed responses to crossposts - **Description**: Idea: bring likes, boosts and replies to StarPunk's own crossposts back to the note as responses. Two options: poll GoToSocial (the token would need a `read` scope), or rely on Bridgy, which sends webmentions. Since v2.8.0, StarPunk ignores webmentions *from* a note's own crossposts, so that rule would need to change to work with Bridgy. @@ -213,6 +185,8 @@ at v2.10.1. Work that has shipped is listed per release in `vX.Y.Z/RELEASE.md` a These items were removed from the backlog because they have shipped: +- v2.11.0: SQLite foreign keys enforced (orphans cleaned up by migration 018); CSRF tokens on admin, logout and consent forms; IndieAuth client and crosspost link-card fetches moved onto the SSRF-safe fetcher; `FLASK_DEBUG`/`FLASK_ENV` default to off/production; rate limits shared by all processes (and old windows pruned, so the webmention limiter no longer grows); dependencies upgraded (Python 3.14, no known vulnerabilities); stale `compose.yaml`, duplicate `dev_auth.py` and `test.ini` removed or fixed; `lxml`/`python-dateutil` pinned, `pytest` out of the runtime requirements + - POSSE: GoToSocial (v2.1.0), Bluesky (v2.2.0), Threads (v2.6.0) - Webmentions: receive, send, display and moderation (v1.6.0); fixed and made spec-compliant in v2.6.8–v2.7.2 - Reply contexts, replies, likes, reposts and bookmarks (v2.4.0, v2.5.0, v2.9.0) diff --git a/docs/projectplan/INDEX.md b/docs/projectplan/INDEX.md index 40abd19..5d41228 100644 --- a/docs/projectplan/INDEX.md +++ b/docs/projectplan/INDEX.md @@ -4,7 +4,7 @@ Future work and per-release plans. See `../DOCUMENTATION.md` (PROJECTPLAN). ## Backlog -- **[BACKLOG.md](BACKLOG.md)** - Open features and bugs by priority (updated for v2.10.1) +- **[BACKLOG.md](BACKLOG.md)** - Open features and bugs by priority (updated for v2.11.0) ## Releases @@ -47,3 +47,4 @@ Future work and per-release plans. See `../DOCUMENTATION.md` (PROJECTPLAN). - **[v2.10.0](v2.10.0/RELEASE.md)** - WebSub (Released 2026-11-03) - **[v2.10.1](v2.10.1/RELEASE.md)** - WebSub hardening (Released 2026-11-03) - **[v2.10.2](v2.10.2/RELEASE.md)** - Reply context fetch fix (Released 2026-11-03) +- **[v2.11.0](v2.11.0/RELEASE.md)** - Maintenance: dependency upgrades, High backlog items (Released 2026-11-03) diff --git a/docs/projectplan/v2.11.0/RELEASE.md b/docs/projectplan/v2.11.0/RELEASE.md new file mode 100644 index 0000000..b8fc913 --- /dev/null +++ b/docs/projectplan/v2.11.0/RELEASE.md @@ -0,0 +1,28 @@ +# v2.11.0 + +- **Date**: 2026-11-03 +- **Status**: Released +- **Focus**: Maintenance: dependency upgrades and the High backlog items + +## Security + +- All dependencies upgraded (97 known vulnerabilities before, none after); Python 3.14 +- CSRF tokens on admin forms, logout and the IndieAuth consent +- IndieAuth client and crosspost link-card fetches use the SSRF-safe fetcher +- Debug off and production environment by default + +## Changes + +- Rate limits shared by all server processes +- `requirements-test.txt`; pytest out of the production image; `lxml` and `python-dateutil` pinned; `uv` pinned + +## Bugs fixed + +- Foreign keys enforced, so deletes cascade; orphan rows removed (migration 018) +- Deprecated `datetime.utcnow()` and sqlite3 adapters replaced +- Stale `compose.yaml`, duplicate `dev_auth.py` and `test.ini` cleaned up + +## Related + +- [CHANGELOG](../../../CHANGELOG.md) +- [Design note](../../design/v2.11.0/2026-11-03-maintenance-release.md) diff --git a/docs/standards/development-setup.md b/docs/standards/development-setup.md index 42f3df3..c49a324 100644 --- a/docs/standards/development-setup.md +++ b/docs/standards/development-setup.md @@ -9,7 +9,7 @@ deployment is in [architecture/deployment.md](../architecture/deployment.md). ## Requirements -- Python 3.11 (the container image uses `python:3.11-slim`) +- Python 3.14 (the container image uses `python:3.14-slim`) - [uv](https://docs.astral.sh/uv/) for the virtual environment and packages ([ADR-006](../decisions/ADR-006-python-virtual-environment-uv.md)) - git - Optional: Docker or Podman, to build the image and run the CI test stage locally @@ -23,7 +23,7 @@ pytest-mock, black, flake8, mypy). ```bash git clone https://git.thesatelliteoflove.com/phil/StarPunk.git cd StarPunk -uv venv --python 3.11 # creates .venv/ (gitignored) +uv venv --python 3.14 # creates .venv/ (gitignored) uv pip install -r requirements-dev.txt ``` diff --git a/docs/standards/git-branching-strategy.md b/docs/standards/git-branching-strategy.md index 0c6fb8e..48378ac 100644 --- a/docs/standards/git-branching-strategy.md +++ b/docs/standards/git-branching-strategy.md @@ -63,7 +63,7 @@ cd ~/Documents/projects/StarPunk # main checkout, on main git pull git worktree add ../StarPunk-worktrees/v2.10.1 -b fix/v2.10.1-websub-hardening main cd ../StarPunk-worktrees/v2.10.1 -uv venv --python 3.11 && uv pip install -r requirements-dev.txt +uv venv --python 3.14 && uv pip install -r requirements-dev.txt ``` ### 2. Make the change diff --git a/docs/standards/python-coding-standards.md b/docs/standards/python-coding-standards.md index c94a9e2..b9fc33a 100644 --- a/docs/standards/python-coding-standards.md +++ b/docs/standards/python-coding-standards.md @@ -10,7 +10,7 @@ out." Patterns specific to small helper functions are in ## Base style -- [PEP 8](https://peps.python.org/pep-0008/) naming and layout, Python 3.11. +- [PEP 8](https://peps.python.org/pep-0008/) naming and layout, Python 3.14. Use `datetime.now(timezone.utc)`, not the deprecated `datetime.utcnow()`. - **Match the file you are editing.** 4-space indents everywhere; quote style varies between modules (newer code uses double quotes); keep lines to roughly 100 characters. diff --git a/migrations/018_foreign_key_cleanup.sql b/migrations/018_foreign_key_cleanup.sql new file mode 100644 index 0000000..6bf448c --- /dev/null +++ b/migrations/018_foreign_key_cleanup.sql @@ -0,0 +1,12 @@ +-- Migration 018: Remove rows orphaned while foreign keys weren't enforced (v2.11.0) +-- From v2.11.0 every pooled connection runs PRAGMA foreign_keys=ON, so these +-- cascades happen on delete. This removes what earlier deletes left behind. + +DELETE FROM note_tags WHERE note_id NOT IN (SELECT id FROM notes); +DELETE FROM note_tags WHERE tag_id NOT IN (SELECT id FROM tags); +DELETE FROM note_media WHERE note_id NOT IN (SELECT id FROM notes); +DELETE FROM note_media WHERE media_id NOT IN (SELECT id FROM media); +DELETE FROM media_variants WHERE media_id NOT IN (SELECT id FROM media); +DELETE FROM webmention_queue WHERE source_note_id IS NOT NULL AND source_note_id NOT IN (SELECT id FROM notes); +UPDATE webmentions SET target_note_id = NULL WHERE target_note_id IS NOT NULL AND target_note_id NOT IN (SELECT id FROM notes); +DELETE FROM websub_deliveries WHERE subscription_id IS NOT NULL AND subscription_id NOT IN (SELECT id FROM websub_subscriptions); diff --git a/migrations/019_shared_rate_limits.sql b/migrations/019_shared_rate_limits.sql new file mode 100644 index 0000000..7457a09 --- /dev/null +++ b/migrations/019_shared_rate_limits.sql @@ -0,0 +1,9 @@ +-- Migration 019: Rate limit counters shared by all server processes (v2.11.0) +-- One row per key and time window; see starpunk/rate_limiter.py SharedRateLimiter. + +CREATE TABLE IF NOT EXISTS rate_limits ( + key TEXT NOT NULL, + window INTEGER NOT NULL, + count INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (key, window) +); diff --git a/requirements-dev.txt b/requirements-dev.txt index a8a6246..a832530 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,17 +1,9 @@ -# StarPunk Development Dependencies -# Includes code quality and testing tools - -# Include production dependencies +# StarPunk development dependencies -r requirements.txt +-r requirements-test.txt -# Testing -pytest-cov>=5.0.0 # Test coverage reporting -pytest-mock>=3.12.0 # Mocking for tests - -# Code Quality -black>=24.0.0 # Code formatting -flake8>=7.0.0 # Linting -mypy>=1.8.0 # Type checking - -# WSGI Server (for production-like testing) -gunicorn>=21.2.0 +pytest-cov==7.* # Coverage reports +black==26.* # Formatting (not enforced, see docs/standards) +flake8==7.* # Linting (not enforced) +mypy==1.* # Type checking (not enforced) +pip-audit==2.* # Known-vulnerability check of the dependency set diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..6f61a92 --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,3 @@ +# Test dependencies: used by the Containerfile test stage and CI (v2.11.0) +pytest==9.2.* +pytest-mock==3.15.* diff --git a/requirements.txt b/requirements.txt index a97e85d..b7650ad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,42 +1,27 @@ -# StarPunk Production Dependencies -# Python 3.11+ required +# StarPunk runtime dependencies (v2.11.0) +# Pinned to a minor series; see docs/architecture/technology-stack.md. -# Web Framework -Flask==3.0.* +# Web framework and server +Flask==3.1.* +gunicorn==26.2.* -# WSGI Server (Production) -gunicorn==21.2.* - -# Content Processing -markdown==3.5.* - -# Feed Generation +# Content +markdown==3.11.* +nh3==0.3.* +beautifulsoup4==4.15.* +lxml==6.1.* +mf2py==2.0.* +python-dateutil==4.0.* feedgen==1.0.* -# HTTP Client (for IndieAuth) -httpx==0.27.* +# HTTP and auth +httpx==0.28.* +PyJWT[crypto]==2.15.* +python-dotenv==1.2.* -# Configuration Management -python-dotenv==1.0.* +# Images +Pillow==13.0.* +pillow-heif==1.8.* -# HTML Parsing (for IndieAuth endpoint discovery) -beautifulsoup4==4.12.* - -# Microformats2 Parsing (v1.2.0) -mf2py==2.0.* - -# Testing Framework -pytest==8.0.* - -# System Monitoring (v1.1.2) -psutil==5.9.* - -# Image Processing (v1.2.0, updated v1.4.2 for HEIC support) -Pillow==10.1.* - -# HEIC/HEIF Support (v1.4.2 - iPhone photos) -pillow-heif==0.18.* - -# HTML Sanitization (v1.6.0 - Webmention content) -nh3==0.2.* -PyJWT[crypto]==2.10.* +# Monitoring +psutil==7.2.* diff --git a/starpunk/__init__.py b/starpunk/__init__.py index 622a7df..e1d59aa 100644 --- a/starpunk/__init__.py +++ b/starpunk/__init__.py @@ -309,6 +309,10 @@ def create_app(config=None): response.headers.add('Link', '; rel="webmention"') return response + # CSRF tokens on admin and consent forms (v2.11.0) + from starpunk import csrf + csrf.init_app(app) + # WebSub discovery: Link rel=hub and rel=self on topics (v2.10.0) from starpunk.websub import add_discovery_headers, template_links app.after_request(add_discovery_headers) @@ -470,5 +474,5 @@ def create_app(config=None): # Package version (Semantic Versioning 2.0.0) # See docs/standards/versioning-strategy.md for details -__version__ = "2.10.2" -__version_info__ = (2, 10, 2) +__version__ = "2.11.0" +__version_info__ = (2, 11, 0) diff --git a/starpunk/auth.py b/starpunk/auth.py index 8093311..6c2afc1 100644 --- a/starpunk/auth.py +++ b/starpunk/auth.py @@ -32,7 +32,7 @@ import base64 import hashlib import secrets import time -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from functools import wraps from typing import Any, Dict, Optional, Tuple from urllib.parse import urlencode @@ -223,7 +223,7 @@ def initiate_login(next_url: Optional[str] = None) -> str: nonce, code_verifier, _safe_next_url(next_url), - datetime.utcnow() + timedelta(minutes=LOGIN_STATE_MINUTES), + datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(minutes=LOGIN_STATE_MINUTES), ), ) db.commit() @@ -361,7 +361,7 @@ def create_session(me: str) -> str: # Calculate expiry (use configured session lifetime or default to 30 days) session_lifetime = current_app.config.get("SESSION_LIFETIME", 30) - expires_at = datetime.utcnow() + timedelta(days=session_lifetime) + expires_at = datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(days=session_lifetime) current_app.logger.debug(f"Auth: Session expiry: {expires_at} ({session_lifetime} days)") diff --git a/starpunk/author_discovery.py b/starpunk/author_discovery.py index b3e1c64..c489012 100644 --- a/starpunk/author_discovery.py +++ b/starpunk/author_discovery.py @@ -22,7 +22,7 @@ Fallback Behavior (per Q14): import json import logging -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from typing import Dict, Optional from urllib.parse import urlparse @@ -236,7 +236,7 @@ def get_author_profile(me_url: str, refresh: bool = False) -> Dict: if cached: # Check if cache is still valid cached_until = datetime.fromisoformat(cached['cached_until']) - if datetime.utcnow() < cached_until: + if datetime.now(timezone.utc).replace(tzinfo=None) < cached_until: current_app.logger.debug(f"Using cached author profile for {me_url}") # Parse rel_me_links from JSON @@ -326,7 +326,7 @@ def save_author_profile(me_url: str, profile: Dict) -> None: db = get_db(current_app) # Calculate cache expiry (24 hours from now) - cached_until = datetime.utcnow() + timedelta(hours=CACHE_TTL_HOURS) + cached_until = datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(hours=CACHE_TTL_HOURS) # Convert rel_me_links to JSON (per Q17) rel_me_json = json.dumps(profile.get('rel_me_links', [])) diff --git a/starpunk/config.py b/starpunk/config.py index bee665f..33f6c1d 100644 --- a/starpunk/config.py +++ b/starpunk/config.py @@ -71,8 +71,10 @@ def load_config(app, config_override=None): app.config["DATABASE_PATH"] = Path(os.getenv("DATABASE_PATH", "./data/starpunk.db")) # Flask environment - app.config["ENV"] = os.getenv("FLASK_ENV", "development") - app.config["DEBUG"] = os.getenv("FLASK_DEBUG", "1") == "1" + # Safe defaults (v2.11.0): production, debug off. `flask run --debug` + # sets FLASK_DEBUG=1 for local development. + app.config["ENV"] = os.getenv("FLASK_ENV", "production") + app.config["DEBUG"] = os.getenv("FLASK_DEBUG", "0") == "1" # Logging app.config["LOG_LEVEL"] = os.getenv("LOG_LEVEL", "INFO") @@ -153,8 +155,14 @@ def load_config(app, config_override=None): app.config["SYNDICATION_POLL_INTERVAL"] = int(os.getenv("SYNDICATION_POLL_INTERVAL", "10")) # Apply overrides if provided + # CSRF protection for admin and consent forms (v2.11.0); on by default. + # Tests turn it off unless they set CSRF_ENABLED themselves. + app.config["CSRF_ENABLED"] = os.getenv("CSRF_ENABLED", "true").lower() == "true" + if config_override: app.config.update(config_override) + if app.config.get("TESTING") and "CSRF_ENABLED" not in config_override: + app.config["CSRF_ENABLED"] = False # Normalize SITE_URL trailing slash (in case override provided URL without slash) if "SITE_URL" in app.config: diff --git a/starpunk/csrf.py b/starpunk/csrf.py new file mode 100644 index 0000000..ee92b87 --- /dev/null +++ b/starpunk/csrf.py @@ -0,0 +1,56 @@ +""" +CSRF protection for the admin and consent forms (v2.11.0) + +The token is an HMAC of the login session cookie under the app secret: no +storage, tied to the session, and different after every login. Forms carry +it as a hidden `csrf_token` field (template global `csrf_token()`); POSTs to +protected endpoints without the right token get 400. + +Protected: every admin POST, logout, and the IndieAuth consent ("approve"). +Endpoints used by apps with bearer tokens (Micropub, token, revoke, +Webmention, WebSub) are not: they have no cookie-based authority. +""" + +import hashlib +import hmac + +from flask import abort, current_app, request + +SESSION_COOKIE = "starpunk_session" +FIELD = "csrf_token" +HEADER = "X-CSRF-Token" +PROTECTED_ENDPOINTS = {"auth.logout", "indieauth.approve"} + + +def csrf_token() -> str: + """The CSRF token for the current login session ('' when not logged in)""" + session = request.cookies.get(SESSION_COOKIE) + if not session: + return "" + key = str(current_app.config["SECRET_KEY"]).encode("utf-8") + return hmac.new(key, b"csrf:" + session.encode("utf-8"), hashlib.sha256).hexdigest() + + +def _protected() -> bool: + return request.blueprint == "admin" or request.endpoint in PROTECTED_ENDPOINTS + + +def check_csrf(): + """before_request: refuse state-changing requests without a valid token""" + if request.method in ("GET", "HEAD", "OPTIONS") or not current_app.config.get("CSRF_ENABLED", True): + return None + if not _protected(): + return None + expected = csrf_token() + if not expected: + return None # not logged in: the route's own auth check refuses it + sent = request.form.get(FIELD) or request.headers.get(HEADER) or "" + if not hmac.compare_digest(sent, expected): + current_app.logger.warning(f"CSRF token missing or wrong for {request.endpoint}") + abort(400, description="The form expired or came from another site. Reload the page and try again.") + return None + + +def init_app(app) -> None: + app.before_request(check_csrf) + app.add_template_global(csrf_token, "csrf_token") diff --git a/starpunk/database/__init__.py b/starpunk/database/__init__.py index 29f48bd..0eb8835 100644 --- a/starpunk/database/__init__.py +++ b/starpunk/database/__init__.py @@ -13,4 +13,12 @@ from starpunk.database.init import init_db from starpunk.database.pool import init_pool, get_db, get_pool_stats from starpunk.database.schema import INITIAL_SCHEMA_SQL +# Explicit datetime adapters (v2.11.0). Python 3.12 deprecated sqlite3's +# built-in ones; these store exactly what they did ("YYYY-MM-DD HH:MM:SS[.ffffff]"). +import datetime as _datetime +import sqlite3 as _sqlite3 + +_sqlite3.register_adapter(_datetime.datetime, lambda value: value.isoformat(" ")) +_sqlite3.register_adapter(_datetime.date, lambda value: value.isoformat()) + __all__ = ['init_db', 'init_pool', 'get_db', 'get_pool_stats', 'INITIAL_SCHEMA_SQL'] diff --git a/starpunk/database/pool.py b/starpunk/database/pool.py index d998fdc..47e865e 100644 --- a/starpunk/database/pool.py +++ b/starpunk/database/pool.py @@ -68,6 +68,9 @@ class ConnectionPool: # Enable WAL mode for better concurrency conn.execute("PRAGMA journal_mode=WAL") + # Enforce foreign keys, so ON DELETE CASCADE / SET NULL actually run + # (SQLite leaves them off per connection; v2.11.0) + conn.execute("PRAGMA foreign_keys=ON") self._stats['connections_created'] += 1 diff --git a/starpunk/feeds/opml.py b/starpunk/feeds/opml.py index a41a7a2..0af0476 100644 --- a/starpunk/feeds/opml.py +++ b/starpunk/feeds/opml.py @@ -14,7 +14,7 @@ Per v1.1.2 Phase 3: Specification: http://opml.org/spec2.opml """ -from datetime import datetime +from datetime import datetime, timezone from xml.sax.saxutils import escape @@ -57,7 +57,7 @@ def generate_opml(site_url: str, site_name: str) -> str: safe_site_name = escape(site_name) # RFC 822 date format (required by OPML spec) - creation_date = datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT') + creation_date = datetime.now(timezone.utc).replace(tzinfo=None).strftime('%a, %d %b %Y %H:%M:%S GMT') # Build OPML document opml_lines = [ diff --git a/starpunk/indieauth.py b/starpunk/indieauth.py index b8f10b9..ed1fa83 100644 --- a/starpunk/indieauth.py +++ b/starpunk/indieauth.py @@ -13,7 +13,7 @@ current IndieAuth spec: https://indieauth.spec.indieweb.org/ import base64 import hashlib import secrets -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, Optional from urllib.parse import urljoin, urlparse @@ -83,41 +83,42 @@ def fetch_client_redirect_uris(client_id: str) -> List[str]: Supports a JSON client metadata document ("redirect_uris") and the older HTML/Link header form (rel="redirect_uri"). Returns [] on any failure. """ - from starpunk.webmention_ssrf import SSRFBlockedError, validate_url_for_fetch + import json + from starpunk.webmention_ssrf import FetchTooLargeError, SSRFBlockedError, safe_get, user_agent + # Every redirect hop is checked and the connection pinned to the checked + # address, with a hard time limit (v2.11.0; DEV_MODE allows localhost) try: - if not current_app.config.get("DEV_MODE"): - validate_url_for_fetch(client_id) - response = httpx.get( + result = safe_get( client_id, - timeout=5.0, - follow_redirects=True, - headers={"Accept": "application/json, text/html;q=0.9"}, + headers={"Accept": "application/json, text/html;q=0.9", + "User-Agent": user_agent(current_app.config.get("SITE_URL", ""))}, + max_bytes=500_000, timeout=5.0, truncate=True, ) - except (httpx.HTTPError, SSRFBlockedError, ValueError) as e: + except (httpx.HTTPError, SSRFBlockedError, ValueError, FetchTooLargeError) as e: current_app.logger.info(f"IndieAuth: could not fetch client {client_id}: {e}") return [] - if response.status_code != 200: + if result.status_code != 200: return [] - base = str(response.url) + base = result.url uris: List[str] = [] - if "json" in response.headers.get("Content-Type", ""): + if "json" in result.headers.get("Content-Type", ""): try: - data = response.json() + data = json.loads(result.text) except ValueError: return [] if isinstance(data, dict) and isinstance(data.get("redirect_uris"), list): uris.extend(u for u in data["redirect_uris"] if isinstance(u, str)) return uris - link = response.links.get("redirect_uri") + link = httpx.Response(200, headers=result.headers).links.get("redirect_uri") if link and link.get("url"): uris.append(urljoin(base, link["url"])) - soup = BeautifulSoup(response.text[:500_000], "html.parser") + soup = BeautifulSoup(result.text, "html.parser") for tag in soup.find_all(["link", "a"], href=True): if "redirect_uri" in (tag.get("rel") or []): uris.append(urljoin(base, tag["href"])) @@ -203,7 +204,7 @@ def create_authorization_code( redirect_uri, scope, code_challenge, - datetime.utcnow() + timedelta(minutes=CODE_LIFETIME_MINUTES), + datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(minutes=CODE_LIFETIME_MINUTES), ), ) db.commit() @@ -258,7 +259,7 @@ def create_access_token(client_id: str, scope: str) -> Dict[str, Any]: """Issue an access token (stored hashed); returns the token response""" token = secrets.token_urlsafe(32) lifetime_days = current_app.config.get("INDIEAUTH_TOKEN_LIFETIME_DAYS", 90) - expires_at = datetime.utcnow() + timedelta(days=lifetime_days) + expires_at = datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(days=lifetime_days) db = get_db(current_app) db.execute( diff --git a/starpunk/models.py b/starpunk/models.py index ef10df2..84a5c06 100644 --- a/starpunk/models.py +++ b/starpunk/models.py @@ -8,7 +8,7 @@ state. All models are immutable and use dataclasses for clean structure. # Standard library imports from dataclasses import dataclass, field, replace -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any, Optional import re @@ -698,7 +698,7 @@ class Session: >>> session.is_expired False """ - return datetime.utcnow() >= self.expires_at + return datetime.now(timezone.utc).replace(tzinfo=None) >= self.expires_at @property def is_active(self) -> bool: @@ -726,7 +726,7 @@ class Session: >>> session.age datetime.timedelta(days=2, seconds=3600) """ - return datetime.utcnow() - self.created_at + return datetime.now(timezone.utc).replace(tzinfo=None) - self.created_at @property def time_until_expiry(self) -> timedelta: @@ -740,7 +740,7 @@ class Session: >>> session.time_until_expiry datetime.timedelta(days=28) """ - return self.expires_at - datetime.utcnow() + return self.expires_at - datetime.now(timezone.utc).replace(tzinfo=None) def is_valid(self) -> bool: """ @@ -787,7 +787,7 @@ class Session: >>> updated.last_used_at datetime.datetime(2024, 11, 18, 15, 30, 0) """ - return replace(self, last_used_at=datetime.utcnow()) + return replace(self, last_used_at=datetime.now(timezone.utc).replace(tzinfo=None)) def to_dict(self) -> dict[str, Any]: """ @@ -863,7 +863,7 @@ class Token: me: str client_id: Optional[str] = None scope: Optional[str] = None - created_at: datetime = field(default_factory=lambda: datetime.utcnow()) + created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc).replace(tzinfo=None)) expires_at: Optional[datetime] = None @classmethod @@ -926,7 +926,7 @@ class Token: # No expiry set, token never expires return False - return datetime.utcnow() >= self.expires_at + return datetime.now(timezone.utc).replace(tzinfo=None) >= self.expires_at @property def is_active(self) -> bool: @@ -1145,7 +1145,7 @@ class AuthState: >>> auth_state.is_expired False """ - return datetime.utcnow() >= self.expires_at + return datetime.now(timezone.utc).replace(tzinfo=None) >= self.expires_at @property def is_active(self) -> bool: @@ -1169,7 +1169,7 @@ class AuthState: >>> auth_state.age datetime.timedelta(seconds=120) """ - return datetime.utcnow() - self.created_at + return datetime.now(timezone.utc).replace(tzinfo=None) - self.created_at def is_valid(self) -> bool: """ diff --git a/starpunk/monitoring/metrics.py b/starpunk/monitoring/metrics.py index 62c5d69..7351d9c 100644 --- a/starpunk/monitoring/metrics.py +++ b/starpunk/monitoring/metrics.py @@ -24,7 +24,7 @@ import random import time from collections import deque from dataclasses import dataclass, field, asdict -from datetime import datetime +from datetime import datetime, timezone from threading import Lock from typing import Any, Deque, Dict, List, Literal, Optional, Union @@ -159,7 +159,7 @@ class MetricsBuffer: metric = Metric( operation_type=operation_type, operation_name=operation_name, - timestamp=datetime.utcnow().isoformat() + "Z", + timestamp=datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + "Z", duration_ms=duration_ms, process_id=self._process_id, metadata=metadata or {} diff --git a/starpunk/notes.py b/starpunk/notes.py index daf4e42..b91532d 100644 --- a/starpunk/notes.py +++ b/starpunk/notes.py @@ -20,7 +20,7 @@ Exceptions: """ # Standard library imports -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path from typing import Optional @@ -226,7 +226,7 @@ def create_note( # 2. SETUP if created_at is None: - created_at = datetime.utcnow() + created_at = datetime.now(timezone.utc).replace(tzinfo=None) updated_at = created_at # Same as created_at for new notes @@ -735,7 +735,7 @@ def update_note( ) # 3. SETUP - updated_at = datetime.utcnow() + updated_at = datetime.now(timezone.utc).replace(tzinfo=None) data_dir = Path(current_app.config["DATA_PATH"]) note_path = data_dir / existing_note.file_path @@ -1052,7 +1052,7 @@ def delete_note( if soft: # SOFT DELETE: Mark as deleted in database - deleted_at = datetime.utcnow() + deleted_at = datetime.now(timezone.utc).replace(tzinfo=None) try: db.execute( diff --git a/starpunk/rate_limiter.py b/starpunk/rate_limiter.py index 5a69f74..08cb05c 100644 --- a/starpunk/rate_limiter.py +++ b/starpunk/rate_limiter.py @@ -138,6 +138,79 @@ _ip_limiter: RateLimiter | None = None _domain_limiter: RateLimiter | None = None +class SharedRateLimiter: + """ + Rate limiter whose counts live in SQLite, shared by every gunicorn + process (v2.11.0). In-memory counters let each of the 4 workers allow the + full limit, so the real limit was four times the setting. + + Sliding-window counter: the current window's count plus the previous + window's, weighted by how much of it still overlaps the last + `window_seconds`. Same interface as RateLimiter; needs an app context. + """ + + CLEANUP_EVERY = 200 # calls between removals of old windows + + def __init__(self, limit: int, window_seconds: int, prefix: str): + self.limit = limit + self.window_seconds = window_seconds + self.prefix = prefix + self._calls = 0 + + def _db(self): + from flask import current_app + from starpunk.database import get_db + return get_db(current_app) + + def _windows(self): + now = time.time() + current = int(now // self.window_seconds) + overlap = 1 - (now % self.window_seconds) / self.window_seconds + return now, current, overlap + + def _estimate(self, db, key: str) -> float: + _, current, overlap = self._windows() + counts = dict(db.execute( + "SELECT window, count FROM rate_limits WHERE key = ? AND window IN (?, ?)", + (f"{self.prefix}:{key}", current - 1, current), + ).fetchall()) + return counts.get(current - 1, 0) * overlap + counts.get(current, 0) + + def is_allowed(self, key: str) -> bool: + db = self._db() + _, current, _ = self._windows() + if self._estimate(db, key) >= self.limit: + return False + db.execute( + "INSERT INTO rate_limits (key, window, count) VALUES (?, ?, 1) " + "ON CONFLICT(key, window) DO UPDATE SET count = count + 1", + (f"{self.prefix}:{key}", current), + ) + db.commit() + self._calls += 1 + if self._calls % self.CLEANUP_EVERY == 0: + self.cleanup() + return True + + def get_retry_after(self, key: str) -> int: + now, current, _ = self._windows() + return max(1, int((current + 1) * self.window_seconds - now) + 1) + + def get_request_count(self, key: str) -> int: + return int(self._estimate(self._db(), key)) + + def cleanup(self, max_idle_seconds: int = 300) -> int: + """Remove windows too old to matter (this limiter's keys only)""" + _, current, _ = self._windows() + db = self._db() + cursor = db.execute( + "DELETE FROM rate_limits WHERE key LIKE ? AND window < ?", + (f"{self.prefix}:%", current - 1), + ) + db.commit() + return cursor.rowcount + + def init_rate_limiters(app) -> None: """ Initialize rate limiters with Flask app configuration. @@ -157,8 +230,9 @@ def init_rate_limiters(app) -> None: domain_limit = app.config.get('WEBMENTION_RATE_LIMIT_DOMAIN', 10) window = app.config.get('WEBMENTION_RATE_LIMIT_WINDOW', 60) - _ip_limiter = RateLimiter(ip_limit, window) - _domain_limiter = RateLimiter(domain_limit, window) + # Shared by all server processes (v2.11.0) + _ip_limiter = SharedRateLimiter(ip_limit, window, "webmention-ip") + _domain_limiter = SharedRateLimiter(domain_limit, window, "webmention-domain") app.logger.info( f"Webmention rate limiter initialized: {ip_limit}/min IP, " diff --git a/starpunk/routes/admin.py b/starpunk/routes/admin.py index 00dc8ca..739d0ad 100644 --- a/starpunk/routes/admin.py +++ b/starpunk/routes/admin.py @@ -9,7 +9,7 @@ from flask import Blueprint, flash, g, jsonify, redirect, render_template, reque import logging import os import shutil -from datetime import datetime +from datetime import datetime, timezone logger = logging.getLogger(__name__) @@ -531,7 +531,7 @@ def metrics(): from starpunk.monitoring.business import get_feed_statistics response = { - "timestamp": datetime.utcnow().isoformat() + "Z", + "timestamp": datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + "Z", "process_id": os.getpid(), "database": {}, "performance": {}, diff --git a/starpunk/routes/public.py b/starpunk/routes/public.py index cd3c51c..5cf4726 100644 --- a/starpunk/routes/public.py +++ b/starpunk/routes/public.py @@ -6,7 +6,7 @@ No authentication required for these routes. """ import hashlib -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from flask import Blueprint, abort, render_template, Response, current_app, request, send_from_directory @@ -54,7 +54,7 @@ def _get_cached_notes(): # Get cache duration from config (in seconds) cache_seconds = current_app.config.get("FEED_CACHE_SECONDS", 300) cache_duration = timedelta(seconds=cache_seconds) - now = datetime.utcnow() + now = datetime.now(timezone.utc).replace(tzinfo=None) # The cached list is stale once any public note changed, in any process # (v2.10.0): every change bumps content_version in the database diff --git a/starpunk/routes/websub.py b/starpunk/routes/websub.py index 4f1019a..02472a3 100644 --- a/starpunk/routes/websub.py +++ b/starpunk/routes/websub.py @@ -9,16 +9,15 @@ optional hub.secret and hub.lease_seconds. Answers 202 and verifies later from flask import Blueprint, current_app, request from starpunk.database import get_db -from starpunk.rate_limiter import RateLimiter +from starpunk.rate_limiter import SharedRateLimiter from starpunk import websub bp = Blueprint("websub", __name__) # Subscription requests per client IP per minute -# (per gunicorn process; needs TRUSTED_PROXY=true behind a reverse proxy, -# or every client shares the proxy's address) -_limiter = RateLimiter(limit=30, window_seconds=60) -_requests = 0 +# Shared by all server processes (v2.11.0). Needs TRUSTED_PROXY=true behind a +# reverse proxy, or every client shares the proxy's address. +_limiter = SharedRateLimiter(limit=30, window_seconds=60, prefix="websub") def _text(message: str, status: int): @@ -31,10 +30,6 @@ def hub(): return _text("WebSub is not enabled on this site", 404) from starpunk.routes.webmention import get_client_ip - global _requests - _requests += 1 - if _requests % 500 == 0: - _limiter.cleanup() # forget idle clients so memory doesn't grow if not _limiter.is_allowed(get_client_ip()): return _text("Too many requests", 429) diff --git a/starpunk/slug_utils.py b/starpunk/slug_utils.py index 5bbc95f..d89c9bb 100644 --- a/starpunk/slug_utils.py +++ b/starpunk/slug_utils.py @@ -14,7 +14,7 @@ Per developer Q&A Q8: import re import unicodedata import logging -from datetime import datetime +from datetime import datetime, timezone from typing import Optional, Set logger = logging.getLogger(__name__) @@ -138,7 +138,7 @@ def sanitize_slug(slug: str, allow_timestamp_fallback: bool = False) -> str: # Check if normalization resulted in empty slug if not slug and allow_timestamp_fallback: # Per Q8: Use timestamp-based fallback - timestamp = datetime.utcnow().strftime('%Y%m%d-%H%M%S') + timestamp = datetime.now(timezone.utc).replace(tzinfo=None).strftime('%Y%m%d-%H%M%S') slug = timestamp logger.warning( f"Slug normalization failed for input '{original_slug}' " @@ -268,7 +268,7 @@ def generate_timestamp_slug( '20251216143052-2' """ if created_at is None: - created_at = datetime.utcnow() + created_at = datetime.now(timezone.utc).replace(tzinfo=None) if existing_slugs is None: existing_slugs = set() @@ -340,7 +340,7 @@ def validate_and_sanitize_custom_slug(custom_slug: str, existing_slugs: Set[str] if not sanitized: # This should never happen with allow_timestamp_fallback=True # but handle it just in case - timestamp = datetime.utcnow().strftime('%Y%m%d-%H%M%S') + timestamp = datetime.now(timezone.utc).replace(tzinfo=None).strftime('%Y%m%d-%H%M%S') sanitized = timestamp logger.error( f"Unexpected empty slug after sanitization with fallback. " @@ -361,7 +361,7 @@ def validate_and_sanitize_custom_slug(custom_slug: str, existing_slugs: Set[str] if not validate_slug(sanitized): # This should rarely happen after sanitization # but if it does, use timestamp fallback - timestamp = datetime.utcnow().strftime('%Y%m%d-%H%M%S') + timestamp = datetime.now(timezone.utc).replace(tzinfo=None).strftime('%Y%m%d-%H%M%S') logger.warning( f"Slug '{sanitized}' (from '{custom_slug}') failed validation. " f"Using timestamp fallback: {timestamp}" @@ -375,7 +375,7 @@ def validate_and_sanitize_custom_slug(custom_slug: str, existing_slugs: Set[str] except ValueError as e: # This should rarely happen, but if it does, use timestamp # Per Q8: Never fail Micropub request - timestamp = datetime.utcnow().strftime('%Y%m%d-%H%M%S') + timestamp = datetime.now(timezone.utc).replace(tzinfo=None).strftime('%Y%m%d-%H%M%S') logger.error( f"Could not create unique slug from '{custom_slug}'. " f"Using timestamp: {timestamp}. Error: {e}" diff --git a/starpunk/syndication.py b/starpunk/syndication.py index 8f0afd8..5ab9743 100644 --- a/starpunk/syndication.py +++ b/starpunk/syndication.py @@ -444,39 +444,37 @@ CARD_IMAGE_LIMIT = 5 * 1024 * 1024 def _fetch_public(http: httpx.Client, url: str, limit: int, truncate: bool) -> tuple: """ - GET a public web URL, checking every redirect hop (no requests into the - LAN). Reads at most `limit` bytes: stops there if `truncate`, else fails. - Returns (data, content_type, final_url). - """ - from urllib.parse import urljoin, urlparse - from starpunk.micropub import _public_address + GET a public web URL (link card pages, images to re-upload). - site = current_app.config.get("SITE_URL", "") - for _ in range(5): - parsed = urlparse(url) - if parsed.scheme not in ("http", "https") or not parsed.hostname: - raise SyndicationError(f"not a web URL: {url}", permanent=True) - if not current_app.config.get("TESTING") and not _public_address(parsed.hostname): - raise SyndicationError(f"not a public address: {url}", permanent=True) - try: - with http.stream("GET", url, follow_redirects=False, timeout=10.0, - headers={"User-Agent": f"StarPunk (+{site})"}) as response: - if response.is_redirect and "location" in response.headers: - url = urljoin(url, response.headers["location"]) - continue - if response.status_code >= 400: - raise SyndicationError(f"{url}: HTTP {response.status_code}") - data = bytearray() - for chunk in response.iter_bytes(): - data += chunk - if len(data) > limit: - if truncate: - break - raise SyndicationError(f"{url}: too large") - return bytes(data), response.headers.get("content-type", ""), url - except httpx.HTTPError as e: - raise SyndicationError(f"{url}: {type(e).__name__}") - raise SyndicationError(f"{url}: too many redirects") + Uses the shared SSRF-safe fetcher (v2.11.0): every redirect hop checked, + the connection pinned to the checked address, a hard 10 s limit. Reads at + most `limit` bytes: stops there if `truncate`, else fails. + Returns (data, content_type, final_url). `http` is no longer used. + """ + from urllib.parse import urlparse + from starpunk.webmention_ssrf import ( + FetchTooLargeError, ResolutionError, SSRFBlockedError, safe_get, user_agent, + ) + + parsed = urlparse(url) + if parsed.scheme not in ("http", "https") or not parsed.hostname: + raise SyndicationError(f"not a web URL: {url}", permanent=True) + try: + result = safe_get(url, headers={"User-Agent": user_agent(current_app.config.get("SITE_URL", ""))}, + max_bytes=limit, max_redirects=5, timeout=10.0, truncate=truncate) + except ResolutionError as e: + raise SyndicationError(f"{url}: {e}") + except (SSRFBlockedError, ValueError) as e: + raise SyndicationError(f"not a public address: {url} ({e})", permanent=True) + except FetchTooLargeError: + raise SyndicationError(f"{url}: too large") + except httpx.TooManyRedirects: + raise SyndicationError(f"{url}: too many redirects") + except httpx.HTTPError as e: + raise SyndicationError(f"{url}: {type(e).__name__}") + if result.status_code >= 400: + raise SyndicationError(f"{url}: HTTP {result.status_code}") + return result.content, result.headers.get("content-type", ""), result.url def page_card(http: httpx.Client, url: str) -> dict: diff --git a/starpunk/utils.py b/starpunk/utils.py index aa18f09..4438ddf 100644 --- a/starpunk/utils.py +++ b/starpunk/utils.py @@ -217,7 +217,7 @@ def generate_slug(content: str, created_at: Optional[datetime] = None) -> str: # If slug is empty or too short, use timestamp fallback if len(slug) < MIN_SLUG_LENGTH: if created_at is None: - created_at = datetime.utcnow() + created_at = datetime.now(timezone.utc).replace(tzinfo=None) slug = created_at.strftime("%Y%m%d-%H%M%S") # Truncate to maximum length @@ -592,7 +592,7 @@ def delete_note_file( month = parts[notes_idx + 2] except (ValueError, IndexError): # If path doesn't follow expected structure, use current date - now = datetime.utcnow() + now = datetime.now(timezone.utc).replace(tzinfo=None) year = now.strftime("%Y") month = now.strftime("%m") diff --git a/starpunk/webmention_ssrf.py b/starpunk/webmention_ssrf.py index 1a4fd6c..fabc47c 100644 --- a/starpunk/webmention_ssrf.py +++ b/starpunk/webmention_ssrf.py @@ -250,7 +250,8 @@ def safe_post(url: str, data: Optional[dict] = None, headers: Optional[dict] = N def safe_get(url: str, headers: dict, max_bytes: int = MAX_BYTES, - max_redirects: int = MAX_REDIRECTS, timeout: float = TIMEOUT) -> FetchResult: + max_redirects: int = MAX_REDIRECTS, timeout: float = TIMEOUT, + truncate: bool = False) -> FetchResult: """ GET a URL, following redirects by hand and checking each hop. @@ -258,6 +259,8 @@ def safe_get(url: str, headers: dict, max_bytes: int = MAX_BYTES, url: URL to fetch headers: Request headers max_bytes: Response body limit + truncate: Stop at max_bytes and return what was read, instead of + raising FetchTooLargeError (v2.11.0) max_redirects: Redirect limit timeout: Total time limit in seconds @@ -287,13 +290,16 @@ def safe_get(url: str, headers: dict, max_bytes: int = MAX_BYTES, continue content_length = response.headers.get('Content-Length', '') - if content_length.isdigit() and int(content_length) > max_bytes: + if not truncate and content_length.isdigit() and int(content_length) > max_bytes: raise FetchTooLargeError(f"Response exceeds size limit ({max_bytes} bytes)") content = b'' for chunk in response.iter_bytes(chunk_size=8192): content += chunk if len(content) > max_bytes: + if truncate: + content = content[:max_bytes] + break raise FetchTooLargeError(f"Response exceeds size limit ({max_bytes} bytes)") if time.monotonic() > deadline: raise httpx.TimeoutException("Fetch exceeded time limit") diff --git a/starpunk/webmention_worker.py b/starpunk/webmention_worker.py index 4af1f01..1042447 100644 --- a/starpunk/webmention_worker.py +++ b/starpunk/webmention_worker.py @@ -19,7 +19,7 @@ import threading import time import sqlite3 import httpx -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Dict, Any, Optional from urllib.parse import urljoin @@ -116,7 +116,7 @@ class WebmentionWorker(threading.Thread): # Update last processed time if processed > 0: - self._last_processed = datetime.utcnow() + self._last_processed = datetime.now(timezone.utc).replace(tzinfo=None) # Sleep if nothing processed if processed == 0: @@ -790,7 +790,7 @@ class WebmentionWorker(threading.Thread): } delay = delays.get(new_retry_count) if delay: - next_retry_at = datetime.utcnow() + delay + next_retry_at = datetime.now(timezone.utc).replace(tzinfo=None) + delay try: conn = sqlite3.connect(self.db_path) diff --git a/starpunk/websub_worker.py b/starpunk/websub_worker.py index b4410fd..2d2b38e 100644 --- a/starpunk/websub_worker.py +++ b/starpunk/websub_worker.py @@ -25,7 +25,7 @@ import hashlib import hmac import logging import threading -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Optional from urllib.parse import urlencode, urlparse, urlsplit, urlunsplit @@ -53,11 +53,11 @@ def signature(secret: str, body: bytes) -> str: def _now() -> str: - return datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S") + return datetime.now(timezone.utc).replace(tzinfo=None).strftime("%Y-%m-%d %H:%M:%S") def _later(seconds: int) -> str: - return (datetime.utcnow() + timedelta(seconds=seconds)).strftime("%Y-%m-%d %H:%M:%S") + return (datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(seconds=seconds)).strftime("%Y-%m-%d %H:%M:%S") class WebSubWorker(threading.Thread): @@ -132,7 +132,7 @@ class WebSubWorker(threading.Thread): ).fetchall(): self._guarded(db, row, "delivery", lambda d, r: self.deliver(d, r, rendered), self._retry_or_drop) done += 1 - if self._last_prune is None or datetime.utcnow() - self._last_prune > PRUNE_EVERY: + if self._last_prune is None or datetime.now(timezone.utc).replace(tzinfo=None) - self._last_prune > PRUNE_EVERY: self.prune(db) return done @@ -348,7 +348,7 @@ class WebSubWorker(threading.Thread): db.execute("DELETE FROM websub_deliveries WHERE subscription_id IS NOT NULL AND " "subscription_id NOT IN (SELECT id FROM websub_subscriptions)") db.commit() - self._last_prune = datetime.utcnow() + self._last_prune = datetime.now(timezone.utc).replace(tzinfo=None) if cursor.rowcount: logger.info(f"WebSub: removed {cursor.rowcount} expired subscriptions") return cursor.rowcount diff --git a/templates/admin/authorize.html b/templates/admin/authorize.html index f4300d0..786986a 100644 --- a/templates/admin/authorize.html +++ b/templates/admin/authorize.html @@ -16,7 +16,7 @@

After approval you will be sent to {{ auth_request.redirect_uri }}

- diff --git a/templates/admin/dashboard.html b/templates/admin/dashboard.html index 03fe735..d935a55 100644 --- a/templates/admin/dashboard.html +++ b/templates/admin/dashboard.html @@ -74,7 +74,7 @@ Edit -
+
diff --git a/templates/admin/edit.html b/templates/admin/edit.html index 16c9ec6..5c19292 100644 --- a/templates/admin/edit.html +++ b/templates/admin/edit.html @@ -10,7 +10,7 @@ Created: {{ note.created_at.strftime('%B %d, %Y at %I:%M %p') }}

-
+
{% if note.reacts_to %}

{{ {'like': '★ Like of', 'repost': '🔁 Repost of', 'bookmark': '🔖 Bookmark of'}[note.kind] }} {{ note.reacts_to }}

@@ -76,7 +76,7 @@ {# Outside the edit form: nested forms are not allowed in HTML #} -
+
@@ -98,7 +98,7 @@ {% endif %} {% if s.last_error %}
{{ s.last_error }}{% endif %} {% if s.state != 'synced' %} -
+
{% endif %} diff --git a/templates/admin/new.html b/templates/admin/new.html index 4875a23..fcc33e8 100644 --- a/templates/admin/new.html +++ b/templates/admin/new.html @@ -6,7 +6,7 @@

Create New Note

-
+