Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
13936749de | ||
|
|
742e2e3ec3 | ||
|
|
d6a916ba7d | ||
|
|
e654dd84de | ||
|
|
abb75f080e | ||
|
|
d5157a6059 | ||
|
|
393228002b | ||
|
|
874465e103 | ||
|
|
3f5e7dc735 | ||
|
|
75946be7b2 |
No files matched your search
+9
-15
@@ -73,10 +73,10 @@ DATABASE_PATH=./data/starpunk.db
|
||||
# FLASK / LOGGING
|
||||
# =============================================================================
|
||||
|
||||
# development or production (default: development)
|
||||
FLASK_ENV=development
|
||||
# 1 = debug on, 0 = off (default: 1). Set 0 in production.
|
||||
FLASK_DEBUG=1
|
||||
# development or production (default: production)
|
||||
FLASK_ENV=production
|
||||
# 1 = debug on, 0 = off (default: 0). `flask run --debug` turns it on locally.
|
||||
FLASK_DEBUG=0
|
||||
# DEBUG, INFO, WARNING, ERROR or CRITICAL (default: INFO)
|
||||
LOG_LEVEL=INFO
|
||||
# Overrides the version shown in the footer and /health (default: the
|
||||
@@ -104,6 +104,10 @@ LOG_LEVEL=INFO
|
||||
# (default: false)
|
||||
TRUSTED_PROXY=false
|
||||
|
||||
# CSRF tokens on admin forms, logout and the IndieAuth consent (default: true).
|
||||
# Leave on; tests turn it off unless they test it.
|
||||
CSRF_ENABLED=true
|
||||
|
||||
# =============================================================================
|
||||
# FEEDS
|
||||
# =============================================================================
|
||||
@@ -195,19 +199,9 @@ THREADS_USERNAME=
|
||||
# SYNDICATION_POLL_INTERVAL=10
|
||||
|
||||
# =============================================================================
|
||||
# MONITORING
|
||||
# DEBUGGING
|
||||
# =============================================================================
|
||||
|
||||
# Metrics shown on /admin/metrics-dashboard (default: true)
|
||||
# METRICS_ENABLED=true
|
||||
# Queries slower than this many seconds are logged (default: 1.0)
|
||||
# METRICS_SLOW_QUERY_THRESHOLD=1.0
|
||||
# Fraction of operations recorded, 0.0-1.0 (default: 1.0)
|
||||
# METRICS_SAMPLING_RATE=1.0
|
||||
# METRICS_BUFFER_SIZE=1000
|
||||
# Seconds between memory samples (default: 30)
|
||||
# METRICS_MEMORY_INTERVAL=30
|
||||
|
||||
# Keep uploads that fail processing in DATA_PATH/debug for diagnosis
|
||||
# (default: false); cleaned up by age (default: 7 days) and total size
|
||||
# (default: 100 MB)
|
||||
|
||||
@@ -7,6 +7,41 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [2.12.0] - 2026-11-03
|
||||
|
||||
The metrics dashboard is gone (ADR-072).
|
||||
|
||||
### Removed
|
||||
|
||||
- The admin metrics dashboard (`/admin/metrics-dashboard`, the "Metrics" admin link) and its JSON endpoint `/admin/metrics`. Its numbers were kept in memory per server process, so they covered a quarter of the traffic and vanished on every restart
|
||||
- Everything that only fed it: request and database-query timing, the memory monitor thread, feed-generation counters (`starpunk/monitoring/`), and the `X-Request-ID` response header. Slow queries are no longer logged
|
||||
- The `METRICS_ENABLED`, `METRICS_SLOW_QUERY_THRESHOLD`, `METRICS_SAMPLING_RATE`, `METRICS_BUFFER_SIZE` and `METRICS_MEMORY_INTERVAL` settings (ignored if still set), and the `psutil` dependency
|
||||
- The `metrics` section of `/admin/health`; its database, pool, filesystem and disk checks are unchanged, as is `/health`
|
||||
|
||||
## [2.11.0] - 2026-11-03
|
||||
|
||||
Maintenance release: dependency upgrades and the High-priority backlog items.
|
||||
|
||||
### Security
|
||||
|
||||
- Every dependency is upgraded to its current release. The previous set had 97 published vulnerabilities (Pillow 64, PyJWT 22, gunicorn 4, and Flask, Markdown, python-dotenv, pytest and pillow-heif); the new set has none known. Notable versions: Flask 3.1, gunicorn 26, Pillow 13, PyJWT 2.15, httpx 0.28, nh3 0.3, Markdown 3.11, pillow-heif 1.8, psutil 7
|
||||
- Admin forms, logout and the IndieAuth consent screen now carry a CSRF token (an HMAC of your login session, no storage needed); requests without it are refused. Endpoints that apps call with tokens (Micropub, Webmention, WebSub, the token endpoint) are unaffected
|
||||
- Fetching an IndieAuth app's page and fetching pages and images for crosspost link cards now use the same protected fetcher as webmentions: every redirect is checked against private addresses, the connection goes to the checked address, and the whole fetch has a time limit
|
||||
- Debug mode is off and the environment is production unless set otherwise (`flask run --debug` still turns debug on locally); the container image sets both explicitly
|
||||
|
||||
### Changed
|
||||
|
||||
- Runs on Python 3.14 (was 3.11, which reaches end of life in 2027)
|
||||
- Rate limits on `/webmention` and `/websub` are counted in the database and shared by all server processes. Before, each of the four processes counted separately, so the real limit was four times the setting
|
||||
- Test dependencies are in `requirements-test.txt` (used by CI); `pytest` is no longer installed in the production image, and `lxml` and `python-dateutil`, which StarPunk uses directly, are now pinned
|
||||
- `uv` is pinned in the Containerfile instead of `latest`
|
||||
|
||||
### Fixed
|
||||
|
||||
- Deleting a note or a photo left its tag, photo and image-size rows behind, because SQLite only applies the schema's cascades when foreign keys are switched on. They now are; migration 018 removes rows left by earlier deletes (production had three)
|
||||
- Deprecated `datetime.utcnow()` and sqlite3's default date adapter (both removed in a future Python) replaced; the stored formats are unchanged
|
||||
- `compose.yaml` pinned version 0.6.0 (which would override the real version shown) and set an unused variable; a stray duplicate `dev_auth.py` and a `test.ini` with a private host name were removed
|
||||
|
||||
## [2.10.2] - 2026-11-03
|
||||
|
||||
### Fixed
|
||||
|
||||
@@ -7,7 +7,7 @@ This file contains operational instructions for Claude agents working on this pr
|
||||
- We use **uv** for Python virtual environment management
|
||||
- All Python commands must be run with `uv run` prefix
|
||||
- Example: `uv run pytest`, `uv run flask --app app.py run --debug`
|
||||
- There is no `pyproject.toml`: create the environment with `uv venv --python 3.11 && uv pip install -r requirements-dev.txt`
|
||||
- There is no `pyproject.toml`: create the environment with `uv venv --python 3.14 && uv pip install -r requirements-dev.txt`
|
||||
- Tests need `SESSION_SECRET` set: `SESSION_SECRET=test uv run pytest`. The full suite takes about 6 minutes, 5 of them in `tests/test_migration_race_condition.py`; add `--ignore=tests/test_migration_race_condition.py` while iterating, but run everything before a release
|
||||
- Local login without OIDC: `DEV_MODE=true` and `DEV_ADMIN_ME` in `.env`, then `/dev/login`
|
||||
- Setup details: `docs/standards/development-setup.md`
|
||||
|
||||
+9
-6
@@ -6,10 +6,10 @@
|
||||
# ============================================================================
|
||||
# Build Stage - Install dependencies in virtual environment
|
||||
# ============================================================================
|
||||
FROM python:3.11-slim AS builder
|
||||
FROM python:3.14-slim AS builder
|
||||
|
||||
# Install uv for fast dependency installation
|
||||
COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.9.28 /uv /usr/local/bin/uv
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
@@ -25,11 +25,12 @@ RUN uv venv /opt/venv && \
|
||||
# ============================================================================
|
||||
# Test Stage - Runs the test suite (CI builds it with --target test)
|
||||
# ============================================================================
|
||||
FROM python:3.11-slim AS test
|
||||
FROM python:3.14-slim AS test
|
||||
|
||||
COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.9.28 /uv /usr/local/bin/uv
|
||||
COPY --from=builder /opt/venv /opt/venv
|
||||
RUN uv pip install --no-cache --python /opt/venv/bin/python pytest pytest-mock
|
||||
COPY requirements-test.txt .
|
||||
RUN uv pip install --no-cache --python /opt/venv/bin/python -r requirements-test.txt
|
||||
|
||||
ENV PATH="/opt/venv/bin:$PATH" \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
@@ -45,7 +46,7 @@ RUN python -m pytest -q -p no:cacheprovider tests
|
||||
# ============================================================================
|
||||
# Runtime Stage - Minimal production image (default target: keep it last)
|
||||
# ============================================================================
|
||||
FROM python:3.11-slim
|
||||
FROM python:3.14-slim
|
||||
|
||||
# Create non-root user for security
|
||||
# UID/GID 1000 is standard for first user on most systems
|
||||
@@ -60,6 +61,8 @@ COPY --from=builder /opt/venv /opt/venv
|
||||
ENV PATH="/opt/venv/bin:$PATH" \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
FLASK_ENV=production \
|
||||
FLASK_DEBUG=0 \
|
||||
FLASK_APP=app.py \
|
||||
DATA_PATH=/data \
|
||||
NOTES_PATH=/data/notes \
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2024 [Your Name]
|
||||
Copyright (c) 2025-2026 Phil Skentelbery
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
||||
@@ -41,7 +41,7 @@ what changed in each release is in [CHANGELOG.md](CHANGELOG.md).
|
||||
|
||||
**Operations**
|
||||
- Migrations applied automatically at startup
|
||||
- `/health` endpoint, admin metrics dashboard, structured logs
|
||||
- `/health` endpoint, `/admin/health` diagnostics, structured logs
|
||||
- Fetches of user-supplied URLs are limited to public addresses, size and
|
||||
time
|
||||
|
||||
@@ -69,7 +69,7 @@ redirect URI with your OIDC provider. Details:
|
||||
### Local development
|
||||
|
||||
```bash
|
||||
uv venv --python 3.11
|
||||
uv venv --python 3.14
|
||||
uv pip install -r requirements-dev.txt
|
||||
cp .env.example .env # set SESSION_SECRET; add DEV_MODE=true and DEV_ADMIN_ME
|
||||
uv run flask --app app.py run --debug # http://localhost:5000
|
||||
@@ -113,4 +113,5 @@ up regularly, and always before upgrading to a release with a migration.
|
||||
|
||||
## License
|
||||
|
||||
MIT, see [LICENSE](LICENSE).
|
||||
[MIT](LICENSE): use, change and share it for anything, including
|
||||
commercially, as long as the copyright and license notice stay with it.
|
||||
+2
-7
@@ -11,12 +11,11 @@
|
||||
# docker compose logs -f
|
||||
# docker compose down
|
||||
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
starpunk:
|
||||
# Container configuration
|
||||
image: starpunk:0.6.0
|
||||
# Built locally from this checkout (release images: see README)
|
||||
image: starpunk:local
|
||||
container_name: starpunk
|
||||
|
||||
# Build configuration
|
||||
@@ -49,10 +48,6 @@ services:
|
||||
- NOTES_PATH=/data/notes
|
||||
- DATABASE_PATH=/data/starpunk.db
|
||||
|
||||
# Application metadata
|
||||
- VERSION=0.6.0
|
||||
- ENVIRONMENT=production
|
||||
|
||||
# Volume mounts for persistent data
|
||||
# All application data stored in ./container-data on host
|
||||
volumes:
|
||||
|
||||
-103
@@ -1,103 +0,0 @@
|
||||
"""
|
||||
Development authentication module for StarPunk
|
||||
|
||||
WARNING: This module provides a development-only authentication mechanism
|
||||
that bypasses IndieLogin. It should NEVER be enabled in production.
|
||||
|
||||
This module is separate from production auth (starpunk/auth.py) to maintain
|
||||
clear architectural boundaries and enable easy security audits.
|
||||
|
||||
Security measures:
|
||||
- Only active when DEV_MODE=true
|
||||
- Returns 404 if DEV_MODE=false
|
||||
- Requires DEV_ADMIN_ME configuration
|
||||
- Logs prominent warnings
|
||||
- Cannot coexist with production SITE_URL
|
||||
- Visual warnings in UI
|
||||
|
||||
Functions:
|
||||
is_dev_mode: Check if development mode is enabled
|
||||
validate_dev_config: Validate development configuration
|
||||
create_dev_session: Create session without authentication
|
||||
"""
|
||||
|
||||
from flask import current_app
|
||||
|
||||
from starpunk.auth import create_session
|
||||
|
||||
|
||||
def is_dev_mode() -> bool:
|
||||
"""
|
||||
Check if development mode is enabled
|
||||
|
||||
Returns:
|
||||
True if DEV_MODE is enabled, False otherwise
|
||||
|
||||
Example:
|
||||
>>> from starpunk.dev_auth import is_dev_mode
|
||||
>>> if is_dev_mode():
|
||||
... print("Development mode active")
|
||||
"""
|
||||
return current_app.config.get("DEV_MODE", False)
|
||||
|
||||
|
||||
def validate_dev_config() -> None:
|
||||
"""
|
||||
Validate development mode configuration
|
||||
|
||||
Checks that DEV_MODE configuration is valid and safe:
|
||||
- DEV_ADMIN_ME must be set if DEV_MODE is true
|
||||
- Warns if DEV_MODE is enabled with production-like SITE_URL
|
||||
|
||||
Raises:
|
||||
ValueError: If DEV_MODE is true but DEV_ADMIN_ME is not set
|
||||
|
||||
Logs:
|
||||
WARNING: If DEV_MODE is enabled with HTTPS SITE_URL
|
||||
"""
|
||||
dev_mode = current_app.config.get("DEV_MODE", False)
|
||||
|
||||
if dev_mode:
|
||||
# Require DEV_ADMIN_ME
|
||||
dev_admin_me = current_app.config.get("DEV_ADMIN_ME")
|
||||
if not dev_admin_me:
|
||||
raise ValueError("DEV_MODE=true requires DEV_ADMIN_ME to be set")
|
||||
|
||||
# Warn if production-like configuration detected
|
||||
site_url = current_app.config.get("SITE_URL", "")
|
||||
if site_url.startswith("https://"):
|
||||
current_app.logger.warning(
|
||||
"WARNING: DEV_MODE is enabled with production SITE_URL. "
|
||||
"This is likely a misconfiguration. "
|
||||
"DEV_MODE should only be used in local development."
|
||||
)
|
||||
|
||||
|
||||
def create_dev_session(me: str) -> str:
|
||||
"""
|
||||
Create development session without authentication
|
||||
|
||||
WARNING: This bypasses IndieLogin authentication entirely.
|
||||
Only use in development environments.
|
||||
|
||||
Args:
|
||||
me: User identity URL (from DEV_ADMIN_ME config)
|
||||
|
||||
Returns:
|
||||
Session token (same format as production sessions)
|
||||
|
||||
Logs:
|
||||
WARNING: Logs that dev session was created without authentication
|
||||
|
||||
Example:
|
||||
>>> token = create_dev_session("https://example.com")
|
||||
>>> # Session created without IndieLogin verification
|
||||
"""
|
||||
current_app.logger.warning(
|
||||
f"DEV MODE: Creating session for {me} without authentication. "
|
||||
f"This should NEVER happen in production!"
|
||||
)
|
||||
|
||||
# Use production session creation (same session format)
|
||||
# This ensures dev sessions work identically to production
|
||||
return create_session(me)
|
||||
@@ -11,7 +11,7 @@ per-release designs and reports are in [../design/](../design/).
|
||||
|----------|-------------|
|
||||
| [overview.md](overview.md) | System overview: purpose, request flow, modules and routes, background workers, storage, auth model, URL surface, caching, security posture |
|
||||
| [technology-stack.md](technology-stack.md) | Current runtime, Python dependencies, standards implemented, testing and CI |
|
||||
| [database.md](database.md) | SQLite schema after migrations 001–017, migration runner and its lock, connection pool |
|
||||
| [database.md](database.md) | SQLite schema after migrations 001–019, migration runner and its lock, connection pool |
|
||||
| [deployment.md](deployment.md) | Container image stages, CI workflow, release process, `/data` layout, reverse proxy, all environment variables |
|
||||
| [simplified-auth-architecture.md](simplified-auth-architecture.md) | Admin login via OpenID Connect and the built-in IndieAuth server that issues Micropub tokens (ADR-064) |
|
||||
| [indieweb-protocols.md](indieweb-protocols.md) | How Micropub, IndieAuth, Webmention, microformats2, WebSub and the feeds are implemented, with compliance status and known gaps |
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
# Database Architecture
|
||||
|
||||
**Version**: 2.10.1 (2026-11-03)
|
||||
**Version**: 2.11.0 (2026-11-03)
|
||||
**Status**: Current. Verified against a database created by `create_app()`
|
||||
with migrations 001–017.
|
||||
with migrations 001–019.
|
||||
|
||||
StarPunk uses one SQLite file, `starpunk.db` under `DATA_PATH` (`/data` in
|
||||
the container), in WAL mode. Note *content* is not in the database: it lives
|
||||
@@ -71,6 +71,8 @@ Background: [ADR-031](../decisions/ADR-031-database-migration-system-redesign.md
|
||||
| `015_bookmarks_and_content_format.sql` | 2.9 | `notes.bookmark_of`, `notes.content_format` |
|
||||
| `016_websub.sql` | 2.10.0 | `websub_subscriptions`, `websub_intents`, `websub_deliveries`, `site_state` |
|
||||
| `017_websub_hardening.sql` | 2.10.1 | `websub_intents.request_id`, `requested_topic`, `callback_ip`; `websub_subscriptions.callback_ip` |
|
||||
| `018_foreign_key_cleanup.sql` | 2.11.0 | Removes rows orphaned while foreign keys weren't enforced |
|
||||
| `019_shared_rate_limits.sql` | 2.11.0 | `rate_limits` (counters shared by all processes) |
|
||||
|
||||
New migrations are numbered `NNN_description.sql`, must be safe to run inside
|
||||
one transaction, and should use `IF NOT EXISTS` where SQLite allows it.
|
||||
@@ -205,14 +207,18 @@ its cached feed data is stale.
|
||||
|
||||
**`schema_migrations`**: applied migrations (see above).
|
||||
|
||||
**`rate_limits`** (v2.11.0): request counts for `/webmention` and `/websub`,
|
||||
one row per key (`<limiter>:<ip or domain>`) and time window, so every
|
||||
server process enforces one shared limit. Old windows are pruned as it goes.
|
||||
|
||||
## Connections
|
||||
|
||||
- **Request connections**: `starpunk/database/pool.py`. `init_pool()` creates
|
||||
one `ConnectionPool` per process (up to 5 idle connections, 10-second busy
|
||||
timeout; `DB_POOL_SIZE` and `DB_TIMEOUT` can be set only through the
|
||||
`create_app()` config dict). Connections use `sqlite3.Row` rows,
|
||||
`check_same_thread=False` and `PRAGMA journal_mode=WAL`, and are wrapped
|
||||
in `MonitoredConnection` for query timing when metrics are on. `get_db()`
|
||||
`check_same_thread=False`, `PRAGMA journal_mode=WAL` and (since v2.11.0)
|
||||
`PRAGMA foreign_keys=ON`, so `ON DELETE CASCADE`/`SET NULL` run. `get_db()`
|
||||
stores one connection in `flask.g` per request or app context; on teardown
|
||||
it goes back to the pool, rolled back first if a transaction was left
|
||||
open.
|
||||
|
||||
@@ -24,12 +24,12 @@ deployment repository, not here.
|
||||
|
||||
## Container image (`Containerfile`)
|
||||
|
||||
A multi-stage build on `python:3.11-slim`; it works with Podman and Docker.
|
||||
A multi-stage build on `python:3.14-slim`; it works with Podman and Docker.
|
||||
|
||||
| Stage | Purpose |
|
||||
|-------|---------|
|
||||
| `builder` | Copies `uv` from `ghcr.io/astral-sh/uv`, creates `/opt/venv` and installs `requirements.txt` into it |
|
||||
| `test` | Copies the venv, adds `pytest` and `pytest-mock`, copies the source and runs `python -m pytest -q -p no:cacheprovider tests` with test values for `SESSION_SECRET`, `SITE_URL` and `ADMIN_ME`. The stage only builds if the tests pass. |
|
||||
| `test` | Copies the venv, installs `requirements-test.txt`, copies the source and runs `python -m pytest -q -p no:cacheprovider tests` with test values for `SESSION_SECRET`, `SITE_URL` and `ADMIN_ME`. The stage only builds if the tests pass. |
|
||||
| runtime (last, the default target) | Creates user `starpunk` (UID/GID 1000) and `/data/notes`, copies the venv and the application to `/app`, sets `DATA_PATH=/data`, `NOTES_PATH=/data/notes`, `DATABASE_PATH=/data/starpunk.db`, runs as `starpunk`, exposes port 8000 |
|
||||
|
||||
The runtime image's command is gunicorn:
|
||||
@@ -172,8 +172,8 @@ ignored with a warning.
|
||||
| `DATA_PATH` | `./data` (image: `/data`) | Root of all stored data |
|
||||
| `DATABASE_PATH` | `./data/starpunk.db` (image: `/data/starpunk.db`) | |
|
||||
| `NOTES_PATH` | `./data/notes` (image: `/data/notes`) | Created at start-up, but note files are always written to `DATA_PATH/notes` |
|
||||
| `FLASK_ENV` | `development` | Set `production` in production |
|
||||
| `FLASK_DEBUG` | `1` | Set `0` in production (the image does not set it) |
|
||||
| `FLASK_ENV` | `production` | Reported in `/health` and admin diagnostics; the image sets `production` |
|
||||
| `FLASK_DEBUG` | `0` | Debug off unless `1` (`flask run --debug` sets it locally); the image sets `0` |
|
||||
| `LOG_LEVEL` | `INFO` | `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL` |
|
||||
| `VERSION` | package `__version__` | Overrides the reported version; normally unset |
|
||||
|
||||
@@ -235,15 +235,12 @@ starts only if at least one is.
|
||||
| `THREADS_MAX_MEDIA` | `4` | |
|
||||
| `SYNDICATION_POLL_INTERVAL` | `10` | Seconds |
|
||||
|
||||
### Metrics and debugging
|
||||
### Debugging
|
||||
|
||||
The `METRICS_*` variables were removed in v2.12.0 and are ignored if set.
|
||||
|
||||
| Variable | Default | Notes |
|
||||
|----------|---------|-------|
|
||||
| `METRICS_ENABLED` | `true` | HTTP/database metrics and the memory monitor |
|
||||
| `METRICS_SLOW_QUERY_THRESHOLD` | `1.0` | Seconds |
|
||||
| `METRICS_SAMPLING_RATE` | `1.0` | |
|
||||
| `METRICS_BUFFER_SIZE` | `1000` | |
|
||||
| `METRICS_MEMORY_INTERVAL` | `30` | Seconds |
|
||||
| `DEBUG_SAVE_FAILED_UPLOADS` | `false` | Keep failed uploads in `debug/` |
|
||||
| `DEBUG_FILE_MAX_AGE_DAYS` | `7` | |
|
||||
| `DEBUG_FILE_MAX_SIZE_MB` | `100` | |
|
||||
|
||||
@@ -197,7 +197,8 @@ route isn't registered at all.
|
||||
|
||||
1. Rate limits: 60 requests per minute per client IP and 10 per minute per
|
||||
source domain (`WEBMENTION_RATE_LIMIT_*`). Over the limit gets 429 with
|
||||
`Retry-After`. The limiter is in memory, per process. The client IP
|
||||
`Retry-After`. Counts are shared by all server processes (`rate_limits`
|
||||
table, v2.11.0). The client IP
|
||||
comes from `X-Forwarded-For` only when `TRUSTED_PROXY=true`.
|
||||
2. Form-encoded (JSON also accepted). `source` and `target` are required,
|
||||
http(s), at most 2048 characters, and different from each other.
|
||||
@@ -445,7 +446,7 @@ Drafts that stay drafts change nothing.
|
||||
|
||||
Limits (429 when hit):
|
||||
|
||||
- 30 requests per minute per client IP (per process; needs
|
||||
- 30 requests per minute per client IP (shared by all processes; needs
|
||||
`TRUSTED_PROXY=true` behind a proxy);
|
||||
- at most 200 pending verifications;
|
||||
- `WEBSUB_MAX_SUBSCRIPTIONS` (1000) in total;
|
||||
@@ -488,7 +489,6 @@ retried.
|
||||
|
||||
- The hub serves only this site's topics, by design. websub.rocks' hub
|
||||
tests use their own topics, so they can't be run against it.
|
||||
- Rate limiting is per process and in memory.
|
||||
- Deliveries always carry the full topic content (no diffs). Fat pings
|
||||
aren't sent to external hubs.
|
||||
|
||||
|
||||
@@ -43,8 +43,6 @@ Key decisions:
|
||||
[ADR-058](../decisions/ADR-058-image-optimization-strategy.md) (media),
|
||||
[ADR-034](../decisions/ADR-034-full-text-search.md) (search),
|
||||
[ADR-052](../decisions/ADR-052-configuration-system-architecture.md) (configuration),
|
||||
[ADR-053](../decisions/ADR-053-performance-monitoring-strategy.md) and
|
||||
[ADR-060](../decisions/ADR-060-production-hotfix-metrics-dashboard.md) (metrics),
|
||||
[ADR-054](../decisions/ADR-054-structured-logging-architecture.md) (logging),
|
||||
[ADR-055](../decisions/ADR-055-error-handling-philosophy.md) (errors).
|
||||
|
||||
@@ -65,7 +63,6 @@ Key decisions:
|
||||
│ webmention worker works its queues only while holding a lock │
|
||||
│ websub worker works its queues only while holding a lock │
|
||||
│ syndication worker claims rows with a lease in SQLite │
|
||||
│ memory monitor per-process metrics │
|
||||
└───────────────┬──────────────────────────────┬─────────────────────┘
|
||||
│ │
|
||||
┌───────▼────────┐ ┌──────────▼──────────────────┐
|
||||
@@ -86,7 +83,7 @@ Key decisions:
|
||||
validates configuration, configures logging, cleans up old debug and
|
||||
temporary upload files, creates the schema and runs migrations (under a
|
||||
lock, see [database.md](database.md)), creates the connection pool,
|
||||
installs the HTTP metrics middleware, configures the feed cache and rate
|
||||
configures the feed cache and rate
|
||||
limiters, fills the FTS index if it is empty, registers blueprints, error
|
||||
handlers and template helpers, and starts the background threads.
|
||||
3. `before_request` assigns a correlation ID (included in every log line) and,
|
||||
@@ -94,8 +91,7 @@ Key decisions:
|
||||
4. The blueprint handler runs. Database access goes through `get_db()`, which
|
||||
hands out a pooled, request-scoped SQLite connection.
|
||||
5. `after_request` hooks add `Link: </webmention>; rel="webmention"` to HTML
|
||||
responses, WebSub `Link rel=hub` / `rel=self` headers to topic responses,
|
||||
and request metrics with an `X-Request-ID` header.
|
||||
responses, WebSub `Link rel=hub` / `rel=self` headers to topic responses.
|
||||
6. Errors are rendered by the central handlers in `starpunk/errors.py`
|
||||
(JSON for Micropub, HTML pages from `templates/<code>.html` elsewhere).
|
||||
|
||||
@@ -141,7 +137,6 @@ wait for outbound network calls; the workers make them later.
|
||||
| `errors.py` | Central error handlers |
|
||||
| `feeds/` | `rss.py` (feedgen), `atom.py`, `json_feed.py`, `opml.py`, `negotiation.py` (Accept header), `cache.py` (LRU + TTL feed cache with ETags) |
|
||||
| `feed.py` | Backward-compatibility re-export of `feeds.rss` |
|
||||
| `monitoring/` | Metrics buffer, HTTP middleware, database timing wrapper, memory monitor, business metrics |
|
||||
| `database/` | `schema.py` (`INITIAL_SCHEMA_SQL`), `init.py` (`init_db`), `pool.py` (connection pool) |
|
||||
| `migrations.py` | Migration runner for `migrations/NNN_*.sql` |
|
||||
|
||||
@@ -196,7 +191,7 @@ Everything works without JavaScript.
|
||||
| `/admin/webmentions`, `/admin/webmentions/<id>`, `…/action`, `…/bulk-action` | Webmention moderation |
|
||||
| `/admin/tokens`, `/admin/tokens/<id>/revoke` | Issued IndieAuth tokens |
|
||||
| `/admin/syndication/<id>/retry` | Retry a failed crosspost |
|
||||
| `/admin/metrics-dashboard`, `/admin/metrics`, `/admin/health` | Monitoring (HTML dashboard, JSON metrics, full diagnostics) |
|
||||
| `/admin/health` | Full diagnostics (JSON): database, pool statistics, filesystem, disk |
|
||||
| `/health` | Public liveness check (`{"status": "ok", "version": …}`); `?detailed=true` requires login |
|
||||
|
||||
## Background workers
|
||||
@@ -210,7 +205,6 @@ coordinated through the data volume:
|
||||
| Webmention worker (`webmention_worker.py`) | `WEBMENTION_WORKER_ENABLED` and not testing | Non-blocking `fcntl.flock` on `webmention-worker.lock` next to the database. Only the holder works the queues; the others try again each poll interval. The OS releases the lock if the holder dies, and the new holder resets rows stuck in `sending`. | Validates received webmentions; discovers endpoints and sends queued outgoing webmentions, with retries |
|
||||
| WebSub worker (`websub_worker.py`) | `WEBSUB_ENABLED` and not testing | Same pattern with `websub-worker.lock` | Verifies subscription intents, delivers topic content (signed with `X-Hub-Signature` when the subscriber gave a secret), retries, expires leases; pings an external hub instead if `WEBSUB_HUB_URL` is set |
|
||||
| Syndication worker (`syndication.py`) | At least one crosspost target configured, and not testing | No file lock. Rows in `syndications` are claimed with a 5-minute `claimed_until` lease by an atomic `UPDATE`, so any process may work a row but only one at a time; a `version` column keeps edits made during a send. Threads token refreshes are serialised with a lock file next to `threads_token.json`. | Creates, updates or deletes the remote post for each (note, target) pair; up to 8 attempts |
|
||||
| Memory monitor (`monitoring/memory.py`) | `METRICS_ENABLED` and not testing | None (per-process metrics) | Samples process memory every `METRICS_MEMORY_INTERVAL` seconds |
|
||||
|
||||
All workers run inside an app context, are daemon threads (they end with the
|
||||
process) and sleep for their poll interval when there is nothing to do. Under
|
||||
@@ -293,13 +287,12 @@ their credentials are set. The full list with defaults is in
|
||||
callbacks, remote Micropub photos, IndieAuth client pages, crosspost link
|
||||
cards) are checked first: every address the host resolves to must be public
|
||||
(private, loopback, link-local, CGNAT, multicast and reserved ranges are
|
||||
refused). The fetcher in `webmention_ssrf.py`, used for webmentions, reply
|
||||
contexts, WebSub and remote Micropub photos, follows redirects by hand and
|
||||
re-checks each hop, connects to the address it checked (no second DNS
|
||||
lookup) and limits response size (1 MB by default) and total time.
|
||||
Crosspost link cards use their own check that also re-checks each redirect.
|
||||
The IndieAuth client-page fetch checks only the initial URL. `DEV_MODE`
|
||||
relaxes the checks for local testing.
|
||||
refused). Every fetch of a user-supplied URL (webmentions, reply contexts,
|
||||
WebSub, remote Micropub photos, IndieAuth client pages and, since v2.11.0,
|
||||
crosspost link cards) uses the fetcher in `webmention_ssrf.py`: it follows
|
||||
redirects by hand and re-checks each hop, connects to the address it
|
||||
checked (no second DNS lookup) and limits response size (1 MB by default)
|
||||
and total time. `DEV_MODE` relaxes the checks for local testing.
|
||||
- **HTML sanitising**: rendered notes (Markdown or HTML sent by a Micropub
|
||||
client) and webmention content are cleaned with nh3 allow-lists; search
|
||||
highlighting escapes everything except `<mark>`.
|
||||
@@ -307,15 +300,17 @@ their credentials are set. The full list with defaults is in
|
||||
(defaults 60 and 10 per 60 seconds); `/websub` 30 requests per IP per
|
||||
minute, plus limits on subscriptions in total, per callback host and per
|
||||
callback address, and on requests waiting for verification. Request-rate
|
||||
counters are in memory and per process.
|
||||
counters live in the `rate_limits` table, shared by all server processes
|
||||
(v2.11.0).
|
||||
- **Client IP behind a proxy**: with `TRUSTED_PROXY=true` the first address in
|
||||
`X-Forwarded-For` is used as the client IP for rate limiting. Set it only
|
||||
when the app is reachable solely through a proxy that sets that header
|
||||
(Caddy does); without it, every client shares the proxy's address.
|
||||
- **Auth**: hashed session and token storage, single-use login state, PKCE
|
||||
for both OIDC and IndieAuth, redirect URIs checked against the client,
|
||||
post-login redirects limited to this site. Admin form POSTs rely on the
|
||||
SameSite=Lax session cookie (there are no per-form CSRF tokens).
|
||||
post-login redirects limited to this site. Admin form POSTs, logout and
|
||||
the IndieAuth consent carry a CSRF token: an HMAC of the session cookie,
|
||||
checked by `starpunk/csrf.py` (v2.11.0), on top of the SameSite=Lax cookie.
|
||||
- **Uploads**: request bodies are capped at 50 MB; images are processed by
|
||||
Pillow and stored under random (UUID) names.
|
||||
- **Container**: runs as a non-root user. HTTPS and security headers (HSTS,
|
||||
@@ -325,14 +320,11 @@ their credentials are set. The full list with defaults is in
|
||||
|
||||
- Logs go to stdout and `logs/starpunk.log` (10 MB × 10 files); each line
|
||||
carries the request's correlation ID.
|
||||
- Metrics (`monitoring/`): HTTP request timings, database query timings
|
||||
(with a slow-query threshold), memory, pool statistics and business
|
||||
counters, in an in-memory buffer per process, shown at
|
||||
`/admin/metrics-dashboard` and `/admin/metrics`.
|
||||
- `/health` reports status, version and webmention-worker statistics;
|
||||
`/health?detailed=true` checks database, filesystem and disk space;
|
||||
`/admin/health` adds metrics, pool statistics and system information.
|
||||
`/admin/health` adds pool statistics and filesystem details. (The metrics
|
||||
dashboard was removed in v2.12.0, see ADR-072.)
|
||||
|
||||
---
|
||||
|
||||
**Last Updated**: 2026-11-03 (v2.10.1)
|
||||
**Last Updated**: 2026-11-03 (v2.12.0)
|
||||
@@ -328,14 +328,16 @@ In place:
|
||||
untrusted redirect URIs never followed.
|
||||
- Post-login redirects are limited to this site.
|
||||
|
||||
CSRF (v2.11.0): admin form POSTs, `/auth/logout` and the consent form's
|
||||
approve button need a `csrf_token` field (or `X-CSRF-Token` header) equal to
|
||||
an HMAC-SHA256 of the `starpunk_session` cookie under `SECRET_KEY`; anything
|
||||
else gets 400. The token changes with every login and needs no storage.
|
||||
Endpoints apps call with bearer tokens are not affected. The client metadata
|
||||
fetch uses `webmention_ssrf.safe_get` (every redirect hop checked, the
|
||||
connection pinned, a 5-second limit).
|
||||
|
||||
Known gaps:
|
||||
|
||||
- Admin forms and the consent form have no CSRF token. They rely on the
|
||||
`SameSite=Lax` session cookie, which cross-site POSTs don't carry.
|
||||
- The client metadata fetch checks the `client_id` URL against private
|
||||
addresses, but then makes a plain `httpx.get` that follows redirects.
|
||||
Redirect hops are not re-checked and the connection is not pinned, unlike
|
||||
`webmention_ssrf.safe_get`.
|
||||
- `scopes_supported` leaves out `draft`, although Micropub honours it and
|
||||
the consent screen grants it when requested. Unknown scopes are also shown
|
||||
and granted as requested; they just have no effect.
|
||||
|
||||
@@ -246,10 +246,6 @@ to itself (v2.8.0).
|
||||
|
||||
- Bluesky and Threads copies don't follow edits (platform limitation).
|
||||
Unticking and re-ticking a target makes a new copy.
|
||||
- Link-card and Threads image fetches check every redirect hop against
|
||||
private addresses but don't pin DNS, and use a per-request rather than an
|
||||
overall time limit. Webmention and WebSub fetching use the stricter
|
||||
`webmention_ssrf` fetcher.
|
||||
- No per-target choice for Micropub updates: `mp-syndicate-to` is read only
|
||||
on create.
|
||||
- Likes of ordinary web pages, and all likes on Threads, stay on the site.
|
||||
@@ -14,9 +14,9 @@ the pieces fit together.
|
||||
|
||||
| Component | Choice |
|
||||
|-----------|--------|
|
||||
| Language | Python 3.11 (`python:3.11-slim` base image) |
|
||||
| Web framework | Flask 3.0 (Werkzeug, Jinja2, itsdangerous, click come with it) |
|
||||
| WSGI server | gunicorn 21.2, 4 sync workers (see [deployment.md](deployment.md)) |
|
||||
| Language | Python 3.14 (`python:3.14-slim` base image; since v2.11.0) |
|
||||
| Web framework | Flask 3.1 (Werkzeug, Jinja2, itsdangerous, click come with it) |
|
||||
| WSGI server | gunicorn 26.2, 4 sync workers (see [deployment.md](deployment.md)) |
|
||||
| Database | SQLite (standard-library `sqlite3`), WAL mode, FTS5 for search |
|
||||
| Content storage | Markdown files on disk |
|
||||
| Background work | `threading` daemon threads; `fcntl.flock` file locks to pick one process |
|
||||
@@ -28,32 +28,35 @@ the pieces fit together.
|
||||
|
||||
## Python dependencies (`requirements.txt`)
|
||||
|
||||
Pinned to a minor series. Upgraded in v2.11.0, when the set had no known
|
||||
vulnerabilities (`pip-audit`); check again before each release.
|
||||
|
||||
| Package | Pin | Used for |
|
||||
|---------|-----|----------|
|
||||
| Flask | `3.0.*` | Web framework |
|
||||
| gunicorn | `21.2.*` | Production WSGI server |
|
||||
| markdown | `3.5.*` | Rendering notes (`extra`, `codehilite`, `nl2br` and a bare-URL linking extension) |
|
||||
| feedgen | `1.0.*` | RSS 2.0 feed (`starpunk/feeds/rss.py`); Atom and JSON Feed are generated by StarPunk's own code |
|
||||
| httpx | `0.27.*` | All outbound HTTP: OIDC, IndieAuth client discovery, webmention and WebSub requests (through the SSRF-safe wrapper in `webmention_ssrf.py`), crossposting APIs, health check |
|
||||
| python-dotenv | `1.0.*` | Loading `.env` |
|
||||
| beautifulsoup4 | `4.12.*` | HTML parsing: IndieAuth client pages, crosspost text and link cards |
|
||||
| Flask | `3.1.*` | Web framework |
|
||||
| gunicorn | `26.2.*` | Production WSGI server |
|
||||
| markdown | `3.11.*` | Rendering notes (`extra`, `codehilite`, `nl2br` and a bare-URL linking extension) |
|
||||
| nh3 | `0.3.*` | HTML sanitising (rendered notes, webmention content) |
|
||||
| beautifulsoup4 | `4.15.*` | HTML parsing: IndieAuth client pages, crosspost text and link cards |
|
||||
| lxml | `6.1.*` | Outgoing link extraction, webmention endpoint discovery and source checks |
|
||||
| mf2py | `2.0.*` | Parsing microformats2 (webmention sources, reply contexts, author h-card) |
|
||||
| pytest | `8.0.*` | Test runner (listed here as well as in the image's test stage) |
|
||||
| psutil | `5.9.*` | Memory monitor and system metrics |
|
||||
| Pillow | `10.1.*` | Image validation, resizing and variants |
|
||||
| pillow-heif | `0.18.*` | HEIC/HEIF uploads (iPhone photos) |
|
||||
| nh3 | `0.2.*` | HTML sanitising (rendered notes, webmention content) |
|
||||
| PyJWT[crypto] | `2.10.*` | Verifying OIDC ID tokens (JWKS, RS/ES signatures via `cryptography`) |
|
||||
| python-dateutil | `4.0.*` | Dates in reply contexts |
|
||||
| feedgen | `1.0.*` | RSS 2.0 feed (`starpunk/feeds/rss.py`); Atom and JSON Feed are generated by StarPunk's own code |
|
||||
| httpx | `0.28.*` | All outbound HTTP; requests to user-supplied URLs go through `webmention_ssrf.safe_get`/`safe_post` |
|
||||
| PyJWT[crypto] | `2.15.*` | Verifying OIDC ID tokens (JWKS, RS/ES signatures via `cryptography`) |
|
||||
| python-dotenv | `1.2.*` | Loading `.env` |
|
||||
| Pillow | `13.0.*` | Image validation, resizing and variants |
|
||||
| pillow-heif | `1.8.*` | HEIC/HEIF uploads (iPhone photos) |
|
||||
|
||||
Packages imported directly but installed only as dependencies of the above:
|
||||
`lxml` (outgoing link extraction, webmention endpoint discovery and source checks; from feedgen) and `python-dateutil` (dates in
|
||||
reply contexts; from feedgen). `markupsafe` comes with Flask/Jinja2.
|
||||
`markupsafe` comes with Flask/Jinja2.
|
||||
|
||||
### Development dependencies (`requirements-dev.txt`)
|
||||
### Test and development dependencies
|
||||
|
||||
Includes `requirements.txt`, plus `pytest-cov`, `pytest-mock`, `black`,
|
||||
`flake8`, `mypy` and `gunicorn`. Coding standards are in
|
||||
[../standards/](../standards/).
|
||||
- `requirements-test.txt`: `pytest` 9.2 and `pytest-mock` 3.15, used by the
|
||||
Containerfile test stage (CI).
|
||||
- `requirements-dev.txt`: runtime and test requirements plus `pytest-cov`,
|
||||
`black`, `flake8`, `mypy` (not enforced) and `pip-audit`. Coding standards
|
||||
are in [../standards/](../standards/).
|
||||
|
||||
### Deliberately not used
|
||||
|
||||
@@ -94,7 +97,7 @@ Protocol details: [indieweb-protocols.md](indieweb-protocols.md),
|
||||
```
|
||||
|
||||
- **In CI**: the `test` stage of the `Containerfile` installs the production
|
||||
requirements plus pytest and pytest-mock and runs the whole suite. The
|
||||
requirements plus `requirements-test.txt` and runs the whole suite. The
|
||||
image is only built and pushed if it passes. Checklist:
|
||||
[../standards/testing-checklist.md](../standards/testing-checklist.md).
|
||||
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
# ADR-053: Performance Monitoring Strategy
|
||||
|
||||
## Status
|
||||
Accepted
|
||||
Superseded by ADR-072 (was: Accepted, v1.1.1)
|
||||
|
||||
> The metrics system was removed in v2.12.0. (status updated 2026-11-03)
|
||||
|
||||
## Context
|
||||
StarPunk v1.1.1 introduces performance monitoring to help operators understand system behavior in production. Currently, we have no visibility into:
|
||||
|
||||
@@ -2,7 +2,9 @@
|
||||
|
||||
## Status
|
||||
|
||||
Accepted (v1.1.1-rc.2, 2025-11-25)
|
||||
Superseded by ADR-072 (was: Accepted, v1.1.1-rc.2, 2025-11-25)
|
||||
|
||||
> The metrics dashboard was removed in v2.12.0. (status updated 2026-11-03)
|
||||
|
||||
> Reconstructed on 2026-11-03 from code, changelog and related documents;
|
||||
> the original was lost. The original
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
# ADR-072: Remove the metrics dashboard
|
||||
|
||||
## Status
|
||||
|
||||
Accepted (v2.12.0, 2026-11-03). Supersedes ADR-053 and ADR-060.
|
||||
|
||||
## Context
|
||||
|
||||
v1.1.1 and v1.1.2 added built-in performance monitoring (ADR-053): HTTP
|
||||
request timing middleware, a wrapper on every database connection that
|
||||
timed queries, a memory-sampling thread, feed-generation counters, an
|
||||
in-memory buffer of the results, and an HTML dashboard at
|
||||
`/admin/metrics-dashboard` with a JSON twin at `/admin/metrics` (ADR-060
|
||||
covered adapting the data for the dashboard template).
|
||||
|
||||
In practice it told us little:
|
||||
|
||||
- The buffer lives in memory in each of the four gunicorn processes, so the
|
||||
dashboard showed whichever process served the request, and everything was
|
||||
lost on restart or deploy.
|
||||
- Nothing in it was ever used to find or fix a problem. Logs (with
|
||||
correlation IDs), `/health` and `/admin/health` were.
|
||||
- It cost something on every request and every query, kept a thread per
|
||||
process, and needed `psutil`, about 1,300 lines of code and three test
|
||||
files.
|
||||
|
||||
## Decision
|
||||
|
||||
Remove the dashboard and everything that only fed it: the `starpunk/monitoring`
|
||||
package, the HTTP metrics middleware (including its `X-Request-ID`
|
||||
response header), `MonitoredConnection`, the memory monitor, the
|
||||
feed-generation tracking calls, `/admin/metrics-dashboard`,
|
||||
`/admin/metrics`, the `METRICS_*` settings and the `psutil` dependency.
|
||||
|
||||
Keep:
|
||||
|
||||
- `/health` and `/health?detailed=true`;
|
||||
- `/admin/health`, without its `metrics` section (database, pool
|
||||
statistics, filesystem and disk checks stay);
|
||||
- connection pool statistics (`get_pool_stats`) and the feed cache, which
|
||||
do work of their own;
|
||||
- structured logs with correlation IDs (ADR-054).
|
||||
|
||||
## Consequences
|
||||
|
||||
- Less code and one fewer dependency; no timing work on each request or
|
||||
query; one fewer thread per process.
|
||||
- Slow database queries are no longer logged. If that is ever needed, a
|
||||
query timer can be added at the pool level, or an external tool used.
|
||||
- `METRICS_*` environment variables are ignored if still set.
|
||||
- Bookmarks to `/admin/metrics-dashboard` now get a 404.
|
||||
@@ -5,8 +5,8 @@ records one decision: its context, what was decided, and the consequences.
|
||||
Superseded ADRs are kept as history; their Status line names the ADR that
|
||||
replaced them.
|
||||
|
||||
**Total ADR files**: 67 (66 numbers: ADR-001 to ADR-044 and ADR-050 to
|
||||
ADR-071, with two files numbered ADR-012). See [Numbering](#numbering).
|
||||
**Total ADR files**: 68 (67 numbers: ADR-001 to ADR-044 and ADR-050 to
|
||||
ADR-072, with two files numbered ADR-012). See [Numbering](#numbering).
|
||||
|
||||
Status values: **Accepted** (in force), **Superseded** (replaced by a later
|
||||
ADR), **Deprecated** (withdrawn, nothing replaces it; none at present),
|
||||
@@ -101,10 +101,9 @@ See also [IndieWeb protocols](../architecture/indieweb-protocols.md).
|
||||
|---|---|---|---|
|
||||
| [012](ADR-012-http-error-handling-policy.md) | HTTP Error Handling Policy | Accepted | Correct HTTP status codes (e.g. 404 for missing notes) across routes. |
|
||||
| [052](ADR-052-configuration-system-architecture.md) | Configuration System Architecture | Accepted | Environment variables with defaults, loaded in one module. |
|
||||
| [053](ADR-053-performance-monitoring-strategy.md) | Performance Monitoring Strategy | Accepted | Built-in, standard-library performance metrics (v1.1.1). |
|
||||
| [054](ADR-054-structured-logging-architecture.md) | Structured Logging Architecture | Accepted | Rotating log files with correlation IDs (v1.1.1). |
|
||||
| [055](ADR-055-error-handling-philosophy.md) | Error Handling Philosophy | Accepted | Layered handling: graceful degradation, clear user messages, detailed logs. |
|
||||
| [060](ADR-060-production-hotfix-metrics-dashboard.md) | Route Adapter for the Metrics Dashboard | Accepted | Adapt monitoring data to the dashboard template in the route (v1.1.1-rc.2). *Reconstructed.* |
|
||||
| [072](ADR-072-remove-metrics-dashboard.md) | Remove the metrics dashboard | Accepted | Drop the in-memory metrics system and dashboard; keep `/health`, `/admin/health` and logs (v2.12.0). |
|
||||
|
||||
See also [deployment](../architecture/deployment.md).
|
||||
|
||||
@@ -117,6 +116,8 @@ designs of v0.x–v1.6.0, all replaced by ADR-064 in v2.0.0.
|
||||
|---|---|---|---|
|
||||
| [005](ADR-005-indielogin-authentication.md) | IndieLogin Authentication Integration | Superseded by 064 | Admin login through indielogin.com. |
|
||||
| [007](ADR-007-slug-generation-algorithm.md) | Slug Generation Algorithm | Superseded by 062 | Slugs from the first words of the note. |
|
||||
| [053](ADR-053-performance-monitoring-strategy.md) | Performance Monitoring Strategy | Superseded by 072 | Built-in, standard-library performance metrics (v1.1.1). |
|
||||
| [060](ADR-060-production-hotfix-metrics-dashboard.md) | Route Adapter for the Metrics Dashboard | Superseded by 072 | Adapt monitoring data to the dashboard template in the route (v1.1.1-rc.2). *Reconstructed.* |
|
||||
| [016](ADR-016-indieauth-client-discovery.md) | IndieAuth Client Discovery Mechanism | Superseded by 019 | h-app markup for client discovery. |
|
||||
| [017](ADR-017-oauth-client-metadata-document.md) | OAuth Client ID Metadata Document Implementation | Superseded by 019 | OAuth client metadata endpoint. |
|
||||
| [018](ADR-018-indieauth-detailed-logging.md) | IndieAuth Detailed Logging Strategy | Superseded by 064 | Redacted HTTP logging of the IndieAuth login flow. |
|
||||
@@ -166,7 +167,7 @@ was carried out and is not itself superseded.
|
||||
|
||||
## How to create a new ADR
|
||||
|
||||
1. Use the next number after the highest existing one (currently ADR-072).
|
||||
1. Use the next number after the highest existing one (currently ADR-073).
|
||||
2. Name it `ADR-NNN-brief-descriptive-title.md`.
|
||||
3. Follow the format of recent ADRs (e.g. ADR-070, ADR-071):
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ is in `../architecture/`; decisions are in `../decisions/` (ADRs).
|
||||
|
||||
## Contents
|
||||
|
||||
[v1.0.0](#v100) · [v1.1.1](#v111) · [v1.1.2](#v112) · [v1.2.0](#v120) · [v1.3.0](#v130) · [v1.3.1](#v131) · [v1.4.0](#v140) · [v1.4.1](#v141) · [v1.4.2](#v142) · [v1.5.0](#v150) · [v1.6.0](#v160) · [v2.0.0](#v200) · [v2.0.1](#v201) · [v2.0.2](#v202) · [v2.1.0](#v210) · [v2.2.0](#v220) · [v2.3.0](#v230) · [v2.3.1](#v231) · [v2.3.2](#v232) · [v2.3.3](#v233) · [v2.4.0](#v240) · [v2.5.0](#v250) · [v2.6.0](#v260) · [v2.6.1](#v261) · [v2.6.2](#v262) · [v2.6.3](#v263) · [v2.6.4](#v264) · [v2.6.5](#v265) · [v2.6.6](#v266) · [v2.6.7](#v267) · [v2.6.8](#v268) · [v2.6.9](#v269) · [v2.7.0](#v270) · [v2.7.1](#v271) · [v2.7.2](#v272) · [v2.8.0](#v280) · [v2.8.1](#v281) · [v2.9.0](#v290) · [v2.10.0](#v2100) · [v2.10.1](#v2101) · [v2.10.2](#v2102) · [hotfix](#hotfix)
|
||||
[v1.0.0](#v100) · [v1.1.1](#v111) · [v1.1.2](#v112) · [v1.2.0](#v120) · [v1.3.0](#v130) · [v1.3.1](#v131) · [v1.4.0](#v140) · [v1.4.1](#v141) · [v1.4.2](#v142) · [v1.5.0](#v150) · [v1.6.0](#v160) · [v2.0.0](#v200) · [v2.0.1](#v201) · [v2.0.2](#v202) · [v2.1.0](#v210) · [v2.2.0](#v220) · [v2.3.0](#v230) · [v2.3.1](#v231) · [v2.3.2](#v232) · [v2.3.3](#v233) · [v2.4.0](#v240) · [v2.5.0](#v250) · [v2.6.0](#v260) · [v2.6.1](#v261) · [v2.6.2](#v262) · [v2.6.3](#v263) · [v2.6.4](#v264) · [v2.6.5](#v265) · [v2.6.6](#v266) · [v2.6.7](#v267) · [v2.6.8](#v268) · [v2.6.9](#v269) · [v2.7.0](#v270) · [v2.7.1](#v271) · [v2.7.2](#v272) · [v2.8.0](#v280) · [v2.8.1](#v281) · [v2.9.0](#v290) · [v2.10.0](#v2100) · [v2.10.1](#v2101) · [v2.10.2](#v2102) · [v2.11.0](#v2110) · [v2.12.0](#v2120) · [hotfix](#hotfix)
|
||||
|
||||
## v1.0.0
|
||||
|
||||
@@ -410,6 +410,14 @@ OIDC admin login and built-in IndieAuth server.
|
||||
|
||||
- [2026-11-03-reply-context-fetch.md](v2.10.2/2026-11-03-reply-context-fetch.md) - Reply contexts fetched again (fetch_source returns three values since v2.7.0)
|
||||
|
||||
## v2.11.0
|
||||
|
||||
- [2026-11-03-maintenance-release.md](v2.11.0/2026-11-03-maintenance-release.md) - Dependency upgrades (Python 3.14), foreign keys, CSRF, fetchers, debug default, shared rate limits
|
||||
|
||||
## v2.12.0
|
||||
|
||||
- [2026-11-03-remove-metrics.md](v2.12.0/2026-11-03-remove-metrics.md) - Metrics dashboard and `starpunk/monitoring` removed (ADR-072)
|
||||
|
||||
## hotfix
|
||||
|
||||
Not a release folder.
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
# v2.11.0: Maintenance release
|
||||
|
||||
Dependency upgrades and the five High items in the backlog (all re-checked
|
||||
against the code first and still present).
|
||||
|
||||
## Dependencies
|
||||
|
||||
`pip-audit` on the v2.10.2 set found 97 published vulnerabilities in eight
|
||||
packages (Pillow 64, PyJWT 22, gunicorn 4, Flask 2, Markdown 2,
|
||||
python-dotenv 2, pytest 2, pillow-heif 1). All direct dependencies were moved
|
||||
to their current minor series (`requirements.txt`); the new set has no known
|
||||
vulnerabilities. Python moved from 3.11 to 3.14 (`python:3.14-slim`); the
|
||||
whole suite passes with `-W error::DeprecationWarning`.
|
||||
|
||||
- `requirements-test.txt` (pytest, pytest-mock) is installed by the
|
||||
Containerfile test stage; pytest left the runtime requirements.
|
||||
- `lxml` and `python-dateutil` are pinned (used directly, previously only
|
||||
transitive). `uv` is pinned to 0.9.28 in the Containerfile.
|
||||
- `datetime.utcnow()` (34 calls in `starpunk/`, more in tests) became
|
||||
`datetime.now(timezone.utc).replace(tzinfo=None)`: same naive-UTC values.
|
||||
sqlite3's default datetime adapters, deprecated in 3.12, are registered
|
||||
explicitly in `starpunk/database/__init__.py` with the same output.
|
||||
|
||||
## High backlog items
|
||||
|
||||
1. **Foreign keys.** `PRAGMA foreign_keys=ON` on every pooled connection
|
||||
(`database/pool.py`); all deletes of notes, media and WebSub subscriptions
|
||||
go through the pool. Every foreign key is `CASCADE` or `SET NULL`, so no
|
||||
delete can be blocked. Migration 018 removes orphans (production: three
|
||||
`note_tags` rows from hard-deleted notes). Crosspost rows still have no
|
||||
foreign key on purpose.
|
||||
2. **CSRF.** `starpunk/csrf.py`: token = HMAC-SHA256(SECRET_KEY,
|
||||
"csrf:" + starpunk_session cookie); `before_request` checks the
|
||||
`csrf_token` field or `X-CSRF-Token` header on every admin POST,
|
||||
`auth.logout` and `indieauth.approve`; 400 otherwise. All 12 POST forms
|
||||
carry it. `CSRF_ENABLED` (default true); test apps turn it off unless they
|
||||
set it.
|
||||
3. **Fetchers.** `indieauth.fetch_client_redirect_uris` and
|
||||
`syndication._fetch_public` use `webmention_ssrf.safe_get` (per-hop
|
||||
checks, DNS pinning, hard deadline). `safe_get` gained `truncate` for
|
||||
reads that stop at the size limit.
|
||||
4. **Debug default.** `FLASK_DEBUG` defaults to 0 and `FLASK_ENV` to
|
||||
production; the runtime image sets both.
|
||||
5. **Shared rate limits.** `SharedRateLimiter` keeps sliding-window counters
|
||||
in `rate_limits` (migration 019): current window plus the previous one
|
||||
weighted by overlap. Used by `/webmention` (per IP and per source domain)
|
||||
and `/websub`; old windows are pruned every 200 calls.
|
||||
|
||||
## Housekeeping
|
||||
|
||||
`compose.yaml` no longer pins 0.6.0 or sets `VERSION`/`ENVIRONMENT`; the
|
||||
unused root `dev_auth.py` and the committed `test.ini` were removed.
|
||||
|
||||
## Tests
|
||||
|
||||
New: `tests/test_csrf.py`, `tests/test_foreign_keys.py`,
|
||||
`tests/test_shared_rate_limit.py`, IndieAuth client-metadata tests through the
|
||||
fetcher (including a redirect to a private address). Crosspost tests route
|
||||
link-card fetches through the fetcher's transport. Full suite: 1264 passed
|
||||
on Python 3.14.
|
||||
@@ -0,0 +1,28 @@
|
||||
# v2.12.0: Remove the metrics dashboard
|
||||
|
||||
Decision: [ADR-072](../../decisions/ADR-072-remove-metrics-dashboard.md).
|
||||
|
||||
## Removed
|
||||
|
||||
| What | Where |
|
||||
|---|---|
|
||||
| Metrics buffer, HTTP middleware, database timing wrapper, memory monitor, business metrics | `starpunk/monitoring/` (deleted) |
|
||||
| HTTP middleware and memory monitor start-up | `starpunk/__init__.py` |
|
||||
| `MonitoredConnection` wrapping, `slow_query_threshold` and `metrics_enabled` arguments | `starpunk/database/pool.py` |
|
||||
| `track_feed_generated` calls and their timing code | `starpunk/feeds/atom.py`, `json_feed.py`, `rss.py` |
|
||||
| `/admin/metrics-dashboard`, `/admin/metrics`, `transform_metrics_for_template`, the `metrics` block of `/admin/health` | `starpunk/routes/admin.py` |
|
||||
| Dashboard template and the admin nav link | `templates/admin/metrics_dashboard.html`, `templates/admin/base.html` |
|
||||
| `METRICS_ENABLED`, `METRICS_SLOW_QUERY_THRESHOLD`, `METRICS_SAMPLING_RATE`, `METRICS_BUFFER_SIZE`, `METRICS_MEMORY_INTERVAL` | `starpunk/config.py`, `.env.example` |
|
||||
| `psutil` | `requirements.txt` |
|
||||
| Tests | `tests/test_monitoring.py`, `tests/test_monitoring_feed_statistics.py`, `tests/test_admin_feed_statistics.py` |
|
||||
|
||||
## Kept
|
||||
|
||||
`/health`, `/health?detailed=true`, `/admin/health` (database, pool,
|
||||
filesystem, disk), pool statistics, the feed cache and its own hit/miss
|
||||
counters, and correlation IDs in logs.
|
||||
|
||||
## Upgrade
|
||||
|
||||
No migration. `METRICS_*` variables can be deleted from the environment;
|
||||
they are ignored if left.
|
||||
@@ -1,9 +1,9 @@
|
||||
# StarPunk Backlog
|
||||
|
||||
**Last Updated**: 2026-11-03 (current release: v2.10.1)
|
||||
**Last Updated**: 2026-11-03 (current release: v2.11.0)
|
||||
|
||||
These are the open features and bugs, sorted by priority. Each item was checked against the code
|
||||
at v2.10.1. Work that has shipped is listed per release in `vX.Y.Z/RELEASE.md` and in
|
||||
at v2.11.0. Work that has shipped is listed per release in `vX.Y.Z/RELEASE.md` and in
|
||||
`CHANGELOG.md`.
|
||||
|
||||
## Priority Levels
|
||||
@@ -23,30 +23,7 @@ at v2.10.1. Work that has shipped is listed per release in `vX.Y.Z/RELEASE.md` a
|
||||
|
||||
## High
|
||||
|
||||
### SQLite foreign keys are not enforced (ON DELETE CASCADE is inert)
|
||||
- **Description**: No connection runs `PRAGMA foreign_keys = ON`, so the `ON DELETE CASCADE` clauses in the schema do nothing. When a note is deleted from the admin (a hard delete), its `note_tags`, `note_media`, `syndications` and webmention rows stay behind. `delete_media()` relies on the cascade to remove `media_variants` and `note_media` rows, so those are left orphaned too. Before turning the pragma on, check for existing orphan rows and add a migration that cleans them up. Each cascade also needs tests.
|
||||
- **Where**: `starpunk/database/` (connection setup/pool), `starpunk/notes.py` (`delete_note`, hard delete), `starpunk/media.py` (`delete_media`), `migrations/007`–`010`, `016`
|
||||
- **Source**: v2.10.x review; confirmed in code (there is no `foreign_keys` pragma anywhere in `starpunk/`)
|
||||
|
||||
### No CSRF tokens on admin and consent forms
|
||||
- **Description**: Admin forms (create/edit/delete notes, webmention moderation, token revocation) and the IndieAuth consent form rely only on the `SameSite=Lax` session cookie. Lax blocks cross-site POSTs in current browsers, but a per-session CSRF token is the standard defence and covers older clients and same-site subdomains.
|
||||
- **Where**: `starpunk/routes/admin.py`, `starpunk/routes/indieauth.py`, `templates/admin/*`
|
||||
- **Source**: Documentation refresh 2026-11-03 (architecture review)
|
||||
|
||||
### Two outbound fetches don't use the hardened fetcher
|
||||
- **Description**: IndieAuth client-metadata discovery (`indieauth.fetch_client_redirect_uris`) checks only the first URL, then uses `httpx.get(follow_redirects=True)`: redirect hops aren't checked and DNS isn't pinned. Crosspost link cards and Threads image fetches (`syndication._fetch_public`) check each hop but don't pin DNS, and their 10 s timeout is per read. Move both to `webmention_ssrf.safe_get`, as Webmention, WebSub and Micropub photo downloads already do.
|
||||
- **Where**: `starpunk/indieauth.py`, `starpunk/syndication.py`
|
||||
- **Source**: Documentation refresh 2026-11-03
|
||||
|
||||
### `FLASK_DEBUG` defaults to on
|
||||
- **Description**: `config.py` reads `FLASK_DEBUG` with default `"1"` (and `FLASK_ENV` defaults to development); the runtime image sets neither. Production is safe only because the deployment sets `FLASK_DEBUG=0`. Default to off, or set it in the `Containerfile` runtime stage.
|
||||
- **Where**: `starpunk/config.py`, `Containerfile`
|
||||
- **Source**: Documentation refresh 2026-11-03 (production verified: DEBUG is False there)
|
||||
|
||||
### Rate limiters are per process
|
||||
- **Description**: `RateLimiter` keeps its counters in memory, and gunicorn runs 4 workers. The WebSub hub limit of 30/min is therefore really 4×30/min, and the webmention endpoint limit is multiplied the same way. Consider a counter stored in SQLite that all workers share.
|
||||
- **Where**: `starpunk/rate_limiter.py`, `starpunk/routes/websub.py`, `starpunk/routes/webmention.py`, `Containerfile` (`--workers 4`)
|
||||
- **Source**: ADR-071 (Consequences: rate limiting bounds subscriber-chosen requests); v2.10.1 review
|
||||
None open.
|
||||
|
||||
---
|
||||
|
||||
@@ -92,21 +69,11 @@ at v2.10.1. Work that has shipped is listed per release in `vX.Y.Z/RELEASE.md` a
|
||||
- **Where**: `starpunk/webmention_processor.py`, `starpunk/webmention_worker.py`
|
||||
- **Source**: v1.6.0 design reconstruction (docs/design/v1.6.0/background-worker.md)
|
||||
|
||||
### Webmention rate limiter never cleaned up
|
||||
- **Description**: `cleanup_rate_limiters()` is never called, so per-IP and per-domain keys accumulate for the life of each process.
|
||||
- **Where**: `starpunk/rate_limiter.py`, `starpunk/routes/webmention.py`
|
||||
- **Source**: v1.6.0 design reconstruction
|
||||
|
||||
### Atom and JSON Feed not linked from pages
|
||||
- **Description**: `base.html` advertises only `/feed` (as RSS) and OPML with `rel=alternate`; add `<link rel="alternate">` for `/feed.atom` and `/feed.json` so readers can pick a format.
|
||||
- **Where**: `templates/base.html`
|
||||
- **Source**: Documentation refresh 2026-11-03
|
||||
|
||||
### Housekeeping in the repository
|
||||
- **Description**: `compose.yaml` is stale (`image: starpunk:0.6.0`, `VERSION=0.6.0` would override the real version, unused `ENVIRONMENT`); the root `dev_auth.py` duplicates `starpunk/dev_auth.py` and is unused; `LICENSE` says "Copyright (c) 2024 [Your Name]"; `test.ini` contains a private host name; `lxml` and `python-dateutil` are used directly but only installed as dependencies of other packages, while `pytest` is pinned in the production `requirements.txt`.
|
||||
- **Where**: `compose.yaml`, `dev_auth.py`, `LICENSE`, `test.ini`, `requirements*.txt`
|
||||
- **Source**: Documentation refresh 2026-11-03
|
||||
|
||||
### Backfeed responses to crossposts
|
||||
- **Description**: Idea: bring likes, boosts and replies to StarPunk's own crossposts back to the note as responses. Two options: poll GoToSocial (the token would need a `read` scope), or rely on Bridgy, which sends webmentions. Since v2.8.0, StarPunk ignores webmentions *from* a note's own crossposts, so that rule would need to change to work with Bridgy.
|
||||
- **Where**: `starpunk/syndication.py`, `starpunk/webmention_worker.py`
|
||||
@@ -213,6 +180,10 @@ at v2.10.1. Work that has shipped is listed per release in `vX.Y.Z/RELEASE.md` a
|
||||
|
||||
These items were removed from the backlog because they have shipped:
|
||||
|
||||
- LICENSE: MIT, copyright Phil Skentelbery 2025-2026 (2026-11-03)
|
||||
|
||||
- v2.11.0: SQLite foreign keys enforced (orphans cleaned up by migration 018); CSRF tokens on admin, logout and consent forms; IndieAuth client and crosspost link-card fetches moved onto the SSRF-safe fetcher; `FLASK_DEBUG`/`FLASK_ENV` default to off/production; rate limits shared by all processes (and old windows pruned, so the webmention limiter no longer grows); dependencies upgraded (Python 3.14, no known vulnerabilities); stale `compose.yaml`, duplicate `dev_auth.py` and `test.ini` removed or fixed; `lxml`/`python-dateutil` pinned, `pytest` out of the runtime requirements
|
||||
|
||||
- POSSE: GoToSocial (v2.1.0), Bluesky (v2.2.0), Threads (v2.6.0)
|
||||
- Webmentions: receive, send, display and moderation (v1.6.0); fixed and made spec-compliant in v2.6.8–v2.7.2
|
||||
- Reply contexts, replies, likes, reposts and bookmarks (v2.4.0, v2.5.0, v2.9.0)
|
||||
|
||||
@@ -4,7 +4,7 @@ Future work and per-release plans. See `../DOCUMENTATION.md` (PROJECTPLAN).
|
||||
|
||||
## Backlog
|
||||
|
||||
- **[BACKLOG.md](BACKLOG.md)** - Open features and bugs by priority (updated for v2.10.1)
|
||||
- **[BACKLOG.md](BACKLOG.md)** - Open features and bugs by priority (updated for v2.11.0)
|
||||
|
||||
## Releases
|
||||
|
||||
@@ -47,3 +47,5 @@ Future work and per-release plans. See `../DOCUMENTATION.md` (PROJECTPLAN).
|
||||
- **[v2.10.0](v2.10.0/RELEASE.md)** - WebSub (Released 2026-11-03)
|
||||
- **[v2.10.1](v2.10.1/RELEASE.md)** - WebSub hardening (Released 2026-11-03)
|
||||
- **[v2.10.2](v2.10.2/RELEASE.md)** - Reply context fetch fix (Released 2026-11-03)
|
||||
- **[v2.11.0](v2.11.0/RELEASE.md)** - Maintenance: dependency upgrades, High backlog items (Released 2026-11-03)
|
||||
- **[v2.12.0](v2.12.0/RELEASE.md)** - Metrics dashboard removed (Released 2026-11-03)
|
||||
@@ -0,0 +1,28 @@
|
||||
# v2.11.0
|
||||
|
||||
- **Date**: 2026-11-03
|
||||
- **Status**: Released
|
||||
- **Focus**: Maintenance: dependency upgrades and the High backlog items
|
||||
|
||||
## Security
|
||||
|
||||
- All dependencies upgraded (97 known vulnerabilities before, none after); Python 3.14
|
||||
- CSRF tokens on admin forms, logout and the IndieAuth consent
|
||||
- IndieAuth client and crosspost link-card fetches use the SSRF-safe fetcher
|
||||
- Debug off and production environment by default
|
||||
|
||||
## Changes
|
||||
|
||||
- Rate limits shared by all server processes
|
||||
- `requirements-test.txt`; pytest out of the production image; `lxml` and `python-dateutil` pinned; `uv` pinned
|
||||
|
||||
## Bugs fixed
|
||||
|
||||
- Foreign keys enforced, so deletes cascade; orphan rows removed (migration 018)
|
||||
- Deprecated `datetime.utcnow()` and sqlite3 adapters replaced
|
||||
- Stale `compose.yaml`, duplicate `dev_auth.py` and `test.ini` cleaned up
|
||||
|
||||
## Related
|
||||
|
||||
- [CHANGELOG](../../../CHANGELOG.md)
|
||||
- [Design note](../../design/v2.11.0/2026-11-03-maintenance-release.md)
|
||||
@@ -0,0 +1,21 @@
|
||||
# v2.12.0
|
||||
|
||||
- **Date**: 2026-11-03
|
||||
- **Status**: Released
|
||||
- **Focus**: Remove the metrics dashboard
|
||||
|
||||
## Removed
|
||||
|
||||
- Admin metrics dashboard (`/admin/metrics-dashboard`) and `/admin/metrics`
|
||||
- The `starpunk/monitoring` package: request timing, query timing, memory monitor thread, feed counters
|
||||
- `METRICS_*` settings and the `psutil` dependency
|
||||
|
||||
## Kept
|
||||
|
||||
- `/health`, `/admin/health` (without the metrics section), logs with correlation IDs
|
||||
|
||||
## Related
|
||||
|
||||
- [CHANGELOG](../../../CHANGELOG.md)
|
||||
- [ADR-072](../../decisions/ADR-072-remove-metrics-dashboard.md)
|
||||
- [Design note](../../design/v2.12.0/2026-11-03-remove-metrics.md)
|
||||
@@ -9,7 +9,7 @@ deployment is in [architecture/deployment.md](../architecture/deployment.md).
|
||||
|
||||
## Requirements
|
||||
|
||||
- Python 3.11 (the container image uses `python:3.11-slim`)
|
||||
- Python 3.14 (the container image uses `python:3.14-slim`)
|
||||
- [uv](https://docs.astral.sh/uv/) for the virtual environment and packages ([ADR-006](../decisions/ADR-006-python-virtual-environment-uv.md))
|
||||
- git
|
||||
- Optional: Docker or Podman, to build the image and run the CI test stage locally
|
||||
@@ -23,7 +23,7 @@ pytest-mock, black, flake8, mypy).
|
||||
```bash
|
||||
git clone https://git.thesatelliteoflove.com/phil/StarPunk.git
|
||||
cd StarPunk
|
||||
uv venv --python 3.11 # creates .venv/ (gitignored)
|
||||
uv venv --python 3.14 # creates .venv/ (gitignored)
|
||||
uv pip install -r requirements-dev.txt
|
||||
```
|
||||
|
||||
@@ -84,10 +84,9 @@ banner to every page. It sets the `starpunk_session` cookie for
|
||||
server.
|
||||
|
||||
Each app process also starts background threads: the webmention worker,
|
||||
the WebSub hub worker, the memory monitor and, when a crossposting target
|
||||
is configured, the syndication worker. Turn off what you don't need locally
|
||||
with `WEBMENTION_WORKER_ENABLED=false`, `WEBSUB_ENABLED=false` and
|
||||
`METRICS_ENABLED=false`. Leave `GOTOSOCIAL_*`, `BLUESKY_*` and `THREADS_*`
|
||||
the WebSub hub worker and, when a crossposting target is configured, the
|
||||
syndication worker. Turn off what you don't need locally with
|
||||
`WEBMENTION_WORKER_ENABLED=false` and `WEBSUB_ENABLED=false`. Leave `GOTOSOCIAL_*`, `BLUESKY_*` and `THREADS_*`
|
||||
empty unless you mean to post to those accounts.
|
||||
|
||||
Production runs the same app under gunicorn (`gunicorn app:app`, see `CMD`
|
||||
@@ -112,7 +111,7 @@ SESSION_SECRET=test uv run pytest tests/test_micropub.py -k delete -x
|
||||
`ADMIN_ME=https://test.example.com`; set them too to reproduce CI exactly.
|
||||
- `tests/conftest.py` builds each app with `TESTING=True` and a temporary
|
||||
`DATA_PATH` and database, so tests never touch `./data`. Under
|
||||
`TESTING=True` the background workers and memory monitor don't start and
|
||||
`TESTING=True` the background workers don't start and
|
||||
the production OIDC check is skipped.
|
||||
- `tests/test_migration_race_condition.py` is slow (about 5 minutes). While
|
||||
iterating, add `--ignore=tests/test_migration_race_condition.py`; run the
|
||||
|
||||
@@ -63,7 +63,7 @@ cd ~/Documents/projects/StarPunk # main checkout, on main
|
||||
git pull
|
||||
git worktree add ../StarPunk-worktrees/v2.10.1 -b fix/v2.10.1-websub-hardening main
|
||||
cd ../StarPunk-worktrees/v2.10.1
|
||||
uv venv --python 3.11 && uv pip install -r requirements-dev.txt
|
||||
uv venv --python 3.14 && uv pip install -r requirements-dev.txt
|
||||
```
|
||||
|
||||
### 2. Make the change
|
||||
|
||||
@@ -10,7 +10,7 @@ out." Patterns specific to small helper functions are in
|
||||
|
||||
## Base style
|
||||
|
||||
- [PEP 8](https://peps.python.org/pep-0008/) naming and layout, Python 3.11.
|
||||
- [PEP 8](https://peps.python.org/pep-0008/) naming and layout, Python 3.14. Use `datetime.now(timezone.utc)`, not the deprecated `datetime.utcnow()`.
|
||||
- **Match the file you are editing.** 4-space indents everywhere; quote
|
||||
style varies between modules (newer code uses double quotes); keep lines
|
||||
to roughly 100 characters.
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
-- Migration 018: Remove rows orphaned while foreign keys weren't enforced (v2.11.0)
|
||||
-- From v2.11.0 every pooled connection runs PRAGMA foreign_keys=ON, so these
|
||||
-- cascades happen on delete. This removes what earlier deletes left behind.
|
||||
|
||||
DELETE FROM note_tags WHERE note_id NOT IN (SELECT id FROM notes);
|
||||
DELETE FROM note_tags WHERE tag_id NOT IN (SELECT id FROM tags);
|
||||
DELETE FROM note_media WHERE note_id NOT IN (SELECT id FROM notes);
|
||||
DELETE FROM note_media WHERE media_id NOT IN (SELECT id FROM media);
|
||||
DELETE FROM media_variants WHERE media_id NOT IN (SELECT id FROM media);
|
||||
DELETE FROM webmention_queue WHERE source_note_id IS NOT NULL AND source_note_id NOT IN (SELECT id FROM notes);
|
||||
UPDATE webmentions SET target_note_id = NULL WHERE target_note_id IS NOT NULL AND target_note_id NOT IN (SELECT id FROM notes);
|
||||
DELETE FROM websub_deliveries WHERE subscription_id IS NOT NULL AND subscription_id NOT IN (SELECT id FROM websub_subscriptions);
|
||||
@@ -0,0 +1,9 @@
|
||||
-- Migration 019: Rate limit counters shared by all server processes (v2.11.0)
|
||||
-- One row per key and time window; see starpunk/rate_limiter.py SharedRateLimiter.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS rate_limits (
|
||||
key TEXT NOT NULL,
|
||||
window INTEGER NOT NULL,
|
||||
count INTEGER NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (key, window)
|
||||
);
|
||||
+7
-15
@@ -1,17 +1,9 @@
|
||||
# StarPunk Development Dependencies
|
||||
# Includes code quality and testing tools
|
||||
|
||||
# Include production dependencies
|
||||
# StarPunk development dependencies
|
||||
-r requirements.txt
|
||||
-r requirements-test.txt
|
||||
|
||||
# Testing
|
||||
pytest-cov>=5.0.0 # Test coverage reporting
|
||||
pytest-mock>=3.12.0 # Mocking for tests
|
||||
|
||||
# Code Quality
|
||||
black>=24.0.0 # Code formatting
|
||||
flake8>=7.0.0 # Linting
|
||||
mypy>=1.8.0 # Type checking
|
||||
|
||||
# WSGI Server (for production-like testing)
|
||||
gunicorn>=21.2.0
|
||||
pytest-cov==7.* # Coverage reports
|
||||
black==26.* # Formatting (not enforced, see docs/standards)
|
||||
flake8==7.* # Linting (not enforced)
|
||||
mypy==1.* # Type checking (not enforced)
|
||||
pip-audit==2.* # Known-vulnerability check of the dependency set
|
||||
@@ -0,0 +1,3 @@
|
||||
# Test dependencies: used by the Containerfile test stage and CI (v2.11.0)
|
||||
pytest==9.2.*
|
||||
pytest-mock==3.15.*
|
||||
+19
-37
@@ -1,42 +1,24 @@
|
||||
# StarPunk Production Dependencies
|
||||
# Python 3.11+ required
|
||||
# StarPunk runtime dependencies (v2.11.0)
|
||||
# Pinned to a minor series; see docs/architecture/technology-stack.md.
|
||||
|
||||
# Web Framework
|
||||
Flask==3.0.*
|
||||
# Web framework and server
|
||||
Flask==3.1.*
|
||||
gunicorn==26.2.*
|
||||
|
||||
# WSGI Server (Production)
|
||||
gunicorn==21.2.*
|
||||
|
||||
# Content Processing
|
||||
markdown==3.5.*
|
||||
|
||||
# Feed Generation
|
||||
# Content
|
||||
markdown==3.11.*
|
||||
nh3==0.3.*
|
||||
beautifulsoup4==4.15.*
|
||||
lxml==6.1.*
|
||||
mf2py==2.0.*
|
||||
python-dateutil==4.0.*
|
||||
feedgen==1.0.*
|
||||
|
||||
# HTTP Client (for IndieAuth)
|
||||
httpx==0.27.*
|
||||
# HTTP and auth
|
||||
httpx==0.28.*
|
||||
PyJWT[crypto]==2.15.*
|
||||
python-dotenv==1.2.*
|
||||
|
||||
# Configuration Management
|
||||
python-dotenv==1.0.*
|
||||
|
||||
# HTML Parsing (for IndieAuth endpoint discovery)
|
||||
beautifulsoup4==4.12.*
|
||||
|
||||
# Microformats2 Parsing (v1.2.0)
|
||||
mf2py==2.0.*
|
||||
|
||||
# Testing Framework
|
||||
pytest==8.0.*
|
||||
|
||||
# System Monitoring (v1.1.2)
|
||||
psutil==5.9.*
|
||||
|
||||
# Image Processing (v1.2.0, updated v1.4.2 for HEIC support)
|
||||
Pillow==10.1.*
|
||||
|
||||
# HEIC/HEIF Support (v1.4.2 - iPhone photos)
|
||||
pillow-heif==0.18.*
|
||||
|
||||
# HTML Sanitization (v1.6.0 - Webmention content)
|
||||
nh3==0.2.*
|
||||
PyJWT[crypto]==2.10.*
|
||||
# Images
|
||||
Pillow==13.0.*
|
||||
pillow-heif==1.8.*
|
||||
+6
-19
@@ -142,12 +142,6 @@ def create_app(config=None):
|
||||
# Initialize connection pool
|
||||
init_pool(app)
|
||||
|
||||
# Setup HTTP metrics middleware (v1.1.2 Phase 1)
|
||||
if app.config.get('METRICS_ENABLED', True):
|
||||
from starpunk.monitoring import setup_http_metrics
|
||||
setup_http_metrics(app)
|
||||
app.logger.info("HTTP metrics middleware enabled")
|
||||
|
||||
# Initialize feed cache (v1.1.2 Phase 3)
|
||||
if app.config.get('FEED_CACHE_ENABLED', True):
|
||||
from starpunk.feeds import configure_cache
|
||||
@@ -309,6 +303,10 @@ def create_app(config=None):
|
||||
response.headers.add('Link', '</webmention>; rel="webmention"')
|
||||
return response
|
||||
|
||||
# CSRF tokens on admin and consent forms (v2.11.0)
|
||||
from starpunk import csrf
|
||||
csrf.init_app(app)
|
||||
|
||||
# WebSub discovery: Link rel=hub and rel=self on topics (v2.10.0)
|
||||
from starpunk.websub import add_discovery_headers, template_links
|
||||
app.after_request(add_discovery_headers)
|
||||
@@ -319,17 +317,6 @@ def create_app(config=None):
|
||||
|
||||
register_error_handlers(app)
|
||||
|
||||
# Start memory monitor thread (v1.1.2 Phase 1)
|
||||
# Per CQ5: Skip in test mode
|
||||
if app.config.get('METRICS_ENABLED', True) and not app.config.get('TESTING', False):
|
||||
from starpunk.monitoring import MemoryMonitor
|
||||
memory_monitor = MemoryMonitor(interval=app.config.get('METRICS_MEMORY_INTERVAL', 30))
|
||||
memory_monitor.start()
|
||||
app.memory_monitor = memory_monitor
|
||||
app.logger.info(f"Memory monitor started (interval={memory_monitor.interval}s)")
|
||||
# No teardown_appcontext stop: that ran after every request and stopped
|
||||
# the monitor straight away (v2.10.0). It is a daemon thread.
|
||||
|
||||
# Start webmention worker thread (v1.6.0 Phase 2)
|
||||
# Per background-worker.md: Skip in test mode
|
||||
if app.config.get('WEBMENTION_WORKER_ENABLED', True) and not app.config.get('TESTING', False):
|
||||
@@ -470,5 +457,5 @@ def create_app(config=None):
|
||||
|
||||
# Package version (Semantic Versioning 2.0.0)
|
||||
# See docs/standards/versioning-strategy.md for details
|
||||
__version__ = "2.10.2"
|
||||
__version_info__ = (2, 10, 2)
|
||||
__version__ = "2.12.0"
|
||||
__version_info__ = (2, 12, 0)
|
||||
+3
-3
@@ -32,7 +32,7 @@ import base64
|
||||
import hashlib
|
||||
import secrets
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from functools import wraps
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
from urllib.parse import urlencode
|
||||
@@ -223,7 +223,7 @@ def initiate_login(next_url: Optional[str] = None) -> str:
|
||||
nonce,
|
||||
code_verifier,
|
||||
_safe_next_url(next_url),
|
||||
datetime.utcnow() + timedelta(minutes=LOGIN_STATE_MINUTES),
|
||||
datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(minutes=LOGIN_STATE_MINUTES),
|
||||
),
|
||||
)
|
||||
db.commit()
|
||||
@@ -361,7 +361,7 @@ def create_session(me: str) -> str:
|
||||
|
||||
# Calculate expiry (use configured session lifetime or default to 30 days)
|
||||
session_lifetime = current_app.config.get("SESSION_LIFETIME", 30)
|
||||
expires_at = datetime.utcnow() + timedelta(days=session_lifetime)
|
||||
expires_at = datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(days=session_lifetime)
|
||||
|
||||
current_app.logger.debug(f"Auth: Session expiry: {expires_at} ({session_lifetime} days)")
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@ Fallback Behavior (per Q14):
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Dict, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
@@ -236,7 +236,7 @@ def get_author_profile(me_url: str, refresh: bool = False) -> Dict:
|
||||
if cached:
|
||||
# Check if cache is still valid
|
||||
cached_until = datetime.fromisoformat(cached['cached_until'])
|
||||
if datetime.utcnow() < cached_until:
|
||||
if datetime.now(timezone.utc).replace(tzinfo=None) < cached_until:
|
||||
current_app.logger.debug(f"Using cached author profile for {me_url}")
|
||||
|
||||
# Parse rel_me_links from JSON
|
||||
@@ -326,7 +326,7 @@ def save_author_profile(me_url: str, profile: Dict) -> None:
|
||||
db = get_db(current_app)
|
||||
|
||||
# Calculate cache expiry (24 hours from now)
|
||||
cached_until = datetime.utcnow() + timedelta(hours=CACHE_TTL_HOURS)
|
||||
cached_until = datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(hours=CACHE_TTL_HOURS)
|
||||
|
||||
# Convert rel_me_links to JSON (per Q17)
|
||||
rel_me_json = json.dumps(profile.get('rel_me_links', []))
|
||||
|
||||
+10
-9
@@ -71,8 +71,10 @@ def load_config(app, config_override=None):
|
||||
app.config["DATABASE_PATH"] = Path(os.getenv("DATABASE_PATH", "./data/starpunk.db"))
|
||||
|
||||
# Flask environment
|
||||
app.config["ENV"] = os.getenv("FLASK_ENV", "development")
|
||||
app.config["DEBUG"] = os.getenv("FLASK_DEBUG", "1") == "1"
|
||||
# Safe defaults (v2.11.0): production, debug off. `flask run --debug`
|
||||
# sets FLASK_DEBUG=1 for local development.
|
||||
app.config["ENV"] = os.getenv("FLASK_ENV", "production")
|
||||
app.config["DEBUG"] = os.getenv("FLASK_DEBUG", "0") == "1"
|
||||
|
||||
# Logging
|
||||
app.config["LOG_LEVEL"] = os.getenv("LOG_LEVEL", "INFO")
|
||||
@@ -97,13 +99,6 @@ def load_config(app, config_override=None):
|
||||
# Flask MAX_CONTENT_LENGTH limits request body size (matches media.py MAX_FILE_SIZE)
|
||||
app.config["MAX_CONTENT_LENGTH"] = 50 * 1024 * 1024 # 50MB
|
||||
|
||||
# Metrics configuration (v1.1.2 Phase 1)
|
||||
app.config["METRICS_ENABLED"] = os.getenv("METRICS_ENABLED", "true").lower() == "true"
|
||||
app.config["METRICS_SLOW_QUERY_THRESHOLD"] = float(os.getenv("METRICS_SLOW_QUERY_THRESHOLD", "1.0"))
|
||||
app.config["METRICS_SAMPLING_RATE"] = float(os.getenv("METRICS_SAMPLING_RATE", "1.0"))
|
||||
app.config["METRICS_BUFFER_SIZE"] = int(os.getenv("METRICS_BUFFER_SIZE", "1000"))
|
||||
app.config["METRICS_MEMORY_INTERVAL"] = int(os.getenv("METRICS_MEMORY_INTERVAL", "30"))
|
||||
|
||||
# Debug file configuration (v1.5.0 Phase 2)
|
||||
app.config["DEBUG_SAVE_FAILED_UPLOADS"] = os.getenv("DEBUG_SAVE_FAILED_UPLOADS", "false").lower() == "true"
|
||||
app.config["DEBUG_FILE_MAX_AGE_DAYS"] = int(os.getenv("DEBUG_FILE_MAX_AGE_DAYS", "7"))
|
||||
@@ -153,8 +148,14 @@ def load_config(app, config_override=None):
|
||||
app.config["SYNDICATION_POLL_INTERVAL"] = int(os.getenv("SYNDICATION_POLL_INTERVAL", "10"))
|
||||
|
||||
# Apply overrides if provided
|
||||
# CSRF protection for admin and consent forms (v2.11.0); on by default.
|
||||
# Tests turn it off unless they set CSRF_ENABLED themselves.
|
||||
app.config["CSRF_ENABLED"] = os.getenv("CSRF_ENABLED", "true").lower() == "true"
|
||||
|
||||
if config_override:
|
||||
app.config.update(config_override)
|
||||
if app.config.get("TESTING") and "CSRF_ENABLED" not in config_override:
|
||||
app.config["CSRF_ENABLED"] = False
|
||||
|
||||
# Normalize SITE_URL trailing slash (in case override provided URL without slash)
|
||||
if "SITE_URL" in app.config:
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
"""
|
||||
CSRF protection for the admin and consent forms (v2.11.0)
|
||||
|
||||
The token is an HMAC of the login session cookie under the app secret: no
|
||||
storage, tied to the session, and different after every login. Forms carry
|
||||
it as a hidden `csrf_token` field (template global `csrf_token()`); POSTs to
|
||||
protected endpoints without the right token get 400.
|
||||
|
||||
Protected: every admin POST, logout, and the IndieAuth consent ("approve").
|
||||
Endpoints used by apps with bearer tokens (Micropub, token, revoke,
|
||||
Webmention, WebSub) are not: they have no cookie-based authority.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
|
||||
from flask import abort, current_app, request
|
||||
|
||||
SESSION_COOKIE = "starpunk_session"
|
||||
FIELD = "csrf_token"
|
||||
HEADER = "X-CSRF-Token"
|
||||
PROTECTED_ENDPOINTS = {"auth.logout", "indieauth.approve"}
|
||||
|
||||
|
||||
def csrf_token() -> str:
|
||||
"""The CSRF token for the current login session ('' when not logged in)"""
|
||||
session = request.cookies.get(SESSION_COOKIE)
|
||||
if not session:
|
||||
return ""
|
||||
key = str(current_app.config["SECRET_KEY"]).encode("utf-8")
|
||||
return hmac.new(key, b"csrf:" + session.encode("utf-8"), hashlib.sha256).hexdigest()
|
||||
|
||||
|
||||
def _protected() -> bool:
|
||||
return request.blueprint == "admin" or request.endpoint in PROTECTED_ENDPOINTS
|
||||
|
||||
|
||||
def check_csrf():
|
||||
"""before_request: refuse state-changing requests without a valid token"""
|
||||
if request.method in ("GET", "HEAD", "OPTIONS") or not current_app.config.get("CSRF_ENABLED", True):
|
||||
return None
|
||||
if not _protected():
|
||||
return None
|
||||
expected = csrf_token()
|
||||
if not expected:
|
||||
return None # not logged in: the route's own auth check refuses it
|
||||
sent = request.form.get(FIELD) or request.headers.get(HEADER) or ""
|
||||
if not hmac.compare_digest(sent, expected):
|
||||
current_app.logger.warning(f"CSRF token missing or wrong for {request.endpoint}")
|
||||
abort(400, description="The form expired or came from another site. Reload the page and try again.")
|
||||
return None
|
||||
|
||||
|
||||
def init_app(app) -> None:
|
||||
app.before_request(check_csrf)
|
||||
app.add_template_global(csrf_token, "csrf_token")
|
||||
@@ -6,11 +6,19 @@ Provides database initialization and connection pooling
|
||||
Per v1.1.1 Phase 1:
|
||||
- Connection pooling for improved performance (ADR-053)
|
||||
- Request-scoped connections via Flask's g object
|
||||
- Pool statistics for monitoring
|
||||
- Pool statistics
|
||||
"""
|
||||
|
||||
from starpunk.database.init import init_db
|
||||
from starpunk.database.pool import init_pool, get_db, get_pool_stats
|
||||
from starpunk.database.schema import INITIAL_SCHEMA_SQL
|
||||
|
||||
# Explicit datetime adapters (v2.11.0). Python 3.12 deprecated sqlite3's
|
||||
# built-in ones; these store exactly what they did ("YYYY-MM-DD HH:MM:SS[.ffffff]").
|
||||
import datetime as _datetime
|
||||
import sqlite3 as _sqlite3
|
||||
|
||||
_sqlite3.register_adapter(_datetime.datetime, lambda value: value.isoformat(" "))
|
||||
_sqlite3.register_adapter(_datetime.date, lambda value: value.isoformat())
|
||||
|
||||
__all__ = ['init_db', 'init_pool', 'get_db', 'get_pool_stats', 'INITIAL_SCHEMA_SQL']
|
||||
@@ -5,7 +5,7 @@ Per ADR-053 and developer Q&A Q2, CQ1:
|
||||
- Provides connection pooling for improved performance
|
||||
- Integrates with Flask's g object for request-scoped connections
|
||||
- Maintains same interface as get_db() for transparency
|
||||
- Pool statistics available for metrics
|
||||
- Pool statistics (shown in /admin/health)
|
||||
- Wraps connections with MonitoredConnection for timing (v1.1.2 Phase 1)
|
||||
|
||||
Note: Migrations use direct connections (not pooled) for isolation
|
||||
@@ -27,7 +27,7 @@ class ConnectionPool:
|
||||
but this provides connection reuse and request-scoped connection management.
|
||||
"""
|
||||
|
||||
def __init__(self, db_path, pool_size=5, timeout=10.0, slow_query_threshold=1.0, metrics_enabled=True):
|
||||
def __init__(self, db_path, pool_size=5, timeout=10.0):
|
||||
"""
|
||||
Initialize connection pool
|
||||
|
||||
@@ -35,14 +35,10 @@ class ConnectionPool:
|
||||
db_path: Path to SQLite database file
|
||||
pool_size: Maximum number of connections in pool
|
||||
timeout: Timeout for getting connection (seconds)
|
||||
slow_query_threshold: Threshold in seconds for slow query detection (v1.1.2)
|
||||
metrics_enabled: Whether to enable metrics collection (v1.1.2)
|
||||
"""
|
||||
self.db_path = Path(db_path)
|
||||
self.pool_size = pool_size
|
||||
self.timeout = timeout
|
||||
self.slow_query_threshold = slow_query_threshold
|
||||
self.metrics_enabled = metrics_enabled
|
||||
self._pool = deque(maxlen=pool_size)
|
||||
self._lock = Lock()
|
||||
self._stats = {
|
||||
@@ -56,8 +52,6 @@ class ConnectionPool:
|
||||
def _create_connection(self):
|
||||
"""
|
||||
Create a new database connection
|
||||
|
||||
Per CQ1: Wraps connection with MonitoredConnection if metrics enabled
|
||||
"""
|
||||
conn = sqlite3.connect(
|
||||
self.db_path,
|
||||
@@ -68,14 +62,11 @@ class ConnectionPool:
|
||||
|
||||
# Enable WAL mode for better concurrency
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
# Enforce foreign keys, so ON DELETE CASCADE / SET NULL actually run
|
||||
# (SQLite leaves them off per connection; v2.11.0)
|
||||
conn.execute("PRAGMA foreign_keys=ON")
|
||||
|
||||
self._stats['connections_created'] += 1
|
||||
|
||||
# Wrap with monitoring if enabled (v1.1.2 Phase 1)
|
||||
if self.metrics_enabled:
|
||||
from starpunk.monitoring import MonitoredConnection
|
||||
return MonitoredConnection(conn, self.slow_query_threshold)
|
||||
|
||||
return conn
|
||||
|
||||
def get_connection(self):
|
||||
@@ -168,8 +159,6 @@ def init_pool(app):
|
||||
"""
|
||||
Initialize the connection pool
|
||||
|
||||
Per CQ2: Passes metrics configuration from app config
|
||||
|
||||
Args:
|
||||
app: Flask application instance
|
||||
"""
|
||||
@@ -178,20 +167,8 @@ def init_pool(app):
|
||||
db_path = app.config['DATABASE_PATH']
|
||||
pool_size = app.config.get('DB_POOL_SIZE', 5)
|
||||
timeout = app.config.get('DB_TIMEOUT', 10.0)
|
||||
slow_query_threshold = app.config.get('METRICS_SLOW_QUERY_THRESHOLD', 1.0)
|
||||
metrics_enabled = app.config.get('METRICS_ENABLED', True)
|
||||
|
||||
_pool = ConnectionPool(
|
||||
db_path,
|
||||
pool_size,
|
||||
timeout,
|
||||
slow_query_threshold,
|
||||
metrics_enabled
|
||||
)
|
||||
app.logger.info(
|
||||
f"Database connection pool initialized "
|
||||
f"(size={pool_size}, metrics={'enabled' if metrics_enabled else 'disabled'})"
|
||||
)
|
||||
_pool = ConnectionPool(db_path, pool_size, timeout)
|
||||
app.logger.info(f"Database connection pool initialized (size={pool_size})")
|
||||
|
||||
# Register teardown handler
|
||||
@app.teardown_appcontext
|
||||
|
||||
@@ -18,12 +18,10 @@ Standards:
|
||||
# Standard library imports
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
import time
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
# Local imports
|
||||
from starpunk.models import Note
|
||||
from starpunk.monitoring.business import track_feed_generated
|
||||
|
||||
|
||||
# ATOM namespace
|
||||
@@ -127,10 +125,6 @@ def generate_atom_streaming(
|
||||
# Remove trailing slash from site_url for consistency
|
||||
site_url = site_url.rstrip("/")
|
||||
|
||||
# Track feed generation timing
|
||||
start_time = time.time()
|
||||
item_count = 0
|
||||
|
||||
# Current timestamp for updated
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
@@ -159,8 +153,6 @@ def generate_atom_streaming(
|
||||
# Yield entries (newest first)
|
||||
# Notes from database are already in DESC order (newest first)
|
||||
for note in notes[:limit]:
|
||||
item_count += 1
|
||||
|
||||
# Build permalink URL
|
||||
permalink = f"{site_url}{note.permalink}"
|
||||
|
||||
@@ -229,15 +221,6 @@ def generate_atom_streaming(
|
||||
# Yield closing tag
|
||||
yield '</feed>\n'
|
||||
|
||||
# Track feed generation metrics
|
||||
duration_ms = (time.time() - start_time) * 1000
|
||||
track_feed_generated(
|
||||
format='atom',
|
||||
item_count=item_count,
|
||||
duration_ms=duration_ms,
|
||||
cached=False
|
||||
)
|
||||
|
||||
|
||||
def _escape_xml(text: str) -> str:
|
||||
"""
|
||||
|
||||
@@ -18,12 +18,10 @@ Standards:
|
||||
# Standard library imports
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional, Dict, Any
|
||||
import time
|
||||
import json
|
||||
|
||||
# Local imports
|
||||
from starpunk.models import Note
|
||||
from starpunk.monitoring.business import track_feed_generated
|
||||
|
||||
|
||||
def generate_json_feed(
|
||||
@@ -74,9 +72,6 @@ def generate_json_feed(
|
||||
# Remove trailing slash from site_url for consistency
|
||||
site_url = site_url.rstrip("/")
|
||||
|
||||
# Track feed generation timing
|
||||
start_time = time.time()
|
||||
|
||||
# Build feed object
|
||||
feed = _build_feed_object(
|
||||
site_url=site_url,
|
||||
@@ -88,14 +83,6 @@ def generate_json_feed(
|
||||
# Serialize to JSON (pretty-printed)
|
||||
feed_json = json.dumps(feed, ensure_ascii=False, indent=2)
|
||||
|
||||
# Track feed generation metrics
|
||||
duration_ms = (time.time() - start_time) * 1000
|
||||
track_feed_generated(
|
||||
format='json',
|
||||
item_count=min(len(notes), limit),
|
||||
duration_ms=duration_ms,
|
||||
cached=False
|
||||
)
|
||||
|
||||
return feed_json
|
||||
|
||||
@@ -147,10 +134,6 @@ def generate_json_feed_streaming(
|
||||
# Remove trailing slash from site_url for consistency
|
||||
site_url = site_url.rstrip("/")
|
||||
|
||||
# Track feed generation timing
|
||||
start_time = time.time()
|
||||
item_count = 0
|
||||
|
||||
# Start feed object
|
||||
yield '{\n'
|
||||
yield f' "version": "https://jsonfeed.org/version/1.1",\n'
|
||||
@@ -170,8 +153,6 @@ def generate_json_feed_streaming(
|
||||
# Notes from database are already in DESC order (newest first)
|
||||
items = notes[:limit]
|
||||
for i, note in enumerate(items):
|
||||
item_count += 1
|
||||
|
||||
# Build item object
|
||||
item = _build_item_object(site_url, note)
|
||||
|
||||
@@ -193,15 +174,6 @@ def generate_json_feed_streaming(
|
||||
yield ' ]\n'
|
||||
yield '}\n'
|
||||
|
||||
# Track feed generation metrics
|
||||
duration_ms = (time.time() - start_time) * 1000
|
||||
track_feed_generated(
|
||||
format='json',
|
||||
item_count=item_count,
|
||||
duration_ms=duration_ms,
|
||||
cached=False
|
||||
)
|
||||
|
||||
|
||||
def _build_feed_object(
|
||||
site_url: str,
|
||||
|
||||
@@ -14,7 +14,7 @@ Per v1.1.2 Phase 3:
|
||||
Specification: http://opml.org/spec2.opml
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timezone
|
||||
from xml.sax.saxutils import escape
|
||||
|
||||
|
||||
@@ -57,7 +57,7 @@ def generate_opml(site_url: str, site_name: str) -> str:
|
||||
safe_site_name = escape(site_name)
|
||||
|
||||
# RFC 822 date format (required by OPML spec)
|
||||
creation_date = datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT')
|
||||
creation_date = datetime.now(timezone.utc).replace(tzinfo=None).strftime('%a, %d %b %Y %H:%M:%S GMT')
|
||||
|
||||
# Build OPML document
|
||||
opml_lines = [
|
||||
|
||||
@@ -22,14 +22,12 @@ Standards:
|
||||
# Standard library imports
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
import time
|
||||
|
||||
# Third-party imports
|
||||
from feedgen.feed import FeedGenerator
|
||||
|
||||
# Local imports
|
||||
from starpunk.models import Note
|
||||
from starpunk.monitoring.business import track_feed_generated
|
||||
|
||||
|
||||
def generate_rss(
|
||||
@@ -98,9 +96,6 @@ def generate_rss(
|
||||
# Set last build date to now
|
||||
fg.lastBuildDate(datetime.now(timezone.utc))
|
||||
|
||||
# Track feed generation timing
|
||||
start_time = time.time()
|
||||
|
||||
# Add items (limit to configured maximum, newest first)
|
||||
# Notes from database are DESC but feedgen reverses them, so we reverse back
|
||||
for note in reversed(notes[:limit]):
|
||||
@@ -165,14 +160,6 @@ def generate_rss(
|
||||
# We need to inject media:content and media:thumbnail elements
|
||||
feed_xml = _inject_media_rss_elements(feed_xml, site_url, notes[:limit])
|
||||
|
||||
# Track feed generation metrics
|
||||
duration_ms = (time.time() - start_time) * 1000
|
||||
track_feed_generated(
|
||||
format='rss',
|
||||
item_count=min(len(notes), limit),
|
||||
duration_ms=duration_ms,
|
||||
cached=False
|
||||
)
|
||||
|
||||
return feed_xml
|
||||
|
||||
@@ -227,10 +214,6 @@ def generate_rss_streaming(
|
||||
# Remove trailing slash from site_url for consistency
|
||||
site_url = site_url.rstrip("/")
|
||||
|
||||
# Track feed generation timing
|
||||
start_time = time.time()
|
||||
item_count = 0
|
||||
|
||||
# Current timestamp for lastBuildDate
|
||||
now = datetime.now(timezone.utc)
|
||||
last_build = format_rfc822_date(now)
|
||||
@@ -251,8 +234,6 @@ def generate_rss_streaming(
|
||||
# Yield items (newest first)
|
||||
# Notes from database are already in DESC order (newest first)
|
||||
for note in notes[:limit]:
|
||||
item_count += 1
|
||||
|
||||
# Build permalink URL
|
||||
permalink = f"{site_url}{note.permalink}"
|
||||
|
||||
@@ -371,15 +352,6 @@ def generate_rss_streaming(
|
||||
yield " </channel>\n"
|
||||
yield "</rss>\n"
|
||||
|
||||
# Track feed generation metrics
|
||||
duration_ms = (time.time() - start_time) * 1000
|
||||
track_feed_generated(
|
||||
format='rss',
|
||||
item_count=item_count,
|
||||
duration_ms=duration_ms,
|
||||
cached=False
|
||||
)
|
||||
|
||||
|
||||
def _inject_media_rss_elements(feed_xml: str, site_url: str, notes: list[Note]) -> str:
|
||||
"""
|
||||
|
||||
+18
-17
@@ -13,7 +13,7 @@ current IndieAuth spec: https://indieauth.spec.indieweb.org/
|
||||
import base64
|
||||
import hashlib
|
||||
import secrets
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Dict, List, Optional
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
@@ -83,41 +83,42 @@ def fetch_client_redirect_uris(client_id: str) -> List[str]:
|
||||
Supports a JSON client metadata document ("redirect_uris") and the older
|
||||
HTML/Link header form (rel="redirect_uri"). Returns [] on any failure.
|
||||
"""
|
||||
from starpunk.webmention_ssrf import SSRFBlockedError, validate_url_for_fetch
|
||||
import json
|
||||
from starpunk.webmention_ssrf import FetchTooLargeError, SSRFBlockedError, safe_get, user_agent
|
||||
|
||||
# Every redirect hop is checked and the connection pinned to the checked
|
||||
# address, with a hard time limit (v2.11.0; DEV_MODE allows localhost)
|
||||
try:
|
||||
if not current_app.config.get("DEV_MODE"):
|
||||
validate_url_for_fetch(client_id)
|
||||
response = httpx.get(
|
||||
result = safe_get(
|
||||
client_id,
|
||||
timeout=5.0,
|
||||
follow_redirects=True,
|
||||
headers={"Accept": "application/json, text/html;q=0.9"},
|
||||
headers={"Accept": "application/json, text/html;q=0.9",
|
||||
"User-Agent": user_agent(current_app.config.get("SITE_URL", ""))},
|
||||
max_bytes=500_000, timeout=5.0, truncate=True,
|
||||
)
|
||||
except (httpx.HTTPError, SSRFBlockedError, ValueError) as e:
|
||||
except (httpx.HTTPError, SSRFBlockedError, ValueError, FetchTooLargeError) as e:
|
||||
current_app.logger.info(f"IndieAuth: could not fetch client {client_id}: {e}")
|
||||
return []
|
||||
|
||||
if response.status_code != 200:
|
||||
if result.status_code != 200:
|
||||
return []
|
||||
|
||||
base = str(response.url)
|
||||
base = result.url
|
||||
uris: List[str] = []
|
||||
|
||||
if "json" in response.headers.get("Content-Type", ""):
|
||||
if "json" in result.headers.get("Content-Type", ""):
|
||||
try:
|
||||
data = response.json()
|
||||
data = json.loads(result.text)
|
||||
except ValueError:
|
||||
return []
|
||||
if isinstance(data, dict) and isinstance(data.get("redirect_uris"), list):
|
||||
uris.extend(u for u in data["redirect_uris"] if isinstance(u, str))
|
||||
return uris
|
||||
|
||||
link = response.links.get("redirect_uri")
|
||||
link = httpx.Response(200, headers=result.headers).links.get("redirect_uri")
|
||||
if link and link.get("url"):
|
||||
uris.append(urljoin(base, link["url"]))
|
||||
|
||||
soup = BeautifulSoup(response.text[:500_000], "html.parser")
|
||||
soup = BeautifulSoup(result.text, "html.parser")
|
||||
for tag in soup.find_all(["link", "a"], href=True):
|
||||
if "redirect_uri" in (tag.get("rel") or []):
|
||||
uris.append(urljoin(base, tag["href"]))
|
||||
@@ -203,7 +204,7 @@ def create_authorization_code(
|
||||
redirect_uri,
|
||||
scope,
|
||||
code_challenge,
|
||||
datetime.utcnow() + timedelta(minutes=CODE_LIFETIME_MINUTES),
|
||||
datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(minutes=CODE_LIFETIME_MINUTES),
|
||||
),
|
||||
)
|
||||
db.commit()
|
||||
@@ -258,7 +259,7 @@ def create_access_token(client_id: str, scope: str) -> Dict[str, Any]:
|
||||
"""Issue an access token (stored hashed); returns the token response"""
|
||||
token = secrets.token_urlsafe(32)
|
||||
lifetime_days = current_app.config.get("INDIEAUTH_TOKEN_LIFETIME_DAYS", 90)
|
||||
expires_at = datetime.utcnow() + timedelta(days=lifetime_days)
|
||||
expires_at = datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(days=lifetime_days)
|
||||
|
||||
db = get_db(current_app)
|
||||
db.execute(
|
||||
|
||||
+9
-9
@@ -8,7 +8,7 @@ state. All models are immutable and use dataclasses for clean structure.
|
||||
|
||||
# Standard library imports
|
||||
from dataclasses import dataclass, field, replace
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
import re
|
||||
@@ -698,7 +698,7 @@ class Session:
|
||||
>>> session.is_expired
|
||||
False
|
||||
"""
|
||||
return datetime.utcnow() >= self.expires_at
|
||||
return datetime.now(timezone.utc).replace(tzinfo=None) >= self.expires_at
|
||||
|
||||
@property
|
||||
def is_active(self) -> bool:
|
||||
@@ -726,7 +726,7 @@ class Session:
|
||||
>>> session.age
|
||||
datetime.timedelta(days=2, seconds=3600)
|
||||
"""
|
||||
return datetime.utcnow() - self.created_at
|
||||
return datetime.now(timezone.utc).replace(tzinfo=None) - self.created_at
|
||||
|
||||
@property
|
||||
def time_until_expiry(self) -> timedelta:
|
||||
@@ -740,7 +740,7 @@ class Session:
|
||||
>>> session.time_until_expiry
|
||||
datetime.timedelta(days=28)
|
||||
"""
|
||||
return self.expires_at - datetime.utcnow()
|
||||
return self.expires_at - datetime.now(timezone.utc).replace(tzinfo=None)
|
||||
|
||||
def is_valid(self) -> bool:
|
||||
"""
|
||||
@@ -787,7 +787,7 @@ class Session:
|
||||
>>> updated.last_used_at
|
||||
datetime.datetime(2024, 11, 18, 15, 30, 0)
|
||||
"""
|
||||
return replace(self, last_used_at=datetime.utcnow())
|
||||
return replace(self, last_used_at=datetime.now(timezone.utc).replace(tzinfo=None))
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""
|
||||
@@ -863,7 +863,7 @@ class Token:
|
||||
me: str
|
||||
client_id: Optional[str] = None
|
||||
scope: Optional[str] = None
|
||||
created_at: datetime = field(default_factory=lambda: datetime.utcnow())
|
||||
created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc).replace(tzinfo=None))
|
||||
expires_at: Optional[datetime] = None
|
||||
|
||||
@classmethod
|
||||
@@ -926,7 +926,7 @@ class Token:
|
||||
# No expiry set, token never expires
|
||||
return False
|
||||
|
||||
return datetime.utcnow() >= self.expires_at
|
||||
return datetime.now(timezone.utc).replace(tzinfo=None) >= self.expires_at
|
||||
|
||||
@property
|
||||
def is_active(self) -> bool:
|
||||
@@ -1145,7 +1145,7 @@ class AuthState:
|
||||
>>> auth_state.is_expired
|
||||
False
|
||||
"""
|
||||
return datetime.utcnow() >= self.expires_at
|
||||
return datetime.now(timezone.utc).replace(tzinfo=None) >= self.expires_at
|
||||
|
||||
@property
|
||||
def is_active(self) -> bool:
|
||||
@@ -1169,7 +1169,7 @@ class AuthState:
|
||||
>>> auth_state.age
|
||||
datetime.timedelta(seconds=120)
|
||||
"""
|
||||
return datetime.utcnow() - self.created_at
|
||||
return datetime.now(timezone.utc).replace(tzinfo=None) - self.created_at
|
||||
|
||||
def is_valid(self) -> bool:
|
||||
"""
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
"""
|
||||
Performance monitoring for StarPunk
|
||||
|
||||
This package provides performance monitoring capabilities including:
|
||||
- Metrics collection with circular buffers
|
||||
- Operation timing (database, HTTP, rendering)
|
||||
- Per-process metrics with aggregation
|
||||
- Configurable sampling rates
|
||||
- Database query monitoring (v1.1.2 Phase 1)
|
||||
- HTTP request/response metrics (v1.1.2 Phase 1)
|
||||
- Memory monitoring (v1.1.2 Phase 1)
|
||||
|
||||
Per ADR-053 and developer Q&A Q6, Q12:
|
||||
- Each process maintains its own circular buffer
|
||||
- Buffers store recent metrics (default 1000 entries)
|
||||
- Metrics include process ID for multi-process deployment
|
||||
- Sampling rates are configurable per operation type
|
||||
"""
|
||||
|
||||
from starpunk.monitoring.metrics import MetricsBuffer, record_metric, get_metrics, get_metrics_stats
|
||||
from starpunk.monitoring.database import MonitoredConnection
|
||||
from starpunk.monitoring.http import setup_http_metrics
|
||||
from starpunk.monitoring.memory import MemoryMonitor
|
||||
from starpunk.monitoring import business
|
||||
|
||||
__all__ = [
|
||||
"MetricsBuffer",
|
||||
"record_metric",
|
||||
"get_metrics",
|
||||
"get_metrics_stats",
|
||||
"MonitoredConnection",
|
||||
"setup_http_metrics",
|
||||
"MemoryMonitor",
|
||||
"business",
|
||||
]
|
||||
@@ -1,298 +0,0 @@
|
||||
"""
|
||||
Business metrics for StarPunk operations
|
||||
|
||||
Per v1.1.2 Phase 1:
|
||||
- Track note operations (create, update, delete)
|
||||
- Track feed generation and cache hits/misses
|
||||
- Track content statistics
|
||||
|
||||
Per v1.1.2 Phase 3:
|
||||
- Track feed statistics by format
|
||||
- Track feed cache hit/miss rates
|
||||
- Provide feed statistics dashboard
|
||||
|
||||
Example usage:
|
||||
>>> from starpunk.monitoring.business import track_note_created
|
||||
>>> track_note_created(note_id=123, content_length=500)
|
||||
"""
|
||||
|
||||
from typing import Optional, Dict, Any
|
||||
|
||||
from starpunk.monitoring.metrics import record_metric, get_metrics_stats
|
||||
|
||||
|
||||
def track_note_created(note_id: int, content_length: int, has_media: bool = False) -> None:
|
||||
"""
|
||||
Track note creation event
|
||||
|
||||
Args:
|
||||
note_id: ID of created note
|
||||
content_length: Length of note content in characters
|
||||
has_media: Whether note has media attachments
|
||||
"""
|
||||
metadata = {
|
||||
'note_id': note_id,
|
||||
'content_length': content_length,
|
||||
'has_media': has_media,
|
||||
}
|
||||
|
||||
record_metric(
|
||||
'render', # Use 'render' for business metrics
|
||||
'note_created',
|
||||
content_length,
|
||||
metadata,
|
||||
force=True # Always track business events
|
||||
)
|
||||
|
||||
|
||||
def track_note_updated(note_id: int, content_length: int, fields_changed: Optional[list] = None) -> None:
|
||||
"""
|
||||
Track note update event
|
||||
|
||||
Args:
|
||||
note_id: ID of updated note
|
||||
content_length: New length of note content
|
||||
fields_changed: List of fields that were changed
|
||||
"""
|
||||
metadata = {
|
||||
'note_id': note_id,
|
||||
'content_length': content_length,
|
||||
}
|
||||
|
||||
if fields_changed:
|
||||
metadata['fields_changed'] = ','.join(fields_changed)
|
||||
|
||||
record_metric(
|
||||
'render',
|
||||
'note_updated',
|
||||
content_length,
|
||||
metadata,
|
||||
force=True
|
||||
)
|
||||
|
||||
|
||||
def track_note_deleted(note_id: int) -> None:
|
||||
"""
|
||||
Track note deletion event
|
||||
|
||||
Args:
|
||||
note_id: ID of deleted note
|
||||
"""
|
||||
metadata = {
|
||||
'note_id': note_id,
|
||||
}
|
||||
|
||||
record_metric(
|
||||
'render',
|
||||
'note_deleted',
|
||||
0, # No meaningful duration for deletion
|
||||
metadata,
|
||||
force=True
|
||||
)
|
||||
|
||||
|
||||
def track_feed_generated(format: str, item_count: int, duration_ms: float, cached: bool = False) -> None:
|
||||
"""
|
||||
Track feed generation event
|
||||
|
||||
Args:
|
||||
format: Feed format (rss, atom, json)
|
||||
item_count: Number of items in feed
|
||||
duration_ms: Time taken to generate feed
|
||||
cached: Whether feed was served from cache
|
||||
"""
|
||||
metadata = {
|
||||
'format': format,
|
||||
'item_count': item_count,
|
||||
'cached': cached,
|
||||
}
|
||||
|
||||
operation = f'feed_{format}{"_cached" if cached else "_generated"}'
|
||||
|
||||
record_metric(
|
||||
'render',
|
||||
operation,
|
||||
duration_ms,
|
||||
metadata,
|
||||
force=True # Always track feed operations
|
||||
)
|
||||
|
||||
|
||||
def track_cache_hit(cache_type: str, key: str) -> None:
|
||||
"""
|
||||
Track cache hit event
|
||||
|
||||
Args:
|
||||
cache_type: Type of cache (feed, etc.)
|
||||
key: Cache key that was hit
|
||||
"""
|
||||
metadata = {
|
||||
'cache_type': cache_type,
|
||||
'key': key,
|
||||
}
|
||||
|
||||
record_metric(
|
||||
'render',
|
||||
f'{cache_type}_cache_hit',
|
||||
0,
|
||||
metadata,
|
||||
force=True
|
||||
)
|
||||
|
||||
|
||||
def track_cache_miss(cache_type: str, key: str) -> None:
|
||||
"""
|
||||
Track cache miss event
|
||||
|
||||
Args:
|
||||
cache_type: Type of cache (feed, etc.)
|
||||
key: Cache key that was missed
|
||||
"""
|
||||
metadata = {
|
||||
'cache_type': cache_type,
|
||||
'key': key,
|
||||
}
|
||||
|
||||
record_metric(
|
||||
'render',
|
||||
f'{cache_type}_cache_miss',
|
||||
0,
|
||||
metadata,
|
||||
force=True
|
||||
)
|
||||
|
||||
|
||||
def get_feed_statistics() -> Dict[str, Any]:
|
||||
"""
|
||||
Get aggregated feed statistics from metrics buffer and feed cache.
|
||||
|
||||
Analyzes metrics to provide feed-specific statistics including:
|
||||
- Total requests by format (RSS, ATOM, JSON)
|
||||
- Cache hit/miss rates by format
|
||||
- Feed generation times by format
|
||||
- Format popularity (percentage breakdown)
|
||||
- Feed cache internal statistics
|
||||
|
||||
Returns:
|
||||
Dictionary with feed statistics:
|
||||
{
|
||||
'by_format': {
|
||||
'rss': {'generated': int, 'cached': int, 'total': int, 'avg_duration_ms': float},
|
||||
'atom': {...},
|
||||
'json': {...}
|
||||
},
|
||||
'cache': {
|
||||
'hits': int,
|
||||
'misses': int,
|
||||
'hit_rate': float (0.0-1.0),
|
||||
'entries': int,
|
||||
'evictions': int
|
||||
},
|
||||
'total_requests': int,
|
||||
'format_percentages': {
|
||||
'rss': float,
|
||||
'atom': float,
|
||||
'json': float
|
||||
}
|
||||
}
|
||||
|
||||
Example:
|
||||
>>> stats = get_feed_statistics()
|
||||
>>> print(f"RSS requests: {stats['by_format']['rss']['total']}")
|
||||
>>> print(f"Cache hit rate: {stats['cache']['hit_rate']:.2%}")
|
||||
"""
|
||||
# Get all metrics
|
||||
all_metrics = get_metrics_stats()
|
||||
|
||||
# Initialize result structure
|
||||
result = {
|
||||
'by_format': {
|
||||
'rss': {'generated': 0, 'cached': 0, 'total': 0, 'avg_duration_ms': 0.0},
|
||||
'atom': {'generated': 0, 'cached': 0, 'total': 0, 'avg_duration_ms': 0.0},
|
||||
'json': {'generated': 0, 'cached': 0, 'total': 0, 'avg_duration_ms': 0.0},
|
||||
},
|
||||
'cache': {
|
||||
'hits': 0,
|
||||
'misses': 0,
|
||||
'hit_rate': 0.0,
|
||||
},
|
||||
'total_requests': 0,
|
||||
'format_percentages': {
|
||||
'rss': 0.0,
|
||||
'atom': 0.0,
|
||||
'json': 0.0,
|
||||
},
|
||||
}
|
||||
|
||||
# Get by_operation metrics if available
|
||||
by_operation = all_metrics.get('by_operation', {})
|
||||
|
||||
# Count feed operations by format
|
||||
for operation_name, op_stats in by_operation.items():
|
||||
# Feed operations are named: feed_rss_generated, feed_rss_cached, etc.
|
||||
if operation_name.startswith('feed_'):
|
||||
parts = operation_name.split('_')
|
||||
if len(parts) >= 3:
|
||||
format_name = parts[1] # rss, atom, or json
|
||||
operation_type = parts[2] # generated or cached
|
||||
|
||||
if format_name in result['by_format']:
|
||||
count = op_stats.get('count', 0)
|
||||
|
||||
if operation_type == 'generated':
|
||||
result['by_format'][format_name]['generated'] = count
|
||||
# Track average duration for generated feeds
|
||||
result['by_format'][format_name]['avg_duration_ms'] = op_stats.get('avg_duration_ms', 0.0)
|
||||
elif operation_type == 'cached':
|
||||
result['by_format'][format_name]['cached'] = count
|
||||
|
||||
# Update total for this format
|
||||
result['by_format'][format_name]['total'] = (
|
||||
result['by_format'][format_name]['generated'] +
|
||||
result['by_format'][format_name]['cached']
|
||||
)
|
||||
|
||||
# Track cache hits/misses
|
||||
elif operation_name == 'feed_cache_hit':
|
||||
result['cache']['hits'] = op_stats.get('count', 0)
|
||||
elif operation_name == 'feed_cache_miss':
|
||||
result['cache']['misses'] = op_stats.get('count', 0)
|
||||
|
||||
# Calculate total requests across all formats
|
||||
result['total_requests'] = sum(
|
||||
fmt['total'] for fmt in result['by_format'].values()
|
||||
)
|
||||
|
||||
# Calculate cache hit rate
|
||||
total_cache_requests = result['cache']['hits'] + result['cache']['misses']
|
||||
if total_cache_requests > 0:
|
||||
result['cache']['hit_rate'] = result['cache']['hits'] / total_cache_requests
|
||||
|
||||
# Calculate format percentages
|
||||
if result['total_requests'] > 0:
|
||||
for format_name, fmt_stats in result['by_format'].items():
|
||||
result['format_percentages'][format_name] = (
|
||||
fmt_stats['total'] / result['total_requests']
|
||||
)
|
||||
|
||||
# Get feed cache statistics if available
|
||||
try:
|
||||
from starpunk.feeds import get_cache
|
||||
feed_cache = get_cache()
|
||||
cache_stats = feed_cache.get_stats()
|
||||
|
||||
# Merge cache stats (prefer FeedCache internal stats over metrics)
|
||||
result['cache']['entries'] = cache_stats.get('entries', 0)
|
||||
result['cache']['evictions'] = cache_stats.get('evictions', 0)
|
||||
|
||||
# Use FeedCache hit rate if available and more accurate
|
||||
if cache_stats.get('hits', 0) + cache_stats.get('misses', 0) > 0:
|
||||
result['cache']['hits'] = cache_stats.get('hits', 0)
|
||||
result['cache']['misses'] = cache_stats.get('misses', 0)
|
||||
result['cache']['hit_rate'] = cache_stats.get('hit_rate', 0.0)
|
||||
|
||||
except ImportError:
|
||||
# Feed cache not available, use defaults
|
||||
pass
|
||||
|
||||
return result
|
||||
@@ -1,236 +0,0 @@
|
||||
"""
|
||||
Database operation monitoring wrapper
|
||||
|
||||
Per ADR-053, v1.1.2 Phase 1, and developer Q&A CQ1, IQ1, IQ3:
|
||||
- Wraps SQLite connections at the pool level
|
||||
- Times all database operations
|
||||
- Extracts query type and table name (best effort)
|
||||
- Detects slow queries based on configurable threshold
|
||||
- Records metrics to the metrics collector
|
||||
|
||||
Example usage:
|
||||
>>> from starpunk.monitoring.database import MonitoredConnection
|
||||
>>> conn = sqlite3.connect(':memory:')
|
||||
>>> monitored = MonitoredConnection(conn, metrics_collector)
|
||||
>>> cursor = monitored.execute('SELECT * FROM notes')
|
||||
"""
|
||||
|
||||
import re
|
||||
import sqlite3
|
||||
import time
|
||||
from typing import Optional, Any, Tuple
|
||||
|
||||
from starpunk.monitoring.metrics import record_metric
|
||||
|
||||
|
||||
class MonitoredConnection:
|
||||
"""
|
||||
Wrapper for SQLite connections that monitors performance
|
||||
|
||||
Per CQ1: Wraps connections at the pool level
|
||||
Per IQ1: Uses simple regex for table name extraction
|
||||
Per IQ3: Single configurable slow query threshold
|
||||
"""
|
||||
|
||||
def __init__(self, connection: sqlite3.Connection, slow_query_threshold: float = 1.0):
|
||||
"""
|
||||
Initialize monitored connection wrapper
|
||||
|
||||
Args:
|
||||
connection: SQLite connection to wrap
|
||||
slow_query_threshold: Threshold in seconds for slow query detection
|
||||
"""
|
||||
self._connection = connection
|
||||
self._slow_query_threshold = slow_query_threshold
|
||||
|
||||
def execute(self, query: str, parameters: Optional[Tuple] = None) -> sqlite3.Cursor:
|
||||
"""
|
||||
Execute a query with performance monitoring
|
||||
|
||||
Args:
|
||||
query: SQL query to execute
|
||||
parameters: Optional query parameters
|
||||
|
||||
Returns:
|
||||
sqlite3.Cursor: Query cursor
|
||||
"""
|
||||
start_time = time.perf_counter()
|
||||
query_type = self._get_query_type(query)
|
||||
table_name = self._extract_table_name(query)
|
||||
|
||||
try:
|
||||
if parameters:
|
||||
cursor = self._connection.execute(query, parameters)
|
||||
else:
|
||||
cursor = self._connection.execute(query)
|
||||
|
||||
duration_sec = time.perf_counter() - start_time
|
||||
duration_ms = duration_sec * 1000
|
||||
|
||||
# Record metric (forced if slow query)
|
||||
is_slow = duration_sec >= self._slow_query_threshold
|
||||
metadata = {
|
||||
'query_type': query_type,
|
||||
'table': table_name,
|
||||
'is_slow': is_slow,
|
||||
}
|
||||
|
||||
# Add query text for slow queries (for debugging)
|
||||
if is_slow:
|
||||
# Truncate query to avoid storing huge queries
|
||||
metadata['query'] = query[:200] if len(query) > 200 else query
|
||||
|
||||
record_metric(
|
||||
'database',
|
||||
f'{query_type} {table_name}',
|
||||
duration_ms,
|
||||
metadata,
|
||||
force=is_slow # Always record slow queries
|
||||
)
|
||||
|
||||
return cursor
|
||||
|
||||
except Exception as e:
|
||||
duration_sec = time.perf_counter() - start_time
|
||||
duration_ms = duration_sec * 1000
|
||||
|
||||
# Record error metric
|
||||
metadata = {
|
||||
'query_type': query_type,
|
||||
'table': table_name,
|
||||
'error': str(e),
|
||||
'query': query[:200] if len(query) > 200 else query
|
||||
}
|
||||
|
||||
record_metric(
|
||||
'database',
|
||||
f'{query_type} {table_name} ERROR',
|
||||
duration_ms,
|
||||
metadata,
|
||||
force=True # Always record errors
|
||||
)
|
||||
|
||||
raise
|
||||
|
||||
def executemany(self, query: str, parameters) -> sqlite3.Cursor:
|
||||
"""
|
||||
Execute a query with multiple parameter sets
|
||||
|
||||
Args:
|
||||
query: SQL query to execute
|
||||
parameters: Sequence of parameter tuples
|
||||
|
||||
Returns:
|
||||
sqlite3.Cursor: Query cursor
|
||||
"""
|
||||
start_time = time.perf_counter()
|
||||
query_type = self._get_query_type(query)
|
||||
table_name = self._extract_table_name(query)
|
||||
|
||||
try:
|
||||
cursor = self._connection.executemany(query, parameters)
|
||||
duration_ms = (time.perf_counter() - start_time) * 1000
|
||||
|
||||
# Record metric
|
||||
metadata = {
|
||||
'query_type': query_type,
|
||||
'table': table_name,
|
||||
'batch': True,
|
||||
}
|
||||
|
||||
record_metric(
|
||||
'database',
|
||||
f'{query_type} {table_name} BATCH',
|
||||
duration_ms,
|
||||
metadata
|
||||
)
|
||||
|
||||
return cursor
|
||||
|
||||
except Exception as e:
|
||||
duration_ms = (time.perf_counter() - start_time) * 1000
|
||||
|
||||
metadata = {
|
||||
'query_type': query_type,
|
||||
'table': table_name,
|
||||
'error': str(e),
|
||||
'batch': True
|
||||
}
|
||||
|
||||
record_metric(
|
||||
'database',
|
||||
f'{query_type} {table_name} BATCH ERROR',
|
||||
duration_ms,
|
||||
metadata,
|
||||
force=True
|
||||
)
|
||||
|
||||
raise
|
||||
|
||||
def _get_query_type(self, query: str) -> str:
|
||||
"""
|
||||
Extract query type from SQL statement
|
||||
|
||||
Args:
|
||||
query: SQL query
|
||||
|
||||
Returns:
|
||||
Query type (SELECT, INSERT, UPDATE, DELETE, etc.)
|
||||
"""
|
||||
query_upper = query.strip().upper()
|
||||
|
||||
for query_type in ['SELECT', 'INSERT', 'UPDATE', 'DELETE', 'CREATE', 'DROP', 'ALTER', 'PRAGMA']:
|
||||
if query_upper.startswith(query_type):
|
||||
return query_type
|
||||
|
||||
return 'OTHER'
|
||||
|
||||
def _extract_table_name(self, query: str) -> str:
|
||||
"""
|
||||
Extract table name from query (best effort)
|
||||
|
||||
Per IQ1: Keep it simple with basic regex patterns.
|
||||
Returns "unknown" for complex queries.
|
||||
|
||||
Note: Complex queries (JOINs, subqueries, CTEs) return "unknown".
|
||||
This covers 90% of queries accurately.
|
||||
|
||||
Args:
|
||||
query: SQL query
|
||||
|
||||
Returns:
|
||||
Table name or "unknown"
|
||||
"""
|
||||
query_lower = query.lower().strip()
|
||||
|
||||
# Simple patterns that cover 90% of cases
|
||||
patterns = [
|
||||
r'from\s+(\w+)',
|
||||
r'update\s+(\w+)',
|
||||
r'insert\s+into\s+(\w+)',
|
||||
r'delete\s+from\s+(\w+)',
|
||||
r'create\s+table\s+(?:if\s+not\s+exists\s+)?(\w+)',
|
||||
r'drop\s+table\s+(?:if\s+exists\s+)?(\w+)',
|
||||
r'alter\s+table\s+(\w+)',
|
||||
]
|
||||
|
||||
for pattern in patterns:
|
||||
match = re.search(pattern, query_lower)
|
||||
if match:
|
||||
return match.group(1)
|
||||
|
||||
# Complex queries (JOINs, subqueries, CTEs)
|
||||
return "unknown"
|
||||
|
||||
# Delegate all other connection methods to the wrapped connection
|
||||
def __getattr__(self, name: str) -> Any:
|
||||
"""Delegate all other methods to the wrapped connection"""
|
||||
return getattr(self._connection, name)
|
||||
|
||||
def __enter__(self):
|
||||
"""Support context manager protocol"""
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
"""Support context manager protocol"""
|
||||
return self._connection.__exit__(exc_type, exc_val, exc_tb)
|
||||
@@ -1,133 +0,0 @@
|
||||
"""
|
||||
HTTP request/response metrics middleware
|
||||
|
||||
Per v1.1.2 Phase 1 and developer Q&A IQ2:
|
||||
- Times all HTTP requests
|
||||
- Generates request IDs for tracking (IQ2)
|
||||
- Records status codes, methods, routes
|
||||
- Tracks request and response sizes
|
||||
- Adds X-Request-ID header to all responses (not just debug mode)
|
||||
|
||||
Example usage:
|
||||
>>> from starpunk.monitoring.http import setup_http_metrics
|
||||
>>> app = Flask(__name__)
|
||||
>>> setup_http_metrics(app)
|
||||
"""
|
||||
|
||||
import time
|
||||
import uuid
|
||||
from flask import g, request, Flask
|
||||
from typing import Any
|
||||
|
||||
from starpunk.monitoring.metrics import record_metric
|
||||
|
||||
|
||||
def setup_http_metrics(app: Flask) -> None:
|
||||
"""
|
||||
Setup HTTP metrics collection for Flask app
|
||||
|
||||
Per IQ2: Generates request IDs and adds X-Request-ID header in all modes
|
||||
|
||||
Args:
|
||||
app: Flask application instance
|
||||
"""
|
||||
|
||||
@app.before_request
|
||||
def start_request_metrics():
|
||||
"""
|
||||
Initialize request metrics tracking
|
||||
|
||||
Per IQ2: Generate UUID request ID and store in g
|
||||
"""
|
||||
# Generate request ID (IQ2: in all modes, not just debug)
|
||||
g.request_id = str(uuid.uuid4())
|
||||
|
||||
# Store request start time and metadata
|
||||
g.request_start_time = time.perf_counter()
|
||||
g.request_metadata = {
|
||||
'method': request.method,
|
||||
'endpoint': request.endpoint or 'unknown',
|
||||
'path': request.path,
|
||||
'content_length': request.content_length or 0,
|
||||
}
|
||||
|
||||
@app.after_request
|
||||
def record_response_metrics(response):
|
||||
"""
|
||||
Record HTTP response metrics
|
||||
|
||||
Args:
|
||||
response: Flask response object
|
||||
|
||||
Returns:
|
||||
Modified response with X-Request-ID header
|
||||
"""
|
||||
# Skip if metrics not initialized (shouldn't happen in normal flow)
|
||||
if not hasattr(g, 'request_start_time'):
|
||||
return response
|
||||
|
||||
# Calculate request duration
|
||||
duration_sec = time.perf_counter() - g.request_start_time
|
||||
duration_ms = duration_sec * 1000
|
||||
|
||||
# Get response size
|
||||
response_size = 0
|
||||
|
||||
# Check if response is in direct passthrough mode (streaming)
|
||||
if hasattr(response, 'direct_passthrough') and response.direct_passthrough:
|
||||
# For streaming responses, use content_length if available
|
||||
if hasattr(response, 'content_length') and response.content_length:
|
||||
response_size = response.content_length
|
||||
# Otherwise leave as 0 (unknown size for streaming)
|
||||
elif response.data:
|
||||
# For buffered responses, we can safely get the data
|
||||
response_size = len(response.data)
|
||||
elif hasattr(response, 'content_length') and response.content_length:
|
||||
response_size = response.content_length
|
||||
|
||||
# Build metadata
|
||||
metadata = {
|
||||
**g.request_metadata,
|
||||
'status_code': response.status_code,
|
||||
'response_size': response_size,
|
||||
}
|
||||
|
||||
# Record metric
|
||||
operation_name = f"{g.request_metadata['method']} {g.request_metadata['endpoint']}"
|
||||
record_metric(
|
||||
'http',
|
||||
operation_name,
|
||||
duration_ms,
|
||||
metadata
|
||||
)
|
||||
|
||||
# Add request ID header (IQ2: in all modes)
|
||||
response.headers['X-Request-ID'] = g.request_id
|
||||
|
||||
return response
|
||||
|
||||
@app.teardown_request
|
||||
def record_error_metrics(error=None):
|
||||
"""
|
||||
Record metrics for requests that result in errors
|
||||
|
||||
Args:
|
||||
error: Exception if request failed
|
||||
"""
|
||||
if error and hasattr(g, 'request_start_time'):
|
||||
duration_ms = (time.perf_counter() - g.request_start_time) * 1000
|
||||
|
||||
metadata = {
|
||||
**g.request_metadata,
|
||||
'error': str(error),
|
||||
'error_type': type(error).__name__,
|
||||
}
|
||||
|
||||
operation_name = f"{g.request_metadata['method']} {g.request_metadata['endpoint']} ERROR"
|
||||
record_metric(
|
||||
'http',
|
||||
operation_name,
|
||||
duration_ms,
|
||||
metadata,
|
||||
force=True # Always record errors
|
||||
)
|
||||
@@ -1,191 +0,0 @@
|
||||
"""
|
||||
Memory monitoring background thread
|
||||
|
||||
Per v1.1.2 Phase 1 and developer Q&A CQ5, IQ8:
|
||||
- Background daemon thread for continuous memory monitoring
|
||||
- Tracks RSS and VMS memory usage
|
||||
- Detects memory growth and potential leaks
|
||||
- 5-second baseline period after startup (IQ8)
|
||||
- Skipped in test mode (CQ5)
|
||||
|
||||
Example usage:
|
||||
>>> from starpunk.monitoring.memory import MemoryMonitor
|
||||
>>> monitor = MemoryMonitor(interval=30)
|
||||
>>> monitor.start() # Runs as daemon thread
|
||||
>>> # ... application runs ...
|
||||
>>> monitor.stop()
|
||||
"""
|
||||
|
||||
import gc
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from typing import Dict, Any
|
||||
|
||||
import psutil
|
||||
|
||||
from starpunk.monitoring.metrics import record_metric
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MemoryMonitor(threading.Thread):
|
||||
"""
|
||||
Background thread for memory monitoring
|
||||
|
||||
Per CQ5: Daemon thread that auto-terminates with main process
|
||||
Per IQ8: 5-second baseline period after startup
|
||||
"""
|
||||
|
||||
def __init__(self, interval: int = 30):
|
||||
"""
|
||||
Initialize memory monitor thread
|
||||
|
||||
Args:
|
||||
interval: Monitoring interval in seconds (default: 30)
|
||||
"""
|
||||
super().__init__(daemon=True) # CQ5: daemon thread
|
||||
self.interval = interval
|
||||
self._stop_event = threading.Event()
|
||||
self._process = psutil.Process()
|
||||
self._baseline_memory = None
|
||||
self._high_water_mark = 0
|
||||
|
||||
def run(self):
|
||||
"""
|
||||
Main monitoring loop
|
||||
|
||||
Per IQ8: Wait 5 seconds for app initialization before setting baseline
|
||||
"""
|
||||
try:
|
||||
# Wait for app initialization (IQ8: 5 seconds)
|
||||
time.sleep(5)
|
||||
|
||||
# Set baseline memory
|
||||
memory_info = self._get_memory_info()
|
||||
self._baseline_memory = memory_info['rss_mb']
|
||||
logger.info(f"Memory monitor baseline set: {self._baseline_memory:.2f} MB RSS")
|
||||
|
||||
# Start monitoring loop
|
||||
while not self._stop_event.is_set():
|
||||
try:
|
||||
self._collect_metrics()
|
||||
except Exception as e:
|
||||
logger.error(f"Memory monitoring error: {e}", exc_info=True)
|
||||
|
||||
# Wait for interval or until stop event
|
||||
self._stop_event.wait(self.interval)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Memory monitor thread failed: {e}", exc_info=True)
|
||||
|
||||
def _collect_metrics(self):
|
||||
"""Collect and record memory metrics"""
|
||||
memory_info = self._get_memory_info()
|
||||
gc_stats = self._get_gc_stats()
|
||||
|
||||
# Update high water mark
|
||||
if memory_info['rss_mb'] > self._high_water_mark:
|
||||
self._high_water_mark = memory_info['rss_mb']
|
||||
|
||||
# Calculate growth rate (MB/hour) if baseline is set
|
||||
growth_rate = 0.0
|
||||
if self._baseline_memory:
|
||||
growth_rate = memory_info['rss_mb'] - self._baseline_memory
|
||||
|
||||
# Record metrics
|
||||
metadata = {
|
||||
'rss_mb': memory_info['rss_mb'],
|
||||
'vms_mb': memory_info['vms_mb'],
|
||||
'percent': memory_info['percent'],
|
||||
'high_water_mb': self._high_water_mark,
|
||||
'growth_mb': growth_rate,
|
||||
'gc_collections': gc_stats['collections'],
|
||||
'gc_collected': gc_stats['collected'],
|
||||
}
|
||||
|
||||
record_metric(
|
||||
'render', # Use 'render' operation type for memory metrics
|
||||
'memory_usage',
|
||||
memory_info['rss_mb'],
|
||||
metadata,
|
||||
force=True # Always record memory metrics
|
||||
)
|
||||
|
||||
# Warn if significant growth detected (>10MB growth from baseline)
|
||||
if growth_rate > 10.0:
|
||||
logger.warning(
|
||||
f"Memory growth detected: +{growth_rate:.2f} MB from baseline "
|
||||
f"(current: {memory_info['rss_mb']:.2f} MB, baseline: {self._baseline_memory:.2f} MB)"
|
||||
)
|
||||
|
||||
def _get_memory_info(self) -> Dict[str, float]:
|
||||
"""
|
||||
Get current process memory usage
|
||||
|
||||
Returns:
|
||||
Dict with memory info in MB
|
||||
"""
|
||||
memory = self._process.memory_info()
|
||||
|
||||
return {
|
||||
'rss_mb': memory.rss / (1024 * 1024), # Resident Set Size
|
||||
'vms_mb': memory.vms / (1024 * 1024), # Virtual Memory Size
|
||||
'percent': self._process.memory_percent(),
|
||||
}
|
||||
|
||||
def _get_gc_stats(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Get garbage collection statistics
|
||||
|
||||
Returns:
|
||||
Dict with GC stats
|
||||
"""
|
||||
# Get collection counts per generation
|
||||
counts = gc.get_count()
|
||||
|
||||
# Perform a quick gen 0 collection and count collected objects
|
||||
collected = gc.collect(0)
|
||||
|
||||
return {
|
||||
'collections': {
|
||||
'gen0': counts[0],
|
||||
'gen1': counts[1],
|
||||
'gen2': counts[2],
|
||||
},
|
||||
'collected': collected,
|
||||
'uncollectable': len(gc.garbage),
|
||||
}
|
||||
|
||||
def stop(self):
|
||||
"""
|
||||
Stop the monitoring thread gracefully
|
||||
|
||||
Sets the stop event to signal the thread to exit
|
||||
"""
|
||||
logger.info("Stopping memory monitor")
|
||||
self._stop_event.set()
|
||||
|
||||
def get_stats(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Get current memory statistics
|
||||
|
||||
Returns:
|
||||
Dict with current memory stats
|
||||
"""
|
||||
if not self._baseline_memory:
|
||||
return {'status': 'initializing'}
|
||||
|
||||
memory_info = self._get_memory_info()
|
||||
|
||||
return {
|
||||
'status': 'running',
|
||||
'current_rss_mb': memory_info['rss_mb'],
|
||||
'baseline_rss_mb': self._baseline_memory,
|
||||
'growth_mb': memory_info['rss_mb'] - self._baseline_memory,
|
||||
'high_water_mb': self._high_water_mark,
|
||||
'percent': memory_info['percent'],
|
||||
}
|
||||
@@ -1,425 +0,0 @@
|
||||
"""
|
||||
Metrics collection and buffering for performance monitoring
|
||||
|
||||
Per ADR-053 and developer Q&A Q6, Q12:
|
||||
- Per-process circular buffers using deque
|
||||
- Configurable buffer size (default 1000 entries)
|
||||
- Include process ID in all metrics
|
||||
- Configuration-based sampling rates
|
||||
- Operation types: database, http, render
|
||||
|
||||
Example usage:
|
||||
>>> from starpunk.monitoring import record_metric, get_metrics
|
||||
>>>
|
||||
>>> # Record a database operation
|
||||
>>> record_metric('database', 'query', duration_ms=45.2, query='SELECT * FROM notes')
|
||||
>>>
|
||||
>>> # Get all metrics
|
||||
>>> metrics = get_metrics()
|
||||
>>> print(f"Collected {len(metrics)} metrics")
|
||||
"""
|
||||
|
||||
import os
|
||||
import random
|
||||
import time
|
||||
from collections import deque
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from datetime import datetime
|
||||
from threading import Lock
|
||||
from typing import Any, Deque, Dict, List, Literal, Optional, Union
|
||||
|
||||
# Operation types for categorizing metrics
|
||||
OperationType = Literal["database", "http", "render"]
|
||||
|
||||
# Module-level circular buffer (per-process)
|
||||
# Each process in a multi-process deployment maintains its own buffer
|
||||
_metrics_buffer: Optional["MetricsBuffer"] = None
|
||||
_buffer_lock = Lock()
|
||||
|
||||
|
||||
@dataclass
|
||||
class Metric:
|
||||
"""
|
||||
Represents a single performance metric
|
||||
|
||||
Attributes:
|
||||
operation_type: Type of operation (database/http/render)
|
||||
operation_name: Name/description of operation
|
||||
timestamp: When the metric was recorded (ISO format)
|
||||
duration_ms: Duration in milliseconds
|
||||
process_id: Process ID that recorded the metric
|
||||
metadata: Additional operation-specific data
|
||||
"""
|
||||
operation_type: OperationType
|
||||
operation_name: str
|
||||
timestamp: str
|
||||
duration_ms: float
|
||||
process_id: int
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert metric to dictionary for serialization"""
|
||||
return asdict(self)
|
||||
|
||||
|
||||
class MetricsBuffer:
|
||||
"""
|
||||
Circular buffer for storing performance metrics
|
||||
|
||||
Per developer Q&A Q6:
|
||||
- Uses deque for efficient circular buffer
|
||||
- Per-process storage (not shared across workers)
|
||||
- Thread-safe with locking
|
||||
- Configurable max size (default 1000)
|
||||
- Automatic eviction of oldest entries when full
|
||||
|
||||
Per developer Q&A Q12:
|
||||
- Configurable sampling rates per operation type
|
||||
- Default 100% sampling (suitable for low-traffic sites)
|
||||
- Slow queries always logged regardless of sampling
|
||||
|
||||
Example:
|
||||
>>> buffer = MetricsBuffer(max_size=1000)
|
||||
>>> buffer.record('database', 'query', 45.2, {'query': 'SELECT ...'})
|
||||
>>> metrics = buffer.get_all()
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
max_size: int = 1000,
|
||||
sampling_rates: Optional[Union[Dict[OperationType, float], float]] = None
|
||||
):
|
||||
"""
|
||||
Initialize metrics buffer
|
||||
|
||||
Args:
|
||||
max_size: Maximum number of metrics to store
|
||||
sampling_rates: Either:
|
||||
- float: Global sampling rate for all operation types (0.0-1.0)
|
||||
- dict: Mapping operation type to sampling rate
|
||||
Default: 1.0 (100% sampling)
|
||||
"""
|
||||
self.max_size = max_size
|
||||
self._buffer: Deque[Metric] = deque(maxlen=max_size)
|
||||
self._lock = Lock()
|
||||
self._process_id = os.getpid()
|
||||
|
||||
# Handle different sampling_rates types
|
||||
if sampling_rates is None:
|
||||
# Default to 100% sampling for all types
|
||||
self._sampling_rates = {
|
||||
"database": 1.0,
|
||||
"http": 1.0,
|
||||
"render": 1.0,
|
||||
}
|
||||
elif isinstance(sampling_rates, (int, float)):
|
||||
# Global rate for all types
|
||||
rate = float(sampling_rates)
|
||||
self._sampling_rates = {
|
||||
"database": rate,
|
||||
"http": rate,
|
||||
"render": rate,
|
||||
}
|
||||
else:
|
||||
# Dict with per-type rates
|
||||
self._sampling_rates = sampling_rates
|
||||
|
||||
def record(
|
||||
self,
|
||||
operation_type: OperationType,
|
||||
operation_name: str,
|
||||
duration_ms: float,
|
||||
metadata: Optional[Dict[str, Any]] = None,
|
||||
force: bool = False
|
||||
) -> bool:
|
||||
"""
|
||||
Record a performance metric
|
||||
|
||||
Args:
|
||||
operation_type: Type of operation (database/http/render)
|
||||
operation_name: Name/description of operation
|
||||
duration_ms: Duration in milliseconds
|
||||
metadata: Additional operation-specific data
|
||||
force: If True, bypass sampling (for slow query logging)
|
||||
|
||||
Returns:
|
||||
True if metric was recorded, False if skipped due to sampling
|
||||
|
||||
Example:
|
||||
>>> buffer.record('database', 'SELECT notes', 45.2,
|
||||
... {'query': 'SELECT * FROM notes LIMIT 10'})
|
||||
True
|
||||
"""
|
||||
# Apply sampling (unless forced)
|
||||
if not force:
|
||||
sampling_rate = self._sampling_rates.get(operation_type, 0.1)
|
||||
if random.random() > sampling_rate:
|
||||
return False
|
||||
|
||||
metric = Metric(
|
||||
operation_type=operation_type,
|
||||
operation_name=operation_name,
|
||||
timestamp=datetime.utcnow().isoformat() + "Z",
|
||||
duration_ms=duration_ms,
|
||||
process_id=self._process_id,
|
||||
metadata=metadata or {}
|
||||
)
|
||||
|
||||
with self._lock:
|
||||
self._buffer.append(metric)
|
||||
|
||||
return True
|
||||
|
||||
def get_all(self) -> List[Metric]:
|
||||
"""
|
||||
Get all metrics from buffer
|
||||
|
||||
Returns:
|
||||
List of metrics (oldest to newest)
|
||||
|
||||
Example:
|
||||
>>> metrics = buffer.get_all()
|
||||
>>> len(metrics)
|
||||
1000
|
||||
"""
|
||||
with self._lock:
|
||||
return list(self._buffer)
|
||||
|
||||
def get_recent(self, count: int) -> List[Metric]:
|
||||
"""
|
||||
Get most recent N metrics
|
||||
|
||||
Args:
|
||||
count: Number of recent metrics to return
|
||||
|
||||
Returns:
|
||||
List of most recent metrics (newest first)
|
||||
|
||||
Example:
|
||||
>>> recent = buffer.get_recent(10)
|
||||
>>> len(recent)
|
||||
10
|
||||
"""
|
||||
with self._lock:
|
||||
# Convert to list, reverse to get newest first, then slice
|
||||
all_metrics = list(self._buffer)
|
||||
all_metrics.reverse()
|
||||
return all_metrics[:count]
|
||||
|
||||
def get_by_type(self, operation_type: OperationType) -> List[Metric]:
|
||||
"""
|
||||
Get all metrics of a specific type
|
||||
|
||||
Args:
|
||||
operation_type: Type to filter by (database/http/render)
|
||||
|
||||
Returns:
|
||||
List of metrics matching the type
|
||||
|
||||
Example:
|
||||
>>> db_metrics = buffer.get_by_type('database')
|
||||
"""
|
||||
with self._lock:
|
||||
return [m for m in self._buffer if m.operation_type == operation_type]
|
||||
|
||||
def get_slow_operations(
|
||||
self,
|
||||
threshold_ms: float = 1000.0,
|
||||
operation_type: Optional[OperationType] = None
|
||||
) -> List[Metric]:
|
||||
"""
|
||||
Get operations that exceeded a duration threshold
|
||||
|
||||
Args:
|
||||
threshold_ms: Duration threshold in milliseconds
|
||||
operation_type: Optional type filter
|
||||
|
||||
Returns:
|
||||
List of slow operations
|
||||
|
||||
Example:
|
||||
>>> slow_queries = buffer.get_slow_operations(1000, 'database')
|
||||
"""
|
||||
with self._lock:
|
||||
metrics = list(self._buffer)
|
||||
|
||||
# Filter by type if specified
|
||||
if operation_type:
|
||||
metrics = [m for m in metrics if m.operation_type == operation_type]
|
||||
|
||||
# Filter by duration threshold
|
||||
return [m for m in metrics if m.duration_ms >= threshold_ms]
|
||||
|
||||
def get_stats(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Get statistics about the buffer
|
||||
|
||||
Returns:
|
||||
Dict with buffer statistics
|
||||
|
||||
Example:
|
||||
>>> stats = buffer.get_stats()
|
||||
>>> stats['total_count']
|
||||
1000
|
||||
"""
|
||||
with self._lock:
|
||||
metrics = list(self._buffer)
|
||||
|
||||
# Calculate stats per operation type
|
||||
type_stats = {}
|
||||
for op_type in ["database", "http", "render"]:
|
||||
type_metrics = [m for m in metrics if m.operation_type == op_type]
|
||||
if type_metrics:
|
||||
durations = [m.duration_ms for m in type_metrics]
|
||||
type_stats[op_type] = {
|
||||
"count": len(type_metrics),
|
||||
"avg_duration_ms": sum(durations) / len(durations),
|
||||
"min_duration_ms": min(durations),
|
||||
"max_duration_ms": max(durations),
|
||||
}
|
||||
else:
|
||||
type_stats[op_type] = {
|
||||
"count": 0,
|
||||
"avg_duration_ms": 0.0,
|
||||
"min_duration_ms": 0.0,
|
||||
"max_duration_ms": 0.0,
|
||||
}
|
||||
|
||||
return {
|
||||
"total_count": len(metrics),
|
||||
"max_size": self.max_size,
|
||||
"process_id": self._process_id,
|
||||
"sampling_rates": self._sampling_rates,
|
||||
"by_type": type_stats,
|
||||
}
|
||||
|
||||
def clear(self) -> None:
|
||||
"""
|
||||
Clear all metrics from buffer
|
||||
|
||||
Example:
|
||||
>>> buffer.clear()
|
||||
"""
|
||||
with self._lock:
|
||||
self._buffer.clear()
|
||||
|
||||
def set_sampling_rate(
|
||||
self,
|
||||
operation_type: OperationType,
|
||||
rate: float
|
||||
) -> None:
|
||||
"""
|
||||
Update sampling rate for an operation type
|
||||
|
||||
Args:
|
||||
operation_type: Type to update
|
||||
rate: New sampling rate (0.0-1.0)
|
||||
|
||||
Example:
|
||||
>>> buffer.set_sampling_rate('database', 0.5) # 50% sampling
|
||||
"""
|
||||
if not 0.0 <= rate <= 1.0:
|
||||
raise ValueError("Sampling rate must be between 0.0 and 1.0")
|
||||
|
||||
with self._lock:
|
||||
self._sampling_rates[operation_type] = rate
|
||||
|
||||
|
||||
def get_buffer() -> MetricsBuffer:
|
||||
"""
|
||||
Get or create the module-level metrics buffer
|
||||
|
||||
This ensures a single buffer per process. In multi-process deployments
|
||||
(e.g., gunicorn), each worker process will have its own buffer.
|
||||
|
||||
Returns:
|
||||
MetricsBuffer instance for this process
|
||||
|
||||
Example:
|
||||
>>> buffer = get_buffer()
|
||||
>>> buffer.record('database', 'query', 45.2)
|
||||
"""
|
||||
global _metrics_buffer
|
||||
|
||||
if _metrics_buffer is None:
|
||||
with _buffer_lock:
|
||||
# Double-check locking pattern
|
||||
if _metrics_buffer is None:
|
||||
# Get configuration from Flask app if available
|
||||
try:
|
||||
from flask import current_app
|
||||
max_size = current_app.config.get('METRICS_BUFFER_SIZE', 1000)
|
||||
sampling_rate = current_app.config.get('METRICS_SAMPLING_RATE', 1.0)
|
||||
except (ImportError, RuntimeError):
|
||||
# Flask not available or no app context
|
||||
max_size = 1000
|
||||
sampling_rate = 1.0 # Default to 100%
|
||||
|
||||
_metrics_buffer = MetricsBuffer(
|
||||
max_size=max_size,
|
||||
sampling_rates=sampling_rate
|
||||
)
|
||||
|
||||
return _metrics_buffer
|
||||
|
||||
|
||||
def record_metric(
|
||||
operation_type: OperationType,
|
||||
operation_name: str,
|
||||
duration_ms: float,
|
||||
metadata: Optional[Dict[str, Any]] = None,
|
||||
force: bool = False
|
||||
) -> bool:
|
||||
"""
|
||||
Record a metric using the module-level buffer
|
||||
|
||||
Convenience function that uses get_buffer() internally.
|
||||
|
||||
Args:
|
||||
operation_type: Type of operation (database/http/render)
|
||||
operation_name: Name/description of operation
|
||||
duration_ms: Duration in milliseconds
|
||||
metadata: Additional operation-specific data
|
||||
force: If True, bypass sampling (for slow query logging)
|
||||
|
||||
Returns:
|
||||
True if metric was recorded, False if skipped due to sampling
|
||||
|
||||
Example:
|
||||
>>> record_metric('database', 'SELECT notes', 45.2,
|
||||
... {'query': 'SELECT * FROM notes LIMIT 10'})
|
||||
True
|
||||
"""
|
||||
buffer = get_buffer()
|
||||
return buffer.record(operation_type, operation_name, duration_ms, metadata, force)
|
||||
|
||||
|
||||
def get_metrics() -> List[Metric]:
|
||||
"""
|
||||
Get all metrics from the module-level buffer
|
||||
|
||||
Returns:
|
||||
List of metrics (oldest to newest)
|
||||
|
||||
Example:
|
||||
>>> metrics = get_metrics()
|
||||
>>> len(metrics)
|
||||
1000
|
||||
"""
|
||||
buffer = get_buffer()
|
||||
return buffer.get_all()
|
||||
|
||||
|
||||
def get_metrics_stats() -> Dict[str, Any]:
|
||||
"""
|
||||
Get statistics from the module-level buffer
|
||||
|
||||
Returns:
|
||||
Dict with buffer statistics
|
||||
|
||||
Example:
|
||||
>>> stats = get_metrics_stats()
|
||||
>>> print(f"Total metrics: {stats['total_count']}")
|
||||
"""
|
||||
buffer = get_buffer()
|
||||
return buffer.get_stats()
|
||||
+4
-4
@@ -20,7 +20,7 @@ Exceptions:
|
||||
"""
|
||||
|
||||
# Standard library imports
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
@@ -226,7 +226,7 @@ def create_note(
|
||||
|
||||
# 2. SETUP
|
||||
if created_at is None:
|
||||
created_at = datetime.utcnow()
|
||||
created_at = datetime.now(timezone.utc).replace(tzinfo=None)
|
||||
|
||||
updated_at = created_at # Same as created_at for new notes
|
||||
|
||||
@@ -735,7 +735,7 @@ def update_note(
|
||||
)
|
||||
|
||||
# 3. SETUP
|
||||
updated_at = datetime.utcnow()
|
||||
updated_at = datetime.now(timezone.utc).replace(tzinfo=None)
|
||||
data_dir = Path(current_app.config["DATA_PATH"])
|
||||
note_path = data_dir / existing_note.file_path
|
||||
|
||||
@@ -1052,7 +1052,7 @@ def delete_note(
|
||||
|
||||
if soft:
|
||||
# SOFT DELETE: Mark as deleted in database
|
||||
deleted_at = datetime.utcnow()
|
||||
deleted_at = datetime.now(timezone.utc).replace(tzinfo=None)
|
||||
|
||||
try:
|
||||
db.execute(
|
||||
|
||||
@@ -138,6 +138,79 @@ _ip_limiter: RateLimiter | None = None
|
||||
_domain_limiter: RateLimiter | None = None
|
||||
|
||||
|
||||
class SharedRateLimiter:
|
||||
"""
|
||||
Rate limiter whose counts live in SQLite, shared by every gunicorn
|
||||
process (v2.11.0). In-memory counters let each of the 4 workers allow the
|
||||
full limit, so the real limit was four times the setting.
|
||||
|
||||
Sliding-window counter: the current window's count plus the previous
|
||||
window's, weighted by how much of it still overlaps the last
|
||||
`window_seconds`. Same interface as RateLimiter; needs an app context.
|
||||
"""
|
||||
|
||||
CLEANUP_EVERY = 200 # calls between removals of old windows
|
||||
|
||||
def __init__(self, limit: int, window_seconds: int, prefix: str):
|
||||
self.limit = limit
|
||||
self.window_seconds = window_seconds
|
||||
self.prefix = prefix
|
||||
self._calls = 0
|
||||
|
||||
def _db(self):
|
||||
from flask import current_app
|
||||
from starpunk.database import get_db
|
||||
return get_db(current_app)
|
||||
|
||||
def _windows(self):
|
||||
now = time.time()
|
||||
current = int(now // self.window_seconds)
|
||||
overlap = 1 - (now % self.window_seconds) / self.window_seconds
|
||||
return now, current, overlap
|
||||
|
||||
def _estimate(self, db, key: str) -> float:
|
||||
_, current, overlap = self._windows()
|
||||
counts = dict(db.execute(
|
||||
"SELECT window, count FROM rate_limits WHERE key = ? AND window IN (?, ?)",
|
||||
(f"{self.prefix}:{key}", current - 1, current),
|
||||
).fetchall())
|
||||
return counts.get(current - 1, 0) * overlap + counts.get(current, 0)
|
||||
|
||||
def is_allowed(self, key: str) -> bool:
|
||||
db = self._db()
|
||||
_, current, _ = self._windows()
|
||||
if self._estimate(db, key) >= self.limit:
|
||||
return False
|
||||
db.execute(
|
||||
"INSERT INTO rate_limits (key, window, count) VALUES (?, ?, 1) "
|
||||
"ON CONFLICT(key, window) DO UPDATE SET count = count + 1",
|
||||
(f"{self.prefix}:{key}", current),
|
||||
)
|
||||
db.commit()
|
||||
self._calls += 1
|
||||
if self._calls % self.CLEANUP_EVERY == 0:
|
||||
self.cleanup()
|
||||
return True
|
||||
|
||||
def get_retry_after(self, key: str) -> int:
|
||||
now, current, _ = self._windows()
|
||||
return max(1, int((current + 1) * self.window_seconds - now) + 1)
|
||||
|
||||
def get_request_count(self, key: str) -> int:
|
||||
return int(self._estimate(self._db(), key))
|
||||
|
||||
def cleanup(self, max_idle_seconds: int = 300) -> int:
|
||||
"""Remove windows too old to matter (this limiter's keys only)"""
|
||||
_, current, _ = self._windows()
|
||||
db = self._db()
|
||||
cursor = db.execute(
|
||||
"DELETE FROM rate_limits WHERE key LIKE ? AND window < ?",
|
||||
(f"{self.prefix}:%", current - 1),
|
||||
)
|
||||
db.commit()
|
||||
return cursor.rowcount
|
||||
|
||||
|
||||
def init_rate_limiters(app) -> None:
|
||||
"""
|
||||
Initialize rate limiters with Flask app configuration.
|
||||
@@ -157,8 +230,9 @@ def init_rate_limiters(app) -> None:
|
||||
domain_limit = app.config.get('WEBMENTION_RATE_LIMIT_DOMAIN', 10)
|
||||
window = app.config.get('WEBMENTION_RATE_LIMIT_WINDOW', 60)
|
||||
|
||||
_ip_limiter = RateLimiter(ip_limit, window)
|
||||
_domain_limiter = RateLimiter(domain_limit, window)
|
||||
# Shared by all server processes (v2.11.0)
|
||||
_ip_limiter = SharedRateLimiter(ip_limit, window, "webmention-ip")
|
||||
_domain_limiter = SharedRateLimiter(domain_limit, window, "webmention-domain")
|
||||
|
||||
app.logger.info(
|
||||
f"Webmention rate limiter initialized: {ip_limit}/min IP, "
|
||||
|
||||
+1
-207
@@ -9,7 +9,6 @@ from flask import Blueprint, flash, g, jsonify, redirect, render_template, reque
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
from datetime import datetime
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -366,202 +365,6 @@ def delete_note_submit(note_id: int):
|
||||
return redirect(url_for("admin.dashboard"))
|
||||
|
||||
|
||||
def transform_metrics_for_template(metrics_stats):
|
||||
"""
|
||||
Transform metrics stats to match template structure
|
||||
|
||||
The template expects direct access to metrics.database.count, but
|
||||
get_metrics_stats() returns metrics.by_type.database.count.
|
||||
This function adapts the data structure to match template expectations.
|
||||
|
||||
Args:
|
||||
metrics_stats: Dict from get_metrics_stats() with nested by_type structure
|
||||
|
||||
Returns:
|
||||
Dict with flattened structure matching template expectations
|
||||
|
||||
Per ADR-060: Route Adapter Pattern for template compatibility
|
||||
"""
|
||||
transformed = {}
|
||||
|
||||
# Map by_type to direct access
|
||||
for op_type in ['database', 'http', 'render']:
|
||||
if 'by_type' in metrics_stats and op_type in metrics_stats['by_type']:
|
||||
type_data = metrics_stats['by_type'][op_type]
|
||||
transformed[op_type] = {
|
||||
'count': type_data.get('count', 0),
|
||||
'avg': type_data.get('avg_duration_ms', 0),
|
||||
'min': type_data.get('min_duration_ms', 0),
|
||||
'max': type_data.get('max_duration_ms', 0)
|
||||
}
|
||||
else:
|
||||
# Provide defaults for missing types or when by_type doesn't exist
|
||||
transformed[op_type] = {
|
||||
'count': 0,
|
||||
'avg': 0,
|
||||
'min': 0,
|
||||
'max': 0
|
||||
}
|
||||
|
||||
# Keep other top-level stats
|
||||
transformed['total_count'] = metrics_stats.get('total_count', 0)
|
||||
transformed['max_size'] = metrics_stats.get('max_size', 1000)
|
||||
transformed['process_id'] = metrics_stats.get('process_id', 0)
|
||||
|
||||
return transformed
|
||||
|
||||
|
||||
@bp.route("/metrics-dashboard")
|
||||
@require_auth
|
||||
def metrics_dashboard():
|
||||
"""
|
||||
Metrics visualization dashboard (Phase 3)
|
||||
|
||||
Displays performance metrics, database statistics, feed statistics,
|
||||
and system health with visual charts and auto-refresh capability.
|
||||
|
||||
Per Q19 requirements:
|
||||
- Server-side rendering with Jinja2
|
||||
- htmx for auto-refresh
|
||||
- Chart.js from CDN for graphs
|
||||
- Progressive enhancement (works without JS)
|
||||
|
||||
Per v1.1.2 Phase 3:
|
||||
- Feed statistics by format
|
||||
- Cache hit/miss rates
|
||||
- Format popularity breakdown
|
||||
|
||||
Returns:
|
||||
Rendered dashboard template with metrics
|
||||
|
||||
Decorator: @require_auth
|
||||
Template: templates/admin/metrics_dashboard.html
|
||||
"""
|
||||
# Defensive imports with graceful degradation for missing modules
|
||||
try:
|
||||
from starpunk.database.pool import get_pool_stats
|
||||
from starpunk.monitoring import get_metrics_stats
|
||||
from starpunk.monitoring.business import get_feed_statistics
|
||||
monitoring_available = True
|
||||
except ImportError:
|
||||
monitoring_available = False
|
||||
# Provide fallback functions that return error messages
|
||||
def get_pool_stats():
|
||||
return {"error": "Database pool monitoring not available"}
|
||||
def get_metrics_stats():
|
||||
return {"error": "Monitoring module not implemented"}
|
||||
def get_feed_statistics():
|
||||
return {"error": "Feed statistics not available"}
|
||||
|
||||
# Get current metrics for initial page load
|
||||
metrics_data = {}
|
||||
pool_stats = {}
|
||||
feed_stats = {}
|
||||
|
||||
try:
|
||||
raw_metrics = get_metrics_stats()
|
||||
metrics_data = transform_metrics_for_template(raw_metrics)
|
||||
except Exception as e:
|
||||
flash(f"Error loading metrics: {e}", "warning")
|
||||
# Provide safe defaults matching template expectations
|
||||
metrics_data = {
|
||||
'database': {'count': 0, 'avg': 0, 'min': 0, 'max': 0},
|
||||
'http': {'count': 0, 'avg': 0, 'min': 0, 'max': 0},
|
||||
'render': {'count': 0, 'avg': 0, 'min': 0, 'max': 0},
|
||||
'total_count': 0,
|
||||
'max_size': 1000,
|
||||
'process_id': 0
|
||||
}
|
||||
|
||||
try:
|
||||
pool_stats = get_pool_stats()
|
||||
except Exception as e:
|
||||
flash(f"Error loading pool stats: {e}", "warning")
|
||||
|
||||
try:
|
||||
feed_stats = get_feed_statistics()
|
||||
except Exception as e:
|
||||
flash(f"Error loading feed stats: {e}", "warning")
|
||||
# Provide safe defaults
|
||||
feed_stats = {
|
||||
'by_format': {
|
||||
'rss': {'generated': 0, 'cached': 0, 'total': 0, 'avg_duration_ms': 0.0},
|
||||
'atom': {'generated': 0, 'cached': 0, 'total': 0, 'avg_duration_ms': 0.0},
|
||||
'json': {'generated': 0, 'cached': 0, 'total': 0, 'avg_duration_ms': 0.0},
|
||||
},
|
||||
'cache': {'hits': 0, 'misses': 0, 'hit_rate': 0.0, 'entries': 0, 'evictions': 0},
|
||||
'total_requests': 0,
|
||||
'format_percentages': {'rss': 0.0, 'atom': 0.0, 'json': 0.0},
|
||||
}
|
||||
|
||||
return render_template(
|
||||
"admin/metrics_dashboard.html",
|
||||
metrics=metrics_data,
|
||||
pool=pool_stats,
|
||||
feeds=feed_stats,
|
||||
user_me=g.me
|
||||
)
|
||||
|
||||
|
||||
@bp.route("/metrics")
|
||||
@require_auth
|
||||
def metrics():
|
||||
"""
|
||||
Performance metrics and database pool statistics endpoint
|
||||
|
||||
Per Phase 2 requirements:
|
||||
- Expose database pool statistics
|
||||
- Show performance metrics from MetricsBuffer
|
||||
- Requires authentication
|
||||
|
||||
Per v1.1.2 Phase 3:
|
||||
- Include feed statistics
|
||||
|
||||
Returns:
|
||||
JSON with metrics, pool statistics, and feed statistics
|
||||
|
||||
Response codes:
|
||||
200: Metrics retrieved successfully
|
||||
|
||||
Decorator: @require_auth
|
||||
"""
|
||||
from flask import current_app
|
||||
from starpunk.database.pool import get_pool_stats
|
||||
from starpunk.monitoring import get_metrics_stats
|
||||
from starpunk.monitoring.business import get_feed_statistics
|
||||
|
||||
response = {
|
||||
"timestamp": datetime.utcnow().isoformat() + "Z",
|
||||
"process_id": os.getpid(),
|
||||
"database": {},
|
||||
"performance": {},
|
||||
"feeds": {}
|
||||
}
|
||||
|
||||
# Get database pool statistics
|
||||
try:
|
||||
pool_stats = get_pool_stats()
|
||||
response["database"]["pool"] = pool_stats
|
||||
except Exception as e:
|
||||
response["database"]["pool"] = {"error": str(e)}
|
||||
|
||||
# Get performance metrics
|
||||
try:
|
||||
metrics_stats = get_metrics_stats()
|
||||
response["performance"] = metrics_stats
|
||||
except Exception as e:
|
||||
response["performance"] = {"error": str(e)}
|
||||
|
||||
# Get feed statistics
|
||||
try:
|
||||
feed_stats = get_feed_statistics()
|
||||
response["feeds"] = feed_stats
|
||||
except Exception as e:
|
||||
response["feeds"] = {"error": str(e)}
|
||||
|
||||
return jsonify(response), 200
|
||||
|
||||
|
||||
@bp.route("/health")
|
||||
@require_auth
|
||||
def health_diagnostics():
|
||||
@@ -571,7 +374,7 @@ def health_diagnostics():
|
||||
Per developer Q&A Q10:
|
||||
- Always requires authentication
|
||||
- Provides comprehensive diagnostics
|
||||
- Includes metrics, database pool statistics, and system info
|
||||
- Includes database pool statistics and system info
|
||||
|
||||
Returns:
|
||||
JSON with complete system diagnostics
|
||||
@@ -591,7 +394,6 @@ def health_diagnostics():
|
||||
"environment": current_app.config.get("ENV", "unknown"),
|
||||
"process_id": os.getpid(),
|
||||
"checks": {},
|
||||
"metrics": {},
|
||||
"database": {}
|
||||
}
|
||||
|
||||
@@ -665,14 +467,6 @@ def health_diagnostics():
|
||||
}
|
||||
overall_healthy = False
|
||||
|
||||
# Performance metrics
|
||||
try:
|
||||
from starpunk.monitoring import get_metrics_stats
|
||||
metrics_stats = get_metrics_stats()
|
||||
diagnostics["metrics"] = metrics_stats
|
||||
except Exception as e:
|
||||
diagnostics["metrics"] = {"error": str(e)}
|
||||
|
||||
# Update overall status
|
||||
diagnostics["status"] = "healthy" if overall_healthy else "unhealthy"
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ No authentication required for these routes.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from flask import Blueprint, abort, render_template, Response, current_app, request, send_from_directory
|
||||
|
||||
@@ -54,7 +54,7 @@ def _get_cached_notes():
|
||||
# Get cache duration from config (in seconds)
|
||||
cache_seconds = current_app.config.get("FEED_CACHE_SECONDS", 300)
|
||||
cache_duration = timedelta(seconds=cache_seconds)
|
||||
now = datetime.utcnow()
|
||||
now = datetime.now(timezone.utc).replace(tzinfo=None)
|
||||
|
||||
# The cached list is stale once any public note changed, in any process
|
||||
# (v2.10.0): every change bumps content_version in the database
|
||||
|
||||
@@ -9,16 +9,15 @@ optional hub.secret and hub.lease_seconds. Answers 202 and verifies later
|
||||
from flask import Blueprint, current_app, request
|
||||
|
||||
from starpunk.database import get_db
|
||||
from starpunk.rate_limiter import RateLimiter
|
||||
from starpunk.rate_limiter import SharedRateLimiter
|
||||
from starpunk import websub
|
||||
|
||||
bp = Blueprint("websub", __name__)
|
||||
|
||||
# Subscription requests per client IP per minute
|
||||
# (per gunicorn process; needs TRUSTED_PROXY=true behind a reverse proxy,
|
||||
# or every client shares the proxy's address)
|
||||
_limiter = RateLimiter(limit=30, window_seconds=60)
|
||||
_requests = 0
|
||||
# Shared by all server processes (v2.11.0). Needs TRUSTED_PROXY=true behind a
|
||||
# reverse proxy, or every client shares the proxy's address.
|
||||
_limiter = SharedRateLimiter(limit=30, window_seconds=60, prefix="websub")
|
||||
|
||||
|
||||
def _text(message: str, status: int):
|
||||
@@ -31,10 +30,6 @@ def hub():
|
||||
return _text("WebSub is not enabled on this site", 404)
|
||||
|
||||
from starpunk.routes.webmention import get_client_ip
|
||||
global _requests
|
||||
_requests += 1
|
||||
if _requests % 500 == 0:
|
||||
_limiter.cleanup() # forget idle clients so memory doesn't grow
|
||||
if not _limiter.is_allowed(get_client_ip()):
|
||||
return _text("Too many requests", 429)
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ Per developer Q&A Q8:
|
||||
import re
|
||||
import unicodedata
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional, Set
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -138,7 +138,7 @@ def sanitize_slug(slug: str, allow_timestamp_fallback: bool = False) -> str:
|
||||
# Check if normalization resulted in empty slug
|
||||
if not slug and allow_timestamp_fallback:
|
||||
# Per Q8: Use timestamp-based fallback
|
||||
timestamp = datetime.utcnow().strftime('%Y%m%d-%H%M%S')
|
||||
timestamp = datetime.now(timezone.utc).replace(tzinfo=None).strftime('%Y%m%d-%H%M%S')
|
||||
slug = timestamp
|
||||
logger.warning(
|
||||
f"Slug normalization failed for input '{original_slug}' "
|
||||
@@ -268,7 +268,7 @@ def generate_timestamp_slug(
|
||||
'20251216143052-2'
|
||||
"""
|
||||
if created_at is None:
|
||||
created_at = datetime.utcnow()
|
||||
created_at = datetime.now(timezone.utc).replace(tzinfo=None)
|
||||
|
||||
if existing_slugs is None:
|
||||
existing_slugs = set()
|
||||
@@ -340,7 +340,7 @@ def validate_and_sanitize_custom_slug(custom_slug: str, existing_slugs: Set[str]
|
||||
if not sanitized:
|
||||
# This should never happen with allow_timestamp_fallback=True
|
||||
# but handle it just in case
|
||||
timestamp = datetime.utcnow().strftime('%Y%m%d-%H%M%S')
|
||||
timestamp = datetime.now(timezone.utc).replace(tzinfo=None).strftime('%Y%m%d-%H%M%S')
|
||||
sanitized = timestamp
|
||||
logger.error(
|
||||
f"Unexpected empty slug after sanitization with fallback. "
|
||||
@@ -361,7 +361,7 @@ def validate_and_sanitize_custom_slug(custom_slug: str, existing_slugs: Set[str]
|
||||
if not validate_slug(sanitized):
|
||||
# This should rarely happen after sanitization
|
||||
# but if it does, use timestamp fallback
|
||||
timestamp = datetime.utcnow().strftime('%Y%m%d-%H%M%S')
|
||||
timestamp = datetime.now(timezone.utc).replace(tzinfo=None).strftime('%Y%m%d-%H%M%S')
|
||||
logger.warning(
|
||||
f"Slug '{sanitized}' (from '{custom_slug}') failed validation. "
|
||||
f"Using timestamp fallback: {timestamp}"
|
||||
@@ -375,7 +375,7 @@ def validate_and_sanitize_custom_slug(custom_slug: str, existing_slugs: Set[str]
|
||||
except ValueError as e:
|
||||
# This should rarely happen, but if it does, use timestamp
|
||||
# Per Q8: Never fail Micropub request
|
||||
timestamp = datetime.utcnow().strftime('%Y%m%d-%H%M%S')
|
||||
timestamp = datetime.now(timezone.utc).replace(tzinfo=None).strftime('%Y%m%d-%H%M%S')
|
||||
logger.error(
|
||||
f"Could not create unique slug from '{custom_slug}'. "
|
||||
f"Using timestamp: {timestamp}. Error: {e}"
|
||||
|
||||
+30
-32
@@ -444,39 +444,37 @@ CARD_IMAGE_LIMIT = 5 * 1024 * 1024
|
||||
|
||||
def _fetch_public(http: httpx.Client, url: str, limit: int, truncate: bool) -> tuple:
|
||||
"""
|
||||
GET a public web URL, checking every redirect hop (no requests into the
|
||||
LAN). Reads at most `limit` bytes: stops there if `truncate`, else fails.
|
||||
Returns (data, content_type, final_url).
|
||||
"""
|
||||
from urllib.parse import urljoin, urlparse
|
||||
from starpunk.micropub import _public_address
|
||||
GET a public web URL (link card pages, images to re-upload).
|
||||
|
||||
site = current_app.config.get("SITE_URL", "")
|
||||
for _ in range(5):
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme not in ("http", "https") or not parsed.hostname:
|
||||
raise SyndicationError(f"not a web URL: {url}", permanent=True)
|
||||
if not current_app.config.get("TESTING") and not _public_address(parsed.hostname):
|
||||
raise SyndicationError(f"not a public address: {url}", permanent=True)
|
||||
try:
|
||||
with http.stream("GET", url, follow_redirects=False, timeout=10.0,
|
||||
headers={"User-Agent": f"StarPunk (+{site})"}) as response:
|
||||
if response.is_redirect and "location" in response.headers:
|
||||
url = urljoin(url, response.headers["location"])
|
||||
continue
|
||||
if response.status_code >= 400:
|
||||
raise SyndicationError(f"{url}: HTTP {response.status_code}")
|
||||
data = bytearray()
|
||||
for chunk in response.iter_bytes():
|
||||
data += chunk
|
||||
if len(data) > limit:
|
||||
if truncate:
|
||||
break
|
||||
raise SyndicationError(f"{url}: too large")
|
||||
return bytes(data), response.headers.get("content-type", ""), url
|
||||
except httpx.HTTPError as e:
|
||||
raise SyndicationError(f"{url}: {type(e).__name__}")
|
||||
raise SyndicationError(f"{url}: too many redirects")
|
||||
Uses the shared SSRF-safe fetcher (v2.11.0): every redirect hop checked,
|
||||
the connection pinned to the checked address, a hard 10 s limit. Reads at
|
||||
most `limit` bytes: stops there if `truncate`, else fails.
|
||||
Returns (data, content_type, final_url). `http` is no longer used.
|
||||
"""
|
||||
from urllib.parse import urlparse
|
||||
from starpunk.webmention_ssrf import (
|
||||
FetchTooLargeError, ResolutionError, SSRFBlockedError, safe_get, user_agent,
|
||||
)
|
||||
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme not in ("http", "https") or not parsed.hostname:
|
||||
raise SyndicationError(f"not a web URL: {url}", permanent=True)
|
||||
try:
|
||||
result = safe_get(url, headers={"User-Agent": user_agent(current_app.config.get("SITE_URL", ""))},
|
||||
max_bytes=limit, max_redirects=5, timeout=10.0, truncate=truncate)
|
||||
except ResolutionError as e:
|
||||
raise SyndicationError(f"{url}: {e}")
|
||||
except (SSRFBlockedError, ValueError) as e:
|
||||
raise SyndicationError(f"not a public address: {url} ({e})", permanent=True)
|
||||
except FetchTooLargeError:
|
||||
raise SyndicationError(f"{url}: too large")
|
||||
except httpx.TooManyRedirects:
|
||||
raise SyndicationError(f"{url}: too many redirects")
|
||||
except httpx.HTTPError as e:
|
||||
raise SyndicationError(f"{url}: {type(e).__name__}")
|
||||
if result.status_code >= 400:
|
||||
raise SyndicationError(f"{url}: HTTP {result.status_code}")
|
||||
return result.content, result.headers.get("content-type", ""), result.url
|
||||
|
||||
|
||||
def page_card(http: httpx.Client, url: str) -> dict:
|
||||
|
||||
+2
-2
@@ -217,7 +217,7 @@ def generate_slug(content: str, created_at: Optional[datetime] = None) -> str:
|
||||
# If slug is empty or too short, use timestamp fallback
|
||||
if len(slug) < MIN_SLUG_LENGTH:
|
||||
if created_at is None:
|
||||
created_at = datetime.utcnow()
|
||||
created_at = datetime.now(timezone.utc).replace(tzinfo=None)
|
||||
slug = created_at.strftime("%Y%m%d-%H%M%S")
|
||||
|
||||
# Truncate to maximum length
|
||||
@@ -592,7 +592,7 @@ def delete_note_file(
|
||||
month = parts[notes_idx + 2]
|
||||
except (ValueError, IndexError):
|
||||
# If path doesn't follow expected structure, use current date
|
||||
now = datetime.utcnow()
|
||||
now = datetime.now(timezone.utc).replace(tzinfo=None)
|
||||
year = now.strftime("%Y")
|
||||
month = now.strftime("%m")
|
||||
|
||||
|
||||
@@ -250,7 +250,8 @@ def safe_post(url: str, data: Optional[dict] = None, headers: Optional[dict] = N
|
||||
|
||||
|
||||
def safe_get(url: str, headers: dict, max_bytes: int = MAX_BYTES,
|
||||
max_redirects: int = MAX_REDIRECTS, timeout: float = TIMEOUT) -> FetchResult:
|
||||
max_redirects: int = MAX_REDIRECTS, timeout: float = TIMEOUT,
|
||||
truncate: bool = False) -> FetchResult:
|
||||
"""
|
||||
GET a URL, following redirects by hand and checking each hop.
|
||||
|
||||
@@ -258,6 +259,8 @@ def safe_get(url: str, headers: dict, max_bytes: int = MAX_BYTES,
|
||||
url: URL to fetch
|
||||
headers: Request headers
|
||||
max_bytes: Response body limit
|
||||
truncate: Stop at max_bytes and return what was read, instead of
|
||||
raising FetchTooLargeError (v2.11.0)
|
||||
max_redirects: Redirect limit
|
||||
timeout: Total time limit in seconds
|
||||
|
||||
@@ -287,13 +290,16 @@ def safe_get(url: str, headers: dict, max_bytes: int = MAX_BYTES,
|
||||
continue
|
||||
|
||||
content_length = response.headers.get('Content-Length', '')
|
||||
if content_length.isdigit() and int(content_length) > max_bytes:
|
||||
if not truncate and content_length.isdigit() and int(content_length) > max_bytes:
|
||||
raise FetchTooLargeError(f"Response exceeds size limit ({max_bytes} bytes)")
|
||||
|
||||
content = b''
|
||||
for chunk in response.iter_bytes(chunk_size=8192):
|
||||
content += chunk
|
||||
if len(content) > max_bytes:
|
||||
if truncate:
|
||||
content = content[:max_bytes]
|
||||
break
|
||||
raise FetchTooLargeError(f"Response exceeds size limit ({max_bytes} bytes)")
|
||||
if time.monotonic() > deadline:
|
||||
raise httpx.TimeoutException("Fetch exceeded time limit")
|
||||
|
||||
@@ -19,7 +19,7 @@ import threading
|
||||
import time
|
||||
import sqlite3
|
||||
import httpx
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, Optional
|
||||
from urllib.parse import urljoin
|
||||
@@ -116,7 +116,7 @@ class WebmentionWorker(threading.Thread):
|
||||
|
||||
# Update last processed time
|
||||
if processed > 0:
|
||||
self._last_processed = datetime.utcnow()
|
||||
self._last_processed = datetime.now(timezone.utc).replace(tzinfo=None)
|
||||
|
||||
# Sleep if nothing processed
|
||||
if processed == 0:
|
||||
@@ -790,7 +790,7 @@ class WebmentionWorker(threading.Thread):
|
||||
}
|
||||
delay = delays.get(new_retry_count)
|
||||
if delay:
|
||||
next_retry_at = datetime.utcnow() + delay
|
||||
next_retry_at = datetime.now(timezone.utc).replace(tzinfo=None) + delay
|
||||
|
||||
try:
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
|
||||
@@ -25,7 +25,7 @@ import hashlib
|
||||
import hmac
|
||||
import logging
|
||||
import threading
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from urllib.parse import urlencode, urlparse, urlsplit, urlunsplit
|
||||
@@ -53,11 +53,11 @@ def signature(secret: str, body: bytes) -> str:
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
|
||||
return datetime.now(timezone.utc).replace(tzinfo=None).strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
|
||||
def _later(seconds: int) -> str:
|
||||
return (datetime.utcnow() + timedelta(seconds=seconds)).strftime("%Y-%m-%d %H:%M:%S")
|
||||
return (datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(seconds=seconds)).strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
|
||||
class WebSubWorker(threading.Thread):
|
||||
@@ -132,7 +132,7 @@ class WebSubWorker(threading.Thread):
|
||||
).fetchall():
|
||||
self._guarded(db, row, "delivery", lambda d, r: self.deliver(d, r, rendered), self._retry_or_drop)
|
||||
done += 1
|
||||
if self._last_prune is None or datetime.utcnow() - self._last_prune > PRUNE_EVERY:
|
||||
if self._last_prune is None or datetime.now(timezone.utc).replace(tzinfo=None) - self._last_prune > PRUNE_EVERY:
|
||||
self.prune(db)
|
||||
return done
|
||||
|
||||
@@ -348,7 +348,7 @@ class WebSubWorker(threading.Thread):
|
||||
db.execute("DELETE FROM websub_deliveries WHERE subscription_id IS NOT NULL AND "
|
||||
"subscription_id NOT IN (SELECT id FROM websub_subscriptions)")
|
||||
db.commit()
|
||||
self._last_prune = datetime.utcnow()
|
||||
self._last_prune = datetime.now(timezone.utc).replace(tzinfo=None)
|
||||
if cursor.rowcount:
|
||||
logger.info(f"WebSub: removed {cursor.rowcount} expired subscriptions")
|
||||
return cursor.rowcount
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
</p>
|
||||
<p><small>After approval you will be sent to {{ auth_request.redirect_uri }}</small></p>
|
||||
|
||||
<form action="{{ url_for('indieauth.approve') }}" method="POST" class="login-form">
|
||||
<form action="{{ url_for('indieauth.approve') }}" method="POST" class="login-form"><input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
|
||||
{% for name, value in fields.items() %}
|
||||
<input type="hidden" name="{{ name }}" value="{{ value }}">
|
||||
{% endfor %}
|
||||
|
||||
@@ -8,9 +8,8 @@
|
||||
<nav class="admin-nav">
|
||||
<a href="{{ url_for('admin.dashboard') }}">Dashboard</a>
|
||||
<a href="{{ url_for('admin.new_note_form') }}">New Note</a>
|
||||
<a href="{{ url_for('admin.metrics_dashboard') }}">Metrics</a>
|
||||
<a href="{{ url_for('admin.tokens') }}">Connected Apps</a>
|
||||
<form action="{{ url_for('auth.logout') }}" method="POST" class="logout-form">
|
||||
<form action="{{ url_for('auth.logout') }}" method="POST" class="logout-form"><input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
|
||||
<button type="submit" class="button button-secondary">Logout</button>
|
||||
</form>
|
||||
</nav>
|
||||
|
||||
@@ -74,7 +74,7 @@
|
||||
<a href="{{ url_for('admin.edit_note_form', note_id=note.id) }}" class="button button-small">
|
||||
Edit
|
||||
</a>
|
||||
<form action="{{ url_for('admin.delete_note_submit', note_id=note.id) }}" method="POST" class="delete-form" onsubmit="return confirm('Are you sure you want to delete this note?');">
|
||||
<form action="{{ url_for('admin.delete_note_submit', note_id=note.id) }}" method="POST" class="delete-form" onsubmit="return confirm('Are you sure you want to delete this note?');"><input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
|
||||
<input type="hidden" name="confirm" value="yes">
|
||||
<button type="submit" class="button button-small button-danger">Delete</button>
|
||||
</form>
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
Created: {{ note.created_at.strftime('%B %d, %Y at %I:%M %p') }}
|
||||
</p>
|
||||
|
||||
<form action="{{ url_for('admin.update_note_submit', note_id=note.id) }}" method="POST" class="note-form">
|
||||
<form action="{{ url_for('admin.update_note_submit', note_id=note.id) }}" method="POST" class="note-form"><input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
|
||||
<div class="form-group">
|
||||
{% if note.reacts_to %}
|
||||
<p class="note-meta">{{ {'like': '★ Like of', 'repost': '🔁 Repost of', 'bookmark': '🔖 Bookmark of'}[note.kind] }} <a href="{{ note.reacts_to }}">{{ note.reacts_to }}</a></p>
|
||||
@@ -76,7 +76,7 @@
|
||||
</form>
|
||||
|
||||
{# Outside the edit form: nested forms are not allowed in HTML #}
|
||||
<form id="delete-note-form" action="{{ url_for('admin.delete_note_submit', note_id=note.id) }}" method="POST" class="delete-form" onsubmit="return confirm('Are you sure you want to delete this note? This cannot be undone.');">
|
||||
<form id="delete-note-form" action="{{ url_for('admin.delete_note_submit', note_id=note.id) }}" method="POST" class="delete-form" onsubmit="return confirm('Are you sure you want to delete this note? This cannot be undone.');"><input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
|
||||
<input type="hidden" name="confirm" value="yes">
|
||||
</form>
|
||||
|
||||
@@ -98,7 +98,7 @@
|
||||
{% endif %}
|
||||
{% if s.last_error %}<br><small>{{ s.last_error }}</small>{% endif %}
|
||||
{% if s.state != 'synced' %}
|
||||
<form action="{{ url_for('admin.retry_syndication', syndication_id=s.id) }}" method="POST" style="display: inline;">
|
||||
<form action="{{ url_for('admin.retry_syndication', syndication_id=s.id) }}" method="POST" style="display: inline;"><input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
|
||||
<button type="submit" class="button button-secondary">Retry now</button>
|
||||
</form>
|
||||
{% endif %}
|
||||
|
||||
@@ -1,585 +0,0 @@
|
||||
{% extends "admin/base.html" %}
|
||||
|
||||
{% block title %}Metrics Dashboard - StarPunk Admin{% endblock %}
|
||||
|
||||
{% block head %}
|
||||
{{ super() }}
|
||||
<!-- Chart.js from CDN for visualizations -->
|
||||
<script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.0/dist/chart.umd.min.js" crossorigin="anonymous"></script>
|
||||
<!-- htmx for auto-refresh -->
|
||||
<script src="https://unpkg.com/htmx.org@1.9.10" crossorigin="anonymous"></script>
|
||||
<style>
|
||||
.metrics-dashboard {
|
||||
max-width: 1200px;
|
||||
}
|
||||
|
||||
.metrics-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(300px, 1fr));
|
||||
gap: 20px;
|
||||
margin-bottom: 30px;
|
||||
}
|
||||
|
||||
.metric-card {
|
||||
background: #fff;
|
||||
border: 1px solid #ddd;
|
||||
border-radius: 8px;
|
||||
padding: 20px;
|
||||
box-shadow: 0 2px 4px rgba(0,0,0,0.05);
|
||||
}
|
||||
|
||||
.metric-card h3 {
|
||||
margin-top: 0;
|
||||
font-size: 1.1em;
|
||||
color: #333;
|
||||
border-bottom: 2px solid #007bff;
|
||||
padding-bottom: 10px;
|
||||
margin-bottom: 15px;
|
||||
}
|
||||
|
||||
.metric-value {
|
||||
font-size: 2em;
|
||||
font-weight: bold;
|
||||
color: #007bff;
|
||||
margin: 10px 0;
|
||||
}
|
||||
|
||||
.metric-label {
|
||||
color: #666;
|
||||
font-size: 0.9em;
|
||||
margin-bottom: 5px;
|
||||
}
|
||||
|
||||
.metric-detail {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
padding: 8px 0;
|
||||
border-bottom: 1px solid #f0f0f0;
|
||||
}
|
||||
|
||||
.metric-detail:last-child {
|
||||
border-bottom: none;
|
||||
}
|
||||
|
||||
.metric-detail-label {
|
||||
color: #666;
|
||||
}
|
||||
|
||||
.metric-detail-value {
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
.chart-container {
|
||||
position: relative;
|
||||
height: 300px;
|
||||
margin-top: 20px;
|
||||
}
|
||||
|
||||
.status-indicator {
|
||||
display: inline-block;
|
||||
width: 12px;
|
||||
height: 12px;
|
||||
border-radius: 50%;
|
||||
margin-right: 8px;
|
||||
}
|
||||
|
||||
.status-healthy {
|
||||
background-color: #28a745;
|
||||
}
|
||||
|
||||
.status-warning {
|
||||
background-color: #ffc107;
|
||||
}
|
||||
|
||||
.status-error {
|
||||
background-color: #dc3545;
|
||||
}
|
||||
|
||||
.refresh-info {
|
||||
color: #666;
|
||||
font-size: 0.9em;
|
||||
text-align: center;
|
||||
margin-top: 20px;
|
||||
padding: 10px;
|
||||
background-color: #f8f9fa;
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
.no-js-message {
|
||||
display: none;
|
||||
background-color: #fff3cd;
|
||||
border: 1px solid #ffeaa7;
|
||||
color: #856404;
|
||||
padding: 15px;
|
||||
border-radius: 4px;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
|
||||
noscript .no-js-message {
|
||||
display: block;
|
||||
}
|
||||
</style>
|
||||
{% endblock %}
|
||||
|
||||
{% block admin_content %}
|
||||
<div class="metrics-dashboard">
|
||||
<h2>Metrics Dashboard</h2>
|
||||
|
||||
<noscript>
|
||||
<div class="no-js-message">
|
||||
Note: Auto-refresh and charts require JavaScript. Data is displayed below in text format.
|
||||
</div>
|
||||
</noscript>
|
||||
|
||||
<!-- Auto-refresh container -->
|
||||
<div hx-get="{{ url_for('admin.metrics') }}" hx-trigger="every 10s" hx-swap="none" hx-on::after-request="updateDashboard(event)"></div>
|
||||
|
||||
<!-- Database Pool Statistics -->
|
||||
<div class="metrics-grid">
|
||||
<div class="metric-card">
|
||||
<h3>Database Connection Pool</h3>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Active Connections</span>
|
||||
<span class="metric-detail-value" id="pool-active">{{ pool.active_connections|default(0) }}</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Idle Connections</span>
|
||||
<span class="metric-detail-value" id="pool-idle">{{ pool.idle_connections|default(0) }}</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Total Connections</span>
|
||||
<span class="metric-detail-value" id="pool-total">{{ pool.total_connections|default(0) }}</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Pool Size</span>
|
||||
<span class="metric-detail-value" id="pool-size">{{ pool.pool_size|default(5) }}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="metric-card">
|
||||
<h3>Database Operations</h3>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Total Queries</span>
|
||||
<span class="metric-detail-value" id="db-total">{{ metrics.database.count|default(0) }}</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Average Time</span>
|
||||
<span class="metric-detail-value" id="db-avg">{{ "%.2f"|format(metrics.database.avg|default(0)) }} ms</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Min Time</span>
|
||||
<span class="metric-detail-value" id="db-min">{{ "%.2f"|format(metrics.database.min|default(0)) }} ms</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Max Time</span>
|
||||
<span class="metric-detail-value" id="db-max">{{ "%.2f"|format(metrics.database.max|default(0)) }} ms</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="metric-card">
|
||||
<h3>HTTP Requests</h3>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Total Requests</span>
|
||||
<span class="metric-detail-value" id="http-total">{{ metrics.http.count|default(0) }}</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Average Time</span>
|
||||
<span class="metric-detail-value" id="http-avg">{{ "%.2f"|format(metrics.http.avg|default(0)) }} ms</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Min Time</span>
|
||||
<span class="metric-detail-value" id="http-min">{{ "%.2f"|format(metrics.http.min|default(0)) }} ms</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Max Time</span>
|
||||
<span class="metric-detail-value" id="http-max">{{ "%.2f"|format(metrics.http.max|default(0)) }} ms</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="metric-card">
|
||||
<h3>Template Rendering</h3>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Total Renders</span>
|
||||
<span class="metric-detail-value" id="render-total">{{ metrics.render.count|default(0) }}</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Average Time</span>
|
||||
<span class="metric-detail-value" id="render-avg">{{ "%.2f"|format(metrics.render.avg|default(0)) }} ms</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Min Time</span>
|
||||
<span class="metric-detail-value" id="render-min">{{ "%.2f"|format(metrics.render.min|default(0)) }} ms</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Max Time</span>
|
||||
<span class="metric-detail-value" id="render-max">{{ "%.2f"|format(metrics.render.max|default(0)) }} ms</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Charts -->
|
||||
<div class="metrics-grid">
|
||||
<div class="metric-card">
|
||||
<h3>Connection Pool Usage</h3>
|
||||
<div class="chart-container">
|
||||
<canvas id="poolChart"></canvas>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="metric-card">
|
||||
<h3>Performance Overview</h3>
|
||||
<div class="chart-container">
|
||||
<canvas id="performanceChart"></canvas>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Feed Statistics (Phase 3) -->
|
||||
<h2 style="margin-top: 40px;">Feed Statistics</h2>
|
||||
<div class="metrics-grid">
|
||||
<div class="metric-card">
|
||||
<h3>Feed Requests by Format</h3>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">RSS</span>
|
||||
<span class="metric-detail-value" id="feed-rss-total">{{ feeds.by_format.rss.total|default(0) }}</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">ATOM</span>
|
||||
<span class="metric-detail-value" id="feed-atom-total">{{ feeds.by_format.atom.total|default(0) }}</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">JSON Feed</span>
|
||||
<span class="metric-detail-value" id="feed-json-total">{{ feeds.by_format.json.total|default(0) }}</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Total Requests</span>
|
||||
<span class="metric-detail-value" id="feed-total">{{ feeds.total_requests|default(0) }}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="metric-card">
|
||||
<h3>Feed Cache Statistics</h3>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Cache Hits</span>
|
||||
<span class="metric-detail-value" id="feed-cache-hits">{{ feeds.cache.hits|default(0) }}</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Cache Misses</span>
|
||||
<span class="metric-detail-value" id="feed-cache-misses">{{ feeds.cache.misses|default(0) }}</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Hit Rate</span>
|
||||
<span class="metric-detail-value" id="feed-cache-hit-rate">{{ "%.1f"|format(feeds.cache.hit_rate|default(0) * 100) }}%</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Cached Entries</span>
|
||||
<span class="metric-detail-value" id="feed-cache-entries">{{ feeds.cache.entries|default(0) }}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="metric-card">
|
||||
<h3>Feed Generation Performance</h3>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">RSS Avg Time</span>
|
||||
<span class="metric-detail-value" id="feed-rss-avg">{{ "%.2f"|format(feeds.by_format.rss.avg_duration_ms|default(0)) }} ms</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">ATOM Avg Time</span>
|
||||
<span class="metric-detail-value" id="feed-atom-avg">{{ "%.2f"|format(feeds.by_format.atom.avg_duration_ms|default(0)) }} ms</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">JSON Avg Time</span>
|
||||
<span class="metric-detail-value" id="feed-json-avg">{{ "%.2f"|format(feeds.by_format.json.avg_duration_ms|default(0)) }} ms</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Feed Charts -->
|
||||
<div class="metrics-grid">
|
||||
<div class="metric-card">
|
||||
<h3>Format Popularity</h3>
|
||||
<div class="chart-container">
|
||||
<canvas id="feedFormatChart"></canvas>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="metric-card">
|
||||
<h3>Cache Efficiency</h3>
|
||||
<div class="chart-container">
|
||||
<canvas id="feedCacheChart"></canvas>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="refresh-info">
|
||||
Auto-refresh every 10 seconds (requires JavaScript)
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
// Initialize charts with current data
|
||||
let poolChart, performanceChart, feedFormatChart, feedCacheChart;
|
||||
|
||||
function initCharts() {
|
||||
// Pool usage chart (doughnut)
|
||||
const poolCtx = document.getElementById('poolChart');
|
||||
if (poolCtx && !poolChart) {
|
||||
poolChart = new Chart(poolCtx, {
|
||||
type: 'doughnut',
|
||||
data: {
|
||||
labels: ['Active', 'Idle'],
|
||||
datasets: [{
|
||||
data: [
|
||||
{{ pool.active_connections|default(0) }},
|
||||
{{ pool.idle_connections|default(0) }}
|
||||
],
|
||||
backgroundColor: ['#007bff', '#6c757d'],
|
||||
borderWidth: 1
|
||||
}]
|
||||
},
|
||||
options: {
|
||||
responsive: true,
|
||||
maintainAspectRatio: false,
|
||||
plugins: {
|
||||
legend: {
|
||||
position: 'bottom'
|
||||
},
|
||||
title: {
|
||||
display: true,
|
||||
text: 'Connection Distribution'
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Performance chart (bar)
|
||||
const perfCtx = document.getElementById('performanceChart');
|
||||
if (perfCtx && !performanceChart) {
|
||||
performanceChart = new Chart(perfCtx, {
|
||||
type: 'bar',
|
||||
data: {
|
||||
labels: ['Database', 'HTTP', 'Render'],
|
||||
datasets: [{
|
||||
label: 'Average Time (ms)',
|
||||
data: [
|
||||
{{ metrics.database.avg|default(0) }},
|
||||
{{ metrics.http.avg|default(0) }},
|
||||
{{ metrics.render.avg|default(0) }}
|
||||
],
|
||||
backgroundColor: ['#007bff', '#28a745', '#ffc107'],
|
||||
borderWidth: 1
|
||||
}]
|
||||
},
|
||||
options: {
|
||||
responsive: true,
|
||||
maintainAspectRatio: false,
|
||||
scales: {
|
||||
y: {
|
||||
beginAtZero: true,
|
||||
title: {
|
||||
display: true,
|
||||
text: 'Milliseconds'
|
||||
}
|
||||
}
|
||||
},
|
||||
plugins: {
|
||||
legend: {
|
||||
display: false
|
||||
},
|
||||
title: {
|
||||
display: true,
|
||||
text: 'Average Response Times'
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Feed format chart (pie)
|
||||
const feedFormatCtx = document.getElementById('feedFormatChart');
|
||||
if (feedFormatCtx && !feedFormatChart) {
|
||||
feedFormatChart = new Chart(feedFormatCtx, {
|
||||
type: 'pie',
|
||||
data: {
|
||||
labels: ['RSS', 'ATOM', 'JSON Feed'],
|
||||
datasets: [{
|
||||
data: [
|
||||
{{ feeds.by_format.rss.total|default(0) }},
|
||||
{{ feeds.by_format.atom.total|default(0) }},
|
||||
{{ feeds.by_format.json.total|default(0) }}
|
||||
],
|
||||
backgroundColor: ['#ff6384', '#36a2eb', '#ffce56'],
|
||||
borderWidth: 1
|
||||
}]
|
||||
},
|
||||
options: {
|
||||
responsive: true,
|
||||
maintainAspectRatio: false,
|
||||
plugins: {
|
||||
legend: {
|
||||
position: 'bottom'
|
||||
},
|
||||
title: {
|
||||
display: true,
|
||||
text: 'Feed Format Distribution'
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Feed cache chart (doughnut)
|
||||
const feedCacheCtx = document.getElementById('feedCacheChart');
|
||||
if (feedCacheCtx && !feedCacheChart) {
|
||||
feedCacheChart = new Chart(feedCacheCtx, {
|
||||
type: 'doughnut',
|
||||
data: {
|
||||
labels: ['Cache Hits', 'Cache Misses'],
|
||||
datasets: [{
|
||||
data: [
|
||||
{{ feeds.cache.hits|default(0) }},
|
||||
{{ feeds.cache.misses|default(0) }}
|
||||
],
|
||||
backgroundColor: ['#28a745', '#dc3545'],
|
||||
borderWidth: 1
|
||||
}]
|
||||
},
|
||||
options: {
|
||||
responsive: true,
|
||||
maintainAspectRatio: false,
|
||||
plugins: {
|
||||
legend: {
|
||||
position: 'bottom'
|
||||
},
|
||||
title: {
|
||||
display: true,
|
||||
text: 'Cache Hit/Miss Ratio'
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Update dashboard with new data from htmx
|
||||
function updateDashboard(event) {
|
||||
if (!event.detail.xhr) return;
|
||||
|
||||
try {
|
||||
const data = JSON.parse(event.detail.xhr.responseText);
|
||||
|
||||
// Update pool statistics
|
||||
if (data.database && data.database.pool) {
|
||||
const pool = data.database.pool;
|
||||
document.getElementById('pool-active').textContent = pool.active_connections || 0;
|
||||
document.getElementById('pool-idle').textContent = pool.idle_connections || 0;
|
||||
document.getElementById('pool-total').textContent = pool.total_connections || 0;
|
||||
document.getElementById('pool-size').textContent = pool.pool_size || 5;
|
||||
|
||||
// Update pool chart
|
||||
if (poolChart) {
|
||||
poolChart.data.datasets[0].data = [
|
||||
pool.active_connections || 0,
|
||||
pool.idle_connections || 0
|
||||
];
|
||||
poolChart.update();
|
||||
}
|
||||
}
|
||||
|
||||
// Update performance metrics
|
||||
if (data.performance) {
|
||||
const perf = data.performance;
|
||||
|
||||
// Database
|
||||
if (perf.database) {
|
||||
document.getElementById('db-total').textContent = perf.database.count || 0;
|
||||
document.getElementById('db-avg').textContent = (perf.database.avg || 0).toFixed(2) + ' ms';
|
||||
document.getElementById('db-min').textContent = (perf.database.min || 0).toFixed(2) + ' ms';
|
||||
document.getElementById('db-max').textContent = (perf.database.max || 0).toFixed(2) + ' ms';
|
||||
}
|
||||
|
||||
// HTTP
|
||||
if (perf.http) {
|
||||
document.getElementById('http-total').textContent = perf.http.count || 0;
|
||||
document.getElementById('http-avg').textContent = (perf.http.avg || 0).toFixed(2) + ' ms';
|
||||
document.getElementById('http-min').textContent = (perf.http.min || 0).toFixed(2) + ' ms';
|
||||
document.getElementById('http-max').textContent = (perf.http.max || 0).toFixed(2) + ' ms';
|
||||
}
|
||||
|
||||
// Render
|
||||
if (perf.render) {
|
||||
document.getElementById('render-total').textContent = perf.render.count || 0;
|
||||
document.getElementById('render-avg').textContent = (perf.render.avg || 0).toFixed(2) + ' ms';
|
||||
document.getElementById('render-min').textContent = (perf.render.min || 0).toFixed(2) + ' ms';
|
||||
document.getElementById('render-max').textContent = (perf.render.max || 0).toFixed(2) + ' ms';
|
||||
}
|
||||
|
||||
// Update performance chart
|
||||
if (performanceChart && perf.database && perf.http && perf.render) {
|
||||
performanceChart.data.datasets[0].data = [
|
||||
perf.database.avg || 0,
|
||||
perf.http.avg || 0,
|
||||
perf.render.avg || 0
|
||||
];
|
||||
performanceChart.update();
|
||||
}
|
||||
}
|
||||
|
||||
// Update feed statistics
|
||||
if (data.feeds) {
|
||||
const feeds = data.feeds;
|
||||
|
||||
// Feed requests by format
|
||||
if (feeds.by_format) {
|
||||
document.getElementById('feed-rss-total').textContent = feeds.by_format.rss?.total || 0;
|
||||
document.getElementById('feed-atom-total').textContent = feeds.by_format.atom?.total || 0;
|
||||
document.getElementById('feed-json-total').textContent = feeds.by_format.json?.total || 0;
|
||||
document.getElementById('feed-total').textContent = feeds.total_requests || 0;
|
||||
|
||||
// Feed generation performance
|
||||
document.getElementById('feed-rss-avg').textContent = (feeds.by_format.rss?.avg_duration_ms || 0).toFixed(2) + ' ms';
|
||||
document.getElementById('feed-atom-avg').textContent = (feeds.by_format.atom?.avg_duration_ms || 0).toFixed(2) + ' ms';
|
||||
document.getElementById('feed-json-avg').textContent = (feeds.by_format.json?.avg_duration_ms || 0).toFixed(2) + ' ms';
|
||||
|
||||
// Update feed format chart
|
||||
if (feedFormatChart) {
|
||||
feedFormatChart.data.datasets[0].data = [
|
||||
feeds.by_format.rss?.total || 0,
|
||||
feeds.by_format.atom?.total || 0,
|
||||
feeds.by_format.json?.total || 0
|
||||
];
|
||||
feedFormatChart.update();
|
||||
}
|
||||
}
|
||||
|
||||
// Feed cache statistics
|
||||
if (feeds.cache) {
|
||||
document.getElementById('feed-cache-hits').textContent = feeds.cache.hits || 0;
|
||||
document.getElementById('feed-cache-misses').textContent = feeds.cache.misses || 0;
|
||||
document.getElementById('feed-cache-hit-rate').textContent = ((feeds.cache.hit_rate || 0) * 100).toFixed(1) + '%';
|
||||
document.getElementById('feed-cache-entries').textContent = feeds.cache.entries || 0;
|
||||
|
||||
// Update feed cache chart
|
||||
if (feedCacheChart) {
|
||||
feedCacheChart.data.datasets[0].data = [
|
||||
feeds.cache.hits || 0,
|
||||
feeds.cache.misses || 0
|
||||
];
|
||||
feedCacheChart.update();
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Error updating dashboard:', e);
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize charts when DOM is ready
|
||||
if (document.readyState === 'loading') {
|
||||
document.addEventListener('DOMContentLoaded', initCharts);
|
||||
} else {
|
||||
initCharts();
|
||||
}
|
||||
</script>
|
||||
{% endblock %}
|
||||
@@ -6,7 +6,7 @@
|
||||
<div class="note-editor">
|
||||
<h2>Create New Note</h2>
|
||||
|
||||
<form action="{{ url_for('admin.create_note_submit') }}" method="POST" enctype="multipart/form-data" class="note-form">
|
||||
<form action="{{ url_for('admin.create_note_submit') }}" method="POST" enctype="multipart/form-data" class="note-form"><input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
|
||||
<div class="form-group">
|
||||
<label for="content">Content (Markdown)</label>
|
||||
<textarea
|
||||
|
||||
@@ -29,7 +29,7 @@
|
||||
<td>{{ t.last_used_at or "never" }}</td>
|
||||
<td>{{ t.expires_at }}</td>
|
||||
<td>
|
||||
<form method="POST" action="{{ url_for('admin.revoke_token', token_id=t.id) }}">
|
||||
<form method="POST" action="{{ url_for('admin.revoke_token', token_id=t.id) }}"><input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
|
||||
<button type="submit" class="button button-danger"
|
||||
onclick="return confirm('Revoke this app\'s access?')">Revoke</button>
|
||||
</form>
|
||||
|
||||
@@ -83,21 +83,21 @@
|
||||
{# Moderation Actions #}
|
||||
<div class="moderation-actions">
|
||||
<h3>Actions</h3>
|
||||
<form method="POST" action="{{ url_for('admin.webmention_action', id=webmention.id) }}" style="display: inline;">
|
||||
<form method="POST" action="{{ url_for('admin.webmention_action', id=webmention.id) }}" style="display: inline;"><input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
|
||||
<input type="hidden" name="action" value="approve">
|
||||
<button type="submit" class="button button-primary" {% if webmention.moderation_status == 'approved' %}disabled{% endif %}>
|
||||
Approve
|
||||
</button>
|
||||
</form>
|
||||
|
||||
<form method="POST" action="{{ url_for('admin.webmention_action', id=webmention.id) }}" style="display: inline;">
|
||||
<form method="POST" action="{{ url_for('admin.webmention_action', id=webmention.id) }}" style="display: inline;"><input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
|
||||
<input type="hidden" name="action" value="hide">
|
||||
<button type="submit" class="button" {% if webmention.moderation_status == 'hidden' %}disabled{% endif %}>
|
||||
Hide
|
||||
</button>
|
||||
</form>
|
||||
|
||||
<form method="POST" action="{{ url_for('admin.webmention_action', id=webmention.id) }}" style="display: inline;">
|
||||
<form method="POST" action="{{ url_for('admin.webmention_action', id=webmention.id) }}" style="display: inline;"><input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
|
||||
<input type="hidden" name="action" value="delete">
|
||||
<button type="submit" class="button button-danger" onclick="return confirm('Delete this webmention permanently?')">
|
||||
Delete
|
||||
|
||||
@@ -29,7 +29,7 @@
|
||||
</div>
|
||||
|
||||
{% if webmentions %}
|
||||
<form method="POST" action="{{ url_for('admin.webmention_bulk_action') }}">
|
||||
<form method="POST" action="{{ url_for('admin.webmention_bulk_action') }}"><input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
|
||||
<div class="bulk-actions">
|
||||
<button type="submit" name="action" value="approve" class="button">
|
||||
Approve Selected
|
||||
|
||||
@@ -1,108 +0,0 @@
|
||||
"""
|
||||
Integration tests for feed statistics in admin dashboard
|
||||
|
||||
Tests the feed statistics features in /admin/metrics-dashboard and /admin/metrics
|
||||
per v1.1.2 Phase 3.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from starpunk.auth import create_session
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def authenticated_client(app, client):
|
||||
"""Client with authenticated session"""
|
||||
with app.test_request_context():
|
||||
# Create a session for the test user
|
||||
session_token = create_session(app.config["ADMIN_ME"])
|
||||
|
||||
# Set session cookie
|
||||
client.set_cookie("starpunk_session", session_token)
|
||||
return client
|
||||
|
||||
|
||||
def test_feed_statistics_dashboard_endpoint(authenticated_client):
|
||||
"""Test metrics dashboard includes feed statistics section"""
|
||||
response = authenticated_client.get("/admin/metrics-dashboard")
|
||||
|
||||
assert response.status_code == 200
|
||||
|
||||
# Should contain feed statistics section
|
||||
assert b"Feed Statistics" in response.data
|
||||
assert b"Feed Requests by Format" in response.data
|
||||
assert b"Feed Cache Statistics" in response.data
|
||||
assert b"Feed Generation Performance" in response.data
|
||||
|
||||
# Should have chart canvases
|
||||
assert b'id="feedFormatChart"' in response.data
|
||||
assert b'id="feedCacheChart"' in response.data
|
||||
|
||||
|
||||
def test_feed_statistics_metrics_endpoint(authenticated_client):
|
||||
"""Test /admin/metrics endpoint includes feed statistics"""
|
||||
response = authenticated_client.get("/admin/metrics")
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.get_json()
|
||||
|
||||
# Should have feeds key
|
||||
assert "feeds" in data
|
||||
|
||||
# Should have expected structure
|
||||
feeds = data["feeds"]
|
||||
if "error" not in feeds:
|
||||
assert "by_format" in feeds
|
||||
assert "cache" in feeds
|
||||
assert "total_requests" in feeds
|
||||
assert "format_percentages" in feeds
|
||||
|
||||
# Check format structure
|
||||
for format_name in ["rss", "atom", "json"]:
|
||||
assert format_name in feeds["by_format"]
|
||||
fmt = feeds["by_format"][format_name]
|
||||
assert "generated" in fmt
|
||||
assert "cached" in fmt
|
||||
assert "total" in fmt
|
||||
assert "avg_duration_ms" in fmt
|
||||
|
||||
# Check cache structure
|
||||
assert "hits" in feeds["cache"]
|
||||
assert "misses" in feeds["cache"]
|
||||
assert "hit_rate" in feeds["cache"]
|
||||
|
||||
|
||||
def test_feed_statistics_after_feed_request(authenticated_client):
|
||||
"""Test feed statistics track actual feed requests"""
|
||||
# Make a feed request
|
||||
response = authenticated_client.get("/feed.rss")
|
||||
assert response.status_code == 200
|
||||
|
||||
# Check metrics endpoint now has data
|
||||
response = authenticated_client.get("/admin/metrics")
|
||||
assert response.status_code == 200
|
||||
data = response.get_json()
|
||||
|
||||
# Should have feeds data
|
||||
assert "feeds" in data
|
||||
feeds = data["feeds"]
|
||||
|
||||
# May have requests tracked (depends on metrics buffer timing)
|
||||
# Just verify structure is correct
|
||||
assert "total_requests" in feeds
|
||||
assert feeds["total_requests"] >= 0
|
||||
|
||||
|
||||
def test_dashboard_requires_auth_for_feed_stats(client):
|
||||
"""Test dashboard requires authentication (even for feed stats)"""
|
||||
response = client.get("/admin/metrics-dashboard")
|
||||
|
||||
# Should redirect to auth or return 401/403
|
||||
assert response.status_code in [302, 401, 403]
|
||||
|
||||
|
||||
def test_metrics_endpoint_requires_auth_for_feed_stats(client):
|
||||
"""Test metrics endpoint requires authentication"""
|
||||
response = client.get("/admin/metrics")
|
||||
|
||||
# Should redirect to auth or return 401/403
|
||||
assert response.status_code in [302, 401, 403]
|
||||
+6
-6
@@ -4,7 +4,7 @@ Tests for authentication module (starpunk/auth.py)
|
||||
|
||||
import hashlib
|
||||
import secrets
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import httpx
|
||||
@@ -102,7 +102,7 @@ class TestCleanup:
|
||||
with app.app_context():
|
||||
# Create expired session
|
||||
token_hash = _hash_token("expired-token")
|
||||
expires_at = datetime.utcnow() - timedelta(days=1)
|
||||
expires_at = datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(days=1)
|
||||
|
||||
db.execute(
|
||||
"""
|
||||
@@ -126,7 +126,7 @@ class TestCleanup:
|
||||
"""Test that cleanup keeps valid sessions"""
|
||||
with app.app_context():
|
||||
token_hash = _hash_token("valid-token")
|
||||
expires_at = datetime.utcnow() + timedelta(days=30)
|
||||
expires_at = datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(days=30)
|
||||
|
||||
db.execute(
|
||||
"""
|
||||
@@ -226,7 +226,7 @@ class TestVerifySession:
|
||||
# Create expired session
|
||||
token = secrets.token_urlsafe(32)
|
||||
token_hash = _hash_token(token)
|
||||
expires_at = datetime.utcnow() - timedelta(days=1)
|
||||
expires_at = datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(days=1)
|
||||
|
||||
db.execute(
|
||||
"""
|
||||
@@ -340,7 +340,7 @@ class TestRequireAuthDecorator:
|
||||
with app.app_context():
|
||||
token = secrets.token_urlsafe(32)
|
||||
token_hash = _hash_token(token)
|
||||
expires_at = datetime.utcnow() - timedelta(days=1)
|
||||
expires_at = datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(days=1)
|
||||
|
||||
db.execute(
|
||||
"""
|
||||
@@ -397,7 +397,7 @@ class TestSecurityFeatures:
|
||||
).fetchone()
|
||||
|
||||
expires_at = datetime.fromisoformat(result["expires_at"])
|
||||
created_at = datetime.utcnow()
|
||||
created_at = datetime.now(timezone.utc).replace(tzinfo=None)
|
||||
|
||||
# Should expire approximately 1 day from now
|
||||
# (allow for minor timing differences)
|
||||
|
||||
@@ -8,7 +8,7 @@ Per v1.2.0 Phase 2 and developer Q&A Q31-Q35:
|
||||
"""
|
||||
|
||||
import json
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
@@ -203,7 +203,7 @@ class TestGetAuthorProfile:
|
||||
from starpunk.database import get_db
|
||||
db = get_db(app)
|
||||
|
||||
expired_time = (datetime.utcnow() - timedelta(hours=48)).isoformat()
|
||||
expired_time = (datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(hours=48)).isoformat()
|
||||
|
||||
db.execute(
|
||||
"""
|
||||
@@ -296,9 +296,9 @@ class TestSaveAuthorProfile:
|
||||
'rel_me_links': [],
|
||||
}
|
||||
|
||||
before_save = datetime.utcnow()
|
||||
before_save = datetime.now(timezone.utc).replace(tzinfo=None)
|
||||
save_author_profile('https://cache.example.com', profile)
|
||||
after_save = datetime.utcnow()
|
||||
after_save = datetime.now(timezone.utc).replace(tzinfo=None)
|
||||
|
||||
# Check cache expiry
|
||||
db = get_db(app)
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
"""
|
||||
CSRF protection on admin and consent forms (v2.11.0)
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
import pytest
|
||||
|
||||
from starpunk import create_app
|
||||
from starpunk.auth import create_session
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def app(tmp_path):
|
||||
return create_app({
|
||||
"TESTING": True, "CSRF_ENABLED": True, # the real check, unlike other tests
|
||||
"SITE_URL": "http://localhost:5000", "ADMIN_ME": "https://test.example.com",
|
||||
"SESSION_SECRET": "x", "DATA_PATH": tmp_path, "NOTES_PATH": tmp_path / "notes",
|
||||
"DATABASE_PATH": tmp_path / "starpunk.db",
|
||||
})
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def admin(app):
|
||||
client = app.test_client()
|
||||
with app.test_request_context():
|
||||
client.set_cookie("starpunk_session", create_session(app.config["ADMIN_ME"]))
|
||||
return client
|
||||
|
||||
|
||||
def token_from(page: bytes) -> str:
|
||||
return re.search(rb'name="csrf_token" value="([0-9a-f]{64})"', page).group(1).decode()
|
||||
|
||||
|
||||
def test_forms_carry_a_token(admin):
|
||||
assert token_from(admin.get("/admin/new").data)
|
||||
|
||||
|
||||
def test_post_without_token_is_refused(app, admin):
|
||||
response = admin.post("/admin/new", data={"content": "Forged"})
|
||||
assert response.status_code == 400
|
||||
from starpunk.notes import list_notes
|
||||
with app.app_context():
|
||||
assert list_notes() == []
|
||||
|
||||
|
||||
def test_post_with_wrong_token_is_refused(admin):
|
||||
assert admin.post("/admin/new", data={"content": "x", "csrf_token": "0" * 64}).status_code == 400
|
||||
|
||||
|
||||
def test_post_with_token_works(app, admin):
|
||||
token = token_from(admin.get("/admin/new").data)
|
||||
response = admin.post("/admin/new", data={"content": "Real", "csrf_token": token, "published": "on"})
|
||||
assert response.status_code == 302
|
||||
from starpunk.notes import list_notes
|
||||
with app.app_context():
|
||||
assert [n.content.strip() for n in list_notes()] == ["Real"]
|
||||
|
||||
|
||||
def test_token_changes_with_the_session(app):
|
||||
tokens = []
|
||||
for _ in range(2):
|
||||
client = app.test_client()
|
||||
with app.test_request_context():
|
||||
client.set_cookie("starpunk_session", create_session(app.config["ADMIN_ME"]))
|
||||
tokens.append(token_from(client.get("/admin/new").data))
|
||||
assert tokens[0] != tokens[1]
|
||||
|
||||
|
||||
def test_logout_and_consent_are_protected(admin):
|
||||
assert admin.post("/auth/logout").status_code == 400
|
||||
assert admin.post("/auth/authorize/approve", data={"client_id": "https://app.example/"}).status_code == 400
|
||||
|
||||
|
||||
def test_app_endpoints_are_not_affected(app):
|
||||
client = app.test_client()
|
||||
# Bearer-token and public endpoints answer normally, without a CSRF token
|
||||
assert client.post("/micropub", data={"h": "entry", "content": "x"}).status_code == 401
|
||||
assert client.post("/auth/token", data={"grant_type": "authorization_code"}).status_code == 400
|
||||
assert client.post("/webmention", data={}).status_code == 400
|
||||
@@ -0,0 +1,66 @@
|
||||
"""
|
||||
Foreign keys are enforced (v2.11.0)
|
||||
|
||||
SQLite leaves foreign keys off per connection, so ON DELETE CASCADE never ran
|
||||
and hard deletes left tag, media and variant rows behind.
|
||||
"""
|
||||
|
||||
import io
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from starpunk.database import get_db
|
||||
from starpunk.migrations import execute_in_transaction
|
||||
|
||||
|
||||
def png():
|
||||
buf = io.BytesIO()
|
||||
Image.new("RGB", (30, 20), "red").save(buf, format="PNG")
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def count(app, sql):
|
||||
return sqlite3.connect(app.config["DATABASE_PATH"]).execute(sql).fetchone()[0]
|
||||
|
||||
|
||||
def test_pooled_connections_enforce_foreign_keys(app):
|
||||
with app.app_context():
|
||||
assert get_db(app).execute("PRAGMA foreign_keys").fetchone()[0] == 1
|
||||
|
||||
|
||||
def test_hard_delete_removes_tags_and_media_links(app):
|
||||
from starpunk.media import attach_media_to_note, save_media
|
||||
from starpunk.notes import create_note, delete_note
|
||||
with app.test_request_context():
|
||||
note = create_note("Tagged with a photo", published=True, tags=["one", "two"], has_media=True)
|
||||
media = save_media(png(), "a.png")
|
||||
attach_media_to_note(note.id, [media["id"]], [""])
|
||||
delete_note(id=note.id, soft=False)
|
||||
assert count(app, "SELECT COUNT(*) FROM note_tags") == 0
|
||||
assert count(app, "SELECT COUNT(*) FROM note_media") == 0
|
||||
|
||||
|
||||
def test_deleting_media_removes_variants(app):
|
||||
from starpunk.media import delete_media, save_media
|
||||
with app.test_request_context():
|
||||
media = save_media(png(), "b.png")
|
||||
delete_media(media["id"])
|
||||
assert count(app, "SELECT COUNT(*) FROM media_variants") == 0
|
||||
|
||||
|
||||
def test_migration_018_removes_existing_orphans(app):
|
||||
conn = sqlite3.connect(app.config["DATABASE_PATH"]) # foreign keys off here
|
||||
conn.execute("INSERT INTO tags (name, display_name) VALUES ('t', 't')")
|
||||
conn.execute("INSERT INTO note_tags (note_id, tag_id) VALUES (999, 1)")
|
||||
conn.execute("INSERT INTO webmentions (source_url, target_url, target_note_id) "
|
||||
"VALUES ('https://s.example/', 'http://localhost:5000/note/x', 999)")
|
||||
conn.commit()
|
||||
sql = (Path(__file__).parent.parent / "migrations" / "018_foreign_key_cleanup.sql").read_text()
|
||||
conn.execute("BEGIN")
|
||||
execute_in_transaction(conn, sql)
|
||||
conn.commit()
|
||||
assert conn.execute("SELECT COUNT(*) FROM note_tags").fetchone()[0] == 0
|
||||
assert conn.execute("SELECT target_note_id FROM webmentions").fetchone()[0] is None
|
||||
assert conn.execute("PRAGMA foreign_key_check").fetchall() == []
|
||||
+41
-14
@@ -129,27 +129,54 @@ class TestValidation:
|
||||
|
||||
|
||||
class TestClientMetadata:
|
||||
def _response(self, content_type, body, links=None):
|
||||
response = MagicMock(status_code=200, text=body, url="https://quill.example/")
|
||||
response.headers = {"Content-Type": content_type}
|
||||
response.links = links or {}
|
||||
response.json.return_value = __import__("json").loads(body) if "json" in content_type else None
|
||||
return response
|
||||
"""Client metadata is fetched through the SSRF-safe fetcher (v2.11.0)"""
|
||||
|
||||
def test_json_metadata(self, app):
|
||||
@staticmethod
|
||||
def _serve(monkeypatch, handler, private=()):
|
||||
import httpx
|
||||
|
||||
def resolve(host, *args, **kwargs):
|
||||
return [(2, 1, 6, "", ("10.0.0.5" if host in private else "93.184.216.34", 0))]
|
||||
|
||||
monkeypatch.setattr("starpunk.webmention_ssrf.TRANSPORT", httpx.MockTransport(handler))
|
||||
monkeypatch.setattr("starpunk.webmention_ssrf.socket.getaddrinfo", resolve)
|
||||
|
||||
def test_json_metadata(self, app, monkeypatch):
|
||||
import httpx
|
||||
body = '{"client_id": "https://quill.example/", "redirect_uris": ["app://cb"]}'
|
||||
with app.test_request_context(), patch(
|
||||
"starpunk.webmention_ssrf.validate_url_for_fetch"
|
||||
), patch("starpunk.indieauth.httpx.get", return_value=self._response("application/json", body)):
|
||||
self._serve(monkeypatch, lambda r: httpx.Response(
|
||||
200, headers={"Content-Type": "application/json"}, text=body))
|
||||
with app.test_request_context():
|
||||
assert fetch_client_redirect_uris(CLIENT_ID) == ["app://cb"]
|
||||
|
||||
def test_html_link(self, app):
|
||||
def test_html_link(self, app, monkeypatch):
|
||||
import httpx
|
||||
body = '<html><head><link rel="redirect_uri" href="/cb"></head></html>'
|
||||
with app.test_request_context(), patch(
|
||||
"starpunk.webmention_ssrf.validate_url_for_fetch"
|
||||
), patch("starpunk.indieauth.httpx.get", return_value=self._response("text/html", body)):
|
||||
self._serve(monkeypatch, lambda r: httpx.Response(
|
||||
200, headers={"Content-Type": "text/html"}, text=body))
|
||||
with app.test_request_context():
|
||||
assert fetch_client_redirect_uris(CLIENT_ID) == ["https://quill.example/cb"]
|
||||
|
||||
def test_link_header(self, app, monkeypatch):
|
||||
import httpx
|
||||
self._serve(monkeypatch, lambda r: httpx.Response(
|
||||
200, headers={"Content-Type": "text/html", "Link": '</cb2>; rel="redirect_uri"'}, text="<p>hi</p>"))
|
||||
with app.test_request_context():
|
||||
assert fetch_client_redirect_uris(CLIENT_ID) == ["https://quill.example/cb2"]
|
||||
|
||||
def test_redirect_to_private_address_refused(self, app, monkeypatch):
|
||||
import httpx
|
||||
fetched = []
|
||||
|
||||
def handler(request):
|
||||
fetched.append(request.headers["host"])
|
||||
return httpx.Response(302, headers={"Location": "http://internal.example/"})
|
||||
|
||||
self._serve(monkeypatch, handler, private={"internal.example"})
|
||||
with app.test_request_context():
|
||||
assert fetch_client_redirect_uris(CLIENT_ID) == []
|
||||
assert "internal.example" not in fetched
|
||||
|
||||
|
||||
class TestCodesAndTokens:
|
||||
def test_full_exchange(self, app):
|
||||
|
||||
+74
-74
@@ -9,7 +9,7 @@ Organized by model:
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from starpunk.models import Note, Session, Token, AuthState, split_title
|
||||
from starpunk.utils import calculate_content_hash
|
||||
@@ -68,8 +68,8 @@ class TestNoteModel:
|
||||
slug="test",
|
||||
file_path="notes/2024/11/test.md",
|
||||
published=True,
|
||||
created_at=datetime.utcnow(),
|
||||
updated_at=datetime.utcnow(),
|
||||
created_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
updated_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
_data_dir=tmp_path,
|
||||
)
|
||||
|
||||
@@ -89,8 +89,8 @@ class TestNoteModel:
|
||||
slug="missing",
|
||||
file_path="notes/2024/11/missing.md",
|
||||
published=True,
|
||||
created_at=datetime.utcnow(),
|
||||
updated_at=datetime.utcnow(),
|
||||
created_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
updated_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
_data_dir=tmp_path,
|
||||
)
|
||||
|
||||
@@ -109,8 +109,8 @@ class TestNoteModel:
|
||||
slug="test",
|
||||
file_path="notes/2024/11/test.md",
|
||||
published=True,
|
||||
created_at=datetime.utcnow(),
|
||||
updated_at=datetime.utcnow(),
|
||||
created_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
updated_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
_data_dir=tmp_path,
|
||||
)
|
||||
|
||||
@@ -130,8 +130,8 @@ class TestNoteModel:
|
||||
slug="test",
|
||||
file_path="notes/2024/11/test.md",
|
||||
published=True,
|
||||
created_at=datetime.utcnow(),
|
||||
updated_at=datetime.utcnow(),
|
||||
created_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
updated_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
_data_dir=tmp_path,
|
||||
)
|
||||
|
||||
@@ -150,8 +150,8 @@ class TestNoteModel:
|
||||
slug="test",
|
||||
file_path="notes/2024/11/test.md",
|
||||
published=True,
|
||||
created_at=datetime.utcnow(),
|
||||
updated_at=datetime.utcnow(),
|
||||
created_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
updated_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
_data_dir=tmp_path,
|
||||
)
|
||||
|
||||
@@ -168,8 +168,8 @@ class TestNoteModel:
|
||||
slug="test",
|
||||
file_path="notes/2024/11/test.md",
|
||||
published=True,
|
||||
created_at=datetime.utcnow(),
|
||||
updated_at=datetime.utcnow(),
|
||||
created_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
updated_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
_data_dir=tmp_path,
|
||||
)
|
||||
|
||||
@@ -182,8 +182,8 @@ class TestNoteModel:
|
||||
slug="my-test-note",
|
||||
file_path="notes/2024/11/test.md",
|
||||
published=True,
|
||||
created_at=datetime.utcnow(),
|
||||
updated_at=datetime.utcnow(),
|
||||
created_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
updated_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
_data_dir=Path("/nonexistent"),
|
||||
)
|
||||
|
||||
@@ -202,8 +202,8 @@ class TestNoteModel:
|
||||
slug="test",
|
||||
file_path="notes/2024/11/test.md",
|
||||
published=True,
|
||||
created_at=datetime.utcnow(),
|
||||
updated_at=datetime.utcnow(),
|
||||
created_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
updated_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
_data_dir=tmp_path,
|
||||
)
|
||||
|
||||
@@ -222,8 +222,8 @@ class TestNoteModel:
|
||||
slug="test",
|
||||
file_path="notes/2024/11/test.md",
|
||||
published=True,
|
||||
created_at=datetime.utcnow(),
|
||||
updated_at=datetime.utcnow(),
|
||||
created_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
updated_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
_data_dir=tmp_path,
|
||||
)
|
||||
|
||||
@@ -238,8 +238,8 @@ class TestNoteModel:
|
||||
slug="my-note",
|
||||
file_path="notes/2024/11/test.md",
|
||||
published=True,
|
||||
created_at=datetime.utcnow(),
|
||||
updated_at=datetime.utcnow(),
|
||||
created_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
updated_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
_data_dir=Path("/nonexistent"),
|
||||
)
|
||||
|
||||
@@ -254,8 +254,8 @@ class TestNoteModel:
|
||||
slug="test",
|
||||
file_path="notes/2024/11/test.md",
|
||||
published=True,
|
||||
created_at=datetime.utcnow(),
|
||||
updated_at=datetime.utcnow(),
|
||||
created_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
updated_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
_data_dir=tmp_path,
|
||||
)
|
||||
|
||||
@@ -289,8 +289,8 @@ class TestNoteModel:
|
||||
slug="my-note",
|
||||
file_path="notes/2024/11/my-note.md",
|
||||
published=True,
|
||||
created_at=datetime.utcnow(),
|
||||
updated_at=datetime.utcnow(),
|
||||
created_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
updated_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
_data_dir=Path("data"),
|
||||
)
|
||||
|
||||
@@ -303,8 +303,8 @@ class TestNoteModel:
|
||||
slug="test",
|
||||
file_path="notes/2024/11/test.md",
|
||||
published=True,
|
||||
created_at=datetime.utcnow(),
|
||||
updated_at=datetime.utcnow(),
|
||||
created_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
updated_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
_data_dir=Path("data"),
|
||||
)
|
||||
|
||||
@@ -342,8 +342,8 @@ class TestNoteModel:
|
||||
slug="test",
|
||||
file_path="notes/2024/11/test.md",
|
||||
published=True,
|
||||
created_at=datetime.utcnow(),
|
||||
updated_at=datetime.utcnow(),
|
||||
created_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
updated_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
_data_dir=tmp_path,
|
||||
)
|
||||
|
||||
@@ -362,8 +362,8 @@ class TestNoteModel:
|
||||
slug="test",
|
||||
file_path="notes/2024/11/test.md",
|
||||
published=True,
|
||||
created_at=datetime.utcnow(),
|
||||
updated_at=datetime.utcnow(),
|
||||
created_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
updated_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
_data_dir=tmp_path,
|
||||
)
|
||||
|
||||
@@ -386,8 +386,8 @@ class TestNoteModel:
|
||||
slug="test",
|
||||
file_path="notes/2024/11/test.md",
|
||||
published=True,
|
||||
created_at=datetime.utcnow(),
|
||||
updated_at=datetime.utcnow(),
|
||||
created_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
updated_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
content_hash=content_hash,
|
||||
_data_dir=tmp_path,
|
||||
)
|
||||
@@ -410,8 +410,8 @@ class TestNoteModel:
|
||||
slug="test",
|
||||
file_path="notes/2024/11/test.md",
|
||||
published=True,
|
||||
created_at=datetime.utcnow(),
|
||||
updated_at=datetime.utcnow(),
|
||||
created_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
updated_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
content_hash=content_hash,
|
||||
_data_dir=tmp_path,
|
||||
)
|
||||
@@ -433,8 +433,8 @@ class TestNoteModel:
|
||||
slug="test",
|
||||
file_path="notes/2024/11/test.md",
|
||||
published=True,
|
||||
created_at=datetime.utcnow(),
|
||||
updated_at=datetime.utcnow(),
|
||||
created_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
updated_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
content_hash=None,
|
||||
_data_dir=tmp_path,
|
||||
)
|
||||
@@ -449,8 +449,8 @@ class TestNoteModel:
|
||||
slug="test",
|
||||
file_path="notes/2024/11/test.md",
|
||||
published=True,
|
||||
created_at=datetime.utcnow(),
|
||||
updated_at=datetime.utcnow(),
|
||||
created_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
updated_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
content_hash="abc123",
|
||||
_data_dir=Path("/nonexistent"),
|
||||
)
|
||||
@@ -499,8 +499,8 @@ class TestSessionModel:
|
||||
id=1,
|
||||
session_token="abc123",
|
||||
me="https://alice.example.com",
|
||||
created_at=datetime.utcnow(),
|
||||
expires_at=datetime.utcnow() + timedelta(days=30),
|
||||
created_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
expires_at=datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(days=30),
|
||||
)
|
||||
assert session.is_expired is False
|
||||
assert session.is_active is True
|
||||
@@ -511,21 +511,21 @@ class TestSessionModel:
|
||||
id=1,
|
||||
session_token="abc123",
|
||||
me="https://alice.example.com",
|
||||
created_at=datetime.utcnow() - timedelta(days=31),
|
||||
expires_at=datetime.utcnow() - timedelta(days=1),
|
||||
created_at=datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(days=31),
|
||||
expires_at=datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(days=1),
|
||||
)
|
||||
assert session.is_expired is True
|
||||
assert session.is_active is False
|
||||
|
||||
def test_age(self):
|
||||
"""Test age calculation"""
|
||||
created = datetime.utcnow() - timedelta(hours=2)
|
||||
created = datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(hours=2)
|
||||
session = Session(
|
||||
id=1,
|
||||
session_token="abc123",
|
||||
me="https://alice.example.com",
|
||||
created_at=created,
|
||||
expires_at=datetime.utcnow() + timedelta(days=30),
|
||||
expires_at=datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(days=30),
|
||||
)
|
||||
age = session.age
|
||||
# Age should be at least 2 hours (account for test execution time)
|
||||
@@ -537,8 +537,8 @@ class TestSessionModel:
|
||||
id=1,
|
||||
session_token="abc123",
|
||||
me="https://alice.example.com",
|
||||
created_at=datetime.utcnow(),
|
||||
expires_at=datetime.utcnow() + timedelta(days=30),
|
||||
created_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
expires_at=datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(days=30),
|
||||
)
|
||||
time_left = session.time_until_expiry
|
||||
# Should be approximately 30 days
|
||||
@@ -550,8 +550,8 @@ class TestSessionModel:
|
||||
id=1,
|
||||
session_token="abc123",
|
||||
me="https://alice.example.com",
|
||||
created_at=datetime.utcnow() - timedelta(days=31),
|
||||
expires_at=datetime.utcnow() - timedelta(days=1),
|
||||
created_at=datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(days=31),
|
||||
expires_at=datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(days=1),
|
||||
)
|
||||
time_left = session.time_until_expiry
|
||||
assert time_left.total_seconds() < 0
|
||||
@@ -562,8 +562,8 @@ class TestSessionModel:
|
||||
id=1,
|
||||
session_token="abc123",
|
||||
me="https://alice.example.com",
|
||||
created_at=datetime.utcnow(),
|
||||
expires_at=datetime.utcnow() + timedelta(days=30),
|
||||
created_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
expires_at=datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(days=30),
|
||||
)
|
||||
assert session.is_valid() is True
|
||||
|
||||
@@ -573,8 +573,8 @@ class TestSessionModel:
|
||||
id=1,
|
||||
session_token="abc123",
|
||||
me="https://alice.example.com",
|
||||
created_at=datetime.utcnow() - timedelta(days=31),
|
||||
expires_at=datetime.utcnow() - timedelta(days=1),
|
||||
created_at=datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(days=31),
|
||||
expires_at=datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(days=1),
|
||||
)
|
||||
assert session.is_valid() is False
|
||||
|
||||
@@ -584,8 +584,8 @@ class TestSessionModel:
|
||||
id=1,
|
||||
session_token="",
|
||||
me="https://alice.example.com",
|
||||
created_at=datetime.utcnow(),
|
||||
expires_at=datetime.utcnow() + timedelta(days=30),
|
||||
created_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
expires_at=datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(days=30),
|
||||
)
|
||||
assert session.is_valid() is False
|
||||
|
||||
@@ -595,8 +595,8 @@ class TestSessionModel:
|
||||
id=1,
|
||||
session_token="abc123",
|
||||
me="",
|
||||
created_at=datetime.utcnow(),
|
||||
expires_at=datetime.utcnow() + timedelta(days=30),
|
||||
created_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
expires_at=datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(days=30),
|
||||
)
|
||||
assert session.is_valid() is False
|
||||
|
||||
@@ -606,8 +606,8 @@ class TestSessionModel:
|
||||
id=1,
|
||||
session_token="abc123",
|
||||
me="https://alice.example.com",
|
||||
created_at=datetime.utcnow(),
|
||||
expires_at=datetime.utcnow() + timedelta(days=30),
|
||||
created_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
expires_at=datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(days=30),
|
||||
last_used_at=None,
|
||||
)
|
||||
|
||||
@@ -721,7 +721,7 @@ class TestTokenModel:
|
||||
token = Token(
|
||||
token="xyz789",
|
||||
me="https://alice.example.com",
|
||||
expires_at=datetime.utcnow() + timedelta(days=90),
|
||||
expires_at=datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(days=90),
|
||||
)
|
||||
assert token.is_expired is False
|
||||
assert token.is_active is True
|
||||
@@ -731,7 +731,7 @@ class TestTokenModel:
|
||||
token = Token(
|
||||
token="xyz789",
|
||||
me="https://alice.example.com",
|
||||
expires_at=datetime.utcnow() - timedelta(days=1),
|
||||
expires_at=datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(days=1),
|
||||
)
|
||||
assert token.is_expired is True
|
||||
assert token.is_active is False
|
||||
@@ -746,7 +746,7 @@ class TestTokenModel:
|
||||
token = Token(
|
||||
token="xyz789",
|
||||
me="https://alice.example.com",
|
||||
expires_at=datetime.utcnow() - timedelta(days=1),
|
||||
expires_at=datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(days=1),
|
||||
)
|
||||
assert token.is_valid() is False
|
||||
|
||||
@@ -827,8 +827,8 @@ class TestAuthStateModel:
|
||||
"""Test is_expired returns False for active state"""
|
||||
auth_state = AuthState(
|
||||
state="random123",
|
||||
created_at=datetime.utcnow(),
|
||||
expires_at=datetime.utcnow() + timedelta(minutes=5),
|
||||
created_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
expires_at=datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(minutes=5),
|
||||
)
|
||||
assert auth_state.is_expired is False
|
||||
assert auth_state.is_active is True
|
||||
@@ -837,19 +837,19 @@ class TestAuthStateModel:
|
||||
"""Test is_expired returns True for expired state"""
|
||||
auth_state = AuthState(
|
||||
state="random123",
|
||||
created_at=datetime.utcnow() - timedelta(minutes=10),
|
||||
expires_at=datetime.utcnow() - timedelta(minutes=5),
|
||||
created_at=datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(minutes=10),
|
||||
expires_at=datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(minutes=5),
|
||||
)
|
||||
assert auth_state.is_expired is True
|
||||
assert auth_state.is_active is False
|
||||
|
||||
def test_age(self):
|
||||
"""Test age calculation"""
|
||||
created = datetime.utcnow() - timedelta(minutes=2)
|
||||
created = datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(minutes=2)
|
||||
auth_state = AuthState(
|
||||
state="random123",
|
||||
created_at=created,
|
||||
expires_at=datetime.utcnow() + timedelta(minutes=3),
|
||||
expires_at=datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(minutes=3),
|
||||
)
|
||||
age = auth_state.age
|
||||
# Age should be at least 2 minutes
|
||||
@@ -859,8 +859,8 @@ class TestAuthStateModel:
|
||||
"""Test validation succeeds for active state"""
|
||||
auth_state = AuthState(
|
||||
state="random123",
|
||||
created_at=datetime.utcnow(),
|
||||
expires_at=datetime.utcnow() + timedelta(minutes=5),
|
||||
created_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
expires_at=datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(minutes=5),
|
||||
)
|
||||
assert auth_state.is_valid() is True
|
||||
|
||||
@@ -868,8 +868,8 @@ class TestAuthStateModel:
|
||||
"""Test validation fails for expired state"""
|
||||
auth_state = AuthState(
|
||||
state="random123",
|
||||
created_at=datetime.utcnow() - timedelta(minutes=10),
|
||||
expires_at=datetime.utcnow() - timedelta(minutes=5),
|
||||
created_at=datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(minutes=10),
|
||||
expires_at=datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(minutes=5),
|
||||
)
|
||||
assert auth_state.is_valid() is False
|
||||
|
||||
@@ -877,8 +877,8 @@ class TestAuthStateModel:
|
||||
"""Test validation fails for empty state"""
|
||||
auth_state = AuthState(
|
||||
state="",
|
||||
created_at=datetime.utcnow(),
|
||||
expires_at=datetime.utcnow() + timedelta(minutes=5),
|
||||
created_at=datetime.now(timezone.utc).replace(tzinfo=None),
|
||||
expires_at=datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(minutes=5),
|
||||
)
|
||||
assert auth_state.is_valid() is False
|
||||
|
||||
|
||||
@@ -1,459 +0,0 @@
|
||||
"""
|
||||
Tests for metrics instrumentation (v1.1.2 Phase 1)
|
||||
|
||||
Tests database monitoring, HTTP metrics, memory monitoring, and business metrics.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import sqlite3
|
||||
import time
|
||||
import threading
|
||||
from unittest.mock import Mock, patch, MagicMock
|
||||
|
||||
from starpunk.monitoring import (
|
||||
MonitoredConnection,
|
||||
MemoryMonitor,
|
||||
get_metrics,
|
||||
get_metrics_stats,
|
||||
business,
|
||||
)
|
||||
from starpunk.monitoring.metrics import get_buffer
|
||||
from starpunk.monitoring.http import setup_http_metrics
|
||||
|
||||
|
||||
class TestMonitoredConnection:
|
||||
"""Tests for database operation monitoring"""
|
||||
|
||||
def test_execute_records_metric(self):
|
||||
"""Test that execute() records a metric"""
|
||||
# Create in-memory database
|
||||
conn = sqlite3.connect(':memory:')
|
||||
conn.execute('CREATE TABLE test (id INTEGER, name TEXT)')
|
||||
|
||||
# Wrap with monitoring
|
||||
monitored = MonitoredConnection(conn, slow_query_threshold=1.0)
|
||||
|
||||
# Clear metrics buffer
|
||||
get_buffer().clear()
|
||||
|
||||
# Execute query
|
||||
monitored.execute('SELECT * FROM test')
|
||||
|
||||
# Check metric was recorded
|
||||
metrics = get_metrics()
|
||||
# Note: May not be recorded due to sampling, but slow queries are forced
|
||||
# So we'll check stats instead
|
||||
stats = get_metrics_stats()
|
||||
assert stats['total_count'] >= 0 # May be 0 due to sampling
|
||||
|
||||
def test_slow_query_always_recorded(self):
|
||||
"""Test that slow queries are always recorded regardless of sampling"""
|
||||
# Create in-memory database
|
||||
conn = sqlite3.connect(':memory:')
|
||||
|
||||
# Set very low threshold so any query is "slow"
|
||||
monitored = MonitoredConnection(conn, slow_query_threshold=0.0)
|
||||
|
||||
# Clear metrics buffer
|
||||
get_buffer().clear()
|
||||
|
||||
# Execute query (will be considered slow)
|
||||
monitored.execute('SELECT 1')
|
||||
|
||||
# Check metric was recorded (forced due to being slow)
|
||||
metrics = get_metrics()
|
||||
assert len(metrics) > 0
|
||||
# Check that is_slow is True in metadata
|
||||
assert any(m.metadata.get('is_slow', False) is True for m in metrics)
|
||||
|
||||
def test_extract_table_name_select(self):
|
||||
"""Test table name extraction from SELECT query"""
|
||||
conn = sqlite3.connect(':memory:')
|
||||
conn.execute('CREATE TABLE notes (id INTEGER)')
|
||||
monitored = MonitoredConnection(conn)
|
||||
|
||||
table_name = monitored._extract_table_name('SELECT * FROM notes WHERE id = 1')
|
||||
assert table_name == 'notes'
|
||||
|
||||
def test_extract_table_name_insert(self):
|
||||
"""Test table name extraction from INSERT query"""
|
||||
conn = sqlite3.connect(':memory:')
|
||||
monitored = MonitoredConnection(conn)
|
||||
|
||||
table_name = monitored._extract_table_name('INSERT INTO users (name) VALUES (?)')
|
||||
assert table_name == 'users'
|
||||
|
||||
def test_extract_table_name_update(self):
|
||||
"""Test table name extraction from UPDATE query"""
|
||||
conn = sqlite3.connect(':memory:')
|
||||
monitored = MonitoredConnection(conn)
|
||||
|
||||
table_name = monitored._extract_table_name('UPDATE posts SET title = ?')
|
||||
assert table_name == 'posts'
|
||||
|
||||
def test_extract_table_name_unknown(self):
|
||||
"""Test that complex queries return 'unknown'"""
|
||||
conn = sqlite3.connect(':memory:')
|
||||
monitored = MonitoredConnection(conn)
|
||||
|
||||
# Complex query with JOIN
|
||||
table_name = monitored._extract_table_name(
|
||||
'SELECT a.* FROM notes a JOIN users b ON a.user_id = b.id'
|
||||
)
|
||||
# Our simple regex will find 'notes' from the first FROM
|
||||
assert table_name in ['notes', 'unknown']
|
||||
|
||||
def test_get_query_type(self):
|
||||
"""Test query type extraction"""
|
||||
conn = sqlite3.connect(':memory:')
|
||||
monitored = MonitoredConnection(conn)
|
||||
|
||||
assert monitored._get_query_type('SELECT * FROM notes') == 'SELECT'
|
||||
assert monitored._get_query_type('INSERT INTO notes VALUES (?)') == 'INSERT'
|
||||
assert monitored._get_query_type('UPDATE notes SET x = 1') == 'UPDATE'
|
||||
assert monitored._get_query_type('DELETE FROM notes') == 'DELETE'
|
||||
assert monitored._get_query_type('CREATE TABLE test (id INT)') == 'CREATE'
|
||||
assert monitored._get_query_type('PRAGMA journal_mode=WAL') == 'PRAGMA'
|
||||
|
||||
def test_execute_with_parameters(self):
|
||||
"""Test execute with query parameters"""
|
||||
conn = sqlite3.connect(':memory:')
|
||||
conn.execute('CREATE TABLE test (id INTEGER, name TEXT)')
|
||||
monitored = MonitoredConnection(conn, slow_query_threshold=1.0)
|
||||
|
||||
# Execute with parameters
|
||||
monitored.execute('INSERT INTO test (id, name) VALUES (?, ?)', (1, 'test'))
|
||||
|
||||
# Verify data was inserted
|
||||
cursor = monitored.execute('SELECT * FROM test WHERE id = ?', (1,))
|
||||
rows = cursor.fetchall()
|
||||
assert len(rows) == 1
|
||||
|
||||
def test_executemany(self):
|
||||
"""Test executemany batch operations"""
|
||||
conn = sqlite3.connect(':memory:')
|
||||
conn.execute('CREATE TABLE test (id INTEGER, name TEXT)')
|
||||
monitored = MonitoredConnection(conn)
|
||||
|
||||
# Clear metrics
|
||||
get_buffer().clear()
|
||||
|
||||
# Execute batch insert
|
||||
data = [(1, 'first'), (2, 'second'), (3, 'third')]
|
||||
monitored.executemany('INSERT INTO test (id, name) VALUES (?, ?)', data)
|
||||
|
||||
# Check metric was recorded
|
||||
metrics = get_metrics()
|
||||
# May not be recorded due to sampling
|
||||
stats = get_metrics_stats()
|
||||
assert stats is not None
|
||||
|
||||
def test_error_recording(self):
|
||||
"""Test that errors are recorded in metrics"""
|
||||
conn = sqlite3.connect(':memory:')
|
||||
monitored = MonitoredConnection(conn)
|
||||
|
||||
# Clear metrics
|
||||
get_buffer().clear()
|
||||
|
||||
# Execute invalid query
|
||||
with pytest.raises(sqlite3.OperationalError):
|
||||
monitored.execute('SELECT * FROM nonexistent_table')
|
||||
|
||||
# Check error was recorded (forced)
|
||||
metrics = get_metrics()
|
||||
assert len(metrics) > 0
|
||||
assert any('ERROR' in m.operation_name for m in metrics)
|
||||
|
||||
|
||||
class TestHTTPMetrics:
|
||||
"""Tests for HTTP request/response monitoring"""
|
||||
|
||||
def test_setup_http_metrics(self, app):
|
||||
"""Test HTTP metrics middleware setup"""
|
||||
# Add a simple test route
|
||||
@app.route('/test')
|
||||
def test_route():
|
||||
return 'OK', 200
|
||||
|
||||
setup_http_metrics(app)
|
||||
|
||||
# Clear metrics
|
||||
get_buffer().clear()
|
||||
|
||||
# Make a request
|
||||
with app.test_client() as client:
|
||||
response = client.get('/test')
|
||||
assert response.status_code == 200
|
||||
|
||||
# Check request ID header was added
|
||||
assert 'X-Request-ID' in response.headers
|
||||
|
||||
# Check metrics were recorded
|
||||
metrics = get_metrics()
|
||||
# May be sampled, so just check structure
|
||||
stats = get_metrics_stats()
|
||||
assert stats is not None
|
||||
|
||||
def test_request_id_generation(self, app):
|
||||
"""Test that unique request IDs are generated"""
|
||||
# Add a simple test route
|
||||
@app.route('/test')
|
||||
def test_route():
|
||||
return 'OK', 200
|
||||
|
||||
setup_http_metrics(app)
|
||||
|
||||
request_ids = set()
|
||||
|
||||
with app.test_client() as client:
|
||||
for _ in range(5):
|
||||
response = client.get('/test')
|
||||
request_id = response.headers.get('X-Request-ID')
|
||||
assert request_id is not None
|
||||
request_ids.add(request_id)
|
||||
|
||||
# All request IDs should be unique
|
||||
assert len(request_ids) == 5
|
||||
|
||||
def test_error_metrics_recorded(self, app):
|
||||
"""Test that errors are recorded in metrics"""
|
||||
# Add a simple test route
|
||||
@app.route('/test')
|
||||
def test_route():
|
||||
return 'OK', 200
|
||||
|
||||
setup_http_metrics(app)
|
||||
|
||||
# Clear metrics
|
||||
get_buffer().clear()
|
||||
|
||||
with app.test_client() as client:
|
||||
# Request non-existent endpoint
|
||||
response = client.get('/this-does-not-exist')
|
||||
assert response.status_code == 404
|
||||
|
||||
# Error metrics should be recorded (forced)
|
||||
# Note: 404 is not necessarily an error in the teardown handler
|
||||
# but will be in metrics as a 404 status code
|
||||
metrics = get_metrics()
|
||||
stats = get_metrics_stats()
|
||||
assert stats is not None
|
||||
|
||||
|
||||
class TestMemoryMonitor:
|
||||
"""Tests for memory monitoring thread"""
|
||||
|
||||
def test_memory_monitor_initialization(self):
|
||||
"""Test memory monitor can be initialized"""
|
||||
monitor = MemoryMonitor(interval=1)
|
||||
assert monitor.interval == 1
|
||||
assert monitor.daemon is True # Per CQ5
|
||||
|
||||
def test_memory_monitor_starts_and_stops(self):
|
||||
"""Test memory monitor thread lifecycle"""
|
||||
monitor = MemoryMonitor(interval=1)
|
||||
|
||||
# Start monitor
|
||||
monitor.start()
|
||||
assert monitor.is_alive()
|
||||
|
||||
# Wait a bit for initialization
|
||||
time.sleep(0.5)
|
||||
|
||||
# Stop monitor gracefully
|
||||
monitor.stop()
|
||||
# Give it time to finish gracefully
|
||||
time.sleep(1.0)
|
||||
monitor.join(timeout=5)
|
||||
# Thread should have stopped
|
||||
# Note: In rare cases daemon thread may still be cleaning up
|
||||
if monitor.is_alive():
|
||||
# Give it one more second
|
||||
time.sleep(1.0)
|
||||
assert not monitor.is_alive()
|
||||
|
||||
def test_memory_monitor_collects_metrics(self):
|
||||
"""Test that memory monitor collects metrics"""
|
||||
# Clear metrics
|
||||
get_buffer().clear()
|
||||
|
||||
monitor = MemoryMonitor(interval=1)
|
||||
monitor.start()
|
||||
|
||||
# Wait for baseline + one collection
|
||||
time.sleep(7) # 5s baseline + 2s for collection
|
||||
|
||||
# Stop monitor
|
||||
monitor.stop()
|
||||
monitor.join(timeout=2)
|
||||
|
||||
# Check metrics were collected
|
||||
metrics = get_metrics()
|
||||
memory_metrics = [m for m in metrics if 'memory' in m.operation_name.lower()]
|
||||
|
||||
# Should have at least one memory metric
|
||||
assert len(memory_metrics) > 0
|
||||
|
||||
def test_memory_monitor_stats(self):
|
||||
"""Test memory monitor statistics"""
|
||||
monitor = MemoryMonitor(interval=1)
|
||||
monitor.start()
|
||||
|
||||
# Wait for baseline
|
||||
time.sleep(6)
|
||||
|
||||
# Get stats
|
||||
stats = monitor.get_stats()
|
||||
assert stats['status'] == 'running'
|
||||
assert 'current_rss_mb' in stats
|
||||
assert 'baseline_rss_mb' in stats
|
||||
assert stats['baseline_rss_mb'] > 0
|
||||
|
||||
monitor.stop()
|
||||
monitor.join(timeout=2)
|
||||
|
||||
|
||||
class TestBusinessMetrics:
|
||||
"""Tests for business metrics tracking"""
|
||||
|
||||
def test_track_note_created(self):
|
||||
"""Test note creation tracking"""
|
||||
get_buffer().clear()
|
||||
|
||||
business.track_note_created(note_id=123, content_length=500, has_media=False)
|
||||
|
||||
metrics = get_metrics()
|
||||
assert len(metrics) > 0
|
||||
|
||||
note_metrics = [m for m in metrics if 'note_created' in m.operation_name]
|
||||
assert len(note_metrics) > 0
|
||||
assert note_metrics[0].metadata['note_id'] == 123
|
||||
assert note_metrics[0].metadata['content_length'] == 500
|
||||
|
||||
def test_track_note_updated(self):
|
||||
"""Test note update tracking"""
|
||||
get_buffer().clear()
|
||||
|
||||
business.track_note_updated(
|
||||
note_id=456,
|
||||
content_length=750,
|
||||
fields_changed=['title', 'content']
|
||||
)
|
||||
|
||||
metrics = get_metrics()
|
||||
note_metrics = [m for m in metrics if 'note_updated' in m.operation_name]
|
||||
assert len(note_metrics) > 0
|
||||
assert note_metrics[0].metadata['note_id'] == 456
|
||||
|
||||
def test_track_note_deleted(self):
|
||||
"""Test note deletion tracking"""
|
||||
get_buffer().clear()
|
||||
|
||||
business.track_note_deleted(note_id=789)
|
||||
|
||||
metrics = get_metrics()
|
||||
note_metrics = [m for m in metrics if 'note_deleted' in m.operation_name]
|
||||
assert len(note_metrics) > 0
|
||||
assert note_metrics[0].metadata['note_id'] == 789
|
||||
|
||||
def test_track_feed_generated(self):
|
||||
"""Test feed generation tracking"""
|
||||
get_buffer().clear()
|
||||
|
||||
business.track_feed_generated(
|
||||
format='rss',
|
||||
item_count=50,
|
||||
duration_ms=45.2,
|
||||
cached=False
|
||||
)
|
||||
|
||||
metrics = get_metrics()
|
||||
feed_metrics = [m for m in metrics if 'feed_rss' in m.operation_name]
|
||||
assert len(feed_metrics) > 0
|
||||
assert feed_metrics[0].metadata['format'] == 'rss'
|
||||
assert feed_metrics[0].metadata['item_count'] == 50
|
||||
|
||||
def test_track_cache_hit(self):
|
||||
"""Test cache hit tracking"""
|
||||
get_buffer().clear()
|
||||
|
||||
business.track_cache_hit(cache_type='feed', key='rss:latest')
|
||||
|
||||
metrics = get_metrics()
|
||||
cache_metrics = [m for m in metrics if 'cache_hit' in m.operation_name]
|
||||
assert len(cache_metrics) > 0
|
||||
|
||||
def test_track_cache_miss(self):
|
||||
"""Test cache miss tracking"""
|
||||
get_buffer().clear()
|
||||
|
||||
business.track_cache_miss(cache_type='feed', key='atom:latest')
|
||||
|
||||
metrics = get_metrics()
|
||||
cache_metrics = [m for m in metrics if 'cache_miss' in m.operation_name]
|
||||
assert len(cache_metrics) > 0
|
||||
|
||||
|
||||
class TestMetricsConfiguration:
|
||||
"""Tests for metrics configuration"""
|
||||
|
||||
def test_metrics_can_be_disabled(self, app):
|
||||
"""Test that metrics can be disabled via configuration"""
|
||||
# This would be tested by setting METRICS_ENABLED=False
|
||||
# and verifying no metrics are collected
|
||||
assert 'METRICS_ENABLED' in app.config
|
||||
|
||||
def test_slow_query_threshold_configurable(self, app):
|
||||
"""Test that slow query threshold is configurable"""
|
||||
assert 'METRICS_SLOW_QUERY_THRESHOLD' in app.config
|
||||
assert isinstance(app.config['METRICS_SLOW_QUERY_THRESHOLD'], float)
|
||||
|
||||
def test_sampling_rate_configurable(self, app):
|
||||
"""Test that sampling rate is configurable"""
|
||||
assert 'METRICS_SAMPLING_RATE' in app.config
|
||||
assert isinstance(app.config['METRICS_SAMPLING_RATE'], float)
|
||||
assert 0.0 <= app.config['METRICS_SAMPLING_RATE'] <= 1.0
|
||||
|
||||
def test_buffer_size_configurable(self, app):
|
||||
"""Test that buffer size is configurable"""
|
||||
assert 'METRICS_BUFFER_SIZE' in app.config
|
||||
assert isinstance(app.config['METRICS_BUFFER_SIZE'], int)
|
||||
assert app.config['METRICS_BUFFER_SIZE'] > 0
|
||||
|
||||
def test_memory_interval_configurable(self, app):
|
||||
"""Test that memory monitor interval is configurable"""
|
||||
assert 'METRICS_MEMORY_INTERVAL' in app.config
|
||||
assert isinstance(app.config['METRICS_MEMORY_INTERVAL'], int)
|
||||
assert app.config['METRICS_MEMORY_INTERVAL'] > 0
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def app():
|
||||
"""Create test Flask app with minimal configuration"""
|
||||
from flask import Flask
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
# Create temp directory for testing
|
||||
temp_dir = tempfile.mkdtemp()
|
||||
temp_path = Path(temp_dir)
|
||||
|
||||
# Minimal configuration to avoid migration issues
|
||||
app.config.update({
|
||||
'TESTING': True,
|
||||
'DATABASE_PATH': temp_path / 'test.db',
|
||||
'DATA_PATH': temp_path,
|
||||
'NOTES_PATH': temp_path / 'notes',
|
||||
'SESSION_SECRET': 'test-secret',
|
||||
'ADMIN_ME': 'https://test.example.com',
|
||||
'METRICS_ENABLED': True,
|
||||
'METRICS_SLOW_QUERY_THRESHOLD': 1.0,
|
||||
'METRICS_SAMPLING_RATE': 1.0,
|
||||
'METRICS_BUFFER_SIZE': 1000,
|
||||
'METRICS_MEMORY_INTERVAL': 30,
|
||||
})
|
||||
|
||||
return app
|
||||
@@ -1,103 +0,0 @@
|
||||
"""
|
||||
Tests for feed statistics tracking
|
||||
|
||||
Tests feed statistics aggregation per v1.1.2 Phase 3.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from starpunk.monitoring.business import get_feed_statistics, track_feed_generated
|
||||
|
||||
|
||||
def test_get_feed_statistics_returns_structure():
|
||||
"""Test get_feed_statistics returns expected structure"""
|
||||
stats = get_feed_statistics()
|
||||
|
||||
# Check top-level keys
|
||||
assert "by_format" in stats
|
||||
assert "cache" in stats
|
||||
assert "total_requests" in stats
|
||||
assert "format_percentages" in stats
|
||||
|
||||
# Check by_format structure
|
||||
assert "rss" in stats["by_format"]
|
||||
assert "atom" in stats["by_format"]
|
||||
assert "json" in stats["by_format"]
|
||||
|
||||
# Check format stats structure
|
||||
for format_name in ["rss", "atom", "json"]:
|
||||
fmt_stats = stats["by_format"][format_name]
|
||||
assert "generated" in fmt_stats
|
||||
assert "cached" in fmt_stats
|
||||
assert "total" in fmt_stats
|
||||
assert "avg_duration_ms" in fmt_stats
|
||||
|
||||
# Check cache structure
|
||||
assert "hits" in stats["cache"]
|
||||
assert "misses" in stats["cache"]
|
||||
assert "hit_rate" in stats["cache"]
|
||||
|
||||
|
||||
def test_get_feed_statistics_empty_metrics():
|
||||
"""Test get_feed_statistics with no metrics returns zeros"""
|
||||
stats = get_feed_statistics()
|
||||
|
||||
# All values should be zero or empty
|
||||
assert stats["total_requests"] >= 0
|
||||
assert stats["cache"]["hit_rate"] >= 0.0
|
||||
assert stats["cache"]["hit_rate"] <= 1.0
|
||||
|
||||
|
||||
def test_feed_statistics_cache_hit_rate_calculation():
|
||||
"""Test cache hit rate is calculated correctly"""
|
||||
stats = get_feed_statistics()
|
||||
|
||||
# Hit rate should be between 0 and 1
|
||||
assert 0.0 <= stats["cache"]["hit_rate"] <= 1.0
|
||||
|
||||
# If there are hits and misses, hit rate should be hits / (hits + misses)
|
||||
if stats["cache"]["hits"] + stats["cache"]["misses"] > 0:
|
||||
expected_rate = stats["cache"]["hits"] / (
|
||||
stats["cache"]["hits"] + stats["cache"]["misses"]
|
||||
)
|
||||
assert abs(stats["cache"]["hit_rate"] - expected_rate) < 0.001
|
||||
|
||||
|
||||
def test_feed_statistics_format_percentages():
|
||||
"""Test format percentages sum to 1.0 when there are requests"""
|
||||
stats = get_feed_statistics()
|
||||
|
||||
if stats["total_requests"] > 0:
|
||||
total_percentage = sum(stats["format_percentages"].values())
|
||||
# Should sum to approximately 1.0 (allowing for floating point errors)
|
||||
assert abs(total_percentage - 1.0) < 0.001
|
||||
|
||||
|
||||
def test_feed_statistics_total_requests_sum():
|
||||
"""Test total_requests equals sum of all format totals"""
|
||||
stats = get_feed_statistics()
|
||||
|
||||
format_total = sum(
|
||||
fmt["total"] for fmt in stats["by_format"].values()
|
||||
)
|
||||
|
||||
assert stats["total_requests"] == format_total
|
||||
|
||||
|
||||
def test_track_feed_generated_records_metrics():
|
||||
"""Test track_feed_generated creates metrics entries"""
|
||||
# Note: This test just verifies the function runs without error.
|
||||
# Actual metrics tracking is tested in integration tests.
|
||||
track_feed_generated(
|
||||
format="rss",
|
||||
item_count=10,
|
||||
duration_ms=50.5,
|
||||
cached=False
|
||||
)
|
||||
|
||||
# Get statistics - may be empty if metrics buffer hasn't persisted yet
|
||||
stats = get_feed_statistics()
|
||||
|
||||
# Verify structure is correct
|
||||
assert "total_requests" in stats
|
||||
assert "by_format" in stats
|
||||
assert "cache" in stats
|
||||
@@ -0,0 +1,56 @@
|
||||
"""
|
||||
Rate limits are shared by all server processes (v2.11.0)
|
||||
|
||||
Each gunicorn process used to keep its own counts, so with 4 workers a client
|
||||
got four times the limit. Two SharedRateLimiter instances on one database
|
||||
stand in for two processes.
|
||||
"""
|
||||
|
||||
import time
|
||||
from unittest.mock import patch
|
||||
|
||||
from starpunk.rate_limiter import SharedRateLimiter
|
||||
|
||||
|
||||
def test_two_processes_share_one_limit(app):
|
||||
first = SharedRateLimiter(limit=3, window_seconds=60, prefix="t")
|
||||
second = SharedRateLimiter(limit=3, window_seconds=60, prefix="t")
|
||||
with app.app_context(), patch("starpunk.rate_limiter.time.time", return_value=6000.0):
|
||||
results = [limiter.is_allowed("1.2.3.4") for limiter in (first, second, first, second)]
|
||||
assert results == [True, True, True, False]
|
||||
|
||||
|
||||
def test_keys_and_prefixes_are_separate(app):
|
||||
a = SharedRateLimiter(limit=1, window_seconds=60, prefix="a")
|
||||
b = SharedRateLimiter(limit=1, window_seconds=60, prefix="b")
|
||||
with app.app_context(), patch("starpunk.rate_limiter.time.time", return_value=6000.0):
|
||||
assert a.is_allowed("x") and a.is_allowed("y") and b.is_allowed("x")
|
||||
assert not a.is_allowed("x")
|
||||
|
||||
|
||||
def test_window_slides(app):
|
||||
limiter = SharedRateLimiter(limit=2, window_seconds=60, prefix="s")
|
||||
with app.app_context():
|
||||
# Window 100 (6000-6059 s): two requests use the limit
|
||||
with patch("starpunk.rate_limiter.time.time", return_value=6000.0):
|
||||
assert limiter.is_allowed("k") and limiter.is_allowed("k")
|
||||
assert not limiter.is_allowed("k")
|
||||
# 6030 s is still window 100: still limited
|
||||
with patch("starpunk.rate_limiter.time.time", return_value=6030.0):
|
||||
assert not limiter.is_allowed("k")
|
||||
# 6090 s is halfway through window 101: window 100 counts half (2 x 0.5 = 1)
|
||||
with patch("starpunk.rate_limiter.time.time", return_value=6090.0):
|
||||
assert limiter.is_allowed("k") # 1 + 0 < 2
|
||||
assert not limiter.is_allowed("k") # 1 + 1 = 2
|
||||
|
||||
|
||||
def test_cleanup_removes_old_windows(app):
|
||||
import sqlite3
|
||||
limiter = SharedRateLimiter(limit=5, window_seconds=60, prefix="c")
|
||||
with app.app_context():
|
||||
with patch("starpunk.rate_limiter.time.time", return_value=6000.0):
|
||||
limiter.is_allowed("k")
|
||||
with patch("starpunk.rate_limiter.time.time", return_value=9000.0):
|
||||
limiter.cleanup()
|
||||
rows = sqlite3.connect(app.config["DATABASE_PATH"]).execute("SELECT COUNT(*) FROM rate_limits").fetchone()[0]
|
||||
assert rows == 0
|
||||
@@ -99,7 +99,8 @@ class FakeWeb:
|
||||
self.fetched = []
|
||||
|
||||
def __call__(self, request):
|
||||
url = str(request.url)
|
||||
# Pinned requests go to the checked IP; the Host header names the site
|
||||
url = f"{request.url.scheme}://{request.headers['host']}{request.url.raw_path.decode()}"
|
||||
self.fetched.append(url)
|
||||
status, headers, body = self.pages.get(url, (404, {}, b"nope"))
|
||||
return httpx.Response(status, headers=headers, content=body)
|
||||
@@ -112,11 +113,20 @@ def web():
|
||||
|
||||
def routed(pds, web):
|
||||
return httpx.MockTransport(
|
||||
lambda r: pds(r) if r.url.host in ("bsky.example", "pds.example") else web(r))
|
||||
lambda r: pds(r) if r.headers.get("host", r.url.host) in ("bsky.example", "pds.example") else web(r))
|
||||
|
||||
|
||||
def resolve(host, *args, **kwargs):
|
||||
"""Every test host is public, except internal.example"""
|
||||
ip = "10.0.0.5" if host == "internal.example" else "93.184.216.34"
|
||||
return [(2, 1, 6, "", (ip, 0))]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def app(tmp_path, pds, web):
|
||||
def app(tmp_path, pds, web, monkeypatch):
|
||||
# Pages and images for link cards go through the shared SSRF-safe fetcher (v2.11.0)
|
||||
monkeypatch.setattr("starpunk.webmention_ssrf.TRANSPORT", routed(pds, web))
|
||||
monkeypatch.setattr("starpunk.webmention_ssrf.socket.getaddrinfo", resolve)
|
||||
data = tmp_path / "data"
|
||||
app = create_app({
|
||||
"TESTING": True,
|
||||
@@ -328,12 +338,7 @@ class TestLinkCards:
|
||||
|
||||
def test_redirect_to_private_address_is_not_followed(self, app, pds, web, monkeypatch):
|
||||
web.pages[ARTICLE] = (302, {"location": "http://internal.example/admin"}, b"")
|
||||
monkeypatch.setattr("starpunk.micropub._public_address", lambda host: host != "internal.example")
|
||||
app.config["TESTING"] = False
|
||||
try:
|
||||
publish(app, content=f"See {ARTICLE}")
|
||||
finally:
|
||||
app.config["TESTING"] = True
|
||||
publish(app, content=f"See {ARTICLE}") # internal.example resolves to 10.0.0.5
|
||||
assert "http://internal.example/admin" not in web.fetched
|
||||
assert only_record(pds)["embed"]["external"]["title"] == ARTICLE
|
||||
|
||||
|
||||
Reference in new issue
Block a user