From 742e2e3ec374302fca1dbd265ee5047c9068f001 Mon Sep 17 00:00:00 2001 From: Phil Date: Tue, 3 Nov 2026 16:07:47 -0700 Subject: [PATCH] Remove the metrics dashboard and monitoring package (v2.12.0) The in-memory metrics system covered one gunicorn process at a time, was lost on restart and was never used. Remove /admin/metrics-dashboard, /admin/metrics, starpunk/monitoring, the HTTP/query timing, memory monitor, feed counters, METRICS_* settings and psutil. Keep /health, /admin/health (without its metrics block), pool stats and the feed cache. ADR-072. Co-Authored-By: Claude Opus 5.5 --- .env.example | 12 +- CHANGELOG.md | 11 + README.md | 2 +- docs/architecture/database.md | 3 +- docs/architecture/deployment.md | 9 +- docs/architecture/overview.md | 21 +- docs/architecture/technology-stack.md | 1 - ...ADR-053-performance-monitoring-strategy.md | 4 +- ...060-production-hotfix-metrics-dashboard.md | 4 +- .../ADR-072-remove-metrics-dashboard.md | 51 ++ docs/decisions/INDEX.md | 11 +- docs/design/INDEX.md | 6 +- .../v2.12.0/2026-11-03-remove-metrics.md | 28 + docs/projectplan/INDEX.md | 1 + docs/projectplan/v2.12.0/RELEASE.md | 21 + docs/standards/development-setup.md | 9 +- requirements.txt | 3 - starpunk/__init__.py | 21 +- starpunk/config.py | 7 - starpunk/database/__init__.py | 2 +- starpunk/database/pool.py | 34 +- starpunk/feeds/atom.py | 17 - starpunk/feeds/json_feed.py | 28 - starpunk/feeds/rss.py | 28 - starpunk/monitoring/__init__.py | 35 -- starpunk/monitoring/business.py | 298 --------- starpunk/monitoring/database.py | 236 ------- starpunk/monitoring/http.py | 133 ---- starpunk/monitoring/memory.py | 191 ------ starpunk/monitoring/metrics.py | 425 ------------- starpunk/routes/admin.py | 208 +------ templates/admin/base.html | 1 - templates/admin/metrics_dashboard.html | 585 ------------------ tests/test_admin_feed_statistics.py | 108 ---- tests/test_monitoring.py | 459 -------------- tests/test_monitoring_feed_statistics.py | 103 --- 36 files changed, 153 insertions(+), 2963 deletions(-) create mode 100644 docs/decisions/ADR-072-remove-metrics-dashboard.md create mode 100644 docs/design/v2.12.0/2026-11-03-remove-metrics.md create mode 100644 docs/projectplan/v2.12.0/RELEASE.md delete mode 100644 starpunk/monitoring/__init__.py delete mode 100644 starpunk/monitoring/business.py delete mode 100644 starpunk/monitoring/database.py delete mode 100644 starpunk/monitoring/http.py delete mode 100644 starpunk/monitoring/memory.py delete mode 100644 starpunk/monitoring/metrics.py delete mode 100644 templates/admin/metrics_dashboard.html delete mode 100644 tests/test_admin_feed_statistics.py delete mode 100644 tests/test_monitoring.py delete mode 100644 tests/test_monitoring_feed_statistics.py diff --git a/.env.example b/.env.example index be1e920..491a4c1 100644 --- a/.env.example +++ b/.env.example @@ -199,19 +199,9 @@ THREADS_USERNAME= # SYNDICATION_POLL_INTERVAL=10 # ============================================================================= -# MONITORING +# DEBUGGING # ============================================================================= -# Metrics shown on /admin/metrics-dashboard (default: true) -# METRICS_ENABLED=true -# Queries slower than this many seconds are logged (default: 1.0) -# METRICS_SLOW_QUERY_THRESHOLD=1.0 -# Fraction of operations recorded, 0.0-1.0 (default: 1.0) -# METRICS_SAMPLING_RATE=1.0 -# METRICS_BUFFER_SIZE=1000 -# Seconds between memory samples (default: 30) -# METRICS_MEMORY_INTERVAL=30 - # Keep uploads that fail processing in DATA_PATH/debug for diagnosis # (default: false); cleaned up by age (default: 7 days) and total size # (default: 100 MB) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0111781..e8dab55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [2.12.0] - 2026-11-03 + +The metrics dashboard is gone (ADR-072). + +### Removed + +- The admin metrics dashboard (`/admin/metrics-dashboard`, the "Metrics" admin link) and its JSON endpoint `/admin/metrics`. Its numbers were kept in memory per server process, so they covered a quarter of the traffic and vanished on every restart +- Everything that only fed it: request and database-query timing, the memory monitor thread, feed-generation counters (`starpunk/monitoring/`), and the `X-Request-ID` response header. Slow queries are no longer logged +- The `METRICS_ENABLED`, `METRICS_SLOW_QUERY_THRESHOLD`, `METRICS_SAMPLING_RATE`, `METRICS_BUFFER_SIZE` and `METRICS_MEMORY_INTERVAL` settings (ignored if still set), and the `psutil` dependency +- The `metrics` section of `/admin/health`; its database, pool, filesystem and disk checks are unchanged, as is `/health` + ## [2.11.0] - 2026-11-03 Maintenance release: dependency upgrades and the High-priority backlog items. diff --git a/README.md b/README.md index 58da7a4..ec65f90 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ what changed in each release is in [CHANGELOG.md](CHANGELOG.md). **Operations** - Migrations applied automatically at startup -- `/health` endpoint, admin metrics dashboard, structured logs +- `/health` endpoint, `/admin/health` diagnostics, structured logs - Fetches of user-supplied URLs are limited to public addresses, size and time diff --git a/docs/architecture/database.md b/docs/architecture/database.md index c05e596..9933cdc 100644 --- a/docs/architecture/database.md +++ b/docs/architecture/database.md @@ -218,8 +218,7 @@ server process enforces one shared limit. Old windows are pruned as it goes. timeout; `DB_POOL_SIZE` and `DB_TIMEOUT` can be set only through the `create_app()` config dict). Connections use `sqlite3.Row` rows, `check_same_thread=False`, `PRAGMA journal_mode=WAL` and (since v2.11.0) - `PRAGMA foreign_keys=ON`, so `ON DELETE CASCADE`/`SET NULL` run, and are wrapped - in `MonitoredConnection` for query timing when metrics are on. `get_db()` + `PRAGMA foreign_keys=ON`, so `ON DELETE CASCADE`/`SET NULL` run. `get_db()` stores one connection in `flask.g` per request or app context; on teardown it goes back to the pool, rolled back first if a transaction was left open. diff --git a/docs/architecture/deployment.md b/docs/architecture/deployment.md index eded392..bd71b33 100644 --- a/docs/architecture/deployment.md +++ b/docs/architecture/deployment.md @@ -235,15 +235,12 @@ starts only if at least one is. | `THREADS_MAX_MEDIA` | `4` | | | `SYNDICATION_POLL_INTERVAL` | `10` | Seconds | -### Metrics and debugging +### Debugging + +The `METRICS_*` variables were removed in v2.12.0 and are ignored if set. | Variable | Default | Notes | |----------|---------|-------| -| `METRICS_ENABLED` | `true` | HTTP/database metrics and the memory monitor | -| `METRICS_SLOW_QUERY_THRESHOLD` | `1.0` | Seconds | -| `METRICS_SAMPLING_RATE` | `1.0` | | -| `METRICS_BUFFER_SIZE` | `1000` | | -| `METRICS_MEMORY_INTERVAL` | `30` | Seconds | | `DEBUG_SAVE_FAILED_UPLOADS` | `false` | Keep failed uploads in `debug/` | | `DEBUG_FILE_MAX_AGE_DAYS` | `7` | | | `DEBUG_FILE_MAX_SIZE_MB` | `100` | | diff --git a/docs/architecture/overview.md b/docs/architecture/overview.md index 75e6efd..7560405 100644 --- a/docs/architecture/overview.md +++ b/docs/architecture/overview.md @@ -43,8 +43,6 @@ Key decisions: [ADR-058](../decisions/ADR-058-image-optimization-strategy.md) (media), [ADR-034](../decisions/ADR-034-full-text-search.md) (search), [ADR-052](../decisions/ADR-052-configuration-system-architecture.md) (configuration), -[ADR-053](../decisions/ADR-053-performance-monitoring-strategy.md) and -[ADR-060](../decisions/ADR-060-production-hotfix-metrics-dashboard.md) (metrics), [ADR-054](../decisions/ADR-054-structured-logging-architecture.md) (logging), [ADR-055](../decisions/ADR-055-error-handling-philosophy.md) (errors). @@ -65,7 +63,6 @@ Key decisions: │ webmention worker works its queues only while holding a lock │ │ websub worker works its queues only while holding a lock │ │ syndication worker claims rows with a lease in SQLite │ -│ memory monitor per-process metrics │ └───────────────┬──────────────────────────────┬─────────────────────┘ │ │ ┌───────▼────────┐ ┌──────────▼──────────────────┐ @@ -86,7 +83,7 @@ Key decisions: validates configuration, configures logging, cleans up old debug and temporary upload files, creates the schema and runs migrations (under a lock, see [database.md](database.md)), creates the connection pool, - installs the HTTP metrics middleware, configures the feed cache and rate + configures the feed cache and rate limiters, fills the FTS index if it is empty, registers blueprints, error handlers and template helpers, and starts the background threads. 3. `before_request` assigns a correlation ID (included in every log line) and, @@ -94,8 +91,7 @@ Key decisions: 4. The blueprint handler runs. Database access goes through `get_db()`, which hands out a pooled, request-scoped SQLite connection. 5. `after_request` hooks add `Link: ; rel="webmention"` to HTML - responses, WebSub `Link rel=hub` / `rel=self` headers to topic responses, - and request metrics with an `X-Request-ID` header. + responses, WebSub `Link rel=hub` / `rel=self` headers to topic responses. 6. Errors are rendered by the central handlers in `starpunk/errors.py` (JSON for Micropub, HTML pages from `templates/.html` elsewhere). @@ -141,7 +137,6 @@ wait for outbound network calls; the workers make them later. | `errors.py` | Central error handlers | | `feeds/` | `rss.py` (feedgen), `atom.py`, `json_feed.py`, `opml.py`, `negotiation.py` (Accept header), `cache.py` (LRU + TTL feed cache with ETags) | | `feed.py` | Backward-compatibility re-export of `feeds.rss` | -| `monitoring/` | Metrics buffer, HTTP middleware, database timing wrapper, memory monitor, business metrics | | `database/` | `schema.py` (`INITIAL_SCHEMA_SQL`), `init.py` (`init_db`), `pool.py` (connection pool) | | `migrations.py` | Migration runner for `migrations/NNN_*.sql` | @@ -196,7 +191,7 @@ Everything works without JavaScript. | `/admin/webmentions`, `/admin/webmentions/`, `…/action`, `…/bulk-action` | Webmention moderation | | `/admin/tokens`, `/admin/tokens//revoke` | Issued IndieAuth tokens | | `/admin/syndication//retry` | Retry a failed crosspost | -| `/admin/metrics-dashboard`, `/admin/metrics`, `/admin/health` | Monitoring (HTML dashboard, JSON metrics, full diagnostics) | +| `/admin/health` | Full diagnostics (JSON): database, pool statistics, filesystem, disk | | `/health` | Public liveness check (`{"status": "ok", "version": …}`); `?detailed=true` requires login | ## Background workers @@ -210,7 +205,6 @@ coordinated through the data volume: | Webmention worker (`webmention_worker.py`) | `WEBMENTION_WORKER_ENABLED` and not testing | Non-blocking `fcntl.flock` on `webmention-worker.lock` next to the database. Only the holder works the queues; the others try again each poll interval. The OS releases the lock if the holder dies, and the new holder resets rows stuck in `sending`. | Validates received webmentions; discovers endpoints and sends queued outgoing webmentions, with retries | | WebSub worker (`websub_worker.py`) | `WEBSUB_ENABLED` and not testing | Same pattern with `websub-worker.lock` | Verifies subscription intents, delivers topic content (signed with `X-Hub-Signature` when the subscriber gave a secret), retries, expires leases; pings an external hub instead if `WEBSUB_HUB_URL` is set | | Syndication worker (`syndication.py`) | At least one crosspost target configured, and not testing | No file lock. Rows in `syndications` are claimed with a 5-minute `claimed_until` lease by an atomic `UPDATE`, so any process may work a row but only one at a time; a `version` column keeps edits made during a send. Threads token refreshes are serialised with a lock file next to `threads_token.json`. | Creates, updates or deletes the remote post for each (note, target) pair; up to 8 attempts | -| Memory monitor (`monitoring/memory.py`) | `METRICS_ENABLED` and not testing | None (per-process metrics) | Samples process memory every `METRICS_MEMORY_INTERVAL` seconds | All workers run inside an app context, are daemon threads (they end with the process) and sleep for their poll interval when there is nothing to do. Under @@ -326,14 +320,11 @@ their credentials are set. The full list with defaults is in - Logs go to stdout and `logs/starpunk.log` (10 MB × 10 files); each line carries the request's correlation ID. -- Metrics (`monitoring/`): HTTP request timings, database query timings - (with a slow-query threshold), memory, pool statistics and business - counters, in an in-memory buffer per process, shown at - `/admin/metrics-dashboard` and `/admin/metrics`. - `/health` reports status, version and webmention-worker statistics; `/health?detailed=true` checks database, filesystem and disk space; - `/admin/health` adds metrics, pool statistics and system information. + `/admin/health` adds pool statistics and filesystem details. (The metrics + dashboard was removed in v2.12.0, see ADR-072.) --- -**Last Updated**: 2026-11-03 (v2.10.1) +**Last Updated**: 2026-11-03 (v2.12.0) diff --git a/docs/architecture/technology-stack.md b/docs/architecture/technology-stack.md index 6705a54..15c9262 100644 --- a/docs/architecture/technology-stack.md +++ b/docs/architecture/technology-stack.md @@ -47,7 +47,6 @@ vulnerabilities (`pip-audit`); check again before each release. | python-dotenv | `1.2.*` | Loading `.env` | | Pillow | `13.0.*` | Image validation, resizing and variants | | pillow-heif | `1.8.*` | HEIC/HEIF uploads (iPhone photos) | -| psutil | `7.2.*` | Memory monitor and system metrics | `markupsafe` comes with Flask/Jinja2. diff --git a/docs/decisions/ADR-053-performance-monitoring-strategy.md b/docs/decisions/ADR-053-performance-monitoring-strategy.md index a49a474..89f0bc6 100644 --- a/docs/decisions/ADR-053-performance-monitoring-strategy.md +++ b/docs/decisions/ADR-053-performance-monitoring-strategy.md @@ -1,7 +1,9 @@ # ADR-053: Performance Monitoring Strategy ## Status -Accepted +Superseded by ADR-072 (was: Accepted, v1.1.1) + +> The metrics system was removed in v2.12.0. (status updated 2026-11-03) ## Context StarPunk v1.1.1 introduces performance monitoring to help operators understand system behavior in production. Currently, we have no visibility into: diff --git a/docs/decisions/ADR-060-production-hotfix-metrics-dashboard.md b/docs/decisions/ADR-060-production-hotfix-metrics-dashboard.md index e71dfca..b045c3e 100644 --- a/docs/decisions/ADR-060-production-hotfix-metrics-dashboard.md +++ b/docs/decisions/ADR-060-production-hotfix-metrics-dashboard.md @@ -2,7 +2,9 @@ ## Status -Accepted (v1.1.1-rc.2, 2025-11-25) +Superseded by ADR-072 (was: Accepted, v1.1.1-rc.2, 2025-11-25) + +> The metrics dashboard was removed in v2.12.0. (status updated 2026-11-03) > Reconstructed on 2026-11-03 from code, changelog and related documents; > the original was lost. The original diff --git a/docs/decisions/ADR-072-remove-metrics-dashboard.md b/docs/decisions/ADR-072-remove-metrics-dashboard.md new file mode 100644 index 0000000..184fec6 --- /dev/null +++ b/docs/decisions/ADR-072-remove-metrics-dashboard.md @@ -0,0 +1,51 @@ +# ADR-072: Remove the metrics dashboard + +## Status + +Accepted (v2.12.0, 2026-11-03). Supersedes ADR-053 and ADR-060. + +## Context + +v1.1.1 and v1.1.2 added built-in performance monitoring (ADR-053): HTTP +request timing middleware, a wrapper on every database connection that +timed queries, a memory-sampling thread, feed-generation counters, an +in-memory buffer of the results, and an HTML dashboard at +`/admin/metrics-dashboard` with a JSON twin at `/admin/metrics` (ADR-060 +covered adapting the data for the dashboard template). + +In practice it told us little: + +- The buffer lives in memory in each of the four gunicorn processes, so the + dashboard showed whichever process served the request, and everything was + lost on restart or deploy. +- Nothing in it was ever used to find or fix a problem. Logs (with + correlation IDs), `/health` and `/admin/health` were. +- It cost something on every request and every query, kept a thread per + process, and needed `psutil`, about 1,300 lines of code and three test + files. + +## Decision + +Remove the dashboard and everything that only fed it: the `starpunk/monitoring` +package, the HTTP metrics middleware (including its `X-Request-ID` +response header), `MonitoredConnection`, the memory monitor, the +feed-generation tracking calls, `/admin/metrics-dashboard`, +`/admin/metrics`, the `METRICS_*` settings and the `psutil` dependency. + +Keep: + +- `/health` and `/health?detailed=true`; +- `/admin/health`, without its `metrics` section (database, pool + statistics, filesystem and disk checks stay); +- connection pool statistics (`get_pool_stats`) and the feed cache, which + do work of their own; +- structured logs with correlation IDs (ADR-054). + +## Consequences + +- Less code and one fewer dependency; no timing work on each request or + query; one fewer thread per process. +- Slow database queries are no longer logged. If that is ever needed, a + query timer can be added at the pool level, or an external tool used. +- `METRICS_*` environment variables are ignored if still set. +- Bookmarks to `/admin/metrics-dashboard` now get a 404. diff --git a/docs/decisions/INDEX.md b/docs/decisions/INDEX.md index 9d98ae6..14159da 100644 --- a/docs/decisions/INDEX.md +++ b/docs/decisions/INDEX.md @@ -5,8 +5,8 @@ records one decision: its context, what was decided, and the consequences. Superseded ADRs are kept as history; their Status line names the ADR that replaced them. -**Total ADR files**: 67 (66 numbers: ADR-001 to ADR-044 and ADR-050 to -ADR-071, with two files numbered ADR-012). See [Numbering](#numbering). +**Total ADR files**: 68 (67 numbers: ADR-001 to ADR-044 and ADR-050 to +ADR-072, with two files numbered ADR-012). See [Numbering](#numbering). Status values: **Accepted** (in force), **Superseded** (replaced by a later ADR), **Deprecated** (withdrawn, nothing replaces it; none at present), @@ -101,10 +101,9 @@ See also [IndieWeb protocols](../architecture/indieweb-protocols.md). |---|---|---|---| | [012](ADR-012-http-error-handling-policy.md) | HTTP Error Handling Policy | Accepted | Correct HTTP status codes (e.g. 404 for missing notes) across routes. | | [052](ADR-052-configuration-system-architecture.md) | Configuration System Architecture | Accepted | Environment variables with defaults, loaded in one module. | -| [053](ADR-053-performance-monitoring-strategy.md) | Performance Monitoring Strategy | Accepted | Built-in, standard-library performance metrics (v1.1.1). | | [054](ADR-054-structured-logging-architecture.md) | Structured Logging Architecture | Accepted | Rotating log files with correlation IDs (v1.1.1). | | [055](ADR-055-error-handling-philosophy.md) | Error Handling Philosophy | Accepted | Layered handling: graceful degradation, clear user messages, detailed logs. | -| [060](ADR-060-production-hotfix-metrics-dashboard.md) | Route Adapter for the Metrics Dashboard | Accepted | Adapt monitoring data to the dashboard template in the route (v1.1.1-rc.2). *Reconstructed.* | +| [072](ADR-072-remove-metrics-dashboard.md) | Remove the metrics dashboard | Accepted | Drop the in-memory metrics system and dashboard; keep `/health`, `/admin/health` and logs (v2.12.0). | See also [deployment](../architecture/deployment.md). @@ -117,6 +116,8 @@ designs of v0.x–v1.6.0, all replaced by ADR-064 in v2.0.0. |---|---|---|---| | [005](ADR-005-indielogin-authentication.md) | IndieLogin Authentication Integration | Superseded by 064 | Admin login through indielogin.com. | | [007](ADR-007-slug-generation-algorithm.md) | Slug Generation Algorithm | Superseded by 062 | Slugs from the first words of the note. | +| [053](ADR-053-performance-monitoring-strategy.md) | Performance Monitoring Strategy | Superseded by 072 | Built-in, standard-library performance metrics (v1.1.1). | +| [060](ADR-060-production-hotfix-metrics-dashboard.md) | Route Adapter for the Metrics Dashboard | Superseded by 072 | Adapt monitoring data to the dashboard template in the route (v1.1.1-rc.2). *Reconstructed.* | | [016](ADR-016-indieauth-client-discovery.md) | IndieAuth Client Discovery Mechanism | Superseded by 019 | h-app markup for client discovery. | | [017](ADR-017-oauth-client-metadata-document.md) | OAuth Client ID Metadata Document Implementation | Superseded by 019 | OAuth client metadata endpoint. | | [018](ADR-018-indieauth-detailed-logging.md) | IndieAuth Detailed Logging Strategy | Superseded by 064 | Redacted HTTP logging of the IndieAuth login flow. | @@ -166,7 +167,7 @@ was carried out and is not itself superseded. ## How to create a new ADR -1. Use the next number after the highest existing one (currently ADR-072). +1. Use the next number after the highest existing one (currently ADR-073). 2. Name it `ADR-NNN-brief-descriptive-title.md`. 3. Follow the format of recent ADRs (e.g. ADR-070, ADR-071): diff --git a/docs/design/INDEX.md b/docs/design/INDEX.md index c702b8d..21c6e09 100644 --- a/docs/design/INDEX.md +++ b/docs/design/INDEX.md @@ -21,7 +21,7 @@ is in `../architecture/`; decisions are in `../decisions/` (ADRs). ## Contents -[v1.0.0](#v100) · [v1.1.1](#v111) · [v1.1.2](#v112) · [v1.2.0](#v120) · [v1.3.0](#v130) · [v1.3.1](#v131) · [v1.4.0](#v140) · [v1.4.1](#v141) · [v1.4.2](#v142) · [v1.5.0](#v150) · [v1.6.0](#v160) · [v2.0.0](#v200) · [v2.0.1](#v201) · [v2.0.2](#v202) · [v2.1.0](#v210) · [v2.2.0](#v220) · [v2.3.0](#v230) · [v2.3.1](#v231) · [v2.3.2](#v232) · [v2.3.3](#v233) · [v2.4.0](#v240) · [v2.5.0](#v250) · [v2.6.0](#v260) · [v2.6.1](#v261) · [v2.6.2](#v262) · [v2.6.3](#v263) · [v2.6.4](#v264) · [v2.6.5](#v265) · [v2.6.6](#v266) · [v2.6.7](#v267) · [v2.6.8](#v268) · [v2.6.9](#v269) · [v2.7.0](#v270) · [v2.7.1](#v271) · [v2.7.2](#v272) · [v2.8.0](#v280) · [v2.8.1](#v281) · [v2.9.0](#v290) · [v2.10.0](#v2100) · [v2.10.1](#v2101) · [v2.10.2](#v2102) · [v2.11.0](#v2110) · [hotfix](#hotfix) +[v1.0.0](#v100) · [v1.1.1](#v111) · [v1.1.2](#v112) · [v1.2.0](#v120) · [v1.3.0](#v130) · [v1.3.1](#v131) · [v1.4.0](#v140) · [v1.4.1](#v141) · [v1.4.2](#v142) · [v1.5.0](#v150) · [v1.6.0](#v160) · [v2.0.0](#v200) · [v2.0.1](#v201) · [v2.0.2](#v202) · [v2.1.0](#v210) · [v2.2.0](#v220) · [v2.3.0](#v230) · [v2.3.1](#v231) · [v2.3.2](#v232) · [v2.3.3](#v233) · [v2.4.0](#v240) · [v2.5.0](#v250) · [v2.6.0](#v260) · [v2.6.1](#v261) · [v2.6.2](#v262) · [v2.6.3](#v263) · [v2.6.4](#v264) · [v2.6.5](#v265) · [v2.6.6](#v266) · [v2.6.7](#v267) · [v2.6.8](#v268) · [v2.6.9](#v269) · [v2.7.0](#v270) · [v2.7.1](#v271) · [v2.7.2](#v272) · [v2.8.0](#v280) · [v2.8.1](#v281) · [v2.9.0](#v290) · [v2.10.0](#v2100) · [v2.10.1](#v2101) · [v2.10.2](#v2102) · [v2.11.0](#v2110) · [v2.12.0](#v2120) · [hotfix](#hotfix) ## v1.0.0 @@ -414,6 +414,10 @@ OIDC admin login and built-in IndieAuth server. - [2026-11-03-maintenance-release.md](v2.11.0/2026-11-03-maintenance-release.md) - Dependency upgrades (Python 3.14), foreign keys, CSRF, fetchers, debug default, shared rate limits +## v2.12.0 + +- [2026-11-03-remove-metrics.md](v2.12.0/2026-11-03-remove-metrics.md) - Metrics dashboard and `starpunk/monitoring` removed (ADR-072) + ## hotfix Not a release folder. diff --git a/docs/design/v2.12.0/2026-11-03-remove-metrics.md b/docs/design/v2.12.0/2026-11-03-remove-metrics.md new file mode 100644 index 0000000..54cce75 --- /dev/null +++ b/docs/design/v2.12.0/2026-11-03-remove-metrics.md @@ -0,0 +1,28 @@ +# v2.12.0: Remove the metrics dashboard + +Decision: [ADR-072](../../decisions/ADR-072-remove-metrics-dashboard.md). + +## Removed + +| What | Where | +|---|---| +| Metrics buffer, HTTP middleware, database timing wrapper, memory monitor, business metrics | `starpunk/monitoring/` (deleted) | +| HTTP middleware and memory monitor start-up | `starpunk/__init__.py` | +| `MonitoredConnection` wrapping, `slow_query_threshold` and `metrics_enabled` arguments | `starpunk/database/pool.py` | +| `track_feed_generated` calls and their timing code | `starpunk/feeds/atom.py`, `json_feed.py`, `rss.py` | +| `/admin/metrics-dashboard`, `/admin/metrics`, `transform_metrics_for_template`, the `metrics` block of `/admin/health` | `starpunk/routes/admin.py` | +| Dashboard template and the admin nav link | `templates/admin/metrics_dashboard.html`, `templates/admin/base.html` | +| `METRICS_ENABLED`, `METRICS_SLOW_QUERY_THRESHOLD`, `METRICS_SAMPLING_RATE`, `METRICS_BUFFER_SIZE`, `METRICS_MEMORY_INTERVAL` | `starpunk/config.py`, `.env.example` | +| `psutil` | `requirements.txt` | +| Tests | `tests/test_monitoring.py`, `tests/test_monitoring_feed_statistics.py`, `tests/test_admin_feed_statistics.py` | + +## Kept + +`/health`, `/health?detailed=true`, `/admin/health` (database, pool, +filesystem, disk), pool statistics, the feed cache and its own hit/miss +counters, and correlation IDs in logs. + +## Upgrade + +No migration. `METRICS_*` variables can be deleted from the environment; +they are ignored if left. diff --git a/docs/projectplan/INDEX.md b/docs/projectplan/INDEX.md index 5d41228..772280d 100644 --- a/docs/projectplan/INDEX.md +++ b/docs/projectplan/INDEX.md @@ -48,3 +48,4 @@ Future work and per-release plans. See `../DOCUMENTATION.md` (PROJECTPLAN). - **[v2.10.1](v2.10.1/RELEASE.md)** - WebSub hardening (Released 2026-11-03) - **[v2.10.2](v2.10.2/RELEASE.md)** - Reply context fetch fix (Released 2026-11-03) - **[v2.11.0](v2.11.0/RELEASE.md)** - Maintenance: dependency upgrades, High backlog items (Released 2026-11-03) +- **[v2.12.0](v2.12.0/RELEASE.md)** - Metrics dashboard removed (Released 2026-11-03) diff --git a/docs/projectplan/v2.12.0/RELEASE.md b/docs/projectplan/v2.12.0/RELEASE.md new file mode 100644 index 0000000..61101bf --- /dev/null +++ b/docs/projectplan/v2.12.0/RELEASE.md @@ -0,0 +1,21 @@ +# v2.12.0 + +- **Date**: 2026-11-03 +- **Status**: Released +- **Focus**: Remove the metrics dashboard + +## Removed + +- Admin metrics dashboard (`/admin/metrics-dashboard`) and `/admin/metrics` +- The `starpunk/monitoring` package: request timing, query timing, memory monitor thread, feed counters +- `METRICS_*` settings and the `psutil` dependency + +## Kept + +- `/health`, `/admin/health` (without the metrics section), logs with correlation IDs + +## Related + +- [CHANGELOG](../../../CHANGELOG.md) +- [ADR-072](../../decisions/ADR-072-remove-metrics-dashboard.md) +- [Design note](../../design/v2.12.0/2026-11-03-remove-metrics.md) diff --git a/docs/standards/development-setup.md b/docs/standards/development-setup.md index c49a324..d29cc6e 100644 --- a/docs/standards/development-setup.md +++ b/docs/standards/development-setup.md @@ -84,10 +84,9 @@ banner to every page. It sets the `starpunk_session` cookie for server. Each app process also starts background threads: the webmention worker, -the WebSub hub worker, the memory monitor and, when a crossposting target -is configured, the syndication worker. Turn off what you don't need locally -with `WEBMENTION_WORKER_ENABLED=false`, `WEBSUB_ENABLED=false` and -`METRICS_ENABLED=false`. Leave `GOTOSOCIAL_*`, `BLUESKY_*` and `THREADS_*` +the WebSub hub worker and, when a crossposting target is configured, the +syndication worker. Turn off what you don't need locally with +`WEBMENTION_WORKER_ENABLED=false` and `WEBSUB_ENABLED=false`. Leave `GOTOSOCIAL_*`, `BLUESKY_*` and `THREADS_*` empty unless you mean to post to those accounts. Production runs the same app under gunicorn (`gunicorn app:app`, see `CMD` @@ -112,7 +111,7 @@ SESSION_SECRET=test uv run pytest tests/test_micropub.py -k delete -x `ADMIN_ME=https://test.example.com`; set them too to reproduce CI exactly. - `tests/conftest.py` builds each app with `TESTING=True` and a temporary `DATA_PATH` and database, so tests never touch `./data`. Under - `TESTING=True` the background workers and memory monitor don't start and + `TESTING=True` the background workers don't start and the production OIDC check is skipped. - `tests/test_migration_race_condition.py` is slow (about 5 minutes). While iterating, add `--ignore=tests/test_migration_race_condition.py`; run the diff --git a/requirements.txt b/requirements.txt index b7650ad..5795ea1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,6 +22,3 @@ python-dotenv==1.2.* # Images Pillow==13.0.* pillow-heif==1.8.* - -# Monitoring -psutil==7.2.* diff --git a/starpunk/__init__.py b/starpunk/__init__.py index e1d59aa..57739bf 100644 --- a/starpunk/__init__.py +++ b/starpunk/__init__.py @@ -142,12 +142,6 @@ def create_app(config=None): # Initialize connection pool init_pool(app) - # Setup HTTP metrics middleware (v1.1.2 Phase 1) - if app.config.get('METRICS_ENABLED', True): - from starpunk.monitoring import setup_http_metrics - setup_http_metrics(app) - app.logger.info("HTTP metrics middleware enabled") - # Initialize feed cache (v1.1.2 Phase 3) if app.config.get('FEED_CACHE_ENABLED', True): from starpunk.feeds import configure_cache @@ -323,17 +317,6 @@ def create_app(config=None): register_error_handlers(app) - # Start memory monitor thread (v1.1.2 Phase 1) - # Per CQ5: Skip in test mode - if app.config.get('METRICS_ENABLED', True) and not app.config.get('TESTING', False): - from starpunk.monitoring import MemoryMonitor - memory_monitor = MemoryMonitor(interval=app.config.get('METRICS_MEMORY_INTERVAL', 30)) - memory_monitor.start() - app.memory_monitor = memory_monitor - app.logger.info(f"Memory monitor started (interval={memory_monitor.interval}s)") - # No teardown_appcontext stop: that ran after every request and stopped - # the monitor straight away (v2.10.0). It is a daemon thread. - # Start webmention worker thread (v1.6.0 Phase 2) # Per background-worker.md: Skip in test mode if app.config.get('WEBMENTION_WORKER_ENABLED', True) and not app.config.get('TESTING', False): @@ -474,5 +457,5 @@ def create_app(config=None): # Package version (Semantic Versioning 2.0.0) # See docs/standards/versioning-strategy.md for details -__version__ = "2.11.0" -__version_info__ = (2, 11, 0) +__version__ = "2.12.0" +__version_info__ = (2, 12, 0) diff --git a/starpunk/config.py b/starpunk/config.py index 33f6c1d..04a5efa 100644 --- a/starpunk/config.py +++ b/starpunk/config.py @@ -99,13 +99,6 @@ def load_config(app, config_override=None): # Flask MAX_CONTENT_LENGTH limits request body size (matches media.py MAX_FILE_SIZE) app.config["MAX_CONTENT_LENGTH"] = 50 * 1024 * 1024 # 50MB - # Metrics configuration (v1.1.2 Phase 1) - app.config["METRICS_ENABLED"] = os.getenv("METRICS_ENABLED", "true").lower() == "true" - app.config["METRICS_SLOW_QUERY_THRESHOLD"] = float(os.getenv("METRICS_SLOW_QUERY_THRESHOLD", "1.0")) - app.config["METRICS_SAMPLING_RATE"] = float(os.getenv("METRICS_SAMPLING_RATE", "1.0")) - app.config["METRICS_BUFFER_SIZE"] = int(os.getenv("METRICS_BUFFER_SIZE", "1000")) - app.config["METRICS_MEMORY_INTERVAL"] = int(os.getenv("METRICS_MEMORY_INTERVAL", "30")) - # Debug file configuration (v1.5.0 Phase 2) app.config["DEBUG_SAVE_FAILED_UPLOADS"] = os.getenv("DEBUG_SAVE_FAILED_UPLOADS", "false").lower() == "true" app.config["DEBUG_FILE_MAX_AGE_DAYS"] = int(os.getenv("DEBUG_FILE_MAX_AGE_DAYS", "7")) diff --git a/starpunk/database/__init__.py b/starpunk/database/__init__.py index 0eb8835..d988d21 100644 --- a/starpunk/database/__init__.py +++ b/starpunk/database/__init__.py @@ -6,7 +6,7 @@ Provides database initialization and connection pooling Per v1.1.1 Phase 1: - Connection pooling for improved performance (ADR-053) - Request-scoped connections via Flask's g object -- Pool statistics for monitoring +- Pool statistics """ from starpunk.database.init import init_db diff --git a/starpunk/database/pool.py b/starpunk/database/pool.py index 47e865e..a7888be 100644 --- a/starpunk/database/pool.py +++ b/starpunk/database/pool.py @@ -5,7 +5,7 @@ Per ADR-053 and developer Q&A Q2, CQ1: - Provides connection pooling for improved performance - Integrates with Flask's g object for request-scoped connections - Maintains same interface as get_db() for transparency -- Pool statistics available for metrics +- Pool statistics (shown in /admin/health) - Wraps connections with MonitoredConnection for timing (v1.1.2 Phase 1) Note: Migrations use direct connections (not pooled) for isolation @@ -27,7 +27,7 @@ class ConnectionPool: but this provides connection reuse and request-scoped connection management. """ - def __init__(self, db_path, pool_size=5, timeout=10.0, slow_query_threshold=1.0, metrics_enabled=True): + def __init__(self, db_path, pool_size=5, timeout=10.0): """ Initialize connection pool @@ -35,14 +35,10 @@ class ConnectionPool: db_path: Path to SQLite database file pool_size: Maximum number of connections in pool timeout: Timeout for getting connection (seconds) - slow_query_threshold: Threshold in seconds for slow query detection (v1.1.2) - metrics_enabled: Whether to enable metrics collection (v1.1.2) """ self.db_path = Path(db_path) self.pool_size = pool_size self.timeout = timeout - self.slow_query_threshold = slow_query_threshold - self.metrics_enabled = metrics_enabled self._pool = deque(maxlen=pool_size) self._lock = Lock() self._stats = { @@ -56,8 +52,6 @@ class ConnectionPool: def _create_connection(self): """ Create a new database connection - - Per CQ1: Wraps connection with MonitoredConnection if metrics enabled """ conn = sqlite3.connect( self.db_path, @@ -73,12 +67,6 @@ class ConnectionPool: conn.execute("PRAGMA foreign_keys=ON") self._stats['connections_created'] += 1 - - # Wrap with monitoring if enabled (v1.1.2 Phase 1) - if self.metrics_enabled: - from starpunk.monitoring import MonitoredConnection - return MonitoredConnection(conn, self.slow_query_threshold) - return conn def get_connection(self): @@ -171,8 +159,6 @@ def init_pool(app): """ Initialize the connection pool - Per CQ2: Passes metrics configuration from app config - Args: app: Flask application instance """ @@ -181,20 +167,8 @@ def init_pool(app): db_path = app.config['DATABASE_PATH'] pool_size = app.config.get('DB_POOL_SIZE', 5) timeout = app.config.get('DB_TIMEOUT', 10.0) - slow_query_threshold = app.config.get('METRICS_SLOW_QUERY_THRESHOLD', 1.0) - metrics_enabled = app.config.get('METRICS_ENABLED', True) - - _pool = ConnectionPool( - db_path, - pool_size, - timeout, - slow_query_threshold, - metrics_enabled - ) - app.logger.info( - f"Database connection pool initialized " - f"(size={pool_size}, metrics={'enabled' if metrics_enabled else 'disabled'})" - ) + _pool = ConnectionPool(db_path, pool_size, timeout) + app.logger.info(f"Database connection pool initialized (size={pool_size})") # Register teardown handler @app.teardown_appcontext diff --git a/starpunk/feeds/atom.py b/starpunk/feeds/atom.py index 6891f70..8d1df3e 100644 --- a/starpunk/feeds/atom.py +++ b/starpunk/feeds/atom.py @@ -18,12 +18,10 @@ Standards: # Standard library imports from datetime import datetime, timezone from typing import Optional -import time import xml.etree.ElementTree as ET # Local imports from starpunk.models import Note -from starpunk.monitoring.business import track_feed_generated # ATOM namespace @@ -127,10 +125,6 @@ def generate_atom_streaming( # Remove trailing slash from site_url for consistency site_url = site_url.rstrip("/") - # Track feed generation timing - start_time = time.time() - item_count = 0 - # Current timestamp for updated now = datetime.now(timezone.utc) @@ -159,8 +153,6 @@ def generate_atom_streaming( # Yield entries (newest first) # Notes from database are already in DESC order (newest first) for note in notes[:limit]: - item_count += 1 - # Build permalink URL permalink = f"{site_url}{note.permalink}" @@ -229,15 +221,6 @@ def generate_atom_streaming( # Yield closing tag yield '\n' - # Track feed generation metrics - duration_ms = (time.time() - start_time) * 1000 - track_feed_generated( - format='atom', - item_count=item_count, - duration_ms=duration_ms, - cached=False - ) - def _escape_xml(text: str) -> str: """ diff --git a/starpunk/feeds/json_feed.py b/starpunk/feeds/json_feed.py index 8aea8e5..44c2ab9 100644 --- a/starpunk/feeds/json_feed.py +++ b/starpunk/feeds/json_feed.py @@ -18,12 +18,10 @@ Standards: # Standard library imports from datetime import datetime, timezone from typing import Optional, Dict, Any -import time import json # Local imports from starpunk.models import Note -from starpunk.monitoring.business import track_feed_generated def generate_json_feed( @@ -74,9 +72,6 @@ def generate_json_feed( # Remove trailing slash from site_url for consistency site_url = site_url.rstrip("/") - # Track feed generation timing - start_time = time.time() - # Build feed object feed = _build_feed_object( site_url=site_url, @@ -88,14 +83,6 @@ def generate_json_feed( # Serialize to JSON (pretty-printed) feed_json = json.dumps(feed, ensure_ascii=False, indent=2) - # Track feed generation metrics - duration_ms = (time.time() - start_time) * 1000 - track_feed_generated( - format='json', - item_count=min(len(notes), limit), - duration_ms=duration_ms, - cached=False - ) return feed_json @@ -147,10 +134,6 @@ def generate_json_feed_streaming( # Remove trailing slash from site_url for consistency site_url = site_url.rstrip("/") - # Track feed generation timing - start_time = time.time() - item_count = 0 - # Start feed object yield '{\n' yield f' "version": "https://jsonfeed.org/version/1.1",\n' @@ -170,8 +153,6 @@ def generate_json_feed_streaming( # Notes from database are already in DESC order (newest first) items = notes[:limit] for i, note in enumerate(items): - item_count += 1 - # Build item object item = _build_item_object(site_url, note) @@ -193,15 +174,6 @@ def generate_json_feed_streaming( yield ' ]\n' yield '}\n' - # Track feed generation metrics - duration_ms = (time.time() - start_time) * 1000 - track_feed_generated( - format='json', - item_count=item_count, - duration_ms=duration_ms, - cached=False - ) - def _build_feed_object( site_url: str, diff --git a/starpunk/feeds/rss.py b/starpunk/feeds/rss.py index 1778aaa..80afcb1 100644 --- a/starpunk/feeds/rss.py +++ b/starpunk/feeds/rss.py @@ -22,14 +22,12 @@ Standards: # Standard library imports from datetime import datetime, timezone from typing import Optional -import time # Third-party imports from feedgen.feed import FeedGenerator # Local imports from starpunk.models import Note -from starpunk.monitoring.business import track_feed_generated def generate_rss( @@ -98,9 +96,6 @@ def generate_rss( # Set last build date to now fg.lastBuildDate(datetime.now(timezone.utc)) - # Track feed generation timing - start_time = time.time() - # Add items (limit to configured maximum, newest first) # Notes from database are DESC but feedgen reverses them, so we reverse back for note in reversed(notes[:limit]): @@ -165,14 +160,6 @@ def generate_rss( # We need to inject media:content and media:thumbnail elements feed_xml = _inject_media_rss_elements(feed_xml, site_url, notes[:limit]) - # Track feed generation metrics - duration_ms = (time.time() - start_time) * 1000 - track_feed_generated( - format='rss', - item_count=min(len(notes), limit), - duration_ms=duration_ms, - cached=False - ) return feed_xml @@ -227,10 +214,6 @@ def generate_rss_streaming( # Remove trailing slash from site_url for consistency site_url = site_url.rstrip("/") - # Track feed generation timing - start_time = time.time() - item_count = 0 - # Current timestamp for lastBuildDate now = datetime.now(timezone.utc) last_build = format_rfc822_date(now) @@ -251,8 +234,6 @@ def generate_rss_streaming( # Yield items (newest first) # Notes from database are already in DESC order (newest first) for note in notes[:limit]: - item_count += 1 - # Build permalink URL permalink = f"{site_url}{note.permalink}" @@ -371,15 +352,6 @@ def generate_rss_streaming( yield " \n" yield "\n" - # Track feed generation metrics - duration_ms = (time.time() - start_time) * 1000 - track_feed_generated( - format='rss', - item_count=item_count, - duration_ms=duration_ms, - cached=False - ) - def _inject_media_rss_elements(feed_xml: str, site_url: str, notes: list[Note]) -> str: """ diff --git a/starpunk/monitoring/__init__.py b/starpunk/monitoring/__init__.py deleted file mode 100644 index 6dd170b..0000000 --- a/starpunk/monitoring/__init__.py +++ /dev/null @@ -1,35 +0,0 @@ -""" -Performance monitoring for StarPunk - -This package provides performance monitoring capabilities including: -- Metrics collection with circular buffers -- Operation timing (database, HTTP, rendering) -- Per-process metrics with aggregation -- Configurable sampling rates -- Database query monitoring (v1.1.2 Phase 1) -- HTTP request/response metrics (v1.1.2 Phase 1) -- Memory monitoring (v1.1.2 Phase 1) - -Per ADR-053 and developer Q&A Q6, Q12: -- Each process maintains its own circular buffer -- Buffers store recent metrics (default 1000 entries) -- Metrics include process ID for multi-process deployment -- Sampling rates are configurable per operation type -""" - -from starpunk.monitoring.metrics import MetricsBuffer, record_metric, get_metrics, get_metrics_stats -from starpunk.monitoring.database import MonitoredConnection -from starpunk.monitoring.http import setup_http_metrics -from starpunk.monitoring.memory import MemoryMonitor -from starpunk.monitoring import business - -__all__ = [ - "MetricsBuffer", - "record_metric", - "get_metrics", - "get_metrics_stats", - "MonitoredConnection", - "setup_http_metrics", - "MemoryMonitor", - "business", -] diff --git a/starpunk/monitoring/business.py b/starpunk/monitoring/business.py deleted file mode 100644 index f3d83c3..0000000 --- a/starpunk/monitoring/business.py +++ /dev/null @@ -1,298 +0,0 @@ -""" -Business metrics for StarPunk operations - -Per v1.1.2 Phase 1: -- Track note operations (create, update, delete) -- Track feed generation and cache hits/misses -- Track content statistics - -Per v1.1.2 Phase 3: -- Track feed statistics by format -- Track feed cache hit/miss rates -- Provide feed statistics dashboard - -Example usage: - >>> from starpunk.monitoring.business import track_note_created - >>> track_note_created(note_id=123, content_length=500) -""" - -from typing import Optional, Dict, Any - -from starpunk.monitoring.metrics import record_metric, get_metrics_stats - - -def track_note_created(note_id: int, content_length: int, has_media: bool = False) -> None: - """ - Track note creation event - - Args: - note_id: ID of created note - content_length: Length of note content in characters - has_media: Whether note has media attachments - """ - metadata = { - 'note_id': note_id, - 'content_length': content_length, - 'has_media': has_media, - } - - record_metric( - 'render', # Use 'render' for business metrics - 'note_created', - content_length, - metadata, - force=True # Always track business events - ) - - -def track_note_updated(note_id: int, content_length: int, fields_changed: Optional[list] = None) -> None: - """ - Track note update event - - Args: - note_id: ID of updated note - content_length: New length of note content - fields_changed: List of fields that were changed - """ - metadata = { - 'note_id': note_id, - 'content_length': content_length, - } - - if fields_changed: - metadata['fields_changed'] = ','.join(fields_changed) - - record_metric( - 'render', - 'note_updated', - content_length, - metadata, - force=True - ) - - -def track_note_deleted(note_id: int) -> None: - """ - Track note deletion event - - Args: - note_id: ID of deleted note - """ - metadata = { - 'note_id': note_id, - } - - record_metric( - 'render', - 'note_deleted', - 0, # No meaningful duration for deletion - metadata, - force=True - ) - - -def track_feed_generated(format: str, item_count: int, duration_ms: float, cached: bool = False) -> None: - """ - Track feed generation event - - Args: - format: Feed format (rss, atom, json) - item_count: Number of items in feed - duration_ms: Time taken to generate feed - cached: Whether feed was served from cache - """ - metadata = { - 'format': format, - 'item_count': item_count, - 'cached': cached, - } - - operation = f'feed_{format}{"_cached" if cached else "_generated"}' - - record_metric( - 'render', - operation, - duration_ms, - metadata, - force=True # Always track feed operations - ) - - -def track_cache_hit(cache_type: str, key: str) -> None: - """ - Track cache hit event - - Args: - cache_type: Type of cache (feed, etc.) - key: Cache key that was hit - """ - metadata = { - 'cache_type': cache_type, - 'key': key, - } - - record_metric( - 'render', - f'{cache_type}_cache_hit', - 0, - metadata, - force=True - ) - - -def track_cache_miss(cache_type: str, key: str) -> None: - """ - Track cache miss event - - Args: - cache_type: Type of cache (feed, etc.) - key: Cache key that was missed - """ - metadata = { - 'cache_type': cache_type, - 'key': key, - } - - record_metric( - 'render', - f'{cache_type}_cache_miss', - 0, - metadata, - force=True - ) - - -def get_feed_statistics() -> Dict[str, Any]: - """ - Get aggregated feed statistics from metrics buffer and feed cache. - - Analyzes metrics to provide feed-specific statistics including: - - Total requests by format (RSS, ATOM, JSON) - - Cache hit/miss rates by format - - Feed generation times by format - - Format popularity (percentage breakdown) - - Feed cache internal statistics - - Returns: - Dictionary with feed statistics: - { - 'by_format': { - 'rss': {'generated': int, 'cached': int, 'total': int, 'avg_duration_ms': float}, - 'atom': {...}, - 'json': {...} - }, - 'cache': { - 'hits': int, - 'misses': int, - 'hit_rate': float (0.0-1.0), - 'entries': int, - 'evictions': int - }, - 'total_requests': int, - 'format_percentages': { - 'rss': float, - 'atom': float, - 'json': float - } - } - - Example: - >>> stats = get_feed_statistics() - >>> print(f"RSS requests: {stats['by_format']['rss']['total']}") - >>> print(f"Cache hit rate: {stats['cache']['hit_rate']:.2%}") - """ - # Get all metrics - all_metrics = get_metrics_stats() - - # Initialize result structure - result = { - 'by_format': { - 'rss': {'generated': 0, 'cached': 0, 'total': 0, 'avg_duration_ms': 0.0}, - 'atom': {'generated': 0, 'cached': 0, 'total': 0, 'avg_duration_ms': 0.0}, - 'json': {'generated': 0, 'cached': 0, 'total': 0, 'avg_duration_ms': 0.0}, - }, - 'cache': { - 'hits': 0, - 'misses': 0, - 'hit_rate': 0.0, - }, - 'total_requests': 0, - 'format_percentages': { - 'rss': 0.0, - 'atom': 0.0, - 'json': 0.0, - }, - } - - # Get by_operation metrics if available - by_operation = all_metrics.get('by_operation', {}) - - # Count feed operations by format - for operation_name, op_stats in by_operation.items(): - # Feed operations are named: feed_rss_generated, feed_rss_cached, etc. - if operation_name.startswith('feed_'): - parts = operation_name.split('_') - if len(parts) >= 3: - format_name = parts[1] # rss, atom, or json - operation_type = parts[2] # generated or cached - - if format_name in result['by_format']: - count = op_stats.get('count', 0) - - if operation_type == 'generated': - result['by_format'][format_name]['generated'] = count - # Track average duration for generated feeds - result['by_format'][format_name]['avg_duration_ms'] = op_stats.get('avg_duration_ms', 0.0) - elif operation_type == 'cached': - result['by_format'][format_name]['cached'] = count - - # Update total for this format - result['by_format'][format_name]['total'] = ( - result['by_format'][format_name]['generated'] + - result['by_format'][format_name]['cached'] - ) - - # Track cache hits/misses - elif operation_name == 'feed_cache_hit': - result['cache']['hits'] = op_stats.get('count', 0) - elif operation_name == 'feed_cache_miss': - result['cache']['misses'] = op_stats.get('count', 0) - - # Calculate total requests across all formats - result['total_requests'] = sum( - fmt['total'] for fmt in result['by_format'].values() - ) - - # Calculate cache hit rate - total_cache_requests = result['cache']['hits'] + result['cache']['misses'] - if total_cache_requests > 0: - result['cache']['hit_rate'] = result['cache']['hits'] / total_cache_requests - - # Calculate format percentages - if result['total_requests'] > 0: - for format_name, fmt_stats in result['by_format'].items(): - result['format_percentages'][format_name] = ( - fmt_stats['total'] / result['total_requests'] - ) - - # Get feed cache statistics if available - try: - from starpunk.feeds import get_cache - feed_cache = get_cache() - cache_stats = feed_cache.get_stats() - - # Merge cache stats (prefer FeedCache internal stats over metrics) - result['cache']['entries'] = cache_stats.get('entries', 0) - result['cache']['evictions'] = cache_stats.get('evictions', 0) - - # Use FeedCache hit rate if available and more accurate - if cache_stats.get('hits', 0) + cache_stats.get('misses', 0) > 0: - result['cache']['hits'] = cache_stats.get('hits', 0) - result['cache']['misses'] = cache_stats.get('misses', 0) - result['cache']['hit_rate'] = cache_stats.get('hit_rate', 0.0) - - except ImportError: - # Feed cache not available, use defaults - pass - - return result diff --git a/starpunk/monitoring/database.py b/starpunk/monitoring/database.py deleted file mode 100644 index 89bf9f9..0000000 --- a/starpunk/monitoring/database.py +++ /dev/null @@ -1,236 +0,0 @@ -""" -Database operation monitoring wrapper - -Per ADR-053, v1.1.2 Phase 1, and developer Q&A CQ1, IQ1, IQ3: -- Wraps SQLite connections at the pool level -- Times all database operations -- Extracts query type and table name (best effort) -- Detects slow queries based on configurable threshold -- Records metrics to the metrics collector - -Example usage: - >>> from starpunk.monitoring.database import MonitoredConnection - >>> conn = sqlite3.connect(':memory:') - >>> monitored = MonitoredConnection(conn, metrics_collector) - >>> cursor = monitored.execute('SELECT * FROM notes') -""" - -import re -import sqlite3 -import time -from typing import Optional, Any, Tuple - -from starpunk.monitoring.metrics import record_metric - - -class MonitoredConnection: - """ - Wrapper for SQLite connections that monitors performance - - Per CQ1: Wraps connections at the pool level - Per IQ1: Uses simple regex for table name extraction - Per IQ3: Single configurable slow query threshold - """ - - def __init__(self, connection: sqlite3.Connection, slow_query_threshold: float = 1.0): - """ - Initialize monitored connection wrapper - - Args: - connection: SQLite connection to wrap - slow_query_threshold: Threshold in seconds for slow query detection - """ - self._connection = connection - self._slow_query_threshold = slow_query_threshold - - def execute(self, query: str, parameters: Optional[Tuple] = None) -> sqlite3.Cursor: - """ - Execute a query with performance monitoring - - Args: - query: SQL query to execute - parameters: Optional query parameters - - Returns: - sqlite3.Cursor: Query cursor - """ - start_time = time.perf_counter() - query_type = self._get_query_type(query) - table_name = self._extract_table_name(query) - - try: - if parameters: - cursor = self._connection.execute(query, parameters) - else: - cursor = self._connection.execute(query) - - duration_sec = time.perf_counter() - start_time - duration_ms = duration_sec * 1000 - - # Record metric (forced if slow query) - is_slow = duration_sec >= self._slow_query_threshold - metadata = { - 'query_type': query_type, - 'table': table_name, - 'is_slow': is_slow, - } - - # Add query text for slow queries (for debugging) - if is_slow: - # Truncate query to avoid storing huge queries - metadata['query'] = query[:200] if len(query) > 200 else query - - record_metric( - 'database', - f'{query_type} {table_name}', - duration_ms, - metadata, - force=is_slow # Always record slow queries - ) - - return cursor - - except Exception as e: - duration_sec = time.perf_counter() - start_time - duration_ms = duration_sec * 1000 - - # Record error metric - metadata = { - 'query_type': query_type, - 'table': table_name, - 'error': str(e), - 'query': query[:200] if len(query) > 200 else query - } - - record_metric( - 'database', - f'{query_type} {table_name} ERROR', - duration_ms, - metadata, - force=True # Always record errors - ) - - raise - - def executemany(self, query: str, parameters) -> sqlite3.Cursor: - """ - Execute a query with multiple parameter sets - - Args: - query: SQL query to execute - parameters: Sequence of parameter tuples - - Returns: - sqlite3.Cursor: Query cursor - """ - start_time = time.perf_counter() - query_type = self._get_query_type(query) - table_name = self._extract_table_name(query) - - try: - cursor = self._connection.executemany(query, parameters) - duration_ms = (time.perf_counter() - start_time) * 1000 - - # Record metric - metadata = { - 'query_type': query_type, - 'table': table_name, - 'batch': True, - } - - record_metric( - 'database', - f'{query_type} {table_name} BATCH', - duration_ms, - metadata - ) - - return cursor - - except Exception as e: - duration_ms = (time.perf_counter() - start_time) * 1000 - - metadata = { - 'query_type': query_type, - 'table': table_name, - 'error': str(e), - 'batch': True - } - - record_metric( - 'database', - f'{query_type} {table_name} BATCH ERROR', - duration_ms, - metadata, - force=True - ) - - raise - - def _get_query_type(self, query: str) -> str: - """ - Extract query type from SQL statement - - Args: - query: SQL query - - Returns: - Query type (SELECT, INSERT, UPDATE, DELETE, etc.) - """ - query_upper = query.strip().upper() - - for query_type in ['SELECT', 'INSERT', 'UPDATE', 'DELETE', 'CREATE', 'DROP', 'ALTER', 'PRAGMA']: - if query_upper.startswith(query_type): - return query_type - - return 'OTHER' - - def _extract_table_name(self, query: str) -> str: - """ - Extract table name from query (best effort) - - Per IQ1: Keep it simple with basic regex patterns. - Returns "unknown" for complex queries. - - Note: Complex queries (JOINs, subqueries, CTEs) return "unknown". - This covers 90% of queries accurately. - - Args: - query: SQL query - - Returns: - Table name or "unknown" - """ - query_lower = query.lower().strip() - - # Simple patterns that cover 90% of cases - patterns = [ - r'from\s+(\w+)', - r'update\s+(\w+)', - r'insert\s+into\s+(\w+)', - r'delete\s+from\s+(\w+)', - r'create\s+table\s+(?:if\s+not\s+exists\s+)?(\w+)', - r'drop\s+table\s+(?:if\s+exists\s+)?(\w+)', - r'alter\s+table\s+(\w+)', - ] - - for pattern in patterns: - match = re.search(pattern, query_lower) - if match: - return match.group(1) - - # Complex queries (JOINs, subqueries, CTEs) - return "unknown" - - # Delegate all other connection methods to the wrapped connection - def __getattr__(self, name: str) -> Any: - """Delegate all other methods to the wrapped connection""" - return getattr(self._connection, name) - - def __enter__(self): - """Support context manager protocol""" - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - """Support context manager protocol""" - return self._connection.__exit__(exc_type, exc_val, exc_tb) diff --git a/starpunk/monitoring/http.py b/starpunk/monitoring/http.py deleted file mode 100644 index fb15a73..0000000 --- a/starpunk/monitoring/http.py +++ /dev/null @@ -1,133 +0,0 @@ -""" -HTTP request/response metrics middleware - -Per v1.1.2 Phase 1 and developer Q&A IQ2: -- Times all HTTP requests -- Generates request IDs for tracking (IQ2) -- Records status codes, methods, routes -- Tracks request and response sizes -- Adds X-Request-ID header to all responses (not just debug mode) - -Example usage: - >>> from starpunk.monitoring.http import setup_http_metrics - >>> app = Flask(__name__) - >>> setup_http_metrics(app) -""" - -import time -import uuid -from flask import g, request, Flask -from typing import Any - -from starpunk.monitoring.metrics import record_metric - - -def setup_http_metrics(app: Flask) -> None: - """ - Setup HTTP metrics collection for Flask app - - Per IQ2: Generates request IDs and adds X-Request-ID header in all modes - - Args: - app: Flask application instance - """ - - @app.before_request - def start_request_metrics(): - """ - Initialize request metrics tracking - - Per IQ2: Generate UUID request ID and store in g - """ - # Generate request ID (IQ2: in all modes, not just debug) - g.request_id = str(uuid.uuid4()) - - # Store request start time and metadata - g.request_start_time = time.perf_counter() - g.request_metadata = { - 'method': request.method, - 'endpoint': request.endpoint or 'unknown', - 'path': request.path, - 'content_length': request.content_length or 0, - } - - @app.after_request - def record_response_metrics(response): - """ - Record HTTP response metrics - - Args: - response: Flask response object - - Returns: - Modified response with X-Request-ID header - """ - # Skip if metrics not initialized (shouldn't happen in normal flow) - if not hasattr(g, 'request_start_time'): - return response - - # Calculate request duration - duration_sec = time.perf_counter() - g.request_start_time - duration_ms = duration_sec * 1000 - - # Get response size - response_size = 0 - - # Check if response is in direct passthrough mode (streaming) - if hasattr(response, 'direct_passthrough') and response.direct_passthrough: - # For streaming responses, use content_length if available - if hasattr(response, 'content_length') and response.content_length: - response_size = response.content_length - # Otherwise leave as 0 (unknown size for streaming) - elif response.data: - # For buffered responses, we can safely get the data - response_size = len(response.data) - elif hasattr(response, 'content_length') and response.content_length: - response_size = response.content_length - - # Build metadata - metadata = { - **g.request_metadata, - 'status_code': response.status_code, - 'response_size': response_size, - } - - # Record metric - operation_name = f"{g.request_metadata['method']} {g.request_metadata['endpoint']}" - record_metric( - 'http', - operation_name, - duration_ms, - metadata - ) - - # Add request ID header (IQ2: in all modes) - response.headers['X-Request-ID'] = g.request_id - - return response - - @app.teardown_request - def record_error_metrics(error=None): - """ - Record metrics for requests that result in errors - - Args: - error: Exception if request failed - """ - if error and hasattr(g, 'request_start_time'): - duration_ms = (time.perf_counter() - g.request_start_time) * 1000 - - metadata = { - **g.request_metadata, - 'error': str(error), - 'error_type': type(error).__name__, - } - - operation_name = f"{g.request_metadata['method']} {g.request_metadata['endpoint']} ERROR" - record_metric( - 'http', - operation_name, - duration_ms, - metadata, - force=True # Always record errors - ) diff --git a/starpunk/monitoring/memory.py b/starpunk/monitoring/memory.py deleted file mode 100644 index 9a541c5..0000000 --- a/starpunk/monitoring/memory.py +++ /dev/null @@ -1,191 +0,0 @@ -""" -Memory monitoring background thread - -Per v1.1.2 Phase 1 and developer Q&A CQ5, IQ8: -- Background daemon thread for continuous memory monitoring -- Tracks RSS and VMS memory usage -- Detects memory growth and potential leaks -- 5-second baseline period after startup (IQ8) -- Skipped in test mode (CQ5) - -Example usage: - >>> from starpunk.monitoring.memory import MemoryMonitor - >>> monitor = MemoryMonitor(interval=30) - >>> monitor.start() # Runs as daemon thread - >>> # ... application runs ... - >>> monitor.stop() -""" - -import gc -import logging -import os -import sys -import threading -import time -from typing import Dict, Any - -import psutil - -from starpunk.monitoring.metrics import record_metric - - -logger = logging.getLogger(__name__) - - -class MemoryMonitor(threading.Thread): - """ - Background thread for memory monitoring - - Per CQ5: Daemon thread that auto-terminates with main process - Per IQ8: 5-second baseline period after startup - """ - - def __init__(self, interval: int = 30): - """ - Initialize memory monitor thread - - Args: - interval: Monitoring interval in seconds (default: 30) - """ - super().__init__(daemon=True) # CQ5: daemon thread - self.interval = interval - self._stop_event = threading.Event() - self._process = psutil.Process() - self._baseline_memory = None - self._high_water_mark = 0 - - def run(self): - """ - Main monitoring loop - - Per IQ8: Wait 5 seconds for app initialization before setting baseline - """ - try: - # Wait for app initialization (IQ8: 5 seconds) - time.sleep(5) - - # Set baseline memory - memory_info = self._get_memory_info() - self._baseline_memory = memory_info['rss_mb'] - logger.info(f"Memory monitor baseline set: {self._baseline_memory:.2f} MB RSS") - - # Start monitoring loop - while not self._stop_event.is_set(): - try: - self._collect_metrics() - except Exception as e: - logger.error(f"Memory monitoring error: {e}", exc_info=True) - - # Wait for interval or until stop event - self._stop_event.wait(self.interval) - - except Exception as e: - logger.error(f"Memory monitor thread failed: {e}", exc_info=True) - - def _collect_metrics(self): - """Collect and record memory metrics""" - memory_info = self._get_memory_info() - gc_stats = self._get_gc_stats() - - # Update high water mark - if memory_info['rss_mb'] > self._high_water_mark: - self._high_water_mark = memory_info['rss_mb'] - - # Calculate growth rate (MB/hour) if baseline is set - growth_rate = 0.0 - if self._baseline_memory: - growth_rate = memory_info['rss_mb'] - self._baseline_memory - - # Record metrics - metadata = { - 'rss_mb': memory_info['rss_mb'], - 'vms_mb': memory_info['vms_mb'], - 'percent': memory_info['percent'], - 'high_water_mb': self._high_water_mark, - 'growth_mb': growth_rate, - 'gc_collections': gc_stats['collections'], - 'gc_collected': gc_stats['collected'], - } - - record_metric( - 'render', # Use 'render' operation type for memory metrics - 'memory_usage', - memory_info['rss_mb'], - metadata, - force=True # Always record memory metrics - ) - - # Warn if significant growth detected (>10MB growth from baseline) - if growth_rate > 10.0: - logger.warning( - f"Memory growth detected: +{growth_rate:.2f} MB from baseline " - f"(current: {memory_info['rss_mb']:.2f} MB, baseline: {self._baseline_memory:.2f} MB)" - ) - - def _get_memory_info(self) -> Dict[str, float]: - """ - Get current process memory usage - - Returns: - Dict with memory info in MB - """ - memory = self._process.memory_info() - - return { - 'rss_mb': memory.rss / (1024 * 1024), # Resident Set Size - 'vms_mb': memory.vms / (1024 * 1024), # Virtual Memory Size - 'percent': self._process.memory_percent(), - } - - def _get_gc_stats(self) -> Dict[str, Any]: - """ - Get garbage collection statistics - - Returns: - Dict with GC stats - """ - # Get collection counts per generation - counts = gc.get_count() - - # Perform a quick gen 0 collection and count collected objects - collected = gc.collect(0) - - return { - 'collections': { - 'gen0': counts[0], - 'gen1': counts[1], - 'gen2': counts[2], - }, - 'collected': collected, - 'uncollectable': len(gc.garbage), - } - - def stop(self): - """ - Stop the monitoring thread gracefully - - Sets the stop event to signal the thread to exit - """ - logger.info("Stopping memory monitor") - self._stop_event.set() - - def get_stats(self) -> Dict[str, Any]: - """ - Get current memory statistics - - Returns: - Dict with current memory stats - """ - if not self._baseline_memory: - return {'status': 'initializing'} - - memory_info = self._get_memory_info() - - return { - 'status': 'running', - 'current_rss_mb': memory_info['rss_mb'], - 'baseline_rss_mb': self._baseline_memory, - 'growth_mb': memory_info['rss_mb'] - self._baseline_memory, - 'high_water_mb': self._high_water_mark, - 'percent': memory_info['percent'], - } diff --git a/starpunk/monitoring/metrics.py b/starpunk/monitoring/metrics.py deleted file mode 100644 index 7351d9c..0000000 --- a/starpunk/monitoring/metrics.py +++ /dev/null @@ -1,425 +0,0 @@ -""" -Metrics collection and buffering for performance monitoring - -Per ADR-053 and developer Q&A Q6, Q12: -- Per-process circular buffers using deque -- Configurable buffer size (default 1000 entries) -- Include process ID in all metrics -- Configuration-based sampling rates -- Operation types: database, http, render - -Example usage: - >>> from starpunk.monitoring import record_metric, get_metrics - >>> - >>> # Record a database operation - >>> record_metric('database', 'query', duration_ms=45.2, query='SELECT * FROM notes') - >>> - >>> # Get all metrics - >>> metrics = get_metrics() - >>> print(f"Collected {len(metrics)} metrics") -""" - -import os -import random -import time -from collections import deque -from dataclasses import dataclass, field, asdict -from datetime import datetime, timezone -from threading import Lock -from typing import Any, Deque, Dict, List, Literal, Optional, Union - -# Operation types for categorizing metrics -OperationType = Literal["database", "http", "render"] - -# Module-level circular buffer (per-process) -# Each process in a multi-process deployment maintains its own buffer -_metrics_buffer: Optional["MetricsBuffer"] = None -_buffer_lock = Lock() - - -@dataclass -class Metric: - """ - Represents a single performance metric - - Attributes: - operation_type: Type of operation (database/http/render) - operation_name: Name/description of operation - timestamp: When the metric was recorded (ISO format) - duration_ms: Duration in milliseconds - process_id: Process ID that recorded the metric - metadata: Additional operation-specific data - """ - operation_type: OperationType - operation_name: str - timestamp: str - duration_ms: float - process_id: int - metadata: Dict[str, Any] = field(default_factory=dict) - - def to_dict(self) -> Dict[str, Any]: - """Convert metric to dictionary for serialization""" - return asdict(self) - - -class MetricsBuffer: - """ - Circular buffer for storing performance metrics - - Per developer Q&A Q6: - - Uses deque for efficient circular buffer - - Per-process storage (not shared across workers) - - Thread-safe with locking - - Configurable max size (default 1000) - - Automatic eviction of oldest entries when full - - Per developer Q&A Q12: - - Configurable sampling rates per operation type - - Default 100% sampling (suitable for low-traffic sites) - - Slow queries always logged regardless of sampling - - Example: - >>> buffer = MetricsBuffer(max_size=1000) - >>> buffer.record('database', 'query', 45.2, {'query': 'SELECT ...'}) - >>> metrics = buffer.get_all() - """ - - def __init__( - self, - max_size: int = 1000, - sampling_rates: Optional[Union[Dict[OperationType, float], float]] = None - ): - """ - Initialize metrics buffer - - Args: - max_size: Maximum number of metrics to store - sampling_rates: Either: - - float: Global sampling rate for all operation types (0.0-1.0) - - dict: Mapping operation type to sampling rate - Default: 1.0 (100% sampling) - """ - self.max_size = max_size - self._buffer: Deque[Metric] = deque(maxlen=max_size) - self._lock = Lock() - self._process_id = os.getpid() - - # Handle different sampling_rates types - if sampling_rates is None: - # Default to 100% sampling for all types - self._sampling_rates = { - "database": 1.0, - "http": 1.0, - "render": 1.0, - } - elif isinstance(sampling_rates, (int, float)): - # Global rate for all types - rate = float(sampling_rates) - self._sampling_rates = { - "database": rate, - "http": rate, - "render": rate, - } - else: - # Dict with per-type rates - self._sampling_rates = sampling_rates - - def record( - self, - operation_type: OperationType, - operation_name: str, - duration_ms: float, - metadata: Optional[Dict[str, Any]] = None, - force: bool = False - ) -> bool: - """ - Record a performance metric - - Args: - operation_type: Type of operation (database/http/render) - operation_name: Name/description of operation - duration_ms: Duration in milliseconds - metadata: Additional operation-specific data - force: If True, bypass sampling (for slow query logging) - - Returns: - True if metric was recorded, False if skipped due to sampling - - Example: - >>> buffer.record('database', 'SELECT notes', 45.2, - ... {'query': 'SELECT * FROM notes LIMIT 10'}) - True - """ - # Apply sampling (unless forced) - if not force: - sampling_rate = self._sampling_rates.get(operation_type, 0.1) - if random.random() > sampling_rate: - return False - - metric = Metric( - operation_type=operation_type, - operation_name=operation_name, - timestamp=datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + "Z", - duration_ms=duration_ms, - process_id=self._process_id, - metadata=metadata or {} - ) - - with self._lock: - self._buffer.append(metric) - - return True - - def get_all(self) -> List[Metric]: - """ - Get all metrics from buffer - - Returns: - List of metrics (oldest to newest) - - Example: - >>> metrics = buffer.get_all() - >>> len(metrics) - 1000 - """ - with self._lock: - return list(self._buffer) - - def get_recent(self, count: int) -> List[Metric]: - """ - Get most recent N metrics - - Args: - count: Number of recent metrics to return - - Returns: - List of most recent metrics (newest first) - - Example: - >>> recent = buffer.get_recent(10) - >>> len(recent) - 10 - """ - with self._lock: - # Convert to list, reverse to get newest first, then slice - all_metrics = list(self._buffer) - all_metrics.reverse() - return all_metrics[:count] - - def get_by_type(self, operation_type: OperationType) -> List[Metric]: - """ - Get all metrics of a specific type - - Args: - operation_type: Type to filter by (database/http/render) - - Returns: - List of metrics matching the type - - Example: - >>> db_metrics = buffer.get_by_type('database') - """ - with self._lock: - return [m for m in self._buffer if m.operation_type == operation_type] - - def get_slow_operations( - self, - threshold_ms: float = 1000.0, - operation_type: Optional[OperationType] = None - ) -> List[Metric]: - """ - Get operations that exceeded a duration threshold - - Args: - threshold_ms: Duration threshold in milliseconds - operation_type: Optional type filter - - Returns: - List of slow operations - - Example: - >>> slow_queries = buffer.get_slow_operations(1000, 'database') - """ - with self._lock: - metrics = list(self._buffer) - - # Filter by type if specified - if operation_type: - metrics = [m for m in metrics if m.operation_type == operation_type] - - # Filter by duration threshold - return [m for m in metrics if m.duration_ms >= threshold_ms] - - def get_stats(self) -> Dict[str, Any]: - """ - Get statistics about the buffer - - Returns: - Dict with buffer statistics - - Example: - >>> stats = buffer.get_stats() - >>> stats['total_count'] - 1000 - """ - with self._lock: - metrics = list(self._buffer) - - # Calculate stats per operation type - type_stats = {} - for op_type in ["database", "http", "render"]: - type_metrics = [m for m in metrics if m.operation_type == op_type] - if type_metrics: - durations = [m.duration_ms for m in type_metrics] - type_stats[op_type] = { - "count": len(type_metrics), - "avg_duration_ms": sum(durations) / len(durations), - "min_duration_ms": min(durations), - "max_duration_ms": max(durations), - } - else: - type_stats[op_type] = { - "count": 0, - "avg_duration_ms": 0.0, - "min_duration_ms": 0.0, - "max_duration_ms": 0.0, - } - - return { - "total_count": len(metrics), - "max_size": self.max_size, - "process_id": self._process_id, - "sampling_rates": self._sampling_rates, - "by_type": type_stats, - } - - def clear(self) -> None: - """ - Clear all metrics from buffer - - Example: - >>> buffer.clear() - """ - with self._lock: - self._buffer.clear() - - def set_sampling_rate( - self, - operation_type: OperationType, - rate: float - ) -> None: - """ - Update sampling rate for an operation type - - Args: - operation_type: Type to update - rate: New sampling rate (0.0-1.0) - - Example: - >>> buffer.set_sampling_rate('database', 0.5) # 50% sampling - """ - if not 0.0 <= rate <= 1.0: - raise ValueError("Sampling rate must be between 0.0 and 1.0") - - with self._lock: - self._sampling_rates[operation_type] = rate - - -def get_buffer() -> MetricsBuffer: - """ - Get or create the module-level metrics buffer - - This ensures a single buffer per process. In multi-process deployments - (e.g., gunicorn), each worker process will have its own buffer. - - Returns: - MetricsBuffer instance for this process - - Example: - >>> buffer = get_buffer() - >>> buffer.record('database', 'query', 45.2) - """ - global _metrics_buffer - - if _metrics_buffer is None: - with _buffer_lock: - # Double-check locking pattern - if _metrics_buffer is None: - # Get configuration from Flask app if available - try: - from flask import current_app - max_size = current_app.config.get('METRICS_BUFFER_SIZE', 1000) - sampling_rate = current_app.config.get('METRICS_SAMPLING_RATE', 1.0) - except (ImportError, RuntimeError): - # Flask not available or no app context - max_size = 1000 - sampling_rate = 1.0 # Default to 100% - - _metrics_buffer = MetricsBuffer( - max_size=max_size, - sampling_rates=sampling_rate - ) - - return _metrics_buffer - - -def record_metric( - operation_type: OperationType, - operation_name: str, - duration_ms: float, - metadata: Optional[Dict[str, Any]] = None, - force: bool = False -) -> bool: - """ - Record a metric using the module-level buffer - - Convenience function that uses get_buffer() internally. - - Args: - operation_type: Type of operation (database/http/render) - operation_name: Name/description of operation - duration_ms: Duration in milliseconds - metadata: Additional operation-specific data - force: If True, bypass sampling (for slow query logging) - - Returns: - True if metric was recorded, False if skipped due to sampling - - Example: - >>> record_metric('database', 'SELECT notes', 45.2, - ... {'query': 'SELECT * FROM notes LIMIT 10'}) - True - """ - buffer = get_buffer() - return buffer.record(operation_type, operation_name, duration_ms, metadata, force) - - -def get_metrics() -> List[Metric]: - """ - Get all metrics from the module-level buffer - - Returns: - List of metrics (oldest to newest) - - Example: - >>> metrics = get_metrics() - >>> len(metrics) - 1000 - """ - buffer = get_buffer() - return buffer.get_all() - - -def get_metrics_stats() -> Dict[str, Any]: - """ - Get statistics from the module-level buffer - - Returns: - Dict with buffer statistics - - Example: - >>> stats = get_metrics_stats() - >>> print(f"Total metrics: {stats['total_count']}") - """ - buffer = get_buffer() - return buffer.get_stats() diff --git a/starpunk/routes/admin.py b/starpunk/routes/admin.py index 739d0ad..7201073 100644 --- a/starpunk/routes/admin.py +++ b/starpunk/routes/admin.py @@ -9,7 +9,6 @@ from flask import Blueprint, flash, g, jsonify, redirect, render_template, reque import logging import os import shutil -from datetime import datetime, timezone logger = logging.getLogger(__name__) @@ -366,202 +365,6 @@ def delete_note_submit(note_id: int): return redirect(url_for("admin.dashboard")) -def transform_metrics_for_template(metrics_stats): - """ - Transform metrics stats to match template structure - - The template expects direct access to metrics.database.count, but - get_metrics_stats() returns metrics.by_type.database.count. - This function adapts the data structure to match template expectations. - - Args: - metrics_stats: Dict from get_metrics_stats() with nested by_type structure - - Returns: - Dict with flattened structure matching template expectations - - Per ADR-060: Route Adapter Pattern for template compatibility - """ - transformed = {} - - # Map by_type to direct access - for op_type in ['database', 'http', 'render']: - if 'by_type' in metrics_stats and op_type in metrics_stats['by_type']: - type_data = metrics_stats['by_type'][op_type] - transformed[op_type] = { - 'count': type_data.get('count', 0), - 'avg': type_data.get('avg_duration_ms', 0), - 'min': type_data.get('min_duration_ms', 0), - 'max': type_data.get('max_duration_ms', 0) - } - else: - # Provide defaults for missing types or when by_type doesn't exist - transformed[op_type] = { - 'count': 0, - 'avg': 0, - 'min': 0, - 'max': 0 - } - - # Keep other top-level stats - transformed['total_count'] = metrics_stats.get('total_count', 0) - transformed['max_size'] = metrics_stats.get('max_size', 1000) - transformed['process_id'] = metrics_stats.get('process_id', 0) - - return transformed - - -@bp.route("/metrics-dashboard") -@require_auth -def metrics_dashboard(): - """ - Metrics visualization dashboard (Phase 3) - - Displays performance metrics, database statistics, feed statistics, - and system health with visual charts and auto-refresh capability. - - Per Q19 requirements: - - Server-side rendering with Jinja2 - - htmx for auto-refresh - - Chart.js from CDN for graphs - - Progressive enhancement (works without JS) - - Per v1.1.2 Phase 3: - - Feed statistics by format - - Cache hit/miss rates - - Format popularity breakdown - - Returns: - Rendered dashboard template with metrics - - Decorator: @require_auth - Template: templates/admin/metrics_dashboard.html - """ - # Defensive imports with graceful degradation for missing modules - try: - from starpunk.database.pool import get_pool_stats - from starpunk.monitoring import get_metrics_stats - from starpunk.monitoring.business import get_feed_statistics - monitoring_available = True - except ImportError: - monitoring_available = False - # Provide fallback functions that return error messages - def get_pool_stats(): - return {"error": "Database pool monitoring not available"} - def get_metrics_stats(): - return {"error": "Monitoring module not implemented"} - def get_feed_statistics(): - return {"error": "Feed statistics not available"} - - # Get current metrics for initial page load - metrics_data = {} - pool_stats = {} - feed_stats = {} - - try: - raw_metrics = get_metrics_stats() - metrics_data = transform_metrics_for_template(raw_metrics) - except Exception as e: - flash(f"Error loading metrics: {e}", "warning") - # Provide safe defaults matching template expectations - metrics_data = { - 'database': {'count': 0, 'avg': 0, 'min': 0, 'max': 0}, - 'http': {'count': 0, 'avg': 0, 'min': 0, 'max': 0}, - 'render': {'count': 0, 'avg': 0, 'min': 0, 'max': 0}, - 'total_count': 0, - 'max_size': 1000, - 'process_id': 0 - } - - try: - pool_stats = get_pool_stats() - except Exception as e: - flash(f"Error loading pool stats: {e}", "warning") - - try: - feed_stats = get_feed_statistics() - except Exception as e: - flash(f"Error loading feed stats: {e}", "warning") - # Provide safe defaults - feed_stats = { - 'by_format': { - 'rss': {'generated': 0, 'cached': 0, 'total': 0, 'avg_duration_ms': 0.0}, - 'atom': {'generated': 0, 'cached': 0, 'total': 0, 'avg_duration_ms': 0.0}, - 'json': {'generated': 0, 'cached': 0, 'total': 0, 'avg_duration_ms': 0.0}, - }, - 'cache': {'hits': 0, 'misses': 0, 'hit_rate': 0.0, 'entries': 0, 'evictions': 0}, - 'total_requests': 0, - 'format_percentages': {'rss': 0.0, 'atom': 0.0, 'json': 0.0}, - } - - return render_template( - "admin/metrics_dashboard.html", - metrics=metrics_data, - pool=pool_stats, - feeds=feed_stats, - user_me=g.me - ) - - -@bp.route("/metrics") -@require_auth -def metrics(): - """ - Performance metrics and database pool statistics endpoint - - Per Phase 2 requirements: - - Expose database pool statistics - - Show performance metrics from MetricsBuffer - - Requires authentication - - Per v1.1.2 Phase 3: - - Include feed statistics - - Returns: - JSON with metrics, pool statistics, and feed statistics - - Response codes: - 200: Metrics retrieved successfully - - Decorator: @require_auth - """ - from flask import current_app - from starpunk.database.pool import get_pool_stats - from starpunk.monitoring import get_metrics_stats - from starpunk.monitoring.business import get_feed_statistics - - response = { - "timestamp": datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + "Z", - "process_id": os.getpid(), - "database": {}, - "performance": {}, - "feeds": {} - } - - # Get database pool statistics - try: - pool_stats = get_pool_stats() - response["database"]["pool"] = pool_stats - except Exception as e: - response["database"]["pool"] = {"error": str(e)} - - # Get performance metrics - try: - metrics_stats = get_metrics_stats() - response["performance"] = metrics_stats - except Exception as e: - response["performance"] = {"error": str(e)} - - # Get feed statistics - try: - feed_stats = get_feed_statistics() - response["feeds"] = feed_stats - except Exception as e: - response["feeds"] = {"error": str(e)} - - return jsonify(response), 200 - - @bp.route("/health") @require_auth def health_diagnostics(): @@ -571,7 +374,7 @@ def health_diagnostics(): Per developer Q&A Q10: - Always requires authentication - Provides comprehensive diagnostics - - Includes metrics, database pool statistics, and system info + - Includes database pool statistics and system info Returns: JSON with complete system diagnostics @@ -591,7 +394,6 @@ def health_diagnostics(): "environment": current_app.config.get("ENV", "unknown"), "process_id": os.getpid(), "checks": {}, - "metrics": {}, "database": {} } @@ -665,14 +467,6 @@ def health_diagnostics(): } overall_healthy = False - # Performance metrics - try: - from starpunk.monitoring import get_metrics_stats - metrics_stats = get_metrics_stats() - diagnostics["metrics"] = metrics_stats - except Exception as e: - diagnostics["metrics"] = {"error": str(e)} - # Update overall status diagnostics["status"] = "healthy" if overall_healthy else "unhealthy" diff --git a/templates/admin/base.html b/templates/admin/base.html index b054bdd..fed5482 100644 --- a/templates/admin/base.html +++ b/templates/admin/base.html @@ -8,7 +8,6 @@