Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
76b81d6ad9 | ||
|
|
44f77a3adf | ||
|
|
b247de011d | ||
|
|
65fc246d55 | ||
|
|
f3ed1b3488 | ||
|
|
d400e4d838 | ||
|
|
55ce6a1fdd | ||
|
|
6aaac982d5 | ||
|
|
2654692d15 | ||
|
|
f8a429cdc1 | ||
|
|
afef1fc9a5 | ||
|
|
b450c0c648 | ||
|
|
13936749de | ||
|
|
742e2e3ec3 | ||
|
|
d6a916ba7d | ||
|
|
e654dd84de | ||
|
|
abb75f080e | ||
|
|
d5157a6059 | ||
|
|
393228002b | ||
|
|
874465e103 |
No files matched your search
+12
-14
@@ -165,7 +165,8 @@ WEBSUB_ENABLED=true
|
||||
|
||||
# GoToSocial or any Mastodon-API server (v2.1.0, ADR-065). Create an app
|
||||
# token with scopes: write:statuses write:media, plus read:search
|
||||
# write:favourites for native likes and boosts (v2.5.0)
|
||||
# write:favourites for native likes and boosts (v2.5.0), plus read:statuses
|
||||
# read:accounts (or read) for backfeed (v2.13.0)
|
||||
GOTOSOCIAL_URL=
|
||||
GOTOSOCIAL_ACCESS_TOKEN=
|
||||
# Name shown in the editor and Micropub apps (default: GoToSocial)
|
||||
@@ -185,7 +186,8 @@ BLUESKY_APP_PASSWORD=
|
||||
# BLUESKY_MAX_CHARS=300
|
||||
|
||||
# Threads (v2.6.0, ADR-069). A long-lived user token with threads_basic,
|
||||
# threads_content_publish and threads_delete; StarPunk refreshes it daily and
|
||||
# threads_content_publish and threads_delete (plus threads_read_replies and
|
||||
# threads_manage_insights for backfeed, v2.13.0); StarPunk refreshes it daily and
|
||||
# keeps the current one in DATA_PATH/threads_token.json.
|
||||
# How to get one: docs/design/v2.6.0/2026-10-31-threads-crossposting.md
|
||||
THREADS_ACCESS_TOKEN=
|
||||
@@ -198,19 +200,15 @@ THREADS_USERNAME=
|
||||
# Seconds between syndication worker passes (default: 10)
|
||||
# SYNDICATION_POLL_INTERVAL=10
|
||||
|
||||
# =============================================================================
|
||||
# MONITORING
|
||||
# =============================================================================
|
||||
# Backfeed (v2.13.0, ADR-073): bring likes, reposts, quotes and replies to the
|
||||
# crossposts back to the note as responses. Needs the read scopes above.
|
||||
# BACKFEED_ENABLED=false
|
||||
# Stop checking a note's responses after this many days (default: 90)
|
||||
# BACKFEED_MAX_AGE_DAYS=90
|
||||
|
||||
# Metrics shown on /admin/metrics-dashboard (default: true)
|
||||
# METRICS_ENABLED=true
|
||||
# Queries slower than this many seconds are logged (default: 1.0)
|
||||
# METRICS_SLOW_QUERY_THRESHOLD=1.0
|
||||
# Fraction of operations recorded, 0.0-1.0 (default: 1.0)
|
||||
# METRICS_SAMPLING_RATE=1.0
|
||||
# METRICS_BUFFER_SIZE=1000
|
||||
# Seconds between memory samples (default: 30)
|
||||
# METRICS_MEMORY_INTERVAL=30
|
||||
# =============================================================================
|
||||
# DEBUGGING
|
||||
# =============================================================================
|
||||
|
||||
# Keep uploads that fail processing in DATA_PATH/debug for diagnosis
|
||||
# (default: false); cleaned up by age (default: 7 days) and total size
|
||||
|
||||
@@ -7,6 +7,62 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [2.13.2] - 2026-11-10
|
||||
|
||||
### Changed
|
||||
|
||||
- Likes, reposts and bookmarks are no longer rows of faces under "Responses". A note shows one line of counts under the post (♥ likes, 🔁 reposts, 💬 replies, bookmarks), and hovering or tapping a count lists who, with where each came from. Threads' anonymous counts are part of the totals ("+1 on Threads"), so the separate "On Threads" line is gone. The Responses section now holds replies, quotes and mentions
|
||||
- Web actions (https://indieweb.org/webactions): each like, repost and reply count is an `<indie-action>` for the note, with links to do the same on the note's crossposts ("Like on Bluesky · GoToSocial · Threads"). The icons show even before anyone has responded, when the note has crossposts
|
||||
|
||||
### Fixed
|
||||
|
||||
- Avatars that GoToSocial hasn't cached (it redirects to the original server) are saved; the redirect is followed through the safe fetcher
|
||||
|
||||
## [2.13.1] - 2026-11-10
|
||||
|
||||
### Fixed
|
||||
|
||||
- Avatars from your own GoToSocial server are saved again. When StarPunk reaches it through the same proxy, its name resolves to an internal address, which the safe fetcher refused. That one server (`GOTOSOCIAL_URL`) is now fetched directly, as crossposting already does
|
||||
- Bluesky avatars are saved again. Bluesky's image CDN refuses some servers' IPv4 addresses (HTTP 403), so avatars are now downloaded from the account's own server (`com.atproto.sync.getBlob`)
|
||||
|
||||
## [2.13.0] - 2026-11-10
|
||||
|
||||
Backfeed: responses to crossposts come back to the note (ADR-073).
|
||||
|
||||
### Added
|
||||
|
||||
- Likes, reposts, quotes and replies to a note's crossposts on GoToSocial and Bluesky, and replies on Threads, are shown on the note as responses, labelled "via Bluesky" (or GoToSocial, Threads). StarPunk checks each crosspost itself: every 15 minutes on a note's first day, hourly in its first week, daily until 30 days, weekly until `BACKFEED_MAX_AGE_DAYS` (90). Withdrawn likes and deleted replies drop off at the next check
|
||||
- Threads like, repost and quote counts, shown as a line such as "On Threads: 12 likes · 3 reposts" (the Threads API doesn't say who)
|
||||
- A Quotes group in a note's responses
|
||||
- **Backfill responses** on the admin Webmentions page checks every crosspost once, however old the note; each note's edit page shows when its responses were last checked, with **Check now**
|
||||
- The admin Webmentions list can be filtered by where a response came from, and shows deleted ones under **Deleted**
|
||||
- Settings `BACKFEED_ENABLED` (default off) and `BACKFEED_MAX_AGE_DAYS`. Backfeed needs read access: a GoToSocial token with `read:statuses` and `read:accounts` (or `read`), and a Threads token with `threads_read_replies` and `threads_manage_insights`. Bluesky app passwords already work
|
||||
|
||||
### Changed
|
||||
|
||||
- Response authors' avatars are saved on this site (96 px WebP in `DATA_PATH/avatars`) and shown from there, instead of being loaded from other sites. Until a copy is saved the initial is shown
|
||||
- Deleting a webmention in the admin keeps it as a deleted record, so a re-sent webmention or the next check doesn't bring it back
|
||||
- People with `#nobridge` or `#nobot` in their profile, and Bluesky accounts that ask not to be shown to logged-out visitors, are not backfed; only public responses are
|
||||
- A webmention for a response that was already backfed (for example a reply also sent by Bridgy) is ignored, so it isn't shown twice
|
||||
- The Threads client uses `graph.threads.com`, the host in Meta's current documentation
|
||||
|
||||
## [2.12.1] - 2026-11-10
|
||||
|
||||
### Fixed
|
||||
|
||||
- Micropub `q=category` (the tag list apps use for autocomplete) only lists tags on posts that still exist. It used to include tags that an edit had removed from every post, and tags only used on deleted posts. Drafts still count; a deleted post's tags come back if it is undeleted
|
||||
|
||||
## [2.12.0] - 2026-11-03
|
||||
|
||||
The metrics dashboard is gone (ADR-072).
|
||||
|
||||
### Removed
|
||||
|
||||
- The admin metrics dashboard (`/admin/metrics-dashboard`, the "Metrics" admin link) and its JSON endpoint `/admin/metrics`. Its numbers were kept in memory per server process, so they covered a quarter of the traffic and vanished on every restart
|
||||
- Everything that only fed it: request and database-query timing, the memory monitor thread, feed-generation counters (`starpunk/monitoring/`), and the `X-Request-ID` response header. Slow queries are no longer logged
|
||||
- The `METRICS_ENABLED`, `METRICS_SLOW_QUERY_THRESHOLD`, `METRICS_SAMPLING_RATE`, `METRICS_BUFFER_SIZE` and `METRICS_MEMORY_INTERVAL` settings (ignored if still set), and the `psutil` dependency
|
||||
- The `metrics` section of `/admin/health`; its database, pool, filesystem and disk checks are unchanged, as is `/health`
|
||||
|
||||
## [2.11.0] - 2026-11-03
|
||||
|
||||
Maintenance release: dependency upgrades and the High-priority backlog items.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2024 [Your Name]
|
||||
Copyright (c) 2025-2026 Phil Skentelbery
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
||||
@@ -41,7 +41,7 @@ what changed in each release is in [CHANGELOG.md](CHANGELOG.md).
|
||||
|
||||
**Operations**
|
||||
- Migrations applied automatically at startup
|
||||
- `/health` endpoint, admin metrics dashboard, structured logs
|
||||
- `/health` endpoint, `/admin/health` diagnostics, structured logs
|
||||
- Fetches of user-supplied URLs are limited to public addresses, size and
|
||||
time
|
||||
|
||||
@@ -113,4 +113,5 @@ up regularly, and always before upgrading to a release with a migration.
|
||||
|
||||
## License
|
||||
|
||||
MIT, see [LICENSE](LICENSE).
|
||||
[MIT](LICENSE): use, change and share it for anything, including
|
||||
commercially, as long as the copyright and license notice stay with it.
|
||||
@@ -218,8 +218,7 @@ server process enforces one shared limit. Old windows are pruned as it goes.
|
||||
timeout; `DB_POOL_SIZE` and `DB_TIMEOUT` can be set only through the
|
||||
`create_app()` config dict). Connections use `sqlite3.Row` rows,
|
||||
`check_same_thread=False`, `PRAGMA journal_mode=WAL` and (since v2.11.0)
|
||||
`PRAGMA foreign_keys=ON`, so `ON DELETE CASCADE`/`SET NULL` run, and are wrapped
|
||||
in `MonitoredConnection` for query timing when metrics are on. `get_db()`
|
||||
`PRAGMA foreign_keys=ON`, so `ON DELETE CASCADE`/`SET NULL` run. `get_db()`
|
||||
stores one connection in `flask.g` per request or app context; on teardown
|
||||
it goes back to the pool, rolled back first if a transaction was left
|
||||
open.
|
||||
|
||||
@@ -235,15 +235,12 @@ starts only if at least one is.
|
||||
| `THREADS_MAX_MEDIA` | `4` | |
|
||||
| `SYNDICATION_POLL_INTERVAL` | `10` | Seconds |
|
||||
|
||||
### Metrics and debugging
|
||||
### Debugging
|
||||
|
||||
The `METRICS_*` variables were removed in v2.12.0 and are ignored if set.
|
||||
|
||||
| Variable | Default | Notes |
|
||||
|----------|---------|-------|
|
||||
| `METRICS_ENABLED` | `true` | HTTP/database metrics and the memory monitor |
|
||||
| `METRICS_SLOW_QUERY_THRESHOLD` | `1.0` | Seconds |
|
||||
| `METRICS_SAMPLING_RATE` | `1.0` | |
|
||||
| `METRICS_BUFFER_SIZE` | `1000` | |
|
||||
| `METRICS_MEMORY_INTERVAL` | `30` | Seconds |
|
||||
| `DEBUG_SAVE_FAILED_UPLOADS` | `false` | Keep failed uploads in `debug/` |
|
||||
| `DEBUG_FILE_MAX_AGE_DAYS` | `7` | |
|
||||
| `DEBUG_FILE_MAX_SIZE_MB` | `100` | |
|
||||
|
||||
@@ -43,8 +43,6 @@ Key decisions:
|
||||
[ADR-058](../decisions/ADR-058-image-optimization-strategy.md) (media),
|
||||
[ADR-034](../decisions/ADR-034-full-text-search.md) (search),
|
||||
[ADR-052](../decisions/ADR-052-configuration-system-architecture.md) (configuration),
|
||||
[ADR-053](../decisions/ADR-053-performance-monitoring-strategy.md) and
|
||||
[ADR-060](../decisions/ADR-060-production-hotfix-metrics-dashboard.md) (metrics),
|
||||
[ADR-054](../decisions/ADR-054-structured-logging-architecture.md) (logging),
|
||||
[ADR-055](../decisions/ADR-055-error-handling-philosophy.md) (errors).
|
||||
|
||||
@@ -65,7 +63,6 @@ Key decisions:
|
||||
│ webmention worker works its queues only while holding a lock │
|
||||
│ websub worker works its queues only while holding a lock │
|
||||
│ syndication worker claims rows with a lease in SQLite │
|
||||
│ memory monitor per-process metrics │
|
||||
└───────────────┬──────────────────────────────┬─────────────────────┘
|
||||
│ │
|
||||
┌───────▼────────┐ ┌──────────▼──────────────────┐
|
||||
@@ -86,7 +83,7 @@ Key decisions:
|
||||
validates configuration, configures logging, cleans up old debug and
|
||||
temporary upload files, creates the schema and runs migrations (under a
|
||||
lock, see [database.md](database.md)), creates the connection pool,
|
||||
installs the HTTP metrics middleware, configures the feed cache and rate
|
||||
configures the feed cache and rate
|
||||
limiters, fills the FTS index if it is empty, registers blueprints, error
|
||||
handlers and template helpers, and starts the background threads.
|
||||
3. `before_request` assigns a correlation ID (included in every log line) and,
|
||||
@@ -94,8 +91,7 @@ Key decisions:
|
||||
4. The blueprint handler runs. Database access goes through `get_db()`, which
|
||||
hands out a pooled, request-scoped SQLite connection.
|
||||
5. `after_request` hooks add `Link: </webmention>; rel="webmention"` to HTML
|
||||
responses, WebSub `Link rel=hub` / `rel=self` headers to topic responses,
|
||||
and request metrics with an `X-Request-ID` header.
|
||||
responses, WebSub `Link rel=hub` / `rel=self` headers to topic responses.
|
||||
6. Errors are rendered by the central handlers in `starpunk/errors.py`
|
||||
(JSON for Micropub, HTML pages from `templates/<code>.html` elsewhere).
|
||||
|
||||
@@ -141,7 +137,6 @@ wait for outbound network calls; the workers make them later.
|
||||
| `errors.py` | Central error handlers |
|
||||
| `feeds/` | `rss.py` (feedgen), `atom.py`, `json_feed.py`, `opml.py`, `negotiation.py` (Accept header), `cache.py` (LRU + TTL feed cache with ETags) |
|
||||
| `feed.py` | Backward-compatibility re-export of `feeds.rss` |
|
||||
| `monitoring/` | Metrics buffer, HTTP middleware, database timing wrapper, memory monitor, business metrics |
|
||||
| `database/` | `schema.py` (`INITIAL_SCHEMA_SQL`), `init.py` (`init_db`), `pool.py` (connection pool) |
|
||||
| `migrations.py` | Migration runner for `migrations/NNN_*.sql` |
|
||||
|
||||
@@ -196,7 +191,7 @@ Everything works without JavaScript.
|
||||
| `/admin/webmentions`, `/admin/webmentions/<id>`, `…/action`, `…/bulk-action` | Webmention moderation |
|
||||
| `/admin/tokens`, `/admin/tokens/<id>/revoke` | Issued IndieAuth tokens |
|
||||
| `/admin/syndication/<id>/retry` | Retry a failed crosspost |
|
||||
| `/admin/metrics-dashboard`, `/admin/metrics`, `/admin/health` | Monitoring (HTML dashboard, JSON metrics, full diagnostics) |
|
||||
| `/admin/health` | Full diagnostics (JSON): database, pool statistics, filesystem, disk |
|
||||
| `/health` | Public liveness check (`{"status": "ok", "version": …}`); `?detailed=true` requires login |
|
||||
|
||||
## Background workers
|
||||
@@ -210,7 +205,6 @@ coordinated through the data volume:
|
||||
| Webmention worker (`webmention_worker.py`) | `WEBMENTION_WORKER_ENABLED` and not testing | Non-blocking `fcntl.flock` on `webmention-worker.lock` next to the database. Only the holder works the queues; the others try again each poll interval. The OS releases the lock if the holder dies, and the new holder resets rows stuck in `sending`. | Validates received webmentions; discovers endpoints and sends queued outgoing webmentions, with retries |
|
||||
| WebSub worker (`websub_worker.py`) | `WEBSUB_ENABLED` and not testing | Same pattern with `websub-worker.lock` | Verifies subscription intents, delivers topic content (signed with `X-Hub-Signature` when the subscriber gave a secret), retries, expires leases; pings an external hub instead if `WEBSUB_HUB_URL` is set |
|
||||
| Syndication worker (`syndication.py`) | At least one crosspost target configured, and not testing | No file lock. Rows in `syndications` are claimed with a 5-minute `claimed_until` lease by an atomic `UPDATE`, so any process may work a row but only one at a time; a `version` column keeps edits made during a send. Threads token refreshes are serialised with a lock file next to `threads_token.json`. | Creates, updates or deletes the remote post for each (note, target) pair; up to 8 attempts |
|
||||
| Memory monitor (`monitoring/memory.py`) | `METRICS_ENABLED` and not testing | None (per-process metrics) | Samples process memory every `METRICS_MEMORY_INTERVAL` seconds |
|
||||
|
||||
All workers run inside an app context, are daemon threads (they end with the
|
||||
process) and sleep for their poll interval when there is nothing to do. Under
|
||||
@@ -326,14 +320,11 @@ their credentials are set. The full list with defaults is in
|
||||
|
||||
- Logs go to stdout and `logs/starpunk.log` (10 MB × 10 files); each line
|
||||
carries the request's correlation ID.
|
||||
- Metrics (`monitoring/`): HTTP request timings, database query timings
|
||||
(with a slow-query threshold), memory, pool statistics and business
|
||||
counters, in an in-memory buffer per process, shown at
|
||||
`/admin/metrics-dashboard` and `/admin/metrics`.
|
||||
- `/health` reports status, version and webmention-worker statistics;
|
||||
`/health?detailed=true` checks database, filesystem and disk space;
|
||||
`/admin/health` adds metrics, pool statistics and system information.
|
||||
`/admin/health` adds pool statistics and filesystem details. (The metrics
|
||||
dashboard was removed in v2.12.0, see ADR-072.)
|
||||
|
||||
---
|
||||
|
||||
**Last Updated**: 2026-11-03 (v2.10.1)
|
||||
**Last Updated**: 2026-11-03 (v2.12.0)
|
||||
@@ -223,6 +223,49 @@ Webmention worker rejects a webmention whose source is one of the note's
|
||||
own copies (e.g. backfed by Bridgy), so a copy is never shown as a response
|
||||
to itself (v2.8.0).
|
||||
|
||||
## Backfeed (v2.13.0, ADR-073)
|
||||
|
||||
`starpunk/backfeed.py` brings responses to the copies back to the note.
|
||||
After each crosspost pass, the worker loop calls `backfeed.process_due()`,
|
||||
which checks up to three synced copies of published notes (not likes or
|
||||
reposts) that are due, each under a lease (`backfeed_claimed_until`):
|
||||
|
||||
- **Reading.** Each client has `responses(remote_id, remote_url)`, which
|
||||
returns a `Harvest`: GoToSocial `favourited_by`, `reblogged_by` and
|
||||
public direct replies from `context`; Bluesky `getLikes`,
|
||||
`getRepostedBy`, `getQuotes` and `getPostThread` (depth 1), plus
|
||||
`getProfiles` for reply and quote authors' bios; Threads `replies` and
|
||||
`insights` (like, repost and quote counts). Lists are paged up to 1,000
|
||||
items; a list cut short makes the harvest incomplete.
|
||||
- **Storing.** Responses are upserted into `webmentions` with `origin` (the
|
||||
target id) and `remote_id`, `validation_status = 'valid'`, keyed by
|
||||
`(source_url, target_url)`. Likes and reposts use the copy's URL plus
|
||||
`#liked-by-<actor>` / `#reposted-by-<actor>`. A new row gets the
|
||||
`WEBMENTION_AUTO_APPROVE` default; an existing row keeps its moderation.
|
||||
A complete harvest marks that silo's rows for the note that weren't
|
||||
found `invalid` ("No longer on <silo>"). Authors with `#nobridge` or
|
||||
`#nobot` in their bio, or Bluesky's `!no-unauthenticated` label, are
|
||||
skipped and their rows deleted. If the note's shown responses changed,
|
||||
its webmentions are re-sent (spec 3.1.4). Threads counts go in
|
||||
`syndications.likes_count`, `reposts_count`, `quotes_count`.
|
||||
- **Schedule.** `backfeed_next_at` follows the note's age: 15 minutes (day
|
||||
1), 1 hour (week 1), 1 day (to 30 days), 1 week (to
|
||||
`BACKFEED_MAX_AGE_DAYS`), with up to 10% jitter; none after. Failures back
|
||||
off like crossposts; after 8 they wait a month.
|
||||
- **Backfill.** `start_backfill()` flags every eligible copy
|
||||
(`backfeed_backfill = 1`). Flagged copies are checked only when no
|
||||
routine check is due, at most one every 5 seconds across workers
|
||||
(`site_state.backfeed_backfill_last`). Start and finish times are in
|
||||
`site_state`.
|
||||
- **Refused credentials.** A 401/403 (or Graph auth error) puts the target
|
||||
in `backfeed.denied` until restart; the admin Webmentions page says which
|
||||
scopes are needed. Crossposting is unaffected.
|
||||
|
||||
Avatars of all responses (received and backfed) are saved by
|
||||
`starpunk/avatars.py` and served from `/avatars/<sha256>.webp`; the
|
||||
webmention worker fills in missing ones every 10 minutes and removes unused
|
||||
ones weekly.
|
||||
|
||||
## Client sessions and tokens
|
||||
|
||||
- **Bluesky**: `createSession` with the app password, then
|
||||
|
||||
@@ -47,7 +47,6 @@ vulnerabilities (`pip-audit`); check again before each release.
|
||||
| python-dotenv | `1.2.*` | Loading `.env` |
|
||||
| Pillow | `13.0.*` | Image validation, resizing and variants |
|
||||
| pillow-heif | `1.8.*` | HEIC/HEIF uploads (iPhone photos) |
|
||||
| psutil | `7.2.*` | Memory monitor and system metrics |
|
||||
|
||||
`markupsafe` comes with Flask/Jinja2.
|
||||
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
# ADR-053: Performance Monitoring Strategy
|
||||
|
||||
## Status
|
||||
Accepted
|
||||
Superseded by ADR-072 (was: Accepted, v1.1.1)
|
||||
|
||||
> The metrics system was removed in v2.12.0. (status updated 2026-11-03)
|
||||
|
||||
## Context
|
||||
StarPunk v1.1.1 introduces performance monitoring to help operators understand system behavior in production. Currently, we have no visibility into:
|
||||
|
||||
@@ -2,7 +2,9 @@
|
||||
|
||||
## Status
|
||||
|
||||
Accepted (v1.1.1-rc.2, 2025-11-25)
|
||||
Superseded by ADR-072 (was: Accepted, v1.1.1-rc.2, 2025-11-25)
|
||||
|
||||
> The metrics dashboard was removed in v2.12.0. (status updated 2026-11-03)
|
||||
|
||||
> Reconstructed on 2026-11-03 from code, changelog and related documents;
|
||||
> the original was lost. The original
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
# ADR-072: Remove the metrics dashboard
|
||||
|
||||
## Status
|
||||
|
||||
Accepted (v2.12.0, 2026-11-03). Supersedes ADR-053 and ADR-060.
|
||||
|
||||
## Context
|
||||
|
||||
v1.1.1 and v1.1.2 added built-in performance monitoring (ADR-053): HTTP
|
||||
request timing middleware, a wrapper on every database connection that
|
||||
timed queries, a memory-sampling thread, feed-generation counters, an
|
||||
in-memory buffer of the results, and an HTML dashboard at
|
||||
`/admin/metrics-dashboard` with a JSON twin at `/admin/metrics` (ADR-060
|
||||
covered adapting the data for the dashboard template).
|
||||
|
||||
In practice it told us little:
|
||||
|
||||
- The buffer lives in memory in each of the four gunicorn processes, so the
|
||||
dashboard showed whichever process served the request, and everything was
|
||||
lost on restart or deploy.
|
||||
- Nothing in it was ever used to find or fix a problem. Logs (with
|
||||
correlation IDs), `/health` and `/admin/health` were.
|
||||
- It cost something on every request and every query, kept a thread per
|
||||
process, and needed `psutil`, about 1,300 lines of code and three test
|
||||
files.
|
||||
|
||||
## Decision
|
||||
|
||||
Remove the dashboard and everything that only fed it: the `starpunk/monitoring`
|
||||
package, the HTTP metrics middleware (including its `X-Request-ID`
|
||||
response header), `MonitoredConnection`, the memory monitor, the
|
||||
feed-generation tracking calls, `/admin/metrics-dashboard`,
|
||||
`/admin/metrics`, the `METRICS_*` settings and the `psutil` dependency.
|
||||
|
||||
Keep:
|
||||
|
||||
- `/health` and `/health?detailed=true`;
|
||||
- `/admin/health`, without its `metrics` section (database, pool
|
||||
statistics, filesystem and disk checks stay);
|
||||
- connection pool statistics (`get_pool_stats`) and the feed cache, which
|
||||
do work of their own;
|
||||
- structured logs with correlation IDs (ADR-054).
|
||||
|
||||
## Consequences
|
||||
|
||||
- Less code and one fewer dependency; no timing work on each request or
|
||||
query; one fewer thread per process.
|
||||
- Slow database queries are no longer logged. If that is ever needed, a
|
||||
query timer can be added at the pool level, or an external tool used.
|
||||
- `METRICS_*` environment variables are ignored if still set.
|
||||
- Bookmarks to `/admin/metrics-dashboard` now get a 404.
|
||||
@@ -0,0 +1,272 @@
|
||||
# ADR-073: Backfeed of responses to crossposts
|
||||
|
||||
## Status
|
||||
|
||||
Accepted (v2.13.0, 2026-11-10)
|
||||
|
||||
Implementation notes (v2.13.0):
|
||||
|
||||
- Until an avatar has a local copy, pages show the initial and keep the
|
||||
original URL only as `<data class="u-photo">` for microformats parsers,
|
||||
so browsers don't load it.
|
||||
- Bluesky post views carry no bio, so reply and quote authors' profiles
|
||||
are fetched (`getProfiles`, 25 at a time) to honour `#nobridge`.
|
||||
- A Threads token without `threads_manage_insights` stops Threads
|
||||
backfeed as a whole, as for any refused credential.
|
||||
- Backfill checks take a shared slot in `site_state`
|
||||
(`backfeed_backfill_last`), so all gunicorn workers together do at most
|
||||
one every 5 seconds.
|
||||
|
||||
## Context
|
||||
|
||||
StarPunk crossposts notes to GoToSocial, Bluesky and Threads (ADR-065,
|
||||
ADR-066, ADR-069). People like, repost, quote and reply to those copies,
|
||||
and none of that reaches the note. [Backfeed](https://indieweb.org/backfeed)
|
||||
brings those responses back to the original post.
|
||||
|
||||
The usual tool is [Bridgy](https://brid.gy/about), which polls the silos and
|
||||
sends webmentions. It is a poor fit here:
|
||||
|
||||
- It supports Mastodon and Bluesky, not Threads.
|
||||
- It only checks the 30 most recent posts, about every 30 minutes, and
|
||||
says nothing when a like is withdrawn or a reply deleted.
|
||||
- Since v2.8.0 StarPunk rejects webmentions from a note's own crossposts
|
||||
(`_is_own_syndicated_copy`, `webmention_worker.py`). Bridgy's like and
|
||||
repost pages most likely carry the crosspost's URL plus a fragment as
|
||||
`u-url`, so they would be rejected. Replies have their own URL and get
|
||||
through (production webmention #1 was a Bridgy reply from Mastodon).
|
||||
- Every Bridgy source is on `brid.gy`, which shares one source-domain rate
|
||||
limit of 10 webmentions a minute (ADR-063, Decision 10).
|
||||
- Every response passes through a third party.
|
||||
|
||||
StarPunk already holds a credential for each silo and knows each copy
|
||||
(`syndications.remote_id`, `remote_url`). What the silos let it read:
|
||||
|
||||
| Silo | Likes | Reposts | Quotes | Replies | Credential change |
|
||||
|---|---|---|---|---|---|
|
||||
| Bluesky | `getLikes` | `getRepostedBy` | `getQuotes` | `getPostThread` | none (app password) |
|
||||
| GoToSocial | `favourited_by` | `reblogged_by` | not supported | `context` | read scopes |
|
||||
| Threads | count only | count only | count only | `replies` | `threads_read_replies`, `threads_manage_insights` |
|
||||
|
||||
The Threads API never says who liked, reposted or quoted a post; the
|
||||
Insights API gives lifetime counts only.
|
||||
|
||||
The production GoToSocial prunes remote posts older than 90 days, but never
|
||||
a thread that a local account started or took part in (GoToSocial's "Post
|
||||
Caching and Pruning" docs), so replies to StarPunk's copies are kept. On
|
||||
2026-11-10 the production database still had 106 remote replies to the
|
||||
account older than 90 days, back to August 2024.
|
||||
|
||||
## Decision
|
||||
|
||||
### 1. Native polling, no Bridgy
|
||||
|
||||
StarPunk polls each silo itself for responses to its own copies. Bridgy is
|
||||
switched off for the Mastodon and Bluesky accounts once this ships. The
|
||||
own-copy rule stays as it is.
|
||||
|
||||
### 2. Backfed responses are webmentions
|
||||
|
||||
Responses are stored in the `webmentions` table, so moderation, the
|
||||
Responses section, and re-sending webmentions when a note's responses
|
||||
change (spec 3.1.4, `note_responses_changed`) work as they do now. A
|
||||
backfed row:
|
||||
|
||||
- `source_url`: the response's own URL where it has one (a reply or quote
|
||||
post). Likes and reposts have no page, so they get the copy's URL plus
|
||||
`#liked-by-<actor id>` or `#reposted-by-<actor id>`. The
|
||||
`(source_url, target_url)` key makes repeated checks update the same row.
|
||||
- `target_url`: the note's permalink. `target_note_id`: the note.
|
||||
- `origin`: new column. `webmention` for received webmentions (the
|
||||
default), else `gotosocial`, `bluesky` or `threads`.
|
||||
- `remote_id`: new column. The silo's id for the response (status id,
|
||||
AT-URI, Threads media id), or the actor's id for a like or repost.
|
||||
- `mention_type`: `like`, `repost`, `reply` or the new `quote`.
|
||||
- Author name, profile URL and avatar from the silo's account data;
|
||||
`content_snippet` from the reply or quote text, sanitised the same way as
|
||||
received webmentions (Mastodon HTML through nh3, Bluesky and Threads text
|
||||
escaped with links kept); `source_published` from the silo's timestamp.
|
||||
- `validation_status = 'valid'`: StarPunk read it from the silo's API
|
||||
itself, so there's nothing to fetch and verify. Moderation follows
|
||||
`WEBMENTION_AUTO_APPROVE` like any first-time webmention.
|
||||
|
||||
Only replies directly to the copy are taken, not deeper replies in the
|
||||
thread. Responses from the accounts StarPunk posts as are skipped.
|
||||
|
||||
### 3. Schedule
|
||||
|
||||
Each synced copy of a published note is checked on a schedule that slows
|
||||
as the note ages:
|
||||
|
||||
| Note age | Check every |
|
||||
|---|---|
|
||||
| under 1 day | 15 minutes |
|
||||
| 1 to 7 days | 1 hour |
|
||||
| 7 to 30 days | 1 day |
|
||||
| 30 to 90 days | 1 week |
|
||||
| over 90 days | not checked |
|
||||
|
||||
A random jitter of up to 10% spreads the checks. The cutoff is
|
||||
`BACKFEED_MAX_AGE_DAYS` (default 90). Copies of likes and reposts
|
||||
(ADR-068) aren't checked: they have no responses of their own. A note that
|
||||
is unpublished or deleted is not checked, and its rows stay as they are.
|
||||
|
||||
### 4. Withdrawn likes and deleted replies
|
||||
|
||||
A check that read the whole list (every page, under a cap of 1,000 items
|
||||
per list) marks that note's rows from that silo that are no longer there
|
||||
as `validation_status = 'invalid'` with the error "No longer on
|
||||
<silo>". They drop off the page but keep their moderation decision, and
|
||||
come back if they reappear. A check that stopped early (an error, or the
|
||||
cap) removes nothing.
|
||||
|
||||
### 5. Privacy
|
||||
|
||||
- Only public responses: GoToSocial `visibility` `public` (not unlisted,
|
||||
followers-only or direct); on Bluesky, nothing from accounts with the
|
||||
`!no-unauthenticated` label.
|
||||
- People who opt out are skipped: `#nobridge` or `#nobot` in their bio
|
||||
(Mastodon `note`, Bluesky `description`), the same signal Bridgy honours.
|
||||
A row already stored for them is removed at the next check.
|
||||
- Admin **Delete** stops being a real `DELETE` and sets
|
||||
`moderation_status = 'deleted'` (the value exists but is unused), so a
|
||||
later check or re-sent webmention can't bring the response back. This
|
||||
applies to all webmentions.
|
||||
|
||||
### 6. Avatars are cached locally
|
||||
|
||||
Received webmentions hotlink author photos today. That leaks visitors'
|
||||
addresses to other sites, and Threads and Bluesky avatar URLs change or
|
||||
expire. Avatars are fetched once with `safe_get` (1 MB limit), resized to
|
||||
96 px WebP with Pillow and saved as `DATA_PATH/avatars/<sha256 of URL>.webp`,
|
||||
served from `/avatars/`. They're refetched when the URL changes, and files
|
||||
no row uses are removed weekly. Received webmentions use the same cache
|
||||
from their validation step. A failed fetch falls back to the coloured
|
||||
initial.
|
||||
|
||||
### 7. Threads counts
|
||||
|
||||
Threads likes, reposts and quotes are shown as anonymous counts, not as
|
||||
responses. The worker reads
|
||||
`/{media-id}/insights?metric=likes,reposts,quotes` with each Threads check
|
||||
and stores the numbers on the copy's `syndications` row. The note shows a
|
||||
plain line such as "On Threads: 12 likes · 3 reposts" after the
|
||||
Responses, with no `u-like` or `h-cite` markup, because these aren't
|
||||
people. Threads `replies` counts aren't used: the replies themselves are
|
||||
shown. `views` and `shares` are left out (Meta marks them "in
|
||||
development").
|
||||
|
||||
### 8. Backfill
|
||||
|
||||
An admin action, **Backfill responses**, checks every synced copy once,
|
||||
whatever the note's age. Nothing runs by itself on deploy; the owner runs
|
||||
it after the first deploy (and may run it again later). It:
|
||||
|
||||
- marks every synced copy due now, flagged as a backfill;
|
||||
- is worked through by the same worker, at most one copy every 5 seconds,
|
||||
ahead of the routine schedule only when nothing routine is due;
|
||||
- records when it started and finished; the admin page shows this, and
|
||||
the button is disabled while one is running.
|
||||
|
||||
Upserts make it safe to repeat.
|
||||
|
||||
### 9. Worker
|
||||
|
||||
The checks run in the syndication worker's loop (ADR-065) after crossposts,
|
||||
reusing its clients, token refresh (`maintain()`) and the lease pattern:
|
||||
a copy is claimed with `backfeed_claimed_until` so only one gunicorn worker
|
||||
checks it. A failed check backs off like a failed crosspost
|
||||
(`min(30 s × 2^attempts, 6 h)`) without changing the copy's crosspost
|
||||
state. If a silo answers with a permission error, backfeed for that silo
|
||||
stops until restart and the admin page says which credential needs which
|
||||
scope; crossposting is unaffected.
|
||||
|
||||
Per check: Bluesky 4 calls (likes, reposts, quotes, thread), GoToSocial 3,
|
||||
Threads 2 (replies, insights), plus extra pages.
|
||||
|
||||
### 10. Data (migration 020)
|
||||
|
||||
- `webmentions`: rebuilt to add `quote` to the `mention_type` CHECK, with
|
||||
new columns `origin TEXT NOT NULL DEFAULT 'webmention'` and
|
||||
`remote_id TEXT`, and an index on `(target_note_id, origin)`.
|
||||
- `syndications`: new columns `backfeed_next_at`, `backfeed_checked_at`,
|
||||
`backfeed_claimed_until`, `backfeed_attempts`, `backfeed_error`,
|
||||
`backfeed_backfill` (flag), and `likes_count`, `reposts_count`,
|
||||
`quotes_count` (Threads only).
|
||||
- The backfill's start and finish times go in the existing `site_state`
|
||||
key/value table (migration 016): `backfeed_backfill_started`,
|
||||
`backfeed_backfill_finished`.
|
||||
|
||||
### 11. Display and admin
|
||||
|
||||
- Each backfed response shows a small "via Bluesky" (or GoToSocial,
|
||||
Threads) label. Quotes get their own group, rendered like mentions with
|
||||
the quote text, as `u-mention h-cite`.
|
||||
- The note editor shows when responses were last checked, with a
|
||||
**Check now** button for each copy.
|
||||
- The webmentions list can be filtered by origin.
|
||||
- The dashboard shows backfeed errors next to crosspost errors.
|
||||
|
||||
### 12. Configuration and credentials
|
||||
|
||||
- `BACKFEED_ENABLED` (default `false`; production sets `true`) and
|
||||
`BACKFEED_MAX_AGE_DAYS` (default 90).
|
||||
- GoToSocial: a new token with read access added to the current scopes
|
||||
(`read:statuses` and `read:accounts`, or plain `read`; checked against
|
||||
GoToSocial's scope list when the token is issued).
|
||||
- Threads: the Meta app is re-authorised with `threads_read_replies` and
|
||||
`threads_manage_insights` added. The new `THREADS_ACCESS_TOKEN` replaces
|
||||
the saved token file automatically (ADR-069, Decision 2).
|
||||
- Bluesky: no change.
|
||||
- Production tokens go in the Ansible vault (`bin/vault-run`).
|
||||
|
||||
While at it, the Threads client moves from `graph.threads.net` to
|
||||
`graph.threads.com`, the host Meta's current documentation uses.
|
||||
|
||||
## Amendments (v2.13.2, 2026-11-10)
|
||||
|
||||
- **Display.** Likes, reposts and bookmarks are a line of counts under the
|
||||
note instead of facepiles in the Responses section. Hovering or focusing
|
||||
a count lists the people (as `u-like` / `u-repost` / `u-bookmark`
|
||||
h-cites) and the silo each came from. Threads' anonymous counts are added
|
||||
to the totals ("+N on Threads") instead of a separate line (Decision 7).
|
||||
- **Web actions.** Each like, repost and reply count is an
|
||||
`<indie-action do="…" with="<permalink>">` whose fallback is a link to
|
||||
the same action on each crosspost. No web-action polyfill is loaded;
|
||||
handlers such as browser extensions can use the markup.
|
||||
|
||||
## Consequences
|
||||
|
||||
- Responses on all three silos show on the note within 15 minutes for new
|
||||
notes, with no third party involved.
|
||||
- Withdrawn likes and deleted replies disappear by themselves for 90 days;
|
||||
after that a note's responses are frozen.
|
||||
- Threads shows replies in full but likes, reposts and quotes only as
|
||||
numbers.
|
||||
- Backfeed stores other people's public posts and avatars. The privacy
|
||||
rules (Decision 5) and moderation are the controls; there is no consent
|
||||
step.
|
||||
- More API calls. At about 50 new notes a month the routine schedule is
|
||||
roughly 5,000 calls a month across the three silos, well inside their
|
||||
limits. The backfill makes one burst of a few calls per copy.
|
||||
- Deleting a webmention in the admin now leaves a tombstone row.
|
||||
- A crosspost removed and posted again gets a new copy; responses to the
|
||||
old copy are marked gone at the next check.
|
||||
|
||||
## Alternatives considered
|
||||
|
||||
- **Bridgy.** No Threads, 30-post window, no withdrawals, a third party in
|
||||
the path, and it needs the own-copy rule and rate limit loosened (see
|
||||
Context). Rejected.
|
||||
- **A separate responses table.** Would duplicate the display, moderation
|
||||
and 3.1.4 re-sending code that `webmentions` already has. Rejected.
|
||||
- **Notification feeds** (Mastodon `/api/v1/notifications`, Bluesky
|
||||
`listNotifications`): one call covers every post, but they only report
|
||||
new events, so withdrawals are never seen and a backfill can't use them.
|
||||
Could be added later as a fast path in front of per-post checks.
|
||||
- **Streaming** (Bluesky Jetstream, Mastodon streaming API): real time, but
|
||||
needs a long-lived connection per gunicorn worker. Not worth it for a
|
||||
personal site.
|
||||
- **Keep the Threads counts out.** They aren't IndieWeb responses, but they
|
||||
are the only signal Threads gives, and showing them anonymously costs one
|
||||
call and identifies no one. Kept.
|
||||
@@ -5,8 +5,8 @@ records one decision: its context, what was decided, and the consequences.
|
||||
Superseded ADRs are kept as history; their Status line names the ADR that
|
||||
replaced them.
|
||||
|
||||
**Total ADR files**: 67 (66 numbers: ADR-001 to ADR-044 and ADR-050 to
|
||||
ADR-071, with two files numbered ADR-012). See [Numbering](#numbering).
|
||||
**Total ADR files**: 69 (68 numbers: ADR-001 to ADR-044 and ADR-050 to
|
||||
ADR-073, with two files numbered ADR-012). See [Numbering](#numbering).
|
||||
|
||||
Status values: **Accepted** (in force), **Superseded** (replaced by a later
|
||||
ADR), **Deprecated** (withdrawn, nothing replaces it; none at present),
|
||||
@@ -85,6 +85,7 @@ See also [IndieWeb protocols](../architecture/indieweb-protocols.md).
|
||||
| [066](ADR-066-bluesky-syndication.md) | Crossposting to Bluesky | Accepted | Bluesky as a second crosspost target (v2.2.0). |
|
||||
| [069](ADR-069-threads-syndication.md) | Crossposting to Threads | Accepted | Threads as a third target, with automatic token refresh (v2.6.0). |
|
||||
| [070](ADR-070-crosspost-links-and-cards.md) | Links and link cards in crossposts | Accepted | Keep note links in crossposts and build link cards for the linked page (v2.6.3). |
|
||||
| [073](ADR-073-backfeed.md) | Backfeed of responses to crossposts | Accepted | Poll GoToSocial, Bluesky and Threads for responses to crossposts and store them as webmentions; admin backfill (v2.13.0). |
|
||||
|
||||
### Authentication (current)
|
||||
|
||||
@@ -101,10 +102,9 @@ See also [IndieWeb protocols](../architecture/indieweb-protocols.md).
|
||||
|---|---|---|---|
|
||||
| [012](ADR-012-http-error-handling-policy.md) | HTTP Error Handling Policy | Accepted | Correct HTTP status codes (e.g. 404 for missing notes) across routes. |
|
||||
| [052](ADR-052-configuration-system-architecture.md) | Configuration System Architecture | Accepted | Environment variables with defaults, loaded in one module. |
|
||||
| [053](ADR-053-performance-monitoring-strategy.md) | Performance Monitoring Strategy | Accepted | Built-in, standard-library performance metrics (v1.1.1). |
|
||||
| [054](ADR-054-structured-logging-architecture.md) | Structured Logging Architecture | Accepted | Rotating log files with correlation IDs (v1.1.1). |
|
||||
| [055](ADR-055-error-handling-philosophy.md) | Error Handling Philosophy | Accepted | Layered handling: graceful degradation, clear user messages, detailed logs. |
|
||||
| [060](ADR-060-production-hotfix-metrics-dashboard.md) | Route Adapter for the Metrics Dashboard | Accepted | Adapt monitoring data to the dashboard template in the route (v1.1.1-rc.2). *Reconstructed.* |
|
||||
| [072](ADR-072-remove-metrics-dashboard.md) | Remove the metrics dashboard | Accepted | Drop the in-memory metrics system and dashboard; keep `/health`, `/admin/health` and logs (v2.12.0). |
|
||||
|
||||
See also [deployment](../architecture/deployment.md).
|
||||
|
||||
@@ -117,6 +117,8 @@ designs of v0.x–v1.6.0, all replaced by ADR-064 in v2.0.0.
|
||||
|---|---|---|---|
|
||||
| [005](ADR-005-indielogin-authentication.md) | IndieLogin Authentication Integration | Superseded by 064 | Admin login through indielogin.com. |
|
||||
| [007](ADR-007-slug-generation-algorithm.md) | Slug Generation Algorithm | Superseded by 062 | Slugs from the first words of the note. |
|
||||
| [053](ADR-053-performance-monitoring-strategy.md) | Performance Monitoring Strategy | Superseded by 072 | Built-in, standard-library performance metrics (v1.1.1). |
|
||||
| [060](ADR-060-production-hotfix-metrics-dashboard.md) | Route Adapter for the Metrics Dashboard | Superseded by 072 | Adapt monitoring data to the dashboard template in the route (v1.1.1-rc.2). *Reconstructed.* |
|
||||
| [016](ADR-016-indieauth-client-discovery.md) | IndieAuth Client Discovery Mechanism | Superseded by 019 | h-app markup for client discovery. |
|
||||
| [017](ADR-017-oauth-client-metadata-document.md) | OAuth Client ID Metadata Document Implementation | Superseded by 019 | OAuth client metadata endpoint. |
|
||||
| [018](ADR-018-indieauth-detailed-logging.md) | IndieAuth Detailed Logging Strategy | Superseded by 064 | Redacted HTTP logging of the IndieAuth login flow. |
|
||||
@@ -166,7 +168,7 @@ was carried out and is not itself superseded.
|
||||
|
||||
## How to create a new ADR
|
||||
|
||||
1. Use the next number after the highest existing one (currently ADR-072).
|
||||
1. Use the next number after the highest existing one (currently ADR-074).
|
||||
2. Name it `ADR-NNN-brief-descriptive-title.md`.
|
||||
3. Follow the format of recent ADRs (e.g. ADR-070, ADR-071):
|
||||
|
||||
|
||||
+21
-1
@@ -21,7 +21,7 @@ is in `../architecture/`; decisions are in `../decisions/` (ADRs).
|
||||
|
||||
## Contents
|
||||
|
||||
[v1.0.0](#v100) · [v1.1.1](#v111) · [v1.1.2](#v112) · [v1.2.0](#v120) · [v1.3.0](#v130) · [v1.3.1](#v131) · [v1.4.0](#v140) · [v1.4.1](#v141) · [v1.4.2](#v142) · [v1.5.0](#v150) · [v1.6.0](#v160) · [v2.0.0](#v200) · [v2.0.1](#v201) · [v2.0.2](#v202) · [v2.1.0](#v210) · [v2.2.0](#v220) · [v2.3.0](#v230) · [v2.3.1](#v231) · [v2.3.2](#v232) · [v2.3.3](#v233) · [v2.4.0](#v240) · [v2.5.0](#v250) · [v2.6.0](#v260) · [v2.6.1](#v261) · [v2.6.2](#v262) · [v2.6.3](#v263) · [v2.6.4](#v264) · [v2.6.5](#v265) · [v2.6.6](#v266) · [v2.6.7](#v267) · [v2.6.8](#v268) · [v2.6.9](#v269) · [v2.7.0](#v270) · [v2.7.1](#v271) · [v2.7.2](#v272) · [v2.8.0](#v280) · [v2.8.1](#v281) · [v2.9.0](#v290) · [v2.10.0](#v2100) · [v2.10.1](#v2101) · [v2.10.2](#v2102) · [v2.11.0](#v2110) · [hotfix](#hotfix)
|
||||
[v1.0.0](#v100) · [v1.1.1](#v111) · [v1.1.2](#v112) · [v1.2.0](#v120) · [v1.3.0](#v130) · [v1.3.1](#v131) · [v1.4.0](#v140) · [v1.4.1](#v141) · [v1.4.2](#v142) · [v1.5.0](#v150) · [v1.6.0](#v160) · [v2.0.0](#v200) · [v2.0.1](#v201) · [v2.0.2](#v202) · [v2.1.0](#v210) · [v2.2.0](#v220) · [v2.3.0](#v230) · [v2.3.1](#v231) · [v2.3.2](#v232) · [v2.3.3](#v233) · [v2.4.0](#v240) · [v2.5.0](#v250) · [v2.6.0](#v260) · [v2.6.1](#v261) · [v2.6.2](#v262) · [v2.6.3](#v263) · [v2.6.4](#v264) · [v2.6.5](#v265) · [v2.6.6](#v266) · [v2.6.7](#v267) · [v2.6.8](#v268) · [v2.6.9](#v269) · [v2.7.0](#v270) · [v2.7.1](#v271) · [v2.7.2](#v272) · [v2.8.0](#v280) · [v2.8.1](#v281) · [v2.9.0](#v290) · [v2.10.0](#v2100) · [v2.10.1](#v2101) · [v2.10.2](#v2102) · [v2.11.0](#v2110) · [v2.12.0](#v2120) · [v2.12.1](#v2121) · [v2.13.0](#v2130) · [v2.13.1](#v2131) · [v2.13.2](#v2132) · [hotfix](#hotfix)
|
||||
|
||||
## v1.0.0
|
||||
|
||||
@@ -414,6 +414,26 @@ OIDC admin login and built-in IndieAuth server.
|
||||
|
||||
- [2026-11-03-maintenance-release.md](v2.11.0/2026-11-03-maintenance-release.md) - Dependency upgrades (Python 3.14), foreign keys, CSRF, fetchers, debug default, shared rate limits
|
||||
|
||||
## v2.12.0
|
||||
|
||||
- [2026-11-03-remove-metrics.md](v2.12.0/2026-11-03-remove-metrics.md) - Metrics dashboard and `starpunk/monitoring` removed (ADR-072)
|
||||
|
||||
## v2.12.1
|
||||
|
||||
- [2026-11-10-category-unused-tags.md](v2.12.1/2026-11-10-category-unused-tags.md) - q=category lists only tags on posts that exist
|
||||
|
||||
## v2.13.0
|
||||
|
||||
- [2026-11-10-backfeed.md](v2.13.0/2026-11-10-backfeed.md) - Backfeed of responses to crossposts (ADR-073), avatar cache, delete tombstones
|
||||
|
||||
## v2.13.1
|
||||
|
||||
- [2026-11-10-avatar-fetch.md](v2.13.1/2026-11-10-avatar-fetch.md) - Avatars from the own GoToSocial server and from Bluesky PDSes
|
||||
|
||||
## v2.13.2
|
||||
|
||||
- [2026-11-10-reaction-counts.md](v2.13.2/2026-11-10-reaction-counts.md) - Likes/reposts as a counts line with hover lists; web actions
|
||||
|
||||
## hotfix
|
||||
|
||||
Not a release folder.
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
# v2.12.0: Remove the metrics dashboard
|
||||
|
||||
Decision: [ADR-072](../../decisions/ADR-072-remove-metrics-dashboard.md).
|
||||
|
||||
## Removed
|
||||
|
||||
| What | Where |
|
||||
|---|---|
|
||||
| Metrics buffer, HTTP middleware, database timing wrapper, memory monitor, business metrics | `starpunk/monitoring/` (deleted) |
|
||||
| HTTP middleware and memory monitor start-up | `starpunk/__init__.py` |
|
||||
| `MonitoredConnection` wrapping, `slow_query_threshold` and `metrics_enabled` arguments | `starpunk/database/pool.py` |
|
||||
| `track_feed_generated` calls and their timing code | `starpunk/feeds/atom.py`, `json_feed.py`, `rss.py` |
|
||||
| `/admin/metrics-dashboard`, `/admin/metrics`, `transform_metrics_for_template`, the `metrics` block of `/admin/health` | `starpunk/routes/admin.py` |
|
||||
| Dashboard template and the admin nav link | `templates/admin/metrics_dashboard.html`, `templates/admin/base.html` |
|
||||
| `METRICS_ENABLED`, `METRICS_SLOW_QUERY_THRESHOLD`, `METRICS_SAMPLING_RATE`, `METRICS_BUFFER_SIZE`, `METRICS_MEMORY_INTERVAL` | `starpunk/config.py`, `.env.example` |
|
||||
| `psutil` | `requirements.txt` |
|
||||
| Tests | `tests/test_monitoring.py`, `tests/test_monitoring_feed_statistics.py`, `tests/test_admin_feed_statistics.py` |
|
||||
|
||||
## Kept
|
||||
|
||||
`/health`, `/health?detailed=true`, `/admin/health` (database, pool,
|
||||
filesystem, disk), pool statistics, the feed cache and its own hit/miss
|
||||
counters, and correlation IDs in logs.
|
||||
|
||||
## Upgrade
|
||||
|
||||
No migration. `METRICS_*` variables can be deleted from the environment;
|
||||
they are ignored if left.
|
||||
@@ -0,0 +1,28 @@
|
||||
# v2.12.1: q=category lists only tags in use
|
||||
|
||||
## Problem
|
||||
|
||||
`q=category` (added in v2.4.0) is what Micropub clients use for tag
|
||||
autocomplete. It read every row of `tags` with a `LEFT JOIN` on
|
||||
`note_tags`, so it also returned:
|
||||
|
||||
- tags no post uses any more (a `tags` row stays after an edit removes
|
||||
the tag from its last post), and
|
||||
- tags only used on soft-deleted posts (`note_tags` rows stay so that
|
||||
undelete can restore them).
|
||||
|
||||
Clients then offered tags that appear nowhere on the site.
|
||||
|
||||
## Fix
|
||||
|
||||
`_all_categories()` in `starpunk/micropub.py` inner-joins `note_tags` and
|
||||
`notes` with `deleted_at IS NULL`. Order is unchanged: most used first,
|
||||
then by name. Drafts count, since they are the owner's posts. Unused
|
||||
`tags` rows are left in place; nothing else reads them as a list.
|
||||
|
||||
## Verification
|
||||
|
||||
`tests/test_micropub_v240.py::TestQueries::test_category_skips_unused_tags`
|
||||
retags one post and deletes another, checks only the live tag is listed,
|
||||
then undeletes and checks the deleted post's tag returns. It fails on
|
||||
v2.12.0.
|
||||
@@ -0,0 +1,53 @@
|
||||
# v2.13.0: Backfeed (implementation report)
|
||||
|
||||
Implements ADR-073.
|
||||
|
||||
## What changed
|
||||
|
||||
- `migrations/020_backfeed.sql`: `webmentions` rebuilt with `origin`,
|
||||
`remote_id` and the `quote` type (rows copied, indexes recreated, plus
|
||||
`(target_note_id, origin)`); `syndications` gains the `backfeed_*`
|
||||
columns and Threads counts.
|
||||
- `starpunk/backfeed.py` (new): `Response`/`Harvest`, schedule, lease,
|
||||
storing and pruning, opt-outs, backfill, `check_now`, `status`.
|
||||
- `starpunk/syndication.py`: `responses()` on `MastodonClient`,
|
||||
`BlueskyClient` and `ThreadsClient`; HTTP status kept on errors
|
||||
(`exc.status`) so refused credentials can be told apart; Mastodon `_call`
|
||||
accepts absolute URLs (pagination links); the worker loop runs backfeed
|
||||
after crossposts; Threads host is `graph.threads.com`.
|
||||
- `starpunk/avatars.py` (new) and `/avatars/<name>`: local avatar copies
|
||||
for every response; `local_avatar` template filter.
|
||||
- Webmentions: Quotes group and "via" label; admin delete is a tombstone
|
||||
(`moderation_status = 'deleted'`, hidden from the list unless filtered);
|
||||
origin filter; a webmention can't reset a backfed row, and one whose
|
||||
source is an already backfed response is rejected ("Already backfed").
|
||||
- Admin: Backfeed panel with **Backfill responses** on the Webmentions
|
||||
page; **Check now** and last-checked time per crosspost on the edit page.
|
||||
- Settings `BACKFEED_ENABLED` (default false), `BACKFEED_MAX_AGE_DAYS` (90).
|
||||
|
||||
## Verification
|
||||
|
||||
- `tests/test_backfeed.py` (22 tests): schedule; text/facet HTML; storing,
|
||||
updating and moderation kept; pruning only on complete harvests and
|
||||
restoring; opt-outs; tombstones; backfill and its pacing; likes/reposts
|
||||
and drafts skipped; refused credentials; backoff; note page (via label,
|
||||
quotes, Threads counts, local avatars); avatar save/serve/cleanup;
|
||||
webmention can't overwrite a backfed row; the three clients against fake
|
||||
APIs; migration 020 on a pre-020 database keeps rows and foreign keys;
|
||||
the admin Backfeed panel, backfill and Check now; Threads avatar URLs
|
||||
(signed, changing per request) aren't refetched while a copy exists.
|
||||
- Tests for the touched areas (webmentions, syndication, microformats,
|
||||
migrations, templates, routes): all pass. The full suite runs in CI.
|
||||
- `test_microformats_v280` expects response authors' `u-photo`: kept via
|
||||
`<data class="u-photo">` until the avatar is saved.
|
||||
|
||||
## Deploying
|
||||
|
||||
1. Issue a new GoToSocial token with `read:statuses read:accounts` added
|
||||
to the current scopes, and re-authorise the Threads app with
|
||||
`threads_read_replies` and `threads_manage_insights`. Put both in the
|
||||
Ansible vault.
|
||||
2. Set `BACKFEED_ENABLED=true` in the StarPunk environment and deploy
|
||||
(migration 020 runs at start: back up the database first).
|
||||
3. Run **Backfill responses** once from Admin → Webmentions. Turn Bridgy
|
||||
off for the Mastodon and Bluesky accounts.
|
||||
@@ -0,0 +1,33 @@
|
||||
# v2.13.1: Avatar fetching in production
|
||||
|
||||
## Problem
|
||||
|
||||
After deploying v2.13.0, backfeed stored responses but no avatar was saved:
|
||||
|
||||
- GoToSocial avatars (`https://social.../fileserver/...`):
|
||||
`SSRFBlockedError: Blocked request to private IP: 172.20.0.5`. Inside the
|
||||
container the GoToSocial host resolves to the reverse proxy's internal
|
||||
address (the deployment's hairpin `extra_hosts` entry).
|
||||
- Bluesky avatars (`https://cdn.bsky.app/img/avatar/...`): HTTP 403 from
|
||||
BunnyCDN. From the server, the CDN answers 403 over IPv4 and 200 over
|
||||
IPv6; the container has no IPv6 route. From elsewhere IPv4 works.
|
||||
|
||||
## Fix (`starpunk/avatars.py`, `_download`)
|
||||
|
||||
- The host of `GOTOSOCIAL_URL` is fetched with a plain HTTP client (no
|
||||
redirects, 1 MB limit), as `MastodonClient` already talks to it. Every
|
||||
other host still goes through `safe_get`, so private addresses stay
|
||||
blocked.
|
||||
- `cdn.bsky.app/img/<kind>/plain/<did>/<cid>` URLs are fetched from the
|
||||
account's PDS with `com.atproto.sync.getBlob`; the PDS comes from the DID
|
||||
document (`plc.directory` for `did:plc`, `/.well-known/did.json` for
|
||||
`did:web`). Both requests use `safe_get`. The cache file is still named
|
||||
after the original URL.
|
||||
|
||||
## Verification
|
||||
|
||||
- `tests/test_backfeed.py::test_avatars_from_own_gotosocial_and_bluesky_pds`:
|
||||
own GoToSocial on a private address is fetched, a Bluesky CDN URL is
|
||||
fetched from its PDS, another private address is still refused.
|
||||
- Checked against the real PLC directory and a bsky.network PDS.
|
||||
- From the production server: getBlob over IPv4 returns the image (200).
|
||||
@@ -0,0 +1,32 @@
|
||||
# v2.13.2: Reaction counts and web actions
|
||||
|
||||
## Why
|
||||
|
||||
With backfeed live, notes showed "5 Likes" and "1 Repost" headings over
|
||||
rows of faces (blank until avatars were saved), plus a separate
|
||||
"On Threads: 1 like" line. The owner asked for the usual presentation: a
|
||||
flat count under the post, with who on hover. Web actions
|
||||
(https://indieweb.org/webactions) fit the same line.
|
||||
|
||||
## What changed
|
||||
|
||||
- `templates/partials/responses.html`: `facepile` and `silo_counts` are
|
||||
replaced by `reactions` / `reaction`. Each count is a focusable
|
||||
`.reaction` with an SVG icon; its `.reaction-who` card lists the people
|
||||
(avatar, name, silo) inside a scrollable `.reaction-people`, Threads'
|
||||
anonymous counts, and an `<indie-action>` with links to the crossposts.
|
||||
The reply count has no people list (replies are listed below).
|
||||
- `templates/note.html`: the counts line follows the note's footer; the
|
||||
Responses section shows only when there are replies, quotes or mentions.
|
||||
- `static/css/style.css`: facepile styles replaced by `.reactions`; the card
|
||||
opens on `:hover` and `:focus-within` (tap on touch screens).
|
||||
- `starpunk/avatars.py`: a redirect from the own GoToSocial server (media it
|
||||
hasn't cached) is followed through `safe_get`.
|
||||
|
||||
## Verification
|
||||
|
||||
- `tests/test_backfeed.py` (24 tests): totals include Threads' anonymous
|
||||
count, `<indie-action>` and crosspost links present; redirected avatar
|
||||
saved. Microformats, template and route tests pass.
|
||||
- Rendered locally in headless Chromium in dark and light themes with the
|
||||
like and reply cards open.
|
||||
@@ -74,16 +74,6 @@ None open.
|
||||
- **Where**: `templates/base.html`
|
||||
- **Source**: Documentation refresh 2026-11-03
|
||||
|
||||
### LICENSE placeholder
|
||||
- **Description**: `LICENSE` still reads "Copyright (c) 2024 [Your Name]". Fill in the holder and year.
|
||||
- **Where**: `LICENSE`
|
||||
- **Source**: Documentation refresh 2026-11-03 (rest of that housekeeping item done in v2.11.0)
|
||||
|
||||
### Backfeed responses to crossposts
|
||||
- **Description**: Idea: bring likes, boosts and replies to StarPunk's own crossposts back to the note as responses. Two options: poll GoToSocial (the token would need a `read` scope), or rely on Bridgy, which sends webmentions. Since v2.8.0, StarPunk ignores webmentions *from* a note's own crossposts, so that rule would need to change to work with Bridgy.
|
||||
- **Where**: `starpunk/syndication.py`, `starpunk/webmention_worker.py`
|
||||
- **Source**: v2.8.0 microformats2 review discussion
|
||||
|
||||
### Tag-filtered feeds
|
||||
- **Description**: Tag pages are h-feeds and WebSub topics, but there are no RSS, Atom or JSON feeds per tag (for example `/tag/python/feed.atom`). Tag pages also have no pagination, and the editor has no tag autocomplete.
|
||||
- **Where**: `starpunk/routes/public.py`, `starpunk/feeds/`, `templates/admin/`
|
||||
@@ -185,6 +175,8 @@ None open.
|
||||
|
||||
These items were removed from the backlog because they have shipped:
|
||||
|
||||
- LICENSE: MIT, copyright Phil Skentelbery 2025-2026 (2026-11-03)
|
||||
|
||||
- v2.11.0: SQLite foreign keys enforced (orphans cleaned up by migration 018); CSRF tokens on admin, logout and consent forms; IndieAuth client and crosspost link-card fetches moved onto the SSRF-safe fetcher; `FLASK_DEBUG`/`FLASK_ENV` default to off/production; rate limits shared by all processes (and old windows pruned, so the webmention limiter no longer grows); dependencies upgraded (Python 3.14, no known vulnerabilities); stale `compose.yaml`, duplicate `dev_auth.py` and `test.ini` removed or fixed; `lxml`/`python-dateutil` pinned, `pytest` out of the runtime requirements
|
||||
|
||||
- POSSE: GoToSocial (v2.1.0), Bluesky (v2.2.0), Threads (v2.6.0)
|
||||
|
||||
@@ -48,3 +48,4 @@ Future work and per-release plans. See `../DOCUMENTATION.md` (PROJECTPLAN).
|
||||
- **[v2.10.1](v2.10.1/RELEASE.md)** - WebSub hardening (Released 2026-11-03)
|
||||
- **[v2.10.2](v2.10.2/RELEASE.md)** - Reply context fetch fix (Released 2026-11-03)
|
||||
- **[v2.11.0](v2.11.0/RELEASE.md)** - Maintenance: dependency upgrades, High backlog items (Released 2026-11-03)
|
||||
- **[v2.12.0](v2.12.0/RELEASE.md)** - Metrics dashboard removed (Released 2026-11-03)
|
||||
@@ -0,0 +1,21 @@
|
||||
# v2.12.0
|
||||
|
||||
- **Date**: 2026-11-03
|
||||
- **Status**: Released
|
||||
- **Focus**: Remove the metrics dashboard
|
||||
|
||||
## Removed
|
||||
|
||||
- Admin metrics dashboard (`/admin/metrics-dashboard`) and `/admin/metrics`
|
||||
- The `starpunk/monitoring` package: request timing, query timing, memory monitor thread, feed counters
|
||||
- `METRICS_*` settings and the `psutil` dependency
|
||||
|
||||
## Kept
|
||||
|
||||
- `/health`, `/admin/health` (without the metrics section), logs with correlation IDs
|
||||
|
||||
## Related
|
||||
|
||||
- [CHANGELOG](../../../CHANGELOG.md)
|
||||
- [ADR-072](../../decisions/ADR-072-remove-metrics-dashboard.md)
|
||||
- [Design note](../../design/v2.12.0/2026-11-03-remove-metrics.md)
|
||||
@@ -84,10 +84,9 @@ banner to every page. It sets the `starpunk_session` cookie for
|
||||
server.
|
||||
|
||||
Each app process also starts background threads: the webmention worker,
|
||||
the WebSub hub worker, the memory monitor and, when a crossposting target
|
||||
is configured, the syndication worker. Turn off what you don't need locally
|
||||
with `WEBMENTION_WORKER_ENABLED=false`, `WEBSUB_ENABLED=false` and
|
||||
`METRICS_ENABLED=false`. Leave `GOTOSOCIAL_*`, `BLUESKY_*` and `THREADS_*`
|
||||
the WebSub hub worker and, when a crossposting target is configured, the
|
||||
syndication worker. Turn off what you don't need locally with
|
||||
`WEBMENTION_WORKER_ENABLED=false` and `WEBSUB_ENABLED=false`. Leave `GOTOSOCIAL_*`, `BLUESKY_*` and `THREADS_*`
|
||||
empty unless you mean to post to those accounts.
|
||||
|
||||
Production runs the same app under gunicorn (`gunicorn app:app`, see `CMD`
|
||||
@@ -112,7 +111,7 @@ SESSION_SECRET=test uv run pytest tests/test_micropub.py -k delete -x
|
||||
`ADMIN_ME=https://test.example.com`; set them too to reproduce CI exactly.
|
||||
- `tests/conftest.py` builds each app with `TESTING=True` and a temporary
|
||||
`DATA_PATH` and database, so tests never touch `./data`. Under
|
||||
`TESTING=True` the background workers and memory monitor don't start and
|
||||
`TESTING=True` the background workers don't start and
|
||||
the production OIDC check is skipped.
|
||||
- `tests/test_migration_race_condition.py` is slow (about 5 minutes). While
|
||||
iterating, add `--ignore=tests/test_migration_race_condition.py`; run the
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
**Last Updated**: 2026-11-03 (v2.10.1)
|
||||
|
||||
StarPunk uses [Semantic Versioning 2.0.0](https://semver.org/):
|
||||
`MAJOR.MINOR.PATCH`, currently **2.10.1**. Every release is a git tag
|
||||
`MAJOR.MINOR.PATCH`, currently **2.13.2**. Every release is a git tag
|
||||
`vX.Y.Z` and a `CHANGELOG.md` entry. Where the number is stored and how to
|
||||
write the changelog entry is in
|
||||
[version-implementation-guide.md](version-implementation-guide.md); the
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
-- Migration 020: Backfeed of responses to crossposts (v2.13.0, ADR-073)
|
||||
--
|
||||
-- webmentions: where a response came from (origin, remote_id) and a new
|
||||
-- 'quote' type. SQLite can't change a CHECK constraint, so the table is
|
||||
-- rebuilt (as in migration 003). Nothing references webmentions.
|
||||
--
|
||||
-- syndications: when each copy's responses were last checked and are next
|
||||
-- due, a lease so one gunicorn worker checks a copy, and Threads counts.
|
||||
|
||||
CREATE TABLE webmentions_new (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
source_url TEXT NOT NULL,
|
||||
target_url TEXT NOT NULL,
|
||||
target_note_id INTEGER,
|
||||
mention_type TEXT NOT NULL DEFAULT 'mention'
|
||||
CHECK (mention_type IN ('reply', 'like', 'repost', 'bookmark', 'mention', 'quote')),
|
||||
author_name TEXT,
|
||||
author_url TEXT,
|
||||
author_photo TEXT,
|
||||
content_snippet TEXT,
|
||||
validation_status TEXT NOT NULL DEFAULT 'pending'
|
||||
CHECK (validation_status IN ('pending', 'valid', 'invalid', 'error')),
|
||||
moderation_status TEXT NOT NULL DEFAULT 'pending'
|
||||
CHECK (moderation_status IN ('pending', 'approved', 'hidden', 'deleted')),
|
||||
validation_error TEXT,
|
||||
validated_at TIMESTAMP,
|
||||
received_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
source_post_url TEXT,
|
||||
source_published TIMESTAMP,
|
||||
origin TEXT NOT NULL DEFAULT 'webmention',
|
||||
remote_id TEXT,
|
||||
FOREIGN KEY (target_note_id) REFERENCES notes(id) ON DELETE SET NULL,
|
||||
UNIQUE(source_url, target_url)
|
||||
);
|
||||
|
||||
INSERT INTO webmentions_new (
|
||||
id, source_url, target_url, target_note_id, mention_type, author_name, author_url,
|
||||
author_photo, content_snippet, validation_status, moderation_status, validation_error,
|
||||
validated_at, received_at, updated_at, source_post_url, source_published
|
||||
)
|
||||
SELECT
|
||||
id, source_url, target_url, target_note_id, mention_type, author_name, author_url,
|
||||
author_photo, content_snippet, validation_status, moderation_status, validation_error,
|
||||
validated_at, received_at, updated_at, source_post_url, source_published
|
||||
FROM webmentions;
|
||||
|
||||
DROP TABLE webmentions;
|
||||
|
||||
ALTER TABLE webmentions_new RENAME TO webmentions;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_webmentions_target_note ON webmentions(target_note_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_webmentions_moderation ON webmentions(moderation_status);
|
||||
CREATE INDEX IF NOT EXISTS idx_webmentions_validation ON webmentions(validation_status);
|
||||
CREATE INDEX IF NOT EXISTS idx_webmentions_received ON webmentions(received_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_webmentions_note_origin ON webmentions(target_note_id, origin);
|
||||
|
||||
ALTER TABLE syndications ADD COLUMN backfeed_next_at TIMESTAMP;
|
||||
ALTER TABLE syndications ADD COLUMN backfeed_checked_at TIMESTAMP;
|
||||
ALTER TABLE syndications ADD COLUMN backfeed_claimed_until TIMESTAMP;
|
||||
ALTER TABLE syndications ADD COLUMN backfeed_attempts INTEGER NOT NULL DEFAULT 0;
|
||||
ALTER TABLE syndications ADD COLUMN backfeed_error TEXT;
|
||||
ALTER TABLE syndications ADD COLUMN backfeed_backfill INTEGER NOT NULL DEFAULT 0;
|
||||
ALTER TABLE syndications ADD COLUMN likes_count INTEGER;
|
||||
ALTER TABLE syndications ADD COLUMN reposts_count INTEGER;
|
||||
ALTER TABLE syndications ADD COLUMN quotes_count INTEGER;
|
||||
@@ -22,6 +22,3 @@ python-dotenv==1.2.*
|
||||
# Images
|
||||
Pillow==13.0.*
|
||||
pillow-heif==1.8.*
|
||||
|
||||
# Monitoring
|
||||
psutil==7.2.*
|
||||
+10
-19
@@ -142,12 +142,6 @@ def create_app(config=None):
|
||||
# Initialize connection pool
|
||||
init_pool(app)
|
||||
|
||||
# Setup HTTP metrics middleware (v1.1.2 Phase 1)
|
||||
if app.config.get('METRICS_ENABLED', True):
|
||||
from starpunk.monitoring import setup_http_metrics
|
||||
setup_http_metrics(app)
|
||||
app.logger.info("HTTP metrics middleware enabled")
|
||||
|
||||
# Initialize feed cache (v1.1.2 Phase 3)
|
||||
if app.config.get('FEED_CACHE_ENABLED', True):
|
||||
from starpunk.feeds import configure_cache
|
||||
@@ -223,6 +217,14 @@ def create_app(config=None):
|
||||
from starpunk.utils import iso_utc
|
||||
app.add_template_filter(iso_utc, 'iso_utc')
|
||||
|
||||
# Template filter - local copy of a response author's avatar, or None (v2.13.0)
|
||||
@app.template_filter('local_avatar')
|
||||
def local_avatar_filter(url):
|
||||
from flask import url_for
|
||||
from starpunk.avatars import cached_name
|
||||
name = cached_name(app.config.get('DATA_PATH', 'data'), url)
|
||||
return url_for('public.avatar', name=name) if name else None
|
||||
|
||||
# Template filter - URL to color for facepile (v1.6.0)
|
||||
@app.template_filter('url_to_color')
|
||||
def url_to_color_filter(url):
|
||||
@@ -323,17 +325,6 @@ def create_app(config=None):
|
||||
|
||||
register_error_handlers(app)
|
||||
|
||||
# Start memory monitor thread (v1.1.2 Phase 1)
|
||||
# Per CQ5: Skip in test mode
|
||||
if app.config.get('METRICS_ENABLED', True) and not app.config.get('TESTING', False):
|
||||
from starpunk.monitoring import MemoryMonitor
|
||||
memory_monitor = MemoryMonitor(interval=app.config.get('METRICS_MEMORY_INTERVAL', 30))
|
||||
memory_monitor.start()
|
||||
app.memory_monitor = memory_monitor
|
||||
app.logger.info(f"Memory monitor started (interval={memory_monitor.interval}s)")
|
||||
# No teardown_appcontext stop: that ran after every request and stopped
|
||||
# the monitor straight away (v2.10.0). It is a daemon thread.
|
||||
|
||||
# Start webmention worker thread (v1.6.0 Phase 2)
|
||||
# Per background-worker.md: Skip in test mode
|
||||
if app.config.get('WEBMENTION_WORKER_ENABLED', True) and not app.config.get('TESTING', False):
|
||||
@@ -474,5 +465,5 @@ def create_app(config=None):
|
||||
|
||||
# Package version (Semantic Versioning 2.0.0)
|
||||
# See docs/standards/versioning-strategy.md for details
|
||||
__version__ = "2.11.0"
|
||||
__version_info__ = (2, 11, 0)
|
||||
__version__ = "2.13.2"
|
||||
__version_info__ = (2, 13, 2)
|
||||
@@ -0,0 +1,217 @@
|
||||
"""
|
||||
Local copies of response authors' avatars (v2.13.0, ADR-073 Decision 6)
|
||||
|
||||
Pages used to show other sites' avatar URLs directly, which tells those
|
||||
sites who is reading, and Bluesky and Threads avatar URLs change or expire.
|
||||
Each avatar is fetched once (SSRF-safe), shrunk to a 96 px WebP and saved
|
||||
as DATA_PATH/avatars/<sha256 of its URL>.webp. Pages show the saved copy,
|
||||
or the coloured initial until there is one.
|
||||
|
||||
Paths and URLs are passed in, so the webmention worker can use it. Fetching
|
||||
needs an app context (the SSRF-safe fetcher reads DEV_MODE).
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import logging
|
||||
import re
|
||||
import sqlite3
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SIZE = 96
|
||||
MAX_BYTES = 1024 * 1024
|
||||
# Bluesky avatar on its CDN: .../img/<kind>/plain/<did>/<cid>[@jpeg]
|
||||
BSKY_CDN_RE = re.compile(
|
||||
r"^https://cdn\.bsky\.app/img/[a-z_]+/plain/(did:(?:plc|web):[A-Za-z0-9._:%-]+)/([a-z2-7]+)(?:@\w+)?$")
|
||||
NAME_RE = re.compile(r"^[0-9a-f]{64}\.webp$")
|
||||
CLEANUP_EVERY = 7 * 24 * 3600
|
||||
|
||||
|
||||
def name_for(url: str) -> str:
|
||||
return hashlib.sha256(url.encode()).hexdigest() + ".webp"
|
||||
|
||||
|
||||
def folder(data_path) -> Path:
|
||||
return Path(data_path) / "avatars"
|
||||
|
||||
|
||||
def cached_name(data_path, url: Optional[str]) -> Optional[str]:
|
||||
"""File name of the saved copy of this avatar, if there is one"""
|
||||
if not url or not url.startswith(("http://", "https://")):
|
||||
return None
|
||||
name = name_for(url)
|
||||
return name if (folder(data_path) / name).is_file() else None
|
||||
|
||||
|
||||
def fetch(data_path, url: Optional[str], site_url: str) -> bool:
|
||||
"""
|
||||
Save a copy of an avatar unless there is one. Returns True if one exists
|
||||
afterwards. Failures are logged and the page keeps showing the initial.
|
||||
"""
|
||||
if not url or not url.startswith(("http://", "https://")):
|
||||
return False
|
||||
path = folder(data_path) / name_for(url)
|
||||
if path.is_file():
|
||||
return True
|
||||
from PIL import Image
|
||||
|
||||
try:
|
||||
with Image.open(io.BytesIO(_download(url, site_url))) as image:
|
||||
image.load()
|
||||
image = image.convert("RGBA" if "A" in image.getbands() else "RGB")
|
||||
side = min(image.size)
|
||||
left, top = (image.width - side) // 2, (image.height - side) // 2
|
||||
image = image.crop((left, top, left + side, top + side))
|
||||
image = image.resize((SIZE, SIZE), Image.LANCZOS)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_suffix(".tmp")
|
||||
image.save(tmp, "WEBP", quality=80)
|
||||
tmp.replace(path)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.info(f"Could not save avatar {url}: {type(e).__name__}: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def _own_instance_host() -> Optional[str]:
|
||||
"""Host of the GoToSocial server StarPunk posts to (it may resolve to an internal address)"""
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from flask import current_app
|
||||
return urlparse(current_app.config.get("GOTOSOCIAL_URL") or "").hostname
|
||||
|
||||
|
||||
def _get(url: str, headers: dict) -> bytes:
|
||||
"""GET a public URL through the SSRF-safe fetcher"""
|
||||
from starpunk.webmention_ssrf import safe_get
|
||||
|
||||
result = safe_get(url, headers=headers, max_bytes=MAX_BYTES, max_redirects=5,
|
||||
timeout=10.0, truncate=False)
|
||||
if result.status_code >= 400:
|
||||
raise ValueError(f"HTTP {result.status_code}")
|
||||
return result.content
|
||||
|
||||
|
||||
def _pds(did: str, headers: dict) -> str:
|
||||
"""The PDS that hosts a Bluesky account, from its DID document"""
|
||||
if did.startswith("did:plc:"):
|
||||
doc_url = f"https://plc.directory/{did}"
|
||||
else: # did:web:<host>
|
||||
doc_url = f"https://{did[len('did:web:'):]}/.well-known/did.json"
|
||||
import json
|
||||
|
||||
for service in json.loads(_get(doc_url, headers)).get("service") or []:
|
||||
if service.get("id", "").endswith("#atproto_pds") and service.get("serviceEndpoint"):
|
||||
return service["serviceEndpoint"].rstrip("/")
|
||||
raise ValueError(f"no PDS for {did}")
|
||||
|
||||
|
||||
def _download(url: str, site_url: str) -> bytes:
|
||||
"""
|
||||
The avatar's bytes (v2.13.1):
|
||||
- from StarPunk's own GoToSocial directly, as the crossposting client
|
||||
does: behind the same proxy it resolves to an internal address, which
|
||||
the SSRF-safe fetcher refuses;
|
||||
- Bluesky CDN avatars from the account's PDS (com.atproto.sync.getBlob),
|
||||
because the CDN refuses some servers' IPv4 addresses;
|
||||
- anything else through the SSRF-safe fetcher.
|
||||
"""
|
||||
from urllib.parse import quote, urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
from starpunk import webmention_ssrf
|
||||
from starpunk.webmention_ssrf import user_agent
|
||||
|
||||
headers = {"User-Agent": user_agent(site_url)}
|
||||
own = _own_instance_host()
|
||||
if own and urlparse(url).hostname == own:
|
||||
with httpx.Client(timeout=10.0, transport=webmention_ssrf.TRANSPORT) as http:
|
||||
with http.stream("GET", url, headers=headers) as response:
|
||||
if response.is_redirect:
|
||||
# GoToSocial sends media it hasn't cached to the original
|
||||
# server: that one is fetched like any outside URL (v2.13.2)
|
||||
from urllib.parse import urljoin
|
||||
return _get(urljoin(url, response.headers["location"]), headers)
|
||||
if response.status_code >= 400:
|
||||
raise ValueError(f"HTTP {response.status_code}")
|
||||
data = b""
|
||||
for chunk in response.iter_bytes():
|
||||
data += chunk
|
||||
if len(data) > MAX_BYTES:
|
||||
raise ValueError("too large")
|
||||
return data
|
||||
match = BSKY_CDN_RE.match(url)
|
||||
if match:
|
||||
did, cid = match.groups()
|
||||
url = (f"{_pds(did, headers)}/xrpc/com.atproto.sync.getBlob"
|
||||
f"?did={quote(did, safe=':')}&cid={cid}")
|
||||
return _get(url, headers)
|
||||
|
||||
|
||||
def fill_missing(db_path, data_path, site_url: str, limit: int = 20) -> int:
|
||||
"""Save avatars of shown responses that have no copy yet. Returns how many were tried."""
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
urls = [row[0] for row in conn.execute(
|
||||
"""
|
||||
SELECT DISTINCT author_photo FROM webmentions
|
||||
WHERE author_photo IS NOT NULL AND author_photo != ''
|
||||
AND validation_status = 'valid' AND moderation_status = 'approved'
|
||||
ORDER BY id DESC
|
||||
"""
|
||||
)]
|
||||
finally:
|
||||
conn.close()
|
||||
tried = 0
|
||||
for url in urls:
|
||||
if tried >= limit:
|
||||
break
|
||||
if cached_name(data_path, url) or not url.startswith(("http://", "https://")):
|
||||
continue
|
||||
fetch(data_path, url, site_url)
|
||||
tried += 1
|
||||
return tried
|
||||
|
||||
|
||||
def remove_unused(db_path, data_path) -> int:
|
||||
"""Delete saved avatars no stored response uses. Returns how many were removed."""
|
||||
directory = folder(data_path)
|
||||
if not directory.is_dir():
|
||||
return 0
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
used = {name_for(row[0]) for row in conn.execute(
|
||||
"SELECT DISTINCT author_photo FROM webmentions WHERE author_photo IS NOT NULL")}
|
||||
finally:
|
||||
conn.close()
|
||||
removed = 0
|
||||
for path in directory.iterdir():
|
||||
if NAME_RE.match(path.name) and path.name not in used:
|
||||
path.unlink(missing_ok=True)
|
||||
removed += 1
|
||||
return removed
|
||||
|
||||
|
||||
class Upkeep:
|
||||
"""Runs fill_missing every `fill_every` seconds and remove_unused weekly"""
|
||||
|
||||
def __init__(self, fill_every: int = 600):
|
||||
self.fill_every = fill_every
|
||||
self._filled_at = 0.0
|
||||
self._cleaned_at = 0.0
|
||||
|
||||
def run(self, db_path, data_path, site_url: str) -> None:
|
||||
now = time.time()
|
||||
if now - self._filled_at >= self.fill_every:
|
||||
self._filled_at = now
|
||||
fill_missing(db_path, data_path, site_url)
|
||||
if now - self._cleaned_at >= CLEANUP_EVERY:
|
||||
self._cleaned_at = now
|
||||
removed = remove_unused(db_path, data_path)
|
||||
if removed:
|
||||
logger.info(f"Removed {removed} unused avatars")
|
||||
@@ -0,0 +1,456 @@
|
||||
"""
|
||||
Backfeed of responses to crossposts (v2.13.0, ADR-073)
|
||||
|
||||
StarPunk asks each silo for the responses to its own copies of a note
|
||||
(likes, reposts, quotes and replies) and stores them in `webmentions`, so
|
||||
moderation, the Responses section and re-sending webmentions when a note's
|
||||
responses change (spec 3.1.4) work as they do for received webmentions.
|
||||
|
||||
Each synced copy of a published note is checked on a schedule that slows as
|
||||
the note ages, and not at all after BACKFEED_MAX_AGE_DAYS. A backfill
|
||||
checks every copy once, whatever its age. Checks run in the syndication
|
||||
worker's loop (`process_due`), one copy at a time under a lease.
|
||||
|
||||
A check that read the whole list marks responses that are no longer there
|
||||
as invalid, so withdrawn likes and deleted replies drop off the page.
|
||||
"""
|
||||
|
||||
import html
|
||||
import random
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from flask import current_app
|
||||
|
||||
from starpunk.database import get_db
|
||||
|
||||
DAY = 24 * 3600
|
||||
SCHEDULE = [ # (note younger than, check every), in seconds
|
||||
(DAY, 15 * 60),
|
||||
(7 * DAY, 3600),
|
||||
(30 * DAY, DAY),
|
||||
]
|
||||
LATE_INTERVAL = 7 * DAY # from 30 days until the cutoff
|
||||
CLAIM_MINUTES = 5
|
||||
MAX_ATTEMPTS = 8
|
||||
BACKFILL_GAP = 5 # seconds between backfill checks, across workers
|
||||
SILO_NAMES = {"gotosocial": "GoToSocial", "bluesky": "Bluesky", "threads": "Threads"}
|
||||
OPT_OUT_RE = re.compile(r"#no(bridge|bot)\b", re.IGNORECASE)
|
||||
|
||||
# Silos whose credential was refused this run: target id -> message
|
||||
denied: Dict[str, str] = {}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Response:
|
||||
"""One like, repost, quote or reply on a silo"""
|
||||
kind: str # like, repost, quote, reply
|
||||
source_url: str # its URL (likes/reposts: the copy + #liked-by-<id>)
|
||||
remote_id: str # silo id of the response, or of the actor
|
||||
author_id: str = ""
|
||||
author_name: str = ""
|
||||
author_url: str = ""
|
||||
author_photo: Optional[str] = None
|
||||
author_bio: str = ""
|
||||
opted_out: bool = False # set by the client from silo-specific signals
|
||||
content_html: Optional[str] = None
|
||||
published: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Harvest:
|
||||
"""What one check found. complete=False: some list was cut short."""
|
||||
responses: List[Response]
|
||||
complete: bool = True
|
||||
counts: Optional[Dict[str, int]] = None # Threads likes/reposts/quotes
|
||||
|
||||
|
||||
# --- Text -----------------------------------------------------------------
|
||||
|
||||
def text_html(text: str) -> str:
|
||||
"""Plain text as HTML: escaped, URLs linked, line breaks kept"""
|
||||
escaped = html.escape(text)
|
||||
linked = re.sub(r"https?://[^\s<>&\"]+",
|
||||
lambda m: f'<a href="{m.group(0)}">{m.group(0)}</a>', escaped)
|
||||
return "<p>" + linked.replace("\n", "<br>") + "</p>" if text.strip() else ""
|
||||
|
||||
|
||||
def facets_html(text: str, facets: list) -> str:
|
||||
"""Bluesky post text as HTML, with its link facets (byte ranges) as links"""
|
||||
data = text.encode()
|
||||
links = []
|
||||
for facet in facets:
|
||||
index = facet.get("index") or {}
|
||||
for feature in facet.get("features") or []:
|
||||
uri = feature.get("uri") or ""
|
||||
if feature.get("$type") == "app.bsky.richtext.facet#link" and uri.startswith(("http://", "https://")):
|
||||
links.append((index.get("byteStart", 0), index.get("byteEnd", 0), uri))
|
||||
links.sort()
|
||||
parts, pos = [], 0
|
||||
for start, end, uri in links:
|
||||
if start < pos or end > len(data) or start >= end:
|
||||
continue
|
||||
parts.append(html.escape(data[pos:start].decode(errors="ignore")))
|
||||
label = html.escape(data[start:end].decode(errors="ignore"))
|
||||
parts.append(f'<a href="{html.escape(uri, quote=True)}">{label}</a>')
|
||||
pos = end
|
||||
parts.append(html.escape(data[pos:].decode(errors="ignore")))
|
||||
body = "".join(parts)
|
||||
return "<p>" + body.replace("\n", "<br>") + "</p>" if text.strip() else ""
|
||||
|
||||
|
||||
# --- Scheduling --------------------------------------------------------------
|
||||
|
||||
def interval_for(age_seconds: float, max_age_days: int) -> Optional[int]:
|
||||
"""Seconds until a note of this age is checked again; None past the cutoff"""
|
||||
if age_seconds >= max_age_days * DAY:
|
||||
return None
|
||||
for younger_than, every in SCHEDULE:
|
||||
if age_seconds < younger_than:
|
||||
return every
|
||||
return LATE_INTERVAL
|
||||
|
||||
|
||||
def _age_seconds(created_at) -> float:
|
||||
if isinstance(created_at, str):
|
||||
created_at = datetime.fromisoformat(created_at.replace("Z", "+00:00"))
|
||||
if created_at.tzinfo is None:
|
||||
created_at = created_at.replace(tzinfo=timezone.utc)
|
||||
return (datetime.now(timezone.utc) - created_at).total_seconds()
|
||||
|
||||
|
||||
_ELIGIBLE = """
|
||||
FROM syndications s JOIN notes n ON n.id = s.note_id
|
||||
WHERE s.state = 'synced' AND s.enabled = 1 AND s.remote_id IS NOT NULL
|
||||
AND n.published = 1 AND n.deleted_at IS NULL
|
||||
AND n.like_of IS NULL AND n.repost_of IS NULL
|
||||
"""
|
||||
|
||||
|
||||
def _claim(db, row_id: int) -> bool:
|
||||
cursor = db.execute(
|
||||
f"""
|
||||
UPDATE syndications SET backfeed_claimed_until = datetime('now', '+{CLAIM_MINUTES} minutes')
|
||||
WHERE id = ? AND (backfeed_claimed_until IS NULL OR backfeed_claimed_until < datetime('now'))
|
||||
""",
|
||||
(row_id,),
|
||||
)
|
||||
db.commit()
|
||||
return cursor.rowcount == 1
|
||||
|
||||
|
||||
def _backfill_turn(db) -> bool:
|
||||
"""Take the next backfill slot: at most one check every BACKFILL_GAP seconds"""
|
||||
cursor = db.execute(
|
||||
f"""
|
||||
UPDATE site_state SET value = datetime('now')
|
||||
WHERE key = 'backfeed_backfill_last' AND value <= datetime('now', '-{BACKFILL_GAP} seconds')
|
||||
"""
|
||||
)
|
||||
if cursor.rowcount == 0:
|
||||
cursor = db.execute(
|
||||
"INSERT OR IGNORE INTO site_state (key, value) VALUES ('backfeed_backfill_last', datetime('now'))")
|
||||
db.commit()
|
||||
return cursor.rowcount == 1
|
||||
|
||||
|
||||
def _due(db, limit: int) -> list:
|
||||
"""Routine checks due now; a backfill check only when none are"""
|
||||
max_age = int(current_app.config.get("BACKFEED_MAX_AGE_DAYS", 90))
|
||||
lease = "(s.backfeed_claimed_until IS NULL OR s.backfeed_claimed_until < datetime('now'))"
|
||||
rows = db.execute(
|
||||
f"""
|
||||
SELECT s.*, n.created_at AS note_created, n.slug AS note_slug {_ELIGIBLE}
|
||||
AND {lease}
|
||||
AND n.created_at >= datetime('now', ?)
|
||||
AND (s.backfeed_next_at IS NULL OR s.backfeed_next_at <= datetime('now'))
|
||||
ORDER BY s.backfeed_next_at IS NOT NULL, s.backfeed_next_at LIMIT ?
|
||||
""",
|
||||
(f"-{max_age} days", limit),
|
||||
).fetchall()
|
||||
if rows:
|
||||
return rows
|
||||
backfill = db.execute(
|
||||
f"""
|
||||
SELECT s.*, n.created_at AS note_created, n.slug AS note_slug {_ELIGIBLE}
|
||||
AND {lease} AND s.backfeed_backfill = 1
|
||||
AND (s.backfeed_next_at IS NULL OR s.backfeed_next_at <= datetime('now'))
|
||||
ORDER BY s.id LIMIT 1
|
||||
"""
|
||||
).fetchall()
|
||||
if backfill and _backfill_turn(db):
|
||||
return backfill
|
||||
return []
|
||||
|
||||
|
||||
def _finish(db, row) -> None:
|
||||
max_age = int(current_app.config.get("BACKFEED_MAX_AGE_DAYS", 90))
|
||||
every = interval_for(_age_seconds(row["note_created"]), max_age)
|
||||
if every is None:
|
||||
next_at = None
|
||||
else:
|
||||
next_at = f"+{int(every * random.uniform(1.0, 1.1))} seconds"
|
||||
db.execute(
|
||||
"""
|
||||
UPDATE syndications
|
||||
SET backfeed_checked_at = datetime('now'), backfeed_claimed_until = NULL,
|
||||
backfeed_attempts = 0, backfeed_error = NULL, backfeed_backfill = 0,
|
||||
backfeed_next_at = CASE WHEN ? IS NULL THEN NULL ELSE datetime('now', ?) END
|
||||
WHERE id = ?
|
||||
""",
|
||||
(next_at, next_at, row["id"]),
|
||||
)
|
||||
db.commit()
|
||||
|
||||
|
||||
def _fail(db, row, message: str) -> None:
|
||||
"""Back off; after MAX_ATTEMPTS drop any backfill and try again in a month"""
|
||||
attempts = row["backfeed_attempts"] + 1
|
||||
stop = attempts >= MAX_ATTEMPTS
|
||||
delay = 30 * DAY if stop else min(30 * (2 ** attempts), 6 * 3600)
|
||||
db.execute(
|
||||
f"""
|
||||
UPDATE syndications
|
||||
SET backfeed_attempts = ?, backfeed_error = ?, backfeed_claimed_until = NULL,
|
||||
backfeed_backfill = CASE WHEN ? THEN 0 ELSE backfeed_backfill END,
|
||||
backfeed_next_at = datetime('now', '+{delay} seconds')
|
||||
WHERE id = ?
|
||||
""",
|
||||
(attempts, message[:500], 1 if stop else 0, row["id"]),
|
||||
)
|
||||
db.commit()
|
||||
current_app.logger.warning(
|
||||
f"Backfeed {row['target']} for note {row['note_id']} failed (attempt {attempts}): {message}")
|
||||
|
||||
|
||||
# --- Storing -----------------------------------------------------------------
|
||||
|
||||
def _opted_out(response: Response) -> bool:
|
||||
return response.opted_out or bool(OPT_OUT_RE.search(html.unescape(response.author_bio or "")))
|
||||
|
||||
|
||||
def _display_state(db, note_id: int) -> list:
|
||||
return [tuple(r) for r in db.execute(
|
||||
"""
|
||||
SELECT id, mention_type, author_name, author_url, author_photo, content_snippet
|
||||
FROM webmentions
|
||||
WHERE target_note_id = ? AND validation_status = 'valid' AND moderation_status = 'approved'
|
||||
ORDER BY id
|
||||
""",
|
||||
(note_id,),
|
||||
)]
|
||||
|
||||
|
||||
def store(db, note_id: int, target_url: str, origin: str, harvest: Harvest) -> None:
|
||||
"""
|
||||
Upsert a check's responses for one note and silo. With a complete
|
||||
harvest, that silo's responses for the note that weren't found are
|
||||
marked invalid. Responses from people who opted out are removed.
|
||||
"""
|
||||
from starpunk.webmention_processor import _snippet
|
||||
|
||||
moderation = "approved" if current_app.config.get("WEBMENTION_AUTO_APPROVE", True) else "pending"
|
||||
silo = SILO_NAMES.get(origin, origin)
|
||||
keep = [r for r in harvest.responses if not _opted_out(r)]
|
||||
dropped = [r.source_url for r in harvest.responses if _opted_out(r)]
|
||||
|
||||
from starpunk.avatars import cached_name
|
||||
|
||||
before = _display_state(db, note_id)
|
||||
for r in keep:
|
||||
snippet = _snippet(r.content_html) if r.content_html else None
|
||||
# Threads avatar URLs are signed and change on every request: keep
|
||||
# the stored URL while its saved copy exists, so it isn't refetched
|
||||
old = db.execute("SELECT author_photo FROM webmentions WHERE source_url = ? AND target_url = ?",
|
||||
(r.source_url, target_url)).fetchone()
|
||||
if old and old["author_photo"] and origin == "threads" \
|
||||
and cached_name(current_app.config["DATA_PATH"], old["author_photo"]):
|
||||
r.author_photo = old["author_photo"]
|
||||
db.execute(
|
||||
"""
|
||||
INSERT INTO webmentions (
|
||||
source_url, target_url, target_note_id, mention_type, author_name, author_url,
|
||||
author_photo, content_snippet, validation_status, moderation_status,
|
||||
validated_at, source_published, origin, remote_id)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'valid', ?, CURRENT_TIMESTAMP, ?, ?, ?)
|
||||
ON CONFLICT(source_url, target_url) DO UPDATE SET
|
||||
target_note_id = excluded.target_note_id,
|
||||
mention_type = excluded.mention_type,
|
||||
author_name = excluded.author_name,
|
||||
author_url = excluded.author_url,
|
||||
author_photo = excluded.author_photo,
|
||||
content_snippet = excluded.content_snippet,
|
||||
source_published = COALESCE(excluded.source_published, webmentions.source_published),
|
||||
origin = excluded.origin,
|
||||
remote_id = excluded.remote_id,
|
||||
validation_status = 'valid',
|
||||
validation_error = NULL,
|
||||
validated_at = CURRENT_TIMESTAMP,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(r.source_url, target_url, note_id, r.kind, r.author_name or None, r.author_url or None,
|
||||
r.author_photo or None, snippet, moderation, r.published, origin, r.remote_id),
|
||||
)
|
||||
if dropped:
|
||||
marks = ",".join("?" * len(dropped))
|
||||
db.execute(f"DELETE FROM webmentions WHERE origin = ? AND target_note_id = ? AND source_url IN ({marks})",
|
||||
[origin, note_id] + dropped)
|
||||
if harvest.complete:
|
||||
found = [r.source_url for r in keep]
|
||||
marks = ",".join("?" * len(found)) or "''"
|
||||
db.execute(
|
||||
f"""
|
||||
UPDATE webmentions
|
||||
SET validation_status = 'invalid', validation_error = ?, updated_at = CURRENT_TIMESTAMP
|
||||
WHERE target_note_id = ? AND origin = ? AND validation_status = 'valid'
|
||||
AND source_url NOT IN ({marks})
|
||||
""",
|
||||
[f"No longer on {silo}", note_id, origin] + found,
|
||||
)
|
||||
db.commit()
|
||||
|
||||
if _display_state(db, note_id) != before:
|
||||
try:
|
||||
from pathlib import Path
|
||||
|
||||
from starpunk.webmention_sender import note_responses_changed
|
||||
note_responses_changed(Path(current_app.config["DATABASE_PATH"]), [note_id])
|
||||
except Exception as e:
|
||||
current_app.logger.warning(f"Failed to re-send webmentions for note {note_id}: {e}")
|
||||
|
||||
from starpunk import avatars
|
||||
for r in keep:
|
||||
avatars.fetch(current_app.config["DATA_PATH"], r.author_photo,
|
||||
current_app.config.get("SITE_URL", ""))
|
||||
|
||||
|
||||
def _store_counts(db, row_id: int, counts: Optional[dict]) -> None:
|
||||
if counts is None:
|
||||
return
|
||||
db.execute(
|
||||
"UPDATE syndications SET likes_count = ?, reposts_count = ?, quotes_count = ? WHERE id = ?",
|
||||
(counts.get("likes"), counts.get("reposts"), counts.get("quotes"), row_id),
|
||||
)
|
||||
db.commit()
|
||||
|
||||
|
||||
# --- Checking ---------------------------------------------------------------
|
||||
|
||||
def _is_denied(error) -> bool:
|
||||
"""The silo refused the credential (missing scope or bad token)"""
|
||||
from starpunk.syndication import _GRAPH_AUTH
|
||||
|
||||
return getattr(error, "status", None) in (401, 403) or getattr(error, "code", None) in _GRAPH_AUTH
|
||||
|
||||
|
||||
SCOPES = {
|
||||
"gotosocial": "a GoToSocial token with read:statuses and read:accounts (or read)",
|
||||
"bluesky": "a working Bluesky app password",
|
||||
"threads": "a Threads token with threads_read_replies and threads_manage_insights",
|
||||
}
|
||||
|
||||
|
||||
def check(row, target) -> None:
|
||||
"""Check one copy now and store what it found (raises SyndicationError)"""
|
||||
db = get_db(current_app)
|
||||
site = current_app.config["SITE_URL"].rstrip("/")
|
||||
harvest = target.client.responses(row["remote_id"], row["remote_url"] or "")
|
||||
store(db, row["note_id"], f"{site}/note/{row['note_slug']}", row["target"], harvest)
|
||||
_store_counts(db, row["id"], harvest.counts)
|
||||
|
||||
|
||||
def process_due(limit: int = 3) -> int:
|
||||
"""Check up to `limit` copies that are due. Returns how many were handled."""
|
||||
if not current_app.config.get("BACKFEED_ENABLED", False):
|
||||
return 0
|
||||
from starpunk.syndication import SyndicationError, get_targets
|
||||
|
||||
targets = get_targets()
|
||||
db = get_db(current_app)
|
||||
handled = 0
|
||||
for row in _due(db, limit):
|
||||
if row["target"] in denied or row["target"] not in targets:
|
||||
continue
|
||||
if not _claim(db, row["id"]):
|
||||
continue
|
||||
handled += 1
|
||||
try:
|
||||
check(row, targets[row["target"]])
|
||||
except SyndicationError as e:
|
||||
if _is_denied(e):
|
||||
denied[row["target"]] = f"Needs {SCOPES.get(row['target'], 'read access')}: {e}"
|
||||
current_app.logger.error(f"Backfeed for {row['target']} stopped: {denied[row['target']]}")
|
||||
db.execute("UPDATE syndications SET backfeed_claimed_until = NULL WHERE id = ?", (row["id"],))
|
||||
db.commit()
|
||||
continue
|
||||
_fail(db, row, str(e))
|
||||
continue
|
||||
except Exception as e: # one bad copy mustn't stop the worker
|
||||
_fail(db, row, f"unexpected error: {e}")
|
||||
continue
|
||||
_finish(db, row)
|
||||
_note_backfill_done(db)
|
||||
return handled
|
||||
|
||||
|
||||
# --- Backfill and admin ---------------------------------------------------------
|
||||
|
||||
def _state(db, key: str) -> Optional[str]:
|
||||
row = db.execute("SELECT value FROM site_state WHERE key = ?", (key,)).fetchone()
|
||||
return row["value"] if row and row["value"] else None
|
||||
|
||||
|
||||
def _set_state(db, key: str, value: str) -> None:
|
||||
db.execute("INSERT INTO site_state (key, value) VALUES (?, ?) "
|
||||
"ON CONFLICT(key) DO UPDATE SET value = excluded.value", (key, value))
|
||||
|
||||
|
||||
def start_backfill() -> int:
|
||||
"""Mark every eligible copy for one check, whatever its age. Returns how many."""
|
||||
db = get_db(current_app)
|
||||
cursor = db.execute(
|
||||
f"""
|
||||
UPDATE syndications
|
||||
SET backfeed_backfill = 1, backfeed_attempts = 0, backfeed_next_at = datetime('now')
|
||||
WHERE id IN (SELECT s.id {_ELIGIBLE})
|
||||
"""
|
||||
)
|
||||
now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
|
||||
_set_state(db, "backfeed_backfill_started", now)
|
||||
_set_state(db, "backfeed_backfill_finished", "")
|
||||
db.commit()
|
||||
return cursor.rowcount
|
||||
|
||||
|
||||
def _note_backfill_done(db) -> None:
|
||||
if _state(db, "backfeed_backfill_started") and not _state(db, "backfeed_backfill_finished"):
|
||||
left = db.execute("SELECT COUNT(*) FROM syndications WHERE backfeed_backfill = 1").fetchone()[0]
|
||||
if left == 0:
|
||||
_set_state(db, "backfeed_backfill_finished",
|
||||
datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S"))
|
||||
db.commit()
|
||||
|
||||
|
||||
def check_now(syndication_id: int) -> None:
|
||||
"""Check this copy's responses at the next worker pass"""
|
||||
db = get_db(current_app)
|
||||
db.execute("UPDATE syndications SET backfeed_next_at = datetime('now'), backfeed_attempts = 0 "
|
||||
"WHERE id = ?", (syndication_id,))
|
||||
db.commit()
|
||||
|
||||
|
||||
def status() -> dict:
|
||||
"""For the admin pages: backfill progress and silos whose credential was refused"""
|
||||
db = get_db(current_app)
|
||||
return {
|
||||
"enabled": bool(current_app.config.get("BACKFEED_ENABLED", False)),
|
||||
"started": _state(db, "backfeed_backfill_started"),
|
||||
"finished": _state(db, "backfeed_backfill_finished"),
|
||||
"remaining": db.execute(
|
||||
"SELECT COUNT(*) FROM syndications WHERE backfeed_backfill = 1").fetchone()[0],
|
||||
"denied": dict(denied),
|
||||
"errors": db.execute(
|
||||
"SELECT COUNT(*) FROM syndications WHERE backfeed_error IS NOT NULL").fetchone()[0],
|
||||
}
|
||||
+3
-7
@@ -99,13 +99,6 @@ def load_config(app, config_override=None):
|
||||
# Flask MAX_CONTENT_LENGTH limits request body size (matches media.py MAX_FILE_SIZE)
|
||||
app.config["MAX_CONTENT_LENGTH"] = 50 * 1024 * 1024 # 50MB
|
||||
|
||||
# Metrics configuration (v1.1.2 Phase 1)
|
||||
app.config["METRICS_ENABLED"] = os.getenv("METRICS_ENABLED", "true").lower() == "true"
|
||||
app.config["METRICS_SLOW_QUERY_THRESHOLD"] = float(os.getenv("METRICS_SLOW_QUERY_THRESHOLD", "1.0"))
|
||||
app.config["METRICS_SAMPLING_RATE"] = float(os.getenv("METRICS_SAMPLING_RATE", "1.0"))
|
||||
app.config["METRICS_BUFFER_SIZE"] = int(os.getenv("METRICS_BUFFER_SIZE", "1000"))
|
||||
app.config["METRICS_MEMORY_INTERVAL"] = int(os.getenv("METRICS_MEMORY_INTERVAL", "30"))
|
||||
|
||||
# Debug file configuration (v1.5.0 Phase 2)
|
||||
app.config["DEBUG_SAVE_FAILED_UPLOADS"] = os.getenv("DEBUG_SAVE_FAILED_UPLOADS", "false").lower() == "true"
|
||||
app.config["DEBUG_FILE_MAX_AGE_DAYS"] = int(os.getenv("DEBUG_FILE_MAX_AGE_DAYS", "7"))
|
||||
@@ -153,6 +146,9 @@ def load_config(app, config_override=None):
|
||||
app.config["THREADS_MAX_CHARS"] = int(os.getenv("THREADS_MAX_CHARS", "500"))
|
||||
app.config["THREADS_MAX_MEDIA"] = int(os.getenv("THREADS_MAX_MEDIA", "4"))
|
||||
app.config["SYNDICATION_POLL_INTERVAL"] = int(os.getenv("SYNDICATION_POLL_INTERVAL", "10"))
|
||||
# Backfeed of responses to crossposts (v2.13.0, ADR-073)
|
||||
app.config["BACKFEED_ENABLED"] = os.getenv("BACKFEED_ENABLED", "false").lower() == "true"
|
||||
app.config["BACKFEED_MAX_AGE_DAYS"] = int(os.getenv("BACKFEED_MAX_AGE_DAYS", "90"))
|
||||
|
||||
# Apply overrides if provided
|
||||
# CSRF protection for admin and consent forms (v2.11.0); on by default.
|
||||
|
||||
@@ -6,7 +6,7 @@ Provides database initialization and connection pooling
|
||||
Per v1.1.1 Phase 1:
|
||||
- Connection pooling for improved performance (ADR-053)
|
||||
- Request-scoped connections via Flask's g object
|
||||
- Pool statistics for monitoring
|
||||
- Pool statistics
|
||||
"""
|
||||
|
||||
from starpunk.database.init import init_db
|
||||
|
||||
@@ -5,7 +5,7 @@ Per ADR-053 and developer Q&A Q2, CQ1:
|
||||
- Provides connection pooling for improved performance
|
||||
- Integrates with Flask's g object for request-scoped connections
|
||||
- Maintains same interface as get_db() for transparency
|
||||
- Pool statistics available for metrics
|
||||
- Pool statistics (shown in /admin/health)
|
||||
- Wraps connections with MonitoredConnection for timing (v1.1.2 Phase 1)
|
||||
|
||||
Note: Migrations use direct connections (not pooled) for isolation
|
||||
@@ -27,7 +27,7 @@ class ConnectionPool:
|
||||
but this provides connection reuse and request-scoped connection management.
|
||||
"""
|
||||
|
||||
def __init__(self, db_path, pool_size=5, timeout=10.0, slow_query_threshold=1.0, metrics_enabled=True):
|
||||
def __init__(self, db_path, pool_size=5, timeout=10.0):
|
||||
"""
|
||||
Initialize connection pool
|
||||
|
||||
@@ -35,14 +35,10 @@ class ConnectionPool:
|
||||
db_path: Path to SQLite database file
|
||||
pool_size: Maximum number of connections in pool
|
||||
timeout: Timeout for getting connection (seconds)
|
||||
slow_query_threshold: Threshold in seconds for slow query detection (v1.1.2)
|
||||
metrics_enabled: Whether to enable metrics collection (v1.1.2)
|
||||
"""
|
||||
self.db_path = Path(db_path)
|
||||
self.pool_size = pool_size
|
||||
self.timeout = timeout
|
||||
self.slow_query_threshold = slow_query_threshold
|
||||
self.metrics_enabled = metrics_enabled
|
||||
self._pool = deque(maxlen=pool_size)
|
||||
self._lock = Lock()
|
||||
self._stats = {
|
||||
@@ -56,8 +52,6 @@ class ConnectionPool:
|
||||
def _create_connection(self):
|
||||
"""
|
||||
Create a new database connection
|
||||
|
||||
Per CQ1: Wraps connection with MonitoredConnection if metrics enabled
|
||||
"""
|
||||
conn = sqlite3.connect(
|
||||
self.db_path,
|
||||
@@ -73,12 +67,6 @@ class ConnectionPool:
|
||||
conn.execute("PRAGMA foreign_keys=ON")
|
||||
|
||||
self._stats['connections_created'] += 1
|
||||
|
||||
# Wrap with monitoring if enabled (v1.1.2 Phase 1)
|
||||
if self.metrics_enabled:
|
||||
from starpunk.monitoring import MonitoredConnection
|
||||
return MonitoredConnection(conn, self.slow_query_threshold)
|
||||
|
||||
return conn
|
||||
|
||||
def get_connection(self):
|
||||
@@ -171,8 +159,6 @@ def init_pool(app):
|
||||
"""
|
||||
Initialize the connection pool
|
||||
|
||||
Per CQ2: Passes metrics configuration from app config
|
||||
|
||||
Args:
|
||||
app: Flask application instance
|
||||
"""
|
||||
@@ -181,20 +167,8 @@ def init_pool(app):
|
||||
db_path = app.config['DATABASE_PATH']
|
||||
pool_size = app.config.get('DB_POOL_SIZE', 5)
|
||||
timeout = app.config.get('DB_TIMEOUT', 10.0)
|
||||
slow_query_threshold = app.config.get('METRICS_SLOW_QUERY_THRESHOLD', 1.0)
|
||||
metrics_enabled = app.config.get('METRICS_ENABLED', True)
|
||||
|
||||
_pool = ConnectionPool(
|
||||
db_path,
|
||||
pool_size,
|
||||
timeout,
|
||||
slow_query_threshold,
|
||||
metrics_enabled
|
||||
)
|
||||
app.logger.info(
|
||||
f"Database connection pool initialized "
|
||||
f"(size={pool_size}, metrics={'enabled' if metrics_enabled else 'disabled'})"
|
||||
)
|
||||
_pool = ConnectionPool(db_path, pool_size, timeout)
|
||||
app.logger.info(f"Database connection pool initialized (size={pool_size})")
|
||||
|
||||
# Register teardown handler
|
||||
@app.teardown_appcontext
|
||||
|
||||
@@ -18,12 +18,10 @@ Standards:
|
||||
# Standard library imports
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
import time
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
# Local imports
|
||||
from starpunk.models import Note
|
||||
from starpunk.monitoring.business import track_feed_generated
|
||||
|
||||
|
||||
# ATOM namespace
|
||||
@@ -127,10 +125,6 @@ def generate_atom_streaming(
|
||||
# Remove trailing slash from site_url for consistency
|
||||
site_url = site_url.rstrip("/")
|
||||
|
||||
# Track feed generation timing
|
||||
start_time = time.time()
|
||||
item_count = 0
|
||||
|
||||
# Current timestamp for updated
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
@@ -159,8 +153,6 @@ def generate_atom_streaming(
|
||||
# Yield entries (newest first)
|
||||
# Notes from database are already in DESC order (newest first)
|
||||
for note in notes[:limit]:
|
||||
item_count += 1
|
||||
|
||||
# Build permalink URL
|
||||
permalink = f"{site_url}{note.permalink}"
|
||||
|
||||
@@ -229,15 +221,6 @@ def generate_atom_streaming(
|
||||
# Yield closing tag
|
||||
yield '</feed>\n'
|
||||
|
||||
# Track feed generation metrics
|
||||
duration_ms = (time.time() - start_time) * 1000
|
||||
track_feed_generated(
|
||||
format='atom',
|
||||
item_count=item_count,
|
||||
duration_ms=duration_ms,
|
||||
cached=False
|
||||
)
|
||||
|
||||
|
||||
def _escape_xml(text: str) -> str:
|
||||
"""
|
||||
|
||||
@@ -18,12 +18,10 @@ Standards:
|
||||
# Standard library imports
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional, Dict, Any
|
||||
import time
|
||||
import json
|
||||
|
||||
# Local imports
|
||||
from starpunk.models import Note
|
||||
from starpunk.monitoring.business import track_feed_generated
|
||||
|
||||
|
||||
def generate_json_feed(
|
||||
@@ -74,9 +72,6 @@ def generate_json_feed(
|
||||
# Remove trailing slash from site_url for consistency
|
||||
site_url = site_url.rstrip("/")
|
||||
|
||||
# Track feed generation timing
|
||||
start_time = time.time()
|
||||
|
||||
# Build feed object
|
||||
feed = _build_feed_object(
|
||||
site_url=site_url,
|
||||
@@ -88,14 +83,6 @@ def generate_json_feed(
|
||||
# Serialize to JSON (pretty-printed)
|
||||
feed_json = json.dumps(feed, ensure_ascii=False, indent=2)
|
||||
|
||||
# Track feed generation metrics
|
||||
duration_ms = (time.time() - start_time) * 1000
|
||||
track_feed_generated(
|
||||
format='json',
|
||||
item_count=min(len(notes), limit),
|
||||
duration_ms=duration_ms,
|
||||
cached=False
|
||||
)
|
||||
|
||||
return feed_json
|
||||
|
||||
@@ -147,10 +134,6 @@ def generate_json_feed_streaming(
|
||||
# Remove trailing slash from site_url for consistency
|
||||
site_url = site_url.rstrip("/")
|
||||
|
||||
# Track feed generation timing
|
||||
start_time = time.time()
|
||||
item_count = 0
|
||||
|
||||
# Start feed object
|
||||
yield '{\n'
|
||||
yield f' "version": "https://jsonfeed.org/version/1.1",\n'
|
||||
@@ -170,8 +153,6 @@ def generate_json_feed_streaming(
|
||||
# Notes from database are already in DESC order (newest first)
|
||||
items = notes[:limit]
|
||||
for i, note in enumerate(items):
|
||||
item_count += 1
|
||||
|
||||
# Build item object
|
||||
item = _build_item_object(site_url, note)
|
||||
|
||||
@@ -193,15 +174,6 @@ def generate_json_feed_streaming(
|
||||
yield ' ]\n'
|
||||
yield '}\n'
|
||||
|
||||
# Track feed generation metrics
|
||||
duration_ms = (time.time() - start_time) * 1000
|
||||
track_feed_generated(
|
||||
format='json',
|
||||
item_count=item_count,
|
||||
duration_ms=duration_ms,
|
||||
cached=False
|
||||
)
|
||||
|
||||
|
||||
def _build_feed_object(
|
||||
site_url: str,
|
||||
|
||||
@@ -22,14 +22,12 @@ Standards:
|
||||
# Standard library imports
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
import time
|
||||
|
||||
# Third-party imports
|
||||
from feedgen.feed import FeedGenerator
|
||||
|
||||
# Local imports
|
||||
from starpunk.models import Note
|
||||
from starpunk.monitoring.business import track_feed_generated
|
||||
|
||||
|
||||
def generate_rss(
|
||||
@@ -98,9 +96,6 @@ def generate_rss(
|
||||
# Set last build date to now
|
||||
fg.lastBuildDate(datetime.now(timezone.utc))
|
||||
|
||||
# Track feed generation timing
|
||||
start_time = time.time()
|
||||
|
||||
# Add items (limit to configured maximum, newest first)
|
||||
# Notes from database are DESC but feedgen reverses them, so we reverse back
|
||||
for note in reversed(notes[:limit]):
|
||||
@@ -165,14 +160,6 @@ def generate_rss(
|
||||
# We need to inject media:content and media:thumbnail elements
|
||||
feed_xml = _inject_media_rss_elements(feed_xml, site_url, notes[:limit])
|
||||
|
||||
# Track feed generation metrics
|
||||
duration_ms = (time.time() - start_time) * 1000
|
||||
track_feed_generated(
|
||||
format='rss',
|
||||
item_count=min(len(notes), limit),
|
||||
duration_ms=duration_ms,
|
||||
cached=False
|
||||
)
|
||||
|
||||
return feed_xml
|
||||
|
||||
@@ -227,10 +214,6 @@ def generate_rss_streaming(
|
||||
# Remove trailing slash from site_url for consistency
|
||||
site_url = site_url.rstrip("/")
|
||||
|
||||
# Track feed generation timing
|
||||
start_time = time.time()
|
||||
item_count = 0
|
||||
|
||||
# Current timestamp for lastBuildDate
|
||||
now = datetime.now(timezone.utc)
|
||||
last_build = format_rfc822_date(now)
|
||||
@@ -251,8 +234,6 @@ def generate_rss_streaming(
|
||||
# Yield items (newest first)
|
||||
# Notes from database are already in DESC order (newest first)
|
||||
for note in notes[:limit]:
|
||||
item_count += 1
|
||||
|
||||
# Build permalink URL
|
||||
permalink = f"{site_url}{note.permalink}"
|
||||
|
||||
@@ -371,15 +352,6 @@ def generate_rss_streaming(
|
||||
yield " </channel>\n"
|
||||
yield "</rss>\n"
|
||||
|
||||
# Track feed generation metrics
|
||||
duration_ms = (time.time() - start_time) * 1000
|
||||
track_feed_generated(
|
||||
format='rss',
|
||||
item_count=item_count,
|
||||
duration_ms=duration_ms,
|
||||
cached=False
|
||||
)
|
||||
|
||||
|
||||
def _inject_media_rss_elements(feed_xml: str, site_url: str, notes: list[Note]) -> str:
|
||||
"""
|
||||
|
||||
@@ -842,13 +842,16 @@ def note_to_mf2(note, wanted: Optional[list] = None) -> dict:
|
||||
|
||||
|
||||
def _all_categories(prefix: str = "") -> list:
|
||||
"""Tags in use, most used first (q=category)"""
|
||||
"""Tags on posts that exist (drafts included), most used first (q=category)"""
|
||||
from starpunk.database import get_db
|
||||
|
||||
# Inner joins: tags left behind by edits or soft-deleted posts don't count
|
||||
rows = get_db(current_app).execute(
|
||||
"""
|
||||
SELECT t.display_name, COUNT(nt.note_id) AS uses
|
||||
FROM tags t LEFT JOIN note_tags nt ON nt.tag_id = t.id
|
||||
SELECT t.display_name, COUNT(*) AS uses
|
||||
FROM tags t
|
||||
JOIN note_tags nt ON nt.tag_id = t.id
|
||||
JOIN notes n ON n.id = nt.note_id AND n.deleted_at IS NULL
|
||||
GROUP BY t.id ORDER BY uses DESC, t.display_name
|
||||
"""
|
||||
).fetchall()
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
"""
|
||||
Performance monitoring for StarPunk
|
||||
|
||||
This package provides performance monitoring capabilities including:
|
||||
- Metrics collection with circular buffers
|
||||
- Operation timing (database, HTTP, rendering)
|
||||
- Per-process metrics with aggregation
|
||||
- Configurable sampling rates
|
||||
- Database query monitoring (v1.1.2 Phase 1)
|
||||
- HTTP request/response metrics (v1.1.2 Phase 1)
|
||||
- Memory monitoring (v1.1.2 Phase 1)
|
||||
|
||||
Per ADR-053 and developer Q&A Q6, Q12:
|
||||
- Each process maintains its own circular buffer
|
||||
- Buffers store recent metrics (default 1000 entries)
|
||||
- Metrics include process ID for multi-process deployment
|
||||
- Sampling rates are configurable per operation type
|
||||
"""
|
||||
|
||||
from starpunk.monitoring.metrics import MetricsBuffer, record_metric, get_metrics, get_metrics_stats
|
||||
from starpunk.monitoring.database import MonitoredConnection
|
||||
from starpunk.monitoring.http import setup_http_metrics
|
||||
from starpunk.monitoring.memory import MemoryMonitor
|
||||
from starpunk.monitoring import business
|
||||
|
||||
__all__ = [
|
||||
"MetricsBuffer",
|
||||
"record_metric",
|
||||
"get_metrics",
|
||||
"get_metrics_stats",
|
||||
"MonitoredConnection",
|
||||
"setup_http_metrics",
|
||||
"MemoryMonitor",
|
||||
"business",
|
||||
]
|
||||
@@ -1,298 +0,0 @@
|
||||
"""
|
||||
Business metrics for StarPunk operations
|
||||
|
||||
Per v1.1.2 Phase 1:
|
||||
- Track note operations (create, update, delete)
|
||||
- Track feed generation and cache hits/misses
|
||||
- Track content statistics
|
||||
|
||||
Per v1.1.2 Phase 3:
|
||||
- Track feed statistics by format
|
||||
- Track feed cache hit/miss rates
|
||||
- Provide feed statistics dashboard
|
||||
|
||||
Example usage:
|
||||
>>> from starpunk.monitoring.business import track_note_created
|
||||
>>> track_note_created(note_id=123, content_length=500)
|
||||
"""
|
||||
|
||||
from typing import Optional, Dict, Any
|
||||
|
||||
from starpunk.monitoring.metrics import record_metric, get_metrics_stats
|
||||
|
||||
|
||||
def track_note_created(note_id: int, content_length: int, has_media: bool = False) -> None:
|
||||
"""
|
||||
Track note creation event
|
||||
|
||||
Args:
|
||||
note_id: ID of created note
|
||||
content_length: Length of note content in characters
|
||||
has_media: Whether note has media attachments
|
||||
"""
|
||||
metadata = {
|
||||
'note_id': note_id,
|
||||
'content_length': content_length,
|
||||
'has_media': has_media,
|
||||
}
|
||||
|
||||
record_metric(
|
||||
'render', # Use 'render' for business metrics
|
||||
'note_created',
|
||||
content_length,
|
||||
metadata,
|
||||
force=True # Always track business events
|
||||
)
|
||||
|
||||
|
||||
def track_note_updated(note_id: int, content_length: int, fields_changed: Optional[list] = None) -> None:
|
||||
"""
|
||||
Track note update event
|
||||
|
||||
Args:
|
||||
note_id: ID of updated note
|
||||
content_length: New length of note content
|
||||
fields_changed: List of fields that were changed
|
||||
"""
|
||||
metadata = {
|
||||
'note_id': note_id,
|
||||
'content_length': content_length,
|
||||
}
|
||||
|
||||
if fields_changed:
|
||||
metadata['fields_changed'] = ','.join(fields_changed)
|
||||
|
||||
record_metric(
|
||||
'render',
|
||||
'note_updated',
|
||||
content_length,
|
||||
metadata,
|
||||
force=True
|
||||
)
|
||||
|
||||
|
||||
def track_note_deleted(note_id: int) -> None:
|
||||
"""
|
||||
Track note deletion event
|
||||
|
||||
Args:
|
||||
note_id: ID of deleted note
|
||||
"""
|
||||
metadata = {
|
||||
'note_id': note_id,
|
||||
}
|
||||
|
||||
record_metric(
|
||||
'render',
|
||||
'note_deleted',
|
||||
0, # No meaningful duration for deletion
|
||||
metadata,
|
||||
force=True
|
||||
)
|
||||
|
||||
|
||||
def track_feed_generated(format: str, item_count: int, duration_ms: float, cached: bool = False) -> None:
|
||||
"""
|
||||
Track feed generation event
|
||||
|
||||
Args:
|
||||
format: Feed format (rss, atom, json)
|
||||
item_count: Number of items in feed
|
||||
duration_ms: Time taken to generate feed
|
||||
cached: Whether feed was served from cache
|
||||
"""
|
||||
metadata = {
|
||||
'format': format,
|
||||
'item_count': item_count,
|
||||
'cached': cached,
|
||||
}
|
||||
|
||||
operation = f'feed_{format}{"_cached" if cached else "_generated"}'
|
||||
|
||||
record_metric(
|
||||
'render',
|
||||
operation,
|
||||
duration_ms,
|
||||
metadata,
|
||||
force=True # Always track feed operations
|
||||
)
|
||||
|
||||
|
||||
def track_cache_hit(cache_type: str, key: str) -> None:
|
||||
"""
|
||||
Track cache hit event
|
||||
|
||||
Args:
|
||||
cache_type: Type of cache (feed, etc.)
|
||||
key: Cache key that was hit
|
||||
"""
|
||||
metadata = {
|
||||
'cache_type': cache_type,
|
||||
'key': key,
|
||||
}
|
||||
|
||||
record_metric(
|
||||
'render',
|
||||
f'{cache_type}_cache_hit',
|
||||
0,
|
||||
metadata,
|
||||
force=True
|
||||
)
|
||||
|
||||
|
||||
def track_cache_miss(cache_type: str, key: str) -> None:
|
||||
"""
|
||||
Track cache miss event
|
||||
|
||||
Args:
|
||||
cache_type: Type of cache (feed, etc.)
|
||||
key: Cache key that was missed
|
||||
"""
|
||||
metadata = {
|
||||
'cache_type': cache_type,
|
||||
'key': key,
|
||||
}
|
||||
|
||||
record_metric(
|
||||
'render',
|
||||
f'{cache_type}_cache_miss',
|
||||
0,
|
||||
metadata,
|
||||
force=True
|
||||
)
|
||||
|
||||
|
||||
def get_feed_statistics() -> Dict[str, Any]:
|
||||
"""
|
||||
Get aggregated feed statistics from metrics buffer and feed cache.
|
||||
|
||||
Analyzes metrics to provide feed-specific statistics including:
|
||||
- Total requests by format (RSS, ATOM, JSON)
|
||||
- Cache hit/miss rates by format
|
||||
- Feed generation times by format
|
||||
- Format popularity (percentage breakdown)
|
||||
- Feed cache internal statistics
|
||||
|
||||
Returns:
|
||||
Dictionary with feed statistics:
|
||||
{
|
||||
'by_format': {
|
||||
'rss': {'generated': int, 'cached': int, 'total': int, 'avg_duration_ms': float},
|
||||
'atom': {...},
|
||||
'json': {...}
|
||||
},
|
||||
'cache': {
|
||||
'hits': int,
|
||||
'misses': int,
|
||||
'hit_rate': float (0.0-1.0),
|
||||
'entries': int,
|
||||
'evictions': int
|
||||
},
|
||||
'total_requests': int,
|
||||
'format_percentages': {
|
||||
'rss': float,
|
||||
'atom': float,
|
||||
'json': float
|
||||
}
|
||||
}
|
||||
|
||||
Example:
|
||||
>>> stats = get_feed_statistics()
|
||||
>>> print(f"RSS requests: {stats['by_format']['rss']['total']}")
|
||||
>>> print(f"Cache hit rate: {stats['cache']['hit_rate']:.2%}")
|
||||
"""
|
||||
# Get all metrics
|
||||
all_metrics = get_metrics_stats()
|
||||
|
||||
# Initialize result structure
|
||||
result = {
|
||||
'by_format': {
|
||||
'rss': {'generated': 0, 'cached': 0, 'total': 0, 'avg_duration_ms': 0.0},
|
||||
'atom': {'generated': 0, 'cached': 0, 'total': 0, 'avg_duration_ms': 0.0},
|
||||
'json': {'generated': 0, 'cached': 0, 'total': 0, 'avg_duration_ms': 0.0},
|
||||
},
|
||||
'cache': {
|
||||
'hits': 0,
|
||||
'misses': 0,
|
||||
'hit_rate': 0.0,
|
||||
},
|
||||
'total_requests': 0,
|
||||
'format_percentages': {
|
||||
'rss': 0.0,
|
||||
'atom': 0.0,
|
||||
'json': 0.0,
|
||||
},
|
||||
}
|
||||
|
||||
# Get by_operation metrics if available
|
||||
by_operation = all_metrics.get('by_operation', {})
|
||||
|
||||
# Count feed operations by format
|
||||
for operation_name, op_stats in by_operation.items():
|
||||
# Feed operations are named: feed_rss_generated, feed_rss_cached, etc.
|
||||
if operation_name.startswith('feed_'):
|
||||
parts = operation_name.split('_')
|
||||
if len(parts) >= 3:
|
||||
format_name = parts[1] # rss, atom, or json
|
||||
operation_type = parts[2] # generated or cached
|
||||
|
||||
if format_name in result['by_format']:
|
||||
count = op_stats.get('count', 0)
|
||||
|
||||
if operation_type == 'generated':
|
||||
result['by_format'][format_name]['generated'] = count
|
||||
# Track average duration for generated feeds
|
||||
result['by_format'][format_name]['avg_duration_ms'] = op_stats.get('avg_duration_ms', 0.0)
|
||||
elif operation_type == 'cached':
|
||||
result['by_format'][format_name]['cached'] = count
|
||||
|
||||
# Update total for this format
|
||||
result['by_format'][format_name]['total'] = (
|
||||
result['by_format'][format_name]['generated'] +
|
||||
result['by_format'][format_name]['cached']
|
||||
)
|
||||
|
||||
# Track cache hits/misses
|
||||
elif operation_name == 'feed_cache_hit':
|
||||
result['cache']['hits'] = op_stats.get('count', 0)
|
||||
elif operation_name == 'feed_cache_miss':
|
||||
result['cache']['misses'] = op_stats.get('count', 0)
|
||||
|
||||
# Calculate total requests across all formats
|
||||
result['total_requests'] = sum(
|
||||
fmt['total'] for fmt in result['by_format'].values()
|
||||
)
|
||||
|
||||
# Calculate cache hit rate
|
||||
total_cache_requests = result['cache']['hits'] + result['cache']['misses']
|
||||
if total_cache_requests > 0:
|
||||
result['cache']['hit_rate'] = result['cache']['hits'] / total_cache_requests
|
||||
|
||||
# Calculate format percentages
|
||||
if result['total_requests'] > 0:
|
||||
for format_name, fmt_stats in result['by_format'].items():
|
||||
result['format_percentages'][format_name] = (
|
||||
fmt_stats['total'] / result['total_requests']
|
||||
)
|
||||
|
||||
# Get feed cache statistics if available
|
||||
try:
|
||||
from starpunk.feeds import get_cache
|
||||
feed_cache = get_cache()
|
||||
cache_stats = feed_cache.get_stats()
|
||||
|
||||
# Merge cache stats (prefer FeedCache internal stats over metrics)
|
||||
result['cache']['entries'] = cache_stats.get('entries', 0)
|
||||
result['cache']['evictions'] = cache_stats.get('evictions', 0)
|
||||
|
||||
# Use FeedCache hit rate if available and more accurate
|
||||
if cache_stats.get('hits', 0) + cache_stats.get('misses', 0) > 0:
|
||||
result['cache']['hits'] = cache_stats.get('hits', 0)
|
||||
result['cache']['misses'] = cache_stats.get('misses', 0)
|
||||
result['cache']['hit_rate'] = cache_stats.get('hit_rate', 0.0)
|
||||
|
||||
except ImportError:
|
||||
# Feed cache not available, use defaults
|
||||
pass
|
||||
|
||||
return result
|
||||
@@ -1,236 +0,0 @@
|
||||
"""
|
||||
Database operation monitoring wrapper
|
||||
|
||||
Per ADR-053, v1.1.2 Phase 1, and developer Q&A CQ1, IQ1, IQ3:
|
||||
- Wraps SQLite connections at the pool level
|
||||
- Times all database operations
|
||||
- Extracts query type and table name (best effort)
|
||||
- Detects slow queries based on configurable threshold
|
||||
- Records metrics to the metrics collector
|
||||
|
||||
Example usage:
|
||||
>>> from starpunk.monitoring.database import MonitoredConnection
|
||||
>>> conn = sqlite3.connect(':memory:')
|
||||
>>> monitored = MonitoredConnection(conn, metrics_collector)
|
||||
>>> cursor = monitored.execute('SELECT * FROM notes')
|
||||
"""
|
||||
|
||||
import re
|
||||
import sqlite3
|
||||
import time
|
||||
from typing import Optional, Any, Tuple
|
||||
|
||||
from starpunk.monitoring.metrics import record_metric
|
||||
|
||||
|
||||
class MonitoredConnection:
|
||||
"""
|
||||
Wrapper for SQLite connections that monitors performance
|
||||
|
||||
Per CQ1: Wraps connections at the pool level
|
||||
Per IQ1: Uses simple regex for table name extraction
|
||||
Per IQ3: Single configurable slow query threshold
|
||||
"""
|
||||
|
||||
def __init__(self, connection: sqlite3.Connection, slow_query_threshold: float = 1.0):
|
||||
"""
|
||||
Initialize monitored connection wrapper
|
||||
|
||||
Args:
|
||||
connection: SQLite connection to wrap
|
||||
slow_query_threshold: Threshold in seconds for slow query detection
|
||||
"""
|
||||
self._connection = connection
|
||||
self._slow_query_threshold = slow_query_threshold
|
||||
|
||||
def execute(self, query: str, parameters: Optional[Tuple] = None) -> sqlite3.Cursor:
|
||||
"""
|
||||
Execute a query with performance monitoring
|
||||
|
||||
Args:
|
||||
query: SQL query to execute
|
||||
parameters: Optional query parameters
|
||||
|
||||
Returns:
|
||||
sqlite3.Cursor: Query cursor
|
||||
"""
|
||||
start_time = time.perf_counter()
|
||||
query_type = self._get_query_type(query)
|
||||
table_name = self._extract_table_name(query)
|
||||
|
||||
try:
|
||||
if parameters:
|
||||
cursor = self._connection.execute(query, parameters)
|
||||
else:
|
||||
cursor = self._connection.execute(query)
|
||||
|
||||
duration_sec = time.perf_counter() - start_time
|
||||
duration_ms = duration_sec * 1000
|
||||
|
||||
# Record metric (forced if slow query)
|
||||
is_slow = duration_sec >= self._slow_query_threshold
|
||||
metadata = {
|
||||
'query_type': query_type,
|
||||
'table': table_name,
|
||||
'is_slow': is_slow,
|
||||
}
|
||||
|
||||
# Add query text for slow queries (for debugging)
|
||||
if is_slow:
|
||||
# Truncate query to avoid storing huge queries
|
||||
metadata['query'] = query[:200] if len(query) > 200 else query
|
||||
|
||||
record_metric(
|
||||
'database',
|
||||
f'{query_type} {table_name}',
|
||||
duration_ms,
|
||||
metadata,
|
||||
force=is_slow # Always record slow queries
|
||||
)
|
||||
|
||||
return cursor
|
||||
|
||||
except Exception as e:
|
||||
duration_sec = time.perf_counter() - start_time
|
||||
duration_ms = duration_sec * 1000
|
||||
|
||||
# Record error metric
|
||||
metadata = {
|
||||
'query_type': query_type,
|
||||
'table': table_name,
|
||||
'error': str(e),
|
||||
'query': query[:200] if len(query) > 200 else query
|
||||
}
|
||||
|
||||
record_metric(
|
||||
'database',
|
||||
f'{query_type} {table_name} ERROR',
|
||||
duration_ms,
|
||||
metadata,
|
||||
force=True # Always record errors
|
||||
)
|
||||
|
||||
raise
|
||||
|
||||
def executemany(self, query: str, parameters) -> sqlite3.Cursor:
|
||||
"""
|
||||
Execute a query with multiple parameter sets
|
||||
|
||||
Args:
|
||||
query: SQL query to execute
|
||||
parameters: Sequence of parameter tuples
|
||||
|
||||
Returns:
|
||||
sqlite3.Cursor: Query cursor
|
||||
"""
|
||||
start_time = time.perf_counter()
|
||||
query_type = self._get_query_type(query)
|
||||
table_name = self._extract_table_name(query)
|
||||
|
||||
try:
|
||||
cursor = self._connection.executemany(query, parameters)
|
||||
duration_ms = (time.perf_counter() - start_time) * 1000
|
||||
|
||||
# Record metric
|
||||
metadata = {
|
||||
'query_type': query_type,
|
||||
'table': table_name,
|
||||
'batch': True,
|
||||
}
|
||||
|
||||
record_metric(
|
||||
'database',
|
||||
f'{query_type} {table_name} BATCH',
|
||||
duration_ms,
|
||||
metadata
|
||||
)
|
||||
|
||||
return cursor
|
||||
|
||||
except Exception as e:
|
||||
duration_ms = (time.perf_counter() - start_time) * 1000
|
||||
|
||||
metadata = {
|
||||
'query_type': query_type,
|
||||
'table': table_name,
|
||||
'error': str(e),
|
||||
'batch': True
|
||||
}
|
||||
|
||||
record_metric(
|
||||
'database',
|
||||
f'{query_type} {table_name} BATCH ERROR',
|
||||
duration_ms,
|
||||
metadata,
|
||||
force=True
|
||||
)
|
||||
|
||||
raise
|
||||
|
||||
def _get_query_type(self, query: str) -> str:
|
||||
"""
|
||||
Extract query type from SQL statement
|
||||
|
||||
Args:
|
||||
query: SQL query
|
||||
|
||||
Returns:
|
||||
Query type (SELECT, INSERT, UPDATE, DELETE, etc.)
|
||||
"""
|
||||
query_upper = query.strip().upper()
|
||||
|
||||
for query_type in ['SELECT', 'INSERT', 'UPDATE', 'DELETE', 'CREATE', 'DROP', 'ALTER', 'PRAGMA']:
|
||||
if query_upper.startswith(query_type):
|
||||
return query_type
|
||||
|
||||
return 'OTHER'
|
||||
|
||||
def _extract_table_name(self, query: str) -> str:
|
||||
"""
|
||||
Extract table name from query (best effort)
|
||||
|
||||
Per IQ1: Keep it simple with basic regex patterns.
|
||||
Returns "unknown" for complex queries.
|
||||
|
||||
Note: Complex queries (JOINs, subqueries, CTEs) return "unknown".
|
||||
This covers 90% of queries accurately.
|
||||
|
||||
Args:
|
||||
query: SQL query
|
||||
|
||||
Returns:
|
||||
Table name or "unknown"
|
||||
"""
|
||||
query_lower = query.lower().strip()
|
||||
|
||||
# Simple patterns that cover 90% of cases
|
||||
patterns = [
|
||||
r'from\s+(\w+)',
|
||||
r'update\s+(\w+)',
|
||||
r'insert\s+into\s+(\w+)',
|
||||
r'delete\s+from\s+(\w+)',
|
||||
r'create\s+table\s+(?:if\s+not\s+exists\s+)?(\w+)',
|
||||
r'drop\s+table\s+(?:if\s+exists\s+)?(\w+)',
|
||||
r'alter\s+table\s+(\w+)',
|
||||
]
|
||||
|
||||
for pattern in patterns:
|
||||
match = re.search(pattern, query_lower)
|
||||
if match:
|
||||
return match.group(1)
|
||||
|
||||
# Complex queries (JOINs, subqueries, CTEs)
|
||||
return "unknown"
|
||||
|
||||
# Delegate all other connection methods to the wrapped connection
|
||||
def __getattr__(self, name: str) -> Any:
|
||||
"""Delegate all other methods to the wrapped connection"""
|
||||
return getattr(self._connection, name)
|
||||
|
||||
def __enter__(self):
|
||||
"""Support context manager protocol"""
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
"""Support context manager protocol"""
|
||||
return self._connection.__exit__(exc_type, exc_val, exc_tb)
|
||||
@@ -1,133 +0,0 @@
|
||||
"""
|
||||
HTTP request/response metrics middleware
|
||||
|
||||
Per v1.1.2 Phase 1 and developer Q&A IQ2:
|
||||
- Times all HTTP requests
|
||||
- Generates request IDs for tracking (IQ2)
|
||||
- Records status codes, methods, routes
|
||||
- Tracks request and response sizes
|
||||
- Adds X-Request-ID header to all responses (not just debug mode)
|
||||
|
||||
Example usage:
|
||||
>>> from starpunk.monitoring.http import setup_http_metrics
|
||||
>>> app = Flask(__name__)
|
||||
>>> setup_http_metrics(app)
|
||||
"""
|
||||
|
||||
import time
|
||||
import uuid
|
||||
from flask import g, request, Flask
|
||||
from typing import Any
|
||||
|
||||
from starpunk.monitoring.metrics import record_metric
|
||||
|
||||
|
||||
def setup_http_metrics(app: Flask) -> None:
|
||||
"""
|
||||
Setup HTTP metrics collection for Flask app
|
||||
|
||||
Per IQ2: Generates request IDs and adds X-Request-ID header in all modes
|
||||
|
||||
Args:
|
||||
app: Flask application instance
|
||||
"""
|
||||
|
||||
@app.before_request
|
||||
def start_request_metrics():
|
||||
"""
|
||||
Initialize request metrics tracking
|
||||
|
||||
Per IQ2: Generate UUID request ID and store in g
|
||||
"""
|
||||
# Generate request ID (IQ2: in all modes, not just debug)
|
||||
g.request_id = str(uuid.uuid4())
|
||||
|
||||
# Store request start time and metadata
|
||||
g.request_start_time = time.perf_counter()
|
||||
g.request_metadata = {
|
||||
'method': request.method,
|
||||
'endpoint': request.endpoint or 'unknown',
|
||||
'path': request.path,
|
||||
'content_length': request.content_length or 0,
|
||||
}
|
||||
|
||||
@app.after_request
|
||||
def record_response_metrics(response):
|
||||
"""
|
||||
Record HTTP response metrics
|
||||
|
||||
Args:
|
||||
response: Flask response object
|
||||
|
||||
Returns:
|
||||
Modified response with X-Request-ID header
|
||||
"""
|
||||
# Skip if metrics not initialized (shouldn't happen in normal flow)
|
||||
if not hasattr(g, 'request_start_time'):
|
||||
return response
|
||||
|
||||
# Calculate request duration
|
||||
duration_sec = time.perf_counter() - g.request_start_time
|
||||
duration_ms = duration_sec * 1000
|
||||
|
||||
# Get response size
|
||||
response_size = 0
|
||||
|
||||
# Check if response is in direct passthrough mode (streaming)
|
||||
if hasattr(response, 'direct_passthrough') and response.direct_passthrough:
|
||||
# For streaming responses, use content_length if available
|
||||
if hasattr(response, 'content_length') and response.content_length:
|
||||
response_size = response.content_length
|
||||
# Otherwise leave as 0 (unknown size for streaming)
|
||||
elif response.data:
|
||||
# For buffered responses, we can safely get the data
|
||||
response_size = len(response.data)
|
||||
elif hasattr(response, 'content_length') and response.content_length:
|
||||
response_size = response.content_length
|
||||
|
||||
# Build metadata
|
||||
metadata = {
|
||||
**g.request_metadata,
|
||||
'status_code': response.status_code,
|
||||
'response_size': response_size,
|
||||
}
|
||||
|
||||
# Record metric
|
||||
operation_name = f"{g.request_metadata['method']} {g.request_metadata['endpoint']}"
|
||||
record_metric(
|
||||
'http',
|
||||
operation_name,
|
||||
duration_ms,
|
||||
metadata
|
||||
)
|
||||
|
||||
# Add request ID header (IQ2: in all modes)
|
||||
response.headers['X-Request-ID'] = g.request_id
|
||||
|
||||
return response
|
||||
|
||||
@app.teardown_request
|
||||
def record_error_metrics(error=None):
|
||||
"""
|
||||
Record metrics for requests that result in errors
|
||||
|
||||
Args:
|
||||
error: Exception if request failed
|
||||
"""
|
||||
if error and hasattr(g, 'request_start_time'):
|
||||
duration_ms = (time.perf_counter() - g.request_start_time) * 1000
|
||||
|
||||
metadata = {
|
||||
**g.request_metadata,
|
||||
'error': str(error),
|
||||
'error_type': type(error).__name__,
|
||||
}
|
||||
|
||||
operation_name = f"{g.request_metadata['method']} {g.request_metadata['endpoint']} ERROR"
|
||||
record_metric(
|
||||
'http',
|
||||
operation_name,
|
||||
duration_ms,
|
||||
metadata,
|
||||
force=True # Always record errors
|
||||
)
|
||||
@@ -1,191 +0,0 @@
|
||||
"""
|
||||
Memory monitoring background thread
|
||||
|
||||
Per v1.1.2 Phase 1 and developer Q&A CQ5, IQ8:
|
||||
- Background daemon thread for continuous memory monitoring
|
||||
- Tracks RSS and VMS memory usage
|
||||
- Detects memory growth and potential leaks
|
||||
- 5-second baseline period after startup (IQ8)
|
||||
- Skipped in test mode (CQ5)
|
||||
|
||||
Example usage:
|
||||
>>> from starpunk.monitoring.memory import MemoryMonitor
|
||||
>>> monitor = MemoryMonitor(interval=30)
|
||||
>>> monitor.start() # Runs as daemon thread
|
||||
>>> # ... application runs ...
|
||||
>>> monitor.stop()
|
||||
"""
|
||||
|
||||
import gc
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from typing import Dict, Any
|
||||
|
||||
import psutil
|
||||
|
||||
from starpunk.monitoring.metrics import record_metric
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MemoryMonitor(threading.Thread):
|
||||
"""
|
||||
Background thread for memory monitoring
|
||||
|
||||
Per CQ5: Daemon thread that auto-terminates with main process
|
||||
Per IQ8: 5-second baseline period after startup
|
||||
"""
|
||||
|
||||
def __init__(self, interval: int = 30):
|
||||
"""
|
||||
Initialize memory monitor thread
|
||||
|
||||
Args:
|
||||
interval: Monitoring interval in seconds (default: 30)
|
||||
"""
|
||||
super().__init__(daemon=True) # CQ5: daemon thread
|
||||
self.interval = interval
|
||||
self._stop_event = threading.Event()
|
||||
self._process = psutil.Process()
|
||||
self._baseline_memory = None
|
||||
self._high_water_mark = 0
|
||||
|
||||
def run(self):
|
||||
"""
|
||||
Main monitoring loop
|
||||
|
||||
Per IQ8: Wait 5 seconds for app initialization before setting baseline
|
||||
"""
|
||||
try:
|
||||
# Wait for app initialization (IQ8: 5 seconds)
|
||||
time.sleep(5)
|
||||
|
||||
# Set baseline memory
|
||||
memory_info = self._get_memory_info()
|
||||
self._baseline_memory = memory_info['rss_mb']
|
||||
logger.info(f"Memory monitor baseline set: {self._baseline_memory:.2f} MB RSS")
|
||||
|
||||
# Start monitoring loop
|
||||
while not self._stop_event.is_set():
|
||||
try:
|
||||
self._collect_metrics()
|
||||
except Exception as e:
|
||||
logger.error(f"Memory monitoring error: {e}", exc_info=True)
|
||||
|
||||
# Wait for interval or until stop event
|
||||
self._stop_event.wait(self.interval)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Memory monitor thread failed: {e}", exc_info=True)
|
||||
|
||||
def _collect_metrics(self):
|
||||
"""Collect and record memory metrics"""
|
||||
memory_info = self._get_memory_info()
|
||||
gc_stats = self._get_gc_stats()
|
||||
|
||||
# Update high water mark
|
||||
if memory_info['rss_mb'] > self._high_water_mark:
|
||||
self._high_water_mark = memory_info['rss_mb']
|
||||
|
||||
# Calculate growth rate (MB/hour) if baseline is set
|
||||
growth_rate = 0.0
|
||||
if self._baseline_memory:
|
||||
growth_rate = memory_info['rss_mb'] - self._baseline_memory
|
||||
|
||||
# Record metrics
|
||||
metadata = {
|
||||
'rss_mb': memory_info['rss_mb'],
|
||||
'vms_mb': memory_info['vms_mb'],
|
||||
'percent': memory_info['percent'],
|
||||
'high_water_mb': self._high_water_mark,
|
||||
'growth_mb': growth_rate,
|
||||
'gc_collections': gc_stats['collections'],
|
||||
'gc_collected': gc_stats['collected'],
|
||||
}
|
||||
|
||||
record_metric(
|
||||
'render', # Use 'render' operation type for memory metrics
|
||||
'memory_usage',
|
||||
memory_info['rss_mb'],
|
||||
metadata,
|
||||
force=True # Always record memory metrics
|
||||
)
|
||||
|
||||
# Warn if significant growth detected (>10MB growth from baseline)
|
||||
if growth_rate > 10.0:
|
||||
logger.warning(
|
||||
f"Memory growth detected: +{growth_rate:.2f} MB from baseline "
|
||||
f"(current: {memory_info['rss_mb']:.2f} MB, baseline: {self._baseline_memory:.2f} MB)"
|
||||
)
|
||||
|
||||
def _get_memory_info(self) -> Dict[str, float]:
|
||||
"""
|
||||
Get current process memory usage
|
||||
|
||||
Returns:
|
||||
Dict with memory info in MB
|
||||
"""
|
||||
memory = self._process.memory_info()
|
||||
|
||||
return {
|
||||
'rss_mb': memory.rss / (1024 * 1024), # Resident Set Size
|
||||
'vms_mb': memory.vms / (1024 * 1024), # Virtual Memory Size
|
||||
'percent': self._process.memory_percent(),
|
||||
}
|
||||
|
||||
def _get_gc_stats(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Get garbage collection statistics
|
||||
|
||||
Returns:
|
||||
Dict with GC stats
|
||||
"""
|
||||
# Get collection counts per generation
|
||||
counts = gc.get_count()
|
||||
|
||||
# Perform a quick gen 0 collection and count collected objects
|
||||
collected = gc.collect(0)
|
||||
|
||||
return {
|
||||
'collections': {
|
||||
'gen0': counts[0],
|
||||
'gen1': counts[1],
|
||||
'gen2': counts[2],
|
||||
},
|
||||
'collected': collected,
|
||||
'uncollectable': len(gc.garbage),
|
||||
}
|
||||
|
||||
def stop(self):
|
||||
"""
|
||||
Stop the monitoring thread gracefully
|
||||
|
||||
Sets the stop event to signal the thread to exit
|
||||
"""
|
||||
logger.info("Stopping memory monitor")
|
||||
self._stop_event.set()
|
||||
|
||||
def get_stats(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Get current memory statistics
|
||||
|
||||
Returns:
|
||||
Dict with current memory stats
|
||||
"""
|
||||
if not self._baseline_memory:
|
||||
return {'status': 'initializing'}
|
||||
|
||||
memory_info = self._get_memory_info()
|
||||
|
||||
return {
|
||||
'status': 'running',
|
||||
'current_rss_mb': memory_info['rss_mb'],
|
||||
'baseline_rss_mb': self._baseline_memory,
|
||||
'growth_mb': memory_info['rss_mb'] - self._baseline_memory,
|
||||
'high_water_mb': self._high_water_mark,
|
||||
'percent': memory_info['percent'],
|
||||
}
|
||||
@@ -1,425 +0,0 @@
|
||||
"""
|
||||
Metrics collection and buffering for performance monitoring
|
||||
|
||||
Per ADR-053 and developer Q&A Q6, Q12:
|
||||
- Per-process circular buffers using deque
|
||||
- Configurable buffer size (default 1000 entries)
|
||||
- Include process ID in all metrics
|
||||
- Configuration-based sampling rates
|
||||
- Operation types: database, http, render
|
||||
|
||||
Example usage:
|
||||
>>> from starpunk.monitoring import record_metric, get_metrics
|
||||
>>>
|
||||
>>> # Record a database operation
|
||||
>>> record_metric('database', 'query', duration_ms=45.2, query='SELECT * FROM notes')
|
||||
>>>
|
||||
>>> # Get all metrics
|
||||
>>> metrics = get_metrics()
|
||||
>>> print(f"Collected {len(metrics)} metrics")
|
||||
"""
|
||||
|
||||
import os
|
||||
import random
|
||||
import time
|
||||
from collections import deque
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from datetime import datetime, timezone
|
||||
from threading import Lock
|
||||
from typing import Any, Deque, Dict, List, Literal, Optional, Union
|
||||
|
||||
# Operation types for categorizing metrics
|
||||
OperationType = Literal["database", "http", "render"]
|
||||
|
||||
# Module-level circular buffer (per-process)
|
||||
# Each process in a multi-process deployment maintains its own buffer
|
||||
_metrics_buffer: Optional["MetricsBuffer"] = None
|
||||
_buffer_lock = Lock()
|
||||
|
||||
|
||||
@dataclass
|
||||
class Metric:
|
||||
"""
|
||||
Represents a single performance metric
|
||||
|
||||
Attributes:
|
||||
operation_type: Type of operation (database/http/render)
|
||||
operation_name: Name/description of operation
|
||||
timestamp: When the metric was recorded (ISO format)
|
||||
duration_ms: Duration in milliseconds
|
||||
process_id: Process ID that recorded the metric
|
||||
metadata: Additional operation-specific data
|
||||
"""
|
||||
operation_type: OperationType
|
||||
operation_name: str
|
||||
timestamp: str
|
||||
duration_ms: float
|
||||
process_id: int
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert metric to dictionary for serialization"""
|
||||
return asdict(self)
|
||||
|
||||
|
||||
class MetricsBuffer:
|
||||
"""
|
||||
Circular buffer for storing performance metrics
|
||||
|
||||
Per developer Q&A Q6:
|
||||
- Uses deque for efficient circular buffer
|
||||
- Per-process storage (not shared across workers)
|
||||
- Thread-safe with locking
|
||||
- Configurable max size (default 1000)
|
||||
- Automatic eviction of oldest entries when full
|
||||
|
||||
Per developer Q&A Q12:
|
||||
- Configurable sampling rates per operation type
|
||||
- Default 100% sampling (suitable for low-traffic sites)
|
||||
- Slow queries always logged regardless of sampling
|
||||
|
||||
Example:
|
||||
>>> buffer = MetricsBuffer(max_size=1000)
|
||||
>>> buffer.record('database', 'query', 45.2, {'query': 'SELECT ...'})
|
||||
>>> metrics = buffer.get_all()
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
max_size: int = 1000,
|
||||
sampling_rates: Optional[Union[Dict[OperationType, float], float]] = None
|
||||
):
|
||||
"""
|
||||
Initialize metrics buffer
|
||||
|
||||
Args:
|
||||
max_size: Maximum number of metrics to store
|
||||
sampling_rates: Either:
|
||||
- float: Global sampling rate for all operation types (0.0-1.0)
|
||||
- dict: Mapping operation type to sampling rate
|
||||
Default: 1.0 (100% sampling)
|
||||
"""
|
||||
self.max_size = max_size
|
||||
self._buffer: Deque[Metric] = deque(maxlen=max_size)
|
||||
self._lock = Lock()
|
||||
self._process_id = os.getpid()
|
||||
|
||||
# Handle different sampling_rates types
|
||||
if sampling_rates is None:
|
||||
# Default to 100% sampling for all types
|
||||
self._sampling_rates = {
|
||||
"database": 1.0,
|
||||
"http": 1.0,
|
||||
"render": 1.0,
|
||||
}
|
||||
elif isinstance(sampling_rates, (int, float)):
|
||||
# Global rate for all types
|
||||
rate = float(sampling_rates)
|
||||
self._sampling_rates = {
|
||||
"database": rate,
|
||||
"http": rate,
|
||||
"render": rate,
|
||||
}
|
||||
else:
|
||||
# Dict with per-type rates
|
||||
self._sampling_rates = sampling_rates
|
||||
|
||||
def record(
|
||||
self,
|
||||
operation_type: OperationType,
|
||||
operation_name: str,
|
||||
duration_ms: float,
|
||||
metadata: Optional[Dict[str, Any]] = None,
|
||||
force: bool = False
|
||||
) -> bool:
|
||||
"""
|
||||
Record a performance metric
|
||||
|
||||
Args:
|
||||
operation_type: Type of operation (database/http/render)
|
||||
operation_name: Name/description of operation
|
||||
duration_ms: Duration in milliseconds
|
||||
metadata: Additional operation-specific data
|
||||
force: If True, bypass sampling (for slow query logging)
|
||||
|
||||
Returns:
|
||||
True if metric was recorded, False if skipped due to sampling
|
||||
|
||||
Example:
|
||||
>>> buffer.record('database', 'SELECT notes', 45.2,
|
||||
... {'query': 'SELECT * FROM notes LIMIT 10'})
|
||||
True
|
||||
"""
|
||||
# Apply sampling (unless forced)
|
||||
if not force:
|
||||
sampling_rate = self._sampling_rates.get(operation_type, 0.1)
|
||||
if random.random() > sampling_rate:
|
||||
return False
|
||||
|
||||
metric = Metric(
|
||||
operation_type=operation_type,
|
||||
operation_name=operation_name,
|
||||
timestamp=datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + "Z",
|
||||
duration_ms=duration_ms,
|
||||
process_id=self._process_id,
|
||||
metadata=metadata or {}
|
||||
)
|
||||
|
||||
with self._lock:
|
||||
self._buffer.append(metric)
|
||||
|
||||
return True
|
||||
|
||||
def get_all(self) -> List[Metric]:
|
||||
"""
|
||||
Get all metrics from buffer
|
||||
|
||||
Returns:
|
||||
List of metrics (oldest to newest)
|
||||
|
||||
Example:
|
||||
>>> metrics = buffer.get_all()
|
||||
>>> len(metrics)
|
||||
1000
|
||||
"""
|
||||
with self._lock:
|
||||
return list(self._buffer)
|
||||
|
||||
def get_recent(self, count: int) -> List[Metric]:
|
||||
"""
|
||||
Get most recent N metrics
|
||||
|
||||
Args:
|
||||
count: Number of recent metrics to return
|
||||
|
||||
Returns:
|
||||
List of most recent metrics (newest first)
|
||||
|
||||
Example:
|
||||
>>> recent = buffer.get_recent(10)
|
||||
>>> len(recent)
|
||||
10
|
||||
"""
|
||||
with self._lock:
|
||||
# Convert to list, reverse to get newest first, then slice
|
||||
all_metrics = list(self._buffer)
|
||||
all_metrics.reverse()
|
||||
return all_metrics[:count]
|
||||
|
||||
def get_by_type(self, operation_type: OperationType) -> List[Metric]:
|
||||
"""
|
||||
Get all metrics of a specific type
|
||||
|
||||
Args:
|
||||
operation_type: Type to filter by (database/http/render)
|
||||
|
||||
Returns:
|
||||
List of metrics matching the type
|
||||
|
||||
Example:
|
||||
>>> db_metrics = buffer.get_by_type('database')
|
||||
"""
|
||||
with self._lock:
|
||||
return [m for m in self._buffer if m.operation_type == operation_type]
|
||||
|
||||
def get_slow_operations(
|
||||
self,
|
||||
threshold_ms: float = 1000.0,
|
||||
operation_type: Optional[OperationType] = None
|
||||
) -> List[Metric]:
|
||||
"""
|
||||
Get operations that exceeded a duration threshold
|
||||
|
||||
Args:
|
||||
threshold_ms: Duration threshold in milliseconds
|
||||
operation_type: Optional type filter
|
||||
|
||||
Returns:
|
||||
List of slow operations
|
||||
|
||||
Example:
|
||||
>>> slow_queries = buffer.get_slow_operations(1000, 'database')
|
||||
"""
|
||||
with self._lock:
|
||||
metrics = list(self._buffer)
|
||||
|
||||
# Filter by type if specified
|
||||
if operation_type:
|
||||
metrics = [m for m in metrics if m.operation_type == operation_type]
|
||||
|
||||
# Filter by duration threshold
|
||||
return [m for m in metrics if m.duration_ms >= threshold_ms]
|
||||
|
||||
def get_stats(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Get statistics about the buffer
|
||||
|
||||
Returns:
|
||||
Dict with buffer statistics
|
||||
|
||||
Example:
|
||||
>>> stats = buffer.get_stats()
|
||||
>>> stats['total_count']
|
||||
1000
|
||||
"""
|
||||
with self._lock:
|
||||
metrics = list(self._buffer)
|
||||
|
||||
# Calculate stats per operation type
|
||||
type_stats = {}
|
||||
for op_type in ["database", "http", "render"]:
|
||||
type_metrics = [m for m in metrics if m.operation_type == op_type]
|
||||
if type_metrics:
|
||||
durations = [m.duration_ms for m in type_metrics]
|
||||
type_stats[op_type] = {
|
||||
"count": len(type_metrics),
|
||||
"avg_duration_ms": sum(durations) / len(durations),
|
||||
"min_duration_ms": min(durations),
|
||||
"max_duration_ms": max(durations),
|
||||
}
|
||||
else:
|
||||
type_stats[op_type] = {
|
||||
"count": 0,
|
||||
"avg_duration_ms": 0.0,
|
||||
"min_duration_ms": 0.0,
|
||||
"max_duration_ms": 0.0,
|
||||
}
|
||||
|
||||
return {
|
||||
"total_count": len(metrics),
|
||||
"max_size": self.max_size,
|
||||
"process_id": self._process_id,
|
||||
"sampling_rates": self._sampling_rates,
|
||||
"by_type": type_stats,
|
||||
}
|
||||
|
||||
def clear(self) -> None:
|
||||
"""
|
||||
Clear all metrics from buffer
|
||||
|
||||
Example:
|
||||
>>> buffer.clear()
|
||||
"""
|
||||
with self._lock:
|
||||
self._buffer.clear()
|
||||
|
||||
def set_sampling_rate(
|
||||
self,
|
||||
operation_type: OperationType,
|
||||
rate: float
|
||||
) -> None:
|
||||
"""
|
||||
Update sampling rate for an operation type
|
||||
|
||||
Args:
|
||||
operation_type: Type to update
|
||||
rate: New sampling rate (0.0-1.0)
|
||||
|
||||
Example:
|
||||
>>> buffer.set_sampling_rate('database', 0.5) # 50% sampling
|
||||
"""
|
||||
if not 0.0 <= rate <= 1.0:
|
||||
raise ValueError("Sampling rate must be between 0.0 and 1.0")
|
||||
|
||||
with self._lock:
|
||||
self._sampling_rates[operation_type] = rate
|
||||
|
||||
|
||||
def get_buffer() -> MetricsBuffer:
|
||||
"""
|
||||
Get or create the module-level metrics buffer
|
||||
|
||||
This ensures a single buffer per process. In multi-process deployments
|
||||
(e.g., gunicorn), each worker process will have its own buffer.
|
||||
|
||||
Returns:
|
||||
MetricsBuffer instance for this process
|
||||
|
||||
Example:
|
||||
>>> buffer = get_buffer()
|
||||
>>> buffer.record('database', 'query', 45.2)
|
||||
"""
|
||||
global _metrics_buffer
|
||||
|
||||
if _metrics_buffer is None:
|
||||
with _buffer_lock:
|
||||
# Double-check locking pattern
|
||||
if _metrics_buffer is None:
|
||||
# Get configuration from Flask app if available
|
||||
try:
|
||||
from flask import current_app
|
||||
max_size = current_app.config.get('METRICS_BUFFER_SIZE', 1000)
|
||||
sampling_rate = current_app.config.get('METRICS_SAMPLING_RATE', 1.0)
|
||||
except (ImportError, RuntimeError):
|
||||
# Flask not available or no app context
|
||||
max_size = 1000
|
||||
sampling_rate = 1.0 # Default to 100%
|
||||
|
||||
_metrics_buffer = MetricsBuffer(
|
||||
max_size=max_size,
|
||||
sampling_rates=sampling_rate
|
||||
)
|
||||
|
||||
return _metrics_buffer
|
||||
|
||||
|
||||
def record_metric(
|
||||
operation_type: OperationType,
|
||||
operation_name: str,
|
||||
duration_ms: float,
|
||||
metadata: Optional[Dict[str, Any]] = None,
|
||||
force: bool = False
|
||||
) -> bool:
|
||||
"""
|
||||
Record a metric using the module-level buffer
|
||||
|
||||
Convenience function that uses get_buffer() internally.
|
||||
|
||||
Args:
|
||||
operation_type: Type of operation (database/http/render)
|
||||
operation_name: Name/description of operation
|
||||
duration_ms: Duration in milliseconds
|
||||
metadata: Additional operation-specific data
|
||||
force: If True, bypass sampling (for slow query logging)
|
||||
|
||||
Returns:
|
||||
True if metric was recorded, False if skipped due to sampling
|
||||
|
||||
Example:
|
||||
>>> record_metric('database', 'SELECT notes', 45.2,
|
||||
... {'query': 'SELECT * FROM notes LIMIT 10'})
|
||||
True
|
||||
"""
|
||||
buffer = get_buffer()
|
||||
return buffer.record(operation_type, operation_name, duration_ms, metadata, force)
|
||||
|
||||
|
||||
def get_metrics() -> List[Metric]:
|
||||
"""
|
||||
Get all metrics from the module-level buffer
|
||||
|
||||
Returns:
|
||||
List of metrics (oldest to newest)
|
||||
|
||||
Example:
|
||||
>>> metrics = get_metrics()
|
||||
>>> len(metrics)
|
||||
1000
|
||||
"""
|
||||
buffer = get_buffer()
|
||||
return buffer.get_all()
|
||||
|
||||
|
||||
def get_metrics_stats() -> Dict[str, Any]:
|
||||
"""
|
||||
Get statistics from the module-level buffer
|
||||
|
||||
Returns:
|
||||
Dict with buffer statistics
|
||||
|
||||
Example:
|
||||
>>> stats = get_metrics_stats()
|
||||
>>> print(f"Total metrics: {stats['total_count']}")
|
||||
"""
|
||||
buffer = get_buffer()
|
||||
return buffer.get_stats()
|
||||
+32
-209
@@ -5,11 +5,10 @@ Handles authenticated admin functionality including dashboard, note creation,
|
||||
editing, and deletion. All routes require authentication.
|
||||
"""
|
||||
|
||||
from flask import Blueprint, flash, g, jsonify, redirect, render_template, request, url_for
|
||||
from flask import Blueprint, current_app, flash, g, jsonify, redirect, render_template, request, url_for
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
from datetime import datetime, timezone
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -260,6 +259,7 @@ def edit_note_form(note_id: int):
|
||||
syndication_targets=get_targets().values(),
|
||||
syndications=syndications,
|
||||
syndication_enabled=enabled,
|
||||
backfeed_enabled=current_app.config.get("BACKFEED_ENABLED", False),
|
||||
)
|
||||
|
||||
|
||||
@@ -366,202 +366,6 @@ def delete_note_submit(note_id: int):
|
||||
return redirect(url_for("admin.dashboard"))
|
||||
|
||||
|
||||
def transform_metrics_for_template(metrics_stats):
|
||||
"""
|
||||
Transform metrics stats to match template structure
|
||||
|
||||
The template expects direct access to metrics.database.count, but
|
||||
get_metrics_stats() returns metrics.by_type.database.count.
|
||||
This function adapts the data structure to match template expectations.
|
||||
|
||||
Args:
|
||||
metrics_stats: Dict from get_metrics_stats() with nested by_type structure
|
||||
|
||||
Returns:
|
||||
Dict with flattened structure matching template expectations
|
||||
|
||||
Per ADR-060: Route Adapter Pattern for template compatibility
|
||||
"""
|
||||
transformed = {}
|
||||
|
||||
# Map by_type to direct access
|
||||
for op_type in ['database', 'http', 'render']:
|
||||
if 'by_type' in metrics_stats and op_type in metrics_stats['by_type']:
|
||||
type_data = metrics_stats['by_type'][op_type]
|
||||
transformed[op_type] = {
|
||||
'count': type_data.get('count', 0),
|
||||
'avg': type_data.get('avg_duration_ms', 0),
|
||||
'min': type_data.get('min_duration_ms', 0),
|
||||
'max': type_data.get('max_duration_ms', 0)
|
||||
}
|
||||
else:
|
||||
# Provide defaults for missing types or when by_type doesn't exist
|
||||
transformed[op_type] = {
|
||||
'count': 0,
|
||||
'avg': 0,
|
||||
'min': 0,
|
||||
'max': 0
|
||||
}
|
||||
|
||||
# Keep other top-level stats
|
||||
transformed['total_count'] = metrics_stats.get('total_count', 0)
|
||||
transformed['max_size'] = metrics_stats.get('max_size', 1000)
|
||||
transformed['process_id'] = metrics_stats.get('process_id', 0)
|
||||
|
||||
return transformed
|
||||
|
||||
|
||||
@bp.route("/metrics-dashboard")
|
||||
@require_auth
|
||||
def metrics_dashboard():
|
||||
"""
|
||||
Metrics visualization dashboard (Phase 3)
|
||||
|
||||
Displays performance metrics, database statistics, feed statistics,
|
||||
and system health with visual charts and auto-refresh capability.
|
||||
|
||||
Per Q19 requirements:
|
||||
- Server-side rendering with Jinja2
|
||||
- htmx for auto-refresh
|
||||
- Chart.js from CDN for graphs
|
||||
- Progressive enhancement (works without JS)
|
||||
|
||||
Per v1.1.2 Phase 3:
|
||||
- Feed statistics by format
|
||||
- Cache hit/miss rates
|
||||
- Format popularity breakdown
|
||||
|
||||
Returns:
|
||||
Rendered dashboard template with metrics
|
||||
|
||||
Decorator: @require_auth
|
||||
Template: templates/admin/metrics_dashboard.html
|
||||
"""
|
||||
# Defensive imports with graceful degradation for missing modules
|
||||
try:
|
||||
from starpunk.database.pool import get_pool_stats
|
||||
from starpunk.monitoring import get_metrics_stats
|
||||
from starpunk.monitoring.business import get_feed_statistics
|
||||
monitoring_available = True
|
||||
except ImportError:
|
||||
monitoring_available = False
|
||||
# Provide fallback functions that return error messages
|
||||
def get_pool_stats():
|
||||
return {"error": "Database pool monitoring not available"}
|
||||
def get_metrics_stats():
|
||||
return {"error": "Monitoring module not implemented"}
|
||||
def get_feed_statistics():
|
||||
return {"error": "Feed statistics not available"}
|
||||
|
||||
# Get current metrics for initial page load
|
||||
metrics_data = {}
|
||||
pool_stats = {}
|
||||
feed_stats = {}
|
||||
|
||||
try:
|
||||
raw_metrics = get_metrics_stats()
|
||||
metrics_data = transform_metrics_for_template(raw_metrics)
|
||||
except Exception as e:
|
||||
flash(f"Error loading metrics: {e}", "warning")
|
||||
# Provide safe defaults matching template expectations
|
||||
metrics_data = {
|
||||
'database': {'count': 0, 'avg': 0, 'min': 0, 'max': 0},
|
||||
'http': {'count': 0, 'avg': 0, 'min': 0, 'max': 0},
|
||||
'render': {'count': 0, 'avg': 0, 'min': 0, 'max': 0},
|
||||
'total_count': 0,
|
||||
'max_size': 1000,
|
||||
'process_id': 0
|
||||
}
|
||||
|
||||
try:
|
||||
pool_stats = get_pool_stats()
|
||||
except Exception as e:
|
||||
flash(f"Error loading pool stats: {e}", "warning")
|
||||
|
||||
try:
|
||||
feed_stats = get_feed_statistics()
|
||||
except Exception as e:
|
||||
flash(f"Error loading feed stats: {e}", "warning")
|
||||
# Provide safe defaults
|
||||
feed_stats = {
|
||||
'by_format': {
|
||||
'rss': {'generated': 0, 'cached': 0, 'total': 0, 'avg_duration_ms': 0.0},
|
||||
'atom': {'generated': 0, 'cached': 0, 'total': 0, 'avg_duration_ms': 0.0},
|
||||
'json': {'generated': 0, 'cached': 0, 'total': 0, 'avg_duration_ms': 0.0},
|
||||
},
|
||||
'cache': {'hits': 0, 'misses': 0, 'hit_rate': 0.0, 'entries': 0, 'evictions': 0},
|
||||
'total_requests': 0,
|
||||
'format_percentages': {'rss': 0.0, 'atom': 0.0, 'json': 0.0},
|
||||
}
|
||||
|
||||
return render_template(
|
||||
"admin/metrics_dashboard.html",
|
||||
metrics=metrics_data,
|
||||
pool=pool_stats,
|
||||
feeds=feed_stats,
|
||||
user_me=g.me
|
||||
)
|
||||
|
||||
|
||||
@bp.route("/metrics")
|
||||
@require_auth
|
||||
def metrics():
|
||||
"""
|
||||
Performance metrics and database pool statistics endpoint
|
||||
|
||||
Per Phase 2 requirements:
|
||||
- Expose database pool statistics
|
||||
- Show performance metrics from MetricsBuffer
|
||||
- Requires authentication
|
||||
|
||||
Per v1.1.2 Phase 3:
|
||||
- Include feed statistics
|
||||
|
||||
Returns:
|
||||
JSON with metrics, pool statistics, and feed statistics
|
||||
|
||||
Response codes:
|
||||
200: Metrics retrieved successfully
|
||||
|
||||
Decorator: @require_auth
|
||||
"""
|
||||
from flask import current_app
|
||||
from starpunk.database.pool import get_pool_stats
|
||||
from starpunk.monitoring import get_metrics_stats
|
||||
from starpunk.monitoring.business import get_feed_statistics
|
||||
|
||||
response = {
|
||||
"timestamp": datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + "Z",
|
||||
"process_id": os.getpid(),
|
||||
"database": {},
|
||||
"performance": {},
|
||||
"feeds": {}
|
||||
}
|
||||
|
||||
# Get database pool statistics
|
||||
try:
|
||||
pool_stats = get_pool_stats()
|
||||
response["database"]["pool"] = pool_stats
|
||||
except Exception as e:
|
||||
response["database"]["pool"] = {"error": str(e)}
|
||||
|
||||
# Get performance metrics
|
||||
try:
|
||||
metrics_stats = get_metrics_stats()
|
||||
response["performance"] = metrics_stats
|
||||
except Exception as e:
|
||||
response["performance"] = {"error": str(e)}
|
||||
|
||||
# Get feed statistics
|
||||
try:
|
||||
feed_stats = get_feed_statistics()
|
||||
response["feeds"] = feed_stats
|
||||
except Exception as e:
|
||||
response["feeds"] = {"error": str(e)}
|
||||
|
||||
return jsonify(response), 200
|
||||
|
||||
|
||||
@bp.route("/health")
|
||||
@require_auth
|
||||
def health_diagnostics():
|
||||
@@ -571,7 +375,7 @@ def health_diagnostics():
|
||||
Per developer Q&A Q10:
|
||||
- Always requires authentication
|
||||
- Provides comprehensive diagnostics
|
||||
- Includes metrics, database pool statistics, and system info
|
||||
- Includes database pool statistics and system info
|
||||
|
||||
Returns:
|
||||
JSON with complete system diagnostics
|
||||
@@ -591,7 +395,6 @@ def health_diagnostics():
|
||||
"environment": current_app.config.get("ENV", "unknown"),
|
||||
"process_id": os.getpid(),
|
||||
"checks": {},
|
||||
"metrics": {},
|
||||
"database": {}
|
||||
}
|
||||
|
||||
@@ -665,14 +468,6 @@ def health_diagnostics():
|
||||
}
|
||||
overall_healthy = False
|
||||
|
||||
# Performance metrics
|
||||
try:
|
||||
from starpunk.monitoring import get_metrics_stats
|
||||
metrics_stats = get_metrics_stats()
|
||||
diagnostics["metrics"] = metrics_stats
|
||||
except Exception as e:
|
||||
diagnostics["metrics"] = {"error": str(e)}
|
||||
|
||||
# Update overall status
|
||||
diagnostics["status"] = "healthy" if overall_healthy else "unhealthy"
|
||||
|
||||
@@ -702,18 +497,24 @@ def webmentions():
|
||||
from starpunk.webmention_queries import list_webmentions
|
||||
|
||||
status_filter = request.args.get('status')
|
||||
origin_filter = request.args.get('origin') or None
|
||||
page = int(request.args.get('page', 1))
|
||||
|
||||
webmentions, total_pages = list_webmentions(
|
||||
status_filter=status_filter,
|
||||
page=page,
|
||||
per_page=25
|
||||
per_page=25,
|
||||
origin_filter=origin_filter,
|
||||
)
|
||||
|
||||
from starpunk import backfeed
|
||||
|
||||
return render_template(
|
||||
"admin/webmentions.html",
|
||||
webmentions=webmentions,
|
||||
status_filter=status_filter,
|
||||
origin_filter=origin_filter,
|
||||
backfeed=backfeed.status(),
|
||||
page=page,
|
||||
total_pages=total_pages,
|
||||
user_me=g.me
|
||||
@@ -880,6 +681,28 @@ def revoke_token(token_id):
|
||||
return redirect(url_for("admin.tokens"))
|
||||
|
||||
|
||||
@bp.route("/backfeed/backfill", methods=["POST"])
|
||||
@require_auth
|
||||
def backfeed_backfill():
|
||||
"""Check every crosspost once for responses, whatever its age (v2.13.0)"""
|
||||
from starpunk import backfeed
|
||||
|
||||
count = backfeed.start_backfill()
|
||||
flash(f"Backfill started: {count} crosspost(s) to check", "success")
|
||||
return redirect(url_for("admin.webmentions"))
|
||||
|
||||
|
||||
@bp.route("/syndication/<int:syndication_id>/check", methods=["POST"])
|
||||
@require_auth
|
||||
def check_syndication_responses(syndication_id):
|
||||
"""Check one crosspost's responses at the next worker pass (v2.13.0)"""
|
||||
from starpunk import backfeed
|
||||
|
||||
backfeed.check_now(syndication_id)
|
||||
flash("Responses will be checked shortly", "success")
|
||||
return redirect(request.referrer or url_for("admin.dashboard"))
|
||||
|
||||
|
||||
@bp.route("/syndication/<int:syndication_id>/retry", methods=["POST"])
|
||||
@require_auth
|
||||
def retry_syndication(syndication_id):
|
||||
|
||||
@@ -250,6 +250,20 @@ def media_file(path):
|
||||
return response
|
||||
|
||||
|
||||
@bp.route("/avatars/<name>")
|
||||
def avatar(name: str):
|
||||
"""A saved copy of a response author's avatar (v2.13.0, ADR-073)"""
|
||||
from pathlib import Path
|
||||
from starpunk.avatars import NAME_RE, folder
|
||||
|
||||
if not NAME_RE.match(name):
|
||||
abort(404)
|
||||
response = send_from_directory(folder(Path(current_app.config.get('DATA_PATH', 'data'))), name)
|
||||
# Named by the hash of the original URL; a changed avatar gets a new name
|
||||
response.headers['Cache-Control'] = 'public, max-age=604800'
|
||||
return response
|
||||
|
||||
|
||||
@bp.route("/")
|
||||
def index():
|
||||
"""
|
||||
|
||||
@@ -258,7 +258,10 @@ def receive_webmention():
|
||||
ON CONFLICT(source_url, target_url) DO UPDATE SET
|
||||
validation_status = 'pending',
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
WHERE webmentions.origin = 'webmention'
|
||||
""", (source, target, target_note_id))
|
||||
# (A backfed response with the same URLs is StarPunk's own copy of
|
||||
# what the silo said; a webmention doesn't overwrite it. v2.13.0)
|
||||
db.commit()
|
||||
|
||||
current_app.logger.info(
|
||||
|
||||
+211
-5
@@ -67,8 +67,9 @@ class MastodonClient:
|
||||
|
||||
def _call(self, method: str, path: str, **kwargs) -> httpx.Response:
|
||||
headers = {**self.headers, **kwargs.pop("headers", {})}
|
||||
url = path if path.startswith(("http://", "https://")) else self.base_url + path
|
||||
try:
|
||||
response = self.http.request(method, self.base_url + path, headers=headers, **kwargs)
|
||||
response = self.http.request(method, url, headers=headers, **kwargs)
|
||||
except httpx.HTTPError as e:
|
||||
raise SyndicationError(f"{method} {path}: {e}")
|
||||
if response.status_code == 404:
|
||||
@@ -76,10 +77,12 @@ class MastodonClient:
|
||||
if response.status_code >= 400:
|
||||
# 4xx other than rate limiting won't fix itself
|
||||
permanent = 400 <= response.status_code < 500 and response.status_code != 429
|
||||
raise SyndicationError(
|
||||
exc = SyndicationError(
|
||||
f"{method} {path}: HTTP {response.status_code} {response.text[:200]}",
|
||||
permanent=permanent,
|
||||
)
|
||||
exc.status = response.status_code
|
||||
raise exc
|
||||
return response
|
||||
|
||||
def upload_media(self, path: Path, mime_type: str, description: str) -> str:
|
||||
@@ -166,6 +169,61 @@ class MastodonClient:
|
||||
text, media_ids = self._prepare(post)
|
||||
return self.update_status(remote_id, text, media_ids)
|
||||
|
||||
# --- Backfeed (v2.13.0, ADR-073) ---
|
||||
|
||||
def _account_id(self) -> str:
|
||||
"""Id of the account StarPunk posts as (its own responses are skipped)"""
|
||||
if not getattr(self, "account_id", None):
|
||||
self.account_id = self._call("GET", "/api/v1/accounts/verify_credentials").json()["id"]
|
||||
return self.account_id
|
||||
|
||||
def _list(self, path: str, cap: int) -> tuple:
|
||||
"""Items of a paginated list (Link: rel="next"). Returns (items, complete)."""
|
||||
items, url, params = [], path, {"limit": 80}
|
||||
while url:
|
||||
response = self._call("GET", url, params=params)
|
||||
items.extend(response.json() or [])
|
||||
if len(items) >= cap:
|
||||
return items[:cap], False
|
||||
url, params = (response.links.get("next") or {}).get("url"), None
|
||||
return items, True
|
||||
|
||||
@staticmethod
|
||||
def _author(account: dict) -> dict:
|
||||
return {
|
||||
"author_id": account.get("id") or "",
|
||||
"author_name": account.get("display_name") or account.get("username") or account.get("acct") or "",
|
||||
"author_url": account.get("url") or "",
|
||||
"author_photo": account.get("avatar_static") or account.get("avatar"),
|
||||
"author_bio": account.get("note") or "",
|
||||
}
|
||||
|
||||
def responses(self, remote_id: str, remote_url: str, cap: int = 1000):
|
||||
"""Likes, boosts and public direct replies of one of our statuses"""
|
||||
from starpunk.backfeed import Harvest, Response
|
||||
|
||||
me = self._account_id()
|
||||
found, complete = [], True
|
||||
for kind, path, fragment in (("like", "favourited_by", "liked-by"),
|
||||
("repost", "reblogged_by", "reposted-by")):
|
||||
accounts, done = self._list(f"/api/v1/statuses/{remote_id}/{path}", cap)
|
||||
complete = complete and done
|
||||
for account in accounts:
|
||||
if account.get("id") == me:
|
||||
continue
|
||||
found.append(Response(kind=kind, source_url=f"{remote_url}#{fragment}-{account['id']}",
|
||||
remote_id=account["id"], **self._author(account)))
|
||||
context = self._call("GET", f"/api/v1/statuses/{remote_id}/context").json() or {}
|
||||
for status in context.get("descendants") or []:
|
||||
account = status.get("account") or {}
|
||||
if (status.get("in_reply_to_id") != remote_id or status.get("visibility") != "public"
|
||||
or account.get("id") == me):
|
||||
continue
|
||||
found.append(Response(kind="reply", source_url=status.get("url") or status["uri"],
|
||||
remote_id=status["id"], content_html=status.get("content") or "",
|
||||
published=status.get("created_at"), **self._author(account)))
|
||||
return Harvest(found, complete)
|
||||
|
||||
def delete(self, remote_id: str) -> None:
|
||||
action, _, status_id = remote_id.partition(":")
|
||||
if action in self.NATIVE and status_id:
|
||||
@@ -227,9 +285,11 @@ class BlueskyClient:
|
||||
if response.status_code < 400:
|
||||
return
|
||||
permanent = 400 <= response.status_code < 500 and response.status_code != 429
|
||||
raise SyndicationError(
|
||||
exc = SyndicationError(
|
||||
f"{nsid}: HTTP {response.status_code} {response.text[:200]}", permanent=permanent
|
||||
)
|
||||
exc.status = response.status_code
|
||||
raise exc
|
||||
|
||||
def _login(self) -> None:
|
||||
response = None
|
||||
@@ -405,6 +465,94 @@ class BlueskyClient:
|
||||
def update(self, remote_id: str, post: "Post") -> dict:
|
||||
raise NotImplementedError("Bluesky posts can't be edited")
|
||||
|
||||
# --- Backfeed (v2.13.0, ADR-073) ---
|
||||
|
||||
def _paged(self, nsid: str, params: dict, key: str, cap: int) -> tuple:
|
||||
"""Items of a cursor-paginated XRPC list. Returns (items, complete)."""
|
||||
items, cursor = [], None
|
||||
while True:
|
||||
query = {**params, "limit": 100}
|
||||
if cursor:
|
||||
query["cursor"] = cursor
|
||||
data = self._call("GET", nsid, params=query).json()
|
||||
page = data.get(key) or []
|
||||
items.extend(page)
|
||||
if len(items) >= cap:
|
||||
return items[:cap], False
|
||||
cursor = data.get("cursor")
|
||||
if not cursor or not page:
|
||||
return items, True
|
||||
|
||||
@staticmethod
|
||||
def _author(profile: dict) -> dict:
|
||||
handle = profile.get("handle") or profile.get("did") or ""
|
||||
return {
|
||||
"author_id": profile.get("did") or "",
|
||||
"author_name": profile.get("displayName") or handle,
|
||||
"author_url": f"https://bsky.app/profile/{handle}",
|
||||
"author_photo": profile.get("avatar"),
|
||||
"author_bio": profile.get("description") or "",
|
||||
# The author asked not to be shown to people who aren't logged in
|
||||
"opted_out": any(label.get("val") == "!no-unauthenticated"
|
||||
for label in profile.get("labels") or []),
|
||||
}
|
||||
|
||||
def _post_response(self, kind: str, view: dict):
|
||||
from starpunk.backfeed import Response, facets_html
|
||||
|
||||
author = view.get("author") or {}
|
||||
record = view.get("record") or {}
|
||||
rkey = view["uri"].rsplit("/", 1)[1]
|
||||
return Response(
|
||||
kind=kind,
|
||||
source_url=f"https://bsky.app/profile/{author.get('handle') or author.get('did')}/post/{rkey}",
|
||||
remote_id=view["uri"],
|
||||
content_html=facets_html(record.get("text") or "", record.get("facets") or []),
|
||||
published=record.get("createdAt") or view.get("indexedAt"),
|
||||
**self._author(author),
|
||||
)
|
||||
|
||||
def responses(self, remote_id: str, remote_url: str, cap: int = 1000):
|
||||
"""Likes, reposts, quotes and direct replies of one of our posts"""
|
||||
from starpunk.backfeed import Harvest, Response
|
||||
|
||||
if not self.did:
|
||||
with self._lock:
|
||||
self._login()
|
||||
found, complete = [], True
|
||||
likes, done = self._paged("app.bsky.feed.getLikes", {"uri": remote_id}, "likes", cap)
|
||||
complete = complete and done
|
||||
for like in likes:
|
||||
actor = like.get("actor") or {}
|
||||
found.append(Response(kind="like", source_url=f"{remote_url}#liked-by-{actor.get('did')}",
|
||||
remote_id=actor.get("did") or "", published=like.get("createdAt"),
|
||||
**self._author(actor)))
|
||||
reposters, done = self._paged("app.bsky.feed.getRepostedBy", {"uri": remote_id},
|
||||
"repostedBy", cap)
|
||||
complete = complete and done
|
||||
for actor in reposters:
|
||||
found.append(Response(kind="repost", source_url=f"{remote_url}#reposted-by-{actor.get('did')}",
|
||||
remote_id=actor.get("did") or "", **self._author(actor)))
|
||||
quotes, done = self._paged("app.bsky.feed.getQuotes", {"uri": remote_id}, "posts", cap)
|
||||
complete = complete and done
|
||||
posts = [self._post_response("quote", view) for view in quotes]
|
||||
thread = self._call("GET", "app.bsky.feed.getPostThread",
|
||||
params={"uri": remote_id, "depth": 1, "parentHeight": 0}).json()
|
||||
for item in (thread.get("thread") or {}).get("replies") or []:
|
||||
if item.get("post"): # skips blocked and deleted replies
|
||||
posts.append(self._post_response("reply", item["post"]))
|
||||
# Post views carry no bio; fetch them to honour #nobridge
|
||||
dids = list({p.author_id for p in posts if p.author_id})
|
||||
bios = {}
|
||||
for i in range(0, len(dids), 25):
|
||||
profiles = self._call("GET", "app.bsky.actor.getProfiles",
|
||||
params={"actors": dids[i:i + 25]}).json().get("profiles") or []
|
||||
bios.update({p["did"]: p.get("description") or "" for p in profiles})
|
||||
for post in posts:
|
||||
post.author_bio = bios.get(post.author_id, "")
|
||||
found.extend(posts)
|
||||
return Harvest([r for r in found if r.author_id != self.did], complete)
|
||||
|
||||
def delete(self, remote_id: str) -> None:
|
||||
# at://<did>/app.bsky.feed.post/<rkey>; deleting a missing record succeeds
|
||||
repo, collection, rkey = remote_id[len("at://"):].split("/", 2)
|
||||
@@ -513,7 +661,7 @@ def page_card(http: httpx.Client, url: str) -> dict:
|
||||
return card
|
||||
|
||||
|
||||
THREADS_API = "https://graph.threads.net/v1.0"
|
||||
THREADS_API = "https://graph.threads.com/v1.0" # Meta's current host (v2.13.0)
|
||||
THREADS_IMAGE_TYPES = {"image/jpeg", "image/png"}
|
||||
THREADS_IMAGE_LIMIT = 8_000_000
|
||||
THREADS_MAX_LINKS = 5
|
||||
@@ -775,6 +923,57 @@ class ThreadsClient:
|
||||
def update(self, remote_id: str, post: "Post") -> dict:
|
||||
raise NotImplementedError("Threads posts can't be edited through the API")
|
||||
|
||||
# --- Backfeed (v2.13.0, ADR-073) ---
|
||||
|
||||
REPLY_FIELDS = "id,text,username,permalink,timestamp,profile_picture_url,hide_status"
|
||||
SHOWN = (None, "NOT_HUSHED", "UNHUSHED") # hide_status values of replies nobody hid
|
||||
|
||||
def responses(self, remote_id: str, remote_url: str, cap: int = 1000):
|
||||
"""
|
||||
Direct replies to one of our posts, and its like/repost/quote counts.
|
||||
The API doesn't say who liked, reposted or quoted (ADR-073 Decision 7).
|
||||
"""
|
||||
from starpunk.backfeed import Harvest, Response, text_html
|
||||
|
||||
self._me()
|
||||
replies, after, complete = [], None, True
|
||||
while True:
|
||||
params = {"fields": self.REPLY_FIELDS}
|
||||
if after:
|
||||
params["after"] = after
|
||||
data = self._call("GET", f"{remote_id}/replies", **params)
|
||||
page = data.get("data") or []
|
||||
replies.extend(page)
|
||||
if len(replies) >= cap:
|
||||
replies, complete = replies[:cap], False
|
||||
break
|
||||
paging = data.get("paging") or {}
|
||||
after = (paging.get("cursors") or {}).get("after")
|
||||
if not page or not after or not paging.get("next"):
|
||||
break
|
||||
found = []
|
||||
for reply in replies:
|
||||
username = reply.get("username")
|
||||
# Private accounts have no username or permalink: nothing public to show
|
||||
if (not username or not reply.get("permalink") or username == self.username
|
||||
or reply.get("hide_status") not in self.SHOWN):
|
||||
continue
|
||||
found.append(Response(
|
||||
kind="reply", source_url=reply["permalink"], remote_id=reply["id"],
|
||||
author_id=username, author_name=username,
|
||||
author_url=f"https://www.threads.com/@{username}",
|
||||
author_photo=reply.get("profile_picture_url"),
|
||||
content_html=text_html(reply.get("text") or ""),
|
||||
published=reply.get("timestamp"),
|
||||
))
|
||||
insights = self._call("GET", f"{remote_id}/insights", metric="likes,reposts,quotes")
|
||||
counts = {}
|
||||
for metric in insights.get("data") or []:
|
||||
values = metric.get("values") or [{}]
|
||||
counts[metric.get("name")] = int((values[0] or {}).get("value") or 0)
|
||||
return Harvest(found, complete, counts={k: counts.get(k, 0)
|
||||
for k in ("likes", "reposts", "quotes")})
|
||||
|
||||
def delete(self, remote_id: str) -> None:
|
||||
try:
|
||||
self._call("DELETE", remote_id)
|
||||
@@ -1381,12 +1580,19 @@ class SyndicationWorker(threading.Thread):
|
||||
|
||||
def run(self):
|
||||
while not self._stop_event.is_set():
|
||||
handled = 0
|
||||
try:
|
||||
with self.app.app_context():
|
||||
handled = process_due()
|
||||
except Exception as e:
|
||||
self.app.logger.error(f"Syndication worker error: {e}", exc_info=True)
|
||||
handled = 0
|
||||
# Responses to the copies (v2.13.0, ADR-073), after any crossposts
|
||||
try:
|
||||
from starpunk import backfeed
|
||||
with self.app.app_context():
|
||||
handled += backfeed.process_due()
|
||||
except Exception as e:
|
||||
self.app.logger.error(f"Backfeed error: {e}", exc_info=True)
|
||||
if handled == 0:
|
||||
self._stop_event.wait(self.poll_interval)
|
||||
|
||||
|
||||
@@ -39,7 +39,8 @@ def get_webmentions_for_note(note_id: int) -> Dict[str, List[Dict[str, Any]]]:
|
||||
note_id: Note database ID
|
||||
|
||||
Returns:
|
||||
Dict with keys: 'replies', 'likes', 'reposts', 'bookmarks', 'mentions'
|
||||
Dict with keys: 'replies', 'likes', 'reposts', 'bookmarks', 'quotes'
|
||||
(backfed quote posts, v2.13.0), 'mentions'
|
||||
Each value is a list of webmention dicts
|
||||
"""
|
||||
db = get_db()
|
||||
@@ -56,24 +57,34 @@ def get_webmentions_for_note(note_id: int) -> Dict[str, List[Dict[str, Any]]]:
|
||||
WHEN 'like' THEN 2
|
||||
WHEN 'repost' THEN 3
|
||||
WHEN 'bookmark' THEN 4
|
||||
WHEN 'mention' THEN 5
|
||||
WHEN 'quote' THEN 5
|
||||
WHEN 'mention' THEN 6
|
||||
END,
|
||||
received_at ASC
|
||||
"""
|
||||
|
||||
rows = db.execute(query, (note_id,)).fetchall()
|
||||
|
||||
# Backfed responses say which silo they came from (v2.13.0)
|
||||
silo_names = {}
|
||||
if any(row['origin'] != 'webmention' for row in rows):
|
||||
from starpunk.backfeed import SILO_NAMES
|
||||
from starpunk.syndication import get_targets
|
||||
silo_names = {**SILO_NAMES, **{t.id: t.name for t in get_targets().values()}}
|
||||
|
||||
# Group by type
|
||||
grouped = {
|
||||
'replies': [],
|
||||
'likes': [],
|
||||
'reposts': [],
|
||||
'bookmarks': [],
|
||||
'quotes': [],
|
||||
'mentions': []
|
||||
}
|
||||
|
||||
for row in rows:
|
||||
wm = dict(row)
|
||||
wm['via'] = silo_names.get(wm['origin']) if wm['origin'] != 'webmention' else None
|
||||
# Parse timestamps (source_published may be missing or malformed)
|
||||
for key in ('received_at', 'source_published'):
|
||||
if isinstance(wm.get(key), str):
|
||||
@@ -91,6 +102,8 @@ def get_webmentions_for_note(note_id: int) -> Dict[str, List[Dict[str, Any]]]:
|
||||
grouped['reposts'].append(wm)
|
||||
elif mention_type == 'bookmark':
|
||||
grouped['bookmarks'].append(wm)
|
||||
elif mention_type == 'quote':
|
||||
grouped['quotes'].append(wm)
|
||||
else:
|
||||
grouped['mentions'].append(wm)
|
||||
|
||||
@@ -100,15 +113,18 @@ def get_webmentions_for_note(note_id: int) -> Dict[str, List[Dict[str, Any]]]:
|
||||
def list_webmentions(
|
||||
status_filter: Optional[str] = None,
|
||||
page: int = 1,
|
||||
per_page: int = 25
|
||||
per_page: int = 25,
|
||||
origin_filter: Optional[str] = None,
|
||||
) -> tuple[List[Dict[str, Any]], int]:
|
||||
"""
|
||||
List webmentions for admin with pagination and filtering.
|
||||
|
||||
Args:
|
||||
status_filter: Filter by moderation_status ('pending', 'approved', 'hidden')
|
||||
status_filter: Filter by moderation_status ('pending', 'approved', 'hidden',
|
||||
'deleted'). Deleted ones are left out unless asked for (v2.13.0).
|
||||
page: Page number (1-indexed)
|
||||
per_page: Items per page (default: 25)
|
||||
origin_filter: 'webmention' or a backfeed silo id (v2.13.0)
|
||||
|
||||
Returns:
|
||||
Tuple of (webmentions list, total_pages)
|
||||
@@ -124,12 +140,16 @@ def list_webmentions(
|
||||
count_query = "SELECT COUNT(*) FROM webmentions"
|
||||
|
||||
params = []
|
||||
where_clause = ""
|
||||
|
||||
conditions = ["moderation_status = ?" if status_filter else "moderation_status != 'deleted'"]
|
||||
if status_filter:
|
||||
where_clause = " WHERE w.moderation_status = ?"
|
||||
count_query += " WHERE moderation_status = ?"
|
||||
params.append(status_filter)
|
||||
if origin_filter:
|
||||
conditions.append("origin = ?")
|
||||
params.append(origin_filter)
|
||||
where = " AND ".join(conditions)
|
||||
where_clause = " WHERE " + where.replace("moderation_status", "w.moderation_status").replace(
|
||||
"origin", "w.origin")
|
||||
count_query += " WHERE " + where
|
||||
|
||||
# Count total
|
||||
total_count = db.execute(count_query, params).fetchone()[0]
|
||||
@@ -206,7 +226,7 @@ def count_webmentions(status_filter: Optional[str] = None) -> int:
|
||||
query = "SELECT COUNT(*) FROM webmentions WHERE moderation_status = ?"
|
||||
result = db.execute(query, (status_filter,)).fetchone()
|
||||
else:
|
||||
query = "SELECT COUNT(*) FROM webmentions"
|
||||
query = "SELECT COUNT(*) FROM webmentions WHERE moderation_status != 'deleted'"
|
||||
result = db.execute(query).fetchone()
|
||||
|
||||
return result[0] if result else 0
|
||||
@@ -260,6 +280,9 @@ def delete_webmentions(webmention_ids: List[int]) -> int:
|
||||
"""
|
||||
Delete webmentions by IDs.
|
||||
|
||||
The row stays as a tombstone (moderation_status 'deleted', v2.13.0), so
|
||||
a re-sent webmention or the next backfeed check doesn't bring it back.
|
||||
|
||||
Args:
|
||||
webmention_ids: List of webmention IDs to delete
|
||||
|
||||
@@ -276,7 +299,8 @@ def delete_webmentions(webmention_ids: List[int]) -> int:
|
||||
)
|
||||
|
||||
placeholders = ','.join('?' * len(webmention_ids))
|
||||
query = f"DELETE FROM webmentions WHERE id IN ({placeholders})"
|
||||
query = (f"UPDATE webmentions SET moderation_status = 'deleted', updated_at = CURRENT_TIMESTAMP "
|
||||
f"WHERE id IN ({placeholders}) AND moderation_status != 'deleted'")
|
||||
|
||||
cursor = db.execute(query, webmention_ids)
|
||||
db.commit()
|
||||
|
||||
@@ -82,6 +82,8 @@ class WebmentionWorker(threading.Thread):
|
||||
self._stop_event = threading.Event()
|
||||
self._last_processed = None
|
||||
self._lock_file = None
|
||||
from starpunk.avatars import Upkeep
|
||||
self._avatar_upkeep = Upkeep()
|
||||
|
||||
def run(self):
|
||||
"""
|
||||
@@ -113,6 +115,9 @@ class WebmentionWorker(threading.Thread):
|
||||
|
||||
# Phase 3: Process outgoing webmentions
|
||||
processed += self._process_pending_sends()
|
||||
# Local copies of response avatars (v2.13.0)
|
||||
self._avatar_upkeep.run(self.db_path, self.app.config['DATA_PATH'],
|
||||
self.site_url)
|
||||
|
||||
# Update last processed time
|
||||
if processed > 0:
|
||||
@@ -281,6 +286,22 @@ class WebmentionWorker(threading.Thread):
|
||||
source_url, result.get('source_post_url'), result.get('source_uid')) if u}
|
||||
return bool(copies & source)
|
||||
|
||||
def _is_backfed(self, note_id: Optional[int], source_url: str, result: Dict[str, Any]) -> bool:
|
||||
"""True if the source is a response already stored by backfeed for this note"""
|
||||
if note_id is None:
|
||||
return False
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
try:
|
||||
backfed = {normalize_url(row[0]) for row in conn.execute(
|
||||
"SELECT source_url FROM webmentions WHERE target_note_id = ? AND origin != 'webmention'",
|
||||
(note_id,),
|
||||
)}
|
||||
finally:
|
||||
conn.close()
|
||||
source = {normalize_url(u) for u in (
|
||||
source_url, result.get('source_post_url'), result.get('source_uid')) if u}
|
||||
return bool(backfed & source)
|
||||
|
||||
def _display_state(self, webmention_id: int) -> Optional[tuple]:
|
||||
"""
|
||||
What the note page shows for this webmention, or None if hidden
|
||||
@@ -328,6 +349,11 @@ class WebmentionWorker(threading.Thread):
|
||||
if self._is_own_syndicated_copy(target_note_id, source_url, result):
|
||||
raise ValidationError("Source is this note's own syndicated copy")
|
||||
|
||||
# A response StarPunk already backfed from the silo (e.g. a reply
|
||||
# also sent by Bridgy) would show twice (v2.13.0, ADR-073)
|
||||
if self._is_backfed(target_note_id, source_url, result):
|
||||
raise ValidationError("Already backfed from the silo")
|
||||
|
||||
# Update database with successful validation
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
cursor = conn.cursor()
|
||||
@@ -372,6 +398,10 @@ class WebmentionWorker(threading.Thread):
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
# Pages show a local copy of the avatar (v2.13.0, ADR-073)
|
||||
from starpunk import avatars
|
||||
avatars.fetch(self.app.config['DATA_PATH'], result['author_photo'], self.site_url)
|
||||
|
||||
logger.info(
|
||||
f"Validated webmention: {source_url} -> {target_url} "
|
||||
f"(type={result['mention_type']})"
|
||||
@@ -422,7 +452,8 @@ class WebmentionWorker(threading.Thread):
|
||||
# All other errors are 'error' status
|
||||
if transient:
|
||||
status = 'error'
|
||||
elif any(x in error.lower() for x in ['does not link', 'returned 4', 'own syndicated copy']):
|
||||
elif any(x in error.lower() for x in ['does not link', 'returned 4', 'own syndicated copy',
|
||||
'already backfed']):
|
||||
status = 'invalid'
|
||||
else:
|
||||
status = 'error'
|
||||
|
||||
+29
-47
@@ -343,57 +343,39 @@ small { display: block; margin-top: var(--spacing-xs); color: var(--color-text-l
|
||||
color: var(--color-text-light);
|
||||
}
|
||||
|
||||
/* Facepile */
|
||||
.facepile {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.facepile .face {
|
||||
display: block;
|
||||
width: 32px;
|
||||
height: 32px;
|
||||
border-radius: 50%;
|
||||
overflow: hidden;
|
||||
border: 2px solid var(--color-bg);
|
||||
margin-left: -8px;
|
||||
}
|
||||
|
||||
.facepile .face:first-child {
|
||||
margin-left: 0;
|
||||
}
|
||||
|
||||
.facepile .face > .h-card,
|
||||
.facepile .face > .h-card > a {
|
||||
display: block;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
.facepile .face img {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
object-fit: cover;
|
||||
}
|
||||
|
||||
.facepile .face-initial {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
font-size: 14px;
|
||||
font-weight: bold;
|
||||
color: white;
|
||||
}
|
||||
|
||||
.facepile .face-overflow {
|
||||
margin-left: var(--spacing-xs);
|
||||
/* Backfed responses (v2.13.0) */
|
||||
.webmention-via {
|
||||
font-size: 0.875rem;
|
||||
color: var(--color-text-light);
|
||||
}
|
||||
|
||||
/* Likes, reposts and bookmarks: counts with a hover list (v2.13.2) */
|
||||
.reactions { display: flex; flex-wrap: wrap; gap: var(--spacing-md); margin-top: var(--spacing-sm); color: var(--color-text-light); font-size: 0.875rem; }
|
||||
.reaction { position: relative; display: inline-flex; align-items: center; gap: 0.3rem; padding-bottom: 0.4rem; cursor: default; }
|
||||
.reaction > svg { width: 1rem; height: 1rem; flex: none; }
|
||||
.reaction:hover, .reaction:focus-within { color: var(--color-text); }
|
||||
.reaction-like:hover > svg, .reaction-like:focus-within > svg { color: #e0245e; }
|
||||
.reaction-repost:hover > svg, .reaction-repost:focus-within > svg { color: #17bf63; }
|
||||
.reaction-who {
|
||||
display: none; position: absolute; left: 0; top: 100%; z-index: 10;
|
||||
flex-direction: column; gap: 0.4rem; min-width: 13rem; max-width: 20rem;
|
||||
padding: var(--spacing-sm); background: var(--color-bg-alt); border: 1px solid var(--color-border);
|
||||
border-radius: var(--border-radius); box-shadow: 0 6px 20px rgba(0, 0, 0, 0.25);
|
||||
}
|
||||
.reaction:hover .reaction-who, .reaction:focus-within .reaction-who { display: flex; }
|
||||
.reaction-title { font-weight: 600; color: var(--color-text); }
|
||||
.reaction-people { display: flex; flex-direction: column; gap: 0.35rem; max-height: 14rem; overflow-y: auto; }
|
||||
.reaction-person { display: flex; align-items: center; gap: 0.4rem; min-width: 0; line-height: 1.3; }
|
||||
.reaction-person .h-card { display: flex; min-width: 0; }
|
||||
.reaction-person .h-card a { display: inline-flex; align-items: center; gap: 0.4rem; color: var(--color-text); min-width: 0; }
|
||||
.reaction-person .p-name { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||
.reaction-person img { width: 20px; height: 20px; border-radius: 50%; object-fit: cover; flex: none; }
|
||||
.reaction-person .webmention-via { margin-left: auto; padding-left: 0.5rem; white-space: nowrap; }
|
||||
.reaction-anon { color: var(--color-text-light); }
|
||||
.reaction-elsewhere { display: block; margin-top: 0.15rem; padding-top: 0.4rem; border-top: 1px solid var(--color-border); color: var(--color-text-light); }
|
||||
.reaction-elsewhere:first-child { margin-top: 0; padding-top: 0; border-top: 0; }
|
||||
.reaction-elsewhere a { color: var(--color-link); }
|
||||
|
||||
/* Generic Mentions */
|
||||
.webmention-mention {
|
||||
margin-bottom: var(--spacing-sm);
|
||||
|
||||
@@ -8,7 +8,6 @@
|
||||
<nav class="admin-nav">
|
||||
<a href="{{ url_for('admin.dashboard') }}">Dashboard</a>
|
||||
<a href="{{ url_for('admin.new_note_form') }}">New Note</a>
|
||||
<a href="{{ url_for('admin.metrics_dashboard') }}">Metrics</a>
|
||||
<a href="{{ url_for('admin.tokens') }}">Connected Apps</a>
|
||||
<form action="{{ url_for('auth.logout') }}" method="POST" class="logout-form"><input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
|
||||
<button type="submit" class="button button-secondary">Logout</button>
|
||||
|
||||
@@ -102,6 +102,12 @@
|
||||
<button type="submit" class="button button-secondary">Retry now</button>
|
||||
</form>
|
||||
{% endif %}
|
||||
{% if backfeed_enabled and s.state == 'synced' and s.remote_id %}
|
||||
<br><small>Responses {% if s.backfeed_checked_at %}checked {{ s.backfeed_checked_at }} UTC{% else %}not checked yet{% endif %}{% if s.backfeed_error %}: {{ s.backfeed_error }}{% endif %}</small>
|
||||
<form action="{{ url_for('admin.check_syndication_responses', syndication_id=s.id) }}" method="POST" style="display: inline;"><input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
|
||||
<button type="submit" class="button button-secondary">Check now</button>
|
||||
</form>
|
||||
{% endif %}
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
|
||||
@@ -1,585 +0,0 @@
|
||||
{% extends "admin/base.html" %}
|
||||
|
||||
{% block title %}Metrics Dashboard - StarPunk Admin{% endblock %}
|
||||
|
||||
{% block head %}
|
||||
{{ super() }}
|
||||
<!-- Chart.js from CDN for visualizations -->
|
||||
<script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.0/dist/chart.umd.min.js" crossorigin="anonymous"></script>
|
||||
<!-- htmx for auto-refresh -->
|
||||
<script src="https://unpkg.com/htmx.org@1.9.10" crossorigin="anonymous"></script>
|
||||
<style>
|
||||
.metrics-dashboard {
|
||||
max-width: 1200px;
|
||||
}
|
||||
|
||||
.metrics-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(300px, 1fr));
|
||||
gap: 20px;
|
||||
margin-bottom: 30px;
|
||||
}
|
||||
|
||||
.metric-card {
|
||||
background: #fff;
|
||||
border: 1px solid #ddd;
|
||||
border-radius: 8px;
|
||||
padding: 20px;
|
||||
box-shadow: 0 2px 4px rgba(0,0,0,0.05);
|
||||
}
|
||||
|
||||
.metric-card h3 {
|
||||
margin-top: 0;
|
||||
font-size: 1.1em;
|
||||
color: #333;
|
||||
border-bottom: 2px solid #007bff;
|
||||
padding-bottom: 10px;
|
||||
margin-bottom: 15px;
|
||||
}
|
||||
|
||||
.metric-value {
|
||||
font-size: 2em;
|
||||
font-weight: bold;
|
||||
color: #007bff;
|
||||
margin: 10px 0;
|
||||
}
|
||||
|
||||
.metric-label {
|
||||
color: #666;
|
||||
font-size: 0.9em;
|
||||
margin-bottom: 5px;
|
||||
}
|
||||
|
||||
.metric-detail {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
padding: 8px 0;
|
||||
border-bottom: 1px solid #f0f0f0;
|
||||
}
|
||||
|
||||
.metric-detail:last-child {
|
||||
border-bottom: none;
|
||||
}
|
||||
|
||||
.metric-detail-label {
|
||||
color: #666;
|
||||
}
|
||||
|
||||
.metric-detail-value {
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
.chart-container {
|
||||
position: relative;
|
||||
height: 300px;
|
||||
margin-top: 20px;
|
||||
}
|
||||
|
||||
.status-indicator {
|
||||
display: inline-block;
|
||||
width: 12px;
|
||||
height: 12px;
|
||||
border-radius: 50%;
|
||||
margin-right: 8px;
|
||||
}
|
||||
|
||||
.status-healthy {
|
||||
background-color: #28a745;
|
||||
}
|
||||
|
||||
.status-warning {
|
||||
background-color: #ffc107;
|
||||
}
|
||||
|
||||
.status-error {
|
||||
background-color: #dc3545;
|
||||
}
|
||||
|
||||
.refresh-info {
|
||||
color: #666;
|
||||
font-size: 0.9em;
|
||||
text-align: center;
|
||||
margin-top: 20px;
|
||||
padding: 10px;
|
||||
background-color: #f8f9fa;
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
.no-js-message {
|
||||
display: none;
|
||||
background-color: #fff3cd;
|
||||
border: 1px solid #ffeaa7;
|
||||
color: #856404;
|
||||
padding: 15px;
|
||||
border-radius: 4px;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
|
||||
noscript .no-js-message {
|
||||
display: block;
|
||||
}
|
||||
</style>
|
||||
{% endblock %}
|
||||
|
||||
{% block admin_content %}
|
||||
<div class="metrics-dashboard">
|
||||
<h2>Metrics Dashboard</h2>
|
||||
|
||||
<noscript>
|
||||
<div class="no-js-message">
|
||||
Note: Auto-refresh and charts require JavaScript. Data is displayed below in text format.
|
||||
</div>
|
||||
</noscript>
|
||||
|
||||
<!-- Auto-refresh container -->
|
||||
<div hx-get="{{ url_for('admin.metrics') }}" hx-trigger="every 10s" hx-swap="none" hx-on::after-request="updateDashboard(event)"></div>
|
||||
|
||||
<!-- Database Pool Statistics -->
|
||||
<div class="metrics-grid">
|
||||
<div class="metric-card">
|
||||
<h3>Database Connection Pool</h3>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Active Connections</span>
|
||||
<span class="metric-detail-value" id="pool-active">{{ pool.active_connections|default(0) }}</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Idle Connections</span>
|
||||
<span class="metric-detail-value" id="pool-idle">{{ pool.idle_connections|default(0) }}</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Total Connections</span>
|
||||
<span class="metric-detail-value" id="pool-total">{{ pool.total_connections|default(0) }}</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Pool Size</span>
|
||||
<span class="metric-detail-value" id="pool-size">{{ pool.pool_size|default(5) }}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="metric-card">
|
||||
<h3>Database Operations</h3>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Total Queries</span>
|
||||
<span class="metric-detail-value" id="db-total">{{ metrics.database.count|default(0) }}</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Average Time</span>
|
||||
<span class="metric-detail-value" id="db-avg">{{ "%.2f"|format(metrics.database.avg|default(0)) }} ms</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Min Time</span>
|
||||
<span class="metric-detail-value" id="db-min">{{ "%.2f"|format(metrics.database.min|default(0)) }} ms</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Max Time</span>
|
||||
<span class="metric-detail-value" id="db-max">{{ "%.2f"|format(metrics.database.max|default(0)) }} ms</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="metric-card">
|
||||
<h3>HTTP Requests</h3>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Total Requests</span>
|
||||
<span class="metric-detail-value" id="http-total">{{ metrics.http.count|default(0) }}</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Average Time</span>
|
||||
<span class="metric-detail-value" id="http-avg">{{ "%.2f"|format(metrics.http.avg|default(0)) }} ms</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Min Time</span>
|
||||
<span class="metric-detail-value" id="http-min">{{ "%.2f"|format(metrics.http.min|default(0)) }} ms</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Max Time</span>
|
||||
<span class="metric-detail-value" id="http-max">{{ "%.2f"|format(metrics.http.max|default(0)) }} ms</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="metric-card">
|
||||
<h3>Template Rendering</h3>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Total Renders</span>
|
||||
<span class="metric-detail-value" id="render-total">{{ metrics.render.count|default(0) }}</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Average Time</span>
|
||||
<span class="metric-detail-value" id="render-avg">{{ "%.2f"|format(metrics.render.avg|default(0)) }} ms</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Min Time</span>
|
||||
<span class="metric-detail-value" id="render-min">{{ "%.2f"|format(metrics.render.min|default(0)) }} ms</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Max Time</span>
|
||||
<span class="metric-detail-value" id="render-max">{{ "%.2f"|format(metrics.render.max|default(0)) }} ms</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Charts -->
|
||||
<div class="metrics-grid">
|
||||
<div class="metric-card">
|
||||
<h3>Connection Pool Usage</h3>
|
||||
<div class="chart-container">
|
||||
<canvas id="poolChart"></canvas>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="metric-card">
|
||||
<h3>Performance Overview</h3>
|
||||
<div class="chart-container">
|
||||
<canvas id="performanceChart"></canvas>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Feed Statistics (Phase 3) -->
|
||||
<h2 style="margin-top: 40px;">Feed Statistics</h2>
|
||||
<div class="metrics-grid">
|
||||
<div class="metric-card">
|
||||
<h3>Feed Requests by Format</h3>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">RSS</span>
|
||||
<span class="metric-detail-value" id="feed-rss-total">{{ feeds.by_format.rss.total|default(0) }}</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">ATOM</span>
|
||||
<span class="metric-detail-value" id="feed-atom-total">{{ feeds.by_format.atom.total|default(0) }}</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">JSON Feed</span>
|
||||
<span class="metric-detail-value" id="feed-json-total">{{ feeds.by_format.json.total|default(0) }}</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Total Requests</span>
|
||||
<span class="metric-detail-value" id="feed-total">{{ feeds.total_requests|default(0) }}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="metric-card">
|
||||
<h3>Feed Cache Statistics</h3>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Cache Hits</span>
|
||||
<span class="metric-detail-value" id="feed-cache-hits">{{ feeds.cache.hits|default(0) }}</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Cache Misses</span>
|
||||
<span class="metric-detail-value" id="feed-cache-misses">{{ feeds.cache.misses|default(0) }}</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Hit Rate</span>
|
||||
<span class="metric-detail-value" id="feed-cache-hit-rate">{{ "%.1f"|format(feeds.cache.hit_rate|default(0) * 100) }}%</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">Cached Entries</span>
|
||||
<span class="metric-detail-value" id="feed-cache-entries">{{ feeds.cache.entries|default(0) }}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="metric-card">
|
||||
<h3>Feed Generation Performance</h3>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">RSS Avg Time</span>
|
||||
<span class="metric-detail-value" id="feed-rss-avg">{{ "%.2f"|format(feeds.by_format.rss.avg_duration_ms|default(0)) }} ms</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">ATOM Avg Time</span>
|
||||
<span class="metric-detail-value" id="feed-atom-avg">{{ "%.2f"|format(feeds.by_format.atom.avg_duration_ms|default(0)) }} ms</span>
|
||||
</div>
|
||||
<div class="metric-detail">
|
||||
<span class="metric-detail-label">JSON Avg Time</span>
|
||||
<span class="metric-detail-value" id="feed-json-avg">{{ "%.2f"|format(feeds.by_format.json.avg_duration_ms|default(0)) }} ms</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Feed Charts -->
|
||||
<div class="metrics-grid">
|
||||
<div class="metric-card">
|
||||
<h3>Format Popularity</h3>
|
||||
<div class="chart-container">
|
||||
<canvas id="feedFormatChart"></canvas>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="metric-card">
|
||||
<h3>Cache Efficiency</h3>
|
||||
<div class="chart-container">
|
||||
<canvas id="feedCacheChart"></canvas>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="refresh-info">
|
||||
Auto-refresh every 10 seconds (requires JavaScript)
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
// Initialize charts with current data
|
||||
let poolChart, performanceChart, feedFormatChart, feedCacheChart;
|
||||
|
||||
function initCharts() {
|
||||
// Pool usage chart (doughnut)
|
||||
const poolCtx = document.getElementById('poolChart');
|
||||
if (poolCtx && !poolChart) {
|
||||
poolChart = new Chart(poolCtx, {
|
||||
type: 'doughnut',
|
||||
data: {
|
||||
labels: ['Active', 'Idle'],
|
||||
datasets: [{
|
||||
data: [
|
||||
{{ pool.active_connections|default(0) }},
|
||||
{{ pool.idle_connections|default(0) }}
|
||||
],
|
||||
backgroundColor: ['#007bff', '#6c757d'],
|
||||
borderWidth: 1
|
||||
}]
|
||||
},
|
||||
options: {
|
||||
responsive: true,
|
||||
maintainAspectRatio: false,
|
||||
plugins: {
|
||||
legend: {
|
||||
position: 'bottom'
|
||||
},
|
||||
title: {
|
||||
display: true,
|
||||
text: 'Connection Distribution'
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Performance chart (bar)
|
||||
const perfCtx = document.getElementById('performanceChart');
|
||||
if (perfCtx && !performanceChart) {
|
||||
performanceChart = new Chart(perfCtx, {
|
||||
type: 'bar',
|
||||
data: {
|
||||
labels: ['Database', 'HTTP', 'Render'],
|
||||
datasets: [{
|
||||
label: 'Average Time (ms)',
|
||||
data: [
|
||||
{{ metrics.database.avg|default(0) }},
|
||||
{{ metrics.http.avg|default(0) }},
|
||||
{{ metrics.render.avg|default(0) }}
|
||||
],
|
||||
backgroundColor: ['#007bff', '#28a745', '#ffc107'],
|
||||
borderWidth: 1
|
||||
}]
|
||||
},
|
||||
options: {
|
||||
responsive: true,
|
||||
maintainAspectRatio: false,
|
||||
scales: {
|
||||
y: {
|
||||
beginAtZero: true,
|
||||
title: {
|
||||
display: true,
|
||||
text: 'Milliseconds'
|
||||
}
|
||||
}
|
||||
},
|
||||
plugins: {
|
||||
legend: {
|
||||
display: false
|
||||
},
|
||||
title: {
|
||||
display: true,
|
||||
text: 'Average Response Times'
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Feed format chart (pie)
|
||||
const feedFormatCtx = document.getElementById('feedFormatChart');
|
||||
if (feedFormatCtx && !feedFormatChart) {
|
||||
feedFormatChart = new Chart(feedFormatCtx, {
|
||||
type: 'pie',
|
||||
data: {
|
||||
labels: ['RSS', 'ATOM', 'JSON Feed'],
|
||||
datasets: [{
|
||||
data: [
|
||||
{{ feeds.by_format.rss.total|default(0) }},
|
||||
{{ feeds.by_format.atom.total|default(0) }},
|
||||
{{ feeds.by_format.json.total|default(0) }}
|
||||
],
|
||||
backgroundColor: ['#ff6384', '#36a2eb', '#ffce56'],
|
||||
borderWidth: 1
|
||||
}]
|
||||
},
|
||||
options: {
|
||||
responsive: true,
|
||||
maintainAspectRatio: false,
|
||||
plugins: {
|
||||
legend: {
|
||||
position: 'bottom'
|
||||
},
|
||||
title: {
|
||||
display: true,
|
||||
text: 'Feed Format Distribution'
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Feed cache chart (doughnut)
|
||||
const feedCacheCtx = document.getElementById('feedCacheChart');
|
||||
if (feedCacheCtx && !feedCacheChart) {
|
||||
feedCacheChart = new Chart(feedCacheCtx, {
|
||||
type: 'doughnut',
|
||||
data: {
|
||||
labels: ['Cache Hits', 'Cache Misses'],
|
||||
datasets: [{
|
||||
data: [
|
||||
{{ feeds.cache.hits|default(0) }},
|
||||
{{ feeds.cache.misses|default(0) }}
|
||||
],
|
||||
backgroundColor: ['#28a745', '#dc3545'],
|
||||
borderWidth: 1
|
||||
}]
|
||||
},
|
||||
options: {
|
||||
responsive: true,
|
||||
maintainAspectRatio: false,
|
||||
plugins: {
|
||||
legend: {
|
||||
position: 'bottom'
|
||||
},
|
||||
title: {
|
||||
display: true,
|
||||
text: 'Cache Hit/Miss Ratio'
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Update dashboard with new data from htmx
|
||||
function updateDashboard(event) {
|
||||
if (!event.detail.xhr) return;
|
||||
|
||||
try {
|
||||
const data = JSON.parse(event.detail.xhr.responseText);
|
||||
|
||||
// Update pool statistics
|
||||
if (data.database && data.database.pool) {
|
||||
const pool = data.database.pool;
|
||||
document.getElementById('pool-active').textContent = pool.active_connections || 0;
|
||||
document.getElementById('pool-idle').textContent = pool.idle_connections || 0;
|
||||
document.getElementById('pool-total').textContent = pool.total_connections || 0;
|
||||
document.getElementById('pool-size').textContent = pool.pool_size || 5;
|
||||
|
||||
// Update pool chart
|
||||
if (poolChart) {
|
||||
poolChart.data.datasets[0].data = [
|
||||
pool.active_connections || 0,
|
||||
pool.idle_connections || 0
|
||||
];
|
||||
poolChart.update();
|
||||
}
|
||||
}
|
||||
|
||||
// Update performance metrics
|
||||
if (data.performance) {
|
||||
const perf = data.performance;
|
||||
|
||||
// Database
|
||||
if (perf.database) {
|
||||
document.getElementById('db-total').textContent = perf.database.count || 0;
|
||||
document.getElementById('db-avg').textContent = (perf.database.avg || 0).toFixed(2) + ' ms';
|
||||
document.getElementById('db-min').textContent = (perf.database.min || 0).toFixed(2) + ' ms';
|
||||
document.getElementById('db-max').textContent = (perf.database.max || 0).toFixed(2) + ' ms';
|
||||
}
|
||||
|
||||
// HTTP
|
||||
if (perf.http) {
|
||||
document.getElementById('http-total').textContent = perf.http.count || 0;
|
||||
document.getElementById('http-avg').textContent = (perf.http.avg || 0).toFixed(2) + ' ms';
|
||||
document.getElementById('http-min').textContent = (perf.http.min || 0).toFixed(2) + ' ms';
|
||||
document.getElementById('http-max').textContent = (perf.http.max || 0).toFixed(2) + ' ms';
|
||||
}
|
||||
|
||||
// Render
|
||||
if (perf.render) {
|
||||
document.getElementById('render-total').textContent = perf.render.count || 0;
|
||||
document.getElementById('render-avg').textContent = (perf.render.avg || 0).toFixed(2) + ' ms';
|
||||
document.getElementById('render-min').textContent = (perf.render.min || 0).toFixed(2) + ' ms';
|
||||
document.getElementById('render-max').textContent = (perf.render.max || 0).toFixed(2) + ' ms';
|
||||
}
|
||||
|
||||
// Update performance chart
|
||||
if (performanceChart && perf.database && perf.http && perf.render) {
|
||||
performanceChart.data.datasets[0].data = [
|
||||
perf.database.avg || 0,
|
||||
perf.http.avg || 0,
|
||||
perf.render.avg || 0
|
||||
];
|
||||
performanceChart.update();
|
||||
}
|
||||
}
|
||||
|
||||
// Update feed statistics
|
||||
if (data.feeds) {
|
||||
const feeds = data.feeds;
|
||||
|
||||
// Feed requests by format
|
||||
if (feeds.by_format) {
|
||||
document.getElementById('feed-rss-total').textContent = feeds.by_format.rss?.total || 0;
|
||||
document.getElementById('feed-atom-total').textContent = feeds.by_format.atom?.total || 0;
|
||||
document.getElementById('feed-json-total').textContent = feeds.by_format.json?.total || 0;
|
||||
document.getElementById('feed-total').textContent = feeds.total_requests || 0;
|
||||
|
||||
// Feed generation performance
|
||||
document.getElementById('feed-rss-avg').textContent = (feeds.by_format.rss?.avg_duration_ms || 0).toFixed(2) + ' ms';
|
||||
document.getElementById('feed-atom-avg').textContent = (feeds.by_format.atom?.avg_duration_ms || 0).toFixed(2) + ' ms';
|
||||
document.getElementById('feed-json-avg').textContent = (feeds.by_format.json?.avg_duration_ms || 0).toFixed(2) + ' ms';
|
||||
|
||||
// Update feed format chart
|
||||
if (feedFormatChart) {
|
||||
feedFormatChart.data.datasets[0].data = [
|
||||
feeds.by_format.rss?.total || 0,
|
||||
feeds.by_format.atom?.total || 0,
|
||||
feeds.by_format.json?.total || 0
|
||||
];
|
||||
feedFormatChart.update();
|
||||
}
|
||||
}
|
||||
|
||||
// Feed cache statistics
|
||||
if (feeds.cache) {
|
||||
document.getElementById('feed-cache-hits').textContent = feeds.cache.hits || 0;
|
||||
document.getElementById('feed-cache-misses').textContent = feeds.cache.misses || 0;
|
||||
document.getElementById('feed-cache-hit-rate').textContent = ((feeds.cache.hit_rate || 0) * 100).toFixed(1) + '%';
|
||||
document.getElementById('feed-cache-entries').textContent = feeds.cache.entries || 0;
|
||||
|
||||
// Update feed cache chart
|
||||
if (feedCacheChart) {
|
||||
feedCacheChart.data.datasets[0].data = [
|
||||
feeds.cache.hits || 0,
|
||||
feeds.cache.misses || 0
|
||||
];
|
||||
feedCacheChart.update();
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.error('Error updating dashboard:', e);
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize charts when DOM is ready
|
||||
if (document.readyState === 'loading') {
|
||||
document.addEventListener('DOMContentLoaded', initCharts);
|
||||
} else {
|
||||
initCharts();
|
||||
}
|
||||
</script>
|
||||
{% endblock %}
|
||||
@@ -16,6 +16,7 @@
|
||||
<div class="detail-row">
|
||||
<strong>Source:</strong>
|
||||
<a href="{{ webmention.source_url }}" target="_blank">{{ webmention.source_url }}</a>
|
||||
{% if webmention.origin and webmention.origin != 'webmention' %}<small>(backfed from {{ webmention.origin }})</small>{% endif %}
|
||||
</div>
|
||||
|
||||
<div class="detail-row">
|
||||
@@ -43,7 +44,9 @@
|
||||
{% if webmention.author_photo %}
|
||||
<div class="detail-row">
|
||||
<strong>Author Photo:</strong>
|
||||
<img src="{{ webmention.author_photo }}" alt="{{ webmention.author_name or 'Author' }}" width="48" height="48" loading="lazy">
|
||||
{% set photo = webmention.author_photo|local_avatar %}
|
||||
{% if photo %}<img src="{{ photo }}" alt="{{ webmention.author_name or 'Author' }}" width="48" height="48" loading="lazy">{% endif %}
|
||||
<a href="{{ webmention.author_photo }}" rel="noopener" target="_blank">{{ webmention.author_photo|truncate(60) }}</a>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
|
||||
@@ -12,6 +12,27 @@
|
||||
</a>
|
||||
</div>
|
||||
|
||||
{# Backfeed (v2.13.0, ADR-073) #}
|
||||
{% if backfeed and backfeed.enabled %}
|
||||
<section class="backfeed-status">
|
||||
<h3>Backfeed</h3>
|
||||
{% for silo, message in backfeed.denied.items() %}
|
||||
<p class="flash flash-error">{{ silo }}: {{ message }}</p>
|
||||
{% endfor %}
|
||||
{% if backfeed.remaining %}
|
||||
<p>Backfill started {{ backfeed.started }} UTC: {{ backfeed.remaining }} crosspost(s) left to check.</p>
|
||||
{% elif backfeed.finished %}
|
||||
<p>Last backfill: started {{ backfeed.started }}, finished {{ backfeed.finished }} UTC.</p>
|
||||
{% else %}
|
||||
<p>No backfill has been run. It checks every crosspost once for responses, however old the note.</p>
|
||||
{% endif %}
|
||||
{% if backfeed.errors %}<p>{{ backfeed.errors }} crosspost(s) had an error at their last check (see each note's edit page).</p>{% endif %}
|
||||
<form method="POST" action="{{ url_for('admin.backfeed_backfill') }}"><input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
|
||||
<button type="submit" class="button"{% if backfeed.remaining %} disabled{% endif %}>Backfill responses</button>
|
||||
</form>
|
||||
</section>
|
||||
{% endif %}
|
||||
|
||||
{# Filter tabs #}
|
||||
<div class="filter-tabs">
|
||||
<a href="{{ url_for('admin.webmentions') }}" class="{% if not status_filter %}active{% endif %}">
|
||||
@@ -26,6 +47,14 @@
|
||||
<a href="{{ url_for('admin.webmentions', status='hidden') }}" class="{% if status_filter == 'hidden' %}active{% endif %}">
|
||||
Hidden
|
||||
</a>
|
||||
<a href="{{ url_for('admin.webmentions', status='deleted') }}" class="{% if status_filter == 'deleted' %}active{% endif %}">
|
||||
Deleted
|
||||
</a>
|
||||
</div>
|
||||
<div class="filter-tabs">
|
||||
{% for value, label in [(None, 'Any source'), ('webmention', 'Webmentions'), ('gotosocial', 'GoToSocial'), ('bluesky', 'Bluesky'), ('threads', 'Threads')] %}
|
||||
<a href="{{ url_for('admin.webmentions', status=status_filter, origin=value) }}" class="{% if origin_filter == value %}active{% endif %}">{{ label }}</a>
|
||||
{% endfor %}
|
||||
</div>
|
||||
|
||||
{% if webmentions %}
|
||||
@@ -93,11 +122,11 @@
|
||||
{% if total_pages > 1 %}
|
||||
<nav class="pagination">
|
||||
{% if page > 1 %}
|
||||
<a href="?page={{ page - 1 }}{% if status_filter %}&status={{ status_filter }}{% endif %}" class="button">Previous</a>
|
||||
<a href="?page={{ page - 1 }}{% if status_filter %}&status={{ status_filter }}{% endif %}{% if origin_filter %}&origin={{ origin_filter }}{% endif %}" class="button">Previous</a>
|
||||
{% endif %}
|
||||
<span>Page {{ page }} of {{ total_pages }}</span>
|
||||
{% if page < total_pages %}
|
||||
<a href="?page={{ page + 1 }}{% if status_filter %}&status={{ status_filter }}{% endif %}" class="button">Next</a>
|
||||
<a href="?page={{ page + 1 }}{% if status_filter %}&status={{ status_filter }}{% endif %}{% if origin_filter %}&origin={{ origin_filter }}{% endif %}" class="button">Next</a>
|
||||
{% endif %}
|
||||
</nav>
|
||||
{% endif %}
|
||||
|
||||
+9
-20
@@ -78,12 +78,15 @@
|
||||
{% endif %}
|
||||
</footer>
|
||||
|
||||
{{ responses.reactions(likes, reposts, replies, bookmarks, syndications, absolute_url_for('public.note', slug=note.slug)) }}
|
||||
|
||||
<nav class="note-nav">
|
||||
<a href="/">← All notes</a>
|
||||
</nav>
|
||||
|
||||
{# Webmentions section (v1.6.0 Phase 5) - only shown if approved webmentions exist #}
|
||||
{% if webmentions %}
|
||||
{# Webmentions section (v1.6.0 Phase 5): replies, quotes and mentions.
|
||||
Likes, reposts and bookmarks are the counts line above (v2.13.2). #}
|
||||
{% if replies or quotes or mentions %}
|
||||
<section class="webmentions">
|
||||
<h2>Responses</h2>
|
||||
|
||||
@@ -94,24 +97,10 @@
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{% if likes %}
|
||||
<div class="webmention-group webmention-likes">
|
||||
<h3>{{ likes|length }} {{ 'Like' if likes|length == 1 else 'Likes' }}</h3>
|
||||
{{ responses.facepile(likes, 'u-like') }}
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{% if reposts %}
|
||||
<div class="webmention-group webmention-reposts">
|
||||
<h3>{{ reposts|length }} {{ 'Repost' if reposts|length == 1 else 'Reposts' }}</h3>
|
||||
{{ responses.facepile(reposts, 'u-repost') }}
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{% if bookmarks %}
|
||||
<div class="webmention-group webmention-bookmarks">
|
||||
<h3>{{ bookmarks|length }} {{ 'Bookmark' if bookmarks|length == 1 else 'Bookmarks' }}</h3>
|
||||
{{ responses.facepile(bookmarks, 'u-bookmark') }}
|
||||
{% if quotes %}
|
||||
<div class="webmention-group webmention-quotes">
|
||||
<h3>{{ quotes|length }} {{ 'Quote' if quotes|length == 1 else 'Quotes' }}</h3>
|
||||
{% for wm in quotes %}{{ responses.mention(wm) }}{% endfor %}
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
|
||||
@@ -12,11 +12,18 @@
|
||||
{# The response's own URL: the source's u-url if it gave one #}
|
||||
{% macro response_url(wm) %}{{ wm.source_post_url or wm.source_url }}{% endmacro %}
|
||||
|
||||
{# Where a backfed response came from, e.g. "via Bluesky" (v2.13.0) #}
|
||||
{% macro via(wm) %}{% if wm.via %}<span class="webmention-via">via {{ wm.via }}</span>{% endif %}{% endmacro %}
|
||||
|
||||
{# Author h-card: name, URL and photo. u-url is on an inner link: on the
|
||||
h-card element itself it would belong to the enclosing h-cite #}
|
||||
h-card element itself it would belong to the enclosing h-cite.
|
||||
The photo is this site's saved copy (v2.13.0); until there is one, the
|
||||
initial is shown instead of loading it from the other site. #}
|
||||
{% macro author_card(wm, class='', photo_size=48, show_name=True) %}
|
||||
<span class="p-author h-card {{ class }}"><a class="u-url" href="{{ wm.author_url or wm.source_url }}" title="{{ wm.author_name or 'Anonymous' }}">
|
||||
{%- if wm.author_photo %}<img class="u-photo" src="{{ wm.author_photo }}" alt="" width="{{ photo_size }}" height="{{ photo_size }}" loading="lazy">
|
||||
{% set photo = wm.author_photo|local_avatar %}
|
||||
<span class="p-author h-card {{ class }}"><a class="u-url" href="{{ wm.author_url or wm.source_url }}" title="{{ wm.author_name or 'Anonymous' }}{% if wm.via %} via {{ wm.via }}{% endif %}">
|
||||
{%- if photo %}<img class="u-photo" src="{{ photo }}" alt="" width="{{ photo_size }}" height="{{ photo_size }}" loading="lazy">
|
||||
{%- elif wm.author_photo %}<data class="u-photo" value="{{ wm.author_photo }}"></data>{% if not show_name %}<span class="face-initial" style="background-color: {{ wm.author_url|url_to_color }}" aria-hidden="true">{{ (wm.author_name or '?')[0]|upper }}</span>{% endif %}
|
||||
{%- elif not show_name %}<span class="face-initial" style="background-color: {{ wm.author_url|url_to_color }}" aria-hidden="true">{{ (wm.author_name or '?')[0]|upper }}</span>
|
||||
{%- endif %}
|
||||
<span class="p-name{{ '' if show_name else ' visually-hidden' }}">{{ wm.author_name or 'Anonymous' }}</span>
|
||||
@@ -30,33 +37,77 @@
|
||||
{% if wm.content_snippet %}
|
||||
<div class="e-content">{{ wm.content_snippet|safe }}</div>
|
||||
{% endif %}
|
||||
<a class="u-url webmention-date" href="{{ response_url(wm) }}">{{ response_time(wm) }}</a>
|
||||
<a class="u-url webmention-date" href="{{ response_url(wm) }}">{{ response_time(wm) }}</a> {{ via(wm) }}
|
||||
</article>
|
||||
{%- endmacro %}
|
||||
|
||||
{# Likes, reposts or bookmarks as a facepile; prop is u-like / u-repost / u-bookmark.
|
||||
Faces past `shown` are hidden on screen but kept for parsers. #}
|
||||
{% macro facepile(items, prop, shown=8) %}
|
||||
<div class="facepile">
|
||||
{% for wm in items %}
|
||||
<span class="face {{ prop }} h-cite"{% if loop.index > shown %} hidden{% endif %}>
|
||||
{{ author_card(wm, photo_size=32, show_name=False) }}
|
||||
<a class="u-url" href="{{ response_url(wm) }}" hidden>{{ response_time(wm) }}</a>
|
||||
</span>
|
||||
{% endfor %}
|
||||
{% if items|length > shown %}
|
||||
<span class="face-overflow">+{{ items|length - shown }} others</span>
|
||||
{% endif %}
|
||||
</div>
|
||||
{%- endmacro %}
|
||||
|
||||
{# A post that mentions this note without replying #}
|
||||
{# A post that mentions this note without replying; also quote posts (v2.13.0) #}
|
||||
{% macro mention(wm) %}
|
||||
<div class="webmention webmention-mention u-mention h-cite">
|
||||
{{ author_card(wm, photo_size=24) }}
|
||||
{% if wm.content_snippet %}
|
||||
<p class="p-content">{{ wm.content_snippet|striptags|truncate(200) }}</p>
|
||||
{% endif %}
|
||||
<a class="u-url webmention-date" href="{{ response_url(wm) }}">{{ response_time(wm) }}</a>
|
||||
<a class="u-url webmention-date" href="{{ response_url(wm) }}">{{ response_time(wm) }}</a> {{ via(wm) }}
|
||||
</div>
|
||||
{%- endmacro %}
|
||||
|
||||
{# Likes, reposts, replies and bookmarks: one line of counts under the note,
|
||||
like other sites (v2.13.2). Hovering or focusing a count lists who, and
|
||||
offers the same action on the note's crossposts. Each action is an
|
||||
<indie-action> (https://indieweb.org/webactions) with those silo links as
|
||||
its fallback, so web action handlers can take over. Counts a silo gives
|
||||
without names (Threads) are added to the total. The people stay in the
|
||||
markup as u-like / u-repost / u-bookmark h-cites. #}
|
||||
{% set ICONS = {
|
||||
'like': '<svg viewBox="0 0 24 24" fill="currentColor" aria-hidden="true"><path d="M12 21.35l-1.45-1.32C5.4 15.36 2 12.28 2 8.5 2 5.42 4.42 3 7.5 3c1.74 0 3.41.81 4.5 2.09C13.09 3.81 14.76 3 16.5 3 19.58 3 22 5.42 22 8.5c0 3.78-3.4 6.86-8.55 11.54L12 21.35z"/></svg>',
|
||||
'repost': '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M17 1l4 4-4 4"/><path d="M3 11V9a4 4 0 0 1 4-4h14"/><path d="M7 23l-4-4 4-4"/><path d="M21 13v2a4 4 0 0 1-4 4H3"/></svg>',
|
||||
'reply': '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M21 11.5a8.4 8.4 0 0 1-9 8.4 8.6 8.6 0 0 1-3.9-.9L3 21l1.9-5.1A8.4 8.4 0 0 1 12 3.1a8.4 8.4 0 0 1 9 8.4z"/></svg>',
|
||||
'bookmark': '<svg viewBox="0 0 24 24" fill="currentColor" aria-hidden="true"><path d="M6 2h12a1 1 0 0 1 1 1v19l-7-4-7 4V3a1 1 0 0 1 1-1z"/></svg>',
|
||||
} %}
|
||||
{% set VERBS = {'like': 'Like', 'repost': 'Repost', 'reply': 'Reply'} %}
|
||||
|
||||
{% macro reaction(kind, items, prop, anon, one, many, copies=[], permalink='', people=True) %}
|
||||
{% set ns = namespace(total=items|length) %}
|
||||
{% for name, n in anon %}{% set ns.total = ns.total + n %}{% endfor %}
|
||||
{% set verb = VERBS.get(kind) %}
|
||||
{% if ns.total or (verb and copies) %}
|
||||
<span class="reaction reaction-{{ kind }}" tabindex="0">
|
||||
{{ ICONS[kind]|safe }}{% if ns.total %}<span class="reaction-count">{{ ns.total }}</span>{% endif %}<span class="visually-hidden"> {{ ns.total }} {{ one if ns.total == 1 else many }}</span>
|
||||
<span class="reaction-who" role="tooltip">
|
||||
{% if ns.total %}<span class="reaction-title">{{ ns.total }} {{ one if ns.total == 1 else many }}</span>{% endif %}
|
||||
{% if (people and items) or anon %}
|
||||
<span class="reaction-people">
|
||||
{% if people %}
|
||||
{% for wm in items %}
|
||||
<span class="reaction-person {{ prop }} h-cite">{{ author_card(wm, photo_size=20) }}{% if wm.via %}<span class="webmention-via">{{ wm.via }}</span>{% endif %}<a class="u-url" href="{{ response_url(wm) }}" hidden>{{ response_time(wm) }}</a></span>
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% for name, n in anon %}<span class="reaction-anon">{{ '+' if items and people else '' }}{{ n }} on {{ name }}</span>{% endfor %}
|
||||
</span>
|
||||
{% endif %}
|
||||
{% if verb and copies %}
|
||||
<indie-action class="reaction-elsewhere" do="{{ kind }}" with="{{ permalink }}">{{ verb }} on {% for s in copies %}<a href="{{ s.remote_url }}" rel="noopener">{{ s.name }}</a>{% if not loop.last %} · {% endif %}{% endfor %}</indie-action>
|
||||
{% endif %}
|
||||
</span>
|
||||
</span>
|
||||
{% endif %}
|
||||
{%- endmacro %}
|
||||
|
||||
{% macro reactions(likes, reposts, replies, bookmarks, syndications, permalink) %}
|
||||
{% set anon_likes, anon_reposts = [], [] %}
|
||||
{% for s in syndications or [] %}
|
||||
{% if s.likes_count %}{% set _ = anon_likes.append((s.name, s.likes_count)) %}{% endif %}
|
||||
{% if s.reposts_count %}{% set _ = anon_reposts.append((s.name, s.reposts_count)) %}{% endif %}
|
||||
{% endfor %}
|
||||
{% set copies = (syndications or [])|selectattr('remote_url')|list %}
|
||||
{% if copies or likes or reposts or replies or bookmarks or anon_likes or anon_reposts %}
|
||||
<div class="reactions">
|
||||
{{ reaction('like', likes or [], 'u-like', anon_likes, 'like', 'likes', copies, permalink) }}
|
||||
{{ reaction('repost', reposts or [], 'u-repost', anon_reposts, 'repost', 'reposts', copies, permalink) }}
|
||||
{# Replies are listed in full below; here just the count and where to reply #}
|
||||
{{ reaction('reply', replies or [], '', [], 'reply', 'replies', copies, permalink, people=False) }}
|
||||
{{ reaction('bookmark', bookmarks or [], 'u-bookmark', [], 'bookmark', 'bookmarks') }}
|
||||
</div>
|
||||
{% endif %}
|
||||
{%- endmacro %}
|
||||
@@ -1,108 +0,0 @@
|
||||
"""
|
||||
Integration tests for feed statistics in admin dashboard
|
||||
|
||||
Tests the feed statistics features in /admin/metrics-dashboard and /admin/metrics
|
||||
per v1.1.2 Phase 3.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from starpunk.auth import create_session
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def authenticated_client(app, client):
|
||||
"""Client with authenticated session"""
|
||||
with app.test_request_context():
|
||||
# Create a session for the test user
|
||||
session_token = create_session(app.config["ADMIN_ME"])
|
||||
|
||||
# Set session cookie
|
||||
client.set_cookie("starpunk_session", session_token)
|
||||
return client
|
||||
|
||||
|
||||
def test_feed_statistics_dashboard_endpoint(authenticated_client):
|
||||
"""Test metrics dashboard includes feed statistics section"""
|
||||
response = authenticated_client.get("/admin/metrics-dashboard")
|
||||
|
||||
assert response.status_code == 200
|
||||
|
||||
# Should contain feed statistics section
|
||||
assert b"Feed Statistics" in response.data
|
||||
assert b"Feed Requests by Format" in response.data
|
||||
assert b"Feed Cache Statistics" in response.data
|
||||
assert b"Feed Generation Performance" in response.data
|
||||
|
||||
# Should have chart canvases
|
||||
assert b'id="feedFormatChart"' in response.data
|
||||
assert b'id="feedCacheChart"' in response.data
|
||||
|
||||
|
||||
def test_feed_statistics_metrics_endpoint(authenticated_client):
|
||||
"""Test /admin/metrics endpoint includes feed statistics"""
|
||||
response = authenticated_client.get("/admin/metrics")
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.get_json()
|
||||
|
||||
# Should have feeds key
|
||||
assert "feeds" in data
|
||||
|
||||
# Should have expected structure
|
||||
feeds = data["feeds"]
|
||||
if "error" not in feeds:
|
||||
assert "by_format" in feeds
|
||||
assert "cache" in feeds
|
||||
assert "total_requests" in feeds
|
||||
assert "format_percentages" in feeds
|
||||
|
||||
# Check format structure
|
||||
for format_name in ["rss", "atom", "json"]:
|
||||
assert format_name in feeds["by_format"]
|
||||
fmt = feeds["by_format"][format_name]
|
||||
assert "generated" in fmt
|
||||
assert "cached" in fmt
|
||||
assert "total" in fmt
|
||||
assert "avg_duration_ms" in fmt
|
||||
|
||||
# Check cache structure
|
||||
assert "hits" in feeds["cache"]
|
||||
assert "misses" in feeds["cache"]
|
||||
assert "hit_rate" in feeds["cache"]
|
||||
|
||||
|
||||
def test_feed_statistics_after_feed_request(authenticated_client):
|
||||
"""Test feed statistics track actual feed requests"""
|
||||
# Make a feed request
|
||||
response = authenticated_client.get("/feed.rss")
|
||||
assert response.status_code == 200
|
||||
|
||||
# Check metrics endpoint now has data
|
||||
response = authenticated_client.get("/admin/metrics")
|
||||
assert response.status_code == 200
|
||||
data = response.get_json()
|
||||
|
||||
# Should have feeds data
|
||||
assert "feeds" in data
|
||||
feeds = data["feeds"]
|
||||
|
||||
# May have requests tracked (depends on metrics buffer timing)
|
||||
# Just verify structure is correct
|
||||
assert "total_requests" in feeds
|
||||
assert feeds["total_requests"] >= 0
|
||||
|
||||
|
||||
def test_dashboard_requires_auth_for_feed_stats(client):
|
||||
"""Test dashboard requires authentication (even for feed stats)"""
|
||||
response = client.get("/admin/metrics-dashboard")
|
||||
|
||||
# Should redirect to auth or return 401/403
|
||||
assert response.status_code in [302, 401, 403]
|
||||
|
||||
|
||||
def test_metrics_endpoint_requires_auth_for_feed_stats(client):
|
||||
"""Test metrics endpoint requires authentication"""
|
||||
response = client.get("/admin/metrics")
|
||||
|
||||
# Should redirect to auth or return 401/403
|
||||
assert response.status_code in [302, 401, 403]
|
||||
@@ -0,0 +1,606 @@
|
||||
"""
|
||||
Tests for backfeed of responses to crossposts (v2.13.0, ADR-073)
|
||||
|
||||
Silo clients are tested against fake APIs (httpx.MockTransport). The
|
||||
worker, storage and pages are tested with a fake client that returns
|
||||
whatever a test puts in it.
|
||||
"""
|
||||
|
||||
import io
|
||||
import json
|
||||
from urllib.parse import parse_qs
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from starpunk import avatars, backfeed, create_app, syndication
|
||||
from starpunk.backfeed import Harvest, Response, facets_html, interval_for, text_html
|
||||
from starpunk.database import get_db
|
||||
from starpunk.notes import create_note
|
||||
from starpunk.syndication import BlueskyClient, MastodonClient, SyndicationError, ThreadsClient
|
||||
|
||||
SITE = "https://blog.example.com"
|
||||
GTS = "https://social.example"
|
||||
COPY = f"{GTS}/@phil/statuses/01COPY"
|
||||
|
||||
|
||||
# --- Fakes --------------------------------------------------------------------
|
||||
|
||||
class FakeClient:
|
||||
"""Stands in for a silo client: returns the harvest a test sets"""
|
||||
config_token = "token"
|
||||
|
||||
def __init__(self):
|
||||
self.harvest = Harvest([])
|
||||
self.error = None
|
||||
self.calls = []
|
||||
|
||||
def responses(self, remote_id, remote_url, cap=1000):
|
||||
self.calls.append(remote_id)
|
||||
if self.error:
|
||||
raise self.error
|
||||
return self.harvest
|
||||
|
||||
def maintain(self):
|
||||
pass
|
||||
|
||||
|
||||
def like(n, bio="", **kw):
|
||||
return Response(kind="like", source_url=f"{COPY}#liked-by-{n}", remote_id=str(n),
|
||||
author_id=str(n), author_name=f"Liker {n}", author_url=f"{GTS}/@l{n}",
|
||||
author_bio=bio, **kw)
|
||||
|
||||
|
||||
def reply(n, text="Nice!"):
|
||||
return Response(kind="reply", source_url=f"{GTS}/@r{n}/statuses/{n}", remote_id=str(n),
|
||||
author_id=f"r{n}", author_name=f"Replier {n}", author_url=f"{GTS}/@r{n}",
|
||||
content_html=f"<p>{text}</p>", published="2026-11-10T12:00:00Z")
|
||||
|
||||
|
||||
def resolve(host, *args, **kwargs):
|
||||
return [(2, 1, 6, "", ("93.184.216.34", 0))]
|
||||
|
||||
|
||||
def png(color="red", size=(200, 120)):
|
||||
buf = io.BytesIO()
|
||||
Image.new("RGB", size, color).save(buf, "PNG")
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
class FakeWeb:
|
||||
"""Avatar images by URL (requests are pinned to an IP; Host names the site)"""
|
||||
def __init__(self):
|
||||
self.files = {}
|
||||
|
||||
def __call__(self, request):
|
||||
url = f"{request.url.scheme}://{request.headers['host']}{request.url.raw_path.decode()}"
|
||||
if url in self.files:
|
||||
return httpx.Response(200, headers={"content-type": "image/png"}, content=self.files[url])
|
||||
return httpx.Response(404)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def web(monkeypatch):
|
||||
fake = FakeWeb()
|
||||
monkeypatch.setattr("starpunk.webmention_ssrf.TRANSPORT", httpx.MockTransport(fake))
|
||||
monkeypatch.setattr("starpunk.webmention_ssrf.socket.getaddrinfo", resolve)
|
||||
return fake
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
return FakeClient()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def app(tmp_path, client, web):
|
||||
data = tmp_path / "data"
|
||||
app = create_app({
|
||||
"TESTING": True,
|
||||
"SITE_URL": SITE,
|
||||
"ADMIN_ME": "https://example.com",
|
||||
"SESSION_SECRET": "test-secret",
|
||||
"DATA_PATH": data,
|
||||
"NOTES_PATH": data / "notes",
|
||||
"DATABASE_PATH": data / "starpunk.db",
|
||||
"GOTOSOCIAL_URL": GTS,
|
||||
"GOTOSOCIAL_ACCESS_TOKEN": "token",
|
||||
"BACKFEED_ENABLED": True,
|
||||
})
|
||||
syndication._client_cache[GTS] = client
|
||||
backfeed.denied.clear()
|
||||
yield app
|
||||
syndication._client_cache.clear()
|
||||
backfeed.denied.clear()
|
||||
|
||||
|
||||
def crossposted(app, age_days=0, content="Hello"):
|
||||
"""A published note with a synced GoToSocial copy; returns (note, syndication id)"""
|
||||
with app.app_context():
|
||||
note = create_note(content=content, published=True)
|
||||
db = get_db(app)
|
||||
if age_days:
|
||||
db.execute("UPDATE notes SET created_at = datetime('now', ?) WHERE id = ?",
|
||||
(f"-{age_days} days", note.id))
|
||||
cursor = db.execute(
|
||||
"INSERT INTO syndications (note_id, target, enabled, remote_id, remote_url, state) "
|
||||
"VALUES (?, 'gotosocial', 1, '01COPY', ?, 'synced')", (note.id, COPY))
|
||||
db.commit()
|
||||
return note, cursor.lastrowid
|
||||
|
||||
|
||||
def rows(app, note_id):
|
||||
with app.app_context():
|
||||
return [dict(r) for r in get_db(app).execute(
|
||||
"SELECT * FROM webmentions WHERE target_note_id = ? ORDER BY source_url", (note_id,))]
|
||||
|
||||
|
||||
def run(app):
|
||||
with app.app_context():
|
||||
return backfeed.process_due()
|
||||
|
||||
|
||||
# --- Schedule and text ------------------------------------------------------------
|
||||
|
||||
def test_schedule_slows_with_age_and_stops_at_cutoff():
|
||||
day = 24 * 3600
|
||||
assert interval_for(3600, 90) == 15 * 60
|
||||
assert interval_for(2 * day, 90) == 3600
|
||||
assert interval_for(10 * day, 90) == day
|
||||
assert interval_for(60 * day, 90) == 7 * day
|
||||
assert interval_for(91 * day, 90) is None
|
||||
|
||||
|
||||
def test_text_and_facets_become_safe_html():
|
||||
assert text_html("hi <b>\nsee https://a.example/x") == \
|
||||
'<p>hi <b><br>see <a href="https://a.example/x">https://a.example/x</a></p>'
|
||||
text = "café: example.com/long…"
|
||||
start = len("café: ".encode())
|
||||
facets = [{"index": {"byteStart": start, "byteEnd": len(text.encode())},
|
||||
"features": [{"$type": "app.bsky.richtext.facet#link", "uri": "https://example.com/long/path"}]}]
|
||||
assert facets_html(text, facets) == \
|
||||
'<p>café: <a href="https://example.com/long/path">example.com/long…</a></p>'
|
||||
assert facets_html("", []) == ""
|
||||
|
||||
|
||||
# --- Worker and storage -------------------------------------------------------------
|
||||
|
||||
def test_check_stores_responses_as_webmentions(app, client):
|
||||
note, sid = crossposted(app)
|
||||
client.harvest = Harvest([like(1), reply(2, "Great <script>x</script>post")])
|
||||
assert run(app) == 1
|
||||
stored = rows(app, note.id)
|
||||
assert [r["mention_type"] for r in stored] == ["like", "reply"]
|
||||
assert all(r["origin"] == "gotosocial" and r["validation_status"] == "valid"
|
||||
and r["moderation_status"] == "approved" for r in stored)
|
||||
assert stored[1]["content_snippet"] == "<p>Great post</p>"
|
||||
assert stored[0]["target_url"] == f"{SITE}/note/{note.slug}"
|
||||
with app.app_context():
|
||||
s = get_db(app).execute("SELECT * FROM syndications WHERE id = ?", (sid,)).fetchone()
|
||||
assert s["backfeed_checked_at"] and s["backfeed_next_at"] and s["backfeed_error"] is None
|
||||
assert run(app) == 0, "not due again for 15 minutes"
|
||||
|
||||
|
||||
def test_repeat_check_updates_rows_and_keeps_moderation(app, client):
|
||||
note, sid = crossposted(app)
|
||||
client.harvest = Harvest([like(1), like(2)])
|
||||
run(app)
|
||||
with app.app_context():
|
||||
db = get_db(app)
|
||||
db.execute("UPDATE webmentions SET moderation_status = 'hidden' WHERE source_url LIKE '%-1'")
|
||||
db.execute("UPDATE syndications SET backfeed_next_at = datetime('now')")
|
||||
db.commit()
|
||||
client.harvest = Harvest([like(1), like(2)])
|
||||
run(app)
|
||||
stored = rows(app, note.id)
|
||||
assert len(stored) == 2
|
||||
assert [r["moderation_status"] for r in stored] == ["hidden", "approved"]
|
||||
|
||||
|
||||
def test_withdrawn_responses_drop_off_and_come_back(app, client):
|
||||
note, _ = crossposted(app)
|
||||
client.harvest = Harvest([like(1), reply(2)])
|
||||
run(app)
|
||||
|
||||
def recheck(harvest):
|
||||
with app.app_context():
|
||||
get_db(app).execute("UPDATE syndications SET backfeed_next_at = datetime('now')")
|
||||
get_db(app).commit()
|
||||
client.harvest = harvest
|
||||
run(app)
|
||||
return {r["source_url"]: (r["validation_status"], r["validation_error"]) for r in rows(app, note.id)}
|
||||
|
||||
# An incomplete list (cap reached, say) removes nothing
|
||||
assert all(v == ("valid", None) for v in recheck(Harvest([like(1)], complete=False)).values())
|
||||
state = recheck(Harvest([like(1)]))
|
||||
assert state[reply(2).source_url] == ("invalid", "No longer on GoToSocial")
|
||||
assert recheck(Harvest([like(1), reply(2)]))[reply(2).source_url] == ("valid", None)
|
||||
|
||||
|
||||
def test_people_who_opt_out_are_skipped_and_removed(app, client):
|
||||
note, _ = crossposted(app)
|
||||
client.harvest = Harvest([like(1), like(2)])
|
||||
run(app)
|
||||
with app.app_context():
|
||||
get_db(app).execute("UPDATE syndications SET backfeed_next_at = datetime('now')")
|
||||
get_db(app).commit()
|
||||
client.harvest = Harvest([like(1, bio="<p>Please #NoBridge thanks</p>"), like(2),
|
||||
like(3, opted_out=True)])
|
||||
run(app)
|
||||
assert [r["source_url"] for r in rows(app, note.id)] == [like(2).source_url]
|
||||
|
||||
|
||||
def test_admin_delete_is_a_tombstone_the_next_check_keeps(app, client):
|
||||
from starpunk.webmention_queries import count_webmentions, delete_webmentions, list_webmentions
|
||||
|
||||
note, _ = crossposted(app)
|
||||
client.harvest = Harvest([like(1)])
|
||||
run(app)
|
||||
with app.test_request_context():
|
||||
wm_id = rows(app, note.id)[0]["id"]
|
||||
assert delete_webmentions([wm_id]) == 1
|
||||
get_db(app).execute("UPDATE syndications SET backfeed_next_at = datetime('now')")
|
||||
get_db(app).commit()
|
||||
run(app)
|
||||
assert rows(app, note.id)[0]["moderation_status"] == "deleted"
|
||||
with app.test_request_context():
|
||||
assert list_webmentions()[0] == [] and count_webmentions() == 0
|
||||
assert len(list_webmentions(status_filter="deleted")[0]) == 1
|
||||
|
||||
|
||||
def test_old_notes_wait_for_a_backfill(app, client):
|
||||
old, sid = crossposted(app, age_days=200)
|
||||
client.harvest = Harvest([reply(1)])
|
||||
assert run(app) == 0
|
||||
with app.app_context():
|
||||
assert backfeed.start_backfill() == 1
|
||||
assert backfeed.status()["remaining"] == 1
|
||||
assert run(app) == 1
|
||||
assert len(rows(app, old.id)) == 1
|
||||
with app.app_context():
|
||||
status = backfeed.status()
|
||||
s = get_db(app).execute("SELECT * FROM syndications WHERE id = ?", (sid,)).fetchone()
|
||||
assert status["remaining"] == 0 and status["finished"]
|
||||
assert s["backfeed_backfill"] == 0 and s["backfeed_next_at"] is None, "past the cutoff: no more checks"
|
||||
|
||||
|
||||
def test_backfill_is_paced(app, client):
|
||||
crossposted(app, age_days=200)
|
||||
crossposted(app, age_days=300)
|
||||
with app.app_context():
|
||||
backfeed.start_backfill()
|
||||
assert run(app) == 1
|
||||
assert run(app) == 0, "the next backfill check waits a few seconds"
|
||||
with app.app_context():
|
||||
get_db(app).execute("UPDATE site_state SET value = datetime('now', '-10 seconds') "
|
||||
"WHERE key = 'backfeed_backfill_last'")
|
||||
get_db(app).commit()
|
||||
assert run(app) == 1
|
||||
|
||||
|
||||
def test_routine_checks_skip_likes_reposts_and_unpublished(app, client):
|
||||
with app.app_context():
|
||||
liked = create_note(content="", published=True, like_of="https://other.example/post")
|
||||
draft = create_note(content="Draft", published=False)
|
||||
db = get_db(app)
|
||||
for note in (liked, draft):
|
||||
db.execute("INSERT INTO syndications (note_id, target, enabled, remote_id, remote_url, state) "
|
||||
"VALUES (?, 'gotosocial', 1, 'x', 'https://s.example/x', 'synced')", (note.id,))
|
||||
db.commit()
|
||||
assert run(app) == 0 and client.calls == []
|
||||
|
||||
|
||||
def test_refused_credential_stops_that_silo(app, client):
|
||||
crossposted(app)
|
||||
error = SyndicationError("GET /api/v1/statuses/01COPY/favourited_by: HTTP 403 forbidden", permanent=True)
|
||||
error.status = 403
|
||||
client.error = error
|
||||
run(app)
|
||||
assert "gotosocial" in backfeed.denied and "read:statuses" in backfeed.denied["gotosocial"]
|
||||
client.error = None
|
||||
assert run(app) == 0, "not retried until restart"
|
||||
|
||||
|
||||
def test_failures_back_off(app, client):
|
||||
_, sid = crossposted(app)
|
||||
client.error = SyndicationError("HTTP 502 bad gateway")
|
||||
run(app)
|
||||
with app.app_context():
|
||||
s = get_db(app).execute("SELECT * FROM syndications WHERE id = ?", (sid,)).fetchone()
|
||||
assert s["backfeed_attempts"] == 1 and "502" in s["backfeed_error"]
|
||||
assert run(app) == 0
|
||||
|
||||
|
||||
def test_disabled_does_nothing(app, client):
|
||||
crossposted(app)
|
||||
app.config["BACKFEED_ENABLED"] = False
|
||||
assert run(app) == 0 and client.calls == []
|
||||
|
||||
|
||||
# --- Pages ------------------------------------------------------------------------------
|
||||
|
||||
def test_note_page_shows_backfed_responses_quotes_and_counts(app, client, web):
|
||||
note, sid = crossposted(app)
|
||||
web.files["https://cdn.example/a.png"] = png()
|
||||
quote = Response(kind="quote", source_url="https://bsky.app/profile/q.example/post/3k",
|
||||
remote_id="at://did:plc:q/app.bsky.feed.post/3k", author_name="Quoter",
|
||||
author_url="https://bsky.app/profile/q.example", content_html="<p>Look at this</p>")
|
||||
client.harvest = Harvest([like(1, author_photo="https://cdn.example/a.png"), reply(2), quote])
|
||||
run(app)
|
||||
with app.app_context():
|
||||
get_db(app).execute("UPDATE syndications SET likes_count = 12, reposts_count = 1 WHERE id = ?", (sid,))
|
||||
get_db(app).commit()
|
||||
page = app.test_client().get(f"/note/{note.slug}").get_data(as_text=True)
|
||||
assert "via GoToSocial" in page
|
||||
assert "1 Quote" in page and "Look at this" in page
|
||||
# Likes and reposts are a counts line (v2.13.2): named likes plus the silo's anonymous count
|
||||
assert '<span class="reaction-count">13</span>' in page and "+12 on GoToSocial" in page
|
||||
assert '<span class="reaction-count">1</span>' in page
|
||||
assert f'<indie-action class="reaction-elsewhere" do="like" with="{SITE}/note/{note.slug}">' in page
|
||||
assert f'Like on <a href="{COPY}"' in page
|
||||
assert 'src="/avatars/' in page and 'src="https://cdn.example/a.png"' not in page
|
||||
|
||||
|
||||
def test_avatars_are_saved_served_and_cleaned_up(app, web):
|
||||
url = "https://cdn.example/face.png"
|
||||
web.files[url] = png(size=(300, 200))
|
||||
data = app.config["DATA_PATH"]
|
||||
assert avatars.cached_name(data, url) is None
|
||||
with app.app_context():
|
||||
assert avatars.fetch(data, url, SITE)
|
||||
assert not avatars.fetch(data, "https://cdn.example/missing.png", SITE)
|
||||
name = avatars.cached_name(data, url)
|
||||
with Image.open(avatars.folder(data) / name) as image:
|
||||
assert image.size == (96, 96) and image.format == "WEBP"
|
||||
response = app.test_client().get(f"/avatars/{name}")
|
||||
assert response.status_code == 200 and response.mimetype == "image/webp"
|
||||
assert app.test_client().get("/avatars/../starpunk.db").status_code == 404
|
||||
# Nothing uses it: the weekly cleanup removes it
|
||||
assert avatars.remove_unused(app.config["DATABASE_PATH"], data) == 1
|
||||
|
||||
|
||||
def test_webmention_cannot_overwrite_a_backfed_response(app, client):
|
||||
note, _ = crossposted(app)
|
||||
client.harvest = Harvest([reply(7)])
|
||||
run(app)
|
||||
response = app.test_client().post("/webmention", data={
|
||||
"source": reply(7).source_url, "target": f"{SITE}/note/{note.slug}"})
|
||||
assert response.status_code == 202
|
||||
assert rows(app, note.id)[0]["validation_status"] == "valid"
|
||||
|
||||
|
||||
# --- Silo clients ---------------------------------------------------------------------------
|
||||
|
||||
def mastodon_api(request):
|
||||
path, query = request.url.path, parse_qs(request.url.query.decode())
|
||||
account = lambda n, **kw: {"id": n, "display_name": f"User {n}", "url": f"{GTS}/@u{n}",
|
||||
"avatar": f"{GTS}/a/{n}.png", "note": "", **kw}
|
||||
if path == "/api/v1/accounts/verify_credentials":
|
||||
return httpx.Response(200, json={"id": "me"})
|
||||
if path == "/api/v1/statuses/01COPY/favourited_by":
|
||||
if "max_id" in query:
|
||||
return httpx.Response(200, json=[account("3")])
|
||||
return httpx.Response(200, json=[account("1"), account("me")],
|
||||
headers={"Link": f'<{GTS}/api/v1/statuses/01COPY/favourited_by?max_id=9>; rel="next"'})
|
||||
if path == "/api/v1/statuses/01COPY/reblogged_by":
|
||||
return httpx.Response(200, json=[account("2")])
|
||||
if path == "/api/v1/statuses/01COPY/context":
|
||||
status = lambda n, parent="01COPY", vis="public", who=None: {
|
||||
"id": n, "url": f"{GTS}/@u/statuses/{n}", "uri": f"{GTS}/u/{n}", "in_reply_to_id": parent,
|
||||
"visibility": vis, "content": f"<p>reply {n}</p>", "created_at": "2026-11-10T00:00:00Z",
|
||||
"account": who or account("r" + n)}
|
||||
return httpx.Response(200, json={"ancestors": [], "descendants": [
|
||||
status("10"), status("11", vis="unlisted"), status("12", parent="10"),
|
||||
status("13", who=account("me"))]})
|
||||
return httpx.Response(404)
|
||||
|
||||
|
||||
def test_mastodon_client_reads_likes_boosts_and_public_direct_replies():
|
||||
gts = MastodonClient(GTS, "t", http=httpx.Client(transport=httpx.MockTransport(mastodon_api)))
|
||||
harvest = gts.responses("01COPY", COPY)
|
||||
assert harvest.complete
|
||||
got = [(r.kind, r.source_url) for r in harvest.responses]
|
||||
assert got == [("like", f"{COPY}#liked-by-1"), ("like", f"{COPY}#liked-by-3"),
|
||||
("repost", f"{COPY}#reposted-by-2"), ("reply", f"{GTS}/@u/statuses/10")]
|
||||
assert harvest.responses[-1].content_html == "<p>reply 10</p>"
|
||||
assert gts.responses("01COPY", COPY, cap=1).complete is False
|
||||
|
||||
|
||||
def test_bluesky_client_reads_likes_reposts_quotes_and_replies():
|
||||
me, post = "did:plc:me", "at://did:plc:me/app.bsky.feed.post/abc"
|
||||
|
||||
def api(request):
|
||||
nsid = request.url.path.removeprefix("/xrpc/")
|
||||
query = parse_qs(request.url.query.decode())
|
||||
profile = lambda d, **kw: {"did": d, "handle": f"{d[8:]}.example", "displayName": d[8:].title(), **kw}
|
||||
view = lambda d, rkey, text: {"uri": f"at://{d}/app.bsky.feed.post/{rkey}", "author": profile(d),
|
||||
"record": {"text": text, "createdAt": "2026-11-10T00:00:00Z"}}
|
||||
if nsid == "com.atproto.server.createSession":
|
||||
return httpx.Response(200, json={"did": me, "handle": "me.example", "accessJwt": "a", "refreshJwt": "r"})
|
||||
if nsid == "app.bsky.feed.getLikes":
|
||||
if query.get("cursor"):
|
||||
return httpx.Response(200, json={"likes": [{"actor": profile("did:plc:carol")}]})
|
||||
return httpx.Response(200, json={"cursor": "c1", "likes": [
|
||||
{"actor": profile("did:plc:alice"), "createdAt": "2026-11-10T01:00:00Z"},
|
||||
{"actor": profile("did:plc:shy", labels=[{"val": "!no-unauthenticated"}])},
|
||||
{"actor": profile(me)}]})
|
||||
if nsid == "app.bsky.feed.getRepostedBy":
|
||||
return httpx.Response(200, json={"repostedBy": [profile("did:plc:bob")]})
|
||||
if nsid == "app.bsky.feed.getQuotes":
|
||||
return httpx.Response(200, json={"posts": [view("did:plc:dan", "q1", "Quoting this")]})
|
||||
if nsid == "app.bsky.feed.getPostThread":
|
||||
return httpx.Response(200, json={"thread": {"post": {}, "replies": [
|
||||
{"post": view("did:plc:eve", "r1", "A reply")},
|
||||
{"$type": "app.bsky.feed.defs#blockedPost", "uri": "at://x"}]}})
|
||||
if nsid == "app.bsky.actor.getProfiles":
|
||||
return httpx.Response(200, json={"profiles": [
|
||||
profile("did:plc:dan", description="no thanks #nobot"), profile("did:plc:eve")]})
|
||||
return httpx.Response(400, json={"error": "Unknown"})
|
||||
|
||||
bsky = BlueskyClient("https://bsky.example", "me.example", "pw",
|
||||
http=httpx.Client(transport=httpx.MockTransport(api)))
|
||||
harvest = bsky.responses(post, "https://bsky.app/profile/me.example/post/abc")
|
||||
by = {(r.kind, r.author_id): r for r in harvest.responses}
|
||||
assert set(by) == {("like", "did:plc:alice"), ("like", "did:plc:shy"), ("like", "did:plc:carol"),
|
||||
("repost", "did:plc:bob"), ("quote", "did:plc:dan"), ("reply", "did:plc:eve")}
|
||||
assert by[("like", "did:plc:shy")].opted_out
|
||||
assert by[("quote", "did:plc:dan")].author_bio == "no thanks #nobot"
|
||||
assert by[("reply", "did:plc:eve")].source_url == "https://bsky.app/profile/eve.example/post/r1"
|
||||
assert by[("like", "did:plc:alice")].source_url.endswith("#liked-by-did:plc:alice")
|
||||
assert harvest.complete
|
||||
|
||||
|
||||
def test_threads_client_reads_replies_and_counts():
|
||||
def api(request):
|
||||
path = request.url.path
|
||||
params = parse_qs(request.url.query.decode())
|
||||
if path.endswith("/me"):
|
||||
return httpx.Response(200, json={"id": "42", "username": "me"})
|
||||
if path.endswith("/111/replies"):
|
||||
assert "profile_picture_url" in params["fields"][0]
|
||||
return httpx.Response(200, json={"data": [
|
||||
{"id": "1", "text": "Hi", "username": "ann", "permalink": "https://www.threads.com/@ann/post/A",
|
||||
"timestamp": "2026-11-10T00:00:00+0000", "hide_status": "NOT_HUSHED"},
|
||||
{"id": "2", "text": "private"},
|
||||
{"id": "3", "text": "hidden", "username": "x", "permalink": "p", "hide_status": "HIDDEN"},
|
||||
{"id": "4", "text": "mine", "username": "me", "permalink": "p"}]})
|
||||
if path.endswith("/111/insights"):
|
||||
assert params["metric"] == ["likes,reposts,quotes"]
|
||||
return httpx.Response(200, json={"data": [
|
||||
{"name": "likes", "values": [{"value": 12}]}, {"name": "reposts", "values": [{"value": 3}]},
|
||||
{"name": "quotes", "values": [{"value": 0}]}]})
|
||||
return httpx.Response(404, json={"error": {"code": 100}})
|
||||
|
||||
threads = ThreadsClient("tok", http=httpx.Client(transport=httpx.MockTransport(api)))
|
||||
harvest = threads.responses("111", "https://www.threads.com/@me/post/X")
|
||||
assert [(r.kind, r.author_name, r.content_html) for r in harvest.responses] == [("reply", "ann", "<p>Hi</p>")]
|
||||
assert harvest.counts == {"likes": 12, "reposts": 3, "quotes": 0}
|
||||
|
||||
|
||||
# --- Migration ---------------------------------------------------------------------------
|
||||
|
||||
def test_migration_020_keeps_existing_webmentions(tmp_path):
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
from starpunk.database.schema import INITIAL_SCHEMA_SQL
|
||||
from starpunk.migrations import execute_in_transaction
|
||||
|
||||
migrations = Path(__file__).parent.parent / "migrations"
|
||||
conn = sqlite3.connect(tmp_path / "old.db")
|
||||
conn.execute("PRAGMA foreign_keys=ON")
|
||||
conn.executescript(INITIAL_SCHEMA_SQL)
|
||||
for name in ("010_add_webmentions.sql", "012_add_syndications.sql", "014_webmention_source_details.sql"):
|
||||
matches = list(migrations.glob(name.split("_")[0] + "_*.sql"))
|
||||
conn.execute("BEGIN")
|
||||
execute_in_transaction(conn, matches[0].read_text())
|
||||
conn.commit()
|
||||
conn.execute("INSERT INTO notes (slug, file_path, published) VALUES ('n', 'n.md', 1)")
|
||||
conn.execute("INSERT INTO webmentions (source_url, target_url, target_note_id, mention_type, "
|
||||
"validation_status, moderation_status, source_published) "
|
||||
"VALUES ('https://a.example/r', 'https://b.example/note/n', 1, 'reply', 'valid', 'hidden', "
|
||||
"'2026-01-01')")
|
||||
conn.commit()
|
||||
|
||||
conn.execute("BEGIN")
|
||||
execute_in_transaction(conn, (migrations / "020_backfeed.sql").read_text())
|
||||
conn.commit()
|
||||
|
||||
row = conn.execute("SELECT mention_type, moderation_status, source_published, origin, target_note_id "
|
||||
"FROM webmentions").fetchone()
|
||||
assert row == ("reply", "hidden", "2026-01-01", "webmention", 1)
|
||||
conn.execute("INSERT INTO webmentions (source_url, target_url, mention_type, origin) "
|
||||
"VALUES ('https://q.example', 'https://b.example/note/n', 'quote', 'bluesky')")
|
||||
columns = {r[1] for r in conn.execute("PRAGMA table_info(syndications)")}
|
||||
assert {"backfeed_next_at", "backfeed_backfill", "likes_count"} <= columns
|
||||
assert conn.execute("PRAGMA foreign_key_check").fetchall() == []
|
||||
|
||||
|
||||
# --- Admin ----------------------------------------------------------------------------------
|
||||
|
||||
def test_admin_backfill_and_check_now(app, client):
|
||||
from starpunk.auth import create_session
|
||||
|
||||
app.config.update(ADMIN_ME="https://example.com", CSRF_ENABLED=False)
|
||||
note, sid = crossposted(app, age_days=200)
|
||||
with app.test_request_context():
|
||||
token = create_session("https://example.com")
|
||||
browser = app.test_client()
|
||||
browser.set_cookie("starpunk_session", token)
|
||||
|
||||
page = browser.get("/admin/webmentions").get_data(as_text=True)
|
||||
assert "No backfill has been run" in page and "Backfill responses" in page
|
||||
response = browser.post("/admin/backfeed/backfill", follow_redirects=True)
|
||||
assert "Backfill started: 1 crosspost(s) to check" in response.get_data(as_text=True)
|
||||
assert "1 crosspost(s) left to check" in browser.get("/admin/webmentions").get_data(as_text=True)
|
||||
|
||||
edit = browser.get(f"/admin/edit/{note.id}").get_data(as_text=True)
|
||||
assert "Responses not checked yet" in edit and "Check now" in edit
|
||||
browser.post(f"/admin/syndication/{sid}/check")
|
||||
with app.app_context():
|
||||
s = get_db(app).execute("SELECT backfeed_next_at FROM syndications WHERE id = ?", (sid,)).fetchone()
|
||||
assert s["backfeed_next_at"]
|
||||
|
||||
backfeed.denied["gotosocial"] = "Needs a GoToSocial token with read:statuses"
|
||||
assert "Needs a GoToSocial token" in browser.get("/admin/webmentions?origin=gotosocial").get_data(as_text=True)
|
||||
|
||||
|
||||
def test_threads_avatar_kept_while_its_copy_exists(app, client, web):
|
||||
note, _ = crossposted(app)
|
||||
with app.app_context():
|
||||
get_db(app).execute("UPDATE syndications SET target = 'threads'")
|
||||
get_db(app).commit()
|
||||
syndication._client_cache["threads"] = client
|
||||
app.config["THREADS_ACCESS_TOKEN"] = "token"
|
||||
web.files["https://cdn.example/p.jpg?sig=1"] = png()
|
||||
first = reply(1)
|
||||
first.author_photo = "https://cdn.example/p.jpg?sig=1"
|
||||
client.harvest = Harvest([first])
|
||||
run(app)
|
||||
with app.app_context():
|
||||
get_db(app).execute("UPDATE syndications SET backfeed_next_at = datetime('now')")
|
||||
get_db(app).commit()
|
||||
again = reply(1)
|
||||
again.author_photo = "https://cdn.example/p.jpg?sig=2" # not fetchable: would be a refetch
|
||||
client.harvest = Harvest([again])
|
||||
run(app)
|
||||
assert rows(app, note.id)[0]["author_photo"] == "https://cdn.example/p.jpg?sig=1"
|
||||
|
||||
|
||||
def test_avatars_from_own_gotosocial_and_bluesky_pds(app, web, monkeypatch):
|
||||
"""v2.13.1: own GoToSocial may resolve to a private address; Bluesky CDN avatars come from the PDS"""
|
||||
did, cid = "did:plc:abc123", "bafkreiabc"
|
||||
web.files[f"{GTS}/fileserver/a/avatar.png"] = png()
|
||||
web.files[f"https://plc.directory/{did}"] = json.dumps(
|
||||
{"service": [{"id": "#atproto_pds", "serviceEndpoint": "https://pds.example"}]}).encode()
|
||||
web.files[f"https://pds.example/xrpc/com.atproto.sync.getBlob?did={did}&cid={cid}"] = png("blue")
|
||||
real_resolve = resolve
|
||||
monkeypatch.setattr("starpunk.webmention_ssrf.socket.getaddrinfo",
|
||||
lambda host, *a, **k: [(2, 1, 6, "", ("172.20.0.5", 0))]
|
||||
if host == "social.example" else real_resolve(host))
|
||||
data = app.config["DATA_PATH"]
|
||||
with app.app_context():
|
||||
assert avatars.fetch(data, f"{GTS}/fileserver/a/avatar.png", SITE)
|
||||
assert avatars.fetch(data, f"https://cdn.bsky.app/img/avatar/plain/{did}/{cid}@jpeg", SITE)
|
||||
# Other private addresses stay blocked
|
||||
web.files["https://internal.example/a.png"] = png()
|
||||
monkeypatch.setattr("starpunk.webmention_ssrf.socket.getaddrinfo",
|
||||
lambda host, *a, **k: [(2, 1, 6, "", ("10.0.0.5", 0))])
|
||||
assert not avatars.fetch(data, "https://internal.example/a.png", SITE)
|
||||
|
||||
|
||||
def test_own_gotosocial_redirect_to_remote_avatar_is_followed(app, web, monkeypatch):
|
||||
"""v2.13.2: GoToSocial redirects uncached remote media to the original server"""
|
||||
class Redirecting(FakeWeb):
|
||||
def __call__(self, request):
|
||||
url = f"{request.url.scheme}://{request.headers['host']}{request.url.raw_path.decode()}"
|
||||
if url == f"{GTS}/fileserver/x/avatar.jpeg":
|
||||
return httpx.Response(302, headers={"location": "https://remote.example/avatar.jpeg"})
|
||||
return super().__call__(request)
|
||||
|
||||
fake = Redirecting()
|
||||
fake.files["https://remote.example/avatar.jpeg"] = png()
|
||||
monkeypatch.setattr("starpunk.webmention_ssrf.TRANSPORT", httpx.MockTransport(fake))
|
||||
with app.app_context():
|
||||
assert avatars.fetch(app.config["DATA_PATH"], f"{GTS}/fileserver/x/avatar.jpeg", SITE)
|
||||
@@ -198,6 +198,18 @@ class TestQueries:
|
||||
assert client.get("/micropub?q=category", headers=h).get_json()["categories"] == ["IndieWeb", "python"]
|
||||
assert client.get("/micropub?q=category&filter=py", headers=h).get_json()["categories"] == ["python"]
|
||||
|
||||
def test_category_skips_unused_tags(self, client, h):
|
||||
"""Tags dropped by an edit or on deleted posts aren't offered (v2.12.1)"""
|
||||
create(client, h, category=["kept"])
|
||||
retagged = create(client, h, category=["dropped"])
|
||||
deleted = create(client, h, category=["trashed"])
|
||||
body = {"action": "update", "url": retagged, "replace": {"category": ["kept"]}}
|
||||
assert client.post("/micropub", json=body, headers=h).status_code in (200, 204)
|
||||
assert client.post("/micropub", json={"action": "delete", "url": deleted}, headers=h).status_code == 204
|
||||
assert client.get("/micropub?q=category", headers=h).get_json()["categories"] == ["kept"]
|
||||
assert client.post("/micropub", json={"action": "undelete", "url": deleted}, headers=h).status_code < 300
|
||||
assert client.get("/micropub?q=category", headers=h).get_json()["categories"] == ["kept", "trashed"]
|
||||
|
||||
def test_config_lists_queries(self, client, h):
|
||||
config = client.get("/micropub?q=config", headers=h).get_json()
|
||||
assert "category" in config["q"] and "media-endpoint" in config
|
||||
|
||||
@@ -1,459 +0,0 @@
|
||||
"""
|
||||
Tests for metrics instrumentation (v1.1.2 Phase 1)
|
||||
|
||||
Tests database monitoring, HTTP metrics, memory monitoring, and business metrics.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import sqlite3
|
||||
import time
|
||||
import threading
|
||||
from unittest.mock import Mock, patch, MagicMock
|
||||
|
||||
from starpunk.monitoring import (
|
||||
MonitoredConnection,
|
||||
MemoryMonitor,
|
||||
get_metrics,
|
||||
get_metrics_stats,
|
||||
business,
|
||||
)
|
||||
from starpunk.monitoring.metrics import get_buffer
|
||||
from starpunk.monitoring.http import setup_http_metrics
|
||||
|
||||
|
||||
class TestMonitoredConnection:
|
||||
"""Tests for database operation monitoring"""
|
||||
|
||||
def test_execute_records_metric(self):
|
||||
"""Test that execute() records a metric"""
|
||||
# Create in-memory database
|
||||
conn = sqlite3.connect(':memory:')
|
||||
conn.execute('CREATE TABLE test (id INTEGER, name TEXT)')
|
||||
|
||||
# Wrap with monitoring
|
||||
monitored = MonitoredConnection(conn, slow_query_threshold=1.0)
|
||||
|
||||
# Clear metrics buffer
|
||||
get_buffer().clear()
|
||||
|
||||
# Execute query
|
||||
monitored.execute('SELECT * FROM test')
|
||||
|
||||
# Check metric was recorded
|
||||
metrics = get_metrics()
|
||||
# Note: May not be recorded due to sampling, but slow queries are forced
|
||||
# So we'll check stats instead
|
||||
stats = get_metrics_stats()
|
||||
assert stats['total_count'] >= 0 # May be 0 due to sampling
|
||||
|
||||
def test_slow_query_always_recorded(self):
|
||||
"""Test that slow queries are always recorded regardless of sampling"""
|
||||
# Create in-memory database
|
||||
conn = sqlite3.connect(':memory:')
|
||||
|
||||
# Set very low threshold so any query is "slow"
|
||||
monitored = MonitoredConnection(conn, slow_query_threshold=0.0)
|
||||
|
||||
# Clear metrics buffer
|
||||
get_buffer().clear()
|
||||
|
||||
# Execute query (will be considered slow)
|
||||
monitored.execute('SELECT 1')
|
||||
|
||||
# Check metric was recorded (forced due to being slow)
|
||||
metrics = get_metrics()
|
||||
assert len(metrics) > 0
|
||||
# Check that is_slow is True in metadata
|
||||
assert any(m.metadata.get('is_slow', False) is True for m in metrics)
|
||||
|
||||
def test_extract_table_name_select(self):
|
||||
"""Test table name extraction from SELECT query"""
|
||||
conn = sqlite3.connect(':memory:')
|
||||
conn.execute('CREATE TABLE notes (id INTEGER)')
|
||||
monitored = MonitoredConnection(conn)
|
||||
|
||||
table_name = monitored._extract_table_name('SELECT * FROM notes WHERE id = 1')
|
||||
assert table_name == 'notes'
|
||||
|
||||
def test_extract_table_name_insert(self):
|
||||
"""Test table name extraction from INSERT query"""
|
||||
conn = sqlite3.connect(':memory:')
|
||||
monitored = MonitoredConnection(conn)
|
||||
|
||||
table_name = monitored._extract_table_name('INSERT INTO users (name) VALUES (?)')
|
||||
assert table_name == 'users'
|
||||
|
||||
def test_extract_table_name_update(self):
|
||||
"""Test table name extraction from UPDATE query"""
|
||||
conn = sqlite3.connect(':memory:')
|
||||
monitored = MonitoredConnection(conn)
|
||||
|
||||
table_name = monitored._extract_table_name('UPDATE posts SET title = ?')
|
||||
assert table_name == 'posts'
|
||||
|
||||
def test_extract_table_name_unknown(self):
|
||||
"""Test that complex queries return 'unknown'"""
|
||||
conn = sqlite3.connect(':memory:')
|
||||
monitored = MonitoredConnection(conn)
|
||||
|
||||
# Complex query with JOIN
|
||||
table_name = monitored._extract_table_name(
|
||||
'SELECT a.* FROM notes a JOIN users b ON a.user_id = b.id'
|
||||
)
|
||||
# Our simple regex will find 'notes' from the first FROM
|
||||
assert table_name in ['notes', 'unknown']
|
||||
|
||||
def test_get_query_type(self):
|
||||
"""Test query type extraction"""
|
||||
conn = sqlite3.connect(':memory:')
|
||||
monitored = MonitoredConnection(conn)
|
||||
|
||||
assert monitored._get_query_type('SELECT * FROM notes') == 'SELECT'
|
||||
assert monitored._get_query_type('INSERT INTO notes VALUES (?)') == 'INSERT'
|
||||
assert monitored._get_query_type('UPDATE notes SET x = 1') == 'UPDATE'
|
||||
assert monitored._get_query_type('DELETE FROM notes') == 'DELETE'
|
||||
assert monitored._get_query_type('CREATE TABLE test (id INT)') == 'CREATE'
|
||||
assert monitored._get_query_type('PRAGMA journal_mode=WAL') == 'PRAGMA'
|
||||
|
||||
def test_execute_with_parameters(self):
|
||||
"""Test execute with query parameters"""
|
||||
conn = sqlite3.connect(':memory:')
|
||||
conn.execute('CREATE TABLE test (id INTEGER, name TEXT)')
|
||||
monitored = MonitoredConnection(conn, slow_query_threshold=1.0)
|
||||
|
||||
# Execute with parameters
|
||||
monitored.execute('INSERT INTO test (id, name) VALUES (?, ?)', (1, 'test'))
|
||||
|
||||
# Verify data was inserted
|
||||
cursor = monitored.execute('SELECT * FROM test WHERE id = ?', (1,))
|
||||
rows = cursor.fetchall()
|
||||
assert len(rows) == 1
|
||||
|
||||
def test_executemany(self):
|
||||
"""Test executemany batch operations"""
|
||||
conn = sqlite3.connect(':memory:')
|
||||
conn.execute('CREATE TABLE test (id INTEGER, name TEXT)')
|
||||
monitored = MonitoredConnection(conn)
|
||||
|
||||
# Clear metrics
|
||||
get_buffer().clear()
|
||||
|
||||
# Execute batch insert
|
||||
data = [(1, 'first'), (2, 'second'), (3, 'third')]
|
||||
monitored.executemany('INSERT INTO test (id, name) VALUES (?, ?)', data)
|
||||
|
||||
# Check metric was recorded
|
||||
metrics = get_metrics()
|
||||
# May not be recorded due to sampling
|
||||
stats = get_metrics_stats()
|
||||
assert stats is not None
|
||||
|
||||
def test_error_recording(self):
|
||||
"""Test that errors are recorded in metrics"""
|
||||
conn = sqlite3.connect(':memory:')
|
||||
monitored = MonitoredConnection(conn)
|
||||
|
||||
# Clear metrics
|
||||
get_buffer().clear()
|
||||
|
||||
# Execute invalid query
|
||||
with pytest.raises(sqlite3.OperationalError):
|
||||
monitored.execute('SELECT * FROM nonexistent_table')
|
||||
|
||||
# Check error was recorded (forced)
|
||||
metrics = get_metrics()
|
||||
assert len(metrics) > 0
|
||||
assert any('ERROR' in m.operation_name for m in metrics)
|
||||
|
||||
|
||||
class TestHTTPMetrics:
|
||||
"""Tests for HTTP request/response monitoring"""
|
||||
|
||||
def test_setup_http_metrics(self, app):
|
||||
"""Test HTTP metrics middleware setup"""
|
||||
# Add a simple test route
|
||||
@app.route('/test')
|
||||
def test_route():
|
||||
return 'OK', 200
|
||||
|
||||
setup_http_metrics(app)
|
||||
|
||||
# Clear metrics
|
||||
get_buffer().clear()
|
||||
|
||||
# Make a request
|
||||
with app.test_client() as client:
|
||||
response = client.get('/test')
|
||||
assert response.status_code == 200
|
||||
|
||||
# Check request ID header was added
|
||||
assert 'X-Request-ID' in response.headers
|
||||
|
||||
# Check metrics were recorded
|
||||
metrics = get_metrics()
|
||||
# May be sampled, so just check structure
|
||||
stats = get_metrics_stats()
|
||||
assert stats is not None
|
||||
|
||||
def test_request_id_generation(self, app):
|
||||
"""Test that unique request IDs are generated"""
|
||||
# Add a simple test route
|
||||
@app.route('/test')
|
||||
def test_route():
|
||||
return 'OK', 200
|
||||
|
||||
setup_http_metrics(app)
|
||||
|
||||
request_ids = set()
|
||||
|
||||
with app.test_client() as client:
|
||||
for _ in range(5):
|
||||
response = client.get('/test')
|
||||
request_id = response.headers.get('X-Request-ID')
|
||||
assert request_id is not None
|
||||
request_ids.add(request_id)
|
||||
|
||||
# All request IDs should be unique
|
||||
assert len(request_ids) == 5
|
||||
|
||||
def test_error_metrics_recorded(self, app):
|
||||
"""Test that errors are recorded in metrics"""
|
||||
# Add a simple test route
|
||||
@app.route('/test')
|
||||
def test_route():
|
||||
return 'OK', 200
|
||||
|
||||
setup_http_metrics(app)
|
||||
|
||||
# Clear metrics
|
||||
get_buffer().clear()
|
||||
|
||||
with app.test_client() as client:
|
||||
# Request non-existent endpoint
|
||||
response = client.get('/this-does-not-exist')
|
||||
assert response.status_code == 404
|
||||
|
||||
# Error metrics should be recorded (forced)
|
||||
# Note: 404 is not necessarily an error in the teardown handler
|
||||
# but will be in metrics as a 404 status code
|
||||
metrics = get_metrics()
|
||||
stats = get_metrics_stats()
|
||||
assert stats is not None
|
||||
|
||||
|
||||
class TestMemoryMonitor:
|
||||
"""Tests for memory monitoring thread"""
|
||||
|
||||
def test_memory_monitor_initialization(self):
|
||||
"""Test memory monitor can be initialized"""
|
||||
monitor = MemoryMonitor(interval=1)
|
||||
assert monitor.interval == 1
|
||||
assert monitor.daemon is True # Per CQ5
|
||||
|
||||
def test_memory_monitor_starts_and_stops(self):
|
||||
"""Test memory monitor thread lifecycle"""
|
||||
monitor = MemoryMonitor(interval=1)
|
||||
|
||||
# Start monitor
|
||||
monitor.start()
|
||||
assert monitor.is_alive()
|
||||
|
||||
# Wait a bit for initialization
|
||||
time.sleep(0.5)
|
||||
|
||||
# Stop monitor gracefully
|
||||
monitor.stop()
|
||||
# Give it time to finish gracefully
|
||||
time.sleep(1.0)
|
||||
monitor.join(timeout=5)
|
||||
# Thread should have stopped
|
||||
# Note: In rare cases daemon thread may still be cleaning up
|
||||
if monitor.is_alive():
|
||||
# Give it one more second
|
||||
time.sleep(1.0)
|
||||
assert not monitor.is_alive()
|
||||
|
||||
def test_memory_monitor_collects_metrics(self):
|
||||
"""Test that memory monitor collects metrics"""
|
||||
# Clear metrics
|
||||
get_buffer().clear()
|
||||
|
||||
monitor = MemoryMonitor(interval=1)
|
||||
monitor.start()
|
||||
|
||||
# Wait for baseline + one collection
|
||||
time.sleep(7) # 5s baseline + 2s for collection
|
||||
|
||||
# Stop monitor
|
||||
monitor.stop()
|
||||
monitor.join(timeout=2)
|
||||
|
||||
# Check metrics were collected
|
||||
metrics = get_metrics()
|
||||
memory_metrics = [m for m in metrics if 'memory' in m.operation_name.lower()]
|
||||
|
||||
# Should have at least one memory metric
|
||||
assert len(memory_metrics) > 0
|
||||
|
||||
def test_memory_monitor_stats(self):
|
||||
"""Test memory monitor statistics"""
|
||||
monitor = MemoryMonitor(interval=1)
|
||||
monitor.start()
|
||||
|
||||
# Wait for baseline
|
||||
time.sleep(6)
|
||||
|
||||
# Get stats
|
||||
stats = monitor.get_stats()
|
||||
assert stats['status'] == 'running'
|
||||
assert 'current_rss_mb' in stats
|
||||
assert 'baseline_rss_mb' in stats
|
||||
assert stats['baseline_rss_mb'] > 0
|
||||
|
||||
monitor.stop()
|
||||
monitor.join(timeout=2)
|
||||
|
||||
|
||||
class TestBusinessMetrics:
|
||||
"""Tests for business metrics tracking"""
|
||||
|
||||
def test_track_note_created(self):
|
||||
"""Test note creation tracking"""
|
||||
get_buffer().clear()
|
||||
|
||||
business.track_note_created(note_id=123, content_length=500, has_media=False)
|
||||
|
||||
metrics = get_metrics()
|
||||
assert len(metrics) > 0
|
||||
|
||||
note_metrics = [m for m in metrics if 'note_created' in m.operation_name]
|
||||
assert len(note_metrics) > 0
|
||||
assert note_metrics[0].metadata['note_id'] == 123
|
||||
assert note_metrics[0].metadata['content_length'] == 500
|
||||
|
||||
def test_track_note_updated(self):
|
||||
"""Test note update tracking"""
|
||||
get_buffer().clear()
|
||||
|
||||
business.track_note_updated(
|
||||
note_id=456,
|
||||
content_length=750,
|
||||
fields_changed=['title', 'content']
|
||||
)
|
||||
|
||||
metrics = get_metrics()
|
||||
note_metrics = [m for m in metrics if 'note_updated' in m.operation_name]
|
||||
assert len(note_metrics) > 0
|
||||
assert note_metrics[0].metadata['note_id'] == 456
|
||||
|
||||
def test_track_note_deleted(self):
|
||||
"""Test note deletion tracking"""
|
||||
get_buffer().clear()
|
||||
|
||||
business.track_note_deleted(note_id=789)
|
||||
|
||||
metrics = get_metrics()
|
||||
note_metrics = [m for m in metrics if 'note_deleted' in m.operation_name]
|
||||
assert len(note_metrics) > 0
|
||||
assert note_metrics[0].metadata['note_id'] == 789
|
||||
|
||||
def test_track_feed_generated(self):
|
||||
"""Test feed generation tracking"""
|
||||
get_buffer().clear()
|
||||
|
||||
business.track_feed_generated(
|
||||
format='rss',
|
||||
item_count=50,
|
||||
duration_ms=45.2,
|
||||
cached=False
|
||||
)
|
||||
|
||||
metrics = get_metrics()
|
||||
feed_metrics = [m for m in metrics if 'feed_rss' in m.operation_name]
|
||||
assert len(feed_metrics) > 0
|
||||
assert feed_metrics[0].metadata['format'] == 'rss'
|
||||
assert feed_metrics[0].metadata['item_count'] == 50
|
||||
|
||||
def test_track_cache_hit(self):
|
||||
"""Test cache hit tracking"""
|
||||
get_buffer().clear()
|
||||
|
||||
business.track_cache_hit(cache_type='feed', key='rss:latest')
|
||||
|
||||
metrics = get_metrics()
|
||||
cache_metrics = [m for m in metrics if 'cache_hit' in m.operation_name]
|
||||
assert len(cache_metrics) > 0
|
||||
|
||||
def test_track_cache_miss(self):
|
||||
"""Test cache miss tracking"""
|
||||
get_buffer().clear()
|
||||
|
||||
business.track_cache_miss(cache_type='feed', key='atom:latest')
|
||||
|
||||
metrics = get_metrics()
|
||||
cache_metrics = [m for m in metrics if 'cache_miss' in m.operation_name]
|
||||
assert len(cache_metrics) > 0
|
||||
|
||||
|
||||
class TestMetricsConfiguration:
|
||||
"""Tests for metrics configuration"""
|
||||
|
||||
def test_metrics_can_be_disabled(self, app):
|
||||
"""Test that metrics can be disabled via configuration"""
|
||||
# This would be tested by setting METRICS_ENABLED=False
|
||||
# and verifying no metrics are collected
|
||||
assert 'METRICS_ENABLED' in app.config
|
||||
|
||||
def test_slow_query_threshold_configurable(self, app):
|
||||
"""Test that slow query threshold is configurable"""
|
||||
assert 'METRICS_SLOW_QUERY_THRESHOLD' in app.config
|
||||
assert isinstance(app.config['METRICS_SLOW_QUERY_THRESHOLD'], float)
|
||||
|
||||
def test_sampling_rate_configurable(self, app):
|
||||
"""Test that sampling rate is configurable"""
|
||||
assert 'METRICS_SAMPLING_RATE' in app.config
|
||||
assert isinstance(app.config['METRICS_SAMPLING_RATE'], float)
|
||||
assert 0.0 <= app.config['METRICS_SAMPLING_RATE'] <= 1.0
|
||||
|
||||
def test_buffer_size_configurable(self, app):
|
||||
"""Test that buffer size is configurable"""
|
||||
assert 'METRICS_BUFFER_SIZE' in app.config
|
||||
assert isinstance(app.config['METRICS_BUFFER_SIZE'], int)
|
||||
assert app.config['METRICS_BUFFER_SIZE'] > 0
|
||||
|
||||
def test_memory_interval_configurable(self, app):
|
||||
"""Test that memory monitor interval is configurable"""
|
||||
assert 'METRICS_MEMORY_INTERVAL' in app.config
|
||||
assert isinstance(app.config['METRICS_MEMORY_INTERVAL'], int)
|
||||
assert app.config['METRICS_MEMORY_INTERVAL'] > 0
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def app():
|
||||
"""Create test Flask app with minimal configuration"""
|
||||
from flask import Flask
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
# Create temp directory for testing
|
||||
temp_dir = tempfile.mkdtemp()
|
||||
temp_path = Path(temp_dir)
|
||||
|
||||
# Minimal configuration to avoid migration issues
|
||||
app.config.update({
|
||||
'TESTING': True,
|
||||
'DATABASE_PATH': temp_path / 'test.db',
|
||||
'DATA_PATH': temp_path,
|
||||
'NOTES_PATH': temp_path / 'notes',
|
||||
'SESSION_SECRET': 'test-secret',
|
||||
'ADMIN_ME': 'https://test.example.com',
|
||||
'METRICS_ENABLED': True,
|
||||
'METRICS_SLOW_QUERY_THRESHOLD': 1.0,
|
||||
'METRICS_SAMPLING_RATE': 1.0,
|
||||
'METRICS_BUFFER_SIZE': 1000,
|
||||
'METRICS_MEMORY_INTERVAL': 30,
|
||||
})
|
||||
|
||||
return app
|
||||
@@ -1,103 +0,0 @@
|
||||
"""
|
||||
Tests for feed statistics tracking
|
||||
|
||||
Tests feed statistics aggregation per v1.1.2 Phase 3.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from starpunk.monitoring.business import get_feed_statistics, track_feed_generated
|
||||
|
||||
|
||||
def test_get_feed_statistics_returns_structure():
|
||||
"""Test get_feed_statistics returns expected structure"""
|
||||
stats = get_feed_statistics()
|
||||
|
||||
# Check top-level keys
|
||||
assert "by_format" in stats
|
||||
assert "cache" in stats
|
||||
assert "total_requests" in stats
|
||||
assert "format_percentages" in stats
|
||||
|
||||
# Check by_format structure
|
||||
assert "rss" in stats["by_format"]
|
||||
assert "atom" in stats["by_format"]
|
||||
assert "json" in stats["by_format"]
|
||||
|
||||
# Check format stats structure
|
||||
for format_name in ["rss", "atom", "json"]:
|
||||
fmt_stats = stats["by_format"][format_name]
|
||||
assert "generated" in fmt_stats
|
||||
assert "cached" in fmt_stats
|
||||
assert "total" in fmt_stats
|
||||
assert "avg_duration_ms" in fmt_stats
|
||||
|
||||
# Check cache structure
|
||||
assert "hits" in stats["cache"]
|
||||
assert "misses" in stats["cache"]
|
||||
assert "hit_rate" in stats["cache"]
|
||||
|
||||
|
||||
def test_get_feed_statistics_empty_metrics():
|
||||
"""Test get_feed_statistics with no metrics returns zeros"""
|
||||
stats = get_feed_statistics()
|
||||
|
||||
# All values should be zero or empty
|
||||
assert stats["total_requests"] >= 0
|
||||
assert stats["cache"]["hit_rate"] >= 0.0
|
||||
assert stats["cache"]["hit_rate"] <= 1.0
|
||||
|
||||
|
||||
def test_feed_statistics_cache_hit_rate_calculation():
|
||||
"""Test cache hit rate is calculated correctly"""
|
||||
stats = get_feed_statistics()
|
||||
|
||||
# Hit rate should be between 0 and 1
|
||||
assert 0.0 <= stats["cache"]["hit_rate"] <= 1.0
|
||||
|
||||
# If there are hits and misses, hit rate should be hits / (hits + misses)
|
||||
if stats["cache"]["hits"] + stats["cache"]["misses"] > 0:
|
||||
expected_rate = stats["cache"]["hits"] / (
|
||||
stats["cache"]["hits"] + stats["cache"]["misses"]
|
||||
)
|
||||
assert abs(stats["cache"]["hit_rate"] - expected_rate) < 0.001
|
||||
|
||||
|
||||
def test_feed_statistics_format_percentages():
|
||||
"""Test format percentages sum to 1.0 when there are requests"""
|
||||
stats = get_feed_statistics()
|
||||
|
||||
if stats["total_requests"] > 0:
|
||||
total_percentage = sum(stats["format_percentages"].values())
|
||||
# Should sum to approximately 1.0 (allowing for floating point errors)
|
||||
assert abs(total_percentage - 1.0) < 0.001
|
||||
|
||||
|
||||
def test_feed_statistics_total_requests_sum():
|
||||
"""Test total_requests equals sum of all format totals"""
|
||||
stats = get_feed_statistics()
|
||||
|
||||
format_total = sum(
|
||||
fmt["total"] for fmt in stats["by_format"].values()
|
||||
)
|
||||
|
||||
assert stats["total_requests"] == format_total
|
||||
|
||||
|
||||
def test_track_feed_generated_records_metrics():
|
||||
"""Test track_feed_generated creates metrics entries"""
|
||||
# Note: This test just verifies the function runs without error.
|
||||
# Actual metrics tracking is tested in integration tests.
|
||||
track_feed_generated(
|
||||
format="rss",
|
||||
item_count=10,
|
||||
duration_ms=50.5,
|
||||
cached=False
|
||||
)
|
||||
|
||||
# Get statistics - may be empty if metrics buffer hasn't persisted yet
|
||||
stats = get_feed_statistics()
|
||||
|
||||
# Verify structure is correct
|
||||
assert "total_requests" in stats
|
||||
assert "by_format" in stats
|
||||
assert "cache" in stats
|
||||
Reference in new issue
Block a user