Files
StarPunk/starpunk/rate_limiter.py
T
PhilandClaude Opus 5.5 75946be7b2 maint: Dependency upgrades and High backlog items (v2.11.0)
- All dependencies to current releases (97 known vulnerabilities -> 0),
  Python 3.14, uv pinned, requirements-test.txt, lxml/dateutil pinned
- CSRF tokens on admin forms, logout and IndieAuth consent
- IndieAuth client and crosspost link-card fetches via safe_get
- FLASK_DEBUG/FLASK_ENV default off/production; image sets them
- Foreign keys enforced on pooled connections; migration 018 removes orphans
- Rate limits shared across processes (rate_limits, migration 019)
- datetime.utcnow() and sqlite3 default adapters replaced
- compose.yaml fixed; duplicate dev_auth.py and test.ini removed

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-03 13:09:52 -07:00

286 lines
9.2 KiB
Python

"""
Rate limiting for webmention endpoint
Implements sliding window log algorithm with thread-safe in-memory storage.
Per ADR-063 (Decision 10), this uses in-memory counters. They are per
gunicorn process, so the effective limit is the setting times the number of
workers (see docs/design/v1.6.0/rate-limiting.md).
References:
- ADR-063: Webmention Implementation (Decision 10: rate limiting)
- Design: /docs/design/v1.6.0/rate-limiting.md
"""
import threading
import time
from datetime import datetime, timedelta
from typing import Dict, List, Tuple
class RateLimiter:
"""
Thread-safe sliding window rate limiter.
Uses sliding window log algorithm:
- Stores timestamp of each request in window
- Removes expired entries on each check
- Rejects if count >= limit
"""
def __init__(self, limit: int, window_seconds: int):
"""
Initialize rate limiter.
Args:
limit: Maximum requests allowed in window
window_seconds: Time window in seconds
"""
self.limit = limit
self.window_seconds = window_seconds
self._requests: Dict[str, List[float]] = {}
self._lock = threading.Lock()
def is_allowed(self, key: str) -> bool:
"""
Check if request is allowed and record it if so.
Args:
key: Identifier (e.g., IP address or domain)
Returns:
True if request is allowed, False if rate limited
"""
with self._lock:
now = time.time()
cutoff = now - self.window_seconds
# Get or create request log for this key
if key not in self._requests:
self._requests[key] = []
# Remove expired entries
self._requests[key] = [ts for ts in self._requests[key] if ts > cutoff]
# Check if at or over limit
if len(self._requests[key]) >= self.limit:
return False
# Allow request and record timestamp
self._requests[key].append(now)
return True
def get_retry_after(self, key: str) -> int:
"""
Get seconds until oldest request expires (for Retry-After header).
Args:
key: Identifier (e.g., IP address or domain)
Returns:
Seconds until rate limit resets, or window_seconds if no requests
"""
with self._lock:
if key not in self._requests or not self._requests[key]:
return self.window_seconds
now = time.time()
oldest_request = self._requests[key][0]
expires_at = oldest_request + self.window_seconds
retry_after = int(expires_at - now)
# Return at least 1 second, max window_seconds
return max(1, min(retry_after, self.window_seconds))
def cleanup(self, max_idle_seconds: int = 300):
"""
Remove keys with no recent requests to prevent memory growth.
Args:
max_idle_seconds: Remove keys idle for this many seconds
"""
with self._lock:
now = time.time()
cutoff = now - max_idle_seconds
keys_to_remove = []
for key, timestamps in self._requests.items():
# Remove expired timestamps first
self._requests[key] = [ts for ts in timestamps if ts > cutoff]
# Mark empty keys for removal
if not self._requests[key]:
keys_to_remove.append(key)
for key in keys_to_remove:
del self._requests[key]
def get_request_count(self, key: str) -> int:
"""
Get current request count for a key (for testing/logging).
Args:
key: Identifier (e.g., IP address or domain)
Returns:
Number of requests in current window
"""
with self._lock:
if key not in self._requests:
return 0
now = time.time()
cutoff = now - self.window_seconds
# Count non-expired requests
return len([ts for ts in self._requests[key] if ts > cutoff])
# Module-level rate limiters (initialized in init_rate_limiters)
_ip_limiter: RateLimiter | None = None
_domain_limiter: RateLimiter | None = None
class SharedRateLimiter:
"""
Rate limiter whose counts live in SQLite, shared by every gunicorn
process (v2.11.0). In-memory counters let each of the 4 workers allow the
full limit, so the real limit was four times the setting.
Sliding-window counter: the current window's count plus the previous
window's, weighted by how much of it still overlaps the last
`window_seconds`. Same interface as RateLimiter; needs an app context.
"""
CLEANUP_EVERY = 200 # calls between removals of old windows
def __init__(self, limit: int, window_seconds: int, prefix: str):
self.limit = limit
self.window_seconds = window_seconds
self.prefix = prefix
self._calls = 0
def _db(self):
from flask import current_app
from starpunk.database import get_db
return get_db(current_app)
def _windows(self):
now = time.time()
current = int(now // self.window_seconds)
overlap = 1 - (now % self.window_seconds) / self.window_seconds
return now, current, overlap
def _estimate(self, db, key: str) -> float:
_, current, overlap = self._windows()
counts = dict(db.execute(
"SELECT window, count FROM rate_limits WHERE key = ? AND window IN (?, ?)",
(f"{self.prefix}:{key}", current - 1, current),
).fetchall())
return counts.get(current - 1, 0) * overlap + counts.get(current, 0)
def is_allowed(self, key: str) -> bool:
db = self._db()
_, current, _ = self._windows()
if self._estimate(db, key) >= self.limit:
return False
db.execute(
"INSERT INTO rate_limits (key, window, count) VALUES (?, ?, 1) "
"ON CONFLICT(key, window) DO UPDATE SET count = count + 1",
(f"{self.prefix}:{key}", current),
)
db.commit()
self._calls += 1
if self._calls % self.CLEANUP_EVERY == 0:
self.cleanup()
return True
def get_retry_after(self, key: str) -> int:
now, current, _ = self._windows()
return max(1, int((current + 1) * self.window_seconds - now) + 1)
def get_request_count(self, key: str) -> int:
return int(self._estimate(self._db(), key))
def cleanup(self, max_idle_seconds: int = 300) -> int:
"""Remove windows too old to matter (this limiter's keys only)"""
_, current, _ = self._windows()
db = self._db()
cursor = db.execute(
"DELETE FROM rate_limits WHERE key LIKE ? AND window < ?",
(f"{self.prefix}:%", current - 1),
)
db.commit()
return cursor.rowcount
def init_rate_limiters(app) -> None:
"""
Initialize rate limiters with Flask app configuration.
Args:
app: Flask application instance
"""
global _ip_limiter, _domain_limiter
if not app.config.get('WEBMENTION_RATE_LIMIT_ENABLED', True):
app.logger.info("Rate limiting disabled")
_ip_limiter = None
_domain_limiter = None
return
ip_limit = app.config.get('WEBMENTION_RATE_LIMIT_IP', 60)
domain_limit = app.config.get('WEBMENTION_RATE_LIMIT_DOMAIN', 10)
window = app.config.get('WEBMENTION_RATE_LIMIT_WINDOW', 60)
# Shared by all server processes (v2.11.0)
_ip_limiter = SharedRateLimiter(ip_limit, window, "webmention-ip")
_domain_limiter = SharedRateLimiter(domain_limit, window, "webmention-domain")
app.logger.info(
f"Webmention rate limiter initialized: {ip_limit}/min IP, "
f"{domain_limit}/min domain"
)
def check_rate_limit(ip: str, source_domain: str | None) -> Tuple[bool, str | None]:
"""
Check rate limits for a request.
Args:
ip: Client IP address
source_domain: Source URL domain (optional)
Returns:
(allowed, error_message) tuple
- allowed: True if request allowed, False if rate limited
- error_message: Error description if rate limited, None otherwise
"""
# If rate limiting disabled, allow all requests
if _ip_limiter is None:
return True, None
# Check IP limit first
if not _ip_limiter.is_allowed(ip):
retry_after = _ip_limiter.get_retry_after(ip)
count = _ip_limiter.get_request_count(ip)
return False, f"Too many requests from your IP. Please retry after {retry_after} seconds."
# Check domain limit if source domain provided
if source_domain and _domain_limiter is not None:
if not _domain_limiter.is_allowed(source_domain):
retry_after = _domain_limiter.get_retry_after(source_domain)
count = _domain_limiter.get_request_count(source_domain)
return False, f"Too many requests from source domain. Please retry after {retry_after} seconds."
return True, None
def cleanup_rate_limiters() -> None:
"""
Cleanup expired entries from rate limiters.
Called periodically to prevent memory growth.
"""
if _ip_limiter:
_ip_limiter.cleanup()
if _domain_limiter:
_domain_limiter.cleanup()