- All dependencies to current releases (97 known vulnerabilities -> 0), Python 3.14, uv pinned, requirements-test.txt, lxml/dateutil pinned - CSRF tokens on admin forms, logout and IndieAuth consent - IndieAuth client and crosspost link-card fetches via safe_get - FLASK_DEBUG/FLASK_ENV default off/production; image sets them - Foreign keys enforced on pooled connections; migration 018 removes orphans - Rate limits shared across processes (rate_limits, migration 019) - datetime.utcnow() and sqlite3 default adapters replaced - compose.yaml fixed; duplicate dev_auth.py and test.ini removed Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
286 lines
9.2 KiB
Python
286 lines
9.2 KiB
Python
"""
|
|
Rate limiting for webmention endpoint
|
|
|
|
Implements sliding window log algorithm with thread-safe in-memory storage.
|
|
Per ADR-063 (Decision 10), this uses in-memory counters. They are per
|
|
gunicorn process, so the effective limit is the setting times the number of
|
|
workers (see docs/design/v1.6.0/rate-limiting.md).
|
|
|
|
References:
|
|
- ADR-063: Webmention Implementation (Decision 10: rate limiting)
|
|
- Design: /docs/design/v1.6.0/rate-limiting.md
|
|
"""
|
|
|
|
import threading
|
|
import time
|
|
from datetime import datetime, timedelta
|
|
from typing import Dict, List, Tuple
|
|
|
|
|
|
class RateLimiter:
|
|
"""
|
|
Thread-safe sliding window rate limiter.
|
|
|
|
Uses sliding window log algorithm:
|
|
- Stores timestamp of each request in window
|
|
- Removes expired entries on each check
|
|
- Rejects if count >= limit
|
|
"""
|
|
|
|
def __init__(self, limit: int, window_seconds: int):
|
|
"""
|
|
Initialize rate limiter.
|
|
|
|
Args:
|
|
limit: Maximum requests allowed in window
|
|
window_seconds: Time window in seconds
|
|
"""
|
|
self.limit = limit
|
|
self.window_seconds = window_seconds
|
|
self._requests: Dict[str, List[float]] = {}
|
|
self._lock = threading.Lock()
|
|
|
|
def is_allowed(self, key: str) -> bool:
|
|
"""
|
|
Check if request is allowed and record it if so.
|
|
|
|
Args:
|
|
key: Identifier (e.g., IP address or domain)
|
|
|
|
Returns:
|
|
True if request is allowed, False if rate limited
|
|
"""
|
|
with self._lock:
|
|
now = time.time()
|
|
cutoff = now - self.window_seconds
|
|
|
|
# Get or create request log for this key
|
|
if key not in self._requests:
|
|
self._requests[key] = []
|
|
|
|
# Remove expired entries
|
|
self._requests[key] = [ts for ts in self._requests[key] if ts > cutoff]
|
|
|
|
# Check if at or over limit
|
|
if len(self._requests[key]) >= self.limit:
|
|
return False
|
|
|
|
# Allow request and record timestamp
|
|
self._requests[key].append(now)
|
|
return True
|
|
|
|
def get_retry_after(self, key: str) -> int:
|
|
"""
|
|
Get seconds until oldest request expires (for Retry-After header).
|
|
|
|
Args:
|
|
key: Identifier (e.g., IP address or domain)
|
|
|
|
Returns:
|
|
Seconds until rate limit resets, or window_seconds if no requests
|
|
"""
|
|
with self._lock:
|
|
if key not in self._requests or not self._requests[key]:
|
|
return self.window_seconds
|
|
|
|
now = time.time()
|
|
oldest_request = self._requests[key][0]
|
|
expires_at = oldest_request + self.window_seconds
|
|
retry_after = int(expires_at - now)
|
|
|
|
# Return at least 1 second, max window_seconds
|
|
return max(1, min(retry_after, self.window_seconds))
|
|
|
|
def cleanup(self, max_idle_seconds: int = 300):
|
|
"""
|
|
Remove keys with no recent requests to prevent memory growth.
|
|
|
|
Args:
|
|
max_idle_seconds: Remove keys idle for this many seconds
|
|
"""
|
|
with self._lock:
|
|
now = time.time()
|
|
cutoff = now - max_idle_seconds
|
|
keys_to_remove = []
|
|
|
|
for key, timestamps in self._requests.items():
|
|
# Remove expired timestamps first
|
|
self._requests[key] = [ts for ts in timestamps if ts > cutoff]
|
|
# Mark empty keys for removal
|
|
if not self._requests[key]:
|
|
keys_to_remove.append(key)
|
|
|
|
for key in keys_to_remove:
|
|
del self._requests[key]
|
|
|
|
def get_request_count(self, key: str) -> int:
|
|
"""
|
|
Get current request count for a key (for testing/logging).
|
|
|
|
Args:
|
|
key: Identifier (e.g., IP address or domain)
|
|
|
|
Returns:
|
|
Number of requests in current window
|
|
"""
|
|
with self._lock:
|
|
if key not in self._requests:
|
|
return 0
|
|
|
|
now = time.time()
|
|
cutoff = now - self.window_seconds
|
|
# Count non-expired requests
|
|
return len([ts for ts in self._requests[key] if ts > cutoff])
|
|
|
|
|
|
# Module-level rate limiters (initialized in init_rate_limiters)
|
|
_ip_limiter: RateLimiter | None = None
|
|
_domain_limiter: RateLimiter | None = None
|
|
|
|
|
|
class SharedRateLimiter:
|
|
"""
|
|
Rate limiter whose counts live in SQLite, shared by every gunicorn
|
|
process (v2.11.0). In-memory counters let each of the 4 workers allow the
|
|
full limit, so the real limit was four times the setting.
|
|
|
|
Sliding-window counter: the current window's count plus the previous
|
|
window's, weighted by how much of it still overlaps the last
|
|
`window_seconds`. Same interface as RateLimiter; needs an app context.
|
|
"""
|
|
|
|
CLEANUP_EVERY = 200 # calls between removals of old windows
|
|
|
|
def __init__(self, limit: int, window_seconds: int, prefix: str):
|
|
self.limit = limit
|
|
self.window_seconds = window_seconds
|
|
self.prefix = prefix
|
|
self._calls = 0
|
|
|
|
def _db(self):
|
|
from flask import current_app
|
|
from starpunk.database import get_db
|
|
return get_db(current_app)
|
|
|
|
def _windows(self):
|
|
now = time.time()
|
|
current = int(now // self.window_seconds)
|
|
overlap = 1 - (now % self.window_seconds) / self.window_seconds
|
|
return now, current, overlap
|
|
|
|
def _estimate(self, db, key: str) -> float:
|
|
_, current, overlap = self._windows()
|
|
counts = dict(db.execute(
|
|
"SELECT window, count FROM rate_limits WHERE key = ? AND window IN (?, ?)",
|
|
(f"{self.prefix}:{key}", current - 1, current),
|
|
).fetchall())
|
|
return counts.get(current - 1, 0) * overlap + counts.get(current, 0)
|
|
|
|
def is_allowed(self, key: str) -> bool:
|
|
db = self._db()
|
|
_, current, _ = self._windows()
|
|
if self._estimate(db, key) >= self.limit:
|
|
return False
|
|
db.execute(
|
|
"INSERT INTO rate_limits (key, window, count) VALUES (?, ?, 1) "
|
|
"ON CONFLICT(key, window) DO UPDATE SET count = count + 1",
|
|
(f"{self.prefix}:{key}", current),
|
|
)
|
|
db.commit()
|
|
self._calls += 1
|
|
if self._calls % self.CLEANUP_EVERY == 0:
|
|
self.cleanup()
|
|
return True
|
|
|
|
def get_retry_after(self, key: str) -> int:
|
|
now, current, _ = self._windows()
|
|
return max(1, int((current + 1) * self.window_seconds - now) + 1)
|
|
|
|
def get_request_count(self, key: str) -> int:
|
|
return int(self._estimate(self._db(), key))
|
|
|
|
def cleanup(self, max_idle_seconds: int = 300) -> int:
|
|
"""Remove windows too old to matter (this limiter's keys only)"""
|
|
_, current, _ = self._windows()
|
|
db = self._db()
|
|
cursor = db.execute(
|
|
"DELETE FROM rate_limits WHERE key LIKE ? AND window < ?",
|
|
(f"{self.prefix}:%", current - 1),
|
|
)
|
|
db.commit()
|
|
return cursor.rowcount
|
|
|
|
|
|
def init_rate_limiters(app) -> None:
|
|
"""
|
|
Initialize rate limiters with Flask app configuration.
|
|
|
|
Args:
|
|
app: Flask application instance
|
|
"""
|
|
global _ip_limiter, _domain_limiter
|
|
|
|
if not app.config.get('WEBMENTION_RATE_LIMIT_ENABLED', True):
|
|
app.logger.info("Rate limiting disabled")
|
|
_ip_limiter = None
|
|
_domain_limiter = None
|
|
return
|
|
|
|
ip_limit = app.config.get('WEBMENTION_RATE_LIMIT_IP', 60)
|
|
domain_limit = app.config.get('WEBMENTION_RATE_LIMIT_DOMAIN', 10)
|
|
window = app.config.get('WEBMENTION_RATE_LIMIT_WINDOW', 60)
|
|
|
|
# Shared by all server processes (v2.11.0)
|
|
_ip_limiter = SharedRateLimiter(ip_limit, window, "webmention-ip")
|
|
_domain_limiter = SharedRateLimiter(domain_limit, window, "webmention-domain")
|
|
|
|
app.logger.info(
|
|
f"Webmention rate limiter initialized: {ip_limit}/min IP, "
|
|
f"{domain_limit}/min domain"
|
|
)
|
|
|
|
|
|
def check_rate_limit(ip: str, source_domain: str | None) -> Tuple[bool, str | None]:
|
|
"""
|
|
Check rate limits for a request.
|
|
|
|
Args:
|
|
ip: Client IP address
|
|
source_domain: Source URL domain (optional)
|
|
|
|
Returns:
|
|
(allowed, error_message) tuple
|
|
- allowed: True if request allowed, False if rate limited
|
|
- error_message: Error description if rate limited, None otherwise
|
|
"""
|
|
# If rate limiting disabled, allow all requests
|
|
if _ip_limiter is None:
|
|
return True, None
|
|
|
|
# Check IP limit first
|
|
if not _ip_limiter.is_allowed(ip):
|
|
retry_after = _ip_limiter.get_retry_after(ip)
|
|
count = _ip_limiter.get_request_count(ip)
|
|
return False, f"Too many requests from your IP. Please retry after {retry_after} seconds."
|
|
|
|
# Check domain limit if source domain provided
|
|
if source_domain and _domain_limiter is not None:
|
|
if not _domain_limiter.is_allowed(source_domain):
|
|
retry_after = _domain_limiter.get_retry_after(source_domain)
|
|
count = _domain_limiter.get_request_count(source_domain)
|
|
return False, f"Too many requests from source domain. Please retry after {retry_after} seconds."
|
|
|
|
return True, None
|
|
|
|
|
|
def cleanup_rate_limiters() -> None:
|
|
"""
|
|
Cleanup expired entries from rate limiters.
|
|
|
|
Called periodically to prevent memory growth.
|
|
"""
|
|
if _ip_limiter:
|
|
_ip_limiter.cleanup()
|
|
if _domain_limiter:
|
|
_domain_limiter.cleanup()
|