phil/StarPunk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Implement v1.1.1 Phase 1 - Core Infrastructure

Browse Source 
Phase 1 of v1.1.1 "Polish" release focusing on production readiness.
Implements logging, connection pooling, validation, and error handling.

Following specs in docs/design/v1.1.1/developer-qa.md and ADRs 052-055.

**Structured Logging** (Q3, ADR-054)
- RotatingFileHandler (10MB files, keep 10)
- Correlation IDs for request tracing
- All print statements replaced with logging
- Context-aware correlation IDs (init/request)
- Logs written to data/logs/starpunk.log

**Database Connection Pooling** (Q2, ADR-053)
- Connection pool with configurable size (default: 5)
- Request-scoped connections via Flask g object
- Pool statistics for monitoring
- WAL mode enabled for concurrency
- Backward compatible get_db() signature

**Configuration Validation** (Q14, ADR-052)
- Validates presence and type of all config values
- Fail-fast startup with clear error messages
- LOG_LEVEL enum validation
- Type checking for strings, integers, paths
- Non-zero exit status on errors

**Centralized Error Handling** (Q4, ADR-055)
- Moved handlers to starpunk/errors.py
- Micropub spec-compliant JSON errors
- HTML templates for browser requests
- All errors logged with correlation IDs
- MicropubError exception class

**Database Module Reorganization**
- Moved database.py to database/ package
- Separated init.py, pool.py, schema.py
- Maintains backward compatibility
- Cleaner separation of concerns

**Testing**
- 580 tests passing
- 1 pre-existing flaky test noted
- No breaking changes to public API

**Documentation**
- CHANGELOG.md updated with v1.1.1 entry
- Version bumped to 1.1.1
- Implementation report in docs/reports/

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Phil Skentelbery
2025-11-25 13:56:30 -07:00
parent f62d3c5382
commit 93d2398c1d
9 changed files with 1011 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

107
starpunk/__init__.py
View File

@@ -4,12 +4,20 @@ Creates and configures the Flask application
"""
import logging
from flask import Flask
from logging.handlers import RotatingFileHandler
from pathlib import Path
from flask import Flask, g
import uuid
def configure_logging(app):
"""
Configure application logging based on LOG_LEVEL
Configure application logging with RotatingFileHandler and structured logging
Per ADR-054 and developer Q&A Q3:
- Uses RotatingFileHandler (10MB files, keep 10)
- Supports correlation IDs for request tracking
- Uses Flask's app.logger for all logging
Args:
app: Flask application instance
@@ -19,12 +27,24 @@ def configure_logging(app):
# Set Flask logger level
app.logger.setLevel(getattr(logging, log_level, logging.INFO))
# Configure handler with detailed format for DEBUG
handler = logging.StreamHandler()
# Configure console handler
console_handler = logging.StreamHandler()
# Configure file handler with rotation (10MB per file, keep 10 files)
log_dir = app.config.get("DATA_PATH", Path("./data")) / "logs"
log_dir.mkdir(parents=True, exist_ok=True)
log_file = log_dir / "starpunk.log"
file_handler = RotatingFileHandler(
log_file,
maxBytes=10 * 1024 * 1024, # 10MB
backupCount=10
)
# Format with correlation ID support
if log_level == "DEBUG":
formatter = logging.Formatter(
"[%(asctime)s] %(levelname)s - %(name)s: %(message)s",
"[%(asctime)s] %(levelname)s - %(name)s [%(correlation_id)s]: %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)
@@ -41,14 +61,48 @@ def configure_logging(app):
)
else:
formatter = logging.Formatter(
"[%(asctime)s] %(levelname)s: %(message)s", datefmt="%Y-%m-%d %H:%M:%S"
"[%(asctime)s] %(levelname)s [%(correlation_id)s]: %(message)s",
datefmt="%Y-%m-%d %H:%M:%S"
)
handler.setFormatter(formatter)
console_handler.setFormatter(formatter)
file_handler.setFormatter(formatter)
# Remove existing handlers and add our configured handler
# Remove existing handlers and add our configured handlers
app.logger.handlers.clear()
app.logger.addHandler(handler)
app.logger.addHandler(console_handler)
app.logger.addHandler(file_handler)
# Add filter to inject correlation ID
# This filter will be added to ALL loggers to ensure consistency
class CorrelationIdFilter(logging.Filter):
def filter(self, record):
# Get correlation ID from Flask's g object, or use fallback
# Handle case where we're outside of request context
if not hasattr(record, 'correlation_id'):
try:
from flask import has_request_context
if has_request_context():
record.correlation_id = getattr(g, 'correlation_id', 'no-request')
else:
record.correlation_id = 'init'
except (RuntimeError, AttributeError):
record.correlation_id = 'init'
return True
# Apply filter to Flask's app logger
correlation_filter = CorrelationIdFilter()
app.logger.addFilter(correlation_filter)
# Also apply to the root logger to catch all logging calls
root_logger = logging.getLogger()
root_logger.addFilter(correlation_filter)
def add_correlation_id():
"""Generate and store correlation ID for the current request"""
if not hasattr(g, 'correlation_id'):
g.correlation_id = str(uuid.uuid4())
def create_app(config=None):
@@ -71,11 +125,14 @@ def create_app(config=None):
# Configure logging
configure_logging(app)
# Initialize database
from starpunk.database import init_db
# Initialize database schema
from starpunk.database import init_db, init_pool
init_db(app)
# Initialize connection pool
init_pool(app)
# Initialize FTS index if needed
from pathlib import Path
from starpunk.search import has_fts_table, rebuild_fts_index
@@ -106,24 +163,16 @@ def create_app(config=None):
register_routes(app)
# Error handlers
@app.errorhandler(404)
def not_found(error):
from flask import render_template, request
# Request middleware - Add correlation ID to each request
@app.before_request
def before_request():
"""Add correlation ID to request context for tracing"""
add_correlation_id()
# Return HTML for browser requests, JSON for API requests
if request.path.startswith("/api/"):
return {"error": "Not found"}, 404
return render_template("404.html"), 404
# Register centralized error handlers
from starpunk.errors import register_error_handlers
@app.errorhandler(500)
def server_error(error):
from flask import render_template, request
# Return HTML for browser requests, JSON for API requests
if request.path.startswith("/api/"):
return {"error": "Internal server error"}, 500
return render_template("500.html"), 500
register_error_handlers(app)
# Health check endpoint for containers and monitoring
@app.route("/health")
@@ -178,5 +227,5 @@ def create_app(config=None):
# Package version (Semantic Versioning 2.0.0)
# See docs/standards/versioning-strategy.md for details
__version__ = "1.1.0"
__version_info__ = (1, 1, 0)
__version__ = "1.1.1"
__version_info__ = (1, 1, 1)

79
starpunk/config.py
View File

@@ -111,6 +111,12 @@ def validate_config(app):
"""
Validate application configuration on startup
Per ADR-052 and developer Q&A Q14:
- Validates at startup (fail fast)
- Checks both presence and type of required values
- Provides clear error messages
- Exits with non-zero status on failure
Ensures required configuration is present based on mode (dev/production)
and warns prominently if development mode is enabled.
@@ -118,8 +124,60 @@ def validate_config(app):
app: Flask application instance
Raises:
ValueError: If required configuration is missing
ValueError: If required configuration is missing or invalid
"""
errors = []
# Validate required string fields
required_strings = {
'SITE_URL': app.config.get('SITE_URL'),
'SITE_NAME': app.config.get('SITE_NAME'),
'SITE_AUTHOR': app.config.get('SITE_AUTHOR'),
'SESSION_SECRET': app.config.get('SESSION_SECRET'),
'SECRET_KEY': app.config.get('SECRET_KEY'),
}
for field, value in required_strings.items():
if not value:
errors.append(f"{field} is required but not set")
elif not isinstance(value, str):
errors.append(f"{field} must be a string, got {type(value).__name__}")
# Validate required integer fields
required_ints = {
'SESSION_LIFETIME': app.config.get('SESSION_LIFETIME'),
'FEED_MAX_ITEMS': app.config.get('FEED_MAX_ITEMS'),
'FEED_CACHE_SECONDS': app.config.get('FEED_CACHE_SECONDS'),
}
for field, value in required_ints.items():
if value is None:
errors.append(f"{field} is required but not set")
elif not isinstance(value, int):
errors.append(f"{field} must be an integer, got {type(value).__name__}")
elif value < 0:
errors.append(f"{field} must be non-negative, got {value}")
# Validate required Path fields
required_paths = {
'DATA_PATH': app.config.get('DATA_PATH'),
'NOTES_PATH': app.config.get('NOTES_PATH'),
'DATABASE_PATH': app.config.get('DATABASE_PATH'),
}
for field, value in required_paths.items():
if not value:
errors.append(f"{field} is required but not set")
elif not isinstance(value, Path):
errors.append(f"{field} must be a Path object, got {type(value).__name__}")
# Validate LOG_LEVEL
log_level = app.config.get('LOG_LEVEL', 'INFO').upper()
valid_log_levels = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
if log_level not in valid_log_levels:
errors.append(f"LOG_LEVEL must be one of {valid_log_levels}, got '{log_level}'")
# Mode-specific validation
dev_mode = app.config.get("DEV_MODE", False)
if dev_mode:
@@ -133,14 +191,29 @@ def validate_config(app):
# Require DEV_ADMIN_ME in dev mode
if not app.config.get("DEV_ADMIN_ME"):
raise ValueError(
errors.append(
"DEV_MODE=true requires DEV_ADMIN_ME to be set. "
"Set DEV_ADMIN_ME=https://your-dev-identity.example.com in .env"
)
else:
# Production mode: ADMIN_ME is required
if not app.config.get("ADMIN_ME"):
raise ValueError(
errors.append(
"Production mode requires ADMIN_ME to be set. "
"Set ADMIN_ME=https://your-site.com in .env"
)
# If there are validation errors, fail fast with clear message
if errors:
error_msg = "\n".join([
"=" * 70,
"CONFIGURATION VALIDATION FAILED",
"=" * 70,
"The following configuration errors were found:",
"",
*[f" - {error}" for error in errors],
"",
"Please fix these errors in your .env file and restart.",
"=" * 70
])
raise ValueError(error_msg)

16
starpunk/database/__init__.py Normal file
View File

@@ -0,0 +1,16 @@
"""
Database package for StarPunk
Provides database initialization and connection pooling
Per v1.1.1 Phase 1:
- Connection pooling for improved performance (ADR-053)
- Request-scoped connections via Flask's g object
- Pool statistics for monitoring
"""
from starpunk.database.init import init_db
from starpunk.database.pool import init_pool, get_db, get_pool_stats
from starpunk.database.schema import INITIAL_SCHEMA_SQL
__all__ = ['init_db', 'init_pool', 'get_db', 'get_pool_stats', 'INITIAL_SCHEMA_SQL']

44
starpunk/database/init.py Normal file
View File

@@ -0,0 +1,44 @@
"""
Database initialization for StarPunk
"""
import sqlite3
from pathlib import Path
from starpunk.database.schema import INITIAL_SCHEMA_SQL
def init_db(app=None):
"""
Initialize database schema and run migrations
Args:
app: Flask application instance (optional, for config access)
"""
if app:
db_path = app.config["DATABASE_PATH"]
logger = app.logger
else:
# Fallback to default path
db_path = Path("./data/starpunk.db")
logger = None
# Ensure parent directory exists
db_path.parent.mkdir(parents=True, exist_ok=True)
# Create database and initial schema
conn = sqlite3.connect(db_path)
try:
conn.executescript(INITIAL_SCHEMA_SQL)
conn.commit()
if logger:
logger.info(f"Database initialized: {db_path}")
else:
# Fallback logging when logger not available (e.g., during testing)
import logging
logging.getLogger(__name__).info(f"Database initialized: {db_path}")
finally:
conn.close()
# Run migrations
from starpunk.migrations import run_migrations
run_migrations(db_path, logger=logger)

196
starpunk/database/pool.py Normal file
View File

@@ -0,0 +1,196 @@
"""
Database connection pool for StarPunk
Per ADR-053 and developer Q&A Q2:
- Provides connection pooling for improved performance
- Integrates with Flask's g object for request-scoped connections
- Maintains same interface as get_db() for transparency
- Pool statistics available for metrics
Note: Migrations use direct connections (not pooled) for isolation
"""
import sqlite3
from pathlib import Path
from threading import Lock
from collections import deque
from flask import g
class ConnectionPool:
"""
Simple connection pool for SQLite
SQLite doesn't benefit from traditional connection pooling like PostgreSQL,
but this provides connection reuse and request-scoped connection management.
"""
def __init__(self, db_path, pool_size=5, timeout=10.0):
"""
Initialize connection pool
Args:
db_path: Path to SQLite database file
pool_size: Maximum number of connections in pool
timeout: Timeout for getting connection (seconds)
"""
self.db_path = Path(db_path)
self.pool_size = pool_size
self.timeout = timeout
self._pool = deque(maxlen=pool_size)
self._lock = Lock()
self._stats = {
'connections_created': 0,
'connections_reused': 0,
'connections_closed': 0,
'pool_hits': 0,
'pool_misses': 0,
}
def _create_connection(self):
"""Create a new database connection"""
conn = sqlite3.connect(
self.db_path,
timeout=self.timeout,
check_same_thread=False # Allow connection reuse across threads
)
conn.row_factory = sqlite3.Row # Return rows as dictionaries
# Enable WAL mode for better concurrency
conn.execute("PRAGMA journal_mode=WAL")
self._stats['connections_created'] += 1
return conn
def get_connection(self):
"""
Get a connection from the pool
Returns:
sqlite3.Connection: Database connection
"""
with self._lock:
if self._pool:
# Reuse existing connection
conn = self._pool.pop()
self._stats['pool_hits'] += 1
self._stats['connections_reused'] += 1
return conn
else:
# Create new connection
self._stats['pool_misses'] += 1
return self._create_connection()
def return_connection(self, conn):
"""
Return a connection to the pool
Args:
conn: Database connection to return
"""
if not conn:
return
with self._lock:
if len(self._pool) < self.pool_size:
# Return to pool
self._pool.append(conn)
else:
# Pool is full, close connection
conn.close()
self._stats['connections_closed'] += 1
def close_connection(self, conn):
"""
Close a connection without returning to pool
Args:
conn: Database connection to close
"""
if conn:
conn.close()
self._stats['connections_closed'] += 1
def get_stats(self):
"""
Get pool statistics
Returns:
dict: Pool statistics for monitoring
"""
with self._lock:
return {
**self._stats,
'pool_size': len(self._pool),
'max_pool_size': self.pool_size,
}
def close_all(self):
"""Close all connections in the pool"""
with self._lock:
while self._pool:
conn = self._pool.pop()
conn.close()
self._stats['connections_closed'] += 1
# Global pool instance (initialized by app factory)
_pool = None
def init_pool(app):
"""
Initialize the connection pool
Args:
app: Flask application instance
"""
global _pool
db_path = app.config['DATABASE_PATH']
pool_size = app.config.get('DB_POOL_SIZE', 5)
timeout = app.config.get('DB_TIMEOUT', 10.0)
_pool = ConnectionPool(db_path, pool_size, timeout)
app.logger.info(f"Database connection pool initialized (size={pool_size})")
# Register teardown handler
@app.teardown_appcontext
def close_connection(error):
"""Return connection to pool when request context ends"""
conn = g.pop('db', None)
if conn:
_pool.return_connection(conn)
def get_db(app=None):
"""
Get database connection for current request
Uses Flask's g object for request-scoped connection management.
Connection is automatically returned to pool at end of request.
Args:
app: Flask application (optional, for backward compatibility with tests)
When provided, this parameter is ignored as we use the pool
Returns:
sqlite3.Connection: Database connection
"""
# Note: app parameter is kept for backward compatibility but ignored
# The pool is request-scoped via Flask's g object
if 'db' not in g:
g.db = _pool.get_connection()
return g.db
def get_pool_stats():
"""
Get connection pool statistics
Returns:
dict: Pool statistics for monitoring
"""
if _pool:
return _pool.get_stats()
return {}

65
starpunk/database.py → starpunk/database/schema.py
View File

@@ -1,15 +1,11 @@
"""
Database initialization and operations for StarPunk
SQLite database for metadata, sessions, and tokens
Database schema definition for StarPunk
Initial database schema (v1.0.0 baseline)
DO NOT MODIFY - This represents the v1.0.0 schema state
All schema changes after v1.0.0 must go in migration files
"""
import sqlite3
from pathlib import Path
# Initial database schema (v1.0.0 baseline)
# DO NOT MODIFY - This represents the v1.0.0 schema state
# All schema changes after v1.0.0 must go in migration files
INITIAL_SCHEMA_SQL = """
-- Notes metadata (content is in files)
CREATE TABLE IF NOT EXISTS notes (
@@ -86,54 +82,3 @@ CREATE TABLE IF NOT EXISTS auth_state (
CREATE INDEX IF NOT EXISTS idx_auth_state_expires ON auth_state(expires_at);
"""
def init_db(app=None):
"""
Initialize database schema and run migrations
Args:
app: Flask application instance (optional, for config access)
"""
if app:
db_path = app.config["DATABASE_PATH"]
logger = app.logger
else:
# Fallback to default path
db_path = Path("./data/starpunk.db")
logger = None
# Ensure parent directory exists
db_path.parent.mkdir(parents=True, exist_ok=True)
# Create database and initial schema
conn = sqlite3.connect(db_path)
try:
conn.executescript(INITIAL_SCHEMA_SQL)
conn.commit()
if logger:
logger.info(f"Database initialized: {db_path}")
else:
print(f"Database initialized: {db_path}")
finally:
conn.close()
# Run migrations
from starpunk.migrations import run_migrations
run_migrations(db_path, logger=logger)
def get_db(app):
"""
Get database connection
Args:
app: Flask application instance
Returns:
sqlite3.Connection
"""
db_path = app.config["DATABASE_PATH"]
conn = sqlite3.connect(db_path)
conn.row_factory = sqlite3.Row # Return rows as dictionaries
return conn

189
starpunk/errors.py Normal file
View File

@@ -0,0 +1,189 @@
"""
Centralized error handling for StarPunk
Per ADR-055 and developer Q&A Q4:
- Uses Flask's @app.errorhandler decorator
- Registered in app factory (centralized)
- Micropub endpoints return spec-compliant JSON errors
- Other endpoints return HTML error pages
- All errors logged with correlation IDs
"""
from flask import request, render_template, jsonify, g
def register_error_handlers(app):
"""
Register centralized error handlers
Checks request path to determine response format:
- /micropub/* returns JSON (Micropub spec compliance)
- All others return HTML templates
All errors are logged with correlation IDs for tracing
Args:
app: Flask application instance
"""
@app.errorhandler(400)
def bad_request(error):
"""Handle 400 Bad Request errors"""
correlation_id = getattr(g, 'correlation_id', 'no-request')
app.logger.warning(f"Bad request: {error}")
if request.path.startswith('/micropub'):
# Micropub spec-compliant error response
return jsonify({
'error': 'invalid_request',
'error_description': str(error) or 'Bad request'
}), 400
return render_template('400.html', error=error), 400
@app.errorhandler(401)
def unauthorized(error):
"""Handle 401 Unauthorized errors"""
correlation_id = getattr(g, 'correlation_id', 'no-request')
app.logger.warning(f"Unauthorized access attempt")
if request.path.startswith('/micropub'):
# Micropub spec-compliant error response
return jsonify({
'error': 'unauthorized',
'error_description': 'Authentication required'
}), 401
return render_template('401.html'), 401
@app.errorhandler(403)
def forbidden(error):
"""Handle 403 Forbidden errors"""
correlation_id = getattr(g, 'correlation_id', 'no-request')
app.logger.warning(f"Forbidden access attempt")
if request.path.startswith('/micropub'):
# Micropub spec-compliant error response
return jsonify({
'error': 'forbidden',
'error_description': 'Insufficient scope or permissions'
}), 403
return render_template('403.html'), 403
@app.errorhandler(404)
def not_found(error):
"""Handle 404 Not Found errors"""
# Don't log 404s at warning level - they're common and not errors
app.logger.debug(f"Resource not found: {request.path}")
if request.path.startswith('/api/') or request.path.startswith('/micropub'):
return jsonify({'error': 'Not found'}), 404
return render_template('404.html'), 404
@app.errorhandler(405)
def method_not_allowed(error):
"""Handle 405 Method Not Allowed errors"""
correlation_id = getattr(g, 'correlation_id', 'no-request')
app.logger.warning(f"Method not allowed: {request.method} {request.path}")
if request.path.startswith('/micropub'):
return jsonify({
'error': 'invalid_request',
'error_description': f'Method {request.method} not allowed'
}), 405
return render_template('405.html'), 405
@app.errorhandler(500)
def internal_server_error(error):
"""Handle 500 Internal Server Error"""
correlation_id = getattr(g, 'correlation_id', 'no-request')
app.logger.error(f"Internal server error: {error}", exc_info=True)
if request.path.startswith('/api/') or request.path.startswith('/micropub'):
# Don't expose internal error details in API responses
if request.path.startswith('/micropub'):
return jsonify({
'error': 'server_error',
'error_description': 'An internal server error occurred'
}), 500
else:
return jsonify({'error': 'Internal server error'}), 500
return render_template('500.html'), 500
@app.errorhandler(503)
def service_unavailable(error):
"""Handle 503 Service Unavailable errors"""
correlation_id = getattr(g, 'correlation_id', 'no-request')
app.logger.error(f"Service unavailable: {error}")
if request.path.startswith('/api/') or request.path.startswith('/micropub'):
return jsonify({
'error': 'temporarily_unavailable',
'error_description': 'Service temporarily unavailable'
}), 503
return render_template('503.html'), 503
# Register generic exception handler
@app.errorhandler(Exception)
def handle_exception(error):
"""
Handle uncaught exceptions
Logs the full exception with correlation ID and returns appropriate error response
"""
correlation_id = getattr(g, 'correlation_id', 'no-request')
app.logger.error(f"Uncaught exception: {error}", exc_info=True)
# If it's an HTTP exception, let Flask handle it
if hasattr(error, 'code'):
return error
# Otherwise, return 500
if request.path.startswith('/micropub'):
return jsonify({
'error': 'server_error',
'error_description': 'An unexpected error occurred'
}), 500
elif request.path.startswith('/api/'):
return jsonify({'error': 'Internal server error'}), 500
else:
return render_template('500.html'), 500
class MicropubError(Exception):
"""
Micropub-specific error class
Automatically formats errors according to Micropub spec
"""
def __init__(self, error_code, description, status_code=400):
"""
Initialize Micropub error
Args:
error_code: Micropub error code (e.g., 'invalid_request', 'insufficient_scope')
description: Human-readable error description
status_code: HTTP status code (default 400)
"""
self.error_code = error_code
self.description = description
self.status_code = status_code
super().__init__(description)
def to_response(self):
"""
Convert to Micropub-compliant JSON response
Returns:
tuple: (dict, int) Flask response tuple
"""
return jsonify({
'error': self.error_code,
'error_description': self.description
}), self.status_code