Compare commits

..
2 Commits
Author SHA1 Message Date
Phil 57682165a4 Merge branch 'fix/v2.8.1-migration-race' for v2.8.1 release
Build Container / build (push) Successful in 7m34s
2026-11-03 09:58:20 -07:00
PhilandClaude Opus 5.5 8c51b52ed6 fix: Hold the migration lock for the whole run (v2.8.1)
create_migrations_table() committed right after BEGIN IMMEDIATE, and
executescript() commits before running, so gunicorn workers applied
migrations concurrently; the v2.8.0 deploy failed to boot once with
'duplicate column name'. Run migration statements with conn.execute inside
the transaction and don't commit when creating the tracking table.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-03 09:58:20 -07:00
6 changed files with 172 additions and 10 deletions

No files matched your search

+6
View File
@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
## [2.8.1] - 2026-11-03
### Fixed
- Starting the server after an upgrade with a database migration could fail: several server processes applied the same migration at once and one stopped with "duplicate column name", taking the server down until it restarted (this happened deploying v2.8.0, for about two seconds). The migration lock was released right after it was taken, and again before each migration was recorded. Migrations now run entirely inside the lock
## [2.8.0] - 2026-11-03
Microformats2 review: the markup now matches the [microformats2](https://microformats.org/wiki/microformats2) vocabularies, so IndieWeb readers and tools see what the page shows.
@@ -0,0 +1,33 @@
# v2.8.1: Migration lock held for the whole run
## Incident
Deploying v2.8.0 (migration 014 adds two columns): one gunicorn worker
applied 014, another failed with `duplicate column name: source_post_url`,
gunicorn shut down ("Worker failed to boot") and Docker restarted the
container. About two seconds of downtime; schema correct afterwards.
## Causes
`run_migrations` takes `BEGIN IMMEDIATE` so one worker migrates at a time,
but the lock was released early twice:
1. `create_migrations_table()`, called right after `BEGIN IMMEDIATE`,
ended with `conn.commit()`. Everything after it ran without the lock.
2. `conn.executescript()` COMMITs any open transaction before running the
script, so the lock was dropped before the migration was recorded in
`schema_migrations`.
## Fix
- `create_migrations_table(conn, commit=True)`; `run_migrations` passes
`commit=False`.
- `execute_in_transaction()` runs each statement from
`split_sql_statements()` (built on `sqlite3.complete_statement`, so
trigger bodies stay whole) with `conn.execute`, inside the transaction.
## Verification
Four processes migrating a database at 013 simultaneously: 6/30 trials
failed before, 0/30 after. `tests/test_migration_concurrency.py` runs this
five times, plus a rollback test showing a migration commits nothing early.
+2 -2
View File
@@ -463,5 +463,5 @@ def create_app(config=None):
# Package version (Semantic Versioning 2.0.0)
# See docs/standards/versioning-strategy.md for details
__version__ = "2.8.0"
__version_info__ = (2, 8, 0)
__version__ = "2.8.1"
__version_info__ = (2, 8, 1)
+42 -6
View File
@@ -31,12 +31,47 @@ class MigrationError(Exception):
pass
def create_migrations_table(conn):
def split_sql_statements(script: str) -> list[str]:
"""
Split a SQL script into complete statements (v2.8.1)
Uses sqlite3.complete_statement, so semicolons inside strings and
trigger bodies (BEGIN ... END) don't split a statement.
"""
statements = []
current = ""
for line in script.splitlines(keepends=True):
current += line
if sqlite3.complete_statement(current):
if current.strip().strip(";").strip():
statements.append(current.strip())
current = ""
if current.strip():
statements.append(current.strip())
return statements
def execute_in_transaction(conn, script: str) -> None:
"""
Run a SQL script inside the connection's open transaction
Not conn.executescript(): that COMMITs first, which released the
migration lock mid-migration and let another gunicorn worker apply the
same migration again ("duplicate column name", v2.8.0 deploy).
"""
for statement in split_sql_statements(script):
conn.execute(statement)
def create_migrations_table(conn, commit=True):
"""
Create schema_migrations tracking table if it doesn't exist
Args:
conn: SQLite connection
commit: Commit afterwards. run_migrations passes False: committing
there released its BEGIN IMMEDIATE lock straight away, so
gunicorn workers applied migrations concurrently (v2.8.1)
"""
conn.execute("""
CREATE TABLE IF NOT EXISTS schema_migrations (
@@ -51,7 +86,8 @@ def create_migrations_table(conn):
ON schema_migrations(migration_name)
""")
conn.commit()
if commit:
conn.commit()
def is_schema_current(conn):
@@ -287,7 +323,7 @@ def apply_migration(conn, migration_name, migration_path, logger=None):
# Execute migration in transaction
conn.execute("BEGIN")
conn.executescript(migration_sql)
execute_in_transaction(conn, migration_sql)
# Record migration as applied
conn.execute(
@@ -374,7 +410,7 @@ def run_migrations(db_path, logger=None):
try:
# Ensure migrations tracking table exists
create_migrations_table(conn)
create_migrations_table(conn, commit=False)
# Quick check: have migrations already been applied by another worker?
cursor = conn.execute("SELECT COUNT(*) FROM schema_migrations")
@@ -472,8 +508,8 @@ def run_migrations(db_path, logger=None):
logger.debug(f"Applying migration: {migration_name}")
# Execute migration (already in transaction)
conn.executescript(migration_sql)
# Execute migration inside the locked transaction
execute_in_transaction(conn, migration_sql)
# Record migration as applied
conn.execute(
+86
View File
@@ -0,0 +1,86 @@
"""
Migrations under concurrent gunicorn workers (v2.8.1)
The v2.8.0 deploy failed to boot: one worker applied migration 014 while
another applied it again ("duplicate column name"), because
conn.executescript() committed and released the migration lock early.
"""
import multiprocessing
import sqlite3
from pathlib import Path
from starpunk.database.schema import INITIAL_SCHEMA_SQL
from starpunk.migrations import execute_in_transaction, run_migrations, split_sql_statements
MIGRATIONS = Path(__file__).parent.parent / "migrations"
def test_script_runs_inside_the_transaction(tmp_path):
conn = sqlite3.connect(tmp_path / "t.db")
conn.execute("CREATE TABLE t (a INTEGER)")
conn.commit()
conn.execute("BEGIN IMMEDIATE")
execute_in_transaction(conn, "ALTER TABLE t ADD COLUMN b TEXT;\nALTER TABLE t ADD COLUMN c TEXT;")
conn.rollback()
assert [row[1] for row in conn.execute("PRAGMA table_info(t)")] == ["a"]
def test_split_keeps_trigger_bodies_together():
script = """
-- comment; with a semicolon
CREATE TABLE t (a TEXT DEFAULT 'x;y');
CREATE TRIGGER tr AFTER DELETE ON t BEGIN
DELETE FROM t WHERE a = old.a;
SELECT 1;
END;
"""
statements = split_sql_statements(script)
assert len(statements) == 2
assert statements[1].startswith("CREATE TRIGGER") and statements[1].endswith("END;")
def _migrate(db_path, barrier, errors):
try:
barrier.wait()
run_migrations(db_path)
except Exception as e: # report to the parent
errors.put(repr(e))
import pytest
@pytest.mark.parametrize("trial", range(5)) # the race hit ~1 in 5 runs
def test_concurrent_workers_apply_each_migration_once(tmp_path, trial):
# A database at migration 013, like production before v2.8.0
db_path = tmp_path / "starpunk.db"
conn = sqlite3.connect(db_path)
conn.executescript(INITIAL_SCHEMA_SQL) # as init_db does
conn.close()
run_migrations(db_path)
conn = sqlite3.connect(db_path)
conn.execute("DELETE FROM schema_migrations WHERE migration_name = '014_webmention_source_details.sql'")
conn.commit()
columns = [row[1] for row in conn.execute("PRAGMA table_info(webmentions)")]
if "source_post_url" in columns:
conn.execute("ALTER TABLE webmentions DROP COLUMN source_post_url")
conn.execute("ALTER TABLE webmentions DROP COLUMN source_published")
conn.commit()
conn.close()
ctx = multiprocessing.get_context("fork")
barrier, errors = ctx.Barrier(4), ctx.Queue()
workers = [ctx.Process(target=_migrate, args=(db_path, barrier, errors)) for _ in range(4)]
for w in workers:
w.start()
for w in workers:
w.join(timeout=60)
assert errors.empty(), errors.get()
conn = sqlite3.connect(db_path)
assert conn.execute(
"SELECT COUNT(*) FROM schema_migrations WHERE migration_name = '014_webmention_source_details.sql'"
).fetchone()[0] == 1
+3 -2
View File
@@ -302,12 +302,13 @@ class TestErrorHandling:
def test_rollback_failure_causes_system_exit(self, temp_db):
"""Test that rollback failure raises SystemExit"""
with patch('sqlite3.connect') as mock_connect:
with patch('sqlite3.connect') as mock_connect, \
patch('starpunk.migrations.execute_in_transaction',
side_effect=Exception("Migration failed")):
mock_conn = MagicMock()
mock_connect.return_value = mock_conn
# Make both migration and rollback fail
mock_conn.executescript.side_effect = Exception("Migration failed")
mock_conn.rollback.side_effect = Exception("Rollback failed")
mock_conn.execute.return_value.fetchone.return_value = (0,)