From 8c51b52ed68184e2d971ab69ff090c5033d9fc1e Mon Sep 17 00:00:00 2001 From: Phil Date: Tue, 3 Nov 2026 09:58:20 -0700 Subject: [PATCH] fix: Hold the migration lock for the whole run (v2.8.1) create_migrations_table() committed right after BEGIN IMMEDIATE, and executescript() commits before running, so gunicorn workers applied migrations concurrently; the v2.8.0 deploy failed to boot once with 'duplicate column name'. Run migration statements with conn.execute inside the transaction and don't commit when creating the tracking table. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 ++ .../v2.8.1/2026-11-03-migration-lock.md | 33 +++++++ starpunk/__init__.py | 4 +- starpunk/migrations.py | 48 +++++++++-- tests/test_migration_concurrency.py | 86 +++++++++++++++++++ tests/test_migration_race_condition.py | 5 +- 6 files changed, 172 insertions(+), 10 deletions(-) create mode 100644 docs/design/v2.8.1/2026-11-03-migration-lock.md create mode 100644 tests/test_migration_concurrency.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 109b3d3..e65a77c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [2.8.1] - 2026-11-03 + +### Fixed + +- Starting the server after an upgrade with a database migration could fail: several server processes applied the same migration at once and one stopped with "duplicate column name", taking the server down until it restarted (this happened deploying v2.8.0, for about two seconds). The migration lock was released right after it was taken, and again before each migration was recorded. Migrations now run entirely inside the lock + ## [2.8.0] - 2026-11-03 Microformats2 review: the markup now matches the [microformats2](https://microformats.org/wiki/microformats2) vocabularies, so IndieWeb readers and tools see what the page shows. diff --git a/docs/design/v2.8.1/2026-11-03-migration-lock.md b/docs/design/v2.8.1/2026-11-03-migration-lock.md new file mode 100644 index 0000000..fbb0962 --- /dev/null +++ b/docs/design/v2.8.1/2026-11-03-migration-lock.md @@ -0,0 +1,33 @@ +# v2.8.1: Migration lock held for the whole run + +## Incident + +Deploying v2.8.0 (migration 014 adds two columns): one gunicorn worker +applied 014, another failed with `duplicate column name: source_post_url`, +gunicorn shut down ("Worker failed to boot") and Docker restarted the +container. About two seconds of downtime; schema correct afterwards. + +## Causes + +`run_migrations` takes `BEGIN IMMEDIATE` so one worker migrates at a time, +but the lock was released early twice: + +1. `create_migrations_table()`, called right after `BEGIN IMMEDIATE`, + ended with `conn.commit()`. Everything after it ran without the lock. +2. `conn.executescript()` COMMITs any open transaction before running the + script, so the lock was dropped before the migration was recorded in + `schema_migrations`. + +## Fix + +- `create_migrations_table(conn, commit=True)`; `run_migrations` passes + `commit=False`. +- `execute_in_transaction()` runs each statement from + `split_sql_statements()` (built on `sqlite3.complete_statement`, so + trigger bodies stay whole) with `conn.execute`, inside the transaction. + +## Verification + +Four processes migrating a database at 013 simultaneously: 6/30 trials +failed before, 0/30 after. `tests/test_migration_concurrency.py` runs this +five times, plus a rollback test showing a migration commits nothing early. diff --git a/starpunk/__init__.py b/starpunk/__init__.py index b51048e..9329421 100644 --- a/starpunk/__init__.py +++ b/starpunk/__init__.py @@ -463,5 +463,5 @@ def create_app(config=None): # Package version (Semantic Versioning 2.0.0) # See docs/standards/versioning-strategy.md for details -__version__ = "2.8.0" -__version_info__ = (2, 8, 0) +__version__ = "2.8.1" +__version_info__ = (2, 8, 1) diff --git a/starpunk/migrations.py b/starpunk/migrations.py index 490ff7b..2276b95 100644 --- a/starpunk/migrations.py +++ b/starpunk/migrations.py @@ -31,12 +31,47 @@ class MigrationError(Exception): pass -def create_migrations_table(conn): +def split_sql_statements(script: str) -> list[str]: + """ + Split a SQL script into complete statements (v2.8.1) + + Uses sqlite3.complete_statement, so semicolons inside strings and + trigger bodies (BEGIN ... END) don't split a statement. + """ + statements = [] + current = "" + for line in script.splitlines(keepends=True): + current += line + if sqlite3.complete_statement(current): + if current.strip().strip(";").strip(): + statements.append(current.strip()) + current = "" + if current.strip(): + statements.append(current.strip()) + return statements + + +def execute_in_transaction(conn, script: str) -> None: + """ + Run a SQL script inside the connection's open transaction + + Not conn.executescript(): that COMMITs first, which released the + migration lock mid-migration and let another gunicorn worker apply the + same migration again ("duplicate column name", v2.8.0 deploy). + """ + for statement in split_sql_statements(script): + conn.execute(statement) + + +def create_migrations_table(conn, commit=True): """ Create schema_migrations tracking table if it doesn't exist Args: conn: SQLite connection + commit: Commit afterwards. run_migrations passes False: committing + there released its BEGIN IMMEDIATE lock straight away, so + gunicorn workers applied migrations concurrently (v2.8.1) """ conn.execute(""" CREATE TABLE IF NOT EXISTS schema_migrations ( @@ -51,7 +86,8 @@ def create_migrations_table(conn): ON schema_migrations(migration_name) """) - conn.commit() + if commit: + conn.commit() def is_schema_current(conn): @@ -287,7 +323,7 @@ def apply_migration(conn, migration_name, migration_path, logger=None): # Execute migration in transaction conn.execute("BEGIN") - conn.executescript(migration_sql) + execute_in_transaction(conn, migration_sql) # Record migration as applied conn.execute( @@ -374,7 +410,7 @@ def run_migrations(db_path, logger=None): try: # Ensure migrations tracking table exists - create_migrations_table(conn) + create_migrations_table(conn, commit=False) # Quick check: have migrations already been applied by another worker? cursor = conn.execute("SELECT COUNT(*) FROM schema_migrations") @@ -472,8 +508,8 @@ def run_migrations(db_path, logger=None): logger.debug(f"Applying migration: {migration_name}") - # Execute migration (already in transaction) - conn.executescript(migration_sql) + # Execute migration inside the locked transaction + execute_in_transaction(conn, migration_sql) # Record migration as applied conn.execute( diff --git a/tests/test_migration_concurrency.py b/tests/test_migration_concurrency.py new file mode 100644 index 0000000..c435bc5 --- /dev/null +++ b/tests/test_migration_concurrency.py @@ -0,0 +1,86 @@ +""" +Migrations under concurrent gunicorn workers (v2.8.1) + +The v2.8.0 deploy failed to boot: one worker applied migration 014 while +another applied it again ("duplicate column name"), because +conn.executescript() committed and released the migration lock early. +""" + +import multiprocessing +import sqlite3 +from pathlib import Path + +from starpunk.database.schema import INITIAL_SCHEMA_SQL +from starpunk.migrations import execute_in_transaction, run_migrations, split_sql_statements + +MIGRATIONS = Path(__file__).parent.parent / "migrations" + + +def test_script_runs_inside_the_transaction(tmp_path): + conn = sqlite3.connect(tmp_path / "t.db") + conn.execute("CREATE TABLE t (a INTEGER)") + conn.commit() + + conn.execute("BEGIN IMMEDIATE") + execute_in_transaction(conn, "ALTER TABLE t ADD COLUMN b TEXT;\nALTER TABLE t ADD COLUMN c TEXT;") + conn.rollback() + + assert [row[1] for row in conn.execute("PRAGMA table_info(t)")] == ["a"] + + +def test_split_keeps_trigger_bodies_together(): + script = """ + -- comment; with a semicolon + CREATE TABLE t (a TEXT DEFAULT 'x;y'); + CREATE TRIGGER tr AFTER DELETE ON t BEGIN + DELETE FROM t WHERE a = old.a; + SELECT 1; + END; + """ + statements = split_sql_statements(script) + assert len(statements) == 2 + assert statements[1].startswith("CREATE TRIGGER") and statements[1].endswith("END;") + + +def _migrate(db_path, barrier, errors): + try: + barrier.wait() + run_migrations(db_path) + except Exception as e: # report to the parent + errors.put(repr(e)) + + +import pytest + + +@pytest.mark.parametrize("trial", range(5)) # the race hit ~1 in 5 runs +def test_concurrent_workers_apply_each_migration_once(tmp_path, trial): + # A database at migration 013, like production before v2.8.0 + db_path = tmp_path / "starpunk.db" + conn = sqlite3.connect(db_path) + conn.executescript(INITIAL_SCHEMA_SQL) # as init_db does + conn.close() + run_migrations(db_path) + conn = sqlite3.connect(db_path) + conn.execute("DELETE FROM schema_migrations WHERE migration_name = '014_webmention_source_details.sql'") + conn.commit() + columns = [row[1] for row in conn.execute("PRAGMA table_info(webmentions)")] + if "source_post_url" in columns: + conn.execute("ALTER TABLE webmentions DROP COLUMN source_post_url") + conn.execute("ALTER TABLE webmentions DROP COLUMN source_published") + conn.commit() + conn.close() + + ctx = multiprocessing.get_context("fork") + barrier, errors = ctx.Barrier(4), ctx.Queue() + workers = [ctx.Process(target=_migrate, args=(db_path, barrier, errors)) for _ in range(4)] + for w in workers: + w.start() + for w in workers: + w.join(timeout=60) + + assert errors.empty(), errors.get() + conn = sqlite3.connect(db_path) + assert conn.execute( + "SELECT COUNT(*) FROM schema_migrations WHERE migration_name = '014_webmention_source_details.sql'" + ).fetchone()[0] == 1 diff --git a/tests/test_migration_race_condition.py b/tests/test_migration_race_condition.py index 7365f18..765fd80 100644 --- a/tests/test_migration_race_condition.py +++ b/tests/test_migration_race_condition.py @@ -302,12 +302,13 @@ class TestErrorHandling: def test_rollback_failure_causes_system_exit(self, temp_db): """Test that rollback failure raises SystemExit""" - with patch('sqlite3.connect') as mock_connect: + with patch('sqlite3.connect') as mock_connect, \ + patch('starpunk.migrations.execute_in_transaction', + side_effect=Exception("Migration failed")): mock_conn = MagicMock() mock_connect.return_value = mock_conn # Make both migration and rollback fail - mock_conn.executescript.side_effect = Exception("Migration failed") mock_conn.rollback.side_effect = Exception("Rollback failed") mock_conn.execute.return_value.fetchone.return_value = (0,)