fix: Hold the migration lock for the whole run (v2.8.1)
create_migrations_table() committed right after BEGIN IMMEDIATE, and executescript() commits before running, so gunicorn workers applied migrations concurrently; the v2.8.0 deploy failed to boot once with 'duplicate column name'. Run migration statements with conn.execute inside the transaction and don't commit when creating the tracking table. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
42c77e34b7
commit
8c51b52ed6
6 files changed
+172
-10
No files matched your search
@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [2.8.1] - 2026-11-03
|
||||
|
||||
### Fixed
|
||||
|
||||
- Starting the server after an upgrade with a database migration could fail: several server processes applied the same migration at once and one stopped with "duplicate column name", taking the server down until it restarted (this happened deploying v2.8.0, for about two seconds). The migration lock was released right after it was taken, and again before each migration was recorded. Migrations now run entirely inside the lock
|
||||
|
||||
## [2.8.0] - 2026-11-03
|
||||
|
||||
Microformats2 review: the markup now matches the [microformats2](https://microformats.org/wiki/microformats2) vocabularies, so IndieWeb readers and tools see what the page shows.
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
# v2.8.1: Migration lock held for the whole run
|
||||
|
||||
## Incident
|
||||
|
||||
Deploying v2.8.0 (migration 014 adds two columns): one gunicorn worker
|
||||
applied 014, another failed with `duplicate column name: source_post_url`,
|
||||
gunicorn shut down ("Worker failed to boot") and Docker restarted the
|
||||
container. About two seconds of downtime; schema correct afterwards.
|
||||
|
||||
## Causes
|
||||
|
||||
`run_migrations` takes `BEGIN IMMEDIATE` so one worker migrates at a time,
|
||||
but the lock was released early twice:
|
||||
|
||||
1. `create_migrations_table()`, called right after `BEGIN IMMEDIATE`,
|
||||
ended with `conn.commit()`. Everything after it ran without the lock.
|
||||
2. `conn.executescript()` COMMITs any open transaction before running the
|
||||
script, so the lock was dropped before the migration was recorded in
|
||||
`schema_migrations`.
|
||||
|
||||
## Fix
|
||||
|
||||
- `create_migrations_table(conn, commit=True)`; `run_migrations` passes
|
||||
`commit=False`.
|
||||
- `execute_in_transaction()` runs each statement from
|
||||
`split_sql_statements()` (built on `sqlite3.complete_statement`, so
|
||||
trigger bodies stay whole) with `conn.execute`, inside the transaction.
|
||||
|
||||
## Verification
|
||||
|
||||
Four processes migrating a database at 013 simultaneously: 6/30 trials
|
||||
failed before, 0/30 after. `tests/test_migration_concurrency.py` runs this
|
||||
five times, plus a rollback test showing a migration commits nothing early.
|
||||
@@ -463,5 +463,5 @@ def create_app(config=None):
|
||||
|
||||
# Package version (Semantic Versioning 2.0.0)
|
||||
# See docs/standards/versioning-strategy.md for details
|
||||
__version__ = "2.8.0"
|
||||
__version_info__ = (2, 8, 0)
|
||||
__version__ = "2.8.1"
|
||||
__version_info__ = (2, 8, 1)
|
||||
+42
-6
@@ -31,12 +31,47 @@ class MigrationError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def create_migrations_table(conn):
|
||||
def split_sql_statements(script: str) -> list[str]:
|
||||
"""
|
||||
Split a SQL script into complete statements (v2.8.1)
|
||||
|
||||
Uses sqlite3.complete_statement, so semicolons inside strings and
|
||||
trigger bodies (BEGIN ... END) don't split a statement.
|
||||
"""
|
||||
statements = []
|
||||
current = ""
|
||||
for line in script.splitlines(keepends=True):
|
||||
current += line
|
||||
if sqlite3.complete_statement(current):
|
||||
if current.strip().strip(";").strip():
|
||||
statements.append(current.strip())
|
||||
current = ""
|
||||
if current.strip():
|
||||
statements.append(current.strip())
|
||||
return statements
|
||||
|
||||
|
||||
def execute_in_transaction(conn, script: str) -> None:
|
||||
"""
|
||||
Run a SQL script inside the connection's open transaction
|
||||
|
||||
Not conn.executescript(): that COMMITs first, which released the
|
||||
migration lock mid-migration and let another gunicorn worker apply the
|
||||
same migration again ("duplicate column name", v2.8.0 deploy).
|
||||
"""
|
||||
for statement in split_sql_statements(script):
|
||||
conn.execute(statement)
|
||||
|
||||
|
||||
def create_migrations_table(conn, commit=True):
|
||||
"""
|
||||
Create schema_migrations tracking table if it doesn't exist
|
||||
|
||||
Args:
|
||||
conn: SQLite connection
|
||||
commit: Commit afterwards. run_migrations passes False: committing
|
||||
there released its BEGIN IMMEDIATE lock straight away, so
|
||||
gunicorn workers applied migrations concurrently (v2.8.1)
|
||||
"""
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
@@ -51,7 +86,8 @@ def create_migrations_table(conn):
|
||||
ON schema_migrations(migration_name)
|
||||
""")
|
||||
|
||||
conn.commit()
|
||||
if commit:
|
||||
conn.commit()
|
||||
|
||||
|
||||
def is_schema_current(conn):
|
||||
@@ -287,7 +323,7 @@ def apply_migration(conn, migration_name, migration_path, logger=None):
|
||||
|
||||
# Execute migration in transaction
|
||||
conn.execute("BEGIN")
|
||||
conn.executescript(migration_sql)
|
||||
execute_in_transaction(conn, migration_sql)
|
||||
|
||||
# Record migration as applied
|
||||
conn.execute(
|
||||
@@ -374,7 +410,7 @@ def run_migrations(db_path, logger=None):
|
||||
|
||||
try:
|
||||
# Ensure migrations tracking table exists
|
||||
create_migrations_table(conn)
|
||||
create_migrations_table(conn, commit=False)
|
||||
|
||||
# Quick check: have migrations already been applied by another worker?
|
||||
cursor = conn.execute("SELECT COUNT(*) FROM schema_migrations")
|
||||
@@ -472,8 +508,8 @@ def run_migrations(db_path, logger=None):
|
||||
|
||||
logger.debug(f"Applying migration: {migration_name}")
|
||||
|
||||
# Execute migration (already in transaction)
|
||||
conn.executescript(migration_sql)
|
||||
# Execute migration inside the locked transaction
|
||||
execute_in_transaction(conn, migration_sql)
|
||||
|
||||
# Record migration as applied
|
||||
conn.execute(
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
"""
|
||||
Migrations under concurrent gunicorn workers (v2.8.1)
|
||||
|
||||
The v2.8.0 deploy failed to boot: one worker applied migration 014 while
|
||||
another applied it again ("duplicate column name"), because
|
||||
conn.executescript() committed and released the migration lock early.
|
||||
"""
|
||||
|
||||
import multiprocessing
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
from starpunk.database.schema import INITIAL_SCHEMA_SQL
|
||||
from starpunk.migrations import execute_in_transaction, run_migrations, split_sql_statements
|
||||
|
||||
MIGRATIONS = Path(__file__).parent.parent / "migrations"
|
||||
|
||||
|
||||
def test_script_runs_inside_the_transaction(tmp_path):
|
||||
conn = sqlite3.connect(tmp_path / "t.db")
|
||||
conn.execute("CREATE TABLE t (a INTEGER)")
|
||||
conn.commit()
|
||||
|
||||
conn.execute("BEGIN IMMEDIATE")
|
||||
execute_in_transaction(conn, "ALTER TABLE t ADD COLUMN b TEXT;\nALTER TABLE t ADD COLUMN c TEXT;")
|
||||
conn.rollback()
|
||||
|
||||
assert [row[1] for row in conn.execute("PRAGMA table_info(t)")] == ["a"]
|
||||
|
||||
|
||||
def test_split_keeps_trigger_bodies_together():
|
||||
script = """
|
||||
-- comment; with a semicolon
|
||||
CREATE TABLE t (a TEXT DEFAULT 'x;y');
|
||||
CREATE TRIGGER tr AFTER DELETE ON t BEGIN
|
||||
DELETE FROM t WHERE a = old.a;
|
||||
SELECT 1;
|
||||
END;
|
||||
"""
|
||||
statements = split_sql_statements(script)
|
||||
assert len(statements) == 2
|
||||
assert statements[1].startswith("CREATE TRIGGER") and statements[1].endswith("END;")
|
||||
|
||||
|
||||
def _migrate(db_path, barrier, errors):
|
||||
try:
|
||||
barrier.wait()
|
||||
run_migrations(db_path)
|
||||
except Exception as e: # report to the parent
|
||||
errors.put(repr(e))
|
||||
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.parametrize("trial", range(5)) # the race hit ~1 in 5 runs
|
||||
def test_concurrent_workers_apply_each_migration_once(tmp_path, trial):
|
||||
# A database at migration 013, like production before v2.8.0
|
||||
db_path = tmp_path / "starpunk.db"
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.executescript(INITIAL_SCHEMA_SQL) # as init_db does
|
||||
conn.close()
|
||||
run_migrations(db_path)
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.execute("DELETE FROM schema_migrations WHERE migration_name = '014_webmention_source_details.sql'")
|
||||
conn.commit()
|
||||
columns = [row[1] for row in conn.execute("PRAGMA table_info(webmentions)")]
|
||||
if "source_post_url" in columns:
|
||||
conn.execute("ALTER TABLE webmentions DROP COLUMN source_post_url")
|
||||
conn.execute("ALTER TABLE webmentions DROP COLUMN source_published")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
ctx = multiprocessing.get_context("fork")
|
||||
barrier, errors = ctx.Barrier(4), ctx.Queue()
|
||||
workers = [ctx.Process(target=_migrate, args=(db_path, barrier, errors)) for _ in range(4)]
|
||||
for w in workers:
|
||||
w.start()
|
||||
for w in workers:
|
||||
w.join(timeout=60)
|
||||
|
||||
assert errors.empty(), errors.get()
|
||||
conn = sqlite3.connect(db_path)
|
||||
assert conn.execute(
|
||||
"SELECT COUNT(*) FROM schema_migrations WHERE migration_name = '014_webmention_source_details.sql'"
|
||||
).fetchone()[0] == 1
|
||||
@@ -302,12 +302,13 @@ class TestErrorHandling:
|
||||
|
||||
def test_rollback_failure_causes_system_exit(self, temp_db):
|
||||
"""Test that rollback failure raises SystemExit"""
|
||||
with patch('sqlite3.connect') as mock_connect:
|
||||
with patch('sqlite3.connect') as mock_connect, \
|
||||
patch('starpunk.migrations.execute_in_transaction',
|
||||
side_effect=Exception("Migration failed")):
|
||||
mock_conn = MagicMock()
|
||||
mock_connect.return_value = mock_conn
|
||||
|
||||
# Make both migration and rollback fail
|
||||
mock_conn.executescript.side_effect = Exception("Migration failed")
|
||||
mock_conn.rollback.side_effect = Exception("Rollback failed")
|
||||
mock_conn.execute.return_value.fetchone.return_value = (0,)
|
||||
|
||||
|
||||
Reference in new issue
Block a user