Merge branch 'fix/v2.8.1-migration-race' for v2.8.1 release
Build Container / build (push) Successful in 7m34s

This commit is contained in:
Phil committed 2026-11-03 09:58:20 -07:00
commit 57682165a4
6 files changed
+172 -10

No files matched your search

+6
View File
@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased] ## [Unreleased]
## [2.8.1] - 2026-11-03
### Fixed
- Starting the server after an upgrade with a database migration could fail: several server processes applied the same migration at once and one stopped with "duplicate column name", taking the server down until it restarted (this happened deploying v2.8.0, for about two seconds). The migration lock was released right after it was taken, and again before each migration was recorded. Migrations now run entirely inside the lock
## [2.8.0] - 2026-11-03 ## [2.8.0] - 2026-11-03
Microformats2 review: the markup now matches the [microformats2](https://microformats.org/wiki/microformats2) vocabularies, so IndieWeb readers and tools see what the page shows. Microformats2 review: the markup now matches the [microformats2](https://microformats.org/wiki/microformats2) vocabularies, so IndieWeb readers and tools see what the page shows.
@@ -0,0 +1,33 @@
# v2.8.1: Migration lock held for the whole run
## Incident
Deploying v2.8.0 (migration 014 adds two columns): one gunicorn worker
applied 014, another failed with `duplicate column name: source_post_url`,
gunicorn shut down ("Worker failed to boot") and Docker restarted the
container. About two seconds of downtime; schema correct afterwards.
## Causes
`run_migrations` takes `BEGIN IMMEDIATE` so one worker migrates at a time,
but the lock was released early twice:
1. `create_migrations_table()`, called right after `BEGIN IMMEDIATE`,
ended with `conn.commit()`. Everything after it ran without the lock.
2. `conn.executescript()` COMMITs any open transaction before running the
script, so the lock was dropped before the migration was recorded in
`schema_migrations`.
## Fix
- `create_migrations_table(conn, commit=True)`; `run_migrations` passes
`commit=False`.
- `execute_in_transaction()` runs each statement from
`split_sql_statements()` (built on `sqlite3.complete_statement`, so
trigger bodies stay whole) with `conn.execute`, inside the transaction.
## Verification
Four processes migrating a database at 013 simultaneously: 6/30 trials
failed before, 0/30 after. `tests/test_migration_concurrency.py` runs this
five times, plus a rollback test showing a migration commits nothing early.
+2 -2
View File
@@ -463,5 +463,5 @@ def create_app(config=None):
# Package version (Semantic Versioning 2.0.0) # Package version (Semantic Versioning 2.0.0)
# See docs/standards/versioning-strategy.md for details # See docs/standards/versioning-strategy.md for details
__version__ = "2.8.0" __version__ = "2.8.1"
__version_info__ = (2, 8, 0) __version_info__ = (2, 8, 1)
+42 -6
View File
@@ -31,12 +31,47 @@ class MigrationError(Exception):
pass pass
def create_migrations_table(conn): def split_sql_statements(script: str) -> list[str]:
"""
Split a SQL script into complete statements (v2.8.1)
Uses sqlite3.complete_statement, so semicolons inside strings and
trigger bodies (BEGIN ... END) don't split a statement.
"""
statements = []
current = ""
for line in script.splitlines(keepends=True):
current += line
if sqlite3.complete_statement(current):
if current.strip().strip(";").strip():
statements.append(current.strip())
current = ""
if current.strip():
statements.append(current.strip())
return statements
def execute_in_transaction(conn, script: str) -> None:
"""
Run a SQL script inside the connection's open transaction
Not conn.executescript(): that COMMITs first, which released the
migration lock mid-migration and let another gunicorn worker apply the
same migration again ("duplicate column name", v2.8.0 deploy).
"""
for statement in split_sql_statements(script):
conn.execute(statement)
def create_migrations_table(conn, commit=True):
""" """
Create schema_migrations tracking table if it doesn't exist Create schema_migrations tracking table if it doesn't exist
Args: Args:
conn: SQLite connection conn: SQLite connection
commit: Commit afterwards. run_migrations passes False: committing
there released its BEGIN IMMEDIATE lock straight away, so
gunicorn workers applied migrations concurrently (v2.8.1)
""" """
conn.execute(""" conn.execute("""
CREATE TABLE IF NOT EXISTS schema_migrations ( CREATE TABLE IF NOT EXISTS schema_migrations (
@@ -51,7 +86,8 @@ def create_migrations_table(conn):
ON schema_migrations(migration_name) ON schema_migrations(migration_name)
""") """)
conn.commit() if commit:
conn.commit()
def is_schema_current(conn): def is_schema_current(conn):
@@ -287,7 +323,7 @@ def apply_migration(conn, migration_name, migration_path, logger=None):
# Execute migration in transaction # Execute migration in transaction
conn.execute("BEGIN") conn.execute("BEGIN")
conn.executescript(migration_sql) execute_in_transaction(conn, migration_sql)
# Record migration as applied # Record migration as applied
conn.execute( conn.execute(
@@ -374,7 +410,7 @@ def run_migrations(db_path, logger=None):
try: try:
# Ensure migrations tracking table exists # Ensure migrations tracking table exists
create_migrations_table(conn) create_migrations_table(conn, commit=False)
# Quick check: have migrations already been applied by another worker? # Quick check: have migrations already been applied by another worker?
cursor = conn.execute("SELECT COUNT(*) FROM schema_migrations") cursor = conn.execute("SELECT COUNT(*) FROM schema_migrations")
@@ -472,8 +508,8 @@ def run_migrations(db_path, logger=None):
logger.debug(f"Applying migration: {migration_name}") logger.debug(f"Applying migration: {migration_name}")
# Execute migration (already in transaction) # Execute migration inside the locked transaction
conn.executescript(migration_sql) execute_in_transaction(conn, migration_sql)
# Record migration as applied # Record migration as applied
conn.execute( conn.execute(
+86
View File
@@ -0,0 +1,86 @@
"""
Migrations under concurrent gunicorn workers (v2.8.1)
The v2.8.0 deploy failed to boot: one worker applied migration 014 while
another applied it again ("duplicate column name"), because
conn.executescript() committed and released the migration lock early.
"""
import multiprocessing
import sqlite3
from pathlib import Path
from starpunk.database.schema import INITIAL_SCHEMA_SQL
from starpunk.migrations import execute_in_transaction, run_migrations, split_sql_statements
MIGRATIONS = Path(__file__).parent.parent / "migrations"
def test_script_runs_inside_the_transaction(tmp_path):
conn = sqlite3.connect(tmp_path / "t.db")
conn.execute("CREATE TABLE t (a INTEGER)")
conn.commit()
conn.execute("BEGIN IMMEDIATE")
execute_in_transaction(conn, "ALTER TABLE t ADD COLUMN b TEXT;\nALTER TABLE t ADD COLUMN c TEXT;")
conn.rollback()
assert [row[1] for row in conn.execute("PRAGMA table_info(t)")] == ["a"]
def test_split_keeps_trigger_bodies_together():
script = """
-- comment; with a semicolon
CREATE TABLE t (a TEXT DEFAULT 'x;y');
CREATE TRIGGER tr AFTER DELETE ON t BEGIN
DELETE FROM t WHERE a = old.a;
SELECT 1;
END;
"""
statements = split_sql_statements(script)
assert len(statements) == 2
assert statements[1].startswith("CREATE TRIGGER") and statements[1].endswith("END;")
def _migrate(db_path, barrier, errors):
try:
barrier.wait()
run_migrations(db_path)
except Exception as e: # report to the parent
errors.put(repr(e))
import pytest
@pytest.mark.parametrize("trial", range(5)) # the race hit ~1 in 5 runs
def test_concurrent_workers_apply_each_migration_once(tmp_path, trial):
# A database at migration 013, like production before v2.8.0
db_path = tmp_path / "starpunk.db"
conn = sqlite3.connect(db_path)
conn.executescript(INITIAL_SCHEMA_SQL) # as init_db does
conn.close()
run_migrations(db_path)
conn = sqlite3.connect(db_path)
conn.execute("DELETE FROM schema_migrations WHERE migration_name = '014_webmention_source_details.sql'")
conn.commit()
columns = [row[1] for row in conn.execute("PRAGMA table_info(webmentions)")]
if "source_post_url" in columns:
conn.execute("ALTER TABLE webmentions DROP COLUMN source_post_url")
conn.execute("ALTER TABLE webmentions DROP COLUMN source_published")
conn.commit()
conn.close()
ctx = multiprocessing.get_context("fork")
barrier, errors = ctx.Barrier(4), ctx.Queue()
workers = [ctx.Process(target=_migrate, args=(db_path, barrier, errors)) for _ in range(4)]
for w in workers:
w.start()
for w in workers:
w.join(timeout=60)
assert errors.empty(), errors.get()
conn = sqlite3.connect(db_path)
assert conn.execute(
"SELECT COUNT(*) FROM schema_migrations WHERE migration_name = '014_webmention_source_details.sql'"
).fetchone()[0] == 1
+3 -2
View File
@@ -302,12 +302,13 @@ class TestErrorHandling:
def test_rollback_failure_causes_system_exit(self, temp_db): def test_rollback_failure_causes_system_exit(self, temp_db):
"""Test that rollback failure raises SystemExit""" """Test that rollback failure raises SystemExit"""
with patch('sqlite3.connect') as mock_connect: with patch('sqlite3.connect') as mock_connect, \
patch('starpunk.migrations.execute_in_transaction',
side_effect=Exception("Migration failed")):
mock_conn = MagicMock() mock_conn = MagicMock()
mock_connect.return_value = mock_conn mock_connect.return_value = mock_conn
# Make both migration and rollback fail # Make both migration and rollback fail
mock_conn.executescript.side_effect = Exception("Migration failed")
mock_conn.rollback.side_effect = Exception("Rollback failed") mock_conn.rollback.side_effect = Exception("Rollback failed")
mock_conn.execute.return_value.fetchone.return_value = (0,) mock_conn.execute.return_value.fetchone.return_value = (0,)