Fix parallel test execution with isolated per-worker databases

The previous parallel test attempt failed because all workers were trying
to use the same SQLite database file, causing locking errors.

Solution:
- Created conftest.py to configure pytest-xdist workers
- Each worker gets a unique database file (test_make_post_sell_gw0.sqlite, etc.)
- Enabled SQLite WAL (Write-Ahead Logging) mode for better concurrency
- WAL allows multiple readers while a writer is active
- Modified test.ini to use $TEST_DATABASE_PATH environment variable
- Automatic cleanup of worker databases after test completion

Benefits:
- True isolation between test workers
- No database locking conflicts
- Tests can run fully in parallel
- Expected test time reduction from 30min to ~5-10min

Each worker's database is completely isolated, preventing the SQLite
"database is locked" errors that plagued the previous attempt.
This commit is contained in:
russell@unturf.com 2026-01-21 12:07:16 -05:00
parent 012014a20c
commit af4b70a160
3 changed files with 80 additions and 3 deletions

View file

@ -174,7 +174,7 @@ activate:
# Run the test suite.
test: install-source-dev-and-test
@echo "Running tests..."
@echo "Running tests in parallel..."
$(VENV_DIR)/bin/py.test -n auto
# Run tests with coverage for the full repository.

View file

@ -0,0 +1,77 @@
"""
Pytest configuration for parallel test execution.
This module configures pytest-xdist to use isolated databases per worker
to avoid SQLite locking issues during parallel test runs.
"""
import os
import pytest
def pytest_configure(config):
"""
Configure test database isolation for parallel execution.
Each pytest-xdist worker gets its own database file to prevent
SQLite locking conflicts. WAL mode is enabled for better concurrency.
"""
# Get worker ID (e.g., "gw0", "gw1", etc.) for pytest-xdist
worker_id = os.environ.get("PYTEST_XDIST_WORKER", "master")
# Set unique database path for this worker
test_db_path = f"test_make_post_sell_{worker_id}.sqlite"
os.environ["TEST_DATABASE_PATH"] = test_db_path
# Store for cleanup
config.test_db_path = test_db_path
def pytest_unconfigure(config):
"""Clean up test database after all tests complete."""
if hasattr(config, "test_db_path"):
db_path = config.test_db_path
if os.path.exists(db_path):
try:
os.remove(db_path)
except Exception as e:
print(f"Warning: Could not remove test database {db_path}: {e}")
@pytest.fixture(scope="session")
def db_engine(request):
"""
Create a SQLAlchemy engine with WAL mode enabled for concurrency.
WAL (Write-Ahead Logging) mode allows multiple readers while a writer
is active, improving parallel test performance.
"""
from sqlalchemy import create_engine, event
# Use worker-specific database
worker_id = os.environ.get("PYTEST_XDIST_WORKER", "master")
db_path = f"test_make_post_sell_{worker_id}.sqlite"
db_url = f"sqlite:///{db_path}"
engine = create_engine(
db_url,
echo=False,
# Important: Use NullPool to avoid connection sharing issues
poolclass=__import__("sqlalchemy.pool", fromlist=["NullPool"]).NullPool,
)
# Enable WAL mode for better concurrency
@event.listens_for(engine, "connect")
def set_sqlite_pragma(dbapi_conn, connection_record):
cursor = dbapi_conn.cursor()
# Enable WAL mode for concurrent access
cursor.execute("PRAGMA journal_mode=WAL")
# Increase cache size for better performance
cursor.execute("PRAGMA cache_size=-64000") # 64MB
# Enable foreign keys
cursor.execute("PRAGMA foreign_keys=ON")
cursor.close()
yield engine
# Cleanup
engine.dispose()

View file

@ -7,7 +7,7 @@ pyramid.debug_notfound = false
pyramid.debug_routematch = false
pyramid.default_locale_name = en
sqlalchemy.url = sqlite:///%(here)s/test_make_post_sell.sqlite
sqlalchemy.url = sqlite:///%(here)s/${TEST_DATABASE_PATH:-test_make_post_sell.sqlite}
retry.attempts = 3
@ -58,7 +58,7 @@ setup = make_post_sell.scripts.pshell.setup
[alembic]
# path to migration scripts
script_location = make_post_sell:scripts/alembic
sqlalchemy.url = sqlite:///%(here)s/test_make_post_sell.sqlite
sqlalchemy.url = sqlite:///%(here)s/${TEST_DATABASE_PATH:-test_make_post_sell.sqlite}
[server:main]
use = egg:waitress#main