Fix parallel test execution with isolated per-worker databases
The previous parallel test attempt failed because all workers were trying to use the same SQLite database file, causing locking errors. Solution: - Created conftest.py to configure pytest-xdist workers - Each worker gets a unique database file (test_make_post_sell_gw0.sqlite, etc.) - Enabled SQLite WAL (Write-Ahead Logging) mode for better concurrency - WAL allows multiple readers while a writer is active - Modified test.ini to use $TEST_DATABASE_PATH environment variable - Automatic cleanup of worker databases after test completion Benefits: - True isolation between test workers - No database locking conflicts - Tests can run fully in parallel - Expected test time reduction from 30min to ~5-10min Each worker's database is completely isolated, preventing the SQLite "database is locked" errors that plagued the previous attempt.
This commit is contained in:
parent
012014a20c
commit
af4b70a160
3 changed files with 80 additions and 3 deletions
2
Makefile
2
Makefile
|
|
@ -174,7 +174,7 @@ activate:
|
|||
|
||||
# Run the test suite.
|
||||
test: install-source-dev-and-test
|
||||
@echo "Running tests..."
|
||||
@echo "Running tests in parallel..."
|
||||
$(VENV_DIR)/bin/py.test -n auto
|
||||
|
||||
# Run tests with coverage for the full repository.
|
||||
|
|
|
|||
77
make_post_sell/tests/conftest.py
Normal file
77
make_post_sell/tests/conftest.py
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
"""
|
||||
Pytest configuration for parallel test execution.
|
||||
|
||||
This module configures pytest-xdist to use isolated databases per worker
|
||||
to avoid SQLite locking issues during parallel test runs.
|
||||
"""
|
||||
import os
|
||||
import pytest
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
"""
|
||||
Configure test database isolation for parallel execution.
|
||||
|
||||
Each pytest-xdist worker gets its own database file to prevent
|
||||
SQLite locking conflicts. WAL mode is enabled for better concurrency.
|
||||
"""
|
||||
# Get worker ID (e.g., "gw0", "gw1", etc.) for pytest-xdist
|
||||
worker_id = os.environ.get("PYTEST_XDIST_WORKER", "master")
|
||||
|
||||
# Set unique database path for this worker
|
||||
test_db_path = f"test_make_post_sell_{worker_id}.sqlite"
|
||||
os.environ["TEST_DATABASE_PATH"] = test_db_path
|
||||
|
||||
# Store for cleanup
|
||||
config.test_db_path = test_db_path
|
||||
|
||||
|
||||
def pytest_unconfigure(config):
|
||||
"""Clean up test database after all tests complete."""
|
||||
if hasattr(config, "test_db_path"):
|
||||
db_path = config.test_db_path
|
||||
if os.path.exists(db_path):
|
||||
try:
|
||||
os.remove(db_path)
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not remove test database {db_path}: {e}")
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def db_engine(request):
|
||||
"""
|
||||
Create a SQLAlchemy engine with WAL mode enabled for concurrency.
|
||||
|
||||
WAL (Write-Ahead Logging) mode allows multiple readers while a writer
|
||||
is active, improving parallel test performance.
|
||||
"""
|
||||
from sqlalchemy import create_engine, event
|
||||
|
||||
# Use worker-specific database
|
||||
worker_id = os.environ.get("PYTEST_XDIST_WORKER", "master")
|
||||
db_path = f"test_make_post_sell_{worker_id}.sqlite"
|
||||
db_url = f"sqlite:///{db_path}"
|
||||
|
||||
engine = create_engine(
|
||||
db_url,
|
||||
echo=False,
|
||||
# Important: Use NullPool to avoid connection sharing issues
|
||||
poolclass=__import__("sqlalchemy.pool", fromlist=["NullPool"]).NullPool,
|
||||
)
|
||||
|
||||
# Enable WAL mode for better concurrency
|
||||
@event.listens_for(engine, "connect")
|
||||
def set_sqlite_pragma(dbapi_conn, connection_record):
|
||||
cursor = dbapi_conn.cursor()
|
||||
# Enable WAL mode for concurrent access
|
||||
cursor.execute("PRAGMA journal_mode=WAL")
|
||||
# Increase cache size for better performance
|
||||
cursor.execute("PRAGMA cache_size=-64000") # 64MB
|
||||
# Enable foreign keys
|
||||
cursor.execute("PRAGMA foreign_keys=ON")
|
||||
cursor.close()
|
||||
|
||||
yield engine
|
||||
|
||||
# Cleanup
|
||||
engine.dispose()
|
||||
4
test.ini
4
test.ini
|
|
@ -7,7 +7,7 @@ pyramid.debug_notfound = false
|
|||
pyramid.debug_routematch = false
|
||||
pyramid.default_locale_name = en
|
||||
|
||||
sqlalchemy.url = sqlite:///%(here)s/test_make_post_sell.sqlite
|
||||
sqlalchemy.url = sqlite:///%(here)s/${TEST_DATABASE_PATH:-test_make_post_sell.sqlite}
|
||||
|
||||
retry.attempts = 3
|
||||
|
||||
|
|
@ -58,7 +58,7 @@ setup = make_post_sell.scripts.pshell.setup
|
|||
[alembic]
|
||||
# path to migration scripts
|
||||
script_location = make_post_sell:scripts/alembic
|
||||
sqlalchemy.url = sqlite:///%(here)s/test_make_post_sell.sqlite
|
||||
sqlalchemy.url = sqlite:///%(here)s/${TEST_DATABASE_PATH:-test_make_post_sell.sqlite}
|
||||
|
||||
[server:main]
|
||||
use = egg:waitress#main
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue