diff --git a/arborist/cli.py b/arborist/cli.py index 03df400..13a36b9 100644 --- a/arborist/cli.py +++ b/arborist/cli.py @@ -2995,10 +2995,29 @@ def _cmd_cold_unpack(args: argparse.Namespace) -> int: # directory; --hydrate-M picks the routing modulus. Each incoming # row lands on shard_for_document(root, M). if args.hydrate_shards_dir: + import sqlite3 target_dir = Path(args.hydrate_shards_dir).expanduser() target_dir.mkdir(parents=True, exist_ok=True) M = args.hydrate_M target_paths = [target_dir / f"{i:03d}.db" for i in range(M)] + + # #000067 bench-max: bump page_size before schema applies so the + # corpus uses 16 KB pages instead of SQLite's 4 KB default. + # Bigger pages → ~4× fewer page writes during chunk-fill → + # ~4× fewer ext4 metadata journal events (jbd2_log_wait_commit + # was the measured bottleneck on the 3090). page_size CAN ONLY + # be set on an empty DB; VACUUM finalises it. Idempotent on + # re-runs since fresh hydrate dirs are clean. + for p in target_paths: + if not p.exists() or p.stat().st_size == 0: + pre = sqlite3.connect(str(p)) + try: + pre.execute("PRAGMA page_size = 16384") + pre.execute("PRAGMA secure_delete = OFF") + pre.execute("VACUUM") # commit the page_size change + finally: + pre.close() + targets: list = [] for p in target_paths: t = connect(p)