From 05541990706543f75591d495d7d1829b9e9ffb1a Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Tue, 26 May 2026 19:24:07 -0400 Subject: [PATCH] #000067 bench-max #5: 16 KB page_size on hydrate target DBs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 3090 genesis bench at 28-min progress mark showed all 4 workers sitting in jbd2_log_wait_commit / do_get_write_access — ext4 filesystem journal serializing every page-extension event across the 4 parallel writers. SQLite synchronous=OFF and journal_mode= MEMORY skip SQLite's own fsync/WAL costs, but file growth still goes through ext4's journal layer. Fix: bump SQLite page_size from default 4 KB to 16 KB on each hydrate target. Bigger pages mean ~4× fewer page writes during chunk-fill, which means ~4× fewer file-extension events through ext4's journal. Direct attack on the measured bottleneck. page_size can only be set on an EMPTY database; VACUUM finalises the change. The unpack CLI now opens each target file raw, PRAGMA page_size = 16384 + PRAGMA secure_delete = OFF + VACUUM, closes, then connect() applies schema as normal on the pre-sized empty DB. Idempotent on re-runs since hydrate dirs are always clean. Also sets secure_delete = OFF (default is OFF in most builds but explicit for hydrate writers): no overwrite-with-zeros on row deletion, less write traffic during the chunk UPDATEs that might free overflow pages. Expected impact: ~30-50% reduction in chunk-fill wall time vs the killed 30-min v2 run. Combined with the prior bench-max round (journal_mode=MEMORY, deferred FTS, batched UPDATEs) and the upcoming fts-pack-skip-rebuild path, full genesis should land under 30 min total instead of the 75-min v2 extrapolation. 4 cold-unpack-routed tests pass unchanged. --- arborist/cli.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/arborist/cli.py b/arborist/cli.py index 03df400..13a36b9 100644 --- a/arborist/cli.py +++ b/arborist/cli.py @@ -2995,10 +2995,29 @@ def _cmd_cold_unpack(args: argparse.Namespace) -> int: # directory; --hydrate-M picks the routing modulus. Each incoming # row lands on shard_for_document(root, M). if args.hydrate_shards_dir: + import sqlite3 target_dir = Path(args.hydrate_shards_dir).expanduser() target_dir.mkdir(parents=True, exist_ok=True) M = args.hydrate_M target_paths = [target_dir / f"{i:03d}.db" for i in range(M)] + + # #000067 bench-max: bump page_size before schema applies so the + # corpus uses 16 KB pages instead of SQLite's 4 KB default. + # Bigger pages → ~4× fewer page writes during chunk-fill → + # ~4× fewer ext4 metadata journal events (jbd2_log_wait_commit + # was the measured bottleneck on the 3090). page_size CAN ONLY + # be set on an empty DB; VACUUM finalises it. Idempotent on + # re-runs since fresh hydrate dirs are clean. + for p in target_paths: + if not p.exists() or p.stat().st_size == 0: + pre = sqlite3.connect(str(p)) + try: + pre.execute("PRAGMA page_size = 16384") + pre.execute("PRAGMA secure_delete = OFF") + pre.execute("VACUUM") # commit the page_size change + finally: + pre.close() + targets: list = [] for p in target_paths: t = connect(p)