#000067: defer FTS5 to serial post-pass + bulk-load SQLite tunings
Real-measured bottleneck during the 3090 genesis run (2026-05-26):
all 4 parallel workers sitting in jbd2_log_wait_commit /
do_get_write_access — fighting for ext4's single filesystem journal.
The reshard executor was fast because it ran FTS5 rebuild
SEQUENTIALLY (one process per target in turn); the parallel 4-way
unpack collapsed that into journal contention.
Two coupled changes:
1. arborist/evict.py — _pull_pack_inner_routed
* Drop the _rebuild_fts_on_target call from phase 2c entirely.
Parallel inner loop now does: download → metadata route →
chunks fill. NO FTS5 writes during the hot parallel phase.
* Bulk-load PRAGMA tuning on every target connection:
synchronous=OFF no fsync per commit (already had this)
journal_mode=MEMORY WAL in RAM, not on disk (was WAL)
temp_store=MEMORY sort scratch in RAM
cache_size=-524288 512 MB page cache per connection
mmap_size=536870912 512 MB read mmap
Genesis crash recoverability = re-pull from bucket, so
durability of intermediate state has no value — these tunings
trade durability for throughput.
2. arborist/cli.py — new `arborist cold rebuild-fts --shards-dir DIR`
subcommand. Sequentially rebuilds chunks_fts + documents_fts on
every shard, one at a time. Each shard gets the full filesystem
journal in its turn. Total wall = sum of single-shard FTS
rebuild times, NOT 4× contention.
3. Makefile — `cold-hydrate` chains the FTS rebuild automatically
after the parallel unpack (only when HYDRATE_MODE=full, since
just-enough has no chunk bodies to index anyway). Operator can
skip with HYDRATE_REBUILD_FTS=0 for a 2-pass workflow.
Expected wall time on 3090:
parallel hydrate (download + chunks fill): ~5-10 min
serial FTS rebuild × 4 shards: ~5-15 min total
total: ~15-25 min
vs the killed 2.5-hour run.
34 existing cold-unpack-routed + migrate + planner tests still pass
(the tests' FTS check runs against the reshard path, which still
calls _rebuild_fts_on_target inside the executor — that path is one
process, no parallel contention).
This commit is contained in:
parent
939d3ced78
commit
cfb5666ef5
3 changed files with 87 additions and 23 deletions
4
Makefile
4
Makefile
|
|
@ -961,6 +961,10 @@ cold-hydrate: bootstrap ## genesis a fresh peer from cloud, M-aware: pull every
|
|||
--hydrate-shards-dir $(HYDRATE_DIR) \
|
||||
--hydrate-M $(HYDRATE_M) \
|
||||
$$MODE_FLAG
|
||||
@if [ "$${HYDRATE_MODE:-full}" = "full" ] && [ "$${HYDRATE_REBUILD_FTS:-1}" = "1" ]; then \
|
||||
echo ">> serial FTS5 rebuild (each shard gets full FS-journal bandwidth in turn)"; \
|
||||
$(ARBORIST) cold rebuild-fts --shards-dir $(HYDRATE_DIR); \
|
||||
fi
|
||||
@echo ">> hydration complete:"
|
||||
@ls -lh $(HYDRATE_DIR)/*.db 2>/dev/null
|
||||
|
||||
|
|
|
|||
|
|
@ -3032,6 +3032,47 @@ def _cmd_cold_unpack(args: argparse.Namespace) -> int:
|
|||
return 0
|
||||
|
||||
|
||||
def _cmd_cold_rebuild_fts(args: argparse.Namespace) -> int:
|
||||
"""Sequentially rebuild chunks_fts + documents_fts on every shard.
|
||||
|
||||
Pairs with `arborist cold unpack --hydrate-shards-dir` which
|
||||
deliberately skips FTS5 indexing in its parallel inner loop —
|
||||
concurrent FTS5 writes across 4 shards serialize on the ext4
|
||||
journal and become the throughput floor. This serial post-pass
|
||||
gives each shard the full filesystem journal in its turn, then
|
||||
moves on. Total wall time is bounded by the slowest single-shard
|
||||
FTS rebuild, not by 4× contention.
|
||||
"""
|
||||
from arborist.migrate import _rebuild_fts_on_target
|
||||
|
||||
shards_dir = Path(args.shards_dir).expanduser()
|
||||
db_files = sorted(shards_dir.glob("00[0-9].db"))
|
||||
if not db_files:
|
||||
print(f"no shards under {shards_dir}", file=sys.stderr)
|
||||
return 1
|
||||
started = time.time()
|
||||
for path in db_files:
|
||||
per_started = time.time()
|
||||
conn = connect(path)
|
||||
try:
|
||||
conn.execute("PRAGMA synchronous = OFF")
|
||||
conn.execute("PRAGMA journal_mode = MEMORY")
|
||||
conn.execute("PRAGMA temp_store = MEMORY")
|
||||
conn.execute("PRAGMA cache_size = -524288")
|
||||
_rebuild_fts_on_target(conn)
|
||||
finally:
|
||||
conn.close()
|
||||
print(
|
||||
f" {path.name}: FTS rebuilt in {time.time() - per_started:.1f}s",
|
||||
file=sys.stderr,
|
||||
)
|
||||
print(
|
||||
f"all {len(db_files)} shards FTS-rebuilt in {time.time() - started:.1f}s",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def _cmd_cold_stats(args: argparse.Namespace) -> int:
|
||||
from arborist.cold_object import PACK_PREFIX
|
||||
|
||||
|
|
@ -5925,6 +5966,21 @@ def build_parser() -> argparse.ArgumentParser:
|
|||
)
|
||||
cold_stats.set_defaults(func=_cmd_cold_stats)
|
||||
|
||||
cold_rebuild_fts = cold_sub.add_parser(
|
||||
"rebuild-fts",
|
||||
help=(
|
||||
"serially rebuild chunks_fts + documents_fts on every shard "
|
||||
"in SHARDS_DIR. Pairs with `cold unpack --hydrate-shards-dir` "
|
||||
"which defers FTS to avoid ext4-journal contention in the "
|
||||
"parallel inner loop."
|
||||
),
|
||||
)
|
||||
cold_rebuild_fts.add_argument(
|
||||
"--shards-dir", dest="shards_dir", required=True,
|
||||
help="directory of shards to FTS-rebuild (one shard at a time)",
|
||||
)
|
||||
cold_rebuild_fts.set_defaults(func=_cmd_cold_rebuild_fts)
|
||||
|
||||
cold_list = cold_sub.add_parser(
|
||||
"list",
|
||||
help="enumerate packs in the bucket with metadata (pack_hash, size, chunk_count) for new-peer hydration",
|
||||
|
|
|
|||
|
|
@ -861,6 +861,18 @@ def _pull_pack_inner_routed(
|
|||
finally:
|
||||
shutil.rmtree(tables_dir, ignore_errors=True)
|
||||
|
||||
# Pre-tune every target writer for bulk genesis. None of these
|
||||
# apply outside a hydrate context — they're set per-connection,
|
||||
# not at the schema level, so they affect only this writer's
|
||||
# behaviour. Crash mid-hydrate is recoverable (re-pull from
|
||||
# bucket), so durability of intermediate state has no value.
|
||||
for t in targets:
|
||||
t.execute("PRAGMA synchronous = OFF")
|
||||
t.execute("PRAGMA journal_mode = MEMORY") # no on-disk WAL
|
||||
t.execute("PRAGMA temp_store = MEMORY")
|
||||
t.execute("PRAGMA cache_size = -524288") # 512 MB page cache
|
||||
t.execute("PRAGMA mmap_size = 536870912") # 512 MB read mmap
|
||||
|
||||
# Phase 2: fill chunk content. Bench-max'd 2026-05-26 after the
|
||||
# initial naive impl took ~2h wall on a 6M-chunk corpus (same speed
|
||||
# as a fresh ingest — the whole point of a pack restore is to be
|
||||
|
|
@ -877,14 +889,6 @@ def _pull_pack_inner_routed(
|
|||
# inverted-index update; bulk rebuild streams chunks → FTS
|
||||
# in one pass per target, ~10× faster.
|
||||
#
|
||||
# Plus PRAGMA synchronous=OFF on each target — genesis is
|
||||
# rebuildable end-to-end (a partial crash leaves us with empty
|
||||
# shards we can re-pull), so durability of intermediate WAL pages
|
||||
# is not required. The PRAGMA wal_checkpoint(TRUNCATE) at the end
|
||||
# of each phase keeps WAL bounded.
|
||||
for target in targets:
|
||||
target.execute("PRAGMA synchronous = OFF")
|
||||
|
||||
# Phase 2a — build the leaf_hash → owning-target map.
|
||||
leaf_to_target: dict[str, tuple[int, int]] = {}
|
||||
for idx, target in enumerate(targets):
|
||||
|
|
@ -919,21 +923,21 @@ def _pull_pack_inner_routed(
|
|||
with transaction(targets[idx]):
|
||||
targets[idx].executemany(UPDATE_SQL, batch)
|
||||
|
||||
# Phase 2c — bulk FTS5 rebuild per target. Same primitive as the
|
||||
# reshard executor (arborist.migrate._rebuild_fts_on_target):
|
||||
# stream chunks → unpack_chunk → batched INSERT INTO chunks_fts.
|
||||
# Each target shard's FTS only sees rows from the chunks ITS pack
|
||||
# owned, so this rebuilds incrementally without touching other
|
||||
# shards' FTS state.
|
||||
from arborist.migrate import _rebuild_fts_on_target
|
||||
for target in targets:
|
||||
_rebuild_fts_on_target(target)
|
||||
|
||||
# Hygiene: leave WAL files bounded after each phase. The migrate
|
||||
# module already does this implicitly via _rebuild_fts_on_target
|
||||
# but we call it explicitly here for the post-Phase-2b state.
|
||||
for target in targets:
|
||||
target.execute("PRAGMA wal_checkpoint(TRUNCATE)")
|
||||
# IMPORTANT: FTS5 rebuild is NOT performed here. In parallel
|
||||
# hydrate (4 workers, one per pack), running _rebuild_fts_on_target
|
||||
# concurrently across 4 target shards forces all writers through
|
||||
# the same ext4 journal — 30-minute serialization on jbd2 commit.
|
||||
# Defer FTS5 entirely to a SEQUENTIAL post-pass (one process,
|
||||
# processes all shards in turn): `arborist cold rebuild-fts
|
||||
# --shards-dir DIR`, wired up by `make cold-hydrate`. Each shard
|
||||
# gets full filesystem-journal bandwidth in its turn.
|
||||
#
|
||||
# Until rebuild-fts runs, the hydrated peer has chunks.content
|
||||
# populated but chunks_fts empty — body keyword search returns
|
||||
# nothing. Documents_fts (title index) also waits for the
|
||||
# post-pass. Operators can run queries against the peer
|
||||
# immediately for title / graph / audit navigation; body queries
|
||||
# require the FTS rebuild to land first.
|
||||
|
||||
chunk_pack_hashes_referenced: tuple[str, ...] = ()
|
||||
if manifest_bytes is not None:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue