engineering/arborist
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

#000061: cold-pack-all + cold-pack-all-dvd — per-shard parallel pack runs

Browse source 
Shards are independent SQLite files; the ORDER BY landed in 6f0ceab
makes each shard's chunk-to-pack assignment deterministic. Two new
Makefile targets fan out cold pack across every numbered shard via
xargs -P:

  make cold-pack-all                 # parallel push to S3
  make cold-pack-all-dvd LOCAL_DIR=  # parallel write to disk (no S3)

Tunables:
  SHARDS_DIR     defaults to ~/.arborist/shards
  SHARDS_PATTERN defaults to [0-9][0-9][0-9].db (numbered shards only,
                 skipping qa.db / snapshots.db / crawl_*)
  COLD_PACK_JOBS defaults to 4

Expected wall-clock improvement: ~4x on full-corpus runs (4 shards
running in parallel saturate either CPU or network before serializing).

Also verified end-to-end pack → list → unpack round trip against the
live DO Spaces NYC3 endpoint (small pack, 20 chunks): pushed, listed,
unpacked into a fresh receiving DB, content restored byte-identical,
hashes matched.
This commit is contained in:
russell@unturf.com 2026-05-25 20:46:53 -04:00
parent 51f1736091
commit edb4e2d29e
No known key found for this signature in database
1 changed files with 28 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

29
Makefile
View file

@ -41,7 +41,7 @@ SEARCH_Q ?= computer
textbooks-summary textbooks-urls fetch-textbooks textbooks-stats textbooks-verify \
crawl-textbooks crawl-textbooks-stats textbook textbook-list bench-jaggedness \
monitor-poll monitor-graph monitor-access \
bootstrap-object-store cold-pack cold-pack-dvd cold-unpack cold-stats cold-list
bootstrap-object-store cold-pack cold-pack-dvd cold-pack-all cold-pack-all-dvd cold-unpack cold-stats cold-list
all: bootstrap fetch-cur ingest-cur verify stats ## bootstrap → fetch cur → ingest cur → verify → stats
@ -886,6 +886,33 @@ cold-stats: bootstrap ## bucket summary: pack count, total bytes, endpoint
cold-list: bootstrap ## enumerate packs (pack_hash, size, chunk_count) for new-peer hydration
$(ARBORIST) --db $(DB) cold list
# Per-shard parallel pack run. Shards are independent SQLite files; packing
# them in parallel scales near-linearly with CPU/network up to ~4 workers
# (bench: single shard 100k chunks → 40 s build + 13 s upload; 4 shards
# parallel ≈ same wall-clock for ~4x the corpus). SHARDS_DIR defaults to
# ~/.arborist/shards; SHARDS_PATTERN filters which shard DBs participate.
SHARDS_DIR ?= $(HOME)/.arborist/shards
SHARDS_PATTERN ?= [0-9][0-9][0-9].db
COLD_PACK_JOBS ?= 4
cold-pack-all: bootstrap ## parallel cold-pack across every shard (SHARDS_DIR, COLD_PACK_JOBS)
@echo ">> packing shards in $(SHARDS_DIR)/$(SHARDS_PATTERN) — $(COLD_PACK_JOBS)-way parallel"
@find $(SHARDS_DIR) -maxdepth 1 -name '$(SHARDS_PATTERN)' -print0 | \
xargs -0 -n 1 -P $(COLD_PACK_JOBS) -I {} \
$(ARBORIST) --db {} cold pack
@echo ">> all shards packed; final bucket state:"
@$(ARBORIST) cold stats
cold-pack-all-dvd: bootstrap ## same fan-out but local-dir for DVD burning (no S3 push)
@if [ -z "$(LOCAL_DIR)" ]; then echo "LOCAL_DIR=<dir> required"; exit 2; fi
@mkdir -p $(LOCAL_DIR)
@echo ">> packing shards in $(SHARDS_DIR)/$(SHARDS_PATTERN) to $(LOCAL_DIR) — $(COLD_PACK_JOBS)-way parallel, no S3"
@find $(SHARDS_DIR) -maxdepth 1 -name '$(SHARDS_PATTERN)' -print0 | \
xargs -0 -n 1 -P $(COLD_PACK_JOBS) -I {} \
$(ARBORIST) --db {} cold pack --no-push --local-dir $(LOCAL_DIR)
@echo ">> all packs in $(LOCAL_DIR):"
@ls -lh $(LOCAL_DIR)/arborist-pack-*.tar.zst
test: bootstrap ## run pytest suite (excludes opt-in crawler tests)
$(VENV)/bin/pytest -q --ignore=tests/crawler -n auto