diff --git a/Makefile b/Makefile index 2efeeda..3238223 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ SEARCH_Q ?= computer textbooks-summary textbooks-urls fetch-textbooks textbooks-stats textbooks-verify \ crawl-textbooks crawl-textbooks-stats textbook textbook-list bench-jaggedness \ monitor-poll monitor-graph monitor-access \ - bootstrap-object-store cold-pack cold-pack-dvd cold-unpack cold-stats cold-list + bootstrap-object-store cold-pack cold-pack-dvd cold-pack-all cold-pack-all-dvd cold-unpack cold-stats cold-list all: bootstrap fetch-cur ingest-cur verify stats ## bootstrap → fetch cur → ingest cur → verify → stats @@ -886,6 +886,33 @@ cold-stats: bootstrap ## bucket summary: pack count, total bytes, endpoint cold-list: bootstrap ## enumerate packs (pack_hash, size, chunk_count) for new-peer hydration $(ARBORIST) --db $(DB) cold list +# Per-shard parallel pack run. Shards are independent SQLite files; packing +# them in parallel scales near-linearly with CPU/network up to ~4 workers +# (bench: single shard 100k chunks → 40 s build + 13 s upload; 4 shards +# parallel ≈ same wall-clock for ~4x the corpus). SHARDS_DIR defaults to +# ~/.arborist/shards; SHARDS_PATTERN filters which shard DBs participate. +SHARDS_DIR ?= $(HOME)/.arborist/shards +SHARDS_PATTERN ?= [0-9][0-9][0-9].db +COLD_PACK_JOBS ?= 4 + +cold-pack-all: bootstrap ## parallel cold-pack across every shard (SHARDS_DIR, COLD_PACK_JOBS) + @echo ">> packing shards in $(SHARDS_DIR)/$(SHARDS_PATTERN) — $(COLD_PACK_JOBS)-way parallel" + @find $(SHARDS_DIR) -maxdepth 1 -name '$(SHARDS_PATTERN)' -print0 | \ + xargs -0 -n 1 -P $(COLD_PACK_JOBS) -I {} \ + $(ARBORIST) --db {} cold pack + @echo ">> all shards packed; final bucket state:" + @$(ARBORIST) cold stats + +cold-pack-all-dvd: bootstrap ## same fan-out but local-dir for DVD burning (no S3 push) + @if [ -z "$(LOCAL_DIR)" ]; then echo "LOCAL_DIR= required"; exit 2; fi + @mkdir -p $(LOCAL_DIR) + @echo ">> packing shards in $(SHARDS_DIR)/$(SHARDS_PATTERN) to $(LOCAL_DIR) — $(COLD_PACK_JOBS)-way parallel, no S3" + @find $(SHARDS_DIR) -maxdepth 1 -name '$(SHARDS_PATTERN)' -print0 | \ + xargs -0 -n 1 -P $(COLD_PACK_JOBS) -I {} \ + $(ARBORIST) --db {} cold pack --no-push --local-dir $(LOCAL_DIR) + @echo ">> all packs in $(LOCAL_DIR):" + @ls -lh $(LOCAL_DIR)/arborist-pack-*.tar.zst + test: bootstrap ## run pytest suite (excludes opt-in crawler tests) $(VENV)/bin/pytest -q --ignore=tests/crawler -n auto