From edb4e2d29e5bc29944d22d025841e9191717208c Mon Sep 17 00:00:00 2001
From: "russell@unturf.com" <russell@unturf.com>
Date: Mon, 25 May 2026 20:46:53 -0400
Subject: [PATCH] =?UTF-8?q?#000061:=20cold-pack-all=20+=20cold-pack-all-dv?=
 =?UTF-8?q?d=20=E2=80=94=20per-shard=20parallel=20pack=20runs?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Shards are independent SQLite files; the ORDER BY landed in 6f0ceab
makes each shard's chunk-to-pack assignment deterministic. Two new
Makefile targets fan out cold pack across every numbered shard via
xargs -P:

  make cold-pack-all                 # parallel push to S3
  make cold-pack-all-dvd LOCAL_DIR=  # parallel write to disk (no S3)

Tunables:
  SHARDS_DIR     defaults to ~/.arborist/shards
  SHARDS_PATTERN defaults to [0-9][0-9][0-9].db (numbered shards only,
                 skipping qa.db / snapshots.db / crawl_*)
  COLD_PACK_JOBS defaults to 4

Expected wall-clock improvement: ~4x on full-corpus runs (4 shards
running in parallel saturate either CPU or network before serializing).

Also verified end-to-end pack → list → unpack round trip against the
live DO Spaces NYC3 endpoint (small pack, 20 chunks): pushed, listed,
unpacked into a fresh receiving DB, content restored byte-identical,
hashes matched.
---
 Makefile | 29 ++++++++++++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 2efeeda..3238223 100644
--- a/Makefile
+++ b/Makefile
@@ -41,7 +41,7 @@ SEARCH_Q     ?= computer
         textbooks-summary textbooks-urls fetch-textbooks textbooks-stats textbooks-verify \
         crawl-textbooks crawl-textbooks-stats textbook textbook-list bench-jaggedness \
         monitor-poll monitor-graph monitor-access \
-        bootstrap-object-store cold-pack cold-pack-dvd cold-unpack cold-stats cold-list
+        bootstrap-object-store cold-pack cold-pack-dvd cold-pack-all cold-pack-all-dvd cold-unpack cold-stats cold-list
 
 all: bootstrap fetch-cur ingest-cur verify stats ## bootstrap → fetch cur → ingest cur → verify → stats
 
@@ -886,6 +886,33 @@ cold-stats: bootstrap ## bucket summary: pack count, total bytes, endpoint
 cold-list: bootstrap ## enumerate packs (pack_hash, size, chunk_count) for new-peer hydration
 	$(ARBORIST) --db $(DB) cold list
 
+# Per-shard parallel pack run. Shards are independent SQLite files; packing
+# them in parallel scales near-linearly with CPU/network up to ~4 workers
+# (bench: single shard 100k chunks → 40 s build + 13 s upload; 4 shards
+# parallel ≈ same wall-clock for ~4x the corpus). SHARDS_DIR defaults to
+# ~/.arborist/shards; SHARDS_PATTERN filters which shard DBs participate.
+SHARDS_DIR     ?= $(HOME)/.arborist/shards
+SHARDS_PATTERN ?= [0-9][0-9][0-9].db
+COLD_PACK_JOBS ?= 4
+
+cold-pack-all: bootstrap ## parallel cold-pack across every shard (SHARDS_DIR, COLD_PACK_JOBS)
+	@echo ">> packing shards in $(SHARDS_DIR)/$(SHARDS_PATTERN) — $(COLD_PACK_JOBS)-way parallel"
+	@find $(SHARDS_DIR) -maxdepth 1 -name '$(SHARDS_PATTERN)' -print0 | \
+	    xargs -0 -n 1 -P $(COLD_PACK_JOBS) -I {} \
+	    $(ARBORIST) --db {} cold pack
+	@echo ">> all shards packed; final bucket state:"
+	@$(ARBORIST) cold stats
+
+cold-pack-all-dvd: bootstrap ## same fan-out but local-dir for DVD burning (no S3 push)
+	@if [ -z "$(LOCAL_DIR)" ]; then echo "LOCAL_DIR=<dir> required"; exit 2; fi
+	@mkdir -p $(LOCAL_DIR)
+	@echo ">> packing shards in $(SHARDS_DIR)/$(SHARDS_PATTERN) to $(LOCAL_DIR) — $(COLD_PACK_JOBS)-way parallel, no S3"
+	@find $(SHARDS_DIR) -maxdepth 1 -name '$(SHARDS_PATTERN)' -print0 | \
+	    xargs -0 -n 1 -P $(COLD_PACK_JOBS) -I {} \
+	    $(ARBORIST) --db {} cold pack --no-push --local-dir $(LOCAL_DIR)
+	@echo ">> all packs in $(LOCAL_DIR):"
+	@ls -lh $(LOCAL_DIR)/arborist-pack-*.tar.zst
+
 test: bootstrap ## run pytest suite (excludes opt-in crawler tests)
 	$(VENV)/bin/pytest -q --ignore=tests/crawler -n auto