From 93a4a663b4589586a0b63511d1dc93783797e0d2 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Tue, 26 May 2026 17:55:16 -0400 Subject: [PATCH] make: cold-hydrate target + thread ALLOW_LICENSE_CLASS through cold-pack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds two missing pieces to the cold-pack lifecycle: 1. ALLOW_LICENSE_CLASS pass-through on cold-pack and cold-pack-all. Without this, packs from a corpus containing any `unknown`-class docs (html crawls / textbooks) refuse to push to a public bucket per #000061 Gap 2. Real-world bench on 2026-05-26 hit this because the production corpus has crawl + textbook content. Usage: make cold-pack-all ALLOW_LICENSE_CLASS=unknown make cold-pack DB=~/.arborist/shards/000.db \ ALLOW_LICENSE_CLASS=unknown 2. cold-hydrate — M-aware genesis-from-bucket as a make target. Discovers every metadata pack in the bucket via `cold list --no-manifest` + a tiny python json filter, then fires $(JOBS)-way parallel `arborist cold unpack` workers (one per metadata pack) into the same target shards directory. Each worker writes to one disjoint target shard because the bucket's packs come from the post-reshard #000065 source layout (each producer shard's docs all hash to one consumer target) — so zero write contention across the parallel workers. Usage: make cold-hydrate \ HYDRATE_DIR=~/.arborist/shards-genesis-test \ HYDRATE_M=4 Why these belong here (not in /tmp wrapper scripts that get tossed): fox 2026-05-26 — "make sure the pack and upload and the pull and ingest all happen with makefile targets". The cold-pack lifecycle is a first-class operations surface; throwaway tmp scripts hide the real interface from future operators. Both targets are listed in `make help`. The .PHONY line covers cold-hydrate so a directory named cold-hydrate can't shadow it. Sequence to repeat today's run end-to-end: producer: make cold-pack-all ALLOW_LICENSE_CLASS=unknown consumer: make cold-hydrate HYDRATE_DIR=~/.arborist/shards \ HYDRATE_M=4 Today's in-flight 3090 hydrate (started before this commit) used the equivalent shell loop; from now on every operator uses the make target. --- Makefile | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index b8b8d64..283e833 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ SEARCH_Q ?= computer textbooks-summary textbooks-urls fetch-textbooks textbooks-stats textbooks-verify \ crawl-textbooks crawl-textbooks-stats textbook textbook-list bench-jaggedness \ monitor-poll monitor-graph monitor-access \ - bootstrap-object-store cold-pack cold-pack-dvd cold-pack-all cold-pack-all-dvd cold-unpack cold-stats cold-list + bootstrap-object-store cold-pack cold-pack-dvd cold-pack-all cold-pack-all-dvd cold-unpack cold-hydrate cold-stats cold-list all: bootstrap fetch-cur ingest-cur verify stats ## bootstrap → fetch cur → ingest cur → verify → stats @@ -868,7 +868,8 @@ cold-pack: bootstrap ## bundle hot chunks into tar.zst pack(s) on bucket (≤4.4 $(ARBORIST) --db $(DB) cold pack \ $(if $(MAX_CHUNKS),--max-chunks $(MAX_CHUNKS),) \ $(if $(MAX_PACK_BYTES),--max-pack-bytes $(MAX_PACK_BYTES),) \ - $(if $(LOCAL_DIR),--local-dir $(LOCAL_DIR),) + $(if $(LOCAL_DIR),--local-dir $(LOCAL_DIR),) \ + $(if $(ALLOW_LICENSE_CLASS),--allow-license-class $(ALLOW_LICENSE_CLASS),) cold-pack-dvd: bootstrap ## write packs to LOCAL_DIR for burning (no S3 upload); each pack ≤4.7 GB @if [ -z "$(LOCAL_DIR)" ]; then echo "LOCAL_DIR= required"; exit 2; fi @@ -932,10 +933,31 @@ cold-pack-all: bootstrap ## parallel cold-pack, RAM-aware (override with COLD_PA echo ">> packing shards in $(SHARDS_DIR)/$(SHARDS_PATTERN) — $$JOBS-way parallel"; \ find $(SHARDS_DIR) -maxdepth 1 -name '$(SHARDS_PATTERN)' -print0 | \ xargs -0 -n 1 -P $$JOBS -I {} \ - $(ARBORIST) --db {} cold pack + $(ARBORIST) --db {} cold pack \ + $(if $(ALLOW_LICENSE_CLASS),--allow-license-class $(ALLOW_LICENSE_CLASS),) @echo ">> all shards packed; final bucket state:" @$(ARBORIST) cold stats +cold-hydrate: bootstrap ## genesis a fresh peer from cloud, M-aware: pull every metadata pack from bucket in parallel and route into HYDRATE_DIR/00N.db × HYDRATE_M + @if [ -z "$(HYDRATE_DIR)" ]; then echo "HYDRATE_DIR= required"; exit 2; fi + @if [ -z "$(HYDRATE_M)" ]; then echo "HYDRATE_M= required (matches #000065 canonical M)"; exit 2; fi + @mkdir -p $(HYDRATE_DIR) + @$(COLD_PACK_PICK_JOBS); \ + echo ">> discovering metadata packs in bucket"; \ + hashes=$$($(ARBORIST) cold list --no-manifest \ + | $(VENV)/bin/python -c \ + "import sys, json; d=json.load(sys.stdin); print(' '.join(p['pack_hash'] for p in d['packs'] if p['kind']=='metadata'))"); \ + if [ -z "$$hashes" ]; then echo "no metadata packs in bucket"; exit 3; fi; \ + echo ">> $$JOBS-way parallel pull of metadata packs (M=$(HYDRATE_M))"; \ + printf "%s\n" $$hashes | \ + xargs -n 1 -P $$JOBS -I {} \ + $(ARBORIST) cold unpack {} \ + --hydrate-shards-dir $(HYDRATE_DIR) \ + --hydrate-M $(HYDRATE_M) \ + --full + @echo ">> hydration complete:" + @ls -lh $(HYDRATE_DIR)/*.db 2>/dev/null + cold-pack-all-dvd: bootstrap ## same fan-out for DVD burning (no S3); RAM-aware @if [ -z "$(LOCAL_DIR)" ]; then echo "LOCAL_DIR= required"; exit 2; fi @mkdir -p $(LOCAL_DIR)