diff --git a/Makefile b/Makefile index b8b8d64..283e833 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ SEARCH_Q ?= computer textbooks-summary textbooks-urls fetch-textbooks textbooks-stats textbooks-verify \ crawl-textbooks crawl-textbooks-stats textbook textbook-list bench-jaggedness \ monitor-poll monitor-graph monitor-access \ - bootstrap-object-store cold-pack cold-pack-dvd cold-pack-all cold-pack-all-dvd cold-unpack cold-stats cold-list + bootstrap-object-store cold-pack cold-pack-dvd cold-pack-all cold-pack-all-dvd cold-unpack cold-hydrate cold-stats cold-list all: bootstrap fetch-cur ingest-cur verify stats ## bootstrap → fetch cur → ingest cur → verify → stats @@ -868,7 +868,8 @@ cold-pack: bootstrap ## bundle hot chunks into tar.zst pack(s) on bucket (≤4.4 $(ARBORIST) --db $(DB) cold pack \ $(if $(MAX_CHUNKS),--max-chunks $(MAX_CHUNKS),) \ $(if $(MAX_PACK_BYTES),--max-pack-bytes $(MAX_PACK_BYTES),) \ - $(if $(LOCAL_DIR),--local-dir $(LOCAL_DIR),) + $(if $(LOCAL_DIR),--local-dir $(LOCAL_DIR),) \ + $(if $(ALLOW_LICENSE_CLASS),--allow-license-class $(ALLOW_LICENSE_CLASS),) cold-pack-dvd: bootstrap ## write packs to LOCAL_DIR for burning (no S3 upload); each pack ≤4.7 GB @if [ -z "$(LOCAL_DIR)" ]; then echo "LOCAL_DIR= required"; exit 2; fi @@ -932,10 +933,31 @@ cold-pack-all: bootstrap ## parallel cold-pack, RAM-aware (override with COLD_PA echo ">> packing shards in $(SHARDS_DIR)/$(SHARDS_PATTERN) — $$JOBS-way parallel"; \ find $(SHARDS_DIR) -maxdepth 1 -name '$(SHARDS_PATTERN)' -print0 | \ xargs -0 -n 1 -P $$JOBS -I {} \ - $(ARBORIST) --db {} cold pack + $(ARBORIST) --db {} cold pack \ + $(if $(ALLOW_LICENSE_CLASS),--allow-license-class $(ALLOW_LICENSE_CLASS),) @echo ">> all shards packed; final bucket state:" @$(ARBORIST) cold stats +cold-hydrate: bootstrap ## genesis a fresh peer from cloud, M-aware: pull every metadata pack from bucket in parallel and route into HYDRATE_DIR/00N.db × HYDRATE_M + @if [ -z "$(HYDRATE_DIR)" ]; then echo "HYDRATE_DIR= required"; exit 2; fi + @if [ -z "$(HYDRATE_M)" ]; then echo "HYDRATE_M= required (matches #000065 canonical M)"; exit 2; fi + @mkdir -p $(HYDRATE_DIR) + @$(COLD_PACK_PICK_JOBS); \ + echo ">> discovering metadata packs in bucket"; \ + hashes=$$($(ARBORIST) cold list --no-manifest \ + | $(VENV)/bin/python -c \ + "import sys, json; d=json.load(sys.stdin); print(' '.join(p['pack_hash'] for p in d['packs'] if p['kind']=='metadata'))"); \ + if [ -z "$$hashes" ]; then echo "no metadata packs in bucket"; exit 3; fi; \ + echo ">> $$JOBS-way parallel pull of metadata packs (M=$(HYDRATE_M))"; \ + printf "%s\n" $$hashes | \ + xargs -n 1 -P $$JOBS -I {} \ + $(ARBORIST) cold unpack {} \ + --hydrate-shards-dir $(HYDRATE_DIR) \ + --hydrate-M $(HYDRATE_M) \ + --full + @echo ">> hydration complete:" + @ls -lh $(HYDRATE_DIR)/*.db 2>/dev/null + cold-pack-all-dvd: bootstrap ## same fan-out for DVD burning (no S3); RAM-aware @if [ -z "$(LOCAL_DIR)" ]; then echo "LOCAL_DIR= required"; exit 2; fi @mkdir -p $(LOCAL_DIR)