# arborist — Makefile entry points # Every workflow lives behind a `make` target. Bare python commands are not # the user interface. # Tools and config PYTHON ?= python3 VENV ?= .venv PIP := $(VENV)/bin/pip PY := $(VENV)/bin/python ARBORIST := $(VENV)/bin/arborist # Data + DB DATA_DIR ?= data WP_BASE_URL ?= https://dumps.wikimedia.org/archive/2003/2003-05-16/en WP_CUR := $(DATA_DIR)/20030516_cur_tablesql.bz2 WP_OLD_1 := $(DATA_DIR)/old_tablesqlbz2.1 WP_OLD_2 := $(DATA_DIR)/old_tablesqlbz2.2 WP_OLD := $(DATA_DIR)/20030516_old_tablesql.bz2 # Back-compat alias (older callers used WP_DUMP for the cur snapshot). WP_DUMP := $(WP_CUR) DB ?= $(HOME)/.arborist/arborist.db # Smoke-test caps so make all stays fast INGEST_LIMIT ?= 500 VERIFY_N ?= 10 SEARCH_Q ?= computer .PHONY: all bootstrap fetch fetch-cur fetch-old fetch-xml fetch-abstract \ ingest ingest-cur ingest-old ingest-xml ingest-xml-history \ ingest-xml-attached ingest-abstract \ ingest-self ingest-self-providence ingest-git ingest-hg \ verify search stats test test-ci test-live docs docs-api docs-api-clean \ docs-one-pager docs-two-pager docs-pagers docs-pagers-clean \ chain-check chain-check-shards \ falsify burn burn-kindergarten inspect bootstrap-crawler test-crawler crawl-ingest \ recrawl-check bench-qa bench-qa-smoke bench-qa-progressive-and \ prometheus-trigger-probe bench-5f-threshold-calibration \ bench-5f-selfmodel-snapshot bench-5f-finetuning-shardchain \ bench-5f-falsification-hard bench-fork-baseline-hard bench-5f-formulate-hard \ bootstrap-math bootstrap-nli bootstrap-nli-only bench-nli-shadow export-nli-onnx bench-nli-backends judge-self-test control-ab control-sweep rapl-access rapl-access-revoke clean clean-db clean-data help \ textbooks-summary textbooks-urls fetch-textbooks textbooks-stats textbooks-verify \ crawl-textbooks crawl-textbooks-stats textbook textbook-list bench-jaggedness bench-spatial-anchor \ monitor-poll monitor-graph monitor-access \ bootstrap-object-store cold-pack cold-pack-dvd cold-pack-all cold-pack-all-dvd cold-unpack cold-hydrate cold-stats cold-list \ wallet-demo wallet-serve wallet-pin wallet-ask all: bootstrap fetch-cur ingest-cur verify stats ## bootstrap → fetch cur → ingest cur → verify → stats help: ## show this help @awk 'BEGIN{FS=":.*##"} /^[a-zA-Z0-9_-]+:.*##/{printf " %-16s %s\n", $$1, $$2}' $(MAKEFILE_LIST) $(VENV)/bin/activate: pyproject.toml $(PYTHON) -m venv $(VENV) $(PIP) install --upgrade pip wheel $(PIP) install -e '.[dev]' @touch $(VENV)/bin/activate bootstrap: $(VENV)/bin/activate ## create venv and install editable package $(DATA_DIR): mkdir -p $(DATA_DIR) $(WP_CUR): | $(DATA_DIR) @echo ">> fetching $(WP_BASE_URL)/$$(basename $@)" curl -fL --retry 3 -o $@ "$(WP_BASE_URL)/$$(basename $@)" $(WP_OLD_1): | $(DATA_DIR) @echo ">> fetching $(WP_BASE_URL)/$$(basename $@)" curl -fL --retry 3 -o $@ "$(WP_BASE_URL)/$$(basename $@)" $(WP_OLD_2): | $(DATA_DIR) @echo ">> fetching $(WP_BASE_URL)/$$(basename $@)" curl -fL --retry 3 -o $@ "$(WP_BASE_URL)/$$(basename $@)" # The two old_tablesqlbz2.{1,2} parts are split halves of a single bzip2 # stream (.1 is exactly 640 MiB). Concatenate to get a working bz2 file. $(WP_OLD): $(WP_OLD_1) $(WP_OLD_2) @echo ">> concatenating old dump parts" cat $(WP_OLD_1) $(WP_OLD_2) > $@ fetch-cur: $(WP_CUR) ## download cur table dump (~82 MB) fetch-old: $(WP_OLD) ## download old (revision history) parts and concatenate (~893 MB) fetch: fetch-cur fetch-old ## download all 3 files (cur + old.1 + old.2 + concat) ingest-cur: bootstrap fetch-cur ## ingest INGEST_LIMIT cur articles $(ARBORIST) --db $(DB) ingest --source wikipedia_cur --path $(WP_CUR) --limit $(INGEST_LIMIT) ingest-old: bootstrap fetch-old ## ingest INGEST_LIMIT old (history) revisions $(ARBORIST) --db $(DB) ingest --source wikipedia_old --path $(WP_OLD) --limit $(INGEST_LIMIT) # Phase 2: attach-forever sharding. Each shard owns its own SQLite file — # no WAL contention. Reads via `arborist --shards-dir ` attach # all shards as UNION views. "Merge cost" = 0. SHARDS ?= 4 SHARDS_DIR ?= $(HOME)/.arborist/shards ingest-cur-attached: bootstrap fetch-cur ## sharded ingest, no WAL contention (Phase 2) @mkdir -p $(SHARDS_DIR) @for i in $$(seq 0 $$(($(SHARDS) - 1))); do \ $(ARBORIST) ingest --source wikipedia_cur --path $(WP_CUR) \ --shards-dir $(SHARDS_DIR) --shard $$i/$(SHARDS) & \ done; wait ingest-old-attached: bootstrap fetch-old ## sharded ingest of old history (Phase 2) @mkdir -p $(SHARDS_DIR) @for i in $$(seq 0 $$(($(SHARDS) - 1))); do \ $(ARBORIST) ingest --source wikipedia_old --path $(WP_OLD) \ --shards-dir $(SHARDS_DIR) --shard $$i/$(SHARDS) & \ done; wait stats-shards: bootstrap ## cross-shard stats via UNION views over $(SHARDS_DIR) $(ARBORIST) --shards-dir $(SHARDS_DIR) stats ACTIVITY_LIMIT ?= 10 activity: bootstrap ## recent Q&A + freshly cached docs (agent timeline) $(ARBORIST) --shards-dir $(SHARDS_DIR) activity --limit $(ACTIVITY_LIMIT) inspect: bootstrap ## sidecar diagnose unverified spans for a cache_key: make inspect KEY=hex [JSON=1] @if [ -z "$(KEY)" ]; then echo "usage: make inspect KEY= [JSON=1]" >&2; exit 2; fi $(ARBORIST) --shards-dir $(SHARDS_DIR) inspect --cache-key $(KEY) $(if $(JSON),--json,) falsify: bootstrap ## mark a cached answer wrong: make falsify KEY=hex REASON='why' @if [ -z "$(KEY)" ]; then echo "usage: make falsify KEY= REASON='why'" >&2; exit 2; fi $(ARBORIST) --shards-dir $(SHARDS_DIR) providence --falsify $(KEY) --reason "$(REASON)" burn: bootstrap ## delete a leaf with no children. providence: KEY=; document/core: KIND=document|core ROOT=. REASON='why' [FORCE=1] @kind="$${KIND:-providence}"; \ if [ "$$kind" = "providence" ]; then \ if [ -z "$(KEY)" ]; then echo "usage: make burn KEY= REASON='why' [FORCE=1]" >&2; exit 2; fi; \ $(ARBORIST) --shards-dir $(SHARDS_DIR) burn --kind providence --cache-key $(KEY) --reason "$(REASON)" $(if $(FORCE),--force,); \ elif [ "$$kind" = "document" ] || [ "$$kind" = "core" ]; then \ if [ -z "$(ROOT)" ]; then echo "usage: make burn KIND=$$kind ROOT= REASON='why' [FORCE=1]" >&2; exit 2; fi; \ $(ARBORIST) --shards-dir $(SHARDS_DIR) burn --kind $$kind --root $(ROOT) --reason "$(REASON)" $(if $(FORCE),--force,); \ else \ echo "unknown KIND: $$kind (expected: providence|document|core)" >&2; exit 2; \ fi # Mass-burn providence_cache rows younger than the kindergarten window. # Mirrors mesh sync's kindergarten so what's still un-broadcast is what's # safe to bust without confusing peers. Useful while iterating on # retrieval/verifier tunings — wipe recent test runs in one shot. KG_SECONDS ?= 3600 burn-kindergarten: bootstrap ## bust providence rows < SECONDS old [SECONDS=3600 FORCE=1 DRY_RUN=1 REASON='why'] $(ARBORIST) --shards-dir $(SHARDS_DIR) burn-kindergarten \ --kindergarten-seconds $(KG_SECONDS) \ $(if $(REASON),--reason "$(REASON)",) \ $(if $(FORCE),--force,) \ $(if $(DRY_RUN),--dry-run,) # Multi-source RAG query against the shard cluster. # Usage: make query Q="What is anarcho-capitalism?" QUERY_TOP_K ?= 8 # G0 / CTI — pointer-mode is the testing default. Library DEFAULT_POLICY # stays "quote" so Python callers aren't surprised; the Makefile # harness ships pointer-mode-on so `make query` exercises the new path # end-to-end. ANSWER_MODE default flipped to claim_lattice (JSON) on # 2026-04-30 after the post-retry bench showed it leading on # strict-rate (50%) and grounded count (54) with 0 errors. Override # with ANSWER_MODE=claim_lattice_pointer for prose-distribution path # (Hermes-3 8B reflexive output without grammar guidance), or # ANSWER_MODE=quote for the legacy substring verifier. ANSWER_MODE= # (empty) defers to DEFAULT_QUERY_POLICY. ANSWER_MODE ?= claim_lattice # BROAD=1 → flip on the Ticket #000008 quantifier-cap apply-gate for # this call. Default-off per §10.11.3 dry-run discipline; operator # opts in here for broad-quantifier shapes (winners-of-all, # tell-me-everything-about-X). Pairs cleanly with the broad- # quantifier reminder which is default-on for lattice modes. # Bench (#000008 §12.10): cap-on JSON wins +14pp on STRICT-rate. # REJECT_BROAD=1 → strict reject for ALL/COMPREHENSIVE/OPEN_REQUEST # unbounded shapes; returns UNGROUNDED before the LLM call. # ALLOW_BROAD=1 → emergent search; classifier on, caps off. # # LAYOUT — user-payload-layout policy knob. Decides where the # question text sits relative to evidence in the final user-turn # message. See docs/user-payload-layout.md. # Recommendation matrix (2026-05-27 n=3 × 76q bench, Hermes-3-8B, # claim_lattice mode): # tail safe default — preserves prior cache (proven) # bookend safer for small-model recovery; n=3 aggregate bench # was a wash vs tail (+0.44pp STRICT, within 5pp floor) # per_chunk best on list/extraction shapes for small models # (the Ballestrini case) BUT regresses -9.78pp on # aggregate STRICT — opt-in only, never a default # Override per-call: LAYOUT=bookend make query Q="..." # Override session-wide: LAYOUT_DEFAULT=bookend make query Q="..." LAYOUT_DEFAULT ?= tail LAYOUT ?= $(LAYOUT_DEFAULT) # LLM endpoint toggle (shared with cloud-query — same ifeq lives down # below; this one runs first because `query:` is defined ABOVE # cloud-query and Make evaluates top-down). `make query LLM=qwen # Q="..."` swaps to Qwen3.6-27B on uncloseai. LLM unset / LLM=hermes # leaves --endpoint/--model unset so the CLI falls back to its # built-in default (Hermes-3-8B on ai.unturf.com). ifeq ($(LLM),qwen) LLM_ENDPOINT ?= https://qwen.ai.unturf.com/v1 LLM_MODEL ?= Qwen3.6-27B-UD-Q4_K_XL.gguf endif query: bootstrap ## ask the corpus a question [JSON=1 BURN=1 LLM=qwen|hermes REPAIR=1 REPROMPTS=N K="extra retrieval keywords" ANSWER_MODE=claim_lattice|claim_lattice_pointer|quote LAYOUT=tail|bookend|per_chunk BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1 WITNESS=1 XLANG=1 XLANG_MT=1]; JSON by default @if [ -z "$$Q" ] && [ -z "$(Q)" ]; then \ echo "usage: make query Q=\"your question\" [JSON=1 BURN=1 LLM=qwen|hermes REPAIR=1 REPROMPTS=N K=\"extra retrieval keywords\" ANSWER_MODE=claim_lattice|claim_lattice_pointer|quote LAYOUT=tail|bookend|per_chunk BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1 WITNESS=1 XLANG=1 XLANG_MT=1]"; exit 2; \ fi $(if $(LLM_ENDPOINT),@echo "# llm: $(LLM_ENDPOINT) / $(LLM_MODEL)" >&2,) $(ARBORIST) --shards-dir $(SHARDS_DIR) query --top-k $(QUERY_TOP_K) $(if $(LLM_ENDPOINT),--endpoint $(LLM_ENDPOINT) --model $(LLM_MODEL),) $(if $(JSON),--json,) $(if $(BURN),--burn,) $(if $(REPAIR),--repair,) $(if $(REPROMPTS),--repair-reprompts $(REPROMPTS),) $(if $(ANSWER_MODE),--answer-mode $(ANSWER_MODE),) --user-payload-layout $(LAYOUT) $(if $(K),--retrieval-keywords "$(K)",) $(if $(BROAD),--apply-quantifier-caps,) $(if $(REJECT_BROAD),--reject-broad,) $(if $(ALLOW_BROAD),--allow-broad,) $(if $(WITNESS),--witness,) $(if $(XLANG),--crosslang-guard,) $(if $(XLANG_MT),--crosslang-translate,) "$(Q)" # Merkle-rooted multi-turn Q&A REPL. Each turn (or fork) mints one # tree node with a Bates id + a session-wide subtree_hash. Forking is # implicit: /cd to a parent node and ask again → sibling under that # parent. See arborist/qa/session.py + docs/sessions.md. # # Session defaults to Qwen3.6-27B (better answers for interactive # multi-turn use). Override with LLM=hermes for the 8B fast path. SESSION_LLM_ENDPOINT ?= https://qwen.ai.unturf.com/v1 SESSION_LLM_MODEL ?= Qwen3.6-27B-UD-Q4_K_XL.gguf ifeq ($(LLM),hermes) SESSION_LLM_ENDPOINT := https://hermes.ai.unturf.com/v1 SESSION_LLM_MODEL := adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic endif session: bootstrap ## interactive multi-turn Q&A REPL [SID=... LLM=qwen|hermes ANSWER_MODE=... BURN=1]; defaults LLM=qwen @echo "# llm: $(SESSION_LLM_ENDPOINT) / $(SESSION_LLM_MODEL)" >&2 $(ARBORIST) --shards-dir $(SHARDS_DIR) session $(if $(SID),--sid $(SID),) --top-k $(QUERY_TOP_K) --endpoint $(SESSION_LLM_ENDPOINT) --model $(SESSION_LLM_MODEL) $(if $(ANSWER_MODE),--answer-mode $(ANSWER_MODE),) $(if $(BURN),--burn,) session-list: bootstrap ## list all sessions in the single shard $(ARBORIST) session --list $(if $(JSON),--json,) session-tree: bootstrap ## print one session's tree [SID=...] @if [ -z "$$SID" ] && [ -z "$(SID)" ]; then \ echo "usage: make session-tree SID= [JSON=1]"; exit 2; \ fi $(ARBORIST) session --sid $(SID) --tree $(if $(JSON),--json,) session-chain-check: bootstrap ## verify the single shard's global audit chain $(ARBORIST) session --chain-check session-find: bootstrap ## FTS5 search over all session turns [Q="..." LIMIT=N JSON=1] @if [ -z "$$Q" ] && [ -z "$(Q)" ]; then \ echo "usage: make session-find Q=\"query text\" [LIMIT=10] [JSON=1]"; exit 2; \ fi $(ARBORIST) session --find "$(Q)" $(if $(LIMIT),--limit $(LIMIT),) $(if $(JSON),--json,) query-dry: bootstrap ## like 'make query' but skip the LLM call (dry-run) [JSON=1 BURN=1 ANSWER_MODE=... BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1 XLANG=1 XLANG_MT=1] @if [ -z "$$Q" ] && [ -z "$(Q)" ]; then \ echo "usage: make query-dry Q=\"your question\" [JSON=1 BURN=1 ANSWER_MODE=claim_lattice|claim_lattice_pointer|quote BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1 XLANG=1 XLANG_MT=1]"; exit 2; \ fi $(ARBORIST) --shards-dir $(SHARDS_DIR) query --top-k $(QUERY_TOP_K) --dry-run $(if $(JSON),--json,) $(if $(BURN),--burn,) $(if $(ANSWER_MODE),--answer-mode $(ANSWER_MODE),) $(if $(BROAD),--apply-quantifier-caps,) $(if $(REJECT_BROAD),--reject-broad,) $(if $(ALLOW_BROAD),--allow-broad,) $(if $(XLANG),--crosslang-guard,) $(if $(XLANG_MT),--crosslang-translate,) "$(Q)" BENCH_QA_QUESTIONS ?= bench/qa_questions.txt BENCH_QA_OUT ?= bench/qa_results BENCH_QA_MODES ?= quote,claim_lattice_pointer,claim_lattice BENCH_QA_LIMIT ?= 0 BENCH_QA_N ?= 3 BENCH_QA_CONCURRENCY ?= 4 bench-qa: bootstrap ## QA-quality sweep: questions × modes × N samples [BENCH_QA_N=3 BENCH_QA_LIMIT=N BENCH_QA_MODES=... BENCH_QA_CONCURRENCY=4] PYTHONUNBUFFERED=1 $(PY) bench/qa_sweep.py \ --questions $(BENCH_QA_QUESTIONS) \ --shards-dir $(SHARDS_DIR) \ --out-dir $(BENCH_QA_OUT) \ --top-k $(QUERY_TOP_K) \ --modes $(BENCH_QA_MODES) \ --limit $(BENCH_QA_LIMIT) \ --n $(BENCH_QA_N) \ --concurrency $(BENCH_QA_CONCURRENCY) # Smoke fixture: 5 questions, all anchor classes, all currently failing # pointer mode 100% while JSON aces 100%. Inner loop for prompt iteration. # ~30s wall-clock at concurrency=4. Use this between full sweeps. bench-qa-smoke: bootstrap ## quick 5-question smoke (all anchor classes; ~30s) PYTHONUNBUFFERED=1 $(PY) bench/qa_sweep.py \ --questions bench/qa_questions_smoke.txt \ --shards-dir $(SHARDS_DIR) \ --out-dir $(BENCH_QA_OUT) \ --top-k $(QUERY_TOP_K) \ --modes $(BENCH_QA_MODES) \ --n 1 \ --concurrency $(BENCH_QA_CONCURRENCY) # #000057 control experiment — the external judge instrument # (Opus via `claude -p`, hermetic/blinded/reference-grounded). This # target IS the instrument-before-experiment gate: it must pass # before any control A/B run trusts the judge (the session's # deepest lesson — verify the measuring tool first). Real `claude -p` # calls; ~4 verdicts, bounded. judge-self-test: ## verify the #000057 external judge on known-verdict triples (gate before control-ab) $(PY) bench/judge.py --self-test # #000057 v1 control experiment: Hermes-solo vs Arborist, same model, # blinded Opus judge vs fixed gold. Depends on judge-self-test so the # instrument-before-experiment gate CANNOT be skipped (make enforces # it). Bounded spend: N items -> 2N Hermes + 2N `claude -p`. # make control-ab N=12 FIXTURE=bench/qa_questions_numeral_map.json CONTROL_AB_FIXTURE ?= bench/qa_questions_numeral_map.json CONTROL_AB_N ?= 12 control-ab: judge-self-test ## #000057 v1: Hermes-solo vs Arborist, blinded Opus judge [N=12 FIXTURE=...] $(PY) bench/control_ab.py \ --fixture $(CONTROL_AB_FIXTURE) \ --n $(CONTROL_AB_N) \ --shards-dir $(SHARDS_DIR) \ --out-dir $(BENCH_QA_OUT) # #000057 control-arm characterization sweep: model × question-framing # matrix (fox: "measure all benchmarks, bring forward for review"; # "qwen with and without reasoning"). NO judge-self-test make dep on # purpose — control_sweep.py runs the gate ONCE in-script and ABORTS # the run on failure (a stronger guarantee than a build-graph edge: # it gates cell execution, not just the target, and avoids paying the # 4-call Opus self-test twice per invocation). # make control-sweep CONTROL_SWEEP_N=5 CONTROL_SWEEP_FIXTURE ?= bench/qa_questions_stale_map.json CONTROL_SWEEP_N ?= 3 CONTROL_SWEEP_ARB_N ?= 40 CONTROL_SWEEP_WORKERS ?= 6 CONTROL_SWEEP_RESUME ?= control-sweep: ## #000057: model×framing control sweep [CONTROL_SWEEP_N / _WORKERS / _RESUME=jsonl ...] $(PY) bench/control_sweep.py \ --fixture $(CONTROL_SWEEP_FIXTURE) \ --n $(CONTROL_SWEEP_N) \ --arborist-n $(CONTROL_SWEEP_ARB_N) \ --max-workers $(CONTROL_SWEEP_WORKERS) \ $(if $(CONTROL_SWEEP_RESUME),--resume $(CONTROL_SWEEP_RESUME),) \ --shards-dir $(SHARDS_DIR) \ --out-dir $(BENCH_QA_OUT) # #000060 retrieval jaggedness: does the substrate surface the SAME # target across surface-preserving question perturbations (numeral / # accent / hyphen / honorific / amp / brit)? Deterministic — query # --dry-run only, no LLM / no verifier / no judge / no n=3 noise (the # recall_at_k discipline). J_norm = canonical-vs-perturbed surfacing # disagreement rate; lower = more non-jagged. Feeds #000012 ForkScore # ΔJaggednessReduction. Reuses recall_at_k.probe + mine_questions. # make bench-jaggedness JAGGED_LIMIT=40 JAGGED_K=8 JAGGED_CLASSES ?= numeral,accent,hyphen,honorific,amp,brit JAGGED_LIMIT ?= 40 JAGGED_K ?= 8 JAGGED_CONC ?= 4 bench-jaggedness: bootstrap ## #000060: deterministic retrieval jaggedness across surface perturbations [JAGGED_LIMIT / _K / _CLASSES ...] $(PY) bench/jaggedness.py \ --classes $(JAGGED_CLASSES) \ --limit $(JAGGED_LIMIT) \ --k $(JAGGED_K) \ --conc $(JAGGED_CONC) \ --shards-dir $(SHARDS_DIR) # Empirical pre-review validation of Joseph @TrudoJo's 6-dim spatial-anchor # framework (ticket #000070). Pure stdlib, ~2s on a workstation. Answers # dav1d's open questions §2.1 / §2.2 / §2.3 / §2.8 with measurements # instead of appeals to PRF authority. Reproducible: same RNG seed → same # byte output. See bench/spatial_anchor_validation.py. # make bench-spatial-anchor # default N=10000 # make bench-spatial-anchor SPATIAL_N=50000 # tighter stderr SPATIAL_N ?= 10000 bench-spatial-anchor: bootstrap ## #000070: pre-review empirical evidence for 6-dim spatial-anchor framework [SPATIAL_N=10000] PYTHONUNBUFFERED=1 $(PY) bench/spatial_anchor_validation.py --n $(SPATIAL_N) # Request-load monitor for the single-slot LLM endpoints (fox 2026-05-21: # "are we being swamped because we're open to internet?"). Stdlib-only # (urllib + sqlite3 + hand-rolled SVG) — no bootstrap, no venv. The backend # /metrics answers HOW MUCH (queue depth = the swamp signal); the proxy # access log answers WHO (real client IPs the backend can't see behind the # Caddy hop). See bench/load_monitor.py. # make monitor-poll # poll forever, Ctrl-C to stop # make monitor-graph MONITOR_HOURS=6 # render last 6h to SVG # make monitor-access LOG=/var/log/caddy/access.log MONITOR_DB ?= $(HOME)/.arborist/load_monitor.db MONITOR_INTERVAL ?= 10 MONITOR_HOURS ?= 6 MONITOR_TOP ?= 20 # GPU sampling: ssh nvidia-smi on the box behind each endpoint so power-draw + # util graph alongside request rate (the 100%-power-cap saturation signal). MONITOR_GPU ?= hermes-3090=3090-ai.foxhop.net monitor-poll: ## poll LLM endpoint /metrics (+GPU power/util) into SQLite [MONITOR_INTERVAL=10 MONITOR_GPU=name=host] PYTHONUNBUFFERED=1 python3 bench/load_monitor.py --db $(MONITOR_DB) \ poll --interval $(MONITOR_INTERVAL) --gpu $(MONITOR_GPU) monitor-graph: ## render stored load samples to SVG [MONITOR_HOURS=6] python3 bench/load_monitor.py --db $(MONITOR_DB) \ graph --since-hours $(MONITOR_HOURS) monitor-access: ## proxy access-log real-client-IP tally + SVG: make monitor-access LOG=path python3 bench/load_monitor.py access $(LOG) --top $(MONITOR_TOP) \ --out bench/results/proxy_access.svg # Progressive-AND / DF-filter fixture: 9 questions chosen to exercise # the OR-fallback and progressive-AND drop paths. Use this for any # retrieval-side A/B (alternative search backends, synonym/rerank # changes, etc.) — the smoke fixture is structurally insensitive # because it only contains queries where full-AND succeeds on every # shard. Default --n 3 to clear the LLM noise floor. ~3-5 min wall # at concurrency=4. Header in bench/qa_questions_progressive_and.txt # documents which chain fires per query. BENCH_PROGRESSIVE_N ?= 3 bench-qa-progressive-and: bootstrap ## retrieval-side fixture exercising progressive-AND + DF filter [BENCH_PROGRESSIVE_N=3] PYTHONUNBUFFERED=1 $(PY) bench/qa_sweep.py \ --questions bench/qa_questions_progressive_and.txt \ --shards-dir $(SHARDS_DIR) \ --out-dir $(BENCH_QA_OUT) \ --top-k $(QUERY_TOP_K) \ --modes $(BENCH_QA_MODES) \ --n $(BENCH_PROGRESSIVE_N) \ --concurrency $(BENCH_QA_CONCURRENCY) test-live: bootstrap ## live QA quality tests against Hermes (gated; -n auto parallel) ARBORIST_LIVE_TESTS=1 ARBORIST_LIVE_SHARDS_DIR=$(SHARDS_DIR) \ .venv/bin/pytest tests/test_qa_quality_live.py -v -n auto # Prometheus-Σ §12 trigger probe (ticket #000037 Phase 0). Read-only # walk of audit_events + capital_ledger across shards; reports # whether any of triggers 1-3 have fired empirically. Output is the # evidence fox uses to decide go/no-go on Phase 1. Pure measurement. PROMETHEUS_PROBE_OUT ?= bench/results/prometheus-sigma-triggers-$(shell date -u +%Y-%m-%d).md prometheus-trigger-probe: bootstrap ## #000037 §12 measured-pressure probe → markdown report PYTHONUNBUFFERED=1 $(PY) bench/prometheus_sigma_trigger_probe.py \ --shards-dir $(SHARDS_DIR) \ --out $(PROMETHEUS_PROBE_OUT) # Prometheus-Σ Phase 3 sleep-sweep dry-run (ticket #000037). Read-only # simulator: classifies Target A (providence_cache) + Target B # (documents) sweep candidates, synthesizes ControllerBranches, runs # the Phase 1 controller, reports decision distribution + Phase-3 # design findings. No LLM calls, no mutations. PROMETHEUS_SWEEP_DRYRUN_OUT ?= bench/results/prometheus-sigma-sweep-dryrun-$(shell date -u +%Y-%m-%d).md # §22 Finding 3 fix: per-audit-mode τ_qa. CP rows are kernel-only # (cheap re-probe → short τ); LLM-witness modes (STRICT/HYBRID/ # UNGROUNDED) re-witness via LLM (expensive → long τ). PROMETHEUS_SWEEP_TAU_DAYS ?= 7 PROMETHEUS_SWEEP_TAU_CP_DAYS ?= 1 PROMETHEUS_SWEEP_B_SAMPLE ?= 500 prometheus-sweep-dryrun: bootstrap ## #000037 Phase 3 sleep-sweep dry-run → markdown report PYTHONUNBUFFERED=1 $(PY) bench/scripts/prometheus_sigma_sweep_dryrun.py \ --shards-dir $(SHARDS_DIR) \ --tau-qa-days $(PROMETHEUS_SWEEP_TAU_DAYS) \ --tau-qa-cp-days $(PROMETHEUS_SWEEP_TAU_CP_DAYS) \ --sample-b $(PROMETHEUS_SWEEP_B_SAMPLE) \ --out $(PROMETHEUS_SWEEP_DRYRUN_OUT) # Harvest #000037 Phase 1 falsification-fixture proposals from # qa.db into a corpus-derived 5F fixture pack. Stratified sample # (20 HYBRID + 20 UNGROUNDED) by cache_key for determinism. # Idempotent — re-running rewrites the pack from scratch. HARVEST_QA_DB ?= $(SHARDS_DIR)/qa.db HARVEST_OUT ?= bench/fixtures/5f/falsification-harvested-v1.jsonl HARVEST_THRESHOLD ?= 0.5 HARVEST_SAMPLE_PER_BUCKET ?= 20 bench-5f-harvest: bootstrap ## harvest live-shard falsification proposals → 5F fixture pack PYTHONUNBUFFERED=1 $(PY) bench/scripts/harvest_falsification_proposals.py \ --qa-db $(HARVEST_QA_DB) \ --threshold $(HARVEST_THRESHOLD) \ --sample-per-bucket $(HARVEST_SAMPLE_PER_BUCKET) \ --out $(HARVEST_OUT) # Concept-layer backfill targets. Each runs an extractor across every # wiki shard; per-shard work is independent so we use GNU-parallel- # style concurrency with `xargs -P` to overlap the slow paths # (link_reciprocity ~50s/shard, token_idf ~12s/shard, documents_fts # ~3s/shard). Total wall-clock with -P 4 vs serial: # serial: link 200s + idf 50s + fts 10s = 260s # parallel: link 50s + idf 12s + fts 3s ≈ 65s (~4× speedup) CONCEPTS_WORKERS ?= 4 backfill-concepts: bootstrap ## backfill all concept extractors in parallel across shards PYTHONUNBUFFERED=1 $(PY) scripts/backfill_concepts.py \ --shards-dir $(SHARDS_DIR) \ --workers $(CONCEPTS_WORKERS) # Quick bench mode — 1 sample, smoke fixture, all 3 modes. ~10s. # For pure smoke after a code change before the longer bench-qa-smoke. # Emergent stress-test: random word triangulation. Pick 3 words from # /usr/share/dict/words, ask Hermes @ temp=0.8 to weave them into a # creative question, send to arborist, append the journey to # bench/emergent_log.jsonl. Designed for blue-moon cadence — surfaces # combinatoric failure modes the curated bench-qa fixture set can't. # Teacher review (Opus) runs separately via `--print-pending`; fox # brings entries here & gets back guidance to append to the log. EMERGENT_N ?= 10 EMERGENT_SEED ?= bench-emergent: bootstrap ## blue-moon emergent stress test (3-word triangulation; N=10) PYTHONUNBUFFERED=1 $(PY) scripts/bench_emergent.py \ --n $(EMERGENT_N) \ $(if $(EMERGENT_SEED),--seed $(EMERGENT_SEED),) \ --shards-dir $(SHARDS_DIR) \ --qa-db $(SHARDS_DIR)/qa.db bench-emergent-pending: bootstrap ## print log entries awaiting teacher review $(PY) scripts/bench_emergent.py --print-pending # Cross-model self-play — same question through multiple models, tabulate # audit_mode, agreement, $/grounded. Drives the "ask twice for two # options" pattern that uncloseai-cli's tool_arborist supports. BENCH_CM_QUESTIONS ?= bench/qa_questions_smoke.txt BENCH_CM_OUT ?= bench/cross_model_results BENCH_CM_TIMEOUT ?= 600 BENCH_CM_LIMIT ?= 0 bench-cross-model: bootstrap ## cross-model self-play (Hermes + Qwen on smoke fixture) [BENCH_CM_QUESTIONS=... LIMIT=N BURN=1] PYTHONUNBUFFERED=1 $(PY) bench/cross_model_selfplay.py \ --questions $(BENCH_CM_QUESTIONS) \ --shards-dir $(SHARDS_DIR) \ --out-dir $(BENCH_CM_OUT) \ --top-k $(QUERY_TOP_K) \ --timeout $(BENCH_CM_TIMEOUT) \ --limit $(BENCH_CM_LIMIT) \ $(if $(BURN),--burn,) bench-5s: bootstrap ## 5S battery (Syntax+Semantics+Syllogism+Synthesis+Semiotics) PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub syntax --fixtures bench/fixtures/5s/syntax-v1.jsonl PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub semantics --fixtures bench/fixtures/5s/semantics-v1.jsonl PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub syllogism --fixtures bench/fixtures/5s/syllogism-v1.jsonl PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub synthesis --fixtures bench/fixtures/5s/synthesis-v1.jsonl PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub semiotics --fixtures bench/fixtures/5s/semiotics-v1.jsonl bench-5t: bootstrap ## 5T battery (Transfer Learning+Triangulation+Truthtables+Transitivity+Time) PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5t --sub transfer-learning --fixtures bench/fixtures/5t/transfer-learning-v2.jsonl PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5t --sub triangulation --fixtures bench/fixtures/5t/triangulation-v1.jsonl PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5t --sub truthtables --fixtures bench/fixtures/5t/truthtables-v1.jsonl PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5t --sub transitivity --fixtures bench/fixtures/5t/transitivity-v1.jsonl PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5t --sub time --fixtures bench/fixtures/5t/time-v1.jsonl bench-5t-legacy: bootstrap ## 5T legacy SQD-name (transfer-v1) for Phase-1a digest stability PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5t --sub transfer --fixtures bench/fixtures/5t/transfer-v1.jsonl bench-5f: bootstrap ## 5F battery (Function+Finetuning+Falsification+Formulate+Feedback Loop) PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub function --fixtures bench/fixtures/5f/function-v1.jsonl PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub finetuning --fixtures bench/fixtures/5f/finetuning-v1.jsonl PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub falsification --fixtures bench/fixtures/5f/falsification-v1.jsonl PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub formulate --fixtures bench/fixtures/5f/formulate-v1.jsonl PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub feedback-loop --fixtures bench/fixtures/5f/feedback-loop-v1.jsonl bench-5s-code: bootstrap ## 5S code-carrier (Syntax + Semantics through code-py-ast@v1) PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub syntax \ --fixtures bench/fixtures/5s/syntax-code-v1.jsonl PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub semantics \ --fixtures bench/fixtures/5s/semantics-code-v1.jsonl bench-5s-arithmetic: bootstrap ## 5S arithmetic π* (SQD §14.1; rational arithmetic canonicalizer) PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub syntax \ --fixtures bench/fixtures/5s/syntax-arithmetic-v1.jsonl PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub semantics \ --fixtures bench/fixtures/5s/semantics-arithmetic-v1.jsonl bench-5s-logic-kernel: bootstrap ## 5S logic-kernel π* (SQD §14.3; CNF canonicalizer) PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub syntax \ --fixtures bench/fixtures/5s/syntax-logic-v1.jsonl PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub semantics \ --fixtures bench/fixtures/5s/semantics-logic-v1.jsonl bench-5s-math: bench-5s-arithmetic bench-5s-logic-kernel ## complete math π* surface (arithmetic + logic-kernel) bench-5s-algebra: bootstrap-math ## 5S algebra-symbolic π* (ticket #000030 Phase 1; SymPy expand+srepr canonicalizer) PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub syntax \ --fixtures bench/fixtures/5s/syntax-algebra-symbolic-v1.jsonl PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub semantics \ --fixtures bench/fixtures/5s/semantics-algebra-symbolic-v1.jsonl bench-5s-combinatorics: bootstrap-math ## 5S combinatorics π* (ticket #000032; pure-integer counting kernel) PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub syntax \ --fixtures bench/fixtures/5s/syntax-combinatorics-v1.jsonl PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub semantics \ --fixtures bench/fixtures/5s/semantics-combinatorics-v1.jsonl bench-5s-time-series: bootstrap ## 5S time-series-quantized π* (SQD §13.5; quantized integer-vector canonicalizer) PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub syntax \ --fixtures bench/fixtures/5s/syntax-time-series-v1.jsonl PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub semantics \ --fixtures bench/fixtures/5s/semantics-time-series-v1.jsonl bench-5s-tabular: bootstrap ## 5S tabular-pinned π* (#000030/Phase tabular; declared-schema 2D structured-data canonicalizer) PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub syntax \ --fixtures bench/fixtures/5s/syntax-tabular-v1.jsonl PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub semantics \ --fixtures bench/fixtures/5s/semantics-tabular-v1.jsonl bench-5s-calculus-limit: bootstrap ## 5S calculus-limit π* (#000030 Phase 4) PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub syntax \ --fixtures bench/fixtures/5s/syntax-calculus-limit-v1.jsonl PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub semantics \ --fixtures bench/fixtures/5s/semantics-calculus-limit-v1.jsonl bench-5s-calculus-series: bootstrap ## 5S calculus-series π* (#000030 Phase 5) PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub syntax \ --fixtures bench/fixtures/5s/syntax-calculus-series-v1.jsonl PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub semantics \ --fixtures bench/fixtures/5s/semantics-calculus-series-v1.jsonl bench-5s-linear-algebra: bootstrap ## 5S linear-algebra π* (#000030 Phase 6) PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub syntax \ --fixtures bench/fixtures/5s/syntax-linear-algebra-v1.jsonl PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub semantics \ --fixtures bench/fixtures/5s/semantics-linear-algebra-v1.jsonl bench-5s-function-sampled: bootstrap ## 5S function-sampled π* (#000030 Phase 7; SymPy → time-series bridge) PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub syntax \ --fixtures bench/fixtures/5s/syntax-function-sampled-v1.jsonl PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub semantics \ --fixtures bench/fixtures/5s/semantics-function-sampled-v1.jsonl bench-real-shard: bootstrap ## #000026 Phase 2 — real-shard workload baseline (latency, audit, primary-source use) PYTHONUNBUFFERED=1 $(PY) -m bench.scripts.real_shard_baseline \ --shards-dir $${ARBORIST_SHARDS_DIR:-$$HOME/.arborist/shards} \ --burn # #000028 follow-up — extract LLM-divergence audit events as 5F # Falsification fixtures. Reads providence_canonical_witness audit # events from a qa.db and writes them to a witness-derived fixture # JSONL the existing 5F-falsification-live runner can consume. WITNESS_QA_DB ?= $$HOME/.arborist/shards/qa.db WITNESS_OUT ?= bench/fixtures/5f/falsification-witness-v1.jsonl bench-witness-divergence: bootstrap ## extract LLM-divergence events as 5F Falsification fixtures PYTHONUNBUFFERED=1 $(PY) -m bench.scripts.witness_to_5f \ --qa-db $(WITNESS_QA_DB) \ --out $(WITNESS_OUT) # End-to-end witness sweep against real shards + Hermes (#000028 # validation). Fires 8 canonical-shape questions with witness=on, # records the agreement matrix per question. Pair with # bench-witness-divergence to extract any divergence as 5F fixtures. WITNESS_SWEEP_OUT ?= bench/results/witness-sweep.json bench-witness-sweep: bootstrap ## fire witness mode on canonical-question corpus against real shards PYTHONUNBUFFERED=1 $(PY) -m bench.scripts.witness_sweep \ --shards-dir $${ARBORIST_SHARDS_DIR:-$$HOME/.arborist/shards} \ --out $(WITNESS_SWEEP_OUT) # function-sampled@v1 demo (#000030 Phase 7) — closes the # opencompletion activity24-math-plot.yaml loop. Canonical bytes # always print; PNG is optional (provide PNG=/path/to/file.png). # Default expression range covers a full sine period. demo-plot: bootstrap ## function-sampled@v1 demo: make demo-plot Q='sin(x)' [PNG=/tmp/out.png] @if [ -z "$(Q)" ]; then echo "usage: make demo-plot Q='' [PNG=/tmp/out.png]" >&2; exit 2; fi PYTHONUNBUFFERED=1 $(PY) -m bench.scripts.demo_plot \ --expr "$(Q)" \ $(if $(PNG),--png $(PNG),) # v8 ForkScore — runs the full bench-suite, then scores the fresh # child output against a previously-pinned PARENT artifact. Default # parent is bench/results/baseline-suite.json (operator pins this # once via `make bench-fork-baseline`); override per-call with # PARENT=path. Fox's iteration loop: # # make bench-fork-baseline # one-shot — pins current state # # make bench-fork-score # compare hacked branch to baseline # # exit 1 on REJECT (CI-gateable) FORK_PARENT ?= bench/results/baseline-suite.json FORK_CHILD ?= bench/results/current-suite.json FORK_REPORT ?= bench/results/fork_score_report.json bench-fork-baseline: bootstrap ## pin current bench-suite output as ForkScore parent @mkdir -p bench/results PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --all \ --out $(FORK_PARENT) @echo ">> baseline pinned: $(FORK_PARENT)" bench-fork-score: bootstrap ## #000012 Phase 1b — score current bench output vs $(FORK_PARENT) @mkdir -p bench/results PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --all \ --out $(FORK_CHILD) $(ARBORIST) substrate score \ --parent $(FORK_PARENT) \ --child $(FORK_CHILD) \ --out $(FORK_REPORT) bench-5f-formulate-live: bootstrap ## 5F Formulate via live arborist.qa.parse_claims (Phase 1b.2) PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub formulate \ --fixtures bench/fixtures/5f/formulate-live-v1.jsonl bench-5f-feedback-loop-live: bootstrap ## 5F Feedback Loop via live arborist memory + audit chain (Phase 1b.2) PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub feedback-loop \ --fixtures bench/fixtures/5f/feedback-loop-live-v1.jsonl bench-5f-function-live: bootstrap ## 5F Function via live arborist.qa.parse_claims (Phase 1b.2) PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub function \ --fixtures bench/fixtures/5f/function-live-v1.jsonl bench-5f-finetuning-live: bootstrap ## 5F Finetuning via real selfmodel store/claims_for round-trip (Phase 1b.2) PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub finetuning \ --fixtures bench/fixtures/5f/finetuning-live-v1.jsonl bench-5f-falsification-live: bootstrap ## 5F Falsification via real arborist.qa.verify.verify_quotes (Phase 1b.2) PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub falsification \ --fixtures bench/fixtures/5f/falsification-live-v1.jsonl bench-5f-live: bench-5f-formulate-live bench-5f-feedback-loop-live bench-5f-function-live bench-5f-finetuning-live bench-5f-falsification-live ## all 5F Phase-1b.2 live wire-ups # #000046 Phase 1 — HARD live-path Falsification tier. Every fixture is # a near-miss the correct verdict is UNGROUNDED on; verify_quotes # over-grounds 2 of 12 today → rate 10/12 = 0.833 at HEAD (#000046's # paraphrase numeric gate caught 2, #000048 step 2.1's entity # salient-token gate caught 4 more; the last 2 are recombination # STRICT_PARAPHRASE — #000048 step 2.2). A real below-ceiling baseline. # NOT part of bench-5f / bench-fork-baseline. bench-fork-baseline-hard # pins the rate so a future verifier upgrade shows up as a larger # positive gamma*Delta5f term. FORK_PARENT_HARD ?= bench/results/baseline-falsification-hard.json # The hard pack has its expected failures (2/12 over-grounds), so the # runner exits 1 — that's the below-ceiling baseline working as # designed, not a build error. `|| true` keeps the make target green; # the JSON / report is written regardless (the --out write precedes # the runner's nonzero-on-failures exit). bench-5f-falsification-hard: bootstrap ## #000046 — HARD Falsification near-miss pack (rate < 1.0 at HEAD by design) PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub falsification \ --fixtures bench/fixtures/5f/falsification-hard-v1.jsonl || true bench-fork-baseline-hard: bootstrap ## #000046 — pin the below-ceiling hard-Falsification rate as a ForkScore parent @mkdir -p bench/results PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub falsification \ --fixtures bench/fixtures/5f/falsification-hard-v1.jsonl --out $(FORK_PARENT_HARD) || true @echo ">> below-ceiling parent pinned: $(FORK_PARENT_HARD) (2/12 over-grounds expected → runner exits 1; the JSON is written)" # #000046 Phase 2 — HARD Formulate tier. Prose that parse_pointer_claims # should segment a particular way. Was 4/12 (line/bullet-only parser # merged multi-claim lines / split wrapped bullets); #000048 step 2.4's # clause segmenter closes all 8 → rate 12/12 now (pack at ceiling — a # harder Formulate tier would re-open headroom). Runner still exits 0 # here (0 failures), but the target is kept for the operator-pack # convention (not in bench-5f / runner --all). bench-5f-formulate-hard: bootstrap ## #000046 Phase 2 / #000048 step 2.4 — HARD Formulate clause-segmentation pack (12/12 after step 2.4) PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub formulate \ --fixtures bench/fixtures/5f/formulate-hard-v1.jsonl || true # #000025 §10.11 — persistent SelfModel-chain lineage. Each run of # bench-5f-selfmodel-snapshot runs the 5S/5T/5F embedded packs and # appends ONE chained snapshot (capability claims = the battery # rates) to SELFMODEL_CHAIN_DB. Run it >= 2x, then # bench-5f-finetuning-shardchain measures improvement between the two # most-recent snapshots — a real cross-run lineage, not a synthetic # parent/child pair. Operator targets: NOT part of `make bench-5f`, # `make test`, or a fresh checkout. SELFMODEL_CHAIN_DB ?= $(SHARDS_DIR)/selfmodel-chain.db bench-5f-selfmodel-snapshot: bootstrap ## #000025 §10.11 — append one SelfModel snapshot (rates as claims) to the chain shard PYTHONUNBUFFERED=1 ARBORIST_SELFMODEL_CHAIN_DB=$(SELFMODEL_CHAIN_DB) \ $(PY) bench/scripts/selfmodel_chain_snapshot.py bench-5f-finetuning-shardchain: bootstrap ## #000025 §10.11 — Finetuning over the two latest chain snapshots (run bench-5f-selfmodel-snapshot >=2x first) PYTHONUNBUFFERED=1 ARBORIST_SELFMODEL_CHAIN_DB=$(SELFMODEL_CHAIN_DB) \ $(PY) -m bench.batteries.runner --battery 5f --sub finetuning \ --fixtures bench/fixtures/5f/finetuning-shardchain-v1.jsonl # #000025 §10.14 — ForkScore threshold-calibration handoff to #000012. # Runs the canonical 5S/5T/5F packs + the 5F live packs, reports # baseline rates / granularity / floor-constant sanity checks. # Pure measurement; output is the deliverable #000012 cites. FIVEF_CALIBRATION_OUT ?= bench/results/5f-threshold-calibration-$(shell date -u +%Y-%m-%d).md bench-5f-threshold-calibration: bootstrap ## #000025 §10.14 — 5S/5T/5F → ForkScore threshold calibration report PYTHONUNBUFFERED=1 $(PY) bench/scripts/fivef_threshold_calibration.py \ --out $(FIVEF_CALIBRATION_OUT) bench-5r: bootstrap ## 5R battery (React+Rearrange+Restore+Replicate+Resonate) PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5r --sub react --fixtures bench/fixtures/5r/react-v1.jsonl PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5r --sub rearrange --fixtures bench/fixtures/5r/rearrange-v1.jsonl PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5r --sub restore --fixtures bench/fixtures/5r/restore-v1.jsonl PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5r --sub replicate --fixtures bench/fixtures/5r/replicate-v1.jsonl PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5r --sub resonate --fixtures bench/fixtures/5r/resonate-v1.jsonl bench-5r-react-live: bootstrap ## 5R React via real audit_events on a temp shard (Phase 1b.2) PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5r --sub react \ --fixtures bench/fixtures/5r/react-live-v1.jsonl bench-5r-restore-live: bootstrap ## 5R Restore via real audit chain query (Phase 1b.2) PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5r --sub restore \ --fixtures bench/fixtures/5r/restore-live-v1.jsonl bench-5r-live: bench-5r-react-live bench-5r-restore-live ## all 5R Phase-1b.2 live wire-ups bench-5s5t: bench-5s bench-5t ## 5S + 5T (Phase-1b vocabulary) bench-5s5t5f: bench-5s bench-5t bench-5f ## 5S + 5T + 5F (operational triad) bench-suite: bench-5s bench-5t bench-5f bench-5r ## complete Dav1DPrometheus suite (5S + 5T + 5F + 5R) verify-shards: bootstrap ## cross-shard Merkle round-trip on a random sample $(ARBORIST) --shards-dir $(SHARDS_DIR) verify -n $(VERIFY_N) analyze-shards: bootstrap ## cross-shard compression spectrum + audit integrity $(ARBORIST) --shards-dir $(SHARDS_DIR) analyze # Audit-chain integrity probe. Cheap (set-arithmetic, no per-row hash walk), # scriptable, and far faster than `analyze`. 0 = chain intact: a single # linear chain with one genesis. Counts, summed: # - dangling prev_event_hash references (a prev that no row's event_hash has) # - forked parents (a prev_event_hash claimed by >1 row — two events built # on the same head; the bug behind qa.db seq 7724/7725) # - extra genesis rows (more than one row with prev_event_hash IS NULL) # The plain LEFT JOIN check alone misses forks — the forked parent IS present. define CHAIN_CHECK_SQL SELECT (SELECT COUNT(*) FROM audit_events a1 LEFT JOIN audit_events a2 ON a2.event_hash = a1.prev_event_hash WHERE a1.prev_event_hash IS NOT NULL AND a2.event_hash IS NULL) + (SELECT COUNT(*) FROM (SELECT 1 FROM audit_events WHERE prev_event_hash IS NOT NULL GROUP BY prev_event_hash HAVING COUNT(*) > 1)) + MAX(0, (SELECT COUNT(*) FROM audit_events WHERE prev_event_hash IS NULL) - 1) AS chain_breaks endef export CHAIN_CHECK_SQL chain-check: ## audit-chain integrity count for $(DB) (0 = intact: linear chain, one genesis) @printf '%s ' "$(DB)"; sqlite3 $(DB) "$$CHAIN_CHECK_SQL" chain-check-shards: ## audit-chain integrity count for every *.db in $(SHARDS_DIR) @for db in $(SHARDS_DIR)/*.db; do \ printf '%s ' "$$db"; sqlite3 "$$db" "$$CHAIN_CHECK_SQL"; \ done # Parallel per-shard distill: one process per shard. No DB contention # because each shard is its own file. distill-shards-parallel: bootstrap ## one distill process per shard (parallel) @for shard in $(SHARDS_DIR)/*.db; do \ $(ARBORIST) --db $$shard distill --process first-sentence-v1 --kind surface & \ done; wait # TF-IDF cores serve as enriched titles for retrieval — distinctive # low-frequency terms that surface in body text get promoted into # something queryable without a real title match. distill-shards-tfidf-parallel: bootstrap ## TF-IDF cores per shard, in parallel @for shard in $(SHARDS_DIR)/*.db; do \ $(ARBORIST) --db $$shard distill --process tfidf-keywords-v1 --kind surface & \ done; wait ingest: ingest-cur ## default ingest = cur (use ingest-old or *-parallel for full) # Grok account-export ETL. # Point GROK_EXPORT at the directory xAI delivered (the one containing # `ttl/30d/export_data//prod-grok-backend.json`). The source class # auto-walks down to find the JSON. GROK_EXPORT ?= $(HOME)/Downloads/ab8ef1f0-0d08-4f87-89c2-d4509e18115b # Grok lives in its own shard inside the attach-forever cluster so cross- # shard queries (`make query`) see it alongside the Wikipedia 2003 corpus. # Single shard (rank 0/1) — Grok conversations are private and small. GROK_SHARD := $(SHARDS_DIR)/grok.db ingest-grok-attached: bootstrap ## ingest Grok conversations into $(GROK_SHARD) @mkdir -p $(SHARDS_DIR) $(ARBORIST) --db $(GROK_SHARD) ingest --source grok_export --path $(GROK_EXPORT) --resume ingest-grok-media-attached: bootstrap ## ingest Grok media prompts into $(GROK_SHARD) @mkdir -p $(SHARDS_DIR) $(ARBORIST) --db $(GROK_SHARD) ingest --source grok_media --path $(GROK_EXPORT) --resume # ---------------------------------------------------------------------------- # Phase IV (2006+) Wikipedia XML dumps. Drop-in for any dated snapshot in # the dumps.wikimedia.org/archive tree by overriding WP_XML_YEAR/MONTH/DATE # (and WP_XML_LANG for non-English wikis). # # Defaults point at enwiki 20101011 (2010-11 archive), the largest snapshot # in the archive — 6.2 GB compressed, ~3.4M articles. # Other useful snapshots: # make fetch-xml WP_XML_YEAR=2006 WP_XML_MONTH=2006-07 WP_XML_DATE=20061104 # make fetch-xml WP_XML_YEAR=2006 WP_XML_MONTH=2006-12 WP_XML_DATE=20061130 # ---------------------------------------------------------------------------- WP_XML_LANG ?= en WP_XML_YEAR ?= 2010 WP_XML_MONTH ?= 2010-11 WP_XML_DATE ?= 20101011 WP_XML_BASE_URL ?= https://dumps.wikimedia.org/archive/$(WP_XML_YEAR)/$(WP_XML_MONTH)/$(WP_XML_LANG)wiki/$(WP_XML_DATE) WP_XML_FILE ?= $(WP_XML_LANG)wiki-$(WP_XML_DATE)-pages-articles.xml.bz2 WP_XML := $(DATA_DIR)/$(WP_XML_FILE) WP_ABSTRACT_FILE ?= $(WP_XML_LANG)wiki-$(WP_XML_DATE)-abstract.xml WP_ABSTRACT := $(DATA_DIR)/$(WP_ABSTRACT_FILE) $(WP_XML): | $(DATA_DIR) @echo ">> fetching $(WP_XML_BASE_URL)/$(WP_XML_FILE)" curl -fL --retry 3 -o $@ "$(WP_XML_BASE_URL)/$(WP_XML_FILE)" $(WP_ABSTRACT): | $(DATA_DIR) @echo ">> fetching $(WP_XML_BASE_URL)/$(WP_ABSTRACT_FILE)" curl -fL --retry 3 -o $@ "$(WP_XML_BASE_URL)/$(WP_ABSTRACT_FILE)" fetch-xml: $(WP_XML) ## download Phase IV XML cur dump (default: enwiki 20101011, 6.2 GB) fetch-abstract: $(WP_ABSTRACT) ## download Phase IV abstract.xml (default: enwiki 20101011, ~3 GB) ingest-xml: bootstrap fetch-xml ## ingest INGEST_LIMIT pages from $(WP_XML) $(ARBORIST) --db $(DB) ingest --source wikipedia_xml --path $(WP_XML) --limit $(INGEST_LIMIT) ingest-xml-history: bootstrap ## ingest every revision (multi-revision mode); set WP_XML to a pages-meta-history file $(ARBORIST) --db $(DB) ingest --source wikipedia_xml_history --path $(WP_XML) --limit $(INGEST_LIMIT) # Sharded XML ingest into the attach-forever cluster — same pattern as # ingest-cur-attached. One process per shard, one SQLite file per shard, # zero WAL contention. ingest-xml-attached: bootstrap fetch-xml ## sharded XML ingest, one process per shard @mkdir -p $(SHARDS_DIR) @for i in $$(seq 0 $$(($(SHARDS) - 1))); do \ $(ARBORIST) ingest --source wikipedia_xml --path $(WP_XML) \ --shards-dir $(SHARDS_DIR) --shard $$i/$(SHARDS) & \ done; wait ingest-abstract: bootstrap fetch-abstract ## ingest INGEST_LIMIT abstract docs from $(WP_ABSTRACT) $(ARBORIST) --db $(DB) ingest --source wikipedia_abstract --path $(WP_ABSTRACT) --limit $(INGEST_LIMIT) # ---------------------------------------------------------------------------- # Self-ingest: arborist consults its own source code as a queryable corpus. # Re-running picks up new commits — same path + new content gets a fresh # document_root chained to the prior version via a `supersedes` edge, so the # audit chain grows as the repo grows. Lands in a dedicated shard file so it # doesn't compete with the wikipedia/grok shards' WAL writer lock. # # Override SELF_REPO to ingest a different repo's tree. # ---------------------------------------------------------------------------- SELF_REPO ?= $(CURDIR) SELF_SHARD := $(SHARDS_DIR)/arborist-self.db ingest-self: bootstrap ## ingest this repo's HEAD into a dedicated shard @mkdir -p $(SHARDS_DIR) $(ARBORIST) --db $(SELF_SHARD) ingest --source git_repo --path $(SELF_REPO) # Self-reference: promote STRICT live providence records past the # kindergarten window into each shard's documents table. Each shard # self-promotes only its own records; cross-shard sharing happens # via the existing shards-dir UNION at retrieval time. Run on a cron # (e.g. hourly) to keep the substrate fresh. # See docs/self-reference-thought-chains-design.md. KG_SECONDS ?= 3600 ingest-self-providence: bootstrap ## promote STRICT live providence records into the document corpus [KG_SECONDS=3600] @mkdir -p $(SHARDS_DIR) @for db in $(SHARDS_DIR)/*.db; do \ echo ">> promoting providence records: $$db"; \ $(ARBORIST) --db $$db ingest --source providence --kindergarten-seconds $(KG_SECONDS); \ done # Generic git-repo ingest: aim it at any local clone via GIT_REPO=... GIT_REPO ?= $(CURDIR) GIT_SHARD := $(SHARDS_DIR)/$(notdir $(GIT_REPO))-git.db ingest-git: bootstrap ## ingest GIT_REPO= into its own shard @mkdir -p $(SHARDS_DIR) $(ARBORIST) --db $(GIT_SHARD) ingest --source git_repo --path $(GIT_REPO) # Generic hg-repo ingest. HG_REPO=. HG_REPO ?= HG_SHARD := $(SHARDS_DIR)/$(notdir $(HG_REPO))-hg.db ingest-hg: bootstrap ## ingest HG_REPO= (mercurial) into its own shard @if [ -z "$(HG_REPO)" ]; then echo "usage: make ingest-hg HG_REPO=/path/to/repo" >&2; exit 2; fi @mkdir -p $(SHARDS_DIR) $(ARBORIST) --db $(HG_SHARD) ingest --source hg_repo --path $(HG_REPO) verify: bootstrap ## round-trip Merkle proofs for VERIFY_N random documents $(ARBORIST) --db $(DB) verify -n $(VERIFY_N) search: bootstrap ## keyword search; override SEARCH_Q (or pass Q=...) $(ARBORIST) --db $(DB) search '$(if $(Q),$(Q),$(SEARCH_Q))' stats: bootstrap ## counts: documents, chunks, edges, audit chain $(ARBORIST) --db $(DB) stats # --- cold-object-store tier (ticket #000061) --------------------------------- # # Push chunk bodies to an S3-compatible bucket so the corpus can grow past # one machine while the Merkle tree stays intact. One backend covers AWS S3, # DO Spaces, R2, B2, GCS (S3 interop), MinIO — different `endpoint_url`. # # Required env (set in your shell, never hard-code into Makefile vars): # AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY — standard boto3 discovery # ARBORIST_COLD_ENDPOINT_URL — e.g. https://nyc3.digitaloceanspaces.com # ARBORIST_COLD_BUCKET — e.g. arborist-corpus # ARBORIST_COLD_REGION — optional (DO Spaces is region-agnostic) # # Operation Voyeur: credentials NEVER appear on argv, in logs, in audit body. # Only endpoint URL + bucket name surface; keys live in env / ~/.aws/credentials. bootstrap-object-store: bootstrap ## install [object-store] (boto3) into the venv $(PIP) install -e '.[object-store]' cold-pack: bootstrap ## bundle hot chunks into tar.zst pack(s) on bucket (≤4.4 GB DVD-R safe-fit each by default) $(ARBORIST) --db $(DB) cold pack \ $(if $(MAX_CHUNKS),--max-chunks $(MAX_CHUNKS),) \ $(if $(MAX_PACK_BYTES),--max-pack-bytes $(MAX_PACK_BYTES),) \ $(if $(LOCAL_DIR),--local-dir $(LOCAL_DIR),) \ $(if $(ALLOW_LICENSE_CLASS),--allow-license-class $(ALLOW_LICENSE_CLASS),) cold-pack-dvd: bootstrap ## write packs to LOCAL_DIR for burning (no S3 upload); each pack ≤4.7 GB @if [ -z "$(LOCAL_DIR)" ]; then echo "LOCAL_DIR= required"; exit 2; fi $(ARBORIST) --db $(DB) cold pack --no-push --local-dir $(LOCAL_DIR) @echo ">> packs written to $(LOCAL_DIR) — burn each .tar.zst with growisofs:" @echo ">> growisofs -dvd-compat -Z /dev/sr0 $(LOCAL_DIR)/arborist-pack-.tar.zst" cold-unpack: bootstrap ## pull pack PACK= and restore chunks locally @if [ -z "$(PACK)" ]; then echo "PACK= required"; exit 2; fi $(ARBORIST) --db $(DB) cold unpack $(PACK) cold-stats: bootstrap ## bucket summary: pack count, total bytes, endpoint $(ARBORIST) --db $(DB) cold stats cold-list: bootstrap ## enumerate packs (pack_hash, size, chunk_count) for new-peer hydration $(ARBORIST) --db $(DB) cold list # Per-shard parallel pack run. Shards are independent SQLite files; packing # them in parallel scales near-linearly with CPU/network up to the point # where summed worker RSS exceeds available RAM (4-way attempt OOM'd at # fork time on a 31 GB box with 4 GB available, 2-way completed cleanly). # SHARDS_DIR defaults to ~/.arborist/shards; SHARDS_PATTERN filters which # shard DBs participate. # # COLD_PACK_JOBS is RAM-aware by default — auto-computed from # `free -m` at run time. Override with `make cold-pack-all COLD_PACK_JOBS=N` # if you have a known-good number. # # COLD_PACK_PER_WORKER_MB — peak RSS a single worker is observed to use # during the metadata dump phase. Set # conservatively from real-bench observation. # COLD_PACK_HEADROOM_MB — keep this much RAM unallocated for the OS # + buff/cache + other processes. # COLD_PACK_JOBS_MAX — never exceed this even on a huge box. SHARDS_DIR ?= $(HOME)/.arborist/shards SHARDS_PATTERN ?= [0-9][0-9][0-9].db COLD_PACK_PER_WORKER_MB ?= 5500 COLD_PACK_HEADROOM_MB ?= 2000 COLD_PACK_JOBS_MAX ?= 8 # Internal helper: compute the RAM-aware concurrency. Honours an explicit # COLD_PACK_JOBS if the caller passed one; otherwise reads free RAM. # This is a recipe-line sub-command (single shell), so all variables are # bash-scoped — no `$$` doubling needed in the actual command lines below # because we run them inline. define COLD_PACK_PICK_JOBS if [ -n "$(COLD_PACK_JOBS)" ]; then \ JOBS=$(COLD_PACK_JOBS); \ echo ">> COLD_PACK_JOBS=$$JOBS (explicit)"; \ else \ AVAIL_MB=$$(free -m | awk '/^Mem:/ {print $$7}'); \ JOBS=$$(( ($$AVAIL_MB - $(COLD_PACK_HEADROOM_MB)) / $(COLD_PACK_PER_WORKER_MB) )); \ if [ $$JOBS -lt 1 ]; then JOBS=1; fi; \ if [ $$JOBS -gt $(COLD_PACK_JOBS_MAX) ]; then JOBS=$(COLD_PACK_JOBS_MAX); fi; \ echo ">> COLD_PACK_JOBS=$$JOBS (auto: avail=$${AVAIL_MB}MB - headroom=$(COLD_PACK_HEADROOM_MB)MB / per-worker=$(COLD_PACK_PER_WORKER_MB)MB, capped at $(COLD_PACK_JOBS_MAX))"; \ fi endef cold-pack-all: bootstrap ## parallel cold-pack, RAM-aware (override with COLD_PACK_JOBS=N) @$(COLD_PACK_PICK_JOBS); \ echo ">> packing shards in $(SHARDS_DIR)/$(SHARDS_PATTERN) — $$JOBS-way parallel"; \ find $(SHARDS_DIR) -maxdepth 1 -name '$(SHARDS_PATTERN)' -print0 | \ xargs -0 -n 1 -P $$JOBS -I {} \ $(ARBORIST) --db {} cold pack \ $(if $(ALLOW_LICENSE_CLASS),--allow-license-class $(ALLOW_LICENSE_CLASS),) @echo ">> all shards packed; final bucket state:" @$(ARBORIST) cold stats cold-hydrate: bootstrap ## genesis a fresh peer from cloud, M-aware: pull every metadata pack and route into HYDRATE_DIR/00N.db × HYDRATE_M. HYDRATE_MODE=full (default) | just-enough @if [ -z "$(HYDRATE_DIR)" ]; then echo "HYDRATE_DIR= required"; exit 2; fi @if [ -z "$(HYDRATE_M)" ]; then echo "HYDRATE_M= required (matches #000065 canonical M)"; exit 2; fi @mkdir -p $(HYDRATE_DIR) @MODE="$${HYDRATE_MODE:-full}"; \ case "$$MODE" in \ full) MODE_FLAG="--full" ;; \ just-enough) MODE_FLAG="--just-enough" ;; \ *) echo "HYDRATE_MODE=$$MODE invalid (full | just-enough)"; exit 2 ;; \ esac; \ JOBS="$(COLD_PACK_JOBS)"; [ -z "$$JOBS" ] && JOBS=1; \ echo ">> COLD_PACK_JOBS=$$JOBS (serial default: M-aware hydrate routes every pack into all M shared target shards, so parallel workers contend on the same files — #54. Override COLD_PACK_JOBS=N at your own risk.)"; \ echo ">> discovering metadata packs in bucket"; \ hashes=$$($(ARBORIST) cold list --no-manifest \ | $(VENV)/bin/python -c \ "import sys, json; d=json.load(sys.stdin); print(' '.join(p['pack_hash'] for p in d['packs'] if p['kind']=='metadata'))"); \ if [ -z "$$hashes" ]; then echo "no metadata packs in bucket"; exit 3; fi; \ echo ">> $$JOBS-way parallel pull of metadata packs (M=$(HYDRATE_M), mode=$$MODE)"; \ printf "%s\n" $$hashes | \ xargs -n 1 -P $$JOBS -I {} \ $(ARBORIST) cold unpack {} \ --hydrate-shards-dir $(HYDRATE_DIR) \ --hydrate-M $(HYDRATE_M) \ $$MODE_FLAG @if [ "$${HYDRATE_MODE:-full}" = "full" ] && [ "$${HYDRATE_REBUILD_FTS:-auto}" != "0" ]; then \ if [ "$${HYDRATE_REBUILD_FTS:-auto}" = "1" ] || ! $(ARBORIST) cold verify --shards-dir $(HYDRATE_DIR) >/dev/null 2>&1; then \ echo ">> FTS not searchable (no fts pack, or rebuild forced) → rebuilding from content (parallel)"; \ $(ARBORIST) cold rebuild-fts --shards-dir $(HYDRATE_DIR); \ else \ echo ">> FTS restored from packs and verified searchable — skipping rebuild"; \ fi; \ fi @echo ">> final verify (content materialized + searchable; fails loudly if not)" @$(ARBORIST) cold verify --shards-dir $(HYDRATE_DIR) @echo ">> hydration complete:" @ls -lh $(HYDRATE_DIR)/*.db 2>/dev/null # ---- Live-snapshot clone tier (raw .db on Spaces; skip pack/unpack entirely) # Producer streams each shard via SQLite Online Backup API + multipart upload; # consumer pulls raw .db files in parallel. Recovery collapses to ~download time # (the FTS index travels inside the .db, no rebuild needed). JUST_ENOUGH=1 # strips chunks.content into per-chunk blobs/ for SPV / mobile peers. cold-stream-snapshot: bootstrap ## producer: snapshot shards to bucket as raw .db [SHARDS_DIR=path JUST_ENOUGH=1 SNAPSHOT_ID=...] @if [ -z "$(SHARDS_DIR)" ]; then echo "SHARDS_DIR= required"; exit 2; fi $(ARBORIST) cold stream-snapshot --shards-dir $(SHARDS_DIR) \ $(if $(JUST_ENOUGH),--just-enough,) \ $(if $(SNAPSHOT_ID),--snapshot-id $(SNAPSHOT_ID),) cold-clone: bootstrap ## consumer: clone raw .db shards from bucket (fast, no restore) [SHARDS_DIR=path SNAPSHOT_ID=... WORKERS=N] @if [ -z "$(SHARDS_DIR)" ]; then echo "SHARDS_DIR= required"; exit 2; fi @mkdir -p $(SHARDS_DIR) $(ARBORIST) cold clone --shards-dir $(SHARDS_DIR) \ $(if $(SNAPSHOT_ID),--snapshot-id $(SNAPSHOT_ID),) \ $(if $(WORKERS),--workers $(WORKERS),) @echo ">> verifying cloned shards" @$(ARBORIST) cold verify --shards-dir $(SHARDS_DIR) cold-pack-all-dvd: bootstrap ## same fan-out for DVD burning (no S3); RAM-aware @if [ -z "$(LOCAL_DIR)" ]; then echo "LOCAL_DIR= required"; exit 2; fi @mkdir -p $(LOCAL_DIR) @$(COLD_PACK_PICK_JOBS); \ echo ">> packing shards in $(SHARDS_DIR)/$(SHARDS_PATTERN) to $(LOCAL_DIR) — $$JOBS-way parallel, no S3"; \ find $(SHARDS_DIR) -maxdepth 1 -name '$(SHARDS_PATTERN)' -print0 | \ xargs -0 -n 1 -P $$JOBS -I {} \ $(ARBORIST) --db {} cold pack --no-push --local-dir $(LOCAL_DIR) @echo ">> all packs in $(LOCAL_DIR):" @ls -lh $(LOCAL_DIR)/arborist-pack-*.tar.zst test: bootstrap ## run pytest suite (excludes opt-in crawler tests) $(VENV)/bin/pytest -q --ignore=tests/crawler -n auto # CI-scoped suite: same as `make test` but also drops the wikipedia # ingest tests (`test_wikipedia_old.py`, `test_wikipedia_xml.py`). # Wikipedia ingest is exercised end-to-end on real shards via # `make ingest-cur` / `make ingest-xml`; running synthetic-fixture # coverage on every CI push duplicates that surface for no signal # the runtime path doesn't already provide. Local `make test` stays # comprehensive — this target is for the gate, not the dev loop. test-ci: bootstrap ## CI gate: full suite minus crawler + wikipedia ingest $(VENV)/bin/pytest -q \ --ignore=tests/crawler \ --ignore=tests/test_wikipedia_old.py \ --ignore=tests/test_wikipedia_xml.py \ -n auto # Crawler tests are off-by-default — they hit the network in many cases # and require the heavy [crawler] extras (aiohttp, bs4, lxml, etc.). # Bootstrap installs the extras into the existing venv idempotently. bootstrap-crawler: bootstrap ## install [crawler] extras into the venv $(PIP) install -e '.[crawler]' # SymPy substrate for algebra/calculus π* canonicalizers (ticket #000030). # Already pulled in transitively by `make bootstrap` via the [dev] extras; # this target is the explicit opt-in for minimal-install users. bootstrap-math: bootstrap ## install [math] extras (sympy) into the venv $(PIP) install -e '.[math]' # #000049 Phase 2 — sentence-pair NLI runtime for the SHADOW path # (arborist/qa/nli/). ~600 MB (transformers + CPU torch); hard out of # core / dev so a fresh checkout stays python3.12 + venv + sqlite3. # Installs the extra, then warms the pinned checkpoint download so the # first `make bench-nli-shadow` doesn't pay for it. Operator target — # NOT part of `make bootstrap`, `make test`, or a fresh checkout. bootstrap-nli: bootstrap ## install [nli] extras + warm the pinned NLI checkpoint $(PIP) install -e '.[nli]' $(PY) -c "from arborist.qa.nli import ShadowNLI; r=ShadowNLI(); r._ensure_loaded(); print('nli runtime ·', 'available:', r.available, '· backend:', r.backend, '· device:', r.device, '·', r._reason)" # #000049 §7 #22 — lean NLI compute-node bootstrap: a venv + `[nli]` ONLY # (no `[dev]` extras — no crawler/hessian/vec/sympy). For a GPU box that # just runs NLI sweeps: on a CUDA host PyPI's torch wheel is the CUDA # build, so `ShadowNLI` auto-detects cuda (ARBORIST_NLI_DEVICE overrides) # and `make bench-nli-shadow` / `make export-nli-onnx` / the candidate # bench all run on the GPU with no further wiring. Measured: an 82M # cross-encoder, batched, on a 4090 ≈ 0.09 ms/pair (~350x onnx-int8-cpu, # ~1300x torch-cpu-batch1) → a full bench-qa shadow sweep is <1 s of NLI. bootstrap-nli-only: ## #000049 §7 #22 — minimal venv + [nli] only (GPU/CPU compute-node setup; no [dev]) [ -d $(VENV) ] || $(PYTHON) -m venv $(VENV) $(PIP) install -q -U pip $(PIP) install -e '.[nli]' pytest $(PY) -c "import torch; from arborist.qa.nli import ShadowNLI; r=ShadowNLI(); r._ensure_loaded(); print('torch', torch.__version__, '· cuda', torch.cuda.is_available(), (torch.cuda.get_device_name(0) if torch.cuda.is_available() else '')); print('nli runtime ·', 'available:', r.available, '· backend:', r.backend, '· device:', r.device, '·', r._reason)" # #000049 §3 speedup — export the pinned shadow-NLI checkpoint to ONNX # (+ int8 dynamic quantization), into ~/.arborist/models/nli//onnx/. # ShadowNLI._ensure_loaded auto-prefers the export if present (~2-4x on # CPU, drops the torch forward path; on cuda uses CUDAExecutionProvider). # Run once after `make bootstrap-nli`. SHADOW infra — same checkpoint, # same labels, nothing about audit_mode changes. export-nli-onnx: bootstrap ## #000049 §3 — export the pinned shadow-NLI checkpoint to ONNX (int8) PYTHONUNBUFFERED=1 $(PY) bench/scripts/export_nli_onnx.py # #000049 §7 #28 — deterministic inference-backend A/B: same pinned # checkpoint under torch / onnx-int8 / tinygrad. Gate is AGREEMENT with # the torch reference first, latency second. SHADOW ONLY (writes only # bench/results/nli-backend-ab.json). tinygrad is not an arborist dep — # run this on the GPU producer box after installing tinygrad + an ONNX # export (`make export-nli-onnx`). Reports backends honestly as # unavailable when they don't load (e.g. tinygrad frontend op-coverage). bench-nli-backends: bootstrap ## #000049 §7 #28 — torch vs onnx-int8 vs tinygrad agreement+latency A/B PYTHONUNBUFFERED=1 $(PY) bench/scripts/nli_backend_ab.py $(if $(REPEATS),--repeats $(REPEATS),) # #000049 Phase 2 / §7 #12 gate item 4 — measure the would-demote rate # of the clause-level shadow check over (answer, context) records. # SHADOW ONLY: writes only bench/results/nli-shadow-sweep.json, never an # audit_mode. With no INPUT it sweeps the 5f falsification packs (every # record there is a FALSE claim, so a high would-demote rate = the model # working). Point INPUT at a legit-answer sample for the real gate-item-4 # false-positive number. Renders a structural report even without the # [nli] extra (marked available:false). Run `make bootstrap-nli` first # for real numbers. bench-nli-shadow: bootstrap ## #000049 Phase 2 — NLI shadow would-demote sweep (INPUT="f1.jsonl f2.jsonl" optional) PYTHONUNBUFFERED=1 $(PY) bench/scripts/nli_shadow_sweep.py $(foreach f,$(INPUT),--input $(f)) # --------------------------------------------------------------------------- # Public-domain + open-licensed textbooks (#000031 / surface-ingest scope). # # Manifest at $(TEXTBOOK_MANIFEST) lists math / logic / CS textbooks with # explicit license tokens (PD / CC-BY-SA / GFDL / etc.). The fetch path # reuses `arborist ingest --source html`, which already does the # consistent "robots.txt → noise-strip → 512-token chunk → Merkle root → # audit-event" pipeline that every other surface uses. No separate # downloader / Merkle process — the existing ingest IS the consistent # process. # # To grow coverage of a textbook, append more chapter URLs to its `urls` # array in the manifest. For deep-BFS of a textbook home, use # `make crawl-ingest URL= DEPTH=N` (the existing crawler) instead. # --------------------------------------------------------------------------- TEXTBOOK_MANIFEST ?= bench/fixtures/textbooks/manifest-v1.jsonl TEXTBOOK_DB ?= $(HOME)/.arborist/textbooks.db TEXTBOOK_URLS_TMP ?= /tmp/arborist-textbook-urls.txt textbooks-summary: bootstrap ## list manifest entries with license + URL counts $(PY) -m bench.scripts.textbooks_manifest summary < $(TEXTBOOK_MANIFEST) textbooks-urls: bootstrap ## emit one textbook URL per line to stdout @$(PY) -m bench.scripts.textbooks_manifest urls < $(TEXTBOOK_MANIFEST) # License-validated fetch + ingest of every URL in $(TEXTBOOK_MANIFEST). # The manifest helper refuses to emit URLs from entries with missing or # disallowed license tokens — license-discipline is fail-closed at the # URL-emit step, so the fetch never sees a non-redistributable URL. fetch-textbooks: bootstrap ## fetch + ingest manifested textbooks → $(TEXTBOOK_DB) @$(PY) -m bench.scripts.textbooks_manifest urls < $(TEXTBOOK_MANIFEST) > $(TEXTBOOK_URLS_TMP) @count=$$(wc -l < $(TEXTBOOK_URLS_TMP)); \ echo ">> ingesting $$count textbook URLs into $(TEXTBOOK_DB)" $(ARBORIST) --db $(TEXTBOOK_DB) ingest --source html --urls-from $(TEXTBOOK_URLS_TMP) @rm -f $(TEXTBOOK_URLS_TMP) textbooks-stats: bootstrap ## stats of the textbook shard $(ARBORIST) --db $(TEXTBOOK_DB) stats textbooks-verify: bootstrap ## sample-Merkle-verify the textbook shard $(ARBORIST) --db $(TEXTBOOK_DB) verify # Full-book ingest via the BFS crawler — one shard per host. The # manifest's `crawl_url` + `crawl_depth` + `crawl_max` fields drive # this; entries without `crawl_url` are skipped (e.g., PG TeX-only # textbooks like Hilbert + Boole). Crawl-Delay headers are honored # automatically by the crawler — appliedcombinatorics.org's 20-second # delay means that one will run for ~25 minutes; everything else # completes in seconds. # # Crawl shards land in $(CRAWL_SHARDS_DIR), separate from the main # $(SHARDS_DIR), so SQLite's 10-attached-database limit doesn't get # tripped when the operator wants to query across the main shards. CRAWL_SHARDS_DIR ?= $(HOME)/.arborist/crawl crawl-textbooks: bootstrap-crawler ## BFS-crawl every textbook with a `crawl_url` field, one shard per host (in $(CRAWL_SHARDS_DIR)) @mkdir -p $(CRAWL_SHARDS_DIR) @$(PY) -m bench.scripts.textbooks_manifest crawl-targets < $(TEXTBOOK_MANIFEST) | \ while IFS=$$'\t' read -r url depth max_pages id; do \ echo ""; \ echo ">>> $$id ($$url, depth=$$depth, max=$$max_pages)"; \ domain=$$(echo "$$url" | sed -E 's,^https?://([^/]+).*$$,\1,' | tr '.' '_'); \ shard="$(CRAWL_SHARDS_DIR)/crawl_$${domain}.db"; \ $(ARBORIST) --db "$$shard" crawl --seed-url "$$url" --depth $$depth --max-pages $$max_pages --ingest > /tmp/crawl-$${id}.log 2>&1; \ docs=$$(.venv/bin/python -c "import sqlite3; print(sqlite3.connect(\"$$shard\").execute(\"SELECT COUNT(*) FROM documents\").fetchone()[0])" 2>/dev/null || echo 0); \ echo " landed: $$docs docs in $$shard"; \ done crawl-textbooks-stats: bootstrap ## docs-per-shard summary across textbook crawl shards @for shard in $(CRAWL_SHARDS_DIR)/crawl_*.db $(SHARDS_DIR)/crawl_*.db; do \ [ -f "$$shard" ] || continue; \ docs=$$(.venv/bin/python -c "import sqlite3; print(sqlite3.connect(\"$$shard\").execute(\"SELECT COUNT(*) FROM documents\").fetchone()[0])"); \ chunks=$$(.venv/bin/python -c "import sqlite3; print(sqlite3.connect(\"$$shard\").execute(\"SELECT COUNT(*) FROM chunks\").fetchone()[0])"); \ size=$$(stat -c%s "$$shard"); \ printf " %-60s docs=%-5s chunks=%-5s size=%s\n" "$$(basename $$shard)" "$$docs" "$$chunks" "$$size"; \ done # Per-book idempotent ingest: `make textbook ID=` looks up one # entry from the manifest and runs the right ingest path (HTML # urls vs BFS crawl). Re-running on a shard already ingested is a # no-op at the document level — same content_root → no insert. The # crawl path re-fetches HTTP every run; pair with `make recrawl-check` # for a HEAD-only freshness probe before re-ingest. textbook: bootstrap-crawler ## ingest one textbook by id: make textbook ID=bogart-ctgd-2017 @if [ -z "$(ID)" ]; then \ echo "usage: make textbook ID= (use 'make textbook-list' for available ids)" >&2; \ exit 2; \ fi @row=$$($(PY) -m bench.scripts.textbooks_manifest lookup $(ID) < $(TEXTBOOK_MANIFEST)) || exit 1; \ id=$$(echo "$$row" | cut -f1); \ url=$$(echo "$$row" | cut -f2); \ depth=$$(echo "$$row" | cut -f3); \ max=$$(echo "$$row" | cut -f4); \ license=$$(echo "$$row" | cut -f5); \ domain=$$(echo "$$row" | cut -f6); \ author=$$(echo "$$row" | cut -f7); \ mkdir -p $(CRAWL_SHARDS_DIR); \ shard="$(CRAWL_SHARDS_DIR)/textbook_$${id}.db"; \ author_flag=""; \ if [ -n "$$author" ]; then author_flag="--author $$author"; fi; \ echo ">> $$id ($$license, $$domain)"; \ echo " crawl: $$url depth=$$depth max=$$max"; \ echo " shard: $$shard"; \ if [ -n "$$author" ]; then echo " author: $$author"; fi; \ if [ -n "$$url" ]; then \ $(ARBORIST) --db "$$shard" crawl --seed-url "$$url" --depth $$depth --max-pages $$max --ingest $$author_flag > /tmp/textbook-$${id}.log 2>&1 && \ docs=$$(.venv/bin/python -c "import sqlite3; print(sqlite3.connect(\"$$shard\").execute(\"SELECT COUNT(*) FROM documents\").fetchone()[0])"); \ chunks=$$(.venv/bin/python -c "import sqlite3; print(sqlite3.connect(\"$$shard\").execute(\"SELECT COUNT(*) FROM chunks\").fetchone()[0])"); \ echo " landed: $$docs docs / $$chunks chunks"; \ else \ echo " no crawl_url for $$id; using shallow ingest path"; \ echo "$$id" | $(PY) -c 'import sys, json, os; sys.path.insert(0, "."); ids = sys.stdin.read().split(); manifest = "$(TEXTBOOK_MANIFEST)"; [print(u) for line in open(manifest) for e in [json.loads(line)] if "_meta" not in e and e.get("id") in ids for u in e.get("urls", [])]' > /tmp/textbook-$${id}-urls.txt; \ $(ARBORIST) --db "$$shard" ingest --source html --urls-from /tmp/textbook-$${id}-urls.txt $$author_flag; \ rm -f /tmp/textbook-$${id}-urls.txt; \ fi textbook-list: bootstrap ## list ingestable textbook ids (entries with urls or crawl_url) @$(PY) -m bench.scripts.textbooks_manifest ids < $(TEXTBOOK_MANIFEST) # Per-book convenience targets — one per active manifest id. Re-running # any of these is idempotent at the database layer (content-addressed # inserts) but does refetch HTTP. Use `make recrawl-check` first if you # want a freshness probe before paying network cost. .PHONY: textbook-bogart textbook-keller-trotter textbook-levin \ textbook-aristotle-prior textbook-aristotle-posterior \ textbook-newton textbook-morin textbook-judson textbook-cantor \ textbook-demorgan textbook-russell-imp textbooks-base-knowledge textbook-bogart: ## Bogart Combinatorics Through Guided Discovery (GFDL) $(MAKE) textbook ID=bogart-ctgd-2017 textbook-keller-trotter: ## Keller-Trotter Applied Combinatorics (CC-BY-SA, slow ~25min crawl-delay) $(MAKE) textbook ID=keller-trotter-applied-comb-2017 textbook-levin: ## Levin Discrete Mathematics (CC-BY-SA) $(MAKE) textbook ID=levin-discrete-math-3rd textbook-aristotle-prior: ## Aristotle Prior Analytics (PD, Wikisource) $(MAKE) textbook ID=aristotle-prior-analytics-jenkinson textbook-aristotle-posterior: ## Aristotle Posterior Analytics (PD, Wikisource) $(MAKE) textbook ID=aristotle-posterior-analytics textbook-newton: ## Newton Principia (PD, Wikisource) $(MAKE) textbook ID=newton-principia-motte textbook-morin: ## Morin Open Data Structures (CC-BY) $(MAKE) textbook ID=morin-open-data-structures textbook-judson: ## Judson Abstract Algebra: Theory and Applications (GFDL) $(MAKE) textbook ID=judson-abstract-algebra textbook-cantor: ## Cantor Contributions to Transfinite Numbers (PD, Jourdain transl. 1915, Wikisource) $(MAKE) textbook ID=cantor-transfinite-numbers-jourdain textbook-demorgan: ## De Morgan First Notions of Logic 1839 (PD, PG #67017) $(MAKE) textbook ID=demorgan-first-notions-logic textbook-russell-imp: ## Russell Introduction to Mathematical Philosophy 1919 (PD, PG #41654) $(MAKE) textbook ID=russell-imp-1919 # Bulk target: ingest every base-knowledge textbook needed for the # claim-pack warrant chains across pillars I/II/III/IX. Skips entries # already ingested at the same content_root (idempotent at DB layer). # Ordered cheap → expensive crawl-delay-wise; Judson last because its # 23-chapter PreTeXt crawl is the heaviest of the new entries. textbooks-base-knowledge: textbook-cantor textbook-demorgan textbook-russell-imp textbook-judson ## ingest the four 2026-05-09 base-knowledge additions (pillars I/II/III/IX) @echo ">>> base-knowledge textbooks ingested: Cantor + De Morgan + Russell IMP + Judson AGT" # TeX-source textbooks: Project Gutenberg eBooks that ship as # LaTeX source only (no clean HTML edition). The textbook_tex # source strips the LaTeX into plain prose via a focused PG-aware # pipeline. Idempotent at the database layer like every other # ingest path. .PHONY: textbooks-tex textbook-hilbert textbook-boole textbooks-tex: bootstrap ## ingest every manifest entry that declares a tex_url (Hilbert + Boole) @mkdir -p $(CRAWL_SHARDS_DIR) @$(PY) -m bench.scripts.textbooks_manifest tex-targets < $(TEXTBOOK_MANIFEST) | \ while IFS=$$'\t' read -r url id; do \ shard="$(CRAWL_SHARDS_DIR)/textbook_$${id}.db"; \ echo ">> $$id (textbook_tex)"; \ echo " url: $$url"; \ echo " shard: $$shard"; \ $(ARBORIST) --db "$$shard" ingest --source textbook_tex --url "$$url"; \ done textbook-hilbert: ## Hilbert Foundations of Geometry (PD, PG TeX) @row=$$(grep '"hilbert-foundations-geometry-1902"' $(TEXTBOOK_MANIFEST) | head -1); \ url=$$(echo "$$row" | $(PY) -c "import json,sys; print(json.loads(sys.stdin.read()).get('tex_url',''))"); \ if [ -z "$$url" ]; then echo "no tex_url for Hilbert" >&2; exit 1; fi; \ mkdir -p $(CRAWL_SHARDS_DIR); \ $(ARBORIST) --db "$(CRAWL_SHARDS_DIR)/textbook_hilbert-foundations-geometry-1902.db" \ ingest --source textbook_tex --url "$$url" textbook-boole: ## Boole Laws of Thought (PD, PG TeX) @row=$$(grep '"boole-laws-of-thought-1854"' $(TEXTBOOK_MANIFEST) | head -1); \ url=$$(echo "$$row" | $(PY) -c "import json,sys; print(json.loads(sys.stdin.read()).get('tex_url',''))"); \ if [ -z "$$url" ]; then echo "no tex_url for Boole" >&2; exit 1; fi; \ mkdir -p $(CRAWL_SHARDS_DIR); \ $(ARBORIST) --db "$(CRAWL_SHARDS_DIR)/textbook_boole-laws-of-thought-1854.db" \ ingest --source textbook_tex --url "$$url" textbook-peano: ## Peano Arithmetices Principia 1889 (CC-BY-SA, Verheyen+Nahas LaTeX from GitHub) @row=$$(grep '"peano-arithmetices-principia-1889"' $(TEXTBOOK_MANIFEST) | head -1); \ url=$$(echo "$$row" | $(PY) -c "import json,sys; print(json.loads(sys.stdin.read()).get('tex_url',''))"); \ if [ -z "$$url" ]; then echo "no tex_url for Peano" >&2; exit 1; fi; \ mkdir -p $(CRAWL_SHARDS_DIR); \ $(ARBORIST) --db "$(CRAWL_SHARDS_DIR)/textbook_peano-arithmetices-principia-1889.db" \ ingest --source textbook_tex --url "$$url" textbook-russell-pom: ## Russell Principles of Mathematics 1903 (PD content + CC-BY-SA-4.0 typesetting, Klement HTML) $(MAKE) textbook ID=russell-pom-1903 textbook-laplace: ## Laplace Philosophical Essay on Probabilities 1814 / 1902 (PD, PG #58881) $(MAKE) textbook ID=laplace-philosophical-essay-probabilities textbook-dedekind: ## Dedekind Essays on the Theory of Numbers 1901 (PD, PG #21016 TeX) @row=$$(grep '"dedekind-essays-theory-numbers"' $(TEXTBOOK_MANIFEST) | head -1); \ url=$$(echo "$$row" | $(PY) -c "import json,sys; print(json.loads(sys.stdin.read()).get('tex_url',''))"); \ if [ -z "$$url" ]; then echo "no tex_url for Dedekind" >&2; exit 1; fi; \ mkdir -p $(CRAWL_SHARDS_DIR); \ $(ARBORIST) --db "$(CRAWL_SHARDS_DIR)/textbook_dedekind-essays-theory-numbers.db" \ ingest --source textbook_tex --url "$$url" textbook-plfa: ## PLFA - Programming Language Foundations in Agda (CC-BY-4.0) $(MAKE) textbook ID=plfa-wadler-kokke-siek textbook-sf-lf: ## Software Foundations Vol 1 Logical Foundations (MIT, Pierce et al.) $(MAKE) textbook ID=sf-pierce-logical-foundations textbook-pm: ## Whitehead-Russell Principia Mathematica Vol 1 (PD, PG #78050 — preface+intro only) $(MAKE) textbook ID=whitehead-russell-pm-vol1-1910 textbook-grinstead-snell: ## Grinstead-Snell Introduction to Probability (GFDL, LibreTexts mirror) $(MAKE) textbook ID=grinstead-snell-intro-probability test-crawler: bootstrap-crawler ## run only the lifted crawler tests $(VENV)/bin/pytest -q tests/crawler # Bridge target: BFS a seed URL, ingest discovered pages into a shard, # capture ETag/Last-Modified for each so recrawl-check can do conditional # HEAD requests later. URL is required; DEPTH and MAX have safe defaults. CRAWL_DEPTH ?= 2 CRAWL_MAX ?= 0 # General web crawls land in ONE central crawl db ($(CRAWL_DB)), inside # $(CRAWL_SHARDS_DIR) so they stay OUT of the peer-shared main $(SHARDS_DIR) # by default and a growing set of crawled domains never trips SQLite's # 10-attached-database limit. Content-addressing lets many domains share # one file safely (idempotent re-ingest; `supersedes` edges on change). # To query it: `--db $(CRAWL_DB)` standalone, or attach it alongside the # main corpus (still just one extra file, under the 10-attach cap). # Override the path with CRAWL_DB=... (or CRAWL_SHARD=... for a one-off). CRAWL_DB ?= $(CRAWL_SHARDS_DIR)/web.db crawl-ingest: bootstrap-crawler ## crawl URL=https://x.com [DEPTH=2 MAX=0 FAST=1 CRAWL_DB=path] into the central crawl db ($(CRAWL_DB)) @if [ -z "$(URL)" ]; then echo "usage: make crawl-ingest URL=https://example.com [DEPTH=2 MAX=0 FAST=1 CRAWL_DB=path]" >&2; exit 2; fi @mkdir -p $(CRAWL_SHARDS_DIR) @shard="$(CRAWL_SHARD)"; \ if [ -z "$$shard" ]; then shard="$(CRAWL_DB)"; fi; \ echo " shard: $$shard" >&2; \ $(ARBORIST) --db "$$shard" crawl --seed-url "$(URL)" --depth $(CRAWL_DEPTH) --max-pages $(CRAWL_MAX) $(if $(FAST),--fast,) --ingest # Fast freshness probe: conditional HEAD per doc, classify fresh/stale/gone. # Send only If-None-Match + If-Modified-Since headers — server returns 304 # with no body when content unchanged. RECRAWL_LIMIT ?= 100 # By default check every shard under $(SHARDS_DIR). Pass CRAWL_SHARD=path # to scope to one. DOMAIN= filters to URLs containing the substring. recrawl-check: bootstrap-crawler ## conditional HEAD per ingested doc [DOMAIN=x.com LIMIT=100 CRAWL_SHARD=path] @if [ -n "$(CRAWL_SHARD)" ]; then \ $(ARBORIST) --db $(CRAWL_SHARD) crawler recrawl-check $(if $(DOMAIN),--domain $(DOMAIN),) --limit $(RECRAWL_LIMIT); \ else \ for db in $(SHARDS_DIR)/*.db; do \ case "$$(basename $$db)" in qa.db|snapshots.db) continue;; esac; \ echo " shard: $$db" >&2; \ $(ARBORIST) --db "$$db" crawler recrawl-check $(if $(DOMAIN),--domain $(DOMAIN),) --limit $(RECRAWL_LIMIT); \ done; \ fi DOT_SRCS := $(wildcard docs/diagrams/*.dot) DOT_PNGS := $(DOT_SRCS:.dot=.png) DOT_SVGS := $(DOT_SRCS:.dot=.svg) docs/diagrams/%.png: docs/diagrams/%.dot dot -Tpng $< -o $@ docs/diagrams/%.svg: docs/diagrams/%.dot dot -Tsvg $< -o $@ docs: $(DOT_PNGS) $(DOT_SVGS) ## render docs/diagrams/*.dot -> .png + .svg via graphviz docs-api: ## generate Sphinx API reference from docstrings (output: docs/_source/_build/html/) $(PY) -m sphinx.cmd.build -b html docs/_source docs/_source/_build/html docs-api-clean: ## remove Sphinx build artifacts rm -rf docs/_source/_build/ # 1-pager / 2-pager standalone PDFs. Source RST lives at # docs/_source/arborist-*-pager.rst (so the same files render on the # Sphinx site AND as standalone PDFs). Style sheet at docs/pager.style. # rst2pdf is installed lazily into $(VENV) on first build. RST2PDF := $(VENV)/bin/rst2pdf DOCS_BUILD := docs/_build PAGER_STYLE := docs/pager.style PAGER_FIGURES := docs/diagrams/pager-arch-stack.png docs/diagrams/pager-verifier-flow.png $(RST2PDF): $(VENV)/bin/activate $(PIP) install rst2pdf $(DOCS_BUILD): mkdir -p $(DOCS_BUILD) $(DOCS_BUILD)/arborist-one-pager.pdf: docs/_source/arborist-one-pager.rst $(PAGER_STYLE) | $(DOCS_BUILD) $(RST2PDF) $(RST2PDF) docs/_source/arborist-one-pager.rst -s $(PAGER_STYLE) -o $@ $(DOCS_BUILD)/arborist-two-pager.pdf: docs/_source/arborist-two-pager.rst $(PAGER_STYLE) $(PAGER_FIGURES) | $(DOCS_BUILD) $(RST2PDF) $(RST2PDF) docs/_source/arborist-two-pager.rst -s $(PAGER_STYLE) -o $@ docs-one-pager: $(DOCS_BUILD)/arborist-one-pager.pdf ## render docs/_build/arborist-one-pager.pdf docs-two-pager: $(DOCS_BUILD)/arborist-two-pager.pdf ## render docs/_build/arborist-two-pager.pdf docs-pagers: docs-one-pager docs-two-pager ## render both 1-pager and 2-pager PDFs docs-pagers-clean: ## remove generated pager PDFs rm -f $(DOCS_BUILD)/arborist-one-pager.pdf $(DOCS_BUILD)/arborist-two-pager.pdf # Reproducible micro-benchmark over a fixed slice of cur. Lets you compare # ETL throughput across configs and catches regressions on optimization # work. Override BENCH_DOCS=N (default 5000). BENCH_DOCS ?= 5000 BENCH_DIR := /tmp/arborist-bench bench: bootstrap fetch-cur ## benchmark serial vs parallel-shared vs attached at $(BENCH_DOCS) docs @bash bench/run.sh $(BENCH_DOCS) # --- GPU/CPU wattage benchmark (#000057 cost axis) --------------------------- # RAPL CPU-energy counters (/sys/class/powercap/intel-rapl:*/energy_uj) are # root-only by default (the PLATYPUS side-channel mitigation, CVE-2020-8694), # so bench/watt_bench.py reads null CPU watts without this. GPU watts via # nvidia-smi need no special perm. Run ONCE per GPU box AS ROOT: # sudo make rapl-access # Installs a udev rule that makes energy_uj world-readable on every powercap # add event (survives reboot), and applies it immediately so no reboot is # needed. Idempotent. Reverse with `sudo make rapl-access-revoke`. RAPL_UDEV_RULE := /etc/udev/rules.d/99-rapl-readable.rules rapl-access: ## [root] make Intel RAPL energy_uj readable for watt_bench CPU power @test "$$(id -u)" = "0" || { echo "run as root: sudo make rapl-access"; exit 1; } @printf '%s\n' \ '# arborist #000057 — let bench/watt_bench.py read CPU package energy.' \ '# RAPL energy_uj is root-only by default (PLATYPUS/CVE-2020-8694).' \ 'SUBSYSTEM=="powercap", ACTION=="add", RUN+="/bin/sh -c '\''chmod a+r /sys/class/powercap/%k/energy_uj 2>/dev/null || true'\''"' \ > $(RAPL_UDEV_RULE) udevadm control --reload-rules @# apply now so a reboot isn't required @for f in /sys/class/powercap/intel-rapl:*/energy_uj; do \ [ -e "$$f" ] && chmod a+r "$$f" && echo "readable: $$f"; \ done @echo "RAPL access installed. Verify: cat /sys/class/powercap/intel-rapl:0/energy_uj" rapl-access-revoke: ## [root] remove the RAPL read-access udev rule @test "$$(id -u)" = "0" || { echo "run as root: sudo make rapl-access-revoke"; exit 1; } rm -f $(RAPL_UDEV_RULE) udevadm control --reload-rules @for f in /sys/class/powercap/intel-rapl:*/energy_uj; do \ [ -e "$$f" ] && chmod 400 "$$f"; \ done @echo "RAPL access revoked (energy_uj back to root-only on next add; reset to 400 now)." clean: ## remove venv + caches (keeps fetched data and db) rm -rf $(VENV) .pytest_cache **/__pycache__ arborist.egg-info find . -type d -name __pycache__ -prune -exec rm -rf {} + clean-db: ## drop the arborist db (keeps fetched data and venv) rm -f $(DB) $(DB)-journal $(DB)-wal $(DB)-shm clean-data: ## remove fetched dumps rm -rf $(DATA_DIR) # --------------------------------------------------------------------------- # Wallet-in-cloud SPV demo # --------------------------------------------------------------------------- # Proves the wallet works from a 'vanilla laptop' that has NO local corpus # data. Two isolated HOMEs: SERVER_HOME holds a tiny ingested corpus + the # arborist serve process; LAPTOP_HOME is empty save for code (the venv is # project-level so HOME=tmp doesn't take it away). The wallet command on # LAPTOP_HOME fetches the trust anchor over HTTP, asks a question, verifies # the Merkle bundle, prints exit code. Then the same call with a deadbeef # anchor proves rejection. Cleans up both HOMEs on exit. WALLET_DEMO_PORT ?= 18780 WALLET_PORT ?= 18780 WALLET_URL ?= http://127.0.0.1:$(WALLET_PORT) WALLET_ANCHOR_FILE ?= $(HOME)/.arborist/wallet.anchor WALLET_QA_DB ?= $(HOME)/.arborist/wallet-qa.db SHARDS_DIR ?= $(HOME)/.arborist/shards wallet-serve: bootstrap ## start a long-running wallet server [WALLET_PORT=18780 SHARDS_DIR=... or DB=... ARBORIST_WALLET_STUB=1] @mkdir -p $(dir $(WALLET_QA_DB)) @if [ -n "$$DB" ]; then \ echo "serving --db $$DB on http://0.0.0.0:$(WALLET_PORT) (qa-db=$(WALLET_QA_DB))"; \ $(ARBORIST) --db "$$DB" serve --host 0.0.0.0 --port $(WALLET_PORT) --qa-db $(WALLET_QA_DB); \ else \ echo "serving --shards-dir $(SHARDS_DIR) on http://0.0.0.0:$(WALLET_PORT) (qa-db=$(WALLET_QA_DB))"; \ $(ARBORIST) --shards-dir $(SHARDS_DIR) serve --host 0.0.0.0 --port $(WALLET_PORT) --qa-db $(WALLET_QA_DB); \ fi wallet-pin: bootstrap ## fetch + save the server's current snapshot_root as trust anchor [WALLET_URL=...] @mkdir -p $(dir $(WALLET_ANCHOR_FILE)) @curl -sS $(WALLET_URL)/snapshot_root | $(PY) -c "import json,sys;print(json.load(sys.stdin)['snapshot_root'])" > $(WALLET_ANCHOR_FILE) @printf 'pinned anchor:\n '; cat $(WALLET_ANCHOR_FILE); printf ' → %s\n' "$(WALLET_ANCHOR_FILE)" wallet-ask: bootstrap ## query the wallet server with your own question [Q="..." WALLET_URL=... ANCHOR=hex (else uses pinned/auto-fetches)] @test -n "$(Q)" || { echo 'usage: make wallet-ask Q="your question" [ANCHOR=hex] [WALLET_URL=...]'; exit 2; } @ANCHOR="$(ANCHOR)"; \ if [ -z "$$ANCHOR" ]; then \ if [ -f $(WALLET_ANCHOR_FILE) ]; then \ ANCHOR=$$(cat $(WALLET_ANCHOR_FILE)); \ echo "# using pinned anchor from $(WALLET_ANCHOR_FILE)" >&2; \ else \ echo "# WARN: no pinned anchor; auto-fetching from server (run 'make wallet-pin' for true SPV trust)" >&2; \ ANCHOR=$$(curl -sS $(WALLET_URL)/snapshot_root | $(PY) -c "import json,sys;print(json.load(sys.stdin)['snapshot_root'])"); \ fi; \ fi; \ $(ARBORIST) wallet ask "$(Q)" --server-url $(WALLET_URL) --trust-anchor $$ANCHOR # --------------------------------------------------------------------------- # Cloud (bucket-direct) — no server, no local DB. apsw + HTTP-range VFS. # --------------------------------------------------------------------------- # `make cloud-search Q="..."` — defaults SHARD_URL to the russell.ballestrini.net # crawl shard on DO Spaces; override with `SHARD_URL=https://other.bucket/path/000.db` # Open the bucket-resident .db in place via SQLite HTTP-range VFS, run FTS5, # print hits. No arborist server in the picture. Requires `arborist[bucket]` # (apsw). The `cloud-demo` target proves the whole stack on a tiny corpus # served from an in-process HTTP server with Range support. # Default bucket = the russell.ballestrini.net web-crawl shard on DO Spaces # (public-read; ~2.5 MB; the "who wrote virt-back?" demo target). Override # SHARD_URL on the command line or in your environment to point cloud-* # commands at any S3-compatible bucket that returns 206 Partial Content # on Range requests. SHARD_URL ?= https://nyc3.digitaloceanspaces.com/arborist/clones/virtback/000.db BLOB_BASE ?= # BUCKET_URL is the manifest the multi-shard cloud-query points at. Client # GETs $BUCKET_URL (treated as a manifest URL if it ends in .json, else # appends `clones/manifest.json`) and queries every listed shard. # Default = sidecar-enabled manifest on DO Spaces: per-shard inverted- # index sidecar.bin where available (sub-second FTS) + bucket-direct # FTS5 fallback for shards still being indexed. BUCKET_URL ?= https://nyc3.digitaloceanspaces.com/arborist/clones/manifest-sidecar.json bootstrap-bucket: bootstrap ## install apsw (bucket-direct extra) $(PIP) install 'apsw>=3.45' cloud-search: bootstrap ## FTS5 search the bucket-resident shard [Q="..." SHARD_URL=https://... LIMIT=N CACHE_MB=N] @test -n "$(Q)" || { echo 'usage: make cloud-search Q="your query" [SHARD_URL=...] [BLOB_BASE=...] [LIMIT=N] [CACHE_MB=N]'; exit 2; } @echo "# shard: $(SHARD_URL)" >&2 @$(ARBORIST) cloud search '$(Q)' --shard-url "$(SHARD_URL)" \ $(if $(BLOB_BASE),--blob-base "$(BLOB_BASE)") \ --limit $(or $(LIMIT),8) \ --cache-mb $(or $(CACHE_MB),32) cloud-snapshot-root: bootstrap ## compute snapshot_root of the bucket-resident shard [SHARD_URL=https://...] @echo "# shard: $(SHARD_URL)" >&2 @$(ARBORIST) cloud snapshot-root --shard-url "$(SHARD_URL)" --cache-mb $(or $(CACHE_MB),32) sidecar-build-fts: bootstrap ## build slim FTS5 sidecar (.idx.db) from a shard [SHARD=path OUT=path] @test -n "$(SHARD)" || { echo 'usage: make sidecar-build-fts SHARD=/path/to/shard.db OUT=/path/to/shard.idx.db'; exit 2; } @test -n "$(OUT)" || { echo 'OUT required'; exit 2; } @$(ARBORIST) sidecar build-fts --shard "$(SHARD)" --out "$(OUT)" sidecar-build-fts-all: bootstrap ## build slim FTS5 sidecar for every shard in SHARDS_DIR [SHARDS_DIR=path OUT_DIR=path] @: $${SHARDS_DIR:=$(HOME)/.arborist/shards} @: $${OUT_DIR:=$(HOME)/.arborist/sidecar-fts} @mkdir -p "$$OUT_DIR" @for SHARD in "$$SHARDS_DIR"/[0-9][0-9][0-9].db; do \ OUT="$$OUT_DIR/$$(basename $$SHARD .db).idx.db" ; \ if [ -f "$$OUT" ]; then echo "skip $$OUT (exists)"; continue; fi ; \ echo ">> building $$OUT" ; \ $(ARBORIST) sidecar build-fts --shard "$$SHARD" --out "$$OUT" ; \ done @ls -lh "$$OUT_DIR"/*.idx.db 2>/dev/null || true cloud-fetch-chunk: bootstrap ## fetch + hash-verify one chunk from a bucket [LEAF_HASH=hex BLOB_BASE=https://...] @test -n "$(LEAF_HASH)" || { echo 'usage: make cloud-fetch-chunk LEAF_HASH=hex BLOB_BASE=https://.../blobs'; exit 2; } @test -n "$(BLOB_BASE)" || { echo 'BLOB_BASE required (per-chunk blobs/ base URL)'; exit 2; } @$(ARBORIST) cloud fetch-chunk "$(LEAF_HASH)" --blob-base "$(BLOB_BASE)" # LLM toggle: `make cloud-query LLM=qwen Q="..."` swaps to Qwen3.6-27B on # uncloseai. Default (LLM unset or LLM=hermes) leaves --endpoint/--model # unset so cloud-query falls back to its built-in default (Hermes-3-8B on # ai.unturf.com). Per-call LLM_ENDPOINT=… / LLM_MODEL=… still override. ifeq ($(LLM),qwen) LLM_ENDPOINT ?= https://qwen.ai.unturf.com/v1 LLM_MODEL ?= Qwen3.6-27B-UD-Q4_K_XL.gguf endif cloud-query: bootstrap ## bucket-direct end-to-end via manifest [Q="..." JSON=1 LLM=qwen|hermes BUCKET_URL=... TOP_K=N MAX_CONTEXT=N CACHE_MB=N] @test -n "$(Q)" || { echo 'usage: make cloud-query Q="your question" [JSON=1] [LLM=qwen|hermes] [BUCKET_URL=...] [TOP_K=4] [MAX_CONTEXT=24000] [CACHE_MB=64]'; exit 2; } @echo "# bucket: $(BUCKET_URL)" >&2 $(if $(LLM_ENDPOINT),@echo "# llm: $(LLM_ENDPOINT) / $(LLM_MODEL)" >&2,) @$(ARBORIST) cloud query '$(Q)' --bucket-url "$(BUCKET_URL)" \ --top-k $(or $(TOP_K),4) \ --max-context-chars $(or $(MAX_CONTEXT),24000) \ --cache-mb $(or $(CACHE_MB),64) \ $(if $(LLM_ENDPOINT),--endpoint $(LLM_ENDPOINT),) \ $(if $(LLM_MODEL),--model $(LLM_MODEL),) \ $(if $(JSON),--json,) corpus-query: bootstrap ## LOCAL shards via the same Corpus pipeline cloud-query uses [Q="..." JSON=1 LLM=qwen|hermes TOP_K=N MAX_CONTEXT=N] @test -n "$(Q)" || { echo 'usage: make corpus-query Q="your question" [JSON=1] [LLM=qwen|hermes] [TOP_K=4] [MAX_CONTEXT=24000]'; exit 2; } $(if $(LLM_ENDPOINT),@echo "# llm: $(LLM_ENDPOINT) / $(LLM_MODEL)" >&2,) @$(ARBORIST) --shards-dir $(SHARDS_DIR) corpus-query '$(Q)' \ --top-k $(or $(TOP_K),4) \ --max-context-chars $(or $(MAX_CONTEXT),24000) \ $(if $(LLM_ENDPOINT),--endpoint $(LLM_ENDPOINT),) \ $(if $(LLM_MODEL),--model $(LLM_MODEL),) \ $(if $(JSON),--json,) # --------------------------------------------------------------------------- # Head-to-head: local query (BURN=1) vs cloud-query, same question, same data. # --------------------------------------------------------------------------- # Defaults align the two endpoints: LOCAL_DB is the same file that was # uploaded to Spaces under SHARD_URL, so the two paths grind on the # byte-identical corpus. --burn busts the local QA cache so we time # a fresh inference both sides. LOCAL_DB ?= $(HOME)/.arborist/crawl/web.db bench-cloud-vs-local: bootstrap ## head-to-head: local query --burn vs cloud-query, same question, timed [Q="..."] @test -n "$(Q)" || { echo 'usage: make bench-cloud-vs-local Q="your question" [LOCAL_DB=path] [SHARD_URL=https://...]'; exit 2; } @echo "=== bench-cloud-vs-local ===" @echo "Q: $(Q)" @echo "LOCAL_DB: $(LOCAL_DB)" @echo "SHARD_URL: $(SHARD_URL)" @echo @echo '--- LOCAL (--burn, fresh inference) ---' @T0=$$(date +%s.%N); \ $(ARBORIST) --db $(LOCAL_DB) query --top-k 4 --burn --json '$(Q)' > /tmp/.bench-local.json 2>&1 || true; \ T1=$$(date +%s.%N); \ LOCAL_DT=$$(awk "BEGIN { print $$T1 - $$T0 }"); \ $(PY) -c "import json; d=json.load(open('/tmp/.bench-local.json')); print('answer: ', d.get('answer_text','').strip()[:200]); print('audit: ', d.get('audit_mode')); print('verifier: ', d.get('verifier_method')); print('n_verified:', d.get('n_verified'), '/', d.get('n_quotes')); s=d.get('sources') or []; print('sources: ', len(s), '→', s[0]['document_uri'] if s else '(none)')"; \ printf 'wall: %.3fs\n' "$$LOCAL_DT"; \ echo; \ echo '--- CLOUD (bucket-direct, always fresh) ---'; \ T0=$$(date +%s.%N); \ $(ARBORIST) cloud query '$(Q)' --shard-url "$(SHARD_URL)" --top-k 4 > /tmp/.bench-cloud.json 2>&1 || true; \ T1=$$(date +%s.%N); \ CLOUD_DT=$$(awk "BEGIN { print $$T1 - $$T0 }"); \ $(PY) -c "import json; d=json.load(open('/tmp/.bench-cloud.json')); print('answer: ', d.get('answer_text','').strip()[:200]); print('audit: ', d.get('audit_mode')); print('verifier: ', d.get('verifier_method')); print('n_verified:', d.get('n_verified'), '/', d.get('n_quotes')); s=d.get('sources') or []; print('sources: ', len(s), '→', s[0]['document_uri'] if s else '(none)'); st=d.get('stats') or {}; print('http_reqs: ', st.get('http_requests')); cs=st.get('cache') or {}; print('bytes: ', cs.get('bytes_fetched'))"; \ printf 'wall: %.3fs\n' "$$CLOUD_DT"; \ echo; \ echo "=== summary ==="; \ awk "BEGIN { printf \" local: %.3fs\\n cloud: %.3fs (delta %+.3fs, ratio %.2fx)\\n\", $$LOCAL_DT, $$CLOUD_DT, $$CLOUD_DT-$$LOCAL_DT, $$CLOUD_DT/$$LOCAL_DT }"; \ rm -f /tmp/.bench-local.json /tmp/.bench-cloud.json CLOUD_DEMO_PORT ?= 18785 cloud-demo: bootstrap ## bucket-direct end-to-end proof on tiny in-process corpus [CLOUD_DEMO_PORT=N] @$(PY) -c "import apsw" 2>/dev/null || { echo 'apsw not installed; run: make bootstrap-bucket'; exit 2; } @set -e; \ BUCKET=$$(mktemp -d -t arborist-bucket-XXXXXX); \ LAPTOP_HOME=$$(mktemp -d -t arborist-lap-XXXXXX); \ trap 'kill $$HTTP_PID 2>/dev/null || true; wait $$HTTP_PID 2>/dev/null || true; rm -rf $$BUCKET $$LAPTOP_HOME; true' EXIT INT TERM; \ printf '=== 1. seed a corpus + externalize chunks to bucket layout ===\n'; \ $(PY) -m arborist.wallet._cloud_demo_seed $$BUCKET; \ printf '\n=== 2. start static HTTP server on 127.0.0.1:$(CLOUD_DEMO_PORT) (Range-aware) ===\n'; \ ( cd $$BUCKET && exec $(abspath $(PY)) -m arborist.wallet._range_http_server $(CLOUD_DEMO_PORT) ) & HTTP_PID=$$!; \ for i in 1 2 3 4 5 6 7 8 9 10; do \ curl -s -o /dev/null http://127.0.0.1:$(CLOUD_DEMO_PORT)/ && break; \ sleep 0.3; \ done; \ BASE=http://127.0.0.1:$(CLOUD_DEMO_PORT); \ SHARD_URL=$$BASE/clones/snap-1/000.db; \ BLOB_BASE=$$BASE/blobs; \ printf '\n=== 3. vanilla laptop has NO arborist data ===\n'; \ HOME=$$LAPTOP_HOME ls -la $$LAPTOP_HOME; \ printf '\n=== 4. bucket-direct: compute snapshot_root from the .db on bucket ===\n'; \ HOME=$$LAPTOP_HOME $(ARBORIST) cloud snapshot-root --shard-url $$SHARD_URL; \ printf '\n=== 5. bucket-direct: FTS5 search against the .db on bucket ===\n'; \ HOME=$$LAPTOP_HOME $(ARBORIST) cloud search '"lemma-3"' --shard-url $$SHARD_URL --limit 4; \ printf '\n=== 6. bucket-direct: fetch one chunk blob + verify hash ===\n'; \ LH=$$(HOME=$$LAPTOP_HOME $(ARBORIST) cloud search '"lemma-3"' --shard-url $$SHARD_URL --limit 1 | $(PY) -c "import json,sys; d=json.load(sys.stdin); droot=d['hits'][0]['document_root']; import urllib.request,json as J; print('-- fetching one chunk_hash via a second cloud search --',file=sys.stderr); print(d.get('first_leaf',''))" 2>/dev/null || true); \ $(PY) -c "\ import sqlite3, sys; \ from arborist.wallet.bucket import BucketClient, BucketEndpoint; \ ep = BucketEndpoint(shard_url='$$SHARD_URL', blob_base='$$BLOB_BASE'); \ c = BucketClient(ep); \ hits = c.fts_search('\"lemma-3\"', limit=1); \ chunks = c.chunks_for_doc(hits[0]['document_root']); \ print('first leaf_hash:', chunks[0]['leaf_hash']); \ print('LH=' + chunks[0]['leaf_hash']) \ " | tee /tmp/.lh.txt; \ LH=$$(grep '^LH=' /tmp/.lh.txt | cut -d= -f2); \ rm -f /tmp/.lh.txt; \ HOME=$$LAPTOP_HOME $(ARBORIST) cloud fetch-chunk $$LH --blob-base $$BLOB_BASE; \ printf '\n=== 7. confirm laptop HOME is STILL empty ===\n'; \ HOME=$$LAPTOP_HOME ls -la $$LAPTOP_HOME; \ printf '\n=== BUCKET-DIRECT DEMO PASSED — no server, no local DB ===\n' wallet-demo: bootstrap ## SPV wallet end-to-end proof: vanilla laptop queries a cloud server, verifies cryptographically [WALLET_DEMO_PORT=18780] @set -e; \ SERVER_HOME=$$(mktemp -d -t arborist-srv-XXXXXX); \ LAPTOP_HOME=$$(mktemp -d -t arborist-lap-XXXXXX); \ trap 'kill $$SRVPID 2>/dev/null || true; wait $$SRVPID 2>/dev/null || true; rm -rf $$SERVER_HOME $$LAPTOP_HOME; true' EXIT INT TERM; \ printf '=== 1. vanilla laptop has NO arborist data ===\n'; \ HOME=$$LAPTOP_HOME ls -la $$LAPTOP_HOME; \ HOME=$$LAPTOP_HOME ls -la $$LAPTOP_HOME/.arborist 2>&1 || true; \ printf '\n=== 2. server ingests a tiny corpus ===\n'; \ SERVER_ANCHOR=$$(HOME=$$SERVER_HOME $(PY) -m arborist.wallet._demo_seed $$SERVER_HOME); \ printf 'SERVER snapshot_root = %s\n' "$$SERVER_ANCHOR"; \ printf '\n=== 3. arborist serve starting on 127.0.0.1:$(WALLET_DEMO_PORT) ===\n'; \ HOME=$$SERVER_HOME ARBORIST_WALLET_STUB=1 $(ARBORIST) --db $$SERVER_HOME/.arborist/corpus.db \ serve --host 127.0.0.1 --port $(WALLET_DEMO_PORT) --qa-db $$SERVER_HOME/.arborist/qa.db & \ SRVPID=$$!; \ sleep 1.5; \ printf '\n=== 4. vanilla laptop bootstraps trust anchor over HTTP ===\n'; \ ANCHOR=$$(curl -sS http://127.0.0.1:$(WALLET_DEMO_PORT)/snapshot_root | $(PY) -c "import json,sys;print(json.load(sys.stdin)['snapshot_root'])"); \ printf 'laptop fetched anchor=%s\n' "$$ANCHOR"; \ printf '\n=== 5. vanilla laptop asks the cloud and verifies locally ===\n'; \ RC=0; HOME=$$LAPTOP_HOME $(ARBORIST) wallet ask "what is anarchism" \ --server-url http://127.0.0.1:$(WALLET_DEMO_PORT) \ --trust-anchor $$ANCHOR || RC=$$?; \ printf 'exit=%d (0 = Merkle-verified)\n' "$$RC"; \ test "$$RC" = "0" || { printf 'FAIL: verification should have passed\n'; exit 1; }; \ printf '\n=== 6. vanilla laptop HOME is STILL empty (no .arborist created) ===\n'; \ HOME=$$LAPTOP_HOME ls -la $$LAPTOP_HOME; \ printf '\n=== 7. tamper protection: same server, same question, wrong anchor ===\n'; \ BAD_ANCHOR=deadbeef0000000000000000000000000000000000000000000000000000000000; \ RC=0; HOME=$$LAPTOP_HOME $(ARBORIST) wallet ask "what is anarchism" \ --server-url http://127.0.0.1:$(WALLET_DEMO_PORT) \ --trust-anchor $$BAD_ANCHOR || RC=$$?; \ printf 'exit=%d (3 = correctly rejected)\n' "$$RC"; \ test "$$RC" = "3" || { printf 'FAIL: wrong anchor should have been rejected with exit 3\n'; exit 1; }; \ printf '\n=== ALL CHECKS PASSED — wallet works from a no-data machine ===\n'