russell@unturf.com
70ecda3d6c
arborist/qa/nli/ — SHADOW ONLY (never an audit_mode input; manifest not yet in governance_policy_hash per §7 #2). manifest.json pins cross-encoder/nli-MiniLM2-L6-H768 @ a fixed HF revision + the bench-validated θc 0.5/θe 0.9 + 2 alternates + the Phase-3 TODO; shadow.py = ShadowNLI/shadow_check (lazy transformers+torch behind a new [nli] extra, clauses() segmenter, the §7 #5 clause-level Demote() decision, degrades to available=False when [nli] absent); bench/scripts/nli_shadow_sweep.py + make bootstrap-nli / bench-nli-shadow (the gate-item-4 instrument); 16 tests. First sweep (116 records — 5f-falsification packs + the arborist-nli-bench eval sets): 28/28 synth recombination demoted, 0/26 FP on legit summaries, 0/9 fires on already-STRICT_SPAN records, 25/50 on UNGROUNDED (the contradiction half; quiet on non-sequiturs). Gate items 1/2/3/5/6 clear on available data; item 4 — shadow FP rate on a real live-bench-qa sample — remains the open measurement. Production verifier unchanged; falsification-hard stays 10/12.
1116 lines
61 KiB
Makefile
1116 lines
61 KiB
Makefile
# arborist — Makefile entry points
|
||
# Every workflow lives behind a `make` target. Bare python commands are not
|
||
# the user interface.
|
||
|
||
# Tools and config
|
||
PYTHON ?= python3
|
||
VENV ?= .venv
|
||
PIP := $(VENV)/bin/pip
|
||
PY := $(VENV)/bin/python
|
||
ARBORIST := $(VENV)/bin/arborist
|
||
|
||
# Data + DB
|
||
DATA_DIR ?= data
|
||
WP_BASE_URL ?= https://dumps.wikimedia.org/archive/2003/2003-05-16/en
|
||
WP_CUR := $(DATA_DIR)/20030516_cur_tablesql.bz2
|
||
WP_OLD_1 := $(DATA_DIR)/old_tablesqlbz2.1
|
||
WP_OLD_2 := $(DATA_DIR)/old_tablesqlbz2.2
|
||
WP_OLD := $(DATA_DIR)/20030516_old_tablesql.bz2
|
||
# Back-compat alias (older callers used WP_DUMP for the cur snapshot).
|
||
WP_DUMP := $(WP_CUR)
|
||
DB ?= $(HOME)/.arborist/arborist.db
|
||
|
||
# Smoke-test caps so make all stays fast
|
||
INGEST_LIMIT ?= 500
|
||
VERIFY_N ?= 10
|
||
SEARCH_Q ?= computer
|
||
|
||
.PHONY: all bootstrap fetch fetch-cur fetch-old fetch-xml fetch-abstract \
|
||
ingest ingest-cur ingest-old ingest-xml ingest-xml-history \
|
||
ingest-xml-attached ingest-abstract \
|
||
ingest-self ingest-self-providence ingest-git ingest-hg \
|
||
verify search stats test test-ci test-live docs docs-api docs-api-clean \
|
||
chain-check chain-check-shards \
|
||
falsify burn burn-kindergarten inspect bootstrap-crawler test-crawler crawl-ingest \
|
||
recrawl-check bench-qa bench-qa-smoke bench-qa-progressive-and \
|
||
prometheus-trigger-probe bench-5f-threshold-calibration \
|
||
bench-5f-selfmodel-snapshot bench-5f-finetuning-shardchain \
|
||
bench-5f-falsification-hard bench-fork-baseline-hard bench-5f-formulate-hard \
|
||
bootstrap-math bootstrap-nli bench-nli-shadow clean clean-db clean-data help \
|
||
textbooks-summary textbooks-urls fetch-textbooks textbooks-stats textbooks-verify \
|
||
crawl-textbooks crawl-textbooks-stats textbook textbook-list
|
||
|
||
all: bootstrap fetch-cur ingest-cur verify stats ## bootstrap → fetch cur → ingest cur → verify → stats
|
||
|
||
help: ## show this help
|
||
@awk 'BEGIN{FS=":.*##"} /^[a-zA-Z0-9_-]+:.*##/{printf " %-16s %s\n", $$1, $$2}' $(MAKEFILE_LIST)
|
||
|
||
$(VENV)/bin/activate: pyproject.toml
|
||
$(PYTHON) -m venv $(VENV)
|
||
$(PIP) install --upgrade pip wheel
|
||
$(PIP) install -e '.[dev]'
|
||
@touch $(VENV)/bin/activate
|
||
|
||
bootstrap: $(VENV)/bin/activate ## create venv and install editable package
|
||
|
||
$(DATA_DIR):
|
||
mkdir -p $(DATA_DIR)
|
||
|
||
$(WP_CUR): | $(DATA_DIR)
|
||
@echo ">> fetching $(WP_BASE_URL)/$$(basename $@)"
|
||
curl -fL --retry 3 -o $@ "$(WP_BASE_URL)/$$(basename $@)"
|
||
|
||
$(WP_OLD_1): | $(DATA_DIR)
|
||
@echo ">> fetching $(WP_BASE_URL)/$$(basename $@)"
|
||
curl -fL --retry 3 -o $@ "$(WP_BASE_URL)/$$(basename $@)"
|
||
|
||
$(WP_OLD_2): | $(DATA_DIR)
|
||
@echo ">> fetching $(WP_BASE_URL)/$$(basename $@)"
|
||
curl -fL --retry 3 -o $@ "$(WP_BASE_URL)/$$(basename $@)"
|
||
|
||
# The two old_tablesqlbz2.{1,2} parts are split halves of a single bzip2
|
||
# stream (.1 is exactly 640 MiB). Concatenate to get a working bz2 file.
|
||
$(WP_OLD): $(WP_OLD_1) $(WP_OLD_2)
|
||
@echo ">> concatenating old dump parts"
|
||
cat $(WP_OLD_1) $(WP_OLD_2) > $@
|
||
|
||
fetch-cur: $(WP_CUR) ## download cur table dump (~82 MB)
|
||
|
||
fetch-old: $(WP_OLD) ## download old (revision history) parts and concatenate (~893 MB)
|
||
|
||
fetch: fetch-cur fetch-old ## download all 3 files (cur + old.1 + old.2 + concat)
|
||
|
||
ingest-cur: bootstrap fetch-cur ## ingest INGEST_LIMIT cur articles
|
||
$(ARBORIST) --db $(DB) ingest --source wikipedia_cur --path $(WP_CUR) --limit $(INGEST_LIMIT)
|
||
|
||
ingest-old: bootstrap fetch-old ## ingest INGEST_LIMIT old (history) revisions
|
||
$(ARBORIST) --db $(DB) ingest --source wikipedia_old --path $(WP_OLD) --limit $(INGEST_LIMIT)
|
||
|
||
# Phase 2: attach-forever sharding. Each shard owns its own SQLite file —
|
||
# no WAL contention. Reads via `arborist --shards-dir <dir> <cmd>` attach
|
||
# all shards as UNION views. "Merge cost" = 0.
|
||
SHARDS ?= 4
|
||
SHARDS_DIR ?= $(HOME)/.arborist/shards
|
||
ingest-cur-attached: bootstrap fetch-cur ## sharded ingest, no WAL contention (Phase 2)
|
||
@mkdir -p $(SHARDS_DIR)
|
||
@for i in $$(seq 0 $$(($(SHARDS) - 1))); do \
|
||
$(ARBORIST) ingest --source wikipedia_cur --path $(WP_CUR) \
|
||
--shards-dir $(SHARDS_DIR) --shard $$i/$(SHARDS) & \
|
||
done; wait
|
||
|
||
ingest-old-attached: bootstrap fetch-old ## sharded ingest of old history (Phase 2)
|
||
@mkdir -p $(SHARDS_DIR)
|
||
@for i in $$(seq 0 $$(($(SHARDS) - 1))); do \
|
||
$(ARBORIST) ingest --source wikipedia_old --path $(WP_OLD) \
|
||
--shards-dir $(SHARDS_DIR) --shard $$i/$(SHARDS) & \
|
||
done; wait
|
||
|
||
stats-shards: bootstrap ## cross-shard stats via UNION views over $(SHARDS_DIR)
|
||
$(ARBORIST) --shards-dir $(SHARDS_DIR) stats
|
||
|
||
ACTIVITY_LIMIT ?= 10
|
||
activity: bootstrap ## recent Q&A + freshly cached docs (agent timeline)
|
||
$(ARBORIST) --shards-dir $(SHARDS_DIR) activity --limit $(ACTIVITY_LIMIT)
|
||
|
||
inspect: bootstrap ## sidecar diagnose unverified spans for a cache_key: make inspect KEY=hex [JSON=1]
|
||
@if [ -z "$(KEY)" ]; then echo "usage: make inspect KEY=<cache_key> [JSON=1]" >&2; exit 2; fi
|
||
$(ARBORIST) --shards-dir $(SHARDS_DIR) inspect --cache-key $(KEY) $(if $(JSON),--json,)
|
||
|
||
falsify: bootstrap ## mark a cached answer wrong: make falsify KEY=hex REASON='why'
|
||
@if [ -z "$(KEY)" ]; then echo "usage: make falsify KEY=<cache_key> REASON='why'" >&2; exit 2; fi
|
||
$(ARBORIST) --shards-dir $(SHARDS_DIR) providence --falsify $(KEY) --reason "$(REASON)"
|
||
|
||
burn: bootstrap ## delete a leaf with no children. providence: KEY=<cache_key>; document/core: KIND=document|core ROOT=<hex>. REASON='why' [FORCE=1]
|
||
@kind="$${KIND:-providence}"; \
|
||
if [ "$$kind" = "providence" ]; then \
|
||
if [ -z "$(KEY)" ]; then echo "usage: make burn KEY=<cache_key> REASON='why' [FORCE=1]" >&2; exit 2; fi; \
|
||
$(ARBORIST) --shards-dir $(SHARDS_DIR) burn --kind providence --cache-key $(KEY) --reason "$(REASON)" $(if $(FORCE),--force,); \
|
||
elif [ "$$kind" = "document" ] || [ "$$kind" = "core" ]; then \
|
||
if [ -z "$(ROOT)" ]; then echo "usage: make burn KIND=$$kind ROOT=<document_root> REASON='why' [FORCE=1]" >&2; exit 2; fi; \
|
||
$(ARBORIST) --shards-dir $(SHARDS_DIR) burn --kind $$kind --root $(ROOT) --reason "$(REASON)" $(if $(FORCE),--force,); \
|
||
else \
|
||
echo "unknown KIND: $$kind (expected: providence|document|core)" >&2; exit 2; \
|
||
fi
|
||
|
||
# Mass-burn providence_cache rows younger than the kindergarten window.
|
||
# Mirrors mesh sync's kindergarten so what's still un-broadcast is what's
|
||
# safe to bust without confusing peers. Useful while iterating on
|
||
# retrieval/verifier tunings — wipe recent test runs in one shot.
|
||
KG_SECONDS ?= 3600
|
||
burn-kindergarten: bootstrap ## bust providence rows < SECONDS old [SECONDS=3600 FORCE=1 DRY_RUN=1 REASON='why']
|
||
$(ARBORIST) --shards-dir $(SHARDS_DIR) burn-kindergarten \
|
||
--kindergarten-seconds $(KG_SECONDS) \
|
||
$(if $(REASON),--reason "$(REASON)",) \
|
||
$(if $(FORCE),--force,) \
|
||
$(if $(DRY_RUN),--dry-run,)
|
||
|
||
# Multi-source RAG query against the shard cluster.
|
||
# Usage: make query Q="What is anarcho-capitalism?"
|
||
QUERY_TOP_K ?= 8
|
||
# G0 / CTI — pointer-mode is the testing default. Library DEFAULT_POLICY
|
||
# stays "quote" so Python callers aren't surprised; the Makefile
|
||
# harness ships pointer-mode-on so `make query` exercises the new path
|
||
# end-to-end. ANSWER_MODE default flipped to claim_lattice (JSON) on
|
||
# 2026-04-30 after the post-retry bench showed it leading on
|
||
# strict-rate (50%) and grounded count (54) with 0 errors. Override
|
||
# with ANSWER_MODE=claim_lattice_pointer for prose-distribution path
|
||
# (Hermes-3 8B reflexive output without grammar guidance), or
|
||
# ANSWER_MODE=quote for the legacy substring verifier. ANSWER_MODE=
|
||
# (empty) defers to DEFAULT_QUERY_POLICY.
|
||
ANSWER_MODE ?= claim_lattice
|
||
# BROAD=1 → flip on the Ticket #000008 quantifier-cap apply-gate for
|
||
# this call. Default-off per §10.11.3 dry-run discipline; operator
|
||
# opts in here for broad-quantifier shapes (winners-of-all,
|
||
# tell-me-everything-about-X). Pairs cleanly with the broad-
|
||
# quantifier reminder which is default-on for lattice modes.
|
||
# Bench (#000008 §12.10): cap-on JSON wins +14pp on STRICT-rate.
|
||
# REJECT_BROAD=1 → strict reject for ALL/COMPREHENSIVE/OPEN_REQUEST
|
||
# unbounded shapes; returns UNGROUNDED before the LLM call.
|
||
# ALLOW_BROAD=1 → emergent search; classifier on, caps off.
|
||
query: bootstrap ## ask the corpus a question [JSON=1 BURN=1 REPAIR=1 REPROMPTS=N K="extra retrieval keywords" ANSWER_MODE=claim_lattice|claim_lattice_pointer|quote BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1 WITNESS=1]; JSON by default
|
||
@if [ -z "$$Q" ] && [ -z "$(Q)" ]; then \
|
||
echo "usage: make query Q=\"your question\" [JSON=1 BURN=1 REPAIR=1 REPROMPTS=N K=\"extra retrieval keywords\" ANSWER_MODE=claim_lattice|claim_lattice_pointer|quote BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1 WITNESS=1]"; exit 2; \
|
||
fi
|
||
$(ARBORIST) --shards-dir $(SHARDS_DIR) query --top-k $(QUERY_TOP_K) $(if $(JSON),--json,) $(if $(BURN),--burn,) $(if $(REPAIR),--repair,) $(if $(REPROMPTS),--repair-reprompts $(REPROMPTS),) $(if $(ANSWER_MODE),--answer-mode $(ANSWER_MODE),) $(if $(K),--retrieval-keywords "$(K)",) $(if $(BROAD),--apply-quantifier-caps,) $(if $(REJECT_BROAD),--reject-broad,) $(if $(ALLOW_BROAD),--allow-broad,) $(if $(WITNESS),--witness,) "$(Q)"
|
||
|
||
query-dry: bootstrap ## like 'make query' but skip the LLM call (dry-run) [JSON=1 BURN=1 ANSWER_MODE=... BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1]
|
||
@if [ -z "$$Q" ] && [ -z "$(Q)" ]; then \
|
||
echo "usage: make query-dry Q=\"your question\" [JSON=1 BURN=1 ANSWER_MODE=claim_lattice|claim_lattice_pointer|quote BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1]"; exit 2; \
|
||
fi
|
||
$(ARBORIST) --shards-dir $(SHARDS_DIR) query --top-k $(QUERY_TOP_K) --dry-run $(if $(JSON),--json,) $(if $(BURN),--burn,) $(if $(ANSWER_MODE),--answer-mode $(ANSWER_MODE),) $(if $(BROAD),--apply-quantifier-caps,) $(if $(REJECT_BROAD),--reject-broad,) $(if $(ALLOW_BROAD),--allow-broad,) "$(Q)"
|
||
|
||
BENCH_QA_QUESTIONS ?= bench/qa_questions.txt
|
||
BENCH_QA_OUT ?= bench/qa_results
|
||
BENCH_QA_MODES ?= quote,claim_lattice_pointer,claim_lattice
|
||
BENCH_QA_LIMIT ?= 0
|
||
BENCH_QA_N ?= 3
|
||
BENCH_QA_CONCURRENCY ?= 4
|
||
bench-qa: bootstrap ## QA-quality sweep: questions × modes × N samples [BENCH_QA_N=3 BENCH_QA_LIMIT=N BENCH_QA_MODES=... BENCH_QA_CONCURRENCY=4]
|
||
PYTHONUNBUFFERED=1 $(PY) bench/qa_sweep.py \
|
||
--questions $(BENCH_QA_QUESTIONS) \
|
||
--shards-dir $(SHARDS_DIR) \
|
||
--out-dir $(BENCH_QA_OUT) \
|
||
--top-k $(QUERY_TOP_K) \
|
||
--modes $(BENCH_QA_MODES) \
|
||
--limit $(BENCH_QA_LIMIT) \
|
||
--n $(BENCH_QA_N) \
|
||
--concurrency $(BENCH_QA_CONCURRENCY)
|
||
|
||
# Smoke fixture: 5 questions, all anchor classes, all currently failing
|
||
# pointer mode 100% while JSON aces 100%. Inner loop for prompt iteration.
|
||
# ~30s wall-clock at concurrency=4. Use this between full sweeps.
|
||
bench-qa-smoke: bootstrap ## quick 5-question smoke (all anchor classes; ~30s)
|
||
PYTHONUNBUFFERED=1 $(PY) bench/qa_sweep.py \
|
||
--questions bench/qa_questions_smoke.txt \
|
||
--shards-dir $(SHARDS_DIR) \
|
||
--out-dir $(BENCH_QA_OUT) \
|
||
--top-k $(QUERY_TOP_K) \
|
||
--modes $(BENCH_QA_MODES) \
|
||
--n 1 \
|
||
--concurrency $(BENCH_QA_CONCURRENCY)
|
||
|
||
# Progressive-AND / DF-filter fixture: 9 questions chosen to exercise
|
||
# the OR-fallback and progressive-AND drop paths. Use this for any
|
||
# retrieval-side A/B (alternative search backends, synonym/rerank
|
||
# changes, etc.) — the smoke fixture is structurally insensitive
|
||
# because it only contains queries where full-AND succeeds on every
|
||
# shard. Default --n 3 to clear the LLM noise floor. ~3-5 min wall
|
||
# at concurrency=4. Header in bench/qa_questions_progressive_and.txt
|
||
# documents which chain fires per query.
|
||
BENCH_PROGRESSIVE_N ?= 3
|
||
bench-qa-progressive-and: bootstrap ## retrieval-side fixture exercising progressive-AND + DF filter [BENCH_PROGRESSIVE_N=3]
|
||
PYTHONUNBUFFERED=1 $(PY) bench/qa_sweep.py \
|
||
--questions bench/qa_questions_progressive_and.txt \
|
||
--shards-dir $(SHARDS_DIR) \
|
||
--out-dir $(BENCH_QA_OUT) \
|
||
--top-k $(QUERY_TOP_K) \
|
||
--modes $(BENCH_QA_MODES) \
|
||
--n $(BENCH_PROGRESSIVE_N) \
|
||
--concurrency $(BENCH_QA_CONCURRENCY)
|
||
|
||
test-live: bootstrap ## live QA quality tests against Hermes (gated; -n auto parallel)
|
||
ARBORIST_LIVE_TESTS=1 ARBORIST_LIVE_SHARDS_DIR=$(SHARDS_DIR) \
|
||
.venv/bin/pytest tests/test_qa_quality_live.py -v -n auto
|
||
|
||
# Prometheus-Σ §12 trigger probe (ticket #000037 Phase 0). Read-only
|
||
# walk of audit_events + capital_ledger across shards; reports
|
||
# whether any of triggers 1-3 have fired empirically. Output is the
|
||
# evidence fox uses to decide go/no-go on Phase 1. Pure measurement.
|
||
PROMETHEUS_PROBE_OUT ?= bench/results/prometheus-sigma-triggers-$(shell date -u +%Y-%m-%d).md
|
||
prometheus-trigger-probe: bootstrap ## #000037 §12 measured-pressure probe → markdown report
|
||
PYTHONUNBUFFERED=1 $(PY) bench/prometheus_sigma_trigger_probe.py \
|
||
--shards-dir $(SHARDS_DIR) \
|
||
--out $(PROMETHEUS_PROBE_OUT)
|
||
|
||
# Prometheus-Σ Phase 3 sleep-sweep dry-run (ticket #000037). Read-only
|
||
# simulator: classifies Target A (providence_cache) + Target B
|
||
# (documents) sweep candidates, synthesizes ControllerBranches, runs
|
||
# the Phase 1 controller, reports decision distribution + Phase-3
|
||
# design findings. No LLM calls, no mutations.
|
||
PROMETHEUS_SWEEP_DRYRUN_OUT ?= bench/results/prometheus-sigma-sweep-dryrun-$(shell date -u +%Y-%m-%d).md
|
||
# §22 Finding 3 fix: per-audit-mode τ_qa. CP rows are kernel-only
|
||
# (cheap re-probe → short τ); LLM-witness modes (STRICT/HYBRID/
|
||
# UNGROUNDED) re-witness via LLM (expensive → long τ).
|
||
PROMETHEUS_SWEEP_TAU_DAYS ?= 7
|
||
PROMETHEUS_SWEEP_TAU_CP_DAYS ?= 1
|
||
PROMETHEUS_SWEEP_B_SAMPLE ?= 500
|
||
prometheus-sweep-dryrun: bootstrap ## #000037 Phase 3 sleep-sweep dry-run → markdown report
|
||
PYTHONUNBUFFERED=1 $(PY) bench/scripts/prometheus_sigma_sweep_dryrun.py \
|
||
--shards-dir $(SHARDS_DIR) \
|
||
--tau-qa-days $(PROMETHEUS_SWEEP_TAU_DAYS) \
|
||
--tau-qa-cp-days $(PROMETHEUS_SWEEP_TAU_CP_DAYS) \
|
||
--sample-b $(PROMETHEUS_SWEEP_B_SAMPLE) \
|
||
--out $(PROMETHEUS_SWEEP_DRYRUN_OUT)
|
||
|
||
# Harvest #000037 Phase 1 falsification-fixture proposals from
|
||
# qa.db into a corpus-derived 5F fixture pack. Stratified sample
|
||
# (20 HYBRID + 20 UNGROUNDED) by cache_key for determinism.
|
||
# Idempotent — re-running rewrites the pack from scratch.
|
||
HARVEST_QA_DB ?= $(SHARDS_DIR)/qa.db
|
||
HARVEST_OUT ?= bench/fixtures/5f/falsification-harvested-v1.jsonl
|
||
HARVEST_THRESHOLD ?= 0.5
|
||
HARVEST_SAMPLE_PER_BUCKET ?= 20
|
||
bench-5f-harvest: bootstrap ## harvest live-shard falsification proposals → 5F fixture pack
|
||
PYTHONUNBUFFERED=1 $(PY) bench/scripts/harvest_falsification_proposals.py \
|
||
--qa-db $(HARVEST_QA_DB) \
|
||
--threshold $(HARVEST_THRESHOLD) \
|
||
--sample-per-bucket $(HARVEST_SAMPLE_PER_BUCKET) \
|
||
--out $(HARVEST_OUT)
|
||
|
||
# Concept-layer backfill targets. Each runs an extractor across every
|
||
# wiki shard; per-shard work is independent so we use GNU-parallel-
|
||
# style concurrency with `xargs -P` to overlap the slow paths
|
||
# (link_reciprocity ~50s/shard, token_idf ~12s/shard, documents_fts
|
||
# ~3s/shard). Total wall-clock with -P 4 vs serial:
|
||
# serial: link 200s + idf 50s + fts 10s = 260s
|
||
# parallel: link 50s + idf 12s + fts 3s ≈ 65s (~4× speedup)
|
||
CONCEPTS_WORKERS ?= 4
|
||
backfill-concepts: bootstrap ## backfill all concept extractors in parallel across shards
|
||
PYTHONUNBUFFERED=1 $(PY) scripts/backfill_concepts.py \
|
||
--shards-dir $(SHARDS_DIR) \
|
||
--workers $(CONCEPTS_WORKERS)
|
||
|
||
# Quick bench mode — 1 sample, smoke fixture, all 3 modes. ~10s.
|
||
# For pure smoke after a code change before the longer bench-qa-smoke.
|
||
# Emergent stress-test: random word triangulation. Pick 3 words from
|
||
# /usr/share/dict/words, ask Hermes @ temp=0.8 to weave them into a
|
||
# creative question, send to arborist, append the journey to
|
||
# bench/emergent_log.jsonl. Designed for blue-moon cadence — surfaces
|
||
# combinatoric failure modes the curated bench-qa fixture set can't.
|
||
# Teacher review (Opus) runs separately via `--print-pending`; fox
|
||
# brings entries here & gets back guidance to append to the log.
|
||
EMERGENT_N ?= 10
|
||
EMERGENT_SEED ?=
|
||
bench-emergent: bootstrap ## blue-moon emergent stress test (3-word triangulation; N=10)
|
||
PYTHONUNBUFFERED=1 $(PY) scripts/bench_emergent.py \
|
||
--n $(EMERGENT_N) \
|
||
$(if $(EMERGENT_SEED),--seed $(EMERGENT_SEED),) \
|
||
--shards-dir $(SHARDS_DIR) \
|
||
--qa-db $(SHARDS_DIR)/qa.db
|
||
|
||
bench-emergent-pending: bootstrap ## print log entries awaiting teacher review
|
||
$(PY) scripts/bench_emergent.py --print-pending
|
||
|
||
|
||
bench-5s: bootstrap ## 5S battery (Syntax+Semantics+Syllogism+Synthesis+Semiotics)
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub syntax --fixtures bench/fixtures/5s/syntax-v1.jsonl
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub semantics --fixtures bench/fixtures/5s/semantics-v1.jsonl
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub syllogism --fixtures bench/fixtures/5s/syllogism-v1.jsonl
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub synthesis --fixtures bench/fixtures/5s/synthesis-v1.jsonl
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub semiotics --fixtures bench/fixtures/5s/semiotics-v1.jsonl
|
||
|
||
bench-5t: bootstrap ## 5T battery (Transfer Learning+Triangulation+Truthtables+Transitivity+Time)
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5t --sub transfer-learning --fixtures bench/fixtures/5t/transfer-learning-v2.jsonl
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5t --sub triangulation --fixtures bench/fixtures/5t/triangulation-v1.jsonl
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5t --sub truthtables --fixtures bench/fixtures/5t/truthtables-v1.jsonl
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5t --sub transitivity --fixtures bench/fixtures/5t/transitivity-v1.jsonl
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5t --sub time --fixtures bench/fixtures/5t/time-v1.jsonl
|
||
|
||
bench-5t-legacy: bootstrap ## 5T legacy SQD-name (transfer-v1) for Phase-1a digest stability
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5t --sub transfer --fixtures bench/fixtures/5t/transfer-v1.jsonl
|
||
|
||
bench-5f: bootstrap ## 5F battery (Function+Finetuning+Falsification+Formulate+Feedback Loop)
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub function --fixtures bench/fixtures/5f/function-v1.jsonl
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub finetuning --fixtures bench/fixtures/5f/finetuning-v1.jsonl
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub falsification --fixtures bench/fixtures/5f/falsification-v1.jsonl
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub formulate --fixtures bench/fixtures/5f/formulate-v1.jsonl
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub feedback-loop --fixtures bench/fixtures/5f/feedback-loop-v1.jsonl
|
||
|
||
bench-5s-code: bootstrap ## 5S code-carrier (Syntax + Semantics through code-py-ast@v1)
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub syntax \
|
||
--fixtures bench/fixtures/5s/syntax-code-v1.jsonl
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub semantics \
|
||
--fixtures bench/fixtures/5s/semantics-code-v1.jsonl
|
||
|
||
bench-5s-arithmetic: bootstrap ## 5S arithmetic π* (SQD §14.1; rational arithmetic canonicalizer)
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub syntax \
|
||
--fixtures bench/fixtures/5s/syntax-arithmetic-v1.jsonl
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub semantics \
|
||
--fixtures bench/fixtures/5s/semantics-arithmetic-v1.jsonl
|
||
|
||
bench-5s-logic-kernel: bootstrap ## 5S logic-kernel π* (SQD §14.3; CNF canonicalizer)
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub syntax \
|
||
--fixtures bench/fixtures/5s/syntax-logic-v1.jsonl
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub semantics \
|
||
--fixtures bench/fixtures/5s/semantics-logic-v1.jsonl
|
||
|
||
bench-5s-math: bench-5s-arithmetic bench-5s-logic-kernel ## complete math π* surface (arithmetic + logic-kernel)
|
||
|
||
bench-5s-algebra: bootstrap-math ## 5S algebra-symbolic π* (ticket #000030 Phase 1; SymPy expand+srepr canonicalizer)
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub syntax \
|
||
--fixtures bench/fixtures/5s/syntax-algebra-symbolic-v1.jsonl
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub semantics \
|
||
--fixtures bench/fixtures/5s/semantics-algebra-symbolic-v1.jsonl
|
||
|
||
bench-5s-combinatorics: bootstrap-math ## 5S combinatorics π* (ticket #000032; pure-integer counting kernel)
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub syntax \
|
||
--fixtures bench/fixtures/5s/syntax-combinatorics-v1.jsonl
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub semantics \
|
||
--fixtures bench/fixtures/5s/semantics-combinatorics-v1.jsonl
|
||
|
||
bench-5s-time-series: bootstrap ## 5S time-series-quantized π* (SQD §13.5; quantized integer-vector canonicalizer)
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub syntax \
|
||
--fixtures bench/fixtures/5s/syntax-time-series-v1.jsonl
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub semantics \
|
||
--fixtures bench/fixtures/5s/semantics-time-series-v1.jsonl
|
||
|
||
bench-5s-tabular: bootstrap ## 5S tabular-pinned π* (#000030/Phase tabular; declared-schema 2D structured-data canonicalizer)
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub syntax \
|
||
--fixtures bench/fixtures/5s/syntax-tabular-v1.jsonl
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub semantics \
|
||
--fixtures bench/fixtures/5s/semantics-tabular-v1.jsonl
|
||
|
||
bench-5s-calculus-limit: bootstrap ## 5S calculus-limit π* (#000030 Phase 4)
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub syntax \
|
||
--fixtures bench/fixtures/5s/syntax-calculus-limit-v1.jsonl
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub semantics \
|
||
--fixtures bench/fixtures/5s/semantics-calculus-limit-v1.jsonl
|
||
|
||
bench-5s-calculus-series: bootstrap ## 5S calculus-series π* (#000030 Phase 5)
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub syntax \
|
||
--fixtures bench/fixtures/5s/syntax-calculus-series-v1.jsonl
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub semantics \
|
||
--fixtures bench/fixtures/5s/semantics-calculus-series-v1.jsonl
|
||
|
||
bench-5s-linear-algebra: bootstrap ## 5S linear-algebra π* (#000030 Phase 6)
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub syntax \
|
||
--fixtures bench/fixtures/5s/syntax-linear-algebra-v1.jsonl
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub semantics \
|
||
--fixtures bench/fixtures/5s/semantics-linear-algebra-v1.jsonl
|
||
|
||
bench-5s-function-sampled: bootstrap ## 5S function-sampled π* (#000030 Phase 7; SymPy → time-series bridge)
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub syntax \
|
||
--fixtures bench/fixtures/5s/syntax-function-sampled-v1.jsonl
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub semantics \
|
||
--fixtures bench/fixtures/5s/semantics-function-sampled-v1.jsonl
|
||
|
||
bench-real-shard: bootstrap ## #000026 Phase 2 — real-shard workload baseline (latency, audit, primary-source use)
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.scripts.real_shard_baseline \
|
||
--shards-dir $${ARBORIST_SHARDS_DIR:-$$HOME/.arborist/shards} \
|
||
--burn
|
||
|
||
# #000028 follow-up — extract LLM-divergence audit events as 5F
|
||
# Falsification fixtures. Reads providence_canonical_witness audit
|
||
# events from a qa.db and writes them to a witness-derived fixture
|
||
# JSONL the existing 5F-falsification-live runner can consume.
|
||
WITNESS_QA_DB ?= $$HOME/.arborist/shards/qa.db
|
||
WITNESS_OUT ?= bench/fixtures/5f/falsification-witness-v1.jsonl
|
||
bench-witness-divergence: bootstrap ## extract LLM-divergence events as 5F Falsification fixtures
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.scripts.witness_to_5f \
|
||
--qa-db $(WITNESS_QA_DB) \
|
||
--out $(WITNESS_OUT)
|
||
|
||
# End-to-end witness sweep against real shards + Hermes (#000028
|
||
# validation). Fires 8 canonical-shape questions with witness=on,
|
||
# records the agreement matrix per question. Pair with
|
||
# bench-witness-divergence to extract any divergence as 5F fixtures.
|
||
WITNESS_SWEEP_OUT ?= bench/results/witness-sweep.json
|
||
bench-witness-sweep: bootstrap ## fire witness mode on canonical-question corpus against real shards
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.scripts.witness_sweep \
|
||
--shards-dir $${ARBORIST_SHARDS_DIR:-$$HOME/.arborist/shards} \
|
||
--out $(WITNESS_SWEEP_OUT)
|
||
|
||
# function-sampled@v1 demo (#000030 Phase 7) — closes the
|
||
# opencompletion activity24-math-plot.yaml loop. Canonical bytes
|
||
# always print; PNG is optional (provide PNG=/path/to/file.png).
|
||
# Default expression range covers a full sine period.
|
||
demo-plot: bootstrap ## function-sampled@v1 demo: make demo-plot Q='sin(x)' [PNG=/tmp/out.png]
|
||
@if [ -z "$(Q)" ]; then echo "usage: make demo-plot Q='<sympy expr>' [PNG=/tmp/out.png]" >&2; exit 2; fi
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.scripts.demo_plot \
|
||
--expr "$(Q)" \
|
||
$(if $(PNG),--png $(PNG),)
|
||
|
||
# v8 ForkScore — runs the full bench-suite, then scores the fresh
|
||
# child output against a previously-pinned PARENT artifact. Default
|
||
# parent is bench/results/baseline-suite.json (operator pins this
|
||
# once via `make bench-fork-baseline`); override per-call with
|
||
# PARENT=path. Fox's iteration loop:
|
||
#
|
||
# make bench-fork-baseline # one-shot — pins current state
|
||
# <hack hack hack>
|
||
# make bench-fork-score # compare hacked branch to baseline
|
||
# # exit 1 on REJECT (CI-gateable)
|
||
FORK_PARENT ?= bench/results/baseline-suite.json
|
||
FORK_CHILD ?= bench/results/current-suite.json
|
||
FORK_REPORT ?= bench/results/fork_score_report.json
|
||
|
||
bench-fork-baseline: bootstrap ## pin current bench-suite output as ForkScore parent
|
||
@mkdir -p bench/results
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --all \
|
||
--out $(FORK_PARENT)
|
||
@echo ">> baseline pinned: $(FORK_PARENT)"
|
||
|
||
bench-fork-score: bootstrap ## #000012 Phase 1b — score current bench output vs $(FORK_PARENT)
|
||
@mkdir -p bench/results
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --all \
|
||
--out $(FORK_CHILD)
|
||
$(ARBORIST) substrate score \
|
||
--parent $(FORK_PARENT) \
|
||
--child $(FORK_CHILD) \
|
||
--out $(FORK_REPORT)
|
||
|
||
bench-5f-formulate-live: bootstrap ## 5F Formulate via live arborist.qa.parse_claims (Phase 1b.2)
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub formulate \
|
||
--fixtures bench/fixtures/5f/formulate-live-v1.jsonl
|
||
|
||
bench-5f-feedback-loop-live: bootstrap ## 5F Feedback Loop via live arborist memory + audit chain (Phase 1b.2)
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub feedback-loop \
|
||
--fixtures bench/fixtures/5f/feedback-loop-live-v1.jsonl
|
||
|
||
bench-5f-function-live: bootstrap ## 5F Function via live arborist.qa.parse_claims (Phase 1b.2)
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub function \
|
||
--fixtures bench/fixtures/5f/function-live-v1.jsonl
|
||
|
||
bench-5f-finetuning-live: bootstrap ## 5F Finetuning via real selfmodel store/claims_for round-trip (Phase 1b.2)
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub finetuning \
|
||
--fixtures bench/fixtures/5f/finetuning-live-v1.jsonl
|
||
|
||
bench-5f-falsification-live: bootstrap ## 5F Falsification via real arborist.qa.verify.verify_quotes (Phase 1b.2)
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub falsification \
|
||
--fixtures bench/fixtures/5f/falsification-live-v1.jsonl
|
||
|
||
bench-5f-live: bench-5f-formulate-live bench-5f-feedback-loop-live bench-5f-function-live bench-5f-finetuning-live bench-5f-falsification-live ## all 5F Phase-1b.2 live wire-ups
|
||
|
||
# #000046 Phase 1 — HARD live-path Falsification tier. Every fixture is
|
||
# a near-miss the correct verdict is UNGROUNDED on; verify_quotes
|
||
# over-grounds 2 of 12 today → rate 10/12 = 0.833 at HEAD (#000046's
|
||
# paraphrase numeric gate caught 2, #000048 step 2.1's entity
|
||
# salient-token gate caught 4 more; the last 2 are recombination
|
||
# STRICT_PARAPHRASE — #000048 step 2.2). A real below-ceiling baseline.
|
||
# NOT part of bench-5f / bench-fork-baseline. bench-fork-baseline-hard
|
||
# pins the rate so a future verifier upgrade shows up as a larger
|
||
# positive gamma*Delta5f term.
|
||
FORK_PARENT_HARD ?= bench/results/baseline-falsification-hard.json
|
||
# The hard pack has its expected failures (2/12 over-grounds), so the
|
||
# runner exits 1 — that's the below-ceiling baseline working as
|
||
# designed, not a build error. `|| true` keeps the make target green;
|
||
# the JSON / report is written regardless (the --out write precedes
|
||
# the runner's nonzero-on-failures exit).
|
||
bench-5f-falsification-hard: bootstrap ## #000046 — HARD Falsification near-miss pack (rate < 1.0 at HEAD by design)
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub falsification \
|
||
--fixtures bench/fixtures/5f/falsification-hard-v1.jsonl || true
|
||
bench-fork-baseline-hard: bootstrap ## #000046 — pin the below-ceiling hard-Falsification rate as a ForkScore parent
|
||
@mkdir -p bench/results
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub falsification \
|
||
--fixtures bench/fixtures/5f/falsification-hard-v1.jsonl --out $(FORK_PARENT_HARD) || true
|
||
@echo ">> below-ceiling parent pinned: $(FORK_PARENT_HARD) (2/12 over-grounds expected → runner exits 1; the JSON is written)"
|
||
# #000046 Phase 2 — HARD Formulate tier. Prose that parse_pointer_claims
|
||
# should segment a particular way. Was 4/12 (line/bullet-only parser
|
||
# merged multi-claim lines / split wrapped bullets); #000048 step 2.4's
|
||
# clause segmenter closes all 8 → rate 12/12 now (pack at ceiling — a
|
||
# harder Formulate tier would re-open headroom). Runner still exits 0
|
||
# here (0 failures), but the target is kept for the operator-pack
|
||
# convention (not in bench-5f / runner --all).
|
||
bench-5f-formulate-hard: bootstrap ## #000046 Phase 2 / #000048 step 2.4 — HARD Formulate clause-segmentation pack (12/12 after step 2.4)
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub formulate \
|
||
--fixtures bench/fixtures/5f/formulate-hard-v1.jsonl || true
|
||
|
||
# #000025 §10.11 — persistent SelfModel-chain lineage. Each run of
|
||
# bench-5f-selfmodel-snapshot runs the 5S/5T/5F embedded packs and
|
||
# appends ONE chained snapshot (capability claims = the battery
|
||
# rates) to SELFMODEL_CHAIN_DB. Run it >= 2x, then
|
||
# bench-5f-finetuning-shardchain measures improvement between the two
|
||
# most-recent snapshots — a real cross-run lineage, not a synthetic
|
||
# parent/child pair. Operator targets: NOT part of `make bench-5f`,
|
||
# `make test`, or a fresh checkout.
|
||
SELFMODEL_CHAIN_DB ?= $(SHARDS_DIR)/selfmodel-chain.db
|
||
bench-5f-selfmodel-snapshot: bootstrap ## #000025 §10.11 — append one SelfModel snapshot (rates as claims) to the chain shard
|
||
PYTHONUNBUFFERED=1 ARBORIST_SELFMODEL_CHAIN_DB=$(SELFMODEL_CHAIN_DB) \
|
||
$(PY) bench/scripts/selfmodel_chain_snapshot.py
|
||
bench-5f-finetuning-shardchain: bootstrap ## #000025 §10.11 — Finetuning over the two latest chain snapshots (run bench-5f-selfmodel-snapshot >=2x first)
|
||
PYTHONUNBUFFERED=1 ARBORIST_SELFMODEL_CHAIN_DB=$(SELFMODEL_CHAIN_DB) \
|
||
$(PY) -m bench.batteries.runner --battery 5f --sub finetuning \
|
||
--fixtures bench/fixtures/5f/finetuning-shardchain-v1.jsonl
|
||
|
||
# #000025 §10.14 — ForkScore threshold-calibration handoff to #000012.
|
||
# Runs the canonical 5S/5T/5F packs + the 5F live packs, reports
|
||
# baseline rates / granularity / floor-constant sanity checks.
|
||
# Pure measurement; output is the deliverable #000012 cites.
|
||
FIVEF_CALIBRATION_OUT ?= bench/results/5f-threshold-calibration-$(shell date -u +%Y-%m-%d).md
|
||
bench-5f-threshold-calibration: bootstrap ## #000025 §10.14 — 5S/5T/5F → ForkScore threshold calibration report
|
||
PYTHONUNBUFFERED=1 $(PY) bench/scripts/fivef_threshold_calibration.py \
|
||
--out $(FIVEF_CALIBRATION_OUT)
|
||
|
||
bench-5r: bootstrap ## 5R battery (React+Rearrange+Restore+Replicate+Resonate)
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5r --sub react --fixtures bench/fixtures/5r/react-v1.jsonl
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5r --sub rearrange --fixtures bench/fixtures/5r/rearrange-v1.jsonl
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5r --sub restore --fixtures bench/fixtures/5r/restore-v1.jsonl
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5r --sub replicate --fixtures bench/fixtures/5r/replicate-v1.jsonl
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5r --sub resonate --fixtures bench/fixtures/5r/resonate-v1.jsonl
|
||
|
||
bench-5r-react-live: bootstrap ## 5R React via real audit_events on a temp shard (Phase 1b.2)
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5r --sub react \
|
||
--fixtures bench/fixtures/5r/react-live-v1.jsonl
|
||
|
||
bench-5r-restore-live: bootstrap ## 5R Restore via real audit chain query (Phase 1b.2)
|
||
PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5r --sub restore \
|
||
--fixtures bench/fixtures/5r/restore-live-v1.jsonl
|
||
|
||
bench-5r-live: bench-5r-react-live bench-5r-restore-live ## all 5R Phase-1b.2 live wire-ups
|
||
|
||
bench-5s5t: bench-5s bench-5t ## 5S + 5T (Phase-1b vocabulary)
|
||
|
||
bench-5s5t5f: bench-5s bench-5t bench-5f ## 5S + 5T + 5F (operational triad)
|
||
|
||
bench-suite: bench-5s bench-5t bench-5f bench-5r ## complete Dav1DPrometheus suite (5S + 5T + 5F + 5R)
|
||
|
||
|
||
verify-shards: bootstrap ## cross-shard Merkle round-trip on a random sample
|
||
$(ARBORIST) --shards-dir $(SHARDS_DIR) verify -n $(VERIFY_N)
|
||
|
||
analyze-shards: bootstrap ## cross-shard compression spectrum + audit integrity
|
||
$(ARBORIST) --shards-dir $(SHARDS_DIR) analyze
|
||
|
||
# Audit-chain integrity probe. Cheap (set-arithmetic, no per-row hash walk),
|
||
# scriptable, and far faster than `analyze`. 0 = chain intact: a single
|
||
# linear chain with one genesis. Counts, summed:
|
||
# - dangling prev_event_hash references (a prev that no row's event_hash has)
|
||
# - forked parents (a prev_event_hash claimed by >1 row — two events built
|
||
# on the same head; the bug behind qa.db seq 7724/7725)
|
||
# - extra genesis rows (more than one row with prev_event_hash IS NULL)
|
||
# The plain LEFT JOIN check alone misses forks — the forked parent IS present.
|
||
define CHAIN_CHECK_SQL
|
||
SELECT
|
||
(SELECT COUNT(*) FROM audit_events a1
|
||
LEFT JOIN audit_events a2 ON a2.event_hash = a1.prev_event_hash
|
||
WHERE a1.prev_event_hash IS NOT NULL AND a2.event_hash IS NULL)
|
||
+ (SELECT COUNT(*) FROM
|
||
(SELECT 1 FROM audit_events WHERE prev_event_hash IS NOT NULL
|
||
GROUP BY prev_event_hash HAVING COUNT(*) > 1))
|
||
+ MAX(0, (SELECT COUNT(*) FROM audit_events WHERE prev_event_hash IS NULL) - 1)
|
||
AS chain_breaks
|
||
endef
|
||
export CHAIN_CHECK_SQL
|
||
|
||
chain-check: ## audit-chain integrity count for $(DB) (0 = intact: linear chain, one genesis)
|
||
@printf '%s ' "$(DB)"; sqlite3 $(DB) "$$CHAIN_CHECK_SQL"
|
||
|
||
chain-check-shards: ## audit-chain integrity count for every *.db in $(SHARDS_DIR)
|
||
@for db in $(SHARDS_DIR)/*.db; do \
|
||
printf '%s ' "$$db"; sqlite3 "$$db" "$$CHAIN_CHECK_SQL"; \
|
||
done
|
||
|
||
# Parallel per-shard distill: one process per shard. No DB contention
|
||
# because each shard is its own file.
|
||
distill-shards-parallel: bootstrap ## one distill process per shard (parallel)
|
||
@for shard in $(SHARDS_DIR)/*.db; do \
|
||
$(ARBORIST) --db $$shard distill --process first-sentence-v1 --kind surface & \
|
||
done; wait
|
||
|
||
# TF-IDF cores serve as enriched titles for retrieval — distinctive
|
||
# low-frequency terms that surface in body text get promoted into
|
||
# something queryable without a real title match.
|
||
distill-shards-tfidf-parallel: bootstrap ## TF-IDF cores per shard, in parallel
|
||
@for shard in $(SHARDS_DIR)/*.db; do \
|
||
$(ARBORIST) --db $$shard distill --process tfidf-keywords-v1 --kind surface & \
|
||
done; wait
|
||
|
||
ingest: ingest-cur ## default ingest = cur (use ingest-old or *-parallel for full)
|
||
|
||
# Grok account-export ETL.
|
||
# Point GROK_EXPORT at the directory xAI delivered (the one containing
|
||
# `ttl/30d/export_data/<user-id>/prod-grok-backend.json`). The source class
|
||
# auto-walks down to find the JSON.
|
||
GROK_EXPORT ?= $(HOME)/Downloads/ab8ef1f0-0d08-4f87-89c2-d4509e18115b
|
||
# Grok lives in its own shard inside the attach-forever cluster so cross-
|
||
# shard queries (`make query`) see it alongside the Wikipedia 2003 corpus.
|
||
# Single shard (rank 0/1) — Grok conversations are private and small.
|
||
GROK_SHARD := $(SHARDS_DIR)/grok.db
|
||
ingest-grok-attached: bootstrap ## ingest Grok conversations into $(GROK_SHARD)
|
||
@mkdir -p $(SHARDS_DIR)
|
||
$(ARBORIST) --db $(GROK_SHARD) ingest --source grok_export --path $(GROK_EXPORT) --resume
|
||
|
||
ingest-grok-media-attached: bootstrap ## ingest Grok media prompts into $(GROK_SHARD)
|
||
@mkdir -p $(SHARDS_DIR)
|
||
$(ARBORIST) --db $(GROK_SHARD) ingest --source grok_media --path $(GROK_EXPORT) --resume
|
||
|
||
# ----------------------------------------------------------------------------
|
||
# Phase IV (2006+) Wikipedia XML dumps. Drop-in for any dated snapshot in
|
||
# the dumps.wikimedia.org/archive tree by overriding WP_XML_YEAR/MONTH/DATE
|
||
# (and WP_XML_LANG for non-English wikis).
|
||
#
|
||
# Defaults point at enwiki 20101011 (2010-11 archive), the largest snapshot
|
||
# in the archive — 6.2 GB compressed, ~3.4M articles.
|
||
# Other useful snapshots:
|
||
# make fetch-xml WP_XML_YEAR=2006 WP_XML_MONTH=2006-07 WP_XML_DATE=20061104
|
||
# make fetch-xml WP_XML_YEAR=2006 WP_XML_MONTH=2006-12 WP_XML_DATE=20061130
|
||
# ----------------------------------------------------------------------------
|
||
WP_XML_LANG ?= en
|
||
WP_XML_YEAR ?= 2010
|
||
WP_XML_MONTH ?= 2010-11
|
||
WP_XML_DATE ?= 20101011
|
||
WP_XML_BASE_URL ?= https://dumps.wikimedia.org/archive/$(WP_XML_YEAR)/$(WP_XML_MONTH)/$(WP_XML_LANG)wiki/$(WP_XML_DATE)
|
||
WP_XML_FILE ?= $(WP_XML_LANG)wiki-$(WP_XML_DATE)-pages-articles.xml.bz2
|
||
WP_XML := $(DATA_DIR)/$(WP_XML_FILE)
|
||
WP_ABSTRACT_FILE ?= $(WP_XML_LANG)wiki-$(WP_XML_DATE)-abstract.xml
|
||
WP_ABSTRACT := $(DATA_DIR)/$(WP_ABSTRACT_FILE)
|
||
|
||
$(WP_XML): | $(DATA_DIR)
|
||
@echo ">> fetching $(WP_XML_BASE_URL)/$(WP_XML_FILE)"
|
||
curl -fL --retry 3 -o $@ "$(WP_XML_BASE_URL)/$(WP_XML_FILE)"
|
||
|
||
$(WP_ABSTRACT): | $(DATA_DIR)
|
||
@echo ">> fetching $(WP_XML_BASE_URL)/$(WP_ABSTRACT_FILE)"
|
||
curl -fL --retry 3 -o $@ "$(WP_XML_BASE_URL)/$(WP_ABSTRACT_FILE)"
|
||
|
||
fetch-xml: $(WP_XML) ## download Phase IV XML cur dump (default: enwiki 20101011, 6.2 GB)
|
||
|
||
fetch-abstract: $(WP_ABSTRACT) ## download Phase IV abstract.xml (default: enwiki 20101011, ~3 GB)
|
||
|
||
ingest-xml: bootstrap fetch-xml ## ingest INGEST_LIMIT pages from $(WP_XML)
|
||
$(ARBORIST) --db $(DB) ingest --source wikipedia_xml --path $(WP_XML) --limit $(INGEST_LIMIT)
|
||
|
||
ingest-xml-history: bootstrap ## ingest every revision (multi-revision mode); set WP_XML to a pages-meta-history file
|
||
$(ARBORIST) --db $(DB) ingest --source wikipedia_xml_history --path $(WP_XML) --limit $(INGEST_LIMIT)
|
||
|
||
# Sharded XML ingest into the attach-forever cluster — same pattern as
|
||
# ingest-cur-attached. One process per shard, one SQLite file per shard,
|
||
# zero WAL contention.
|
||
ingest-xml-attached: bootstrap fetch-xml ## sharded XML ingest, one process per shard
|
||
@mkdir -p $(SHARDS_DIR)
|
||
@for i in $$(seq 0 $$(($(SHARDS) - 1))); do \
|
||
$(ARBORIST) ingest --source wikipedia_xml --path $(WP_XML) \
|
||
--shards-dir $(SHARDS_DIR) --shard $$i/$(SHARDS) & \
|
||
done; wait
|
||
|
||
ingest-abstract: bootstrap fetch-abstract ## ingest INGEST_LIMIT abstract docs from $(WP_ABSTRACT)
|
||
$(ARBORIST) --db $(DB) ingest --source wikipedia_abstract --path $(WP_ABSTRACT) --limit $(INGEST_LIMIT)
|
||
|
||
# ----------------------------------------------------------------------------
|
||
# Self-ingest: arborist consults its own source code as a queryable corpus.
|
||
# Re-running picks up new commits — same path + new content gets a fresh
|
||
# document_root chained to the prior version via a `supersedes` edge, so the
|
||
# audit chain grows as the repo grows. Lands in a dedicated shard file so it
|
||
# doesn't compete with the wikipedia/grok shards' WAL writer lock.
|
||
#
|
||
# Override SELF_REPO to ingest a different repo's tree.
|
||
# ----------------------------------------------------------------------------
|
||
SELF_REPO ?= $(CURDIR)
|
||
SELF_SHARD := $(SHARDS_DIR)/arborist-self.db
|
||
|
||
ingest-self: bootstrap ## ingest this repo's HEAD into a dedicated shard
|
||
@mkdir -p $(SHARDS_DIR)
|
||
$(ARBORIST) --db $(SELF_SHARD) ingest --source git_repo --path $(SELF_REPO)
|
||
|
||
# Self-reference: promote STRICT live providence records past the
|
||
# kindergarten window into each shard's documents table. Each shard
|
||
# self-promotes only its own records; cross-shard sharing happens
|
||
# via the existing shards-dir UNION at retrieval time. Run on a cron
|
||
# (e.g. hourly) to keep the substrate fresh.
|
||
# See docs/self-reference-thought-chains-design.md.
|
||
KG_SECONDS ?= 3600
|
||
ingest-self-providence: bootstrap ## promote STRICT live providence records into the document corpus [KG_SECONDS=3600]
|
||
@mkdir -p $(SHARDS_DIR)
|
||
@for db in $(SHARDS_DIR)/*.db; do \
|
||
echo ">> promoting providence records: $$db"; \
|
||
$(ARBORIST) --db $$db ingest --source providence --kindergarten-seconds $(KG_SECONDS); \
|
||
done
|
||
|
||
# Generic git-repo ingest: aim it at any local clone via GIT_REPO=...
|
||
GIT_REPO ?= $(CURDIR)
|
||
GIT_SHARD := $(SHARDS_DIR)/$(notdir $(GIT_REPO))-git.db
|
||
ingest-git: bootstrap ## ingest GIT_REPO=<path> into its own shard
|
||
@mkdir -p $(SHARDS_DIR)
|
||
$(ARBORIST) --db $(GIT_SHARD) ingest --source git_repo --path $(GIT_REPO)
|
||
|
||
# Generic hg-repo ingest. HG_REPO=<path>.
|
||
HG_REPO ?=
|
||
HG_SHARD := $(SHARDS_DIR)/$(notdir $(HG_REPO))-hg.db
|
||
ingest-hg: bootstrap ## ingest HG_REPO=<path> (mercurial) into its own shard
|
||
@if [ -z "$(HG_REPO)" ]; then echo "usage: make ingest-hg HG_REPO=/path/to/repo" >&2; exit 2; fi
|
||
@mkdir -p $(SHARDS_DIR)
|
||
$(ARBORIST) --db $(HG_SHARD) ingest --source hg_repo --path $(HG_REPO)
|
||
|
||
verify: bootstrap ## round-trip Merkle proofs for VERIFY_N random documents
|
||
$(ARBORIST) --db $(DB) verify -n $(VERIFY_N)
|
||
|
||
search: bootstrap ## keyword search; override SEARCH_Q (or pass Q=...)
|
||
$(ARBORIST) --db $(DB) search '$(if $(Q),$(Q),$(SEARCH_Q))'
|
||
|
||
stats: bootstrap ## counts: documents, chunks, edges, audit chain
|
||
$(ARBORIST) --db $(DB) stats
|
||
|
||
test: bootstrap ## run pytest suite (excludes opt-in crawler tests)
|
||
$(VENV)/bin/pytest -q --ignore=tests/crawler -n auto
|
||
|
||
# CI-scoped suite: same as `make test` but also drops the wikipedia
|
||
# ingest tests (`test_wikipedia_old.py`, `test_wikipedia_xml.py`).
|
||
# Wikipedia ingest is exercised end-to-end on real shards via
|
||
# `make ingest-cur` / `make ingest-xml`; running synthetic-fixture
|
||
# coverage on every CI push duplicates that surface for no signal
|
||
# the runtime path doesn't already provide. Local `make test` stays
|
||
# comprehensive — this target is for the gate, not the dev loop.
|
||
test-ci: bootstrap ## CI gate: full suite minus crawler + wikipedia ingest
|
||
$(VENV)/bin/pytest -q \
|
||
--ignore=tests/crawler \
|
||
--ignore=tests/test_wikipedia_old.py \
|
||
--ignore=tests/test_wikipedia_xml.py \
|
||
-n auto
|
||
|
||
# Crawler tests are off-by-default — they hit the network in many cases
|
||
# and require the heavy [crawler] extras (aiohttp, bs4, lxml, etc.).
|
||
# Bootstrap installs the extras into the existing venv idempotently.
|
||
bootstrap-crawler: bootstrap ## install [crawler] extras into the venv
|
||
$(PIP) install -e '.[crawler]'
|
||
|
||
# SymPy substrate for algebra/calculus π* canonicalizers (ticket #000030).
|
||
# Already pulled in transitively by `make bootstrap` via the [dev] extras;
|
||
# this target is the explicit opt-in for minimal-install users.
|
||
bootstrap-math: bootstrap ## install [math] extras (sympy) into the venv
|
||
$(PIP) install -e '.[math]'
|
||
|
||
# #000049 Phase 2 — sentence-pair NLI runtime for the SHADOW path
|
||
# (arborist/qa/nli/). ~600 MB (transformers + CPU torch); hard out of
|
||
# core / dev so a fresh checkout stays python3.12 + venv + sqlite3.
|
||
# Installs the extra, then warms the pinned checkpoint download so the
|
||
# first `make bench-nli-shadow` doesn't pay for it. Operator target —
|
||
# NOT part of `make bootstrap`, `make test`, or a fresh checkout.
|
||
bootstrap-nli: bootstrap ## install [nli] extras + warm the pinned NLI checkpoint
|
||
$(PIP) install -e '.[nli]'
|
||
$(PY) -c "from arborist.qa.nli import ShadowNLI; r=ShadowNLI(); r._ensure_loaded(); print('nli runtime ·', 'available:', r.available, '·', r._reason)"
|
||
|
||
# #000049 Phase 2 / §7 #12 gate item 4 — measure the would-demote rate
|
||
# of the clause-level shadow check over (answer, context) records.
|
||
# SHADOW ONLY: writes only bench/results/nli-shadow-sweep.json, never an
|
||
# audit_mode. With no INPUT it sweeps the 5f falsification packs (every
|
||
# record there is a FALSE claim, so a high would-demote rate = the model
|
||
# working). Point INPUT at a legit-answer sample for the real gate-item-4
|
||
# false-positive number. Renders a structural report even without the
|
||
# [nli] extra (marked available:false). Run `make bootstrap-nli` first
|
||
# for real numbers.
|
||
bench-nli-shadow: bootstrap ## #000049 Phase 2 — NLI shadow would-demote sweep (INPUT="f1.jsonl f2.jsonl" optional)
|
||
PYTHONUNBUFFERED=1 $(PY) bench/scripts/nli_shadow_sweep.py $(foreach f,$(INPUT),--input $(f))
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Public-domain + open-licensed textbooks (#000031 / surface-ingest scope).
|
||
#
|
||
# Manifest at $(TEXTBOOK_MANIFEST) lists math / logic / CS textbooks with
|
||
# explicit license tokens (PD / CC-BY-SA / GFDL / etc.). The fetch path
|
||
# reuses `arborist ingest --source html`, which already does the
|
||
# consistent "robots.txt → noise-strip → 512-token chunk → Merkle root →
|
||
# audit-event" pipeline that every other surface uses. No separate
|
||
# downloader / Merkle process — the existing ingest IS the consistent
|
||
# process.
|
||
#
|
||
# To grow coverage of a textbook, append more chapter URLs to its `urls`
|
||
# array in the manifest. For deep-BFS of a textbook home, use
|
||
# `make crawl-ingest URL=<base> DEPTH=N` (the existing crawler) instead.
|
||
# ---------------------------------------------------------------------------
|
||
|
||
TEXTBOOK_MANIFEST ?= bench/fixtures/textbooks/manifest-v1.jsonl
|
||
TEXTBOOK_DB ?= $(HOME)/.arborist/textbooks.db
|
||
TEXTBOOK_URLS_TMP ?= /tmp/arborist-textbook-urls.txt
|
||
|
||
textbooks-summary: bootstrap ## list manifest entries with license + URL counts
|
||
$(PY) -m bench.scripts.textbooks_manifest summary < $(TEXTBOOK_MANIFEST)
|
||
|
||
textbooks-urls: bootstrap ## emit one textbook URL per line to stdout
|
||
@$(PY) -m bench.scripts.textbooks_manifest urls < $(TEXTBOOK_MANIFEST)
|
||
|
||
# License-validated fetch + ingest of every URL in $(TEXTBOOK_MANIFEST).
|
||
# The manifest helper refuses to emit URLs from entries with missing or
|
||
# disallowed license tokens — license-discipline is fail-closed at the
|
||
# URL-emit step, so the fetch never sees a non-redistributable URL.
|
||
fetch-textbooks: bootstrap ## fetch + ingest manifested textbooks → $(TEXTBOOK_DB)
|
||
@$(PY) -m bench.scripts.textbooks_manifest urls < $(TEXTBOOK_MANIFEST) > $(TEXTBOOK_URLS_TMP)
|
||
@count=$$(wc -l < $(TEXTBOOK_URLS_TMP)); \
|
||
echo ">> ingesting $$count textbook URLs into $(TEXTBOOK_DB)"
|
||
$(ARBORIST) --db $(TEXTBOOK_DB) ingest --source html --urls-from $(TEXTBOOK_URLS_TMP)
|
||
@rm -f $(TEXTBOOK_URLS_TMP)
|
||
|
||
textbooks-stats: bootstrap ## stats of the textbook shard
|
||
$(ARBORIST) --db $(TEXTBOOK_DB) stats
|
||
|
||
textbooks-verify: bootstrap ## sample-Merkle-verify the textbook shard
|
||
$(ARBORIST) --db $(TEXTBOOK_DB) verify
|
||
|
||
# Full-book ingest via the BFS crawler — one shard per host. The
|
||
# manifest's `crawl_url` + `crawl_depth` + `crawl_max` fields drive
|
||
# this; entries without `crawl_url` are skipped (e.g., PG TeX-only
|
||
# textbooks like Hilbert + Boole). Crawl-Delay headers are honored
|
||
# automatically by the crawler — appliedcombinatorics.org's 20-second
|
||
# delay means that one will run for ~25 minutes; everything else
|
||
# completes in seconds.
|
||
#
|
||
# Crawl shards land in $(CRAWL_SHARDS_DIR), separate from the main
|
||
# $(SHARDS_DIR), so SQLite's 10-attached-database limit doesn't get
|
||
# tripped when the operator wants to query across the main shards.
|
||
CRAWL_SHARDS_DIR ?= $(HOME)/.arborist/crawl
|
||
|
||
crawl-textbooks: bootstrap-crawler ## BFS-crawl every textbook with a `crawl_url` field, one shard per host (in $(CRAWL_SHARDS_DIR))
|
||
@mkdir -p $(CRAWL_SHARDS_DIR)
|
||
@$(PY) -m bench.scripts.textbooks_manifest crawl-targets < $(TEXTBOOK_MANIFEST) | \
|
||
while IFS=$$'\t' read -r url depth max_pages id; do \
|
||
echo ""; \
|
||
echo ">>> $$id ($$url, depth=$$depth, max=$$max_pages)"; \
|
||
domain=$$(echo "$$url" | sed -E 's,^https?://([^/]+).*$$,\1,' | tr '.' '_'); \
|
||
shard="$(CRAWL_SHARDS_DIR)/crawl_$${domain}.db"; \
|
||
$(ARBORIST) --db "$$shard" crawl --seed-url "$$url" --depth $$depth --max-pages $$max_pages --ingest > /tmp/crawl-$${id}.log 2>&1; \
|
||
docs=$$(.venv/bin/python -c "import sqlite3; print(sqlite3.connect(\"$$shard\").execute(\"SELECT COUNT(*) FROM documents\").fetchone()[0])" 2>/dev/null || echo 0); \
|
||
echo " landed: $$docs docs in $$shard"; \
|
||
done
|
||
|
||
crawl-textbooks-stats: bootstrap ## docs-per-shard summary across textbook crawl shards
|
||
@for shard in $(CRAWL_SHARDS_DIR)/crawl_*.db $(SHARDS_DIR)/crawl_*.db; do \
|
||
[ -f "$$shard" ] || continue; \
|
||
docs=$$(.venv/bin/python -c "import sqlite3; print(sqlite3.connect(\"$$shard\").execute(\"SELECT COUNT(*) FROM documents\").fetchone()[0])"); \
|
||
chunks=$$(.venv/bin/python -c "import sqlite3; print(sqlite3.connect(\"$$shard\").execute(\"SELECT COUNT(*) FROM chunks\").fetchone()[0])"); \
|
||
size=$$(stat -c%s "$$shard"); \
|
||
printf " %-60s docs=%-5s chunks=%-5s size=%s\n" "$$(basename $$shard)" "$$docs" "$$chunks" "$$size"; \
|
||
done
|
||
|
||
# Per-book idempotent ingest: `make textbook ID=<id>` looks up one
|
||
# entry from the manifest and runs the right ingest path (HTML
|
||
# urls vs BFS crawl). Re-running on a shard already ingested is a
|
||
# no-op at the document level — same content_root → no insert. The
|
||
# crawl path re-fetches HTTP every run; pair with `make recrawl-check`
|
||
# for a HEAD-only freshness probe before re-ingest.
|
||
textbook: bootstrap-crawler ## ingest one textbook by id: make textbook ID=bogart-ctgd-2017
|
||
@if [ -z "$(ID)" ]; then \
|
||
echo "usage: make textbook ID=<id> (use 'make textbook-list' for available ids)" >&2; \
|
||
exit 2; \
|
||
fi
|
||
@row=$$($(PY) -m bench.scripts.textbooks_manifest lookup $(ID) < $(TEXTBOOK_MANIFEST)) || exit 1; \
|
||
id=$$(echo "$$row" | cut -f1); \
|
||
url=$$(echo "$$row" | cut -f2); \
|
||
depth=$$(echo "$$row" | cut -f3); \
|
||
max=$$(echo "$$row" | cut -f4); \
|
||
license=$$(echo "$$row" | cut -f5); \
|
||
domain=$$(echo "$$row" | cut -f6); \
|
||
author=$$(echo "$$row" | cut -f7); \
|
||
mkdir -p $(CRAWL_SHARDS_DIR); \
|
||
shard="$(CRAWL_SHARDS_DIR)/textbook_$${id}.db"; \
|
||
author_flag=""; \
|
||
if [ -n "$$author" ]; then author_flag="--author $$author"; fi; \
|
||
echo ">> $$id ($$license, $$domain)"; \
|
||
echo " crawl: $$url depth=$$depth max=$$max"; \
|
||
echo " shard: $$shard"; \
|
||
if [ -n "$$author" ]; then echo " author: $$author"; fi; \
|
||
if [ -n "$$url" ]; then \
|
||
$(ARBORIST) --db "$$shard" crawl --seed-url "$$url" --depth $$depth --max-pages $$max --ingest $$author_flag > /tmp/textbook-$${id}.log 2>&1 && \
|
||
docs=$$(.venv/bin/python -c "import sqlite3; print(sqlite3.connect(\"$$shard\").execute(\"SELECT COUNT(*) FROM documents\").fetchone()[0])"); \
|
||
chunks=$$(.venv/bin/python -c "import sqlite3; print(sqlite3.connect(\"$$shard\").execute(\"SELECT COUNT(*) FROM chunks\").fetchone()[0])"); \
|
||
echo " landed: $$docs docs / $$chunks chunks"; \
|
||
else \
|
||
echo " no crawl_url for $$id; using shallow ingest path"; \
|
||
echo "$$id" | $(PY) -c 'import sys, json, os; sys.path.insert(0, "."); ids = sys.stdin.read().split(); manifest = "$(TEXTBOOK_MANIFEST)"; [print(u) for line in open(manifest) for e in [json.loads(line)] if "_meta" not in e and e.get("id") in ids for u in e.get("urls", [])]' > /tmp/textbook-$${id}-urls.txt; \
|
||
$(ARBORIST) --db "$$shard" ingest --source html --urls-from /tmp/textbook-$${id}-urls.txt $$author_flag; \
|
||
rm -f /tmp/textbook-$${id}-urls.txt; \
|
||
fi
|
||
|
||
textbook-list: bootstrap ## list ingestable textbook ids (entries with urls or crawl_url)
|
||
@$(PY) -m bench.scripts.textbooks_manifest ids < $(TEXTBOOK_MANIFEST)
|
||
|
||
# Per-book convenience targets — one per active manifest id. Re-running
|
||
# any of these is idempotent at the database layer (content-addressed
|
||
# inserts) but does refetch HTTP. Use `make recrawl-check` first if you
|
||
# want a freshness probe before paying network cost.
|
||
.PHONY: textbook-bogart textbook-keller-trotter textbook-levin \
|
||
textbook-aristotle-prior textbook-aristotle-posterior \
|
||
textbook-newton textbook-morin textbook-judson textbook-cantor \
|
||
textbook-demorgan textbook-russell-imp textbooks-base-knowledge
|
||
|
||
textbook-bogart: ## Bogart Combinatorics Through Guided Discovery (GFDL)
|
||
$(MAKE) textbook ID=bogart-ctgd-2017
|
||
|
||
textbook-keller-trotter: ## Keller-Trotter Applied Combinatorics (CC-BY-SA, slow ~25min crawl-delay)
|
||
$(MAKE) textbook ID=keller-trotter-applied-comb-2017
|
||
|
||
textbook-levin: ## Levin Discrete Mathematics (CC-BY-SA)
|
||
$(MAKE) textbook ID=levin-discrete-math-3rd
|
||
|
||
textbook-aristotle-prior: ## Aristotle Prior Analytics (PD, Wikisource)
|
||
$(MAKE) textbook ID=aristotle-prior-analytics-jenkinson
|
||
|
||
textbook-aristotle-posterior: ## Aristotle Posterior Analytics (PD, Wikisource)
|
||
$(MAKE) textbook ID=aristotle-posterior-analytics
|
||
|
||
textbook-newton: ## Newton Principia (PD, Wikisource)
|
||
$(MAKE) textbook ID=newton-principia-motte
|
||
|
||
textbook-morin: ## Morin Open Data Structures (CC-BY)
|
||
$(MAKE) textbook ID=morin-open-data-structures
|
||
|
||
textbook-judson: ## Judson Abstract Algebra: Theory and Applications (GFDL)
|
||
$(MAKE) textbook ID=judson-abstract-algebra
|
||
|
||
textbook-cantor: ## Cantor Contributions to Transfinite Numbers (PD, Jourdain transl. 1915, Wikisource)
|
||
$(MAKE) textbook ID=cantor-transfinite-numbers-jourdain
|
||
|
||
textbook-demorgan: ## De Morgan First Notions of Logic 1839 (PD, PG #67017)
|
||
$(MAKE) textbook ID=demorgan-first-notions-logic
|
||
|
||
textbook-russell-imp: ## Russell Introduction to Mathematical Philosophy 1919 (PD, PG #41654)
|
||
$(MAKE) textbook ID=russell-imp-1919
|
||
|
||
# Bulk target: ingest every base-knowledge textbook needed for the
|
||
# claim-pack warrant chains across pillars I/II/III/IX. Skips entries
|
||
# already ingested at the same content_root (idempotent at DB layer).
|
||
# Ordered cheap → expensive crawl-delay-wise; Judson last because its
|
||
# 23-chapter PreTeXt crawl is the heaviest of the new entries.
|
||
textbooks-base-knowledge: textbook-cantor textbook-demorgan textbook-russell-imp textbook-judson ## ingest the four 2026-05-09 base-knowledge additions (pillars I/II/III/IX)
|
||
@echo ">>> base-knowledge textbooks ingested: Cantor + De Morgan + Russell IMP + Judson AGT"
|
||
|
||
# TeX-source textbooks: Project Gutenberg eBooks that ship as
|
||
# LaTeX source only (no clean HTML edition). The textbook_tex
|
||
# source strips the LaTeX into plain prose via a focused PG-aware
|
||
# pipeline. Idempotent at the database layer like every other
|
||
# ingest path.
|
||
.PHONY: textbooks-tex textbook-hilbert textbook-boole
|
||
|
||
textbooks-tex: bootstrap ## ingest every manifest entry that declares a tex_url (Hilbert + Boole)
|
||
@mkdir -p $(CRAWL_SHARDS_DIR)
|
||
@$(PY) -m bench.scripts.textbooks_manifest tex-targets < $(TEXTBOOK_MANIFEST) | \
|
||
while IFS=$$'\t' read -r url id; do \
|
||
shard="$(CRAWL_SHARDS_DIR)/textbook_$${id}.db"; \
|
||
echo ">> $$id (textbook_tex)"; \
|
||
echo " url: $$url"; \
|
||
echo " shard: $$shard"; \
|
||
$(ARBORIST) --db "$$shard" ingest --source textbook_tex --url "$$url"; \
|
||
done
|
||
|
||
textbook-hilbert: ## Hilbert Foundations of Geometry (PD, PG TeX)
|
||
@row=$$(grep '"hilbert-foundations-geometry-1902"' $(TEXTBOOK_MANIFEST) | head -1); \
|
||
url=$$(echo "$$row" | $(PY) -c "import json,sys; print(json.loads(sys.stdin.read()).get('tex_url',''))"); \
|
||
if [ -z "$$url" ]; then echo "no tex_url for Hilbert" >&2; exit 1; fi; \
|
||
mkdir -p $(CRAWL_SHARDS_DIR); \
|
||
$(ARBORIST) --db "$(CRAWL_SHARDS_DIR)/textbook_hilbert-foundations-geometry-1902.db" \
|
||
ingest --source textbook_tex --url "$$url"
|
||
|
||
textbook-boole: ## Boole Laws of Thought (PD, PG TeX)
|
||
@row=$$(grep '"boole-laws-of-thought-1854"' $(TEXTBOOK_MANIFEST) | head -1); \
|
||
url=$$(echo "$$row" | $(PY) -c "import json,sys; print(json.loads(sys.stdin.read()).get('tex_url',''))"); \
|
||
if [ -z "$$url" ]; then echo "no tex_url for Boole" >&2; exit 1; fi; \
|
||
mkdir -p $(CRAWL_SHARDS_DIR); \
|
||
$(ARBORIST) --db "$(CRAWL_SHARDS_DIR)/textbook_boole-laws-of-thought-1854.db" \
|
||
ingest --source textbook_tex --url "$$url"
|
||
|
||
textbook-peano: ## Peano Arithmetices Principia 1889 (CC-BY-SA, Verheyen+Nahas LaTeX from GitHub)
|
||
@row=$$(grep '"peano-arithmetices-principia-1889"' $(TEXTBOOK_MANIFEST) | head -1); \
|
||
url=$$(echo "$$row" | $(PY) -c "import json,sys; print(json.loads(sys.stdin.read()).get('tex_url',''))"); \
|
||
if [ -z "$$url" ]; then echo "no tex_url for Peano" >&2; exit 1; fi; \
|
||
mkdir -p $(CRAWL_SHARDS_DIR); \
|
||
$(ARBORIST) --db "$(CRAWL_SHARDS_DIR)/textbook_peano-arithmetices-principia-1889.db" \
|
||
ingest --source textbook_tex --url "$$url"
|
||
|
||
textbook-russell-pom: ## Russell Principles of Mathematics 1903 (PD content + CC-BY-SA-4.0 typesetting, Klement HTML)
|
||
$(MAKE) textbook ID=russell-pom-1903
|
||
|
||
textbook-laplace: ## Laplace Philosophical Essay on Probabilities 1814 / 1902 (PD, PG #58881)
|
||
$(MAKE) textbook ID=laplace-philosophical-essay-probabilities
|
||
|
||
textbook-dedekind: ## Dedekind Essays on the Theory of Numbers 1901 (PD, PG #21016 TeX)
|
||
@row=$$(grep '"dedekind-essays-theory-numbers"' $(TEXTBOOK_MANIFEST) | head -1); \
|
||
url=$$(echo "$$row" | $(PY) -c "import json,sys; print(json.loads(sys.stdin.read()).get('tex_url',''))"); \
|
||
if [ -z "$$url" ]; then echo "no tex_url for Dedekind" >&2; exit 1; fi; \
|
||
mkdir -p $(CRAWL_SHARDS_DIR); \
|
||
$(ARBORIST) --db "$(CRAWL_SHARDS_DIR)/textbook_dedekind-essays-theory-numbers.db" \
|
||
ingest --source textbook_tex --url "$$url"
|
||
|
||
textbook-plfa: ## PLFA - Programming Language Foundations in Agda (CC-BY-4.0)
|
||
$(MAKE) textbook ID=plfa-wadler-kokke-siek
|
||
|
||
textbook-sf-lf: ## Software Foundations Vol 1 Logical Foundations (MIT, Pierce et al.)
|
||
$(MAKE) textbook ID=sf-pierce-logical-foundations
|
||
|
||
textbook-pm: ## Whitehead-Russell Principia Mathematica Vol 1 (PD, PG #78050 — preface+intro only)
|
||
$(MAKE) textbook ID=whitehead-russell-pm-vol1-1910
|
||
|
||
textbook-grinstead-snell: ## Grinstead-Snell Introduction to Probability (GFDL, LibreTexts mirror)
|
||
$(MAKE) textbook ID=grinstead-snell-intro-probability
|
||
|
||
test-crawler: bootstrap-crawler ## run only the lifted crawler tests
|
||
$(VENV)/bin/pytest -q tests/crawler
|
||
|
||
# Bridge target: BFS a seed URL, ingest discovered pages into a shard,
|
||
# capture ETag/Last-Modified for each so recrawl-check can do conditional
|
||
# HEAD requests later. URL is required; DEPTH and MAX have safe defaults.
|
||
CRAWL_DEPTH ?= 2
|
||
CRAWL_MAX ?= 0
|
||
# Per-domain shard so `make query` (which reads $(SHARDS_DIR)) sees the
|
||
# crawled content. Shard filename derived from the seed URL's hostname:
|
||
# https://russell.ballestrini.net -> $(SHARDS_DIR)/crawl_russell_ballestrini_net.db
|
||
# Override with CRAWL_SHARD=... when you want a custom path.
|
||
crawl-ingest: bootstrap-crawler ## crawl URL=https://x.com [DEPTH=2 MAX=0 FAST=1] into $(SHARDS_DIR)/crawl_<domain>.db
|
||
@if [ -z "$(URL)" ]; then echo "usage: make crawl-ingest URL=https://example.com [DEPTH=2 MAX=0 FAST=1 CRAWL_SHARD=path]" >&2; exit 2; fi
|
||
@mkdir -p $(SHARDS_DIR)
|
||
@shard="$(CRAWL_SHARD)"; \
|
||
if [ -z "$$shard" ]; then \
|
||
domain=$$(echo "$(URL)" | sed -E 's,^https?://([^/]+).*$$,\1,' | tr '.' '_'); \
|
||
shard="$(SHARDS_DIR)/crawl_$${domain}.db"; \
|
||
fi; \
|
||
echo " shard: $$shard" >&2; \
|
||
$(ARBORIST) --db "$$shard" crawl --seed-url "$(URL)" --depth $(CRAWL_DEPTH) --max-pages $(CRAWL_MAX) $(if $(FAST),--fast,) --ingest
|
||
|
||
# Fast freshness probe: conditional HEAD per doc, classify fresh/stale/gone.
|
||
# Send only If-None-Match + If-Modified-Since headers — server returns 304
|
||
# with no body when content unchanged.
|
||
RECRAWL_LIMIT ?= 100
|
||
# By default check every shard under $(SHARDS_DIR). Pass CRAWL_SHARD=path
|
||
# to scope to one. DOMAIN= filters to URLs containing the substring.
|
||
recrawl-check: bootstrap-crawler ## conditional HEAD per ingested doc [DOMAIN=x.com LIMIT=100 CRAWL_SHARD=path]
|
||
@if [ -n "$(CRAWL_SHARD)" ]; then \
|
||
$(ARBORIST) --db $(CRAWL_SHARD) crawler recrawl-check $(if $(DOMAIN),--domain $(DOMAIN),) --limit $(RECRAWL_LIMIT); \
|
||
else \
|
||
for db in $(SHARDS_DIR)/*.db; do \
|
||
case "$$(basename $$db)" in qa.db|snapshots.db) continue;; esac; \
|
||
echo " shard: $$db" >&2; \
|
||
$(ARBORIST) --db "$$db" crawler recrawl-check $(if $(DOMAIN),--domain $(DOMAIN),) --limit $(RECRAWL_LIMIT); \
|
||
done; \
|
||
fi
|
||
|
||
DOT_SRCS := $(wildcard docs/diagrams/*.dot)
|
||
DOT_PNGS := $(DOT_SRCS:.dot=.png)
|
||
DOT_SVGS := $(DOT_SRCS:.dot=.svg)
|
||
|
||
docs/diagrams/%.png: docs/diagrams/%.dot
|
||
dot -Tpng $< -o $@
|
||
|
||
docs/diagrams/%.svg: docs/diagrams/%.dot
|
||
dot -Tsvg $< -o $@
|
||
|
||
docs: $(DOT_PNGS) $(DOT_SVGS) ## render docs/diagrams/*.dot -> .png + .svg via graphviz
|
||
|
||
docs-api: ## generate Sphinx API reference from docstrings (output: docs/_source/_build/html/)
|
||
$(PY) -m sphinx.cmd.build -b html docs/_source docs/_source/_build/html
|
||
|
||
docs-api-clean: ## remove Sphinx build artifacts
|
||
rm -rf docs/_source/_build/
|
||
|
||
# Reproducible micro-benchmark over a fixed slice of cur. Lets you compare
|
||
# ETL throughput across configs and catches regressions on optimization
|
||
# work. Override BENCH_DOCS=N (default 5000).
|
||
BENCH_DOCS ?= 5000
|
||
BENCH_DIR := /tmp/arborist-bench
|
||
bench: bootstrap fetch-cur ## benchmark serial vs parallel-shared vs attached at $(BENCH_DOCS) docs
|
||
@bash bench/run.sh $(BENCH_DOCS)
|
||
|
||
clean: ## remove venv + caches (keeps fetched data and db)
|
||
rm -rf $(VENV) .pytest_cache **/__pycache__ arborist.egg-info
|
||
find . -type d -name __pycache__ -prune -exec rm -rf {} +
|
||
|
||
clean-db: ## drop the arborist db (keeps fetched data and venv)
|
||
rm -f $(DB) $(DB)-journal $(DB)-wal $(DB)-shm
|
||
|
||
clean-data: ## remove fetched dumps
|
||
rm -rf $(DATA_DIR)
|