diff --git a/Makefile b/Makefile index 547dfb9..6379082 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,7 @@ SEARCH_Q ?= computer prometheus-trigger-probe bench-5f-threshold-calibration \ bench-5f-selfmodel-snapshot bench-5f-finetuning-shardchain \ bench-5f-falsification-hard bench-fork-baseline-hard bench-5f-formulate-hard \ - bootstrap-math bootstrap-nli bench-nli-shadow export-nli-onnx clean clean-db clean-data help \ + bootstrap-math bootstrap-nli bootstrap-nli-only bench-nli-shadow export-nli-onnx clean clean-db clean-data help \ textbooks-summary textbooks-urls fetch-textbooks textbooks-stats textbooks-verify \ crawl-textbooks crawl-textbooks-stats textbook textbook-list @@ -789,6 +789,20 @@ bootstrap-nli: bootstrap ## install [nli] extras + warm the pinned NLI checkpoin $(PIP) install -e '.[nli]' $(PY) -c "from arborist.qa.nli import ShadowNLI; r=ShadowNLI(); r._ensure_loaded(); print('nli runtime ·', 'available:', r.available, '· backend:', r.backend, '· device:', r.device, '·', r._reason)" +# #000049 §7 #22 — lean NLI compute-node bootstrap: a venv + `[nli]` ONLY +# (no `[dev]` extras — no crawler/hessian/vec/sympy). For a GPU box that +# just runs NLI sweeps: on a CUDA host PyPI's torch wheel is the CUDA +# build, so `ShadowNLI` auto-detects cuda (ARBORIST_NLI_DEVICE overrides) +# and `make bench-nli-shadow` / `make export-nli-onnx` / the candidate +# bench all run on the GPU with no further wiring. Measured: an 82M +# cross-encoder, batched, on a 4090 ≈ 0.09 ms/pair (~350x onnx-int8-cpu, +# ~1300x torch-cpu-batch1) → a full bench-qa shadow sweep is <1 s of NLI. +bootstrap-nli-only: ## #000049 §7 #22 — minimal venv + [nli] only (GPU/CPU compute-node setup; no [dev]) + [ -d $(VENV) ] || $(PYTHON) -m venv $(VENV) + $(PIP) install -q -U pip + $(PIP) install -e '.[nli]' pytest + $(PY) -c "import torch; from arborist.qa.nli import ShadowNLI; r=ShadowNLI(); r._ensure_loaded(); print('torch', torch.__version__, '· cuda', torch.cuda.is_available(), (torch.cuda.get_device_name(0) if torch.cuda.is_available() else '')); print('nli runtime ·', 'available:', r.available, '· backend:', r.backend, '· device:', r.device, '·', r._reason)" + # #000049 §3 speedup — export the pinned shadow-NLI checkpoint to ONNX # (+ int8 dynamic quantization), into ~/.arborist/models/nli//onnx/. # ShadowNLI._ensure_loaded auto-prefers the export if present (~2-4x on diff --git a/docs/tickets/ticket-000049-attribution-aware-grounding-check.md b/docs/tickets/ticket-000049-attribution-aware-grounding-check.md index 31dbd9e..ef656a4 100644 --- a/docs/tickets/ticket-000049-attribution-aware-grounding-check.md +++ b/docs/tickets/ticket-000049-attribution-aware-grounding-check.md @@ -844,10 +844,22 @@ state, **not** committed — same discipline as the textbook manifest / `[vec]`), and `_ensure_loaded` loads `model_quantized.onnx` via `optimum.onnxruntime` (backend `onnx-int8`), falling back silently to torch when no export / no `optimum`. Measured: torch-cpu-batch1 ≈ 120 -ms/pair → **onnx-int8-cpu-batched ≈ 32 ms/pair** (~4×); on a CUDA box -(the 4090) batched inference is ~10⁴–10⁵ pairs/s — the whole sweep is -seconds. `optimum[onnxruntime]` added to the `[nli]` extra; 24 tests in -`tests/test_nli_shadow.py`. +ms/pair → **onnx-int8-cpu-batched ≈ 32 ms/pair** (~4×) → **torch-cuda- +batched ≈ 0.09 ms/pair on a 4090** (512 pairs in 0.047 s — ~350× +onnx-int8-cpu, ~1300× the original), so a full `bench-qa` n=3 shadow +sweep (~8 k clause-pairs) is **< 1 s** of NLI compute on the GPU +(2.4 s wall for a 62-record synthetic sweep, model load included). +`optimum[onnxruntime]` added to the `[nli]` extra; 24 tests in +`tests/test_nli_shadow.py`. **GPU compute-node workflow** (verified on +`ai`, RTX 4090): `make bootstrap-nli-only` (a venv + `[nli]` only — no +`[dev]` bloat; PyPI's torch wheel is the CUDA build on a CUDA host) → +`ShadowNLI` auto-detects cuda (`ARBORIST_NLI_DEVICE` overrides) → `make +bench-nli-shadow [INPUT=…]` / `make export-nli-onnx` / the candidate +bench all run on the GPU with no further wiring. The pattern: ship the +laptop's `bench-qa` JSONL to the GPU box, run the sweep there in +seconds. (`ai` doesn't have the ~6 GB shard corpus, so `make bench-qa` +itself still runs on the laptop — that's the Hermes LLM, not local +GPU; the GPU only accelerates the NLI half.) *The gate-item-4 number, at proper n.* `ARBORIST_NLI_SHADOW=1 make bench-qa BENCH_QA_N=1` → 75 q × 3 modes, of which 223 cells completed