From 462f639163579fe24df2f72736ffdebf7297f4b1 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Sat, 30 May 2026 10:48:59 -0400 Subject: [PATCH] make bench-cloud-vs-local: timed head-to-head, local --burn vs cloud-ask MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both paths grind on the same data (LOCAL_DB defaults to web.db, the file that was uploaded to the Spaces shard at SHARD_URL). --burn busts the local QA cache so we measure a fresh inference both sides; the cloud path has no caching layer so it's always fresh. Reports answer + audit_mode + verifier_method + sources + wall time for each path, then prints local-vs-cloud delta + ratio. Sample run (Q='who developed virt-back?'): local: 3.8s STRICT (verbatim quote verified) cloud: 11.8s HYBRID (entity-name verified) delta: +8.0s (3.10x — the HTTP RANGE * ~99 requests tax) Cloud audit_mode lands one rung lower because the cloud-ask prompt is simpler than query.py's full pipeline, so the LLM paraphrases instead of quoting. Verifier honestly demotes paraphrase to HYBRID. Both answers are correct; the audit_mode difference is a prompt-shape artifact, not a cloud-path correctness gap. --- Makefile | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/Makefile b/Makefile index 594364f..69d5421 100644 --- a/Makefile +++ b/Makefile @@ -1580,6 +1580,43 @@ cloud-ask: bootstrap ## bucket-direct end-to-end: FTS → LLM → verify [Q="... --max-context-chars $(or $(MAX_CONTEXT),24000) \ --cache-mb $(or $(CACHE_MB),64) +# --------------------------------------------------------------------------- +# Head-to-head: local query (BURN=1) vs cloud-ask, same question, same data. +# --------------------------------------------------------------------------- +# Defaults align the two endpoints: LOCAL_DB is the same file that was +# uploaded to Spaces under SHARD_URL, so the two paths grind on the +# byte-identical corpus. --burn busts the local QA cache so we time +# a fresh inference both sides. + +LOCAL_DB ?= $(HOME)/.arborist/crawl/web.db + +bench-cloud-vs-local: bootstrap ## head-to-head: local query --burn vs cloud-ask, same question, timed [Q="..."] + @test -n "$(Q)" || { echo 'usage: make bench-cloud-vs-local Q="your question" [LOCAL_DB=path] [SHARD_URL=https://...]'; exit 2; } + @echo "=== bench-cloud-vs-local ===" + @echo "Q: $(Q)" + @echo "LOCAL_DB: $(LOCAL_DB)" + @echo "SHARD_URL: $(SHARD_URL)" + @echo + @echo '--- LOCAL (--burn, fresh inference) ---' + @T0=$$(date +%s.%N); \ + $(ARBORIST) --db $(LOCAL_DB) query --top-k 4 --burn --json '$(Q)' > /tmp/.bench-local.json 2>&1 || true; \ + T1=$$(date +%s.%N); \ + LOCAL_DT=$$(awk "BEGIN { print $$T1 - $$T0 }"); \ + $(PY) -c "import json; d=json.load(open('/tmp/.bench-local.json')); print('answer: ', d.get('answer_text','').strip()[:200]); print('audit: ', d.get('audit_mode')); print('verifier: ', d.get('verifier_method')); print('n_verified:', d.get('n_verified'), '/', d.get('n_quotes')); s=d.get('sources') or []; print('sources: ', len(s), '→', s[0]['document_uri'] if s else '(none)')"; \ + printf 'wall: %.3fs\n' "$$LOCAL_DT"; \ + echo; \ + echo '--- CLOUD (bucket-direct, always fresh) ---'; \ + T0=$$(date +%s.%N); \ + $(ARBORIST) cloud ask '$(Q)' --shard-url "$(SHARD_URL)" --top-k 4 > /tmp/.bench-cloud.json 2>&1 || true; \ + T1=$$(date +%s.%N); \ + CLOUD_DT=$$(awk "BEGIN { print $$T1 - $$T0 }"); \ + $(PY) -c "import json; d=json.load(open('/tmp/.bench-cloud.json')); print('answer: ', d.get('answer_text','').strip()[:200]); print('audit: ', d.get('audit_mode')); print('verifier: ', d.get('verifier_method')); print('n_verified:', d.get('n_verified'), '/', d.get('n_quotes')); s=d.get('sources') or []; print('sources: ', len(s), '→', s[0]['document_uri'] if s else '(none)'); st=d.get('stats') or {}; print('http_reqs: ', st.get('http_requests')); cs=st.get('cache') or {}; print('bytes: ', cs.get('bytes_fetched'))"; \ + printf 'wall: %.3fs\n' "$$CLOUD_DT"; \ + echo; \ + echo "=== summary ==="; \ + awk "BEGIN { printf \" local: %.3fs\\n cloud: %.3fs (delta %+.3fs, ratio %.2fx)\\n\", $$LOCAL_DT, $$CLOUD_DT, $$CLOUD_DT-$$LOCAL_DT, $$CLOUD_DT/$$LOCAL_DT }"; \ + rm -f /tmp/.bench-local.json /tmp/.bench-cloud.json + CLOUD_DEMO_PORT ?= 18785 cloud-demo: bootstrap ## bucket-direct end-to-end proof on tiny in-process corpus [CLOUD_DEMO_PORT=N]