make bench-cloud-vs-local: timed head-to-head, local --burn vs cloud-ask
Both paths grind on the same data (LOCAL_DB defaults to web.db, the file that was uploaded to the Spaces shard at SHARD_URL). --burn busts the local QA cache so we measure a fresh inference both sides; the cloud path has no caching layer so it's always fresh. Reports answer + audit_mode + verifier_method + sources + wall time for each path, then prints local-vs-cloud delta + ratio. Sample run (Q='who developed virt-back?'): local: 3.8s STRICT (verbatim quote verified) cloud: 11.8s HYBRID (entity-name verified) delta: +8.0s (3.10x — the HTTP RANGE * ~99 requests tax) Cloud audit_mode lands one rung lower because the cloud-ask prompt is simpler than query.py's full pipeline, so the LLM paraphrases instead of quoting. Verifier honestly demotes paraphrase to HYBRID. Both answers are correct; the audit_mode difference is a prompt-shape artifact, not a cloud-path correctness gap.
This commit is contained in:
parent
65b6fe697d
commit
462f639163
1 changed files with 37 additions and 0 deletions
37
Makefile
37
Makefile
|
|
@ -1580,6 +1580,43 @@ cloud-ask: bootstrap ## bucket-direct end-to-end: FTS → LLM → verify [Q="...
|
|||
--max-context-chars $(or $(MAX_CONTEXT),24000) \
|
||||
--cache-mb $(or $(CACHE_MB),64)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Head-to-head: local query (BURN=1) vs cloud-ask, same question, same data.
|
||||
# ---------------------------------------------------------------------------
|
||||
# Defaults align the two endpoints: LOCAL_DB is the same file that was
|
||||
# uploaded to Spaces under SHARD_URL, so the two paths grind on the
|
||||
# byte-identical corpus. --burn busts the local QA cache so we time
|
||||
# a fresh inference both sides.
|
||||
|
||||
LOCAL_DB ?= $(HOME)/.arborist/crawl/web.db
|
||||
|
||||
bench-cloud-vs-local: bootstrap ## head-to-head: local query --burn vs cloud-ask, same question, timed [Q="..."]
|
||||
@test -n "$(Q)" || { echo 'usage: make bench-cloud-vs-local Q="your question" [LOCAL_DB=path] [SHARD_URL=https://...]'; exit 2; }
|
||||
@echo "=== bench-cloud-vs-local ==="
|
||||
@echo "Q: $(Q)"
|
||||
@echo "LOCAL_DB: $(LOCAL_DB)"
|
||||
@echo "SHARD_URL: $(SHARD_URL)"
|
||||
@echo
|
||||
@echo '--- LOCAL (--burn, fresh inference) ---'
|
||||
@T0=$$(date +%s.%N); \
|
||||
$(ARBORIST) --db $(LOCAL_DB) query --top-k 4 --burn --json '$(Q)' > /tmp/.bench-local.json 2>&1 || true; \
|
||||
T1=$$(date +%s.%N); \
|
||||
LOCAL_DT=$$(awk "BEGIN { print $$T1 - $$T0 }"); \
|
||||
$(PY) -c "import json; d=json.load(open('/tmp/.bench-local.json')); print('answer: ', d.get('answer_text','').strip()[:200]); print('audit: ', d.get('audit_mode')); print('verifier: ', d.get('verifier_method')); print('n_verified:', d.get('n_verified'), '/', d.get('n_quotes')); s=d.get('sources') or []; print('sources: ', len(s), '→', s[0]['document_uri'] if s else '(none)')"; \
|
||||
printf 'wall: %.3fs\n' "$$LOCAL_DT"; \
|
||||
echo; \
|
||||
echo '--- CLOUD (bucket-direct, always fresh) ---'; \
|
||||
T0=$$(date +%s.%N); \
|
||||
$(ARBORIST) cloud ask '$(Q)' --shard-url "$(SHARD_URL)" --top-k 4 > /tmp/.bench-cloud.json 2>&1 || true; \
|
||||
T1=$$(date +%s.%N); \
|
||||
CLOUD_DT=$$(awk "BEGIN { print $$T1 - $$T0 }"); \
|
||||
$(PY) -c "import json; d=json.load(open('/tmp/.bench-cloud.json')); print('answer: ', d.get('answer_text','').strip()[:200]); print('audit: ', d.get('audit_mode')); print('verifier: ', d.get('verifier_method')); print('n_verified:', d.get('n_verified'), '/', d.get('n_quotes')); s=d.get('sources') or []; print('sources: ', len(s), '→', s[0]['document_uri'] if s else '(none)'); st=d.get('stats') or {}; print('http_reqs: ', st.get('http_requests')); cs=st.get('cache') or {}; print('bytes: ', cs.get('bytes_fetched'))"; \
|
||||
printf 'wall: %.3fs\n' "$$CLOUD_DT"; \
|
||||
echo; \
|
||||
echo "=== summary ==="; \
|
||||
awk "BEGIN { printf \" local: %.3fs\\n cloud: %.3fs (delta %+.3fs, ratio %.2fx)\\n\", $$LOCAL_DT, $$CLOUD_DT, $$CLOUD_DT-$$LOCAL_DT, $$CLOUD_DT/$$LOCAL_DT }"; \
|
||||
rm -f /tmp/.bench-local.json /tmp/.bench-cloud.json
|
||||
|
||||
CLOUD_DEMO_PORT ?= 18785
|
||||
|
||||
cloud-demo: bootstrap ## bucket-direct end-to-end proof on tiny in-process corpus [CLOUD_DEMO_PORT=N]
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue