diff --git a/Makefile b/Makefile index 0c473f9..0f10f1a 100644 --- a/Makefile +++ b/Makefile @@ -1552,7 +1552,7 @@ wallet-ask: bootstrap ## query the wallet server with your own question [Q="..." SHARD_URL ?= https://nyc3.digitaloceanspaces.com/arborist/clones/virtback/000.db BLOB_BASE ?= -# BUCKET_URL is the manifest the multi-shard cloud-ask points at. Client +# BUCKET_URL is the manifest the multi-shard cloud-query points at. Client # GETs $BUCKET_URL (treated as a manifest URL if it ends in .json, else # appends `clones/manifest.json`) and queries every listed shard. # Default = sidecar-enabled manifest on DO Spaces: per-shard inverted- @@ -1590,17 +1590,17 @@ cloud-fetch-chunk: bootstrap ## fetch + hash-verify one chunk from a bucket [LEA @test -n "$(BLOB_BASE)" || { echo 'BLOB_BASE required (per-chunk blobs/ base URL)'; exit 2; } @$(ARBORIST) cloud fetch-chunk "$(LEAF_HASH)" --blob-base "$(BLOB_BASE)" -cloud-ask: bootstrap ## bucket-direct end-to-end via manifest [Q="..." JSON=1 BUCKET_URL=... TOP_K=N MAX_CONTEXT=N CACHE_MB=N] - @test -n "$(Q)" || { echo 'usage: make cloud-ask Q="your question" [JSON=1] [BUCKET_URL=...] [TOP_K=4] [MAX_CONTEXT=24000] [CACHE_MB=64]'; exit 2; } +cloud-query: bootstrap ## bucket-direct end-to-end via manifest [Q="..." JSON=1 BUCKET_URL=... TOP_K=N MAX_CONTEXT=N CACHE_MB=N] + @test -n "$(Q)" || { echo 'usage: make cloud-query Q="your question" [JSON=1] [BUCKET_URL=...] [TOP_K=4] [MAX_CONTEXT=24000] [CACHE_MB=64]'; exit 2; } @echo "# bucket: $(BUCKET_URL)" >&2 - @$(ARBORIST) cloud ask '$(Q)' --bucket-url "$(BUCKET_URL)" \ + @$(ARBORIST) cloud query '$(Q)' --bucket-url "$(BUCKET_URL)" \ --top-k $(or $(TOP_K),4) \ --max-context-chars $(or $(MAX_CONTEXT),24000) \ --cache-mb $(or $(CACHE_MB),64) \ $(if $(JSON),--json,) # --------------------------------------------------------------------------- -# Head-to-head: local query (BURN=1) vs cloud-ask, same question, same data. +# Head-to-head: local query (BURN=1) vs cloud-query, same question, same data. # --------------------------------------------------------------------------- # Defaults align the two endpoints: LOCAL_DB is the same file that was # uploaded to Spaces under SHARD_URL, so the two paths grind on the @@ -1609,7 +1609,7 @@ cloud-ask: bootstrap ## bucket-direct end-to-end via manifest [Q="..." JSON=1 BU LOCAL_DB ?= $(HOME)/.arborist/crawl/web.db -bench-cloud-vs-local: bootstrap ## head-to-head: local query --burn vs cloud-ask, same question, timed [Q="..."] +bench-cloud-vs-local: bootstrap ## head-to-head: local query --burn vs cloud-query, same question, timed [Q="..."] @test -n "$(Q)" || { echo 'usage: make bench-cloud-vs-local Q="your question" [LOCAL_DB=path] [SHARD_URL=https://...]'; exit 2; } @echo "=== bench-cloud-vs-local ===" @echo "Q: $(Q)" @@ -1626,7 +1626,7 @@ bench-cloud-vs-local: bootstrap ## head-to-head: local query --burn vs cloud-ask echo; \ echo '--- CLOUD (bucket-direct, always fresh) ---'; \ T0=$$(date +%s.%N); \ - $(ARBORIST) cloud ask '$(Q)' --shard-url "$(SHARD_URL)" --top-k 4 > /tmp/.bench-cloud.json 2>&1 || true; \ + $(ARBORIST) cloud query '$(Q)' --shard-url "$(SHARD_URL)" --top-k 4 > /tmp/.bench-cloud.json 2>&1 || true; \ T1=$$(date +%s.%N); \ CLOUD_DT=$$(awk "BEGIN { print $$T1 - $$T0 }"); \ $(PY) -c "import json; d=json.load(open('/tmp/.bench-cloud.json')); print('answer: ', d.get('answer_text','').strip()[:200]); print('audit: ', d.get('audit_mode')); print('verifier: ', d.get('verifier_method')); print('n_verified:', d.get('n_verified'), '/', d.get('n_quotes')); s=d.get('sources') or []; print('sources: ', len(s), '→', s[0]['document_uri'] if s else '(none)'); st=d.get('stats') or {}; print('http_reqs: ', st.get('http_requests')); cs=st.get('cache') or {}; print('bytes: ', cs.get('bytes_fetched'))"; \ diff --git a/arborist/cli.py b/arborist/cli.py index abf9467..dcd9d5a 100644 --- a/arborist/cli.py +++ b/arborist/cli.py @@ -7264,48 +7264,48 @@ def build_parser() -> argparse.ArgumentParser: cloud_fetch.add_argument("--blob-base", required=True) cloud_fetch.set_defaults(func=_cmd_cloud_fetch_chunk) - cloud_ask = cloud_sub.add_parser( - "ask", + cloud_query = cloud_sub.add_parser( + "query", help=( - "bucket-direct end-to-end ask: FTS → pull chunks via SQL → " + "bucket-direct end-to-end query: FTS → pull chunks via SQL → " "LLM call → verify quotes locally. Same audited-answer shape " "as `arborist query` but no local DB." ), ) - cloud_ask.add_argument("question", type=str) - cloud_ask.add_argument( + cloud_query.add_argument("question", type=str) + cloud_query.add_argument( "--bucket-url", default=None, help="bucket root URL (e.g. https://nyc3.digitaloceanspaces.com/arborist). " "The client fetches `clones/manifest.json` from this URL and queries " "every listed shard. Use this for multi-shard corpus access.", ) - cloud_ask.add_argument( + cloud_query.add_argument( "--shard-url", default=None, help="single shard URL (mutually exclusive with --bucket-url). " "Use when you want to point at exactly one .db file on a bucket.", ) - cloud_ask.add_argument("--top-k", type=int, default=4) - cloud_ask.add_argument("--max-context-chars", type=int, default=24_000) - cloud_ask.add_argument( + cloud_query.add_argument("--top-k", type=int, default=4) + cloud_query.add_argument("--max-context-chars", type=int, default=24_000) + cloud_query.add_argument( "--cache-mb", type=int, default=64, help="LRU page cache size in MB (default 64 — bigger than search " "default; chunk-content reads dominate).", ) - cloud_ask.add_argument( + cloud_query.add_argument( "--endpoint", default=None, help="LLM endpoint (default: $ARBORIST_LLM_ENDPOINT or " "https://hermes.ai.unturf.com/v1).", ) - cloud_ask.add_argument( + cloud_query.add_argument( "--model", default=None, help="LLM model id (default: $ARBORIST_LLM_MODEL or Hermes-3-8B).", ) - cloud_ask.add_argument( + cloud_query.add_argument( "--json", action="store_true", help="emit the full result as JSON. Default: human-rendered " "(matches `arborist query` output).", ) - cloud_ask.set_defaults(func=_cmd_cloud_ask) + cloud_query.set_defaults(func=_cmd_cloud_query) sidecar_cmd = sub.add_parser( "sidecar", @@ -7481,7 +7481,7 @@ def _cmd_cloud_snapshot_root(args: argparse.Namespace) -> int: return 0 -def _cmd_cloud_ask(args: argparse.Namespace) -> int: +def _cmd_cloud_query(args: argparse.Namespace) -> int: """Bucket-direct end-to-end ask: FTS → fetch chunks → LLM (claim- lattice mode) → verify. Same audited-answer shape as local `query()` — EVIDENCE blocks with E1/E2 pointer tags, pointer-line @@ -7535,7 +7535,7 @@ def _cmd_cloud_ask(args: argparse.Namespace) -> int: else: if not getattr(args, "shard_url", None): print( - "cloud ask needs --bucket-url (multi-shard manifest) " + "cloud query needs --bucket-url (multi-shard manifest) " "or --shard-url (single .db file).", file=sys.stderr, ) @@ -7738,12 +7738,12 @@ def _cmd_cloud_ask(args: argparse.Namespace) -> int: if getattr(args, "json", False): print(json.dumps(result, indent=2, ensure_ascii=False)) else: - print(_render_cloud_ask_human(result, args.question)) + print(_render_cloud_query_human(result, args.question)) return 0 -def _render_cloud_ask_human(result: dict, question: str) -> str: - """Pretty-print a cloud-ask result. Mirrors `_render_query_human`'s +def _render_cloud_query_human(result: dict, question: str) -> str: + """Pretty-print a cloud-query result. Mirrors `_render_query_human`'s layout — same audit-label ladder, same source roles + used / pointer annotations — but trims sections the bucket-direct path doesn't produce (warrant tails, run-DAG, retrieval-purity).""" diff --git a/arborist/wallet/sidecar.py b/arborist/wallet/sidecar.py index 6a15e40..dce5459 100644 --- a/arborist/wallet/sidecar.py +++ b/arborist/wallet/sidecar.py @@ -1,6 +1,6 @@ """HTTP-optimized inverted-index sidecar for bucket-direct queries. -The bucket-direct cloud-ask path runs FTS5 against a SQLite .db file +The bucket-direct cloud-query path runs FTS5 against a SQLite .db file served over HTTP RANGE. That's correct but slow on multi-GB shards: FTS5's b-tree walk touches hundreds of random pages per query, each costing a WAN round-trip. We measured 4+ minutes per query on a 9 GB @@ -55,7 +55,7 @@ Conventions: - term comparison: case-folded UTF-8 byte order (matches tokenize_text) - doc_ids are dense [0, docs_count) per sidecar - varint = LEB128 unsigned (per-byte 0..127; high bit = more bytes) -- multi-term AND via posting-list intersection (default cloud-ask +- multi-term AND via posting-list intersection (default cloud-query semantics is OR; sidecar supports both via SidecarReader API) This is NOT a Merkle-bound artifact. It's a soft index — the doc_root