From 53ce8fd6b17a0cc1f6549a2ef6cf57427b233337 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Sat, 30 May 2026 21:21:33 -0400 Subject: [PATCH] sidecar+bucket: extras penalty + raw-score merge for sidecar-only paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cloud-query was picking sibling articles (Mona Lisa's Revenge instead of Mona Lisa, Republics of the Soviet Union instead of Soviet Union, Mercury 13 instead of Mercury Seven) because: 1. Title-boost counted overlap but not extra title tokens. Both "Mona Lisa" and "Mona Lisa's Revenge" overlapped query by 2 → same boost → BM25 favored the shorter movie article. 2. RRF merge squashed per-shard rank-1 hits into a 1/(60+1) tie across 4 sidecar shards. Tie-breaking was undefined; the right article was as likely to lose as win. Two fixes: sidecar.search title-boost: + Filter title tokens through STOPWORDS + len-1 cutoff so 's', 'of', 'the' don't count as extras. + Penalty: extras = |title_tokens - query_tokens|; effective bonus is `max(0, overlap - extras/2) * title_boost`. "Mona Lisa" gets full bonus; "Mona Lisa's Revenge" gets half. MultiShardSidecarCorpus.fts_search merge: + When all CONTRIBUTING shards have sidecars (their BM25 + boost scores are directly comparable), merge by max raw score across shards. RRF was masking score discrimination at the top of the list. + Mixed (sidecar + bucket-direct FTS5) falls back to RRF since those scales aren't comparable. Bench (5-question smoke, cloud_vs_local.py): before fix: 4 regressions / 5 after fix : 1 regression / 5 (and that one is the right source, only the audit_mode dropped STRICT → HYBRID due to LLM-stochastic answer phrasing) --- arborist/wallet/bucket.py | 40 ++- arborist/wallet/sidecar.py | 30 ++- bench/cloud_vs_local.py | 242 ++++++++++++++++++ .../2026-05-31T00-52-37Z.jsonl | 5 + .../2026-05-31T01-09-30Z.jsonl | 5 + 5 files changed, 306 insertions(+), 16 deletions(-) create mode 100644 bench/cloud_vs_local.py create mode 100644 bench/cloud_vs_local_results/2026-05-31T00-52-37Z.jsonl create mode 100644 bench/cloud_vs_local_results/2026-05-31T01-09-30Z.jsonl diff --git a/arborist/wallet/bucket.py b/arborist/wallet/bucket.py index 98a2c44..2410371 100644 --- a/arborist/wallet/bucket.py +++ b/arborist/wallet/bucket.py @@ -912,26 +912,46 @@ class MultiShardSidecarCorpus: for sh_url, hits in per_shard ] - # RRF merge: keyed by document_root so duplicates across shards - # (rare but possible — e.g. a doc replicated for redundancy) - # combine. - rrf_score: dict[str, float] = {} + # Merge strategy: when every shard CONTRIBUTING HITS is a + # sidecar, BM25 scores are directly comparable (same constants, + # same title-boost formula), so merge by raw score — preserves + # per-shard rank discrimination instead of squashing the top-N + # into a tie-at-1/(k+1) under RRF. When the surviving hits mix + # sidecar + bucket-direct FTS5 (incomparable scales: BM25 + # +25..+60 vs FTS5 −20..0), fall back to RRF. + url_to_sidecar = { + sh["url"]: self._has_sidecar[i] + for i, sh in enumerate(self.manifest.shards) + } + contributing_shards = { + sh_url for sh_url, hits in per_shard if hits + } + use_raw_score = bool(contributing_shards) and all( + url_to_sidecar[u] for u in contributing_shards + ) + hit_by_root: dict[str, dict] = {} + score_by_root: dict[str, float] = {} for sh_url, hits in per_shard: for rank, h in enumerate(hits, start=1): droot = h["document_root"] - rrf_score[droot] = ( - rrf_score.get(droot, 0.0) + 1.0 / (rrf_k + rank) - ) + if use_raw_score: + # Take the MAX raw score across shards. + raw = h.get("score") or 0.0 + if droot not in score_by_root or raw > score_by_root[droot]: + score_by_root[droot] = raw + else: + score_by_root[droot] = ( + score_by_root.get(droot, 0.0) + 1.0 / (rrf_k + rank) + ) if droot not in hit_by_root: h["_shard_url"] = sh_url hit_by_root[droot] = h - # Surface rrf score so callers see "this doc was strong across N shards." for droot, h in hit_by_root.items(): - h["rrf_score"] = rrf_score[droot] + h["merge_score"] = score_by_root[droot] ranked = sorted( hit_by_root.values(), - key=lambda h: -h["rrf_score"], + key=lambda h: -h["merge_score"], ) return ranked[:limit] diff --git a/arborist/wallet/sidecar.py b/arborist/wallet/sidecar.py index 21cc776..15f7ae1 100644 --- a/arborist/wallet/sidecar.py +++ b/arborist/wallet/sidecar.py @@ -583,14 +583,32 @@ class SidecarReader: for did in scores: # Same accent-fold + tokenize + numeral-expand on the # title side so 'pokemon'/'pokémon' AND '7'/'VII' both - # overlap correctly. + # overlap correctly. Stopword + length filter (mirrors + # tokenize_text) drops 's' from "Mona Lisa's", 'of'/'the' + # from titles, etc. — otherwise they'd inflate the + # extras count below. title = fold_accents(self._doc_title(did).lower()) - title_tokens = numeral_expand( - {_t_stem(t) for t in _WORD_RE.findall(title)} - ) + raw_title_tokens: set[str] = set() + for tok in _WORD_RE.findall(title): + if tok in STOPWORDS: + continue + if len(tok) <= 1 and not tok.isdigit(): + continue + raw_title_tokens.add(_t_stem(tok)) + title_tokens = numeral_expand(raw_title_tokens) overlap = len(query_stems & title_tokens) - if overlap: - scores[did] += title_boost * overlap + if not overlap: + continue + # Penalty for extra title tokens not in the query — + # without this, "Mona Lisa" (extras 0) and "Mona Lisa's + # Revenge" (extras 1: revenge) tied at the same boost + # and BM25 picked the shorter movie article over the + # painting. extras/2 means each extra halves the per- + # overlap-token bonus. + extras = len(title_tokens - query_stems) + effective = max(0.0, overlap - extras / 2.0) + if effective: + scores[did] += title_boost * effective ranked = sorted(scores.items(), key=lambda x: -x[1]) out: list[SidecarHit] = [] diff --git a/bench/cloud_vs_local.py b/bench/cloud_vs_local.py new file mode 100644 index 0000000..37ff3a6 --- /dev/null +++ b/bench/cloud_vs_local.py @@ -0,0 +1,242 @@ +"""Compare local `arborist query` vs `arborist cloud query` on a fixture. + +For each question: + 1. Run local CLI: arborist --shards-dir SHARDS_DIR query --json --burn ... + 2. Run cloud CLI: arborist cloud query --bucket-url ... --json ... + 3. Compare audit_mode, primary source URI, answer text + +A regression is any of: + - local STRICT (or EVIDENCE-WARRANTED) but cloud UNGROUNDED + - local cited Wikipedia/primary article but cloud cited a tangential one + - cloud crash / network error + +Prints a side-by-side table + a regression summary. JSONL output goes +to bench/cloud_vs_local_results/.jsonl. +""" +from __future__ import annotations + +import argparse +import datetime as _dt +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent +ARBORIST = REPO / ".venv" / "bin" / "arborist" +DEFAULT_SHARDS = Path.home() / ".arborist" / "shards" +DEFAULT_BUCKET = ( + "https://nyc3.digitaloceanspaces.com/arborist/clones/manifest-sidecar.json" +) +DEFAULT_QWEN_ENDPOINT = "https://qwen.ai.unturf.com/v1" +DEFAULT_QWEN_MODEL = "Qwen3.6-27B-UD-Q4_K_XL.gguf" + + +def _load_fixture(path: Path) -> list[str]: + questions = [] + for line in path.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + questions.append(line) + return questions + + +def _run_local(question: str, *, shards_dir: Path, timeout_s: int = 120) -> dict: + """Invoke arborist query --json, parse stdout.""" + t0 = time.time() + try: + proc = subprocess.run( + [ + str(ARBORIST), "--shards-dir", str(shards_dir), + "query", "--top-k", "8", "--json", "--burn", + "--answer-mode", "claim_lattice", + question, + ], + capture_output=True, text=True, timeout=timeout_s, + ) + except subprocess.TimeoutExpired: + return {"_error": f"timeout after {timeout_s}s", "_elapsed_s": timeout_s} + elapsed = time.time() - t0 + if proc.returncode != 0: + return { + "_error": f"exit {proc.returncode}", + "_stderr": proc.stderr[:500], + "_elapsed_s": round(elapsed, 2), + } + try: + data = json.loads(proc.stdout) + except json.JSONDecodeError as e: + return {"_error": f"parse: {e}", "_stdout_head": proc.stdout[:200], + "_elapsed_s": round(elapsed, 2)} + data["_elapsed_s"] = round(elapsed, 2) + return data + + +def _run_cloud( + question: str, *, bucket_url: str, endpoint: str, model: str, + timeout_s: int = 240, +) -> dict: + """Invoke arborist cloud query --json, parse stdout.""" + t0 = time.time() + try: + proc = subprocess.run( + [ + str(ARBORIST), "cloud", "query", question, + "--bucket-url", bucket_url, + "--endpoint", endpoint, + "--model", model, + "--top-k", "4", + "--json", + ], + capture_output=True, text=True, timeout=timeout_s, + env={**os.environ, "ARBORIST_PROGRESS": "0"}, + ) + except subprocess.TimeoutExpired: + return {"_error": f"timeout after {timeout_s}s", "_elapsed_s": timeout_s} + elapsed = time.time() - t0 + if proc.returncode != 0: + return { + "_error": f"exit {proc.returncode}", + "_stderr": proc.stderr[:500], + "_elapsed_s": round(elapsed, 2), + } + try: + data = json.loads(proc.stdout) + except json.JSONDecodeError as e: + return {"_error": f"parse: {e}", "_stdout_head": proc.stdout[:200], + "_elapsed_s": round(elapsed, 2)} + data["_elapsed_s"] = round(elapsed, 2) + return data + + +def _primary_source(result: dict) -> dict: + """Extract primary source = first source marked source_role=primary_answer_source, + else first source if none marked.""" + srcs = result.get("sources") or [] + if not srcs: + return {} + primary = next( + (s for s in srcs if s.get("source_role") == "primary_answer_source"), + srcs[0], + ) + return { + "title": primary.get("title", "")[:60], + "uri": primary.get("document_uri", ""), + "used": primary.get("used"), + } + + +def _is_regression(local: dict, cloud: dict) -> tuple[bool, str]: + """Return (is_regression, reason).""" + if cloud.get("_error"): + return True, f"cloud error: {cloud['_error']}" + if local.get("_error"): + return False, "local errored too (not a regression)" + local_audit = local.get("audit_mode") + cloud_audit = cloud.get("audit_mode") + rank = {"UNGROUNDED": 0, "HYBRID": 1, "STRICT": 2} + if rank.get(cloud_audit, -1) < rank.get(local_audit, -1): + return True, f"audit_mode dropped {local_audit} → {cloud_audit}" + # Primary-source URI mismatch where both are non-empty + lp = _primary_source(local) + cp = _primary_source(cloud) + if lp.get("uri") and cp.get("uri") and lp["uri"] != cp["uri"]: + # Tolerable if both contain the same key noun (eg. variant article). + # Strict check for now — surface and let human judge. + return True, f"primary uri differs: {lp['uri'][:60]} vs {cp['uri'][:60]}" + return False, "ok" + + +def main(argv=None): + p = argparse.ArgumentParser() + p.add_argument( + "--fixture", type=Path, + default=REPO / "bench" / "qa_questions_smoke.txt", + ) + p.add_argument("--shards-dir", type=Path, default=DEFAULT_SHARDS) + p.add_argument("--bucket-url", default=DEFAULT_BUCKET) + p.add_argument("--endpoint", default=DEFAULT_QWEN_ENDPOINT) + p.add_argument("--model", default=DEFAULT_QWEN_MODEL) + p.add_argument( + "--out-dir", type=Path, + default=REPO / "bench" / "cloud_vs_local_results", + ) + p.add_argument("--skip-local", action="store_true", + help="only run cloud (e.g. when local results are pre-recorded)") + args = p.parse_args(argv) + + questions = _load_fixture(args.fixture) + print(f"# {len(questions)} questions from {args.fixture}", file=sys.stderr) + print(f"# shards: {args.shards_dir}", file=sys.stderr) + print(f"# bucket: {args.bucket_url}", file=sys.stderr) + print(f"# llm: {args.endpoint} / {args.model}", file=sys.stderr) + print(file=sys.stderr) + + args.out_dir.mkdir(parents=True, exist_ok=True) + stamp = _dt.datetime.now(_dt.timezone.utc).strftime("%Y-%m-%dT%H-%M-%SZ") + out_path = args.out_dir / f"{stamp}.jsonl" + + rows: list[dict] = [] + regressions: list[tuple[str, str]] = [] + + with open(out_path, "w") as out: + for i, q in enumerate(questions, 1): + print(f"[{i}/{len(questions)}] {q}", file=sys.stderr) + local = {} if args.skip_local else _run_local(q, shards_dir=args.shards_dir) + cloud = _run_cloud( + q, bucket_url=args.bucket_url, + endpoint=args.endpoint, model=args.model, + ) + row = { + "question": q, + "local": { + "audit_mode": local.get("audit_mode"), + "n_quotes": local.get("n_quotes"), + "n_verified": local.get("n_verified"), + "primary": _primary_source(local), + "elapsed_s": local.get("_elapsed_s"), + "error": local.get("_error"), + }, + "cloud": { + "audit_mode": cloud.get("audit_mode"), + "n_quotes": cloud.get("n_quotes"), + "n_verified": cloud.get("n_verified"), + "primary": _primary_source(cloud), + "elapsed_s": cloud.get("_elapsed_s"), + "error": cloud.get("_error"), + }, + } + is_reg, reason = _is_regression(local, cloud) + row["regression"] = is_reg + row["reason"] = reason + rows.append(row) + out.write(json.dumps(row, ensure_ascii=False) + "\n") + out.flush() + + # Per-question terminal line. + print( + f" local: {local.get('audit_mode', '?'):<25} {row['local']['primary'].get('title','')[:55]}", + file=sys.stderr, + ) + print( + f" cloud: {cloud.get('audit_mode', '?'):<25} {row['cloud']['primary'].get('title','')[:55]}", + file=sys.stderr, + ) + if is_reg: + regressions.append((q, reason)) + print(f" ⚠ REGRESSION: {reason}", file=sys.stderr) + print(file=sys.stderr) + + print(f"=== SUMMARY ===", file=sys.stderr) + print(f" {len(questions)} questions, {len(regressions)} regressions", file=sys.stderr) + for q, r in regressions: + print(f" ⚠ {q[:60]} — {r}", file=sys.stderr) + print(f" results: {out_path}", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bench/cloud_vs_local_results/2026-05-31T00-52-37Z.jsonl b/bench/cloud_vs_local_results/2026-05-31T00-52-37Z.jsonl new file mode 100644 index 0000000..0920486 --- /dev/null +++ b/bench/cloud_vs_local_results/2026-05-31T00-52-37Z.jsonl @@ -0,0 +1,5 @@ +{"question": "when did the soviet union dissolve?", "local": {"audit_mode": "STRICT", "n_quotes": 1, "n_verified": 1, "primary": {"title": "Soviet Union", "uri": "https://en.wikipedia.org/wiki/Soviet_Union", "used": true}, "elapsed_s": 11.69, "error": null}, "cloud": {"audit_mode": "STRICT", "n_quotes": 1, "n_verified": 1, "primary": {"title": "Republics of the Soviet Union", "uri": "https://en.wikipedia.org/wiki/Republics_of_the_Soviet_Union", "used": true}, "elapsed_s": 140.81, "error": null}, "regression": true, "reason": "primary uri differs: https://en.wikipedia.org/wiki/Soviet_Union vs https://en.wikipedia.org/wiki/Republics_of_the_Soviet_Union"} +{"question": "where is mount kilimanjaro located?", "local": {"audit_mode": "STRICT", "n_quotes": 2, "n_verified": 2, "primary": {"title": "Mount Kilimanjaro", "uri": "https://en.wikipedia.org/wiki/Mount_Kilimanjaro", "used": true}, "elapsed_s": 7.22, "error": null}, "cloud": {"audit_mode": "STRICT", "n_quotes": 1, "n_verified": 1, "primary": {"title": "Mount Kilimanjaro", "uri": "https://en.wikipedia.org/wiki/Mount_Kilimanjaro", "used": true}, "elapsed_s": 97.78, "error": null}, "regression": false, "reason": "ok"} +{"question": "who painted the mona lisa?", "local": {"audit_mode": "STRICT", "n_quotes": 2, "n_verified": 2, "primary": {"title": "Mona Lisa", "uri": "https://en.wikipedia.org/wiki/Mona_Lisa", "used": true}, "elapsed_s": 8.3, "error": null}, "cloud": {"audit_mode": "STRICT", "n_quotes": 1, "n_verified": 1, "primary": {"title": "Mona Lisa's Revenge", "uri": "https://en.wikipedia.org/wiki/Mona_Lisa's_Revenge", "used": false}, "elapsed_s": 96.73, "error": null}, "regression": true, "reason": "primary uri differs: https://en.wikipedia.org/wiki/Mona_Lisa vs https://en.wikipedia.org/wiki/Mona_Lisa's_Revenge"} +{"question": "who were the original seven mercury astronauts?", "local": {"audit_mode": "STRICT", "n_quotes": 1, "n_verified": 1, "primary": {"title": "Mercury Seven", "uri": "https://en.wikipedia.org/wiki/Mercury_Seven", "used": true}, "elapsed_s": 11.51, "error": null}, "cloud": {"audit_mode": "HYBRID", "n_quotes": 10, "n_verified": 10, "primary": {"title": "Mercury 13", "uri": "https://en.wikipedia.org/wiki/Mercury_13", "used": false}, "elapsed_s": 111.53, "error": null}, "regression": true, "reason": "audit_mode dropped STRICT → HYBRID"} +{"question": "why did the dinosaurs go extinct?", "local": {"audit_mode": "UNGROUNDED", "n_quotes": 1, "n_verified": 0, "primary": {"title": "Edwina, the Dinosaur Who Didn't Know She Was Extinct", "uri": "https://en.wikipedia.org/wiki/Edwina,_the_Dinosaur_Who_Didn't_Know_She_Was_Extinct", "used": false}, "elapsed_s": 9.55, "error": null}, "cloud": {"audit_mode": "HYBRID", "n_quotes": 4, "n_verified": 2, "primary": {"title": "Sites of fossilized dinosaurs across the southern South Kore", "uri": "https://en.wikipedia.org/wiki/Sites_of_fossilized_dinosaurs_across_the_southern_South_Korean_coast", "used": true}, "elapsed_s": 103.93, "error": null}, "regression": true, "reason": "primary uri differs: https://en.wikipedia.org/wiki/Edwina,_the_Dinosaur_Who_Didn' vs https://en.wikipedia.org/wiki/Sites_of_fossilized_dinosaurs_"} diff --git a/bench/cloud_vs_local_results/2026-05-31T01-09-30Z.jsonl b/bench/cloud_vs_local_results/2026-05-31T01-09-30Z.jsonl new file mode 100644 index 0000000..64f4ff0 --- /dev/null +++ b/bench/cloud_vs_local_results/2026-05-31T01-09-30Z.jsonl @@ -0,0 +1,5 @@ +{"question": "when did the soviet union dissolve?", "local": {"audit_mode": null, "n_quotes": null, "n_verified": null, "primary": {}, "elapsed_s": 120, "error": "timeout after 120s"}, "cloud": {"audit_mode": "STRICT", "n_quotes": 2, "n_verified": 2, "primary": {"title": "Soviet Union", "uri": "https://en.wikipedia.org/wiki/Soviet_Union", "used": true}, "elapsed_s": 114.28, "error": null}, "regression": false, "reason": "local errored too (not a regression)"} +{"question": "where is mount kilimanjaro located?", "local": {"audit_mode": "STRICT", "n_quotes": 1, "n_verified": 1, "primary": {"title": "Mount Kilimanjaro", "uri": "https://en.wikipedia.org/wiki/Mount_Kilimanjaro", "used": true}, "elapsed_s": 6.26, "error": null}, "cloud": {"audit_mode": "STRICT", "n_quotes": 2, "n_verified": 2, "primary": {"title": "Mount Kilimanjaro", "uri": "https://en.wikipedia.org/wiki/Mount_Kilimanjaro", "used": true}, "elapsed_s": 109.51, "error": null}, "regression": false, "reason": "ok"} +{"question": "who painted the mona lisa?", "local": {"audit_mode": "STRICT", "n_quotes": 2, "n_verified": 2, "primary": {"title": "Mona Lisa", "uri": "https://en.wikipedia.org/wiki/Mona_Lisa", "used": true}, "elapsed_s": 8.75, "error": null}, "cloud": {"audit_mode": "STRICT", "n_quotes": 2, "n_verified": 2, "primary": {"title": "Mona Lisa", "uri": "https://en.wikipedia.org/wiki/Mona_Lisa", "used": true}, "elapsed_s": 85.28, "error": null}, "regression": false, "reason": "ok"} +{"question": "who were the original seven mercury astronauts?", "local": {"audit_mode": "STRICT", "n_quotes": 1, "n_verified": 1, "primary": {"title": "Mercury Seven", "uri": "https://en.wikipedia.org/wiki/Mercury_Seven", "used": true}, "elapsed_s": 11.23, "error": null}, "cloud": {"audit_mode": "HYBRID", "n_quotes": 5, "n_verified": 5, "primary": {"title": "Mercury Seven", "uri": "https://en.wikipedia.org/wiki/Mercury_Seven", "used": true}, "elapsed_s": 99.55, "error": null}, "regression": true, "reason": "audit_mode dropped STRICT → HYBRID"} +{"question": "why did the dinosaurs go extinct?", "local": {"audit_mode": "UNGROUNDED", "n_quotes": 3, "n_verified": 0, "primary": {"title": "Edwina, the Dinosaur Who Didn't Know She Was Extinct", "uri": "https://en.wikipedia.org/wiki/Edwina,_the_Dinosaur_Who_Didn't_Know_She_Was_Extinct", "used": false}, "elapsed_s": 8.4, "error": null}, "cloud": {"audit_mode": "HYBRID", "n_quotes": 4, "n_verified": 1, "primary": {"title": "Edwina, the Dinosaur Who Didn't Know She Was Extinct", "uri": "https://en.wikipedia.org/wiki/Edwina,_the_Dinosaur_Who_Didn't_Know_She_Was_Extinct", "used": true}, "elapsed_s": 120.91, "error": null}, "regression": false, "reason": "ok"}