diff --git a/bench/chunk_fetch_speed.py b/bench/chunk_fetch_speed.py new file mode 100644 index 0000000..d7ce5c1 --- /dev/null +++ b/bench/chunk_fetch_speed.py @@ -0,0 +1,156 @@ +"""Measure per-chunk fetch latency for two cloud paths: + +1. apsw HttpRangeVFS on the big shard .db — current live path + (FtsSidecarShardClient falls back here today because blobs/ + isn't published). +2. Direct HTTP GET — proxied by GETting a known small bucket object + (the slim FTS5 sidecar's first bytes) to capture real RTT + + transfer time per chunk. + +Output: per-chunk wallclock for both paths, plus the speedup ratio +the JUST_ENOUGH=1 blob-publish move would buy us. + +Reads-only; never writes to the bucket. Uses real cloud manifest + +real chunks. ~30 s on first run (cold cache), ~5 s on warm. +""" +from __future__ import annotations + +import argparse +import sqlite3 +import statistics +import time +import urllib.request +from pathlib import Path + + +DEFAULT_MANIFEST = ( + "https://nyc3.digitaloceanspaces.com/arborist/clones/manifest-fts.json" +) +DEFAULT_N = 10 + + +def _pick_leaf_hashes(sidecar_path: Path, n: int) -> list[str]: + """Pull n random leaf_hashes from a local slim sidecar.""" + c = sqlite3.connect(f"file:{sidecar_path}?mode=ro", uri=True) + rows = c.execute( + "SELECT leaf_hash FROM chunks ORDER BY RANDOM() LIMIT ?", + (n,), + ).fetchall() + c.close() + return [r[0] for r in rows] + + +def _time_apsw_fetches(shard_url: str, leaf_hashes: list[str]) -> list[float]: + """For each leaf_hash, time the FtsSidecarShardClient fallback path: + apsw HttpRangeVFS on the big shard + SELECT content FROM chunks + WHERE leaf_hash = ?. + """ + from arborist.wallet.bucket import BucketClient, BucketEndpoint + # Cache_bytes=64 MB matches the default in MultiShardSidecarCorpus. + # First chunk pays connection setup + page-cache cold misses; later + # chunks may benefit from cached pages — report per-chunk times so + # the asymmetry is visible. + client = BucketClient( + BucketEndpoint(shard_url=shard_url, blob_base=""), + cache_bytes=64 * 1024 * 1024, + ) + times: list[float] = [] + try: + for h in leaf_hashes: + t0 = time.time() + rows = list(client.conn.execute( + "SELECT content FROM chunks WHERE leaf_hash = ? " + "AND content IS NOT NULL LIMIT 1", + (h,), + )) + elapsed = time.time() - t0 + times.append(elapsed) + if not rows: + # No content — leaf_hash missing in this shard. Skip + # but keep timing the trip. + continue + finally: + client.close() + return times + + +def _time_blob_fetches(probe_url: str, n: int) -> list[float]: + """Proxy direct-blob-fetch cost by GETting the same bucket object + n times (cold connection each time so RTT + small-payload transfer + dominate, same shape a per-chunk blob GET would have). + + Uses a small known object (sidecar manifest JSON or first 4 KB of + a published file). We're measuring "open-HTTP-connection-to-bucket + + transfer ~few-KB" — not the apsw multi-roundtrip cost. + """ + times: list[float] = [] + for _ in range(n): + req = urllib.request.Request(probe_url, headers={"Range": "bytes=0-2048"}) + t0 = time.time() + with urllib.request.urlopen(req, timeout=30) as resp: + _ = resp.read() + elapsed = time.time() - t0 + times.append(elapsed) + return times + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--manifest", default=DEFAULT_MANIFEST) + p.add_argument("--shard-idx", type=int, default=0, + help="which manifest shard to probe (0..n-1)") + p.add_argument("--n", type=int, default=DEFAULT_N, + help="number of chunks to fetch on each path") + args = p.parse_args() + + import json + with urllib.request.urlopen(args.manifest, timeout=30) as resp: + m = json.loads(resp.read()) + sh = m["shards"][args.shard_idx] + shard_url = sh["url"] + fts_url = sh["fts_sidecar_url"] + fts_basename = fts_url.rsplit("/", 1)[-1] # 000.idx.db + n = fts_basename.split(".")[0] # 000 + local_sidecar = Path.home() / ".arborist" / "sidecar-fts" / f"{n}.idx.db" + if not local_sidecar.exists(): + print(f"need local slim sidecar at {local_sidecar} to pick " + f"known-good leaf_hashes; build via `make sidecar-build-fts-all` " + f"or download from {fts_url}") + return 2 + + print(f"shard: {shard_url}") + print(f"sample-leaf-hashes drawn from local: {local_sidecar}") + print(f"n={args.n} per path") + print() + + leaf_hashes = _pick_leaf_hashes(local_sidecar, args.n) + + print("=== apsw HttpRangeVFS path (FtsSidecarShardClient fallback today) ===") + apsw_times = _time_apsw_fetches(shard_url, leaf_hashes) + for i, t in enumerate(apsw_times, 1): + print(f" chunk {i:2d}: {t*1000:7.1f} ms") + print(f" apsw median: {statistics.median(apsw_times)*1000:.1f} ms") + print(f" apsw mean : {statistics.mean(apsw_times)*1000:.1f} ms") + print(f" apsw total: {sum(apsw_times):.2f} s") + print() + + print("=== direct blob GET path (proxied by 2KB Range GET on a same-bucket object) ===") + blob_times = _time_blob_fetches(fts_url, args.n) + for i, t in enumerate(blob_times, 1): + print(f" blob {i:2d}: {t*1000:7.1f} ms") + print(f" blob median: {statistics.median(blob_times)*1000:.1f} ms") + print(f" blob mean : {statistics.mean(blob_times)*1000:.1f} ms") + print(f" blob total: {sum(blob_times):.2f} s") + print() + + apsw_med = statistics.median(apsw_times) + blob_med = statistics.median(blob_times) + speedup = apsw_med / blob_med if blob_med > 0 else float("inf") + print(f"=== speedup ===") + print(f" per-chunk median: apsw {apsw_med*1000:.1f} ms / blob {blob_med*1000:.1f} ms = {speedup:.1f}×") + return 0 + + +if __name__ == "__main__": + import sys + sys.exit(main()) diff --git a/bench/slim_fts_parity_results/2026-05-31T13-44-36Z.jsonl b/bench/slim_fts_parity_results/2026-05-31T13-44-36Z.jsonl new file mode 100644 index 0000000..f5dc45b --- /dev/null +++ b/bench/slim_fts_parity_results/2026-05-31T13-44-36Z.jsonl @@ -0,0 +1,5 @@ +{"question": "when did the soviet union dissolve?", "legacy": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Soviet Union", "uri": "https://en.wikipedia.org/wiki/Soviet_Union"}, "elapsed_s": 6.82, "error": null}, "corpus": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Soviet Union", "uri": "https://en.wikipedia.org/wiki/Soviet_Union"}, "elapsed_s": 23.88, "error": null}, "slim_fts": {"audit_mode": "UNGROUNDED", "n_verified": 0, "n_quotes": 0, "primary": {"title": "Soviet Union", "uri": "https://en.wikipedia.org/wiki/Soviet_Union"}, "elapsed_s": 1.92, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 1, "all_agree": true}} +{"question": "where is mount kilimanjaro located?", "legacy": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Mount Kilimanjaro", "uri": "https://en.wikipedia.org/wiki/Mount_Kilimanjaro"}, "elapsed_s": 5.31, "error": null}, "corpus": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Mount Kilimanjaro", "uri": "https://en.wikipedia.org/wiki/Mount_Kilimanjaro"}, "elapsed_s": 34.83, "error": null}, "slim_fts": {"audit_mode": "UNGROUNDED", "n_verified": 0, "n_quotes": 0, "primary": {"title": "Mount Kilimanjaro", "uri": "https://en.wikipedia.org/wiki/Mount_Kilimanjaro"}, "elapsed_s": 3.07, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 1, "all_agree": true}} +{"question": "who painted the mona lisa?", "legacy": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Mona Lisa", "uri": "https://en.wikipedia.org/wiki/Mona_Lisa"}, "elapsed_s": 5.57, "error": null}, "corpus": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Mona Lisa", "uri": "https://en.wikipedia.org/wiki/Mona_Lisa"}, "elapsed_s": 4.95, "error": null}, "slim_fts": {"audit_mode": "UNGROUNDED", "n_verified": 0, "n_quotes": 0, "primary": {"title": "Mona Lisa", "uri": "https://en.wikipedia.org/wiki/Mona_Lisa"}, "elapsed_s": 1.27, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 1, "all_agree": true}} +{"question": "who were the original seven mercury astronauts?", "legacy": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Mercury Seven", "uri": "https://en.wikipedia.org/wiki/Mercury_Seven"}, "elapsed_s": 7.09, "error": null}, "corpus": {"audit_mode": "HYBRID", "n_verified": 5, "n_quotes": 5, "primary": {"title": "Mercury Seven", "uri": "https://en.wikipedia.org/wiki/Mercury_Seven"}, "elapsed_s": 18.18, "error": null}, "slim_fts": {"audit_mode": "UNGROUNDED", "n_verified": 0, "n_quotes": 0, "primary": {"title": "Mercury Seven", "uri": "https://en.wikipedia.org/wiki/Mercury_Seven"}, "elapsed_s": 2.59, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 1, "all_agree": true}} +{"question": "why did the dinosaurs go extinct?", "legacy": {"audit_mode": "UNGROUNDED", "n_verified": 0, "n_quotes": 1, "primary": {"title": "Edwina, the Dinosaur Who Didn't Know She Was Extinct", "uri": "https://en.wikipedia.org/wiki/Edwina,_the_Dinosaur_Who_Didn't_Know_She_Was_Extinct"}, "elapsed_s": 6.49, "error": null}, "corpus": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Dinosaur", "uri": "https://en.wikipedia.org/wiki/Dinosaur"}, "elapsed_s": 12.56, "error": null}, "slim_fts": {"audit_mode": "UNGROUNDED", "n_verified": 0, "n_quotes": 0, "primary": {"title": "Dinosaur", "uri": "https://en.wikipedia.org/wiki/Dinosaur"}, "elapsed_s": 1.77, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 2, "all_agree": false}} diff --git a/bench/slim_fts_parity_results/2026-05-31T13-49-17Z.jsonl b/bench/slim_fts_parity_results/2026-05-31T13-49-17Z.jsonl new file mode 100644 index 0000000..17befea --- /dev/null +++ b/bench/slim_fts_parity_results/2026-05-31T13-49-17Z.jsonl @@ -0,0 +1,5 @@ +{"question": "when did the soviet union dissolve?", "legacy": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Soviet Union", "uri": "https://en.wikipedia.org/wiki/Soviet_Union"}, "elapsed_s": 4.47, "error": null}, "corpus": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Soviet Union", "uri": "https://en.wikipedia.org/wiki/Soviet_Union"}, "elapsed_s": 5.13, "error": null}, "slim_fts": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Soviet Union", "uri": "https://en.wikipedia.org/wiki/Soviet_Union"}, "elapsed_s": 7.39, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 1, "all_agree": true}} +{"question": "where is mount kilimanjaro located?", "legacy": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Mount Kilimanjaro", "uri": "https://en.wikipedia.org/wiki/Mount_Kilimanjaro"}, "elapsed_s": 3.14, "error": null}, "corpus": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Mount Kilimanjaro", "uri": "https://en.wikipedia.org/wiki/Mount_Kilimanjaro"}, "elapsed_s": 10.61, "error": null}, "slim_fts": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Mount Kilimanjaro", "uri": "https://en.wikipedia.org/wiki/Mount_Kilimanjaro"}, "elapsed_s": 9.99, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 1, "all_agree": true}} +{"question": "who painted the mona lisa?", "legacy": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Mona Lisa", "uri": "https://en.wikipedia.org/wiki/Mona_Lisa"}, "elapsed_s": 3.85, "error": null}, "corpus": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Mona Lisa", "uri": "https://en.wikipedia.org/wiki/Mona_Lisa"}, "elapsed_s": 3.17, "error": null}, "slim_fts": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Mona Lisa", "uri": "https://en.wikipedia.org/wiki/Mona_Lisa"}, "elapsed_s": 9.51, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 1, "all_agree": true}} +{"question": "who were the original seven mercury astronauts?", "legacy": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Mercury Seven", "uri": "https://en.wikipedia.org/wiki/Mercury_Seven"}, "elapsed_s": 4.98, "error": null}, "corpus": {"audit_mode": "HYBRID", "n_verified": 5, "n_quotes": 5, "primary": {"title": "Mercury Seven", "uri": "https://en.wikipedia.org/wiki/Mercury_Seven"}, "elapsed_s": 15.42, "error": null}, "slim_fts": {"audit_mode": "HYBRID", "n_verified": 15, "n_quotes": 15, "primary": {"title": "Mercury Seven", "uri": "https://en.wikipedia.org/wiki/Mercury_Seven"}, "elapsed_s": 11.15, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 1, "all_agree": true}} +{"question": "why did the dinosaurs go extinct?", "legacy": {"audit_mode": "UNGROUNDED", "n_verified": 0, "n_quotes": 1, "primary": {"title": "Edwina, the Dinosaur Who Didn't Know She Was Extinct", "uri": "https://en.wikipedia.org/wiki/Edwina,_the_Dinosaur_Who_Didn't_Know_She_Was_Extinct"}, "elapsed_s": 3.55, "error": null}, "corpus": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Dinosaur", "uri": "https://en.wikipedia.org/wiki/Dinosaur"}, "elapsed_s": 8.4, "error": null}, "slim_fts": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Dinosaur", "uri": "https://en.wikipedia.org/wiki/Dinosaur"}, "elapsed_s": 10.29, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 2, "all_agree": false}} diff --git a/bench/three_way_results/2026-05-31T12-44-54Z.jsonl b/bench/three_way_results/2026-05-31T12-44-54Z.jsonl new file mode 100644 index 0000000..00b2b5d --- /dev/null +++ b/bench/three_way_results/2026-05-31T12-44-54Z.jsonl @@ -0,0 +1,5 @@ +{"question": "when did the soviet union dissolve?", "legacy": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Soviet Union", "uri": "https://en.wikipedia.org/wiki/Soviet_Union"}, "elapsed_s": 6.49, "error": null}, "corpus": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Soviet Union", "uri": "https://en.wikipedia.org/wiki/Soviet_Union"}, "elapsed_s": 11.84, "error": null}, "cloud": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Soviet Union", "uri": "https://en.wikipedia.org/wiki/Soviet_Union"}, "elapsed_s": 54.06, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 1, "all_agree": true}} +{"question": "where is mount kilimanjaro located?", "legacy": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Mount Kilimanjaro", "uri": "https://en.wikipedia.org/wiki/Mount_Kilimanjaro"}, "elapsed_s": 5.46, "error": null}, "corpus": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Mount Kilimanjaro", "uri": "https://en.wikipedia.org/wiki/Mount_Kilimanjaro"}, "elapsed_s": 28.03, "error": null}, "cloud": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Mount Kilimanjaro", "uri": "https://en.wikipedia.org/wiki/Mount_Kilimanjaro"}, "elapsed_s": 49.75, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 1, "all_agree": true}} +{"question": "who painted the mona lisa?", "legacy": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Mona Lisa", "uri": "https://en.wikipedia.org/wiki/Mona_Lisa"}, "elapsed_s": 5.58, "error": null}, "corpus": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Mona Lisa", "uri": "https://en.wikipedia.org/wiki/Mona_Lisa"}, "elapsed_s": 7.64, "error": null}, "cloud": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Mona Lisa", "uri": "https://en.wikipedia.org/wiki/Mona_Lisa"}, "elapsed_s": 47.93, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 1, "all_agree": true}} +{"question": "who were the original seven mercury astronauts?", "legacy": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Mercury Seven", "uri": "https://en.wikipedia.org/wiki/Mercury_Seven"}, "elapsed_s": 7.32, "error": null}, "corpus": {"audit_mode": "HYBRID", "n_verified": 4, "n_quotes": 4, "primary": {"title": "Mercury Seven", "uri": "https://en.wikipedia.org/wiki/Mercury_Seven"}, "elapsed_s": 31.54, "error": null}, "cloud": {"audit_mode": "HYBRID", "n_verified": 5, "n_quotes": 5, "primary": {"title": "Mercury Seven", "uri": "https://en.wikipedia.org/wiki/Mercury_Seven"}, "elapsed_s": 51.29, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 1, "all_agree": true}} +{"question": "why did the dinosaurs go extinct?", "legacy": {"audit_mode": "UNGROUNDED", "n_verified": 0, "n_quotes": 3, "primary": {"title": "Edwina, the Dinosaur Who Didn't Know She Was Extinct", "uri": "https://en.wikipedia.org/wiki/Edwina,_the_Dinosaur_Who_Didn't_Know_She_Was_Extinct"}, "elapsed_s": 6.05, "error": null}, "corpus": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Dinosaur", "uri": "https://en.wikipedia.org/wiki/Dinosaur"}, "elapsed_s": 19.56, "error": null}, "cloud": {"audit_mode": "HYBRID", "n_verified": 1, "n_quotes": 4, "primary": {"title": "Edwina, the Dinosaur Who Didn't Know She Was Extinct", "uri": "https://en.wikipedia.org/wiki/Edwina,_the_Dinosaur_Who_Didn't_Know_She_Was_Extinct"}, "elapsed_s": 46.01, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 2, "all_agree": false}} diff --git a/bench/three_way_results/2026-05-31T12-52-43Z.jsonl b/bench/three_way_results/2026-05-31T12-52-43Z.jsonl new file mode 100644 index 0000000..4000ec6 --- /dev/null +++ b/bench/three_way_results/2026-05-31T12-52-43Z.jsonl @@ -0,0 +1,5 @@ +{"question": "when did the soviet union dissolve?", "legacy": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Soviet Union", "uri": "https://en.wikipedia.org/wiki/Soviet_Union"}, "elapsed_s": 6.5, "error": null}, "corpus": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Soviet Union", "uri": "https://en.wikipedia.org/wiki/Soviet_Union"}, "elapsed_s": 15.9, "error": null}, "cloud": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Soviet Union", "uri": "https://en.wikipedia.org/wiki/Soviet_Union"}, "elapsed_s": 50.77, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 1, "all_agree": true}} +{"question": "where is mount kilimanjaro located?", "legacy": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Mount Kilimanjaro", "uri": "https://en.wikipedia.org/wiki/Mount_Kilimanjaro"}, "elapsed_s": 5.3, "error": null}, "corpus": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Mount Kilimanjaro", "uri": "https://en.wikipedia.org/wiki/Mount_Kilimanjaro"}, "elapsed_s": 28.96, "error": null}, "cloud": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Mount Kilimanjaro", "uri": "https://en.wikipedia.org/wiki/Mount_Kilimanjaro"}, "elapsed_s": 48.71, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 1, "all_agree": true}} +{"question": "who painted the mona lisa?", "legacy": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Mona Lisa", "uri": "https://en.wikipedia.org/wiki/Mona_Lisa"}, "elapsed_s": 4.57, "error": null}, "corpus": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Mona Lisa", "uri": "https://en.wikipedia.org/wiki/Mona_Lisa"}, "elapsed_s": 8.53, "error": null}, "cloud": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Mona Lisa", "uri": "https://en.wikipedia.org/wiki/Mona_Lisa"}, "elapsed_s": 46.45, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 1, "all_agree": true}} +{"question": "who were the original seven mercury astronauts?", "legacy": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Mercury Seven", "uri": "https://en.wikipedia.org/wiki/Mercury_Seven"}, "elapsed_s": 4.29, "error": null}, "corpus": {"audit_mode": "HYBRID", "n_verified": 5, "n_quotes": 5, "primary": {"title": "Mercury Seven", "uri": "https://en.wikipedia.org/wiki/Mercury_Seven"}, "elapsed_s": 30.65, "error": null}, "cloud": {"audit_mode": "HYBRID", "n_verified": 3, "n_quotes": 3, "primary": {"title": "Mercury Seven", "uri": "https://en.wikipedia.org/wiki/Mercury_Seven"}, "elapsed_s": 50.91, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 1, "all_agree": true}} +{"question": "why did the dinosaurs go extinct?", "legacy": {"audit_mode": "UNGROUNDED", "n_verified": 0, "n_quotes": 1, "primary": {"title": "Edwina, the Dinosaur Who Didn't Know She Was Extinct", "uri": "https://en.wikipedia.org/wiki/Edwina,_the_Dinosaur_Who_Didn't_Know_She_Was_Extinct"}, "elapsed_s": 4.08, "error": null}, "corpus": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Dinosaur", "uri": "https://en.wikipedia.org/wiki/Dinosaur"}, "elapsed_s": 22.39, "error": null}, "cloud": {"audit_mode": "HYBRID", "n_verified": 1, "n_quotes": 3, "primary": {"title": "Sites of fossilized dinosaurs across the southern South", "uri": "https://en.wikipedia.org/wiki/Sites_of_fossilized_dinosaurs_across_the_southern_South_Korean_coast"}, "elapsed_s": 52.16, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 3, "all_agree": false}} diff --git a/bench/three_way_results/2026-05-31T13-12-06Z.jsonl b/bench/three_way_results/2026-05-31T13-12-06Z.jsonl new file mode 100644 index 0000000..e69de29