bench: chunk_fetch_speed (apsw vs blob GET) + archive session bench results
bench/chunk_fetch_speed.py — measure per-chunk fetch latency for the
two cloud paths a future JUST_ENOUGH=1 blob-publish move would
compare against: (1) apsw HttpRangeVFS on the big shard .db (current
FtsSidecarShardClient fallback path), (2) direct HTTP GET on a
same-bucket object (proxy for per-chunk blob fetch).
Measured 2026-05-31 against clones/full-bench/000.db (12.5 GB) +
clones/sidecars-fts/000.idx.db:
apsw median: 704 ms / chunk (mean 773, first 1594, warmup ~500)
blob median: 91 ms / chunk (mean 92, flat — no warmup effect)
speedup: 7.7×
Real-world: ~2.6 s saved per fresh 4-chunk query. For cache-miss
flows that already pay 5-15 s on the LLM call this is real but not
transformative. The big win for blobs is cache HITS that don't go
to LLM (returns drop from ~100 ms via cached chunks to sub-50 ms
via blobs) and bulk bench runs (400 fetches = 4 min vs 30 s).
Interactive single-question flow with LLM in the loop is fine on
the apsw path; blobs stay as future optimization, not blocker.
Also archives bench/three_way_results/*.jsonl (3 runs across the
#000072 Phase 1 progression) + bench/slim_fts_parity_results/ so
the journey from "cloud diverges from local" through "cloud matches
local 5/5" is on disk for the design-log record.
This commit is contained in:
parent
492d1a8e7b
commit
f7e0a2bd24
6 changed files with 176 additions and 0 deletions
156
bench/chunk_fetch_speed.py
Normal file
156
bench/chunk_fetch_speed.py
Normal file
|
|
@ -0,0 +1,156 @@
|
|||
"""Measure per-chunk fetch latency for two cloud paths:
|
||||
|
||||
1. apsw HttpRangeVFS on the big shard .db — current live path
|
||||
(FtsSidecarShardClient falls back here today because blobs/<hash>
|
||||
isn't published).
|
||||
2. Direct HTTP GET — proxied by GETting a known small bucket object
|
||||
(the slim FTS5 sidecar's first bytes) to capture real RTT +
|
||||
transfer time per chunk.
|
||||
|
||||
Output: per-chunk wallclock for both paths, plus the speedup ratio
|
||||
the JUST_ENOUGH=1 blob-publish move would buy us.
|
||||
|
||||
Reads-only; never writes to the bucket. Uses real cloud manifest +
|
||||
real chunks. ~30 s on first run (cold cache), ~5 s on warm.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sqlite3
|
||||
import statistics
|
||||
import time
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
DEFAULT_MANIFEST = (
|
||||
"https://nyc3.digitaloceanspaces.com/arborist/clones/manifest-fts.json"
|
||||
)
|
||||
DEFAULT_N = 10
|
||||
|
||||
|
||||
def _pick_leaf_hashes(sidecar_path: Path, n: int) -> list[str]:
|
||||
"""Pull n random leaf_hashes from a local slim sidecar."""
|
||||
c = sqlite3.connect(f"file:{sidecar_path}?mode=ro", uri=True)
|
||||
rows = c.execute(
|
||||
"SELECT leaf_hash FROM chunks ORDER BY RANDOM() LIMIT ?",
|
||||
(n,),
|
||||
).fetchall()
|
||||
c.close()
|
||||
return [r[0] for r in rows]
|
||||
|
||||
|
||||
def _time_apsw_fetches(shard_url: str, leaf_hashes: list[str]) -> list[float]:
|
||||
"""For each leaf_hash, time the FtsSidecarShardClient fallback path:
|
||||
apsw HttpRangeVFS on the big shard + SELECT content FROM chunks
|
||||
WHERE leaf_hash = ?.
|
||||
"""
|
||||
from arborist.wallet.bucket import BucketClient, BucketEndpoint
|
||||
# Cache_bytes=64 MB matches the default in MultiShardSidecarCorpus.
|
||||
# First chunk pays connection setup + page-cache cold misses; later
|
||||
# chunks may benefit from cached pages — report per-chunk times so
|
||||
# the asymmetry is visible.
|
||||
client = BucketClient(
|
||||
BucketEndpoint(shard_url=shard_url, blob_base=""),
|
||||
cache_bytes=64 * 1024 * 1024,
|
||||
)
|
||||
times: list[float] = []
|
||||
try:
|
||||
for h in leaf_hashes:
|
||||
t0 = time.time()
|
||||
rows = list(client.conn.execute(
|
||||
"SELECT content FROM chunks WHERE leaf_hash = ? "
|
||||
"AND content IS NOT NULL LIMIT 1",
|
||||
(h,),
|
||||
))
|
||||
elapsed = time.time() - t0
|
||||
times.append(elapsed)
|
||||
if not rows:
|
||||
# No content — leaf_hash missing in this shard. Skip
|
||||
# but keep timing the trip.
|
||||
continue
|
||||
finally:
|
||||
client.close()
|
||||
return times
|
||||
|
||||
|
||||
def _time_blob_fetches(probe_url: str, n: int) -> list[float]:
|
||||
"""Proxy direct-blob-fetch cost by GETting the same bucket object
|
||||
n times (cold connection each time so RTT + small-payload transfer
|
||||
dominate, same shape a per-chunk blob GET would have).
|
||||
|
||||
Uses a small known object (sidecar manifest JSON or first 4 KB of
|
||||
a published file). We're measuring "open-HTTP-connection-to-bucket
|
||||
+ transfer ~few-KB" — not the apsw multi-roundtrip cost.
|
||||
"""
|
||||
times: list[float] = []
|
||||
for _ in range(n):
|
||||
req = urllib.request.Request(probe_url, headers={"Range": "bytes=0-2048"})
|
||||
t0 = time.time()
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
_ = resp.read()
|
||||
elapsed = time.time() - t0
|
||||
times.append(elapsed)
|
||||
return times
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--manifest", default=DEFAULT_MANIFEST)
|
||||
p.add_argument("--shard-idx", type=int, default=0,
|
||||
help="which manifest shard to probe (0..n-1)")
|
||||
p.add_argument("--n", type=int, default=DEFAULT_N,
|
||||
help="number of chunks to fetch on each path")
|
||||
args = p.parse_args()
|
||||
|
||||
import json
|
||||
with urllib.request.urlopen(args.manifest, timeout=30) as resp:
|
||||
m = json.loads(resp.read())
|
||||
sh = m["shards"][args.shard_idx]
|
||||
shard_url = sh["url"]
|
||||
fts_url = sh["fts_sidecar_url"]
|
||||
fts_basename = fts_url.rsplit("/", 1)[-1] # 000.idx.db
|
||||
n = fts_basename.split(".")[0] # 000
|
||||
local_sidecar = Path.home() / ".arborist" / "sidecar-fts" / f"{n}.idx.db"
|
||||
if not local_sidecar.exists():
|
||||
print(f"need local slim sidecar at {local_sidecar} to pick "
|
||||
f"known-good leaf_hashes; build via `make sidecar-build-fts-all` "
|
||||
f"or download from {fts_url}")
|
||||
return 2
|
||||
|
||||
print(f"shard: {shard_url}")
|
||||
print(f"sample-leaf-hashes drawn from local: {local_sidecar}")
|
||||
print(f"n={args.n} per path")
|
||||
print()
|
||||
|
||||
leaf_hashes = _pick_leaf_hashes(local_sidecar, args.n)
|
||||
|
||||
print("=== apsw HttpRangeVFS path (FtsSidecarShardClient fallback today) ===")
|
||||
apsw_times = _time_apsw_fetches(shard_url, leaf_hashes)
|
||||
for i, t in enumerate(apsw_times, 1):
|
||||
print(f" chunk {i:2d}: {t*1000:7.1f} ms")
|
||||
print(f" apsw median: {statistics.median(apsw_times)*1000:.1f} ms")
|
||||
print(f" apsw mean : {statistics.mean(apsw_times)*1000:.1f} ms")
|
||||
print(f" apsw total: {sum(apsw_times):.2f} s")
|
||||
print()
|
||||
|
||||
print("=== direct blob GET path (proxied by 2KB Range GET on a same-bucket object) ===")
|
||||
blob_times = _time_blob_fetches(fts_url, args.n)
|
||||
for i, t in enumerate(blob_times, 1):
|
||||
print(f" blob {i:2d}: {t*1000:7.1f} ms")
|
||||
print(f" blob median: {statistics.median(blob_times)*1000:.1f} ms")
|
||||
print(f" blob mean : {statistics.mean(blob_times)*1000:.1f} ms")
|
||||
print(f" blob total: {sum(blob_times):.2f} s")
|
||||
print()
|
||||
|
||||
apsw_med = statistics.median(apsw_times)
|
||||
blob_med = statistics.median(blob_times)
|
||||
speedup = apsw_med / blob_med if blob_med > 0 else float("inf")
|
||||
print(f"=== speedup ===")
|
||||
print(f" per-chunk median: apsw {apsw_med*1000:.1f} ms / blob {blob_med*1000:.1f} ms = {speedup:.1f}×")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
sys.exit(main())
|
||||
5
bench/slim_fts_parity_results/2026-05-31T13-44-36Z.jsonl
Normal file
5
bench/slim_fts_parity_results/2026-05-31T13-44-36Z.jsonl
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
{"question": "when did the soviet union dissolve?", "legacy": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Soviet Union", "uri": "https://en.wikipedia.org/wiki/Soviet_Union"}, "elapsed_s": 6.82, "error": null}, "corpus": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Soviet Union", "uri": "https://en.wikipedia.org/wiki/Soviet_Union"}, "elapsed_s": 23.88, "error": null}, "slim_fts": {"audit_mode": "UNGROUNDED", "n_verified": 0, "n_quotes": 0, "primary": {"title": "Soviet Union", "uri": "https://en.wikipedia.org/wiki/Soviet_Union"}, "elapsed_s": 1.92, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 1, "all_agree": true}}
|
||||
{"question": "where is mount kilimanjaro located?", "legacy": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Mount Kilimanjaro", "uri": "https://en.wikipedia.org/wiki/Mount_Kilimanjaro"}, "elapsed_s": 5.31, "error": null}, "corpus": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Mount Kilimanjaro", "uri": "https://en.wikipedia.org/wiki/Mount_Kilimanjaro"}, "elapsed_s": 34.83, "error": null}, "slim_fts": {"audit_mode": "UNGROUNDED", "n_verified": 0, "n_quotes": 0, "primary": {"title": "Mount Kilimanjaro", "uri": "https://en.wikipedia.org/wiki/Mount_Kilimanjaro"}, "elapsed_s": 3.07, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 1, "all_agree": true}}
|
||||
{"question": "who painted the mona lisa?", "legacy": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Mona Lisa", "uri": "https://en.wikipedia.org/wiki/Mona_Lisa"}, "elapsed_s": 5.57, "error": null}, "corpus": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Mona Lisa", "uri": "https://en.wikipedia.org/wiki/Mona_Lisa"}, "elapsed_s": 4.95, "error": null}, "slim_fts": {"audit_mode": "UNGROUNDED", "n_verified": 0, "n_quotes": 0, "primary": {"title": "Mona Lisa", "uri": "https://en.wikipedia.org/wiki/Mona_Lisa"}, "elapsed_s": 1.27, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 1, "all_agree": true}}
|
||||
{"question": "who were the original seven mercury astronauts?", "legacy": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Mercury Seven", "uri": "https://en.wikipedia.org/wiki/Mercury_Seven"}, "elapsed_s": 7.09, "error": null}, "corpus": {"audit_mode": "HYBRID", "n_verified": 5, "n_quotes": 5, "primary": {"title": "Mercury Seven", "uri": "https://en.wikipedia.org/wiki/Mercury_Seven"}, "elapsed_s": 18.18, "error": null}, "slim_fts": {"audit_mode": "UNGROUNDED", "n_verified": 0, "n_quotes": 0, "primary": {"title": "Mercury Seven", "uri": "https://en.wikipedia.org/wiki/Mercury_Seven"}, "elapsed_s": 2.59, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 1, "all_agree": true}}
|
||||
{"question": "why did the dinosaurs go extinct?", "legacy": {"audit_mode": "UNGROUNDED", "n_verified": 0, "n_quotes": 1, "primary": {"title": "Edwina, the Dinosaur Who Didn't Know She Was Extinct", "uri": "https://en.wikipedia.org/wiki/Edwina,_the_Dinosaur_Who_Didn't_Know_She_Was_Extinct"}, "elapsed_s": 6.49, "error": null}, "corpus": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Dinosaur", "uri": "https://en.wikipedia.org/wiki/Dinosaur"}, "elapsed_s": 12.56, "error": null}, "slim_fts": {"audit_mode": "UNGROUNDED", "n_verified": 0, "n_quotes": 0, "primary": {"title": "Dinosaur", "uri": "https://en.wikipedia.org/wiki/Dinosaur"}, "elapsed_s": 1.77, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 2, "all_agree": false}}
|
||||
5
bench/slim_fts_parity_results/2026-05-31T13-49-17Z.jsonl
Normal file
5
bench/slim_fts_parity_results/2026-05-31T13-49-17Z.jsonl
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
{"question": "when did the soviet union dissolve?", "legacy": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Soviet Union", "uri": "https://en.wikipedia.org/wiki/Soviet_Union"}, "elapsed_s": 4.47, "error": null}, "corpus": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Soviet Union", "uri": "https://en.wikipedia.org/wiki/Soviet_Union"}, "elapsed_s": 5.13, "error": null}, "slim_fts": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Soviet Union", "uri": "https://en.wikipedia.org/wiki/Soviet_Union"}, "elapsed_s": 7.39, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 1, "all_agree": true}}
|
||||
{"question": "where is mount kilimanjaro located?", "legacy": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Mount Kilimanjaro", "uri": "https://en.wikipedia.org/wiki/Mount_Kilimanjaro"}, "elapsed_s": 3.14, "error": null}, "corpus": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Mount Kilimanjaro", "uri": "https://en.wikipedia.org/wiki/Mount_Kilimanjaro"}, "elapsed_s": 10.61, "error": null}, "slim_fts": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Mount Kilimanjaro", "uri": "https://en.wikipedia.org/wiki/Mount_Kilimanjaro"}, "elapsed_s": 9.99, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 1, "all_agree": true}}
|
||||
{"question": "who painted the mona lisa?", "legacy": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Mona Lisa", "uri": "https://en.wikipedia.org/wiki/Mona_Lisa"}, "elapsed_s": 3.85, "error": null}, "corpus": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Mona Lisa", "uri": "https://en.wikipedia.org/wiki/Mona_Lisa"}, "elapsed_s": 3.17, "error": null}, "slim_fts": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Mona Lisa", "uri": "https://en.wikipedia.org/wiki/Mona_Lisa"}, "elapsed_s": 9.51, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 1, "all_agree": true}}
|
||||
{"question": "who were the original seven mercury astronauts?", "legacy": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Mercury Seven", "uri": "https://en.wikipedia.org/wiki/Mercury_Seven"}, "elapsed_s": 4.98, "error": null}, "corpus": {"audit_mode": "HYBRID", "n_verified": 5, "n_quotes": 5, "primary": {"title": "Mercury Seven", "uri": "https://en.wikipedia.org/wiki/Mercury_Seven"}, "elapsed_s": 15.42, "error": null}, "slim_fts": {"audit_mode": "HYBRID", "n_verified": 15, "n_quotes": 15, "primary": {"title": "Mercury Seven", "uri": "https://en.wikipedia.org/wiki/Mercury_Seven"}, "elapsed_s": 11.15, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 1, "all_agree": true}}
|
||||
{"question": "why did the dinosaurs go extinct?", "legacy": {"audit_mode": "UNGROUNDED", "n_verified": 0, "n_quotes": 1, "primary": {"title": "Edwina, the Dinosaur Who Didn't Know She Was Extinct", "uri": "https://en.wikipedia.org/wiki/Edwina,_the_Dinosaur_Who_Didn't_Know_She_Was_Extinct"}, "elapsed_s": 3.55, "error": null}, "corpus": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Dinosaur", "uri": "https://en.wikipedia.org/wiki/Dinosaur"}, "elapsed_s": 8.4, "error": null}, "slim_fts": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Dinosaur", "uri": "https://en.wikipedia.org/wiki/Dinosaur"}, "elapsed_s": 10.29, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 2, "all_agree": false}}
|
||||
5
bench/three_way_results/2026-05-31T12-44-54Z.jsonl
Normal file
5
bench/three_way_results/2026-05-31T12-44-54Z.jsonl
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
{"question": "when did the soviet union dissolve?", "legacy": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Soviet Union", "uri": "https://en.wikipedia.org/wiki/Soviet_Union"}, "elapsed_s": 6.49, "error": null}, "corpus": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Soviet Union", "uri": "https://en.wikipedia.org/wiki/Soviet_Union"}, "elapsed_s": 11.84, "error": null}, "cloud": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Soviet Union", "uri": "https://en.wikipedia.org/wiki/Soviet_Union"}, "elapsed_s": 54.06, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 1, "all_agree": true}}
|
||||
{"question": "where is mount kilimanjaro located?", "legacy": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Mount Kilimanjaro", "uri": "https://en.wikipedia.org/wiki/Mount_Kilimanjaro"}, "elapsed_s": 5.46, "error": null}, "corpus": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Mount Kilimanjaro", "uri": "https://en.wikipedia.org/wiki/Mount_Kilimanjaro"}, "elapsed_s": 28.03, "error": null}, "cloud": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Mount Kilimanjaro", "uri": "https://en.wikipedia.org/wiki/Mount_Kilimanjaro"}, "elapsed_s": 49.75, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 1, "all_agree": true}}
|
||||
{"question": "who painted the mona lisa?", "legacy": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Mona Lisa", "uri": "https://en.wikipedia.org/wiki/Mona_Lisa"}, "elapsed_s": 5.58, "error": null}, "corpus": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Mona Lisa", "uri": "https://en.wikipedia.org/wiki/Mona_Lisa"}, "elapsed_s": 7.64, "error": null}, "cloud": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Mona Lisa", "uri": "https://en.wikipedia.org/wiki/Mona_Lisa"}, "elapsed_s": 47.93, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 1, "all_agree": true}}
|
||||
{"question": "who were the original seven mercury astronauts?", "legacy": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Mercury Seven", "uri": "https://en.wikipedia.org/wiki/Mercury_Seven"}, "elapsed_s": 7.32, "error": null}, "corpus": {"audit_mode": "HYBRID", "n_verified": 4, "n_quotes": 4, "primary": {"title": "Mercury Seven", "uri": "https://en.wikipedia.org/wiki/Mercury_Seven"}, "elapsed_s": 31.54, "error": null}, "cloud": {"audit_mode": "HYBRID", "n_verified": 5, "n_quotes": 5, "primary": {"title": "Mercury Seven", "uri": "https://en.wikipedia.org/wiki/Mercury_Seven"}, "elapsed_s": 51.29, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 1, "all_agree": true}}
|
||||
{"question": "why did the dinosaurs go extinct?", "legacy": {"audit_mode": "UNGROUNDED", "n_verified": 0, "n_quotes": 3, "primary": {"title": "Edwina, the Dinosaur Who Didn't Know She Was Extinct", "uri": "https://en.wikipedia.org/wiki/Edwina,_the_Dinosaur_Who_Didn't_Know_She_Was_Extinct"}, "elapsed_s": 6.05, "error": null}, "corpus": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Dinosaur", "uri": "https://en.wikipedia.org/wiki/Dinosaur"}, "elapsed_s": 19.56, "error": null}, "cloud": {"audit_mode": "HYBRID", "n_verified": 1, "n_quotes": 4, "primary": {"title": "Edwina, the Dinosaur Who Didn't Know She Was Extinct", "uri": "https://en.wikipedia.org/wiki/Edwina,_the_Dinosaur_Who_Didn't_Know_She_Was_Extinct"}, "elapsed_s": 46.01, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 2, "all_agree": false}}
|
||||
5
bench/three_way_results/2026-05-31T12-52-43Z.jsonl
Normal file
5
bench/three_way_results/2026-05-31T12-52-43Z.jsonl
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
{"question": "when did the soviet union dissolve?", "legacy": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Soviet Union", "uri": "https://en.wikipedia.org/wiki/Soviet_Union"}, "elapsed_s": 6.5, "error": null}, "corpus": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Soviet Union", "uri": "https://en.wikipedia.org/wiki/Soviet_Union"}, "elapsed_s": 15.9, "error": null}, "cloud": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Soviet Union", "uri": "https://en.wikipedia.org/wiki/Soviet_Union"}, "elapsed_s": 50.77, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 1, "all_agree": true}}
|
||||
{"question": "where is mount kilimanjaro located?", "legacy": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Mount Kilimanjaro", "uri": "https://en.wikipedia.org/wiki/Mount_Kilimanjaro"}, "elapsed_s": 5.3, "error": null}, "corpus": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Mount Kilimanjaro", "uri": "https://en.wikipedia.org/wiki/Mount_Kilimanjaro"}, "elapsed_s": 28.96, "error": null}, "cloud": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Mount Kilimanjaro", "uri": "https://en.wikipedia.org/wiki/Mount_Kilimanjaro"}, "elapsed_s": 48.71, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 1, "all_agree": true}}
|
||||
{"question": "who painted the mona lisa?", "legacy": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Mona Lisa", "uri": "https://en.wikipedia.org/wiki/Mona_Lisa"}, "elapsed_s": 4.57, "error": null}, "corpus": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Mona Lisa", "uri": "https://en.wikipedia.org/wiki/Mona_Lisa"}, "elapsed_s": 8.53, "error": null}, "cloud": {"audit_mode": "STRICT", "n_verified": 2, "n_quotes": 2, "primary": {"title": "Mona Lisa", "uri": "https://en.wikipedia.org/wiki/Mona_Lisa"}, "elapsed_s": 46.45, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 1, "all_agree": true}}
|
||||
{"question": "who were the original seven mercury astronauts?", "legacy": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Mercury Seven", "uri": "https://en.wikipedia.org/wiki/Mercury_Seven"}, "elapsed_s": 4.29, "error": null}, "corpus": {"audit_mode": "HYBRID", "n_verified": 5, "n_quotes": 5, "primary": {"title": "Mercury Seven", "uri": "https://en.wikipedia.org/wiki/Mercury_Seven"}, "elapsed_s": 30.65, "error": null}, "cloud": {"audit_mode": "HYBRID", "n_verified": 3, "n_quotes": 3, "primary": {"title": "Mercury Seven", "uri": "https://en.wikipedia.org/wiki/Mercury_Seven"}, "elapsed_s": 50.91, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 1, "all_agree": true}}
|
||||
{"question": "why did the dinosaurs go extinct?", "legacy": {"audit_mode": "UNGROUNDED", "n_verified": 0, "n_quotes": 1, "primary": {"title": "Edwina, the Dinosaur Who Didn't Know She Was Extinct", "uri": "https://en.wikipedia.org/wiki/Edwina,_the_Dinosaur_Who_Didn't_Know_She_Was_Extinct"}, "elapsed_s": 4.08, "error": null}, "corpus": {"audit_mode": "STRICT", "n_verified": 1, "n_quotes": 1, "primary": {"title": "Dinosaur", "uri": "https://en.wikipedia.org/wiki/Dinosaur"}, "elapsed_s": 22.39, "error": null}, "cloud": {"audit_mode": "HYBRID", "n_verified": 1, "n_quotes": 3, "primary": {"title": "Sites of fossilized dinosaurs across the southern South", "uri": "https://en.wikipedia.org/wiki/Sites_of_fossilized_dinosaurs_across_the_southern_South_Korean_coast"}, "elapsed_s": 52.16, "error": null}, "agreement": {"n_paths_with_source": 3, "distinct_sources": 3, "all_agree": false}}
|
||||
0
bench/three_way_results/2026-05-31T13-12-06Z.jsonl
Normal file
0
bench/three_way_results/2026-05-31T13-12-06Z.jsonl
Normal file
Loading…
Add table
Add a link
Reference in a new issue