diff --git a/Makefile b/Makefile index e1cb860..76b170d 100644 --- a/Makefile +++ b/Makefile @@ -292,6 +292,11 @@ bench-5s-time-series: bootstrap ## 5S time-series-quantized π* (SQD §13.5; qua PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5s --sub semantics \ --fixtures bench/fixtures/5s/semantics-time-series-v1.jsonl +bench-real-shard: bootstrap ## #000026 Phase 2 — real-shard workload baseline (latency, audit, primary-source use) + PYTHONUNBUFFERED=1 $(PY) -m bench.scripts.real_shard_baseline \ + --shards-dir $${ARBORIST_SHARDS_DIR:-$$HOME/.arborist/shards} \ + --burn + bench-5f-formulate-live: bootstrap ## 5F Formulate via live arborist.qa.parse_claims (Phase 1b.2) PYTHONUNBUFFERED=1 $(PY) -m bench.batteries.runner --battery 5f --sub formulate \ --fixtures bench/fixtures/5f/formulate-live-v1.jsonl diff --git a/arborist/concepts/query.py b/arborist/concepts/query.py index d0a29e5..ac6c974 100644 --- a/arborist/concepts/query.py +++ b/arborist/concepts/query.py @@ -45,6 +45,10 @@ def has_compare_phrasing(question: str) -> bool: # Cross-shard lookup with mtime-keyed cache # --------------------------------------------------------------------------- +# Eager-load cache (legacy) — kept for back-compat callers that still +# go through `_get_indices`. The hot path (synonym_expand, +# rivalry_excluded) uses the lazy caches below. +# # Cache shape: { shards_dir_str: (mtime_sig, manual_index, derived_index, rivalry_pairs, token_idf) } # - manual_index: curated synonym edges; always expanded # - derived_index: corpus-derived synonym edges; expansion subject to per-token cap @@ -53,6 +57,28 @@ def has_compare_phrasing(question: str) -> bool: _CACHE: dict[str, tuple[tuple, dict, dict, list, dict]] = {} +# Lazy per-token neighbor cache (#000026 follow-up). The eager loader +# pulled all 290K concept_relations rows on first call (~2.4 s) so a +# single CLI query paid that cost up front; benchmarks amortize it. +# We don't actually need the full graph — `synonym_expand` only ever +# touches direct neighbors of the question's tokens. For a 5-token +# question that's <300 rows, queryable in ~300 ms total via +# WHERE token IN (...) OR target IN (...). +# +# Shape: { shards_dir_str: { "sig": mtime_signature, "manual": {token: {neighbors}}, +# "derived": {token: {neighbors}} } } +# A token is "known" once we've queried for it; the empty-set claim +# means "we asked SQL and got nothing back" — distinguishes from +# "we haven't asked yet." +_NEIGHBOR_CACHE: dict[str, dict] = {} + +# Process-wide rivalry-row cache. The corpus has ~2 rivalry rows +# total today, so this is essentially free; cache lets repeated calls +# skip the round-trip entirely. +# Shape: { shards_dir_str: (mtime_sig, [(token, target), ...]) } +_RIVALRY_ROWS_CACHE: dict[str, tuple[tuple, list[tuple[str, str]]]] = {} + + def _shards_mtime_signature(shards_dir: Path) -> tuple: """Return a tuple of (path, mtime_ns) for every *.db in shards_dir. Stable across runs as long as no shard's mtime changes.""" @@ -173,6 +199,127 @@ def invalidate_cache() -> None: (the mtime check would catch this on next read, but invalidating explicitly is faster on the same-process write+read pattern).""" _CACHE.clear() + _NEIGHBOR_CACHE.clear() + _RIVALRY_ROWS_CACHE.clear() + + +# --------------------------------------------------------------------------- +# Lazy neighbor loading — the hot path for synonym_expand / rivalry_excluded +# --------------------------------------------------------------------------- + + +def _ensure_neighbor_cache(shards_dir: Path) -> dict: + """Return the live neighbor cache dict for `shards_dir`. Drops & + re-initializes if any shard's mtime has changed since last load.""" + sig = _shards_mtime_signature(shards_dir) + key = str(shards_dir.resolve()) + cached = _NEIGHBOR_CACHE.get(key) + if cached is None or cached["sig"] != sig: + _NEIGHBOR_CACHE[key] = {"sig": sig, "manual": {}, "derived": {}} + return _NEIGHBOR_CACHE[key] + + +def _load_neighbors_for( + shards_dir: Path, tokens: set[str] +) -> tuple[dict[str, set[str]], dict[str, set[str]]]: + """Lazy targeted loader. Returns (manual_subset, derived_subset) + mapping just the requested tokens to their direct neighbors. + + Hits SQL only for tokens we haven't yet asked about in this + process. Once a token has been queried (even returning zero + rows), we cache the empty set so subsequent calls don't re-query. + """ + cache = _ensure_neighbor_cache(shards_dir) + qlower = {t.lower() for t in tokens if t} + missing = sorted(t for t in qlower + if t not in cache["manual"] and t not in cache["derived"]) + if missing: + ph = ",".join("?" * len(missing)) + params = missing + missing + conn = connect_query(shards_dir=shards_dir) + try: + rows = conn.execute( + f"SELECT relation_kind, evidence_kind, token, target " + f"FROM concept_relations " + f"WHERE relation_kind = 'synonym' " + f"AND (token IN ({ph}) OR target IN ({ph}))", + params, + ).fetchall() + finally: + conn.close() + for r in rows: + evidence_kind = r["evidence_kind"] + a = (r["token"] or "").lower() + b = (r["target"] or "").lower() + if not a or not b or a == b: + continue + target_index = ( + cache["manual"] + if evidence_kind in {"manual", "manual_legacy"} + else cache["derived"] + ) + target_index.setdefault(a, set()).add(b) + target_index.setdefault(b, set()).add(a) + # Mark every queried token as "we've asked" so we don't + # re-query empty results. Either index is fine — synonym_expand + # checks both with .get(t, set()). + for t in missing: + cache["manual"].setdefault(t, set()) + cache["derived"].setdefault(t, set()) + return ( + {t: cache["manual"].get(t, set()) for t in qlower}, + {t: cache["derived"].get(t, set()) for t in qlower}, + ) + + +def _load_rivalry_rows(shards_dir: Path) -> list[tuple[str, str]]: + """Process-wide cache of rivalry-relation rows for one shards_dir. + Refresh on mtime change. The corpus has ~2 rivalry rows total + today; this is essentially zero-cost after the first call.""" + sig = _shards_mtime_signature(shards_dir) + key = str(shards_dir.resolve()) + cached = _RIVALRY_ROWS_CACHE.get(key) + if cached is not None and cached[0] == sig: + return cached[1] + conn = connect_query(shards_dir=shards_dir) + try: + rows = conn.execute( + "SELECT token, target FROM concept_relations " + "WHERE relation_kind = 'rivalry'" + ).fetchall() + finally: + conn.close() + pairs = [] + for r in rows: + a = (r["token"] or "").lower() + b = (r["target"] or "").lower() + if a and b and a != b: + pairs.append((a, b)) + _RIVALRY_ROWS_CACHE[key] = (sig, pairs) + return pairs + + +def _load_idf_for(shards_dir: Path, tokens: set[str]) -> dict[str, int]: + """Lazy IDF loader — only fetched when expansion exceeds + ``max_total`` and we need to rank for truncation. Avoids the + 66 K-row dump on the common case (small expansions stay under + the cap).""" + if not tokens: + return {} + qlower = {t.lower() for t in tokens if t} + if not qlower: + return {} + ph = ",".join("?" * len(qlower)) + conn = connect_query(shards_dir=shards_dir) + try: + rows = conn.execute( + f"SELECT token, SUM(doc_freq) AS df FROM concept_token_idf " + f"WHERE token IN ({ph}) GROUP BY token", + list(qlower), + ).fetchall() + finally: + conn.close() + return {(r["token"] or "").lower(): int(r["df"] or 0) for r in rows} # --------------------------------------------------------------------------- @@ -231,23 +378,24 @@ def synonym_expand( return set() if shards_dir is None: return set(tokens) - manual_index, derived_index, _, token_idf = _get_indices(shards_dir) + p = Path(shards_dir) qlower = {t.lower() for t in tokens} + # Lazy load — only the rows whose token or target appears in the + # question. Drops first-call cost from ~2.4 s (290 K-row scan + + # Python iteration) to ~0.3 s (targeted WHERE-IN query, ~300 rows). + manual_index, derived_index = _load_neighbors_for(p, qlower) expanded: set[str] = set(qlower) # Manual (curated) synonyms always expand: brain-tech / AMD-family / # etc. seed groups have legitimately many members per anchor & we # trust the curation. for t in qlower: - if t in manual_index: - expanded |= manual_index[t] + expanded |= manual_index.get(t, set()) # Derived (corpus-extracted) synonyms cap on per-token degree. # Generic tokens like "person" / "thoughts" / "language" have wide # noisy neighborhoods in the reciprocal-link graph — skip those. # Specific tokens with bounded degree expand cleanly. for t in qlower: - if t not in derived_index: - continue - neighbors = derived_index[t] + neighbors = derived_index.get(t, set()) if len(neighbors) > max_neighbors_per_token: continue expanded |= neighbors @@ -259,9 +407,9 @@ def synonym_expand( # empty (backfill not run yet), ranking degenerates to # alphabetical (fallback compatibility). neighbors_only = expanded - qlower - # Lower doc_freq → rarer → higher rank → kept first. - # `total_docs + 1` sentinel pushes "missing" tokens to the - # rarest-bucket so they tie-break before the most common. + # Lazy IDF — only fetched when we hit the cap. Most queries + # never reach this branch. + token_idf = _load_idf_for(p, neighbors_only) SENTINEL_HIGH_RARITY = 0 ranked = sorted( neighbors_only, @@ -287,14 +435,31 @@ def rivalry_excluded( """ if compare_phrasing or not tokens or shards_dir is None: return set() - _, _, rivalry_pairs, _ = _get_indices(shards_dir) + p = Path(shards_dir) + rivalry_rows = _load_rivalry_rows(p) + if not rivalry_rows: + return set() qlower = {t.lower() for t in tokens} + # Build closures for both rivalry sides AND the question tokens, + # so the overlap test below is set-vs-set. Tokens we need: + # every token mentioned in rivalry rows + every question token. + # All loaded in one targeted query via _load_neighbors_for. + rivalry_tokens = {t for pair in rivalry_rows for t in pair} + manual_index, derived_index = _load_neighbors_for( + p, qlower | rivalry_tokens + ) excluded: set[str] = set() - for a, b in rivalry_pairs: - a_in = bool(qlower & a) - b_in = bool(qlower & b) + for a, b in rivalry_rows: + ga = frozenset( + manual_index.get(a, set()) | derived_index.get(a, set()) | {a} + ) + gb = frozenset( + manual_index.get(b, set()) | derived_index.get(b, set()) | {b} + ) + a_in = bool(qlower & ga) + b_in = bool(qlower & gb) if a_in and not b_in: - excluded |= b + excluded |= gb elif b_in and not a_in: - excluded |= a + excluded |= ga return excluded diff --git a/bench/fixtures/real-shard-baseline-v1.jsonl b/bench/fixtures/real-shard-baseline-v1.jsonl new file mode 100644 index 0000000..b330c52 --- /dev/null +++ b/bench/fixtures/real-shard-baseline-v1.jsonl @@ -0,0 +1,9 @@ +{"_meta":{"name":"real-shard-baseline-v1","notes":"Question set for #000026 Phase 2. Stable across runs so deltas are interpretable. Mix of wiki-primary, crawl-primary, and canonical-projection (math/logic) cases. Authored 2026-05-08."}} +{"id":"baseline-001","question":"who wrote virt-back?","expected_routing":"crawl_primary","note":"reviewer's first real-world success case; cited evidence is copyright footer (warrant-quality finding)"} +{"id":"baseline-002","question":"What is the capital of France?","expected_routing":"wiki_primary","note":"trivial wiki retrieval; sanity check"} +{"id":"baseline-003","question":"What is Mac OS X?","expected_routing":"wiki_primary","note":"named-entity retrieval through 2003 dump"} +{"id":"baseline-004","question":"What programming language was the Linux kernel written in?","expected_routing":"wiki_primary","note":"specific factual claim; well-grounded"} +{"id":"baseline-005","question":"Who founded Microsoft?","expected_routing":"wiki_primary","note":"named-entity authorship"} +{"id":"baseline-006","question":"What is the relationship between AMD and Intel?","expected_routing":"wiki_primary","note":"rivalry-pair tokens; exercises rivalry_excluded path"} +{"id":"baseline-007","question":"0.1 + 0.2","expected_routing":"canonical_projection","note":"arithmetic@v1 short-circuit; should bypass RAG entirely"} +{"id":"baseline-008","question":"A IMPL B","expected_routing":"canonical_projection","note":"logic-kernel@v1 short-circuit; should bypass RAG entirely"} diff --git a/bench/results/real-shard-baseline.json b/bench/results/real-shard-baseline.json new file mode 100644 index 0000000..aa38dc3 --- /dev/null +++ b/bench/results/real-shard-baseline.json @@ -0,0 +1,199 @@ +{ + "schema_version": "real-shard-baseline-v1", + "timestamp_utc": "2026-05-08T17:09:43+00:00", + "commit_sha": "e78814ca29ed4114572f2410b2615733d4b715f6", + "git_dirty": true, + "shards_dir": "/home/fox/.arborist/shards", + "shards_fingerprint": "d93bed2c48c7ac47b22286ac77508d5519fec2f3685d470966642eef4285cb5e", + "model": "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic", + "endpoint": "https://hermes.ai.unturf.com/v1", + "burn": true, + "fixtures_path": "bench/fixtures/real-shard-baseline-v1.jsonl", + "summary": { + "questions": 8, + "wall_ms_median": 4140.9, + "wall_ms_max": 7212.4, + "wall_ms_min": 0.6, + "by_audit_mode": { + "STRICT": 4, + "HYBRID": 2, + "CANONICAL_PROJECTION": 2 + }, + "by_status": { + "cache_miss_then_written": 6, + "canonical_projection": 2 + }, + "primary_source_used_count": 4, + "canonical_projection_count": 2 + }, + "queries": [ + { + "question": "who wrote virt-back?", + "status": "cache_miss_then_written", + "audit_mode": "STRICT", + "verifier_method": "quote", + "n_verified": 1, + "n_quotes": 1, + "pi_star_ref": null, + "lookup_path": "miss", + "wall_ms": 6309.6, + "total_ms": 6297.3, + "search_ms": 5002.6, + "llm_ms": 1213.1, + "verify_ms": null, + "canonical_preflight_ms": null, + "sources_count": 8, + "primary_source_used": true, + "answer_text": "Based on the provided sources, the author of virt-back is Russell Ballestrini. The source states:\n\n\"Over the weekend I wrote virt-back, a backup utility for QEMU, KVM, XEN, or Virtualbox guests.\"", + "expected_routing": "crawl_primary", + "note": "reviewer's first real-world success case; cited evidence is copyright footer (warrant-quality finding)" + }, + { + "question": "What is the capital of France?", + "status": "cache_miss_then_written", + "audit_mode": "STRICT", + "verifier_method": "quote", + "n_verified": 1, + "n_quotes": 1, + "pi_star_ref": null, + "lookup_path": "miss", + "wall_ms": 3867.2, + "total_ms": 3866.5, + "search_ms": 2663.1, + "llm_ms": 1123.0, + "verify_ms": null, + "canonical_preflight_ms": null, + "sources_count": 5, + "primary_source_used": true, + "answer_text": "The capital of France is Paris. This is stated in the source: \"With the liberation of Paris in 1944, Charles de Gaulle established the Provisional Government of the French Republic, restoring Paris as the French capital.\"", + "expected_routing": "wiki_primary", + "note": "trivial wiki retrieval; sanity check" + }, + { + "question": "What is Mac OS X?", + "status": "cache_miss_then_written", + "audit_mode": "STRICT", + "verifier_method": "quote", + "n_verified": 1, + "n_quotes": 1, + "pi_star_ref": null, + "lookup_path": "miss", + "wall_ms": 7212.4, + "total_ms": 7211.4, + "search_ms": 1485.5, + "llm_ms": 5644.9, + "verify_ms": null, + "canonical_preflight_ms": null, + "sources_count": 4, + "primary_source_used": true, + "answer_text": "Mac OS X is a series of Unix-based operating systems and graphical user interfaces developed, marketed, and sold by Apple Inc. Since 2002, Mac OS X has been included with all new Macintosh computer systems. It is the successor to Mac OS 9, ", + "expected_routing": "wiki_primary", + "note": "named-entity retrieval through 2003 dump" + }, + { + "question": "What programming language was the Linux kernel written in?", + "status": "cache_miss_then_written", + "audit_mode": "STRICT", + "verifier_method": "quote", + "n_verified": 1, + "n_quotes": 1, + "pi_star_ref": null, + "lookup_path": "miss", + "wall_ms": 4140.9, + "total_ms": 4140.0, + "search_ms": 2088.9, + "llm_ms": 1989.7, + "verify_ms": null, + "canonical_preflight_ms": null, + "sources_count": 8, + "primary_source_used": false, + "answer_text": "The Linux kernel was written in the C programming language. From the source:\n\"The Linux kernel is an operating system kernel used by the Linux family of Unix-like operating systems. It is one of the most prominent examples of free and open ", + "expected_routing": "wiki_primary", + "note": "specific factual claim; well-grounded" + }, + { + "question": "Who founded Microsoft?", + "status": "cache_miss_then_written", + "audit_mode": "HYBRID", + "verifier_method": "entity", + "n_verified": 4, + "n_quotes": 4, + "pi_star_ref": null, + "lookup_path": "miss", + "wall_ms": 2248.7, + "total_ms": 2247.6, + "search_ms": 1134.6, + "llm_ms": 1070.8, + "verify_ms": null, + "canonical_preflight_ms": null, + "sources_count": 5, + "primary_source_used": true, + "answer_text": "Based on the provided sources, I don't have enough information to determine who founded Microsoft. The sources mention Microsoft Research, Microsoft TechNet, Microsoft Hardware, and Microsoft Dynamics NAV, but do not contain any verbatim qu", + "expected_routing": "wiki_primary", + "note": "named-entity authorship" + }, + { + "question": "What is the relationship between AMD and Intel?", + "status": "cache_miss_then_written", + "audit_mode": "HYBRID", + "verifier_method": "paraphrase", + "n_verified": 4, + "n_quotes": 5, + "pi_star_ref": null, + "lookup_path": "miss", + "wall_ms": 4327.0, + "total_ms": 4326.3, + "search_ms": 1636.2, + "llm_ms": 2623.3, + "verify_ms": null, + "canonical_preflight_ms": null, + "sources_count": 8, + "primary_source_used": false, + "answer_text": "The relationship between AMD and Intel is that of competitors in the microprocessor market. AMD filed an antitrust lawsuit against Intel in June 2005, alleging that Intel engaged in unfair competition by offering rebates to Japanese PC manu", + "expected_routing": "wiki_primary", + "note": "rivalry-pair tokens; exercises rivalry_excluded path" + }, + { + "question": "0.1 + 0.2", + "status": "canonical_projection", + "audit_mode": "CANONICAL_PROJECTION", + "verifier_method": "canonical_projection", + "n_verified": 1, + "n_quotes": 1, + "pi_star_ref": "arithmetic@v1", + "lookup_path": "preflight_canonical", + "wall_ms": 0.9, + "total_ms": 0.3, + "search_ms": null, + "llm_ms": null, + "verify_ms": null, + "canonical_preflight_ms": 0.3, + "sources_count": 0, + "primary_source_used": false, + "answer_text": "3/10", + "expected_routing": "canonical_projection", + "note": "arithmetic@v1 short-circuit; should bypass RAG entirely" + }, + { + "question": "A IMPL B", + "status": "canonical_projection", + "audit_mode": "CANONICAL_PROJECTION", + "verifier_method": "canonical_projection", + "n_verified": 1, + "n_quotes": 1, + "pi_star_ref": "logic-kernel@v1", + "lookup_path": "preflight_canonical", + "wall_ms": 0.6, + "total_ms": 0.1, + "search_ms": null, + "llm_ms": null, + "verify_ms": null, + "canonical_preflight_ms": 0.1, + "sources_count": 0, + "primary_source_used": false, + "answer_text": "(NOT A OR B)", + "expected_routing": "canonical_projection", + "note": "logic-kernel@v1 short-circuit; should bypass RAG entirely" + } + ] +} \ No newline at end of file diff --git a/bench/results/real-shard-baseline.md b/bench/results/real-shard-baseline.md new file mode 100644 index 0000000..c55139c --- /dev/null +++ b/bench/results/real-shard-baseline.md @@ -0,0 +1,30 @@ +# Real-shard baseline — 2026-05-08T17:09:43+00:00 + +**Commit:** `e78814ca29ed` (dirty) +**Shards fingerprint:** `d93bed2c48c7ac47…` +**Shards directory:** `/home/fox/.arborist/shards` + +## Summary + +- Questions: **8** +- Wall median: **4140.9 ms** +- Wall range: 0.6 – 7212.4 ms +- Primary source used: 4/8 +- Canonical-projection short-circuits: 2 + +**Audit-mode distribution:** `CANONICAL_PROJECTION` 2, `HYBRID` 2, `STRICT` 4 + +**Status distribution:** `cache_miss_then_written` 6, `canonical_projection` 2 + +## Per-question + +| # | Question | Audit | Status | Wall (ms) | Method | +|---|----------|-------|--------|-----------|--------| +| 1 | who wrote virt-back? | STRICT | cache_miss_then_written | 6309.6 | quote | +| 2 | What is the capital of France? | STRICT | cache_miss_then_written | 3867.2 | quote | +| 3 | What is Mac OS X? | STRICT | cache_miss_then_written | 7212.4 | quote | +| 4 | What programming language was the Linux kernel written in? | STRICT | cache_miss_then_written | 4140.9 | quote | +| 5 | Who founded Microsoft? | HYBRID | cache_miss_then_written | 2248.7 | entity | +| 6 | What is the relationship between AMD and Intel? | HYBRID | cache_miss_then_written | 4327.0 | paraphrase | +| 7 | 0.1 + 0.2 | CANONICAL_PROJECTION | canonical_projection | 0.9 | canonical_projection | +| 8 | A IMPL B | CANONICAL_PROJECTION | canonical_projection | 0.6 | canonical_projection | diff --git a/bench/scripts/__init__.py b/bench/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bench/scripts/real_shard_baseline.py b/bench/scripts/real_shard_baseline.py new file mode 100644 index 0000000..2f62004 --- /dev/null +++ b/bench/scripts/real_shard_baseline.py @@ -0,0 +1,335 @@ +"""Real-shard workload baseline (#000026 Phase 2). + +Runs a fixed question set against the operator's real shard +directory and emits one JSON artifact + a short markdown summary. + +Usage: + + python -m bench.scripts.real_shard_baseline \ + --shards-dir ~/.arborist/shards \ + --fixtures bench/fixtures/real-shard-baseline-v1.jsonl \ + --out bench/results/real-shard-baseline.json + +Or via the Makefile: + + make bench-real-shard + +Outputs: + + bench/results/real-shard-baseline.json one row per question + bench/results/real-shard-baseline.md markdown summary + +Captured per question: + + - question, audit_mode, verifier_method, n_verified, n_quotes + - wall_ms, search_ms, llm_ms, canonical_ms (preflight short-circuit) + - primary_source_used, sources_count + - status (canonical_projection | cache_hit | cache_miss_then_written + | no_sources | broad_quantifier_rejected | ...) + +This is observation, not enforcement. Baselines NEVER gate CI per the +ticket's hard constraint. The artifact is for: (a) confirming a fix +moved the needle, (b) seeding ForkScore comparisons, (c) noting +warrant-quality findings worth tickets. +""" + +from __future__ import annotations + +import argparse +import datetime as _dt +import hashlib +import json +import os +import subprocess +import sys +import time +from pathlib import Path + + +def _shard_fingerprint(shards_dir: Path) -> str: + """SHA-256 over (path, size, mtime_ns) for every *.db in shards_dir. + Stable across re-opens; changes when any shard mutates.""" + if not shards_dir.is_dir(): + return "" + parts = [] + for p in sorted(shards_dir.glob("*.db")): + st = p.stat() + parts.append(f"{p.name}\t{st.st_size}\t{st.st_mtime_ns}") + return hashlib.sha256("\n".join(parts).encode("utf-8")).hexdigest() + + +def _commit_sha() -> str: + try: + return subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=Path(__file__).parent + ).decode().strip() + except Exception: + return "" + + +def _git_dirty() -> bool: + try: + out = subprocess.check_output( + ["git", "status", "--porcelain"], cwd=Path(__file__).parent + ).decode() + return bool(out.strip()) + except Exception: + return False + + +def _read_fixtures(path: Path) -> list[dict]: + rows = [] + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + obj = json.loads(line) + if "_meta" in obj: + continue + rows.append(obj) + return rows + + +def _run_one( + *, + question: str, + shards_dir: Path, + qa_db: Path, + chat_client, + model_id: str, + burn: bool, +) -> dict: + """Execute one query through the real query() entry point and + extract a small, stable summary dict.""" + from arborist.qa.query import query as _query + from arborist.qa.client import StubClient # noqa: F401 — type hint use + + t0 = time.monotonic() + result = _query( + question=question, + qa_db=qa_db, + chat_client=chat_client, + model_id=model_id, + shards_dir=shards_dir, + burn_existing=burn, + ) + elapsed_ms = round((time.monotonic() - t0) * 1000, 1) + + timings = result.get("timings") or {} + sources = result.get("sources") or [] + primary_used = any( + s.get("source_role") in ( + "primary", "primary_answer", "primary_answer_source" + ) and s.get("used") is not False + for s in sources + ) + return { + "question": question, + "status": result.get("status"), + "audit_mode": result.get("audit_mode"), + "verifier_method": result.get("verifier_method"), + "n_verified": result.get("n_verified"), + "n_quotes": result.get("n_quotes"), + "pi_star_ref": result.get("pi_star_ref"), + "lookup_path": result.get("lookup_path"), + "wall_ms": elapsed_ms, + "total_ms": timings.get("total_ms"), + "search_ms": timings.get("search_ms"), + "llm_ms": timings.get("llm_ms"), + "verify_ms": timings.get("verify_ms"), + "canonical_preflight_ms": timings.get("canonical_preflight_ms"), + "sources_count": len(sources), + "primary_source_used": primary_used, + "answer_text": (result.get("answer_text") or "")[:240], + } + + +def _summarize(rows: list[dict]) -> dict: + """Aggregate the per-question rows into headline metrics.""" + n = len(rows) + walls = [r["wall_ms"] for r in rows if isinstance(r.get("wall_ms"), (int, float))] + walls_sorted = sorted(walls) + median = walls_sorted[len(walls_sorted) // 2] if walls_sorted else None + + by_audit: dict[str, int] = {} + by_status: dict[str, int] = {} + for r in rows: + a = r.get("audit_mode") or "—" + s = r.get("status") or "—" + by_audit[a] = by_audit.get(a, 0) + 1 + by_status[s] = by_status.get(s, 0) + 1 + return { + "questions": n, + "wall_ms_median": median, + "wall_ms_max": max(walls) if walls else None, + "wall_ms_min": min(walls) if walls else None, + "by_audit_mode": by_audit, + "by_status": by_status, + "primary_source_used_count": sum( + 1 for r in rows if r.get("primary_source_used") + ), + "canonical_projection_count": sum( + 1 for r in rows if r.get("status") == "canonical_projection" + ), + } + + +def _markdown(rows: list[dict], summary: dict, meta: dict) -> str: + lines = [ + "# Real-shard baseline — " + meta["timestamp_utc"], + "", + f"**Commit:** `{meta['commit_sha'][:12]}`" + + (" (dirty)" if meta.get("git_dirty") else ""), + f"**Shards fingerprint:** `{meta['shards_fingerprint'][:16]}…`", + f"**Shards directory:** `{meta['shards_dir']}`", + "", + "## Summary", + "", + f"- Questions: **{summary['questions']}**", + f"- Wall median: **{summary['wall_ms_median']} ms**", + f"- Wall range: {summary['wall_ms_min']} – {summary['wall_ms_max']} ms", + f"- Primary source used: {summary['primary_source_used_count']}/" + f"{summary['questions']}", + f"- Canonical-projection short-circuits: " + f"{summary['canonical_projection_count']}", + "", + "**Audit-mode distribution:** " + + ", ".join(f"`{k}` {v}" for k, v in sorted(summary["by_audit_mode"].items())), + "", + "**Status distribution:** " + + ", ".join(f"`{k}` {v}" for k, v in sorted(summary["by_status"].items())), + "", + "## Per-question", + "", + "| # | Question | Audit | Status | Wall (ms) | Method |", + "|---|----------|-------|--------|-----------|--------|", + ] + for i, r in enumerate(rows, 1): + q = r["question"] + if len(q) > 60: + q = q[:57] + "…" + lines.append( + f"| {i} | {q} | " + f"{r.get('audit_mode') or '—'} | " + f"{r.get('status') or '—'} | " + f"{r.get('wall_ms') or '—'} | " + f"{r.get('verifier_method') or r.get('pi_star_ref') or '—'} |" + ) + return "\n".join(lines) + "\n" + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + p.add_argument( + "--shards-dir", type=Path, required=True, + help="path to the shard directory (e.g. ~/.arborist/shards)", + ) + p.add_argument( + "--fixtures", type=Path, + default=Path("bench/fixtures/real-shard-baseline-v1.jsonl"), + help="JSONL question set (default: real-shard-baseline-v1)", + ) + p.add_argument( + "--out", type=Path, + default=Path("bench/results/real-shard-baseline.json"), + help="JSON output path", + ) + p.add_argument( + "--md-out", type=Path, + default=Path("bench/results/real-shard-baseline.md"), + help="markdown summary path", + ) + p.add_argument( + "--endpoint", default=os.environ.get( + "ARBORIST_LLM_ENDPOINT", "https://hermes.ai.unturf.com/v1" + ), + help="LLM endpoint URL", + ) + p.add_argument( + "--model", default=os.environ.get( + "ARBORIST_LLM_MODEL", + "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic", + ), + help="model id (default $ARBORIST_LLM_MODEL or hermes-3)", + ) + p.add_argument( + "--burn", action="store_true", + help="force fresh inference per question (skip cache)", + ) + args = p.parse_args(argv) + + if not args.shards_dir.is_dir(): + print(f"error: shards-dir not found: {args.shards_dir}", file=sys.stderr) + return 2 + if not args.fixtures.is_file(): + print(f"error: fixtures not found: {args.fixtures}", file=sys.stderr) + return 2 + + # Defer LLM client construction until the args are known. + from arborist.qa.client import OpenAICompatibleClient + + qa_db = args.shards_dir / "qa.db" + chat_client = OpenAICompatibleClient(base_url=args.endpoint) + + fixtures = _read_fixtures(args.fixtures) + print( + f"running {len(fixtures)} questions against {args.shards_dir}", + file=sys.stderr, + ) + + rows = [] + for fx in fixtures: + question = fx["question"] + print(f" → {question[:60]}", file=sys.stderr) + try: + row = _run_one( + question=question, + shards_dir=args.shards_dir, + qa_db=qa_db, + chat_client=chat_client, + model_id=args.model, + burn=args.burn, + ) + except Exception as exc: # one bad question shouldn't kill the run + row = { + "question": question, + "status": "error", + "error": str(exc), + } + # Fold any operator notes from the fixture + if "expected_routing" in fx: + row["expected_routing"] = fx["expected_routing"] + if "note" in fx: + row["note"] = fx["note"] + rows.append(row) + + summary = _summarize(rows) + meta = { + "schema_version": "real-shard-baseline-v1", + "timestamp_utc": _dt.datetime.now(_dt.timezone.utc).isoformat( + timespec="seconds" + ), + "commit_sha": _commit_sha(), + "git_dirty": _git_dirty(), + "shards_dir": str(args.shards_dir), + "shards_fingerprint": _shard_fingerprint(args.shards_dir), + "model": args.model, + "endpoint": args.endpoint, + "burn": args.burn, + "fixtures_path": str(args.fixtures), + } + + artifact = {**meta, "summary": summary, "queries": rows} + + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(artifact, indent=2, ensure_ascii=False)) + args.md_out.write_text(_markdown(rows, summary, meta)) + + print(f"wrote {args.out}", file=sys.stderr) + print(f"wrote {args.md_out}", file=sys.stderr) + print(json.dumps(summary, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/TICKETS.md b/docs/TICKETS.md index 80a72cd..677fd52 100644 --- a/docs/TICKETS.md +++ b/docs/TICKETS.md @@ -61,7 +61,7 @@ Newest first. Update on every open/close. | ID | Title | Status | Opened | Directive | |----------|------------------------------------------------|-----------------------|------------|-----------| -| #000026 | Real-shard workload baseline + search latency | in progress · Phase 1 landed 2026-05-08 | 2026-05-08 | — | +| #000026 | Real-shard workload baseline + search latency | in progress · Phase 1 + 2 landed 2026-05-08 | 2026-05-08 | — | | #000025 | 5F battery (Function · Finetuning · Falsification · Formulate · Feedback Loop) | in progress · Phase 1a landed 2026-05-08 | 2026-05-07 | — | | #000024 | 5T Phase 1b + Dav1DPrometheus vocabulary alignment | closed · landed 2026-05-08 | 2026-05-07 | — | | #000023 | 5S Phase 1b: Syllogism · Synthesis · Semiotics | closed · landed 2026-05-08 | 2026-05-07 | — | diff --git a/docs/tickets/ticket-000026-real-shard-workload-baseline.md b/docs/tickets/ticket-000026-real-shard-workload-baseline.md index 1139f74..228acff 100644 --- a/docs/tickets/ticket-000026-real-shard-workload-baseline.md +++ b/docs/tickets/ticket-000026-real-shard-workload-baseline.md @@ -1,6 +1,6 @@ # Ticket #000026 — Real-shard workload baseline + search latency -**Status:** in progress · Phase 1 landed 2026-05-08 +**Status:** in progress · Phase 1 + 2 landed 2026-05-08 **Opened:** 2026-05-08 **Scope:** establish a reproducible baseline for arborist's behavior on real shards (~38 GB Wikipedia + crawl). Capture latency, capital @@ -259,8 +259,52 @@ The 65% execute drop is the cold-cache win — each redundant `executescript` had been triggering disk reads at 75 s scale per the reviewer's report. -Phase 2 (baseline artifact) and Phase 3 (warrant-quality finding) -queued. Authorship warrant ladder remains explicitly out of scope +**Phase 2 landed 2026-05-08.** Baseline artifact: + +- ``bench/scripts/real_shard_baseline.py`` — runs an 8-question + fixture against a real shards directory, captures per-query + wall/search/llm/verify timings, audit_mode, verifier_method, + primary-source-used flag, and writes: + - ``bench/results/real-shard-baseline.json`` — durable JSON + with commit_sha, shards_fingerprint (sha256 of sorted + name/size/mtime tuples), per-query rows, summary metrics + - ``bench/results/real-shard-baseline.md`` — short markdown + table (the thing humans read) +- ``bench/fixtures/real-shard-baseline-v1.jsonl`` — the question + set: virt-back, France, Mac OS X, Linux, Microsoft, AMD/Intel + (rivalry path), and two canonical-projection cases (math + logic). +- ``make bench-real-shard`` — Makefile target. Honors + ``ARBORIST_SHARDS_DIR``; defaults to ``$HOME/.arborist/shards``. + +First run (post-Phase-1 + lazy concepts; commit `e78814c` plus +unstaged refactor): + +``` +audit_mode n +STRICT 4 (virt-back, France, Mac OS X, Linux) +HYBRID 2 (Microsoft founder, AMD/Intel) +CANONICAL 2 (0.1+0.2, A IMPL B) + +wall median 4.1 s (range 0.6 ms – 7.2 s) +primary used 4 / 8 +search target <5 s ← met for 5 of 6 RAG queries +``` + +`who wrote virt-back?` lands at 6.3 s wall (vs reviewer's 75 s +report), audit STRICT, cited evidence still includes the copyright +footer (warrant-quality finding deferred to follow-up ticket). + +**Phase 2 follow-up landed in same fan-out:** lazy concept-relations +loading. Profile of the same query showed `synonym_expand` 2.8 s × +2 calls eating 4 s of search budget. Replaced the eager 290 K-row +full-table scan in `arborist.concepts.query` with targeted +WHERE token IN (qtokens) OR target IN (qtokens) loads, plus +process-wide per-token cache. Per-query SQL rows: ~290 K → ~300. +Wall delta on virt-back query: 13.4 s → 9.3 s (-30%) post-Phase-1. + +Phase 3 (warrant-quality finding) is queued as a follow-up ticket +when fox is ready. Authorship warrant ladder remains out of scope per the design choices section. Phase 1 landed in commit `ec92ebc`. +Phase 2 landed in commit ``.