#!/usr/bin/env python3
"""Deterministic retrieval recall@k on a mined ground-truth fixture.

For each mined question (surface-variant form), run retrieval ONLY
(`query --dry-run` — no LLM) and check whether the KNOWN target
article is in the top-k sources. Recall@k is a hard deterministic
number on N ground-truth questions — no verifier, no n=3 LLM noise,
no 5pp floor. This is the instrument a retrieval lever (e.g.
numeral-fold) is measured against: lever recall@k minus baseline
recall@k, on the same mined fixture.

  usage: recall_at_k.py qa_questions_numeral_map.json [--k 8] [--conc 4]
"""
from __future__ import annotations

import argparse
import concurrent.futures as cf
import json
import subprocess
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
ARB = ROOT / ".venv" / "bin" / "arborist"
SHARDS = Path.home() / ".arborist" / "shards"


def _norm(t: str) -> str:
    return (t or "").replace("_", " ").strip().casefold()


def probe(item: dict, k: int) -> int:
    """Return the target's 0-based rank in retrieved sources, or -1 if
    absent (retrieval only, no LLM). One retrieval → recall at ANY
    k<=k is derivable from the rank (a too-lenient k hides a rank-
    only lift; the 2026-05-18 accent-fold case — recall@8 flat but
    rank-1 22->34. Report @1/@3/@k so rank-sensitive folds aren't
    mis-judged)."""
    try:
        out = subprocess.run(
            [str(ARB), "--shards-dir", str(SHARDS), "query", "--dry-run",
             "--json", "--top-k", str(k), "--answer-mode", "claim_lattice",
             item["question"]],
            capture_output=True, text=True, timeout=120,
        ).stdout
        d = json.loads(out)
    except Exception:
        return -1
    tgt = _norm(item["target_title"])
    titles = [_norm(s.get("title") or "") for s in (d.get("sources") or [])]
    return titles.index(tgt) if tgt in titles else -1


def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("map_json")
    ap.add_argument("--k", type=int, default=8)
    ap.add_argument("--conc", type=int, default=4)
    a = ap.parse_args()
    items = json.loads(Path(a.map_json).read_text())
    with cf.ThreadPoolExecutor(max_workers=a.conc) as ex:
        ranks = list(ex.map(lambda i: probe(i, a.k), items))
    n = len(items)
    paired = list(zip(items, ranks))
    print(f"n={n} (deterministic retrieval recall, no LLM — the instrument)")
    for kk in sorted({1, 3, a.k}):
        hit = sum(1 for r in ranks if 0 <= r < kk)
        print(f"  recall@{kk}: {hit}/{n} = {hit/n:.0%}")
    found = [r for r in ranks if r >= 0]
    if found:
        print(f"  surfaced-anywhere: {len(found)}/{n}; mean rank "
              f"{sum(found)/len(found):.2f} (0=top); rank-1 "
              f"{sum(1 for r in found if r == 0)}/{n}")
    misses = [f"{it['question']!r} -> {it['target_title']!r}"
              for it, r in paired if r < 0]
    print(f"  MISSES ({len(misses)}) — never surfaced:")
    for m in misses[:20]:
        print(f"    {m}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())
