diff --git a/CLAUDE.md b/CLAUDE.md index bc38128..a0230d1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -483,6 +483,25 @@ Full discipline + worked examples in `docs/bench-maxing.md`. Headlines: are answerable-by-construction so they **complement, never replace**, the curated adversarial set (the verifier-honesty/trap gate). Conflating the two is itself a bench-maxing error. +- **Report recall@1/@3/@k, not one lenient k — a coarse k hides a + rank-only lift.** `recall_at_k.py` returns the target's rank, so + recall at every k is free from one retrieval. Measured 2026-05-18: + accent-fold looked inert at recall@8 (95→98, noise) but the OFF + baseline was recall@1 55% vs @8 95% — a too-lenient k flattered + it to a near-ceiling and nearly got a real lever wrongly reverted. + recall@1/@3 is the resolution that matters (primary-source + selection keys on rank, not mere top-k presence). Prevalence ≠ + miss-rate either: the corpus survey ranked accent #1 at 8.1% of + *titles*, but the measured miss-rate (recall@1) is what decides — + measure headroom, never rank candidates by raw prevalence. +- **Fan out independent measurements; serial-by-caution is halting + in disguise.** Mined recall is deterministic per query (read-only, + no LLM, no shared state) — concurrency cannot change which sources + rank; the only risk is the per-probe timeout, and dry-run + retrieval (~2 s) has huge margin under the 120 s cap. Run the whole + fold-search backlog (accent / hyphen / honorific / …) as parallel + background sweeps; only same-fixture A/B that toggles `query.py` + state needs git-stash serialisation. - Avoid negation in prompts (Hermes-3-8B inverts under attention). - Bench is the scoreboard; live fixtures are the gates. - Self-heal beats retry (preserve partial output, never fabricate). diff --git a/arborist/qa/query.py b/arborist/qa/query.py index 639bcc2..d9a2a6e 100644 --- a/arborist/qa/query.py +++ b/arborist/qa/query.py @@ -37,6 +37,7 @@ import json import os import re import time +import unicodedata from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from pathlib import Path @@ -197,6 +198,40 @@ def _numeral_fold_variants(s: str) -> set[str]: return out +def _ascii_fold(s: str) -> str: + """Strip combining diacritics: 'Béla Bartók' -> 'Bela Bartok'.""" + return "".join( + ch for ch in unicodedata.normalize("NFKD", s) + if not unicodedata.combining(ch) + ) + + +def _accent_fold_variants(s: str) -> set[str]: + """ASCII-folded word variants for diacritic text. + + "Béla Bartók" -> {"bela", "bartok"} + "what is X?" -> set() (already ASCII — no-op) + + Why this is load-bearing, not cosmetic: `_TITLE_TOKEN_RE` is + `[A-Za-z]…`, so a diacritic title fragments ("Béla" -> "B","la") + and never matches the ASCII form a user types. Folding then + re-tokenising recovers the clean tokens. Additive+symmetric, + same discipline as `_hyphen_fold_variants` (#000007) / + `_numeral_fold_variants`: a pure-ASCII `s` folds to itself -> + empty -> zero effect on non-accent queries/titles. Measured + 2026-05-18 (fold-search #1; 8.1% of corpus titles carry + diacritics). + """ + folded = _ascii_fold(s) + if folded == s: + return set() + return { + t.lower() + for t in _TITLE_TOKEN_RE.findall(folded) + if t.lower() not in _TITLE_STOPWORDS and len(t) > 1 + } + + def _title_query_tokens(s: str) -> set[str]: base = { t.lower() @@ -213,19 +248,13 @@ def _title_query_tokens(s: str) -> set[str]: # Numeral-fold: ordinal-word <-> Roman-numeral, same additive+ # symmetric discipline (measured 2026-05-18; see above). base |= _numeral_fold_variants(s) - return base - base = { - t.lower() - for t in _TITLE_TOKEN_RE.findall(s) - if t.lower() not in _TITLE_STOPWORDS and len(t) > 1 - } - # Hyphen-fold: additively include joined-no-hyphen variants for - # hyphenated runs in `s`. Symmetric — the function is called on - # both queries and titles, and additive fold preserves existing - # match patterns (e.g. `Coca-Cola history` query keeps {coca, - # cola, cocacola, history} so a `Coca-Cola` title still passes - # title-breadth via {coca, cola, cocacola}). See Ticket #000007. - base |= _hyphen_fold_variants(s) + # Accent-fold: ASCII-folded variants of diacritic words. The + # token regex is [A-Za-z]+, so an accented title ("Béla Bartók") + # otherwise fragments into junk and never matches the ASCII form + # a user types. Additive+symmetric, same discipline; no-op when + # `s` is already ASCII (measured 2026-05-18; fold-search #1, + # 8.1% of titles). See `_accent_fold_variants`. + base |= _accent_fold_variants(s) return base diff --git a/bench/mine_questions.py b/bench/mine_questions.py index 087ed7a..bad5165 100644 --- a/bench/mine_questions.py +++ b/bench/mine_questions.py @@ -26,8 +26,16 @@ import glob import json import re import sqlite3 +import unicodedata from pathlib import Path + +def _ascii_fold(s: str) -> str: + return "".join( + ch for ch in unicodedata.normalize("NFKD", s) + if not unicodedata.combining(ch) + ) + # Strict Roman set 1..40 (covers monarchs/popes/wars); membership test # avoids English-word collisions ("DID"/"MIX"/"CI" are not in here). _ROMAN = { @@ -58,7 +66,86 @@ _BAD = ("list of", "(disambiguation)", "(album)", "(song)", "(film)", "(band)", "(novel)", "(video game)") -def mine(shards_dir: str, limit: int) -> list[dict]: +_ACCENT_LETTER = re.compile(r"[À-ÖØ-öø-ÿĀ-ž]") + + +def _surface_variant(title: str, cls: str): + """Return (question, ok) — the surface form a user TYPES vs the + form the corpus STORES. None when this title isn't in-class.""" + if cls == "numeral": + m = _MONARCH.match(title) + if not m: + return None + name, roman, place = m.group(1), m.group(2), (m.group(3) or "") + n = _ROMAN_TO_INT[roman] + if n not in _ORD: + return None + return f"who was {name} the {_ORD[n]}{place.lower()}?" + if cls == "accent": + # Title has diacritics; user types the ASCII-folded form. + if not _ACCENT_LETTER.search(title): + return None + ascii_t = _ascii_fold(title) + toks = title.split() + if (ascii_t == title or len(toks) > 5 or len(toks) == 0 + or not title[0].isupper()): + return None + return f"what is {ascii_t}?" + if cls == "hyphen": + # Title is hyphenated ("Jean-Paul Sartre", "Coca-Cola"); user + # types the de-hyphenated (space) form. Measures whether the + # EXISTING _hyphen_fold_variants (#000007) actually delivers + # on this 162K-prevalence class (never quantified). + if "-" not in title: + return None + dehy = title.replace("-", " ") + toks = title.split() + if (dehy == title or len(toks) > 5 or len(toks) == 0 + or not title[0].isupper() or title.endswith("-")): + return None + return f"what is {dehy}?" + if cls == "honorific": + # Title spells the honorific in full ("Mount Everest", "Saint + # Petersburg", "Doctor Who"); user types the abbreviation. + # 22K-prevalence; concretely real (curated bench has "mount + # kilimanjaro"). Tests whether retrieval bridges St/Mt/Dr <-> + # Saint/Mount/Doctor (no fold for this exists yet). + abbr = {"Mount": "Mt", "Saint": "St", "Doctor": "Dr", + "Fort": "Ft", "General": "Gen", "President": "Pres", + "Captain": "Capt", "Senator": "Sen"} + toks = title.split() + if len(toks) < 2 or len(toks) > 5 or toks[0] not in abbr: + return None # need >=2 tokens (drop degenerate "Mount") + short = " ".join([abbr[toks[0]]] + toks[1:]) + return f"what is {short}?" + if cls == "amp": + # Title has "&"; user types "and". "AT&T" -> "AT and T". + if "&" not in title: + return None + v = re.sub(r"\s*&\s*", " and ", title).strip() + toks = title.split() + if v == title or len(toks) > 6 or not title[0].isalnum(): + return None + return f"what is {v}?" + if cls == "brit": + # Title uses a British spelling; user types the American one. + bm = {"colour": "color", "honour": "honor", "behaviour": "behavior", + "organisation": "organization", "defence": "defense", + "centre": "center", "theatre": "theater", + "catalogue": "catalog", "programme": "program", + "labour": "labor", "favour": "favor"} + low = title.lower() + hit = next((b for b in bm if b in low), None) + if hit is None or len(title.split()) > 5 or not title[0].isupper(): + return None + v = re.sub(hit, bm[hit], title, flags=re.IGNORECASE) + if v == title: + return None + return f"what is {v}?" + raise SystemExit(f"unknown class {cls!r}") + + +def mine(shards_dir: str, limit: int, cls: str) -> list[dict]: out: list[dict] = [] seen: set[str] = set() for db in sorted(glob.glob(f"{shards_dir}/00*.db")): @@ -77,17 +164,10 @@ def mine(shards_dir: str, limit: int) -> list[dict]: tl = title.lower() if any(b in tl for b in _BAD) or title in seen: continue - m = _MONARCH.match(title) - if not m: - continue - name, roman, place = m.group(1), m.group(2), (m.group(3) or "") - n = _ROMAN_TO_INT[roman] - if n not in _ORD: # keep natural ordinal phrasing only + q = _surface_variant(title, cls) + if q is None: continue seen.add(title) - # Surface variant: the Arabic/ordinal form a user types, - # vs the Roman-numeral title the corpus stores. - q = f"who was {name} the {_ORD[n]}{place.lower()}?" out.append({ "question": q, "target_title": title, @@ -108,7 +188,7 @@ def main() -> int: ap.add_argument("--limit", type=int, default=40) ap.add_argument("--cls", default="numeral") a = ap.parse_args() - rows = mine(a.shards_dir, a.limit) + rows = mine(a.shards_dir, a.limit, a.cls) root = Path(__file__).resolve().parents[1] / "bench" txt = root / f"qa_questions_{a.cls}.txt" mp = root / f"qa_questions_{a.cls}_map.json" diff --git a/bench/qa_questions_accent.txt b/bench/qa_questions_accent.txt new file mode 100644 index 0000000..cbb7bba --- /dev/null +++ b/bench/qa_questions_accent.txt @@ -0,0 +1,43 @@ +# AUTO-MINED (accent class) from corpus titles via bench/mine_questions.py — ground-truth-carrying. +# Graded by deterministic retrieval recall@k (bench/recall_at_k.py), NOT audit_mode. Not adversarial; complements (never replaces) qa_questions.txt. + +what is Casa Batllo? +what is Andre-Marie Ampere? +what is Antoni Gaudi? +what is Alcobaca (Portugal)? +what is Bifrost? +what is Bootes? +what is Bela Bartok? +what is Blue Oyster Cult? +what is Bjorn Borg? +what is Barıs Manco? +what is Transport in Cote d'Ivoire? +what is Crannog? +what is Emily Bronte? +what is Evariste Galois? +what is Elblag? +what is Erwin Schrodinger? +what is Elisabeth-Louise Vigee-Le Brun? +what is Emperor Shomu? +what is Emperor Yomei? +what is Emperor Kotoku? +what is Empress Koken? +what is Emperor Go-En'yu? +what is Emperor Koan? +what is Emperor Chuai? +what is Francisco Alvares? +what is Felix Guattari? +what is Gota Canal? +what is Goran Bregovic? +what is Godel's completeness theorem? +what is Gilbert Arthur a Beckett? +what is GEANT? +what is Herge? +what is Ismet Inonu? +what is Gyula Andrassy? +what is Jean-Francois Millet? +what is House of Karađorđevic? +what is Kunstlerroman? +what is Musee du Louvre? +what is Lubeck? +what is La Tene culture? diff --git a/bench/qa_questions_accent_map.json b/bench/qa_questions_accent_map.json new file mode 100644 index 0000000..bead86b --- /dev/null +++ b/bench/qa_questions_accent_map.json @@ -0,0 +1,242 @@ +[ + { + "question": "what is Casa Batllo?", + "target_title": "Casa Batlló", + "target_root": "c7cb48984f93abd73b6d9166fae8adfa1c11cf93240a2a5673c7006d729366f6", + "shard": "000.db" + }, + { + "question": "what is Andre-Marie Ampere?", + "target_title": "André-Marie Ampère", + "target_root": "46c0ca49bbebb1895e66a621afeb28f207766d76be1a2d40d7ffd567fa757478", + "shard": "000.db" + }, + { + "question": "what is Antoni Gaudi?", + "target_title": "Antoni Gaudí", + "target_root": "70ca4151b7cb1b6c5d9198ca53cb60d2ea932cfd2db67ab0e196ec95b27c9ffe", + "shard": "000.db" + }, + { + "question": "what is Alcobaca (Portugal)?", + "target_title": "Alcobaça (Portugal)", + "target_root": "1bf97449bb4d7932e7b0ede1048dc59ff032d85ffc320c55635a78f3f3eee848", + "shard": "000.db" + }, + { + "question": "what is Bifrost?", + "target_title": "Bifröst", + "target_root": "5866b76e99adc35eb76df8369ffc816af9c3731bfbe253e4f2773792bacf322a", + "shard": "000.db" + }, + { + "question": "what is Bootes?", + "target_title": "Boötes", + "target_root": "f4666faa11a94258c05853bc4fa77d95f4934b0db93072aa1ffe44436351910c", + "shard": "000.db" + }, + { + "question": "what is Bela Bartok?", + "target_title": "Béla Bartók", + "target_root": "9c32c437787b598486b73ca1c3d163e8c1078d371e44a2342b4dff00a8da9b2c", + "shard": "000.db" + }, + { + "question": "what is Blue Oyster Cult?", + "target_title": "Blue Öyster Cult", + "target_root": "f1164863229cba552651c4e21efabe7f8036a40292128a2353909438b7a4a240", + "shard": "000.db" + }, + { + "question": "what is Bjorn Borg?", + "target_title": "Björn Borg", + "target_root": "7e42aee6fc7f3e1f8a768c8675165263d9e38f0adf1540944750638b19a6d259", + "shard": "000.db" + }, + { + "question": "what is Barıs Manco?", + "target_title": "Barış Manço", + "target_root": "38403d8732dcb022ebe2d09710ac90f0616bb6710b8033ca8ed1f1770ef5803b", + "shard": "000.db" + }, + { + "question": "what is Transport in Cote d'Ivoire?", + "target_title": "Transport in Côte d'Ivoire", + "target_root": "2e3411d102c03f245e9906ef61da6ff3fede9600feaac53c39df1f69afd5790c", + "shard": "000.db" + }, + { + "question": "what is Crannog?", + "target_title": "Crannóg", + "target_root": "12b46b289c0db835b8d95c7485f6b86e2b7b104b7c39e597ec532458345550a8", + "shard": "000.db" + }, + { + "question": "what is Emily Bronte?", + "target_title": "Emily Brontë", + "target_root": "78c53cd81b568539d6e38454482a0ba97fd86bfa4a590db8670c99dc215c2b58", + "shard": "000.db" + }, + { + "question": "what is Evariste Galois?", + "target_title": "Évariste Galois", + "target_root": "3da640319defb1ffe7b178a4a424de2b283d96fc62672d7c36e91a3ce9943036", + "shard": "000.db" + }, + { + "question": "what is Elblag?", + "target_title": "Elbląg", + "target_root": "b5ec92e119576a7c04a7600fb60ac6058edcd971d29e798699215065404a3580", + "shard": "000.db" + }, + { + "question": "what is Erwin Schrodinger?", + "target_title": "Erwin Schrödinger", + "target_root": "0ead5314753e1894bc2eaa220bf2cc93c639997128dac3a517d8b4661be228cc", + "shard": "000.db" + }, + { + "question": "what is Elisabeth-Louise Vigee-Le Brun?", + "target_title": "Élisabeth-Louise Vigée-Le Brun", + "target_root": "73e60587c35c69d31312f9585ae1a43dcf0e2b9ba100ce573fcbd95fc98bb778", + "shard": "000.db" + }, + { + "question": "what is Emperor Shomu?", + "target_title": "Emperor Shōmu", + "target_root": "61bc5d5b019b07bb92f9572dab03afedc236dfad43f87d9c0381718f243afd2d", + "shard": "000.db" + }, + { + "question": "what is Emperor Yomei?", + "target_title": "Emperor Yōmei", + "target_root": "42559a42ca8213a5a87187b7c2a05d44ea979bc0bdb8e891ad8b8d5b138db69c", + "shard": "000.db" + }, + { + "question": "what is Emperor Kotoku?", + "target_title": "Emperor Kōtoku", + "target_root": "4aeed736732080574b552f04ec71097b4889c0bef3c33de5a74593a7f3e90c49", + "shard": "000.db" + }, + { + "question": "what is Empress Koken?", + "target_title": "Empress Kōken", + "target_root": "52a095549a02229caa4bde6bcd2c564173219329b36929cb9507d74ea103f49a", + "shard": "000.db" + }, + { + "question": "what is Emperor Go-En'yu?", + "target_title": "Emperor Go-En'yū", + "target_root": "1eda22a89b6650822efc097067dbc87e5d74a42b3f93dd1184c9f25f97e0e6a3", + "shard": "000.db" + }, + { + "question": "what is Emperor Koan?", + "target_title": "Emperor Kōan", + "target_root": "511757327322ff07b858641fdb97a561bf5601c43bafcff5ee56c42fc84d20d9", + "shard": "000.db" + }, + { + "question": "what is Emperor Chuai?", + "target_title": "Emperor Chūai", + "target_root": "eb7ebfeb5117d4982f657d52604321eb95b6aec001a7d44e081769471285a160", + "shard": "000.db" + }, + { + "question": "what is Francisco Alvares?", + "target_title": "Francisco Álvares", + "target_root": "06cb19fc595ef25636dcb7a25354109e82322a59d732dfab8d2104bbabda5a3f", + "shard": "000.db" + }, + { + "question": "what is Felix Guattari?", + "target_title": "Félix Guattari", + "target_root": "53ce58083d07db46e928cfc6021a3e8af2e8d6a3e04d93a530ac25dbec672d17", + "shard": "000.db" + }, + { + "question": "what is Gota Canal?", + "target_title": "Göta Canal", + "target_root": "be181ec1af0cd06845a315b1454c766a8b70d5755b432733ee313cf3cf7e222f", + "shard": "000.db" + }, + { + "question": "what is Goran Bregovic?", + "target_title": "Goran Bregović", + "target_root": "671c9e75c1d1f3b7139bfa328b2b45dd0bbfbaca8b8e26e6a3e4dae806f25164", + "shard": "000.db" + }, + { + "question": "what is Godel's completeness theorem?", + "target_title": "Gödel's completeness theorem", + "target_root": "8b5321e865294e0aa60353714ad65687ffd1a9afa4eb35ab351bfbf17dc8b2c7", + "shard": "000.db" + }, + { + "question": "what is Gilbert Arthur a Beckett?", + "target_title": "Gilbert Arthur à Beckett", + "target_root": "973b6bfbe56a78a864b821048f8068b4dcfe602d97de264b21e40e23bc73b9f7", + "shard": "000.db" + }, + { + "question": "what is GEANT?", + "target_title": "GÉANT", + "target_root": "68a6935d77e47fab693eff9f5cbf1ee85ccb88ce0f6f9296b3904457f5e057dc", + "shard": "000.db" + }, + { + "question": "what is Herge?", + "target_title": "Hergé", + "target_root": "6e442f3603465ea8802aed2bcdeb4cf6843fb3d313c4c469a3e4b24ee611a081", + "shard": "000.db" + }, + { + "question": "what is Ismet Inonu?", + "target_title": "İsmet İnönü", + "target_root": "0eee497696dccd84a7da7b6eb30437f8f16815213f115e7a0bbfd49252336052", + "shard": "000.db" + }, + { + "question": "what is Gyula Andrassy?", + "target_title": "Gyula Andrássy", + "target_root": "669fc54d24191a8d02430773239c69e8c5e96219dc895edaec8bfcf3d60aa316", + "shard": "000.db" + }, + { + "question": "what is Jean-Francois Millet?", + "target_title": "Jean-François Millet", + "target_root": "9f7f942db3b566855a28ec765cf10ff66302dc26e2716788e4db54977827e9ea", + "shard": "000.db" + }, + { + "question": "what is House of Karađorđevic?", + "target_title": "House of Karađorđević", + "target_root": "6cf69d71367fa69d540fe16d85a25f5a3ef5763108dfb9e0974fd0171d1f9e5a", + "shard": "000.db" + }, + { + "question": "what is Kunstlerroman?", + "target_title": "Künstlerroman", + "target_root": "3185b8bc72db6fc984585ae896fe090cc58528c8f7d6f93a26f15201cbfbcbc2", + "shard": "000.db" + }, + { + "question": "what is Musee du Louvre?", + "target_title": "Musée du Louvre", + "target_root": "f41c5777f3ca280ce9fc74cd5d2b58e4da947db5e1d10e5ca30a801943224100", + "shard": "000.db" + }, + { + "question": "what is Lubeck?", + "target_title": "Lübeck", + "target_root": "be78b002f1ccd5ab64f738ed8e550f5439a11827d8ccb9d84081c4383201e4ab", + "shard": "000.db" + }, + { + "question": "what is La Tene culture?", + "target_title": "La Tène culture", + "target_root": "494f1dc71c13f5fbeb6aa114377104f1bcbbb824df1776fd1d5c424754570ae3", + "shard": "000.db" + } +] diff --git a/bench/qa_questions_amp.txt b/bench/qa_questions_amp.txt new file mode 100644 index 0000000..e26af85 --- /dev/null +++ b/bench/qa_questions_amp.txt @@ -0,0 +1,43 @@ +# AUTO-MINED (amp class) from corpus titles via bench/mine_questions.py — ground-truth-carrying. +# Graded by deterministic retrieval recall@k (bench/recall_at_k.py), NOT audit_mode. Not adversarial; complements (never replaces) qa_questions.txt. + +what is Heckler and Koch? +what is Science and Environmental Policy Project? +what is Texas A and M University? +what is Pratt and Whitney? +what is Ernst and Young? +what is Waterloo and City line? +what is Duany Plater-Zyberk and Company? +what is The Sandman: Fables and Reflections? +what is Rape, Abuse and Incest National Network? +what is Hilton Hotels and Resorts? +what is North Walsham and Dilham Canal? +what is Bill and Melinda Gates Foundation? +what is Question Mark and the Mysterians? +what is A and B? +what is Chivalry and Sorcery? +what is II and III? +what is Standard and Poor's? +what is Cheech and Chong? +what is Sasha and John Digweed? +what is Murat and Jose? +what is Valleys and Cardiff Local Routes? +what is The College of William and Mary? +what is Industrial Light and Magic? +what is Barnes and Noble? +what is Law and Order: Special Victims Unit? +what is Grammy Award for Best R and B Song? +what is Ike and Tina Turner? +what is H and M? +what is Yesterday and Today? +what is Open Fire (Y and T album)? +what is Funk and Wagnalls? +what is South Park: Bigger, Longer and Uncut? +what is Lewis and Clark College? +what is Love and Pop? +what is Barnes and Barnes? +what is Sky (UK and Ireland)? +what is Sergio and The Ladies? +what is B and B? +what is Starwood Hotels and Resorts Worldwide? +what is FRANC 2D and 3D? diff --git a/bench/qa_questions_amp_map.json b/bench/qa_questions_amp_map.json new file mode 100644 index 0000000..32d1aa5 --- /dev/null +++ b/bench/qa_questions_amp_map.json @@ -0,0 +1,242 @@ +[ + { + "question": "what is Heckler and Koch?", + "target_title": "Heckler & Koch", + "target_root": "978210367f209c583186c0c826506da54b9ca1414390325960b6c9535d9c34e5", + "shard": "000.db" + }, + { + "question": "what is Science and Environmental Policy Project?", + "target_title": "Science & Environmental Policy Project", + "target_root": "0ea78023a0dd8f2c10334fd1e5aa659c50b1ed2b21d0565186969d9a55a194e2", + "shard": "000.db" + }, + { + "question": "what is Texas A and M University?", + "target_title": "Texas A&M University", + "target_root": "996f9a9e7841fc4b27b8b58f08feedaefcf8be72337abc267526390c21d10554", + "shard": "000.db" + }, + { + "question": "what is Pratt and Whitney?", + "target_title": "Pratt & Whitney", + "target_root": "dc90f35ad29a9c39d2885005ebaff3877bc4d5715d6e82e92ac0ff332682fc10", + "shard": "000.db" + }, + { + "question": "what is Ernst and Young?", + "target_title": "Ernst & Young", + "target_root": "6a1a662c541927efcee315bc87a20875debbadbfc3ff976e6e146697a3025f54", + "shard": "000.db" + }, + { + "question": "what is Waterloo and City line?", + "target_title": "Waterloo & City line", + "target_root": "afd255f827b62f26d8ed616ef73a8d5761ac3957a3095dd1a60216983f363d29", + "shard": "000.db" + }, + { + "question": "what is Duany Plater-Zyberk and Company?", + "target_title": "Duany Plater-Zyberk & Company", + "target_root": "b8b97a7f838ac2116c6de410b628bc9cc8ef7e6aecc9e1edbbd3d9d482ba7d71", + "shard": "000.db" + }, + { + "question": "what is The Sandman: Fables and Reflections?", + "target_title": "The Sandman: Fables & Reflections", + "target_root": "2a7e6bcd2a102495062337590b910d2cc5231b970304658bfc246c4e2df43bdf", + "shard": "000.db" + }, + { + "question": "what is Rape, Abuse and Incest National Network?", + "target_title": "Rape, Abuse & Incest National Network", + "target_root": "95954855b7ffc3a93eb567ec14f992fa42e8208777a1abc1a1bd19c41541c721", + "shard": "000.db" + }, + { + "question": "what is Hilton Hotels and Resorts?", + "target_title": "Hilton Hotels & Resorts", + "target_root": "f06e50b70959ea4df96f549200fc94c7602443905a3b8deee32942f6b7f53b2f", + "shard": "000.db" + }, + { + "question": "what is North Walsham and Dilham Canal?", + "target_title": "North Walsham & Dilham Canal", + "target_root": "3c43a24f38fb9ab784094bf8683c79de6fb0db425ec6d6729785472424e93812", + "shard": "000.db" + }, + { + "question": "what is Bill and Melinda Gates Foundation?", + "target_title": "Bill & Melinda Gates Foundation", + "target_root": "ff90a5fd4baca71f65c1d1886bb7aafbea7808b9fbdc47d423c8f077f612f4b1", + "shard": "000.db" + }, + { + "question": "what is Question Mark and the Mysterians?", + "target_title": "Question Mark & the Mysterians", + "target_root": "140e8733583e32753ac5284c3d9fa19a83f99c779e3fb7eac6ce24678d3eebf5", + "shard": "000.db" + }, + { + "question": "what is A and B?", + "target_title": "A&B", + "target_root": "89792d9631cb85464c18a217608029866f263e7fef32a802b4a16ccaca0daff5", + "shard": "000.db" + }, + { + "question": "what is Chivalry and Sorcery?", + "target_title": "Chivalry & Sorcery", + "target_root": "b54eb248a89978066bae0f64111f02124132d8a52b3c60669f7cc0055d8ced62", + "shard": "000.db" + }, + { + "question": "what is II and III?", + "target_title": "II & III", + "target_root": "2c044b0037189e81caa0c3ba5d65fc5bce0c00e4538eba57b02a221dc005d203", + "shard": "000.db" + }, + { + "question": "what is Standard and Poor's?", + "target_title": "Standard & Poor's", + "target_root": "49a1135e45aa1ceccdc4c6b1f40b76032ac8753801960aa5bdcc63a613d343ef", + "shard": "000.db" + }, + { + "question": "what is Cheech and Chong?", + "target_title": "Cheech & Chong", + "target_root": "22d3997ec9786757c58879ca7d16c6662b1ea4c2dc0f8a6104025b5a645b0c7f", + "shard": "000.db" + }, + { + "question": "what is Sasha and John Digweed?", + "target_title": "Sasha & John Digweed", + "target_root": "f23eeac55103a33e8ed53dda226e96428ee043b0fe8f6b34447a8432d19742ff", + "shard": "000.db" + }, + { + "question": "what is Murat and Jose?", + "target_title": "Murat & Jose", + "target_root": "7edb97a9c375502a60e93a7d8fa4317e12895339aab516d21d219cafa0336655", + "shard": "000.db" + }, + { + "question": "what is Valleys and Cardiff Local Routes?", + "target_title": "Valleys & Cardiff Local Routes", + "target_root": "01f5e16c6b6ef71f60d798b4c8bb79dedbaca8fa9746289859a48120738f46dd", + "shard": "000.db" + }, + { + "question": "what is The College of William and Mary?", + "target_title": "The College of William & Mary", + "target_root": "2686b673858f3310b59052e230c381ac7ae687632041409368cb23c628c556ca", + "shard": "000.db" + }, + { + "question": "what is Industrial Light and Magic?", + "target_title": "Industrial Light & Magic", + "target_root": "bef0cc77efa128c3c89796f1a8f0f26c1f491e34fbd13440ddc9b3b3193883f0", + "shard": "000.db" + }, + { + "question": "what is Barnes and Noble?", + "target_title": "Barnes & Noble", + "target_root": "94b4e7e483c8ca38521e8f85e4c7bdd31d9f8b1b51a599d303eea2f362b8206f", + "shard": "000.db" + }, + { + "question": "what is Law and Order: Special Victims Unit?", + "target_title": "Law & Order: Special Victims Unit", + "target_root": "ab5b911577a527a4e526613d35d2f7dfd5bd6fb44dc1fb513ddffa8fee30c0eb", + "shard": "000.db" + }, + { + "question": "what is Grammy Award for Best R and B Song?", + "target_title": "Grammy Award for Best R&B Song", + "target_root": "2ea1b11647fbdb0f9918000c159750e4220cf9fdf219b0ef3434be86148ed8af", + "shard": "000.db" + }, + { + "question": "what is Ike and Tina Turner?", + "target_title": "Ike & Tina Turner", + "target_root": "9ed9c3e0d8fd1b42b94647ef22f70b3bf4b5a21ec90aa604d26c4892e126accc", + "shard": "000.db" + }, + { + "question": "what is H and M?", + "target_title": "H&M", + "target_root": "cf790fda061f316a16d9fd07e5db1af8a0ca8bc82bed84e9eb0fd2b573017845", + "shard": "000.db" + }, + { + "question": "what is Yesterday and Today?", + "target_title": "Yesterday & Today", + "target_root": "ff1cc6c11ec37effc4a0e202b1197ec4c670b5359b5b8c8f0dfb36a655ba8d6d", + "shard": "000.db" + }, + { + "question": "what is Open Fire (Y and T album)?", + "target_title": "Open Fire (Y&T album)", + "target_root": "ea2873e3ae0ba0c5b8efbef26795eafc6b94b5354caea5a6c1fb800cca6b951d", + "shard": "000.db" + }, + { + "question": "what is Funk and Wagnalls?", + "target_title": "Funk & Wagnalls", + "target_root": "2e420c61ea81778cb0ad6df0d24edc5383f3d0c12528f5fd93d609cbf9ca4a27", + "shard": "000.db" + }, + { + "question": "what is South Park: Bigger, Longer and Uncut?", + "target_title": "South Park: Bigger, Longer & Uncut", + "target_root": "46dc76de50c40c2597ad010bd21890889ddaa9b83cfb2f693703202a94323490", + "shard": "000.db" + }, + { + "question": "what is Lewis and Clark College?", + "target_title": "Lewis & Clark College", + "target_root": "5f52f2383a2eadf7fcc1835a18fd4de66a8a7eaec547356bc82242268efa064c", + "shard": "000.db" + }, + { + "question": "what is Love and Pop?", + "target_title": "Love & Pop", + "target_root": "417083ad66a3ce0972933d83191921b94101001abf159b49c8e055233aa22b2c", + "shard": "000.db" + }, + { + "question": "what is Barnes and Barnes?", + "target_title": "Barnes & Barnes", + "target_root": "a9251d47e8449b5bc94f4b1ebe5bbe0aaf95e9f1df0ec760546ecdaba1208de3", + "shard": "000.db" + }, + { + "question": "what is Sky (UK and Ireland)?", + "target_title": "Sky (UK & Ireland)", + "target_root": "eeeed19fb1fbe2ffd96abce62ba11b50846960a3e7039350d2d92ffa5db73f7a", + "shard": "000.db" + }, + { + "question": "what is Sergio and The Ladies?", + "target_title": "Sergio & The Ladies", + "target_root": "931c0417680eeef901977d8b03b82b9476bd03d7c16b388612e748f3f88ec1b8", + "shard": "000.db" + }, + { + "question": "what is B and B?", + "target_title": "B&B", + "target_root": "bbf6e2ce9136d564c504c596b648094a8672e75f7e99b3c990630c00736c5f41", + "shard": "000.db" + }, + { + "question": "what is Starwood Hotels and Resorts Worldwide?", + "target_title": "Starwood Hotels & Resorts Worldwide", + "target_root": "ba0402f6975a9ab58bbcd23b6ae94ac824c648852a1070ec06814d56a377bd26", + "shard": "000.db" + }, + { + "question": "what is FRANC 2D and 3D?", + "target_title": "FRANC 2D&3D", + "target_root": "6af75043e187533640266428c432cf209a3895b19ad85d9c3b090069adfd9e8a", + "shard": "000.db" + } +] diff --git a/bench/qa_questions_brit.txt b/bench/qa_questions_brit.txt new file mode 100644 index 0000000..2056837 --- /dev/null +++ b/bench/qa_questions_brit.txt @@ -0,0 +1,43 @@ +# AUTO-MINED (brit class) from corpus titles via bench/mine_questions.py — ground-truth-carrying. +# Graded by deterministic retrieval recall@k (bench/recall_at_k.py), NOT audit_mode. Not adversarial; complements (never replaces) qa_questions.txt. + +what is Finnish defense Forces? +what is Hopewell center, Hong Kong? +what is labor economics? +what is World organization for Animal Health? +what is Papua New Guinea defense Force? +what is programr? +what is Tonga defense Services? +what is Trinidad and Tobago defense Force? +what is Zambian defense Force? +what is defense Signals Directorate? +what is English Renaissance theater? +what is labor Party (Netherlands)? +what is The Luzhin defense? +what is Asia-Pacific Network Information center? +what is Soyuz program? +what is center Georges Pompidou? +what is Amphitheater? +what is Australian defense Force? +what is center County, Pennsylvania? +what is organization internationale de la Francophonie? +what is Luna program? +what is Man and the Biosphere program? +what is centerville, Maryland? +what is Boggs Township, center County, Pennsylvania? +what is Ferguson Township, center County, Pennsylvania? +what is Harris Township, center County, Pennsylvania? +what is Huston Township, center County, Pennsylvania? +what is Patton Township, center County, Pennsylvania? +what is Union Township, center County, Pennsylvania? +what is Worth Township, center County, Pennsylvania? +what is center Township, Perry County, Pennsylvania? +what is center Township, Pennsylvania? +what is Tricorn center? +what is Living color? +what is Philipsburg, center County, Pennsylvania? +what is German labor Front? +what is Rogers center? +what is Ulster defense Association? +what is Guiana Space center? +what is Community theater? diff --git a/bench/qa_questions_brit_map.json b/bench/qa_questions_brit_map.json new file mode 100644 index 0000000..8e481be --- /dev/null +++ b/bench/qa_questions_brit_map.json @@ -0,0 +1,242 @@ +[ + { + "question": "what is Finnish defense Forces?", + "target_title": "Finnish Defence Forces", + "target_root": "6114cd5bf6cfcd9b69fdb7dbc07842d6e610c3e2dced64cf61c8f25c6cbc5948", + "shard": "000.db" + }, + { + "question": "what is Hopewell center, Hong Kong?", + "target_title": "Hopewell Centre, Hong Kong", + "target_root": "1e6245dfd251badf7caf1022a673a9935f036853b958c58f76e4432dd353bf91", + "shard": "000.db" + }, + { + "question": "what is labor economics?", + "target_title": "Labour economics", + "target_root": "ad7a31d34d70fb673a30deb8a458a6e924fc3c41970b30752ce5d14d55996260", + "shard": "000.db" + }, + { + "question": "what is World organization for Animal Health?", + "target_title": "World Organisation for Animal Health", + "target_root": "e0e1f1c1d3e206192d3e74278426af1587c055ab12d35211fbefa356b827e3f4", + "shard": "000.db" + }, + { + "question": "what is Papua New Guinea defense Force?", + "target_title": "Papua New Guinea Defence Force", + "target_root": "1d6da0aa77ad422ce9f23baba3832c67547b09ac6945733c4d484491f96ca3a6", + "shard": "000.db" + }, + { + "question": "what is programr?", + "target_title": "Programmer", + "target_root": "f7109a928748c48b88bd213a59bcf1205777c7aaed49209d5c57f4884d578c10", + "shard": "000.db" + }, + { + "question": "what is Tonga defense Services?", + "target_title": "Tonga Defence Services", + "target_root": "f5754650b7149e5e5c3cbf23d8876eafb5070c3571c6f8dae6322c75e09f8342", + "shard": "000.db" + }, + { + "question": "what is Trinidad and Tobago defense Force?", + "target_title": "Trinidad and Tobago Defence Force", + "target_root": "a816b09a0aab2bcf777039b3ac5a26bbc20fb812b2a45bebb317333d21aa5670", + "shard": "000.db" + }, + { + "question": "what is Zambian defense Force?", + "target_title": "Zambian Defence Force", + "target_root": "0a81b1428df734fefcf16d56d4bc33e9121720974e03238eff2c2d0d3b7aaaad", + "shard": "000.db" + }, + { + "question": "what is defense Signals Directorate?", + "target_title": "Defence Signals Directorate", + "target_root": "d9abb78507855e0318888f23e1a5f513cb3c3e5710649577e0874d6b54039725", + "shard": "000.db" + }, + { + "question": "what is English Renaissance theater?", + "target_title": "English Renaissance theatre", + "target_root": "dd17095f643aa6fcbe309f3d28df76cea80817c5a26c77f3934caaa04ee7d650", + "shard": "000.db" + }, + { + "question": "what is labor Party (Netherlands)?", + "target_title": "Labour Party (Netherlands)", + "target_root": "76aec0d0f059d87a91d121b83a16b4e39ecb4e84974f2490871e417603e83286", + "shard": "000.db" + }, + { + "question": "what is The Luzhin defense?", + "target_title": "The Luzhin Defence", + "target_root": "cc09adf74f7a1c960b1051af7685e9e9510a7400cb878ba05185a9872d170996", + "shard": "000.db" + }, + { + "question": "what is Asia-Pacific Network Information center?", + "target_title": "Asia-Pacific Network Information Centre", + "target_root": "1eb9e5765f58ae5c3dca51be14997c998dab3b8faaae727d97f17cc934dd0345", + "shard": "000.db" + }, + { + "question": "what is Soyuz program?", + "target_title": "Soyuz programme", + "target_root": "acf6a8b2a5f9ec258861702edeaa7f3e4cc7af298a84fa3a6294e356bc47a44a", + "shard": "000.db" + }, + { + "question": "what is center Georges Pompidou?", + "target_title": "Centre Georges Pompidou", + "target_root": "3b7f02b2b5ddf5dac61a054aef90fabd44be2629de79d8b4165eaa7ecaba1201", + "shard": "000.db" + }, + { + "question": "what is Amphitheater?", + "target_title": "Amphitheatre", + "target_root": "c63e6c7a60bbaf19dc65d2b869d86274ba1268bd3790366fdf0a14476907a7eb", + "shard": "000.db" + }, + { + "question": "what is Australian defense Force?", + "target_title": "Australian Defence Force", + "target_root": "566a69ad2196a5e5d5f084a375dee9231f0344e0d4c085a79990b2291265edd0", + "shard": "000.db" + }, + { + "question": "what is center County, Pennsylvania?", + "target_title": "Centre County, Pennsylvania", + "target_root": "80487e3ec0141594504f4bc68cd858d67d6ef085c10125e40fc5533a234c5c94", + "shard": "000.db" + }, + { + "question": "what is organization internationale de la Francophonie?", + "target_title": "Organisation internationale de la Francophonie", + "target_root": "8b02358ebc82c033b55c85972344cc54310b34a5a2a0a921d7f97f59677377e2", + "shard": "000.db" + }, + { + "question": "what is Luna program?", + "target_title": "Luna programme", + "target_root": "add3b8760cbe907a2a3076b91466afa7f0e5957d47c56fdcf7e72d9e37e7c1ba", + "shard": "000.db" + }, + { + "question": "what is Man and the Biosphere program?", + "target_title": "Man and the Biosphere Programme", + "target_root": "c92585fe422ad5ff0467a57f0c854319985e15b66bb5fd56ac0664b19650ec70", + "shard": "000.db" + }, + { + "question": "what is centerville, Maryland?", + "target_title": "Centreville, Maryland", + "target_root": "148a7ab7213c794ebdc8875f6ff9e5697a1699bc1923e08857e81f133ea99bbd", + "shard": "000.db" + }, + { + "question": "what is Boggs Township, center County, Pennsylvania?", + "target_title": "Boggs Township, Centre County, Pennsylvania", + "target_root": "ff94137ff22e6330cfc72972ff949b6f45a56e590d08251ec6d14c8341cd8e2d", + "shard": "000.db" + }, + { + "question": "what is Ferguson Township, center County, Pennsylvania?", + "target_title": "Ferguson Township, Centre County, Pennsylvania", + "target_root": "b73fc3e57cc5a9d6574f772f3fb1c921a04b08518b0ff725ddaa8ba983496ede", + "shard": "000.db" + }, + { + "question": "what is Harris Township, center County, Pennsylvania?", + "target_title": "Harris Township, Centre County, Pennsylvania", + "target_root": "37bc6558d805cbc30ef085cacc0c407142745e994a38daa0f0d64b9397e5eac6", + "shard": "000.db" + }, + { + "question": "what is Huston Township, center County, Pennsylvania?", + "target_title": "Huston Township, Centre County, Pennsylvania", + "target_root": "e633a8476cb1d547718cf6952b235762c2762a0d49a4dc8c0e313e19c3ad0d2e", + "shard": "000.db" + }, + { + "question": "what is Patton Township, center County, Pennsylvania?", + "target_title": "Patton Township, Centre County, Pennsylvania", + "target_root": "5746ee9bb4d96fed1695555c3e0c910b1f2f701046ebdaf229aaeb035b41e513", + "shard": "000.db" + }, + { + "question": "what is Union Township, center County, Pennsylvania?", + "target_title": "Union Township, Centre County, Pennsylvania", + "target_root": "a16566edef459502e4a188d8edb01710d9644732ab8b69b0baa47fdb556042d6", + "shard": "000.db" + }, + { + "question": "what is Worth Township, center County, Pennsylvania?", + "target_title": "Worth Township, Centre County, Pennsylvania", + "target_root": "3e2050e2372af027615ddc47e8999aa179f58b92cc0bc992f25b444b7daf7cc7", + "shard": "000.db" + }, + { + "question": "what is center Township, Perry County, Pennsylvania?", + "target_title": "Centre Township, Perry County, Pennsylvania", + "target_root": "f9e0aef04d0933d69a7c4e67d3f48d784ee6e7c05c9a1f2d5d9485930f15e84b", + "shard": "000.db" + }, + { + "question": "what is center Township, Pennsylvania?", + "target_title": "Centre Township, Pennsylvania", + "target_root": "afb66d096c4dce06b655923fd2f05f124d95651f93d9c02e5589a5b9720a24bd", + "shard": "000.db" + }, + { + "question": "what is Tricorn center?", + "target_title": "Tricorn Centre", + "target_root": "1b35fe0e9b63e317624587f1adf3da28084e6dbe922767c778c609521002b130", + "shard": "000.db" + }, + { + "question": "what is Living color?", + "target_title": "Living Colour", + "target_root": "3f19efdb859270607248ae4d4402d2ba8b382159085c1e9e51763b2e098d9d14", + "shard": "000.db" + }, + { + "question": "what is Philipsburg, center County, Pennsylvania?", + "target_title": "Philipsburg, Centre County, Pennsylvania", + "target_root": "7febde5ddf0b0a33830b22ae4c46f9f86531c9128c61f9708f5260317de87f37", + "shard": "000.db" + }, + { + "question": "what is German labor Front?", + "target_title": "German Labour Front", + "target_root": "1603c2b80ed5c6795e940d20de1ab6db4fcf8bac2ae37371f91c5674521347a1", + "shard": "000.db" + }, + { + "question": "what is Rogers center?", + "target_title": "Rogers Centre", + "target_root": "091f7ff1f01402924989f7f93165a257f25c8708e39820d94c2b03803be4a3a8", + "shard": "000.db" + }, + { + "question": "what is Ulster defense Association?", + "target_title": "Ulster Defence Association", + "target_root": "5cb5157f5c4ee9f910b29c49303b8b2f51c644b30de4c39b0104665d5e6bef87", + "shard": "000.db" + }, + { + "question": "what is Guiana Space center?", + "target_title": "Guiana Space Centre", + "target_root": "50db2798791be79ef25a6ed0082909ea268f6bad25763ffd4c6e60feb19d8c30", + "shard": "000.db" + }, + { + "question": "what is Community theater?", + "target_title": "Community theatre", + "target_root": "5c745645b88927ed021df3b3ee2de14926efc8be020f0fc85b534126cb653bc4", + "shard": "000.db" + } +] diff --git a/bench/qa_questions_honorific.txt b/bench/qa_questions_honorific.txt new file mode 100644 index 0000000..3b4d5e5 --- /dev/null +++ b/bench/qa_questions_honorific.txt @@ -0,0 +1,43 @@ +# AUTO-MINED (honorific class) from corpus titles via bench/mine_questions.py — ground-truth-carrying. +# Graded by deterministic retrieval recall@k (bench/recall_at_k.py), NOT audit_mode. Not adversarial; complements (never replaces) qa_questions.txt. + +what is Dr Who? +what is Dr Syn? +what is Dr V64? +what is Ft Wayne, Indiana? +what is Ft Collins, Colorado? +what is Pres of France? +what is St Lawrence Seaway? +what is St Kitts and Nevis? +what is St Lucia? +what is St Adrian? +what is St Boniface? +what is St Ninian? +what is St Andrew? +what is St Timothy? +what is St Anselm? +what is St Paul, Minnesota? +what is Mt Everest? +what is Ft Montgomery (Hudson River)? +what is St Nicholas? +what is St Patrick's Battalion? +what is Mt Sinai? +what is St Charles? +what is Mt Ephraim? +what is Gen Electric Company plc? +what is Mt Joy, Pennsylvania? +what is Ft Pitt? +what is Mt Clunie National Park? +what is Mt Nothofagus National Park? +what is Mt Field National Park? +what is Mt Richmond National Park? +what is Mt Aberdeen National Park? +what is Mt Colosseum National Park? +what is Mt Etna Caves National Park? +what is Mt O'Connell National Park? +what is Mt Webb National Park? +what is Ft Rucker? +what is Mt Olive, Alabama? +what is Ft Yukon, Alaska? +what is Mt Ida, Arkansas? +what is Ft Smith, Arkansas? diff --git a/bench/qa_questions_honorific_map.json b/bench/qa_questions_honorific_map.json new file mode 100644 index 0000000..e050736 --- /dev/null +++ b/bench/qa_questions_honorific_map.json @@ -0,0 +1,242 @@ +[ + { + "question": "what is Dr Who?", + "target_title": "Doctor Who", + "target_root": "3bda6eaddefbf7c72119270622302f55ed440de84f8dcfbc8c3fa59bf39f2286", + "shard": "000.db" + }, + { + "question": "what is Dr Syn?", + "target_title": "Doctor Syn", + "target_root": "f36fe260f689bc797a9afe95fc30a3dd7bd9376b3569fd8df572a2002a35287d", + "shard": "000.db" + }, + { + "question": "what is Dr V64?", + "target_title": "Doctor V64", + "target_root": "18972d20548cec458b36984c89aa833ee230e9f696bfed6419ed4172a1413289", + "shard": "000.db" + }, + { + "question": "what is Ft Wayne, Indiana?", + "target_title": "Fort Wayne, Indiana", + "target_root": "cf688a2a18a47316030b7c6f6d3f51e2be398607fc4e20542da534daaab323ff", + "shard": "000.db" + }, + { + "question": "what is Ft Collins, Colorado?", + "target_title": "Fort Collins, Colorado", + "target_root": "c0c464c0315c4e2dd9ef08cf768f93f074ed21f98df39a65ec3f78cd6030aa69", + "shard": "000.db" + }, + { + "question": "what is Pres of France?", + "target_title": "President of France", + "target_root": "382bba1c25dc090e24e4e54a5233509a0eb4b65d5862458fb660e4bf733b3395", + "shard": "000.db" + }, + { + "question": "what is St Lawrence Seaway?", + "target_title": "Saint Lawrence Seaway", + "target_root": "a891a80fef6b9248ecae5cc009a12372738c41b5278716a492e455c2b3509f6b", + "shard": "000.db" + }, + { + "question": "what is St Kitts and Nevis?", + "target_title": "Saint Kitts and Nevis", + "target_root": "c90a8ef06705b60431de77db01a4e2a5964214f51273f54c8e70bdfc5a16e53e", + "shard": "000.db" + }, + { + "question": "what is St Lucia?", + "target_title": "Saint Lucia", + "target_root": "bcc26d405565ba560c68aa2dd21906f3700a3b1d7fef2071af7f853b93ceb5ed", + "shard": "000.db" + }, + { + "question": "what is St Adrian?", + "target_title": "Saint Adrian", + "target_root": "5f8a343d61f4167c96e4e7908bd74e2585ddff4753f6ee29da8832360a395c48", + "shard": "000.db" + }, + { + "question": "what is St Boniface?", + "target_title": "Saint Boniface", + "target_root": "9d36b35a5b47ab475a08626c1eee22c1a6bfe5303b6f8eb036cf9daa8fb2cadd", + "shard": "000.db" + }, + { + "question": "what is St Ninian?", + "target_title": "Saint Ninian", + "target_root": "0ca827e20ca4efabd8509bac52be4f85df7338defb97b1894e5ac869e82986b9", + "shard": "000.db" + }, + { + "question": "what is St Andrew?", + "target_title": "Saint Andrew", + "target_root": "d2294e4c76a3d636ded6e30365ebb1e51a91c671b76603079c024d028ffe9ba1", + "shard": "000.db" + }, + { + "question": "what is St Timothy?", + "target_title": "Saint Timothy", + "target_root": "d0a2f78451581da3ff7146d2e214cd9d0d2b99a6720a2d93d3a86fd3409dd581", + "shard": "000.db" + }, + { + "question": "what is St Anselm?", + "target_title": "Saint Anselm", + "target_root": "7f768c8ccc45617e57f02c9c65bd82c61fb9f43aed4de9abaf67063e746999ac", + "shard": "000.db" + }, + { + "question": "what is St Paul, Minnesota?", + "target_title": "Saint Paul, Minnesota", + "target_root": "22cf7aec47b900ad13865870db6e5e4312da9c9dc4687f33fca9b77e21e1e5e3", + "shard": "000.db" + }, + { + "question": "what is Mt Everest?", + "target_title": "Mount Everest", + "target_root": "aaf3b423eb2ccb4e8cabe42f0bbea24547186b079c939562e48383cbec3928ee", + "shard": "000.db" + }, + { + "question": "what is Ft Montgomery (Hudson River)?", + "target_title": "Fort Montgomery (Hudson River)", + "target_root": "4b89b15f53a746a077d3bd1f5f8d6f8c6602e9781c4201fed1b15f7871b05ced", + "shard": "000.db" + }, + { + "question": "what is St Nicholas?", + "target_title": "Saint Nicholas", + "target_root": "7e6d8087d3ef4eaaf4b869ec97dfc4a46d8fea78f959211f533e9fff1f3da336", + "shard": "000.db" + }, + { + "question": "what is St Patrick's Battalion?", + "target_title": "Saint Patrick's Battalion", + "target_root": "1e32490a958c1c792e8cca33abe46ca0ce6dc9619d694028b4cc67785bf59717", + "shard": "000.db" + }, + { + "question": "what is Mt Sinai?", + "target_title": "Mount Sinai", + "target_root": "b62bac7647911166f1916785423dc42af025a4a0522944ff35053dfa902839e5", + "shard": "000.db" + }, + { + "question": "what is St Charles?", + "target_title": "Saint Charles", + "target_root": "996b4d3dd28814ae12b26553d4746abfd478016a1fe36266fa304f401ffe7abe", + "shard": "000.db" + }, + { + "question": "what is Mt Ephraim?", + "target_title": "Mount Ephraim", + "target_root": "148f2298bf9353a1817e63c27333955adc1adfc5640fdd071e65f77d98327de5", + "shard": "000.db" + }, + { + "question": "what is Gen Electric Company plc?", + "target_title": "General Electric Company plc", + "target_root": "b59db856e9ee268008282cb3a5b49771452b9d7a1a6ec426c829bfb8c204ca27", + "shard": "000.db" + }, + { + "question": "what is Mt Joy, Pennsylvania?", + "target_title": "Mount Joy, Pennsylvania", + "target_root": "4962e4fbe5b0801ae95f81d10c190c51ccaf1df8895e3aec385c95ecbe571328", + "shard": "000.db" + }, + { + "question": "what is Ft Pitt?", + "target_title": "Fort Pitt", + "target_root": "61d1a97836cac809be53e674772d23f0bf2b1dd2ae1b48ad38ecc0eaa6b0b9d9", + "shard": "000.db" + }, + { + "question": "what is Mt Clunie National Park?", + "target_title": "Mount Clunie National Park", + "target_root": "9ac60e06140c6408071d8cfb8cd0df754631833bbf3e53310fef18c08caf95d1", + "shard": "000.db" + }, + { + "question": "what is Mt Nothofagus National Park?", + "target_title": "Mount Nothofagus National Park", + "target_root": "387bdcbe2332bbe299faed4a29cb44253a48bb5a503ce58ceec97c14658aab79", + "shard": "000.db" + }, + { + "question": "what is Mt Field National Park?", + "target_title": "Mount Field National Park", + "target_root": "363059003b548b228c9160860650b753b6a0ecdbeaafd67f00f771c6d7935a58", + "shard": "000.db" + }, + { + "question": "what is Mt Richmond National Park?", + "target_title": "Mount Richmond National Park", + "target_root": "725b6ce0749788d851c843799608160d32165d2e67dedee6fc68097a6e5f889e", + "shard": "000.db" + }, + { + "question": "what is Mt Aberdeen National Park?", + "target_title": "Mount Aberdeen National Park", + "target_root": "2a0423b99faa14704a81c8eecb52162efd1e3a813d0e9ffa865880f3773bda3b", + "shard": "000.db" + }, + { + "question": "what is Mt Colosseum National Park?", + "target_title": "Mount Colosseum National Park", + "target_root": "617498381698f11c8dd5188e651992a278fa772b45e275370e4ecfc0fd6dd7ae", + "shard": "000.db" + }, + { + "question": "what is Mt Etna Caves National Park?", + "target_title": "Mount Etna Caves National Park", + "target_root": "737b9514367c77ee232e88fd1e3ecb69d4d56bd78543d953e0ffcc966ae3c7c7", + "shard": "000.db" + }, + { + "question": "what is Mt O'Connell National Park?", + "target_title": "Mount O'Connell National Park", + "target_root": "1d7bacc37bce42982e531efd1b3bfa138acdfe3d84cbfebbcb564c5400ed3041", + "shard": "000.db" + }, + { + "question": "what is Mt Webb National Park?", + "target_title": "Mount Webb National Park", + "target_root": "b435f7f449dd02cba620200dd860d963ebb993feeb8f2a65b20bb5e4fe9961ca", + "shard": "000.db" + }, + { + "question": "what is Ft Rucker?", + "target_title": "Fort Rucker", + "target_root": "630a70bb0e6019709a11891d28960881d25d90cf20de58b533e031d3ce935ba8", + "shard": "000.db" + }, + { + "question": "what is Mt Olive, Alabama?", + "target_title": "Mount Olive, Alabama", + "target_root": "3ce537cd880daa793b27d6864bfaeb013b40132fa3ea4770c11a148d3fd6c2db", + "shard": "000.db" + }, + { + "question": "what is Ft Yukon, Alaska?", + "target_title": "Fort Yukon, Alaska", + "target_root": "a6dd1002a9fe6827936c461ac4f657fa569a9c5cbc16d56c0b6e50cba91b9c0b", + "shard": "000.db" + }, + { + "question": "what is Mt Ida, Arkansas?", + "target_title": "Mount Ida, Arkansas", + "target_root": "e9e5baae06c14cbc1aa2f23fe45f47372ad59d20386d6f023ad7e0a3f978f315", + "shard": "000.db" + }, + { + "question": "what is Ft Smith, Arkansas?", + "target_title": "Fort Smith, Arkansas", + "target_root": "e4ebbc4a7b0f1c63001e5247a8aa23b5b9046ae6f16cc4ff63b67537f9ceefd1", + "shard": "000.db" + } +] diff --git a/bench/qa_questions_hyphen.txt b/bench/qa_questions_hyphen.txt new file mode 100644 index 0000000..7b27cf4 --- /dev/null +++ b/bench/qa_questions_hyphen.txt @@ -0,0 +1,43 @@ +# AUTO-MINED (hyphen class) from corpus titles via bench/mine_questions.py — ground-truth-carrying. +# Graded by deterministic retrieval recall@k (bench/recall_at_k.py), NOT audit_mode. Not adversarial; complements (never replaces) qa_questions.txt. + +what is The Amazing Spider Man (comic book)? +what is Augustin Jean Fresnel? +what is André Marie Ampère? +what is Aster CT 80? +what is Anti globalization movement? +what is Abd ar Rahman I? +what is Abd ar Rahman V? +what is Anti Ballistic Missile Treaty? +what is AGM 88 HARM? +what is Lockheed AC 130? +what is CIM 10 Bomarc? +what is North American B 25 Mitchell? +what is Bain marie? +what is Cross dressing? +what is Call of Cthulhu (role playing game)? +what is Context sensitive? +what is Boeing C 17 Globemaster III? +what is Chiang Kai shek? +what is Context free language? +what is C* algebra? +what is Computer generated imagery? +what is Lockheed C 130 Hercules? +what is Covenant breaker? +what is Comprehensive Nuclear Test Ban Treaty? +what is Double slit experiment? +what is Double ended queue? +what is Cost push inflation? +what is Eductor jet pump? +what is Eight ball? +what is Élisabeth Louise Vigée Le Brun? +what is Evidence based medicine? +what is E Prime? +what is Boeing E 3 Sentry? +what is Emperor Go En'yū? +what is Field programmable gate array? +what is Five spice powder? +what is Four poster? +what is Flip flop (electronics)? +what is Guinea Bissau? +what is Politics of Guinea Bissau? diff --git a/bench/qa_questions_hyphen_map.json b/bench/qa_questions_hyphen_map.json new file mode 100644 index 0000000..041fd8a --- /dev/null +++ b/bench/qa_questions_hyphen_map.json @@ -0,0 +1,242 @@ +[ + { + "question": "what is The Amazing Spider Man (comic book)?", + "target_title": "The Amazing Spider-Man (comic book)", + "target_root": "d53601e2e4f7660788840ffc97520b3c8f8fd9cbe72e868274096ab16e08baee", + "shard": "000.db" + }, + { + "question": "what is Augustin Jean Fresnel?", + "target_title": "Augustin-Jean Fresnel", + "target_root": "afa063eca8b1cc23f2d8b474a79e0f7abab1613e5a6156cf9ab3ab79805f3c2f", + "shard": "000.db" + }, + { + "question": "what is André Marie Ampère?", + "target_title": "André-Marie Ampère", + "target_root": "46c0ca49bbebb1895e66a621afeb28f207766d76be1a2d40d7ffd567fa757478", + "shard": "000.db" + }, + { + "question": "what is Aster CT 80?", + "target_title": "Aster CT-80", + "target_root": "3bd6a0d66582ca9aafc97acc0e4e8c41608baa22fd218ea21d7cf83e9a90b4e9", + "shard": "000.db" + }, + { + "question": "what is Anti globalization movement?", + "target_title": "Anti-globalization movement", + "target_root": "e0a5d7170fce542cae4f00c1b98acb98b6fe7b8f6033140419fa023318ae65bd", + "shard": "000.db" + }, + { + "question": "what is Abd ar Rahman I?", + "target_title": "Abd ar-Rahman I", + "target_root": "1e62aee7c9143bcf070585b0af30e0ea6d3fd3e434b6a6c4fc1c37dfe2fde651", + "shard": "000.db" + }, + { + "question": "what is Abd ar Rahman V?", + "target_title": "Abd ar-Rahman V", + "target_root": "de82c6f4adfa9b8721e708c5056a0fd6a4c4755711b60e9c16ae1d02e3e224db", + "shard": "000.db" + }, + { + "question": "what is Anti Ballistic Missile Treaty?", + "target_title": "Anti-Ballistic Missile Treaty", + "target_root": "b68d02841d328befe2c4cba80d60d3d45130718925b284c2941d2521e37937b9", + "shard": "000.db" + }, + { + "question": "what is AGM 88 HARM?", + "target_title": "AGM-88 HARM", + "target_root": "cc0bf6f28be2f070cb8514ae32ef439a5f7eda8e00aa2903ebbb29754bbb8e35", + "shard": "000.db" + }, + { + "question": "what is Lockheed AC 130?", + "target_title": "Lockheed AC-130", + "target_root": "5c1118c0c162c30a7cc774f069c4b631c19a7572146cc6afce724efdbabf4ed3", + "shard": "000.db" + }, + { + "question": "what is CIM 10 Bomarc?", + "target_title": "CIM-10 Bomarc", + "target_root": "cde6d4f22963a409a2aab12f784c97915c48fb40fd5bff8f3c5ca569322e192d", + "shard": "000.db" + }, + { + "question": "what is North American B 25 Mitchell?", + "target_title": "North American B-25 Mitchell", + "target_root": "76551e89d1e598a858947a01e843506ae8c46c73734a05f91e75355303a06971", + "shard": "000.db" + }, + { + "question": "what is Bain marie?", + "target_title": "Bain-marie", + "target_root": "bbe25b304971c272e0414cbf48d3cc1fd864b808cc681ab95dac9280c1d000a3", + "shard": "000.db" + }, + { + "question": "what is Cross dressing?", + "target_title": "Cross-dressing", + "target_root": "f888c381efe454debf20d54f19dc8963ae259725570e46e8d786043ddd23617f", + "shard": "000.db" + }, + { + "question": "what is Call of Cthulhu (role playing game)?", + "target_title": "Call of Cthulhu (role-playing game)", + "target_root": "c1ac04f507ac1f7794ef465ae5091d39eddb030c0329a20e5bb3576c3bd6dc36", + "shard": "000.db" + }, + { + "question": "what is Context sensitive?", + "target_title": "Context-sensitive", + "target_root": "daf91b754de7c52fce3c8f23741a4595df935403ddff744ab3fd00f500d0da1a", + "shard": "000.db" + }, + { + "question": "what is Boeing C 17 Globemaster III?", + "target_title": "Boeing C-17 Globemaster III", + "target_root": "abcaff97ccfc0bdcccf3ac107ddd4e4b9931ad4a8696987c589e79f1b1444e29", + "shard": "000.db" + }, + { + "question": "what is Chiang Kai shek?", + "target_title": "Chiang Kai-shek", + "target_root": "28b23bbba487cb03e004896c67af97fec6ae2ef92dc2becea94f0bb2bd392c3c", + "shard": "000.db" + }, + { + "question": "what is Context free language?", + "target_title": "Context-free language", + "target_root": "ecbf405046a393e614f37cc3a0135238e554011bb314c457cb088a581e28f342", + "shard": "000.db" + }, + { + "question": "what is C* algebra?", + "target_title": "C*-algebra", + "target_root": "f286b4b5bc856209b3f6daea90947ac70f0cc3a5f8e719cb5d00bd0c8daa4006", + "shard": "000.db" + }, + { + "question": "what is Computer generated imagery?", + "target_title": "Computer-generated imagery", + "target_root": "107e71d0d1efc8c3ffecb5baff42e1a340aa8c2a65c1e14d410d7caffe679f24", + "shard": "000.db" + }, + { + "question": "what is Lockheed C 130 Hercules?", + "target_title": "Lockheed C-130 Hercules", + "target_root": "f55294ffdfdaedad52f197b098a8c5684e5534a60d31c40f94ba3102645ea311", + "shard": "000.db" + }, + { + "question": "what is Covenant breaker?", + "target_title": "Covenant-breaker", + "target_root": "1602c6273b5239feca1ed84bc62e1730b1d3004d03428235ff46de62d2ce19e1", + "shard": "000.db" + }, + { + "question": "what is Comprehensive Nuclear Test Ban Treaty?", + "target_title": "Comprehensive Nuclear-Test-Ban Treaty", + "target_root": "a9e1899f647960cb33f437b5a2be04cfc739aac3da9bbc6a3a724864390a66d4", + "shard": "000.db" + }, + { + "question": "what is Double slit experiment?", + "target_title": "Double-slit experiment", + "target_root": "8e9293f1bb3a10a8266ceecad25ef30227f029b03aa7681ba5ceb219780e374d", + "shard": "000.db" + }, + { + "question": "what is Double ended queue?", + "target_title": "Double-ended queue", + "target_root": "b1d5f50ef8b84ecf3a20600f9c5394cda71d2eb3f5b9de11ea54b02b33534889", + "shard": "000.db" + }, + { + "question": "what is Cost push inflation?", + "target_title": "Cost-push inflation", + "target_root": "890e9c85d9567ab08aa9aebb5249a805b135d2d9db9e41133d5802cac55c474e", + "shard": "000.db" + }, + { + "question": "what is Eductor jet pump?", + "target_title": "Eductor-jet pump", + "target_root": "135f3206dda2874696ba3417514338a05b2bc187de3076524446864d59b3242b", + "shard": "000.db" + }, + { + "question": "what is Eight ball?", + "target_title": "Eight-ball", + "target_root": "cf8dfa5eb3a35215e5555de9e695bd4b142bb8cc9efa1757655859c42f0dd22b", + "shard": "000.db" + }, + { + "question": "what is Élisabeth Louise Vigée Le Brun?", + "target_title": "Élisabeth-Louise Vigée-Le Brun", + "target_root": "73e60587c35c69d31312f9585ae1a43dcf0e2b9ba100ce573fcbd95fc98bb778", + "shard": "000.db" + }, + { + "question": "what is Evidence based medicine?", + "target_title": "Evidence-based medicine", + "target_root": "6db654c484d855d8c24dcc9d2ac163709ec054c8083bb333e4bdea6e7edd361f", + "shard": "000.db" + }, + { + "question": "what is E Prime?", + "target_title": "E-Prime", + "target_root": "20c3570fa3140fc3429f4a95695fca2afeae1cf686d15fadb4929bca5fd9eb91", + "shard": "000.db" + }, + { + "question": "what is Boeing E 3 Sentry?", + "target_title": "Boeing E-3 Sentry", + "target_root": "efc1f15bc037ec39561326cc240100b3f39528ca782b7a634408c80f2599bb77", + "shard": "000.db" + }, + { + "question": "what is Emperor Go En'yū?", + "target_title": "Emperor Go-En'yū", + "target_root": "1eda22a89b6650822efc097067dbc87e5d74a42b3f93dd1184c9f25f97e0e6a3", + "shard": "000.db" + }, + { + "question": "what is Field programmable gate array?", + "target_title": "Field-programmable gate array", + "target_root": "a000892caf082200a3fcc3a6f1b3126f2c5d4151fe189f99808fde02b632aa11", + "shard": "000.db" + }, + { + "question": "what is Five spice powder?", + "target_title": "Five-spice powder", + "target_root": "bf0de4cc5ed75d9c655f008e3a9114409e8cfabec49b84aa95d0832ff529c584", + "shard": "000.db" + }, + { + "question": "what is Four poster?", + "target_title": "Four-poster", + "target_root": "a1c0787760f6eb2b01d3f681b39b132227de485f18cc64914282829cdacfea8d", + "shard": "000.db" + }, + { + "question": "what is Flip flop (electronics)?", + "target_title": "Flip-flop (electronics)", + "target_root": "56d5cb2d7ff9fcf3206af52f7d7d86e0c30478b940988725c89e6a863d63dc98", + "shard": "000.db" + }, + { + "question": "what is Guinea Bissau?", + "target_title": "Guinea-Bissau", + "target_root": "1b76ef7f4e81065dd11292e4399fddc8facf1711fc93a962096e3082713d20a2", + "shard": "000.db" + }, + { + "question": "what is Politics of Guinea Bissau?", + "target_title": "Politics of Guinea-Bissau", + "target_root": "d8a22dbea22e127ce9bd141dbf2996b8b9efe94f632d0b9c2cf2f47e497f0106", + "shard": "000.db" + } +] diff --git a/bench/qa_questions_numeral.txt b/bench/qa_questions_numeral.txt index 8ed8f41..fece450 100644 --- a/bench/qa_questions_numeral.txt +++ b/bench/qa_questions_numeral.txt @@ -6,38 +6,3 @@ who was Ahmed the third? who was Alaric the first? who was Alexander the first of epirus? who was Alexander the second of scotland? -who was Alexander the second? -who was Alexander the fourth? -who was Alyattes the second? -who was Afonso the fourth of portugal? -who was Alfonso the second of asturias? -who was Alfonso the fourth of aragon? -who was Alfonso the third? -who was Alfonso the fifth? -who was Anastasius the second? -who was Abbas the second of egypt? -who was Charles the fifth? -who was Constantius the second? -who was Constantine the second of scotland? -who was Charles the first of england? -who was Frederick the fifth? -who was Henry the seventh? -who was Mehmed the first? -who was Mustafa the first? -who was Mieszko the first of poland? -who was Malcolm the first of scotland? -who was Osman the second? -who was Quake the second? -who was Stephen the third? -who was Oscar the first of sweden? -who was Charles the fifteenth of sweden? -who was Sviatoslav the first of kiev? -who was Catherine the second of russia? -who was Childeric the first? -who was Rudolph the first of germany? -who was Xerxes the second of persia? -who was Richard the second of england? -who was Gustav the first of sweden? -who was Photios the first of constantinople? -who was James the fifth of scotland? -who was Basarab the first of wallachia? diff --git a/bench/qa_questions_numeral_map.json b/bench/qa_questions_numeral_map.json index af86d9d..76e79f5 100644 --- a/bench/qa_questions_numeral_map.json +++ b/bench/qa_questions_numeral_map.json @@ -28,215 +28,5 @@ "target_title": "Alexander II of Scotland", "target_root": "4d5885f25ab6d7ebd0054dfdb4783f00db96f210ea33cdff2c0dbb0b92b45a13", "shard": "000.db" - }, - { - "question": "who was Alexander the second?", - "target_title": "Alexander II", - "target_root": "190bf209efb2f096f9b93c7dd2d0b0be9a64d6aa469d27cf2272664e4f624de2", - "shard": "000.db" - }, - { - "question": "who was Alexander the fourth?", - "target_title": "Alexander IV", - "target_root": "cc3134f31a0d69f03aa37a5821fc179082f34316e782d464b0685f3ed0e90f0a", - "shard": "000.db" - }, - { - "question": "who was Alyattes the second?", - "target_title": "Alyattes II", - "target_root": "464c0878c6fcabaf67f37389aa343b180b2762325a9aa916aa0d95dd7529e853", - "shard": "000.db" - }, - { - "question": "who was Afonso the fourth of portugal?", - "target_title": "Afonso IV of Portugal", - "target_root": "88c2a881546a4cfa59c8eba1e037d2092ab73c9f09feb9524801d64763dfbd3e", - "shard": "000.db" - }, - { - "question": "who was Alfonso the second of asturias?", - "target_title": "Alfonso II of Asturias", - "target_root": "1b2073183930900d65f1433efb70e8080628b0aa440817a3eb235d70ddf18f61", - "shard": "000.db" - }, - { - "question": "who was Alfonso the fourth of aragon?", - "target_title": "Alfonso IV of Aragon", - "target_root": "fb96fb9d074fefd80da864c412a9f07cab26b1a217b54a378442143c147316d7", - "shard": "000.db" - }, - { - "question": "who was Alfonso the third?", - "target_title": "Alfonso III", - "target_root": "555a58a9f7c947f32e18dcd6dfb03fb13ba7cc29f144c5b98f8af087892d9ac0", - "shard": "000.db" - }, - { - "question": "who was Alfonso the fifth?", - "target_title": "Alfonso V", - "target_root": "c7aaf0276378cefbbb1a44dc1f90b4cb5928411880c7869ea68628f09f1ebde2", - "shard": "000.db" - }, - { - "question": "who was Anastasius the second?", - "target_title": "Anastasius II", - "target_root": "2cb452df76d8d15494550c8070b51bca9f369647c95574e6a4f9cb441f865c0c", - "shard": "000.db" - }, - { - "question": "who was Abbas the second of egypt?", - "target_title": "Abbas II of Egypt", - "target_root": "8c3378f3375831933b0c0865074747c264599ceb2650f866c5d61d3635126035", - "shard": "000.db" - }, - { - "question": "who was Charles the fifth?", - "target_title": "Charles V", - "target_root": "22d8e5eb9bebef543cecda7b33a1f0d300cd24e3a13427c6a297524c9813718f", - "shard": "000.db" - }, - { - "question": "who was Constantius the second?", - "target_title": "Constantius II", - "target_root": "5f8bd920302cc83ee5aef59abfda0aafd31b6e1cb48cfc4f5c7d4c3e69bedd4a", - "shard": "000.db" - }, - { - "question": "who was Constantine the second of scotland?", - "target_title": "Constantine II of Scotland", - "target_root": "24e398f6b012f01c0e1b1cb7fad7b4ee2fda8898a15975e5d1c75a4ee77bc3df", - "shard": "000.db" - }, - { - "question": "who was Charles the first of england?", - "target_title": "Charles I of England", - "target_root": "ad6bc7ccd05e08e3aa9fd3db04dc85e52fc64e54c22532fe1af00ec956bef907", - "shard": "000.db" - }, - { - "question": "who was Frederick the fifth?", - "target_title": "Frederick V", - "target_root": "ac3d08404d5e30a1642647512b5e5170253fb46577a244390c909dd8679922c0", - "shard": "000.db" - }, - { - "question": "who was Henry the seventh?", - "target_title": "Henry VII", - "target_root": "91e7879a395c0a43c218964ccc28bcff93ec3e3e141483af3b50e2d596b185df", - "shard": "000.db" - }, - { - "question": "who was Mehmed the first?", - "target_title": "Mehmed I", - "target_root": "9a526b048fe00f930da0b4128eb30dde55ebfee10490cea2322768c8d1f18023", - "shard": "000.db" - }, - { - "question": "who was Mustafa the first?", - "target_title": "Mustafa I", - "target_root": "6d4d3e3d401f3b1651fc3d779d71b8084a5ff69ba2f95516dcefec63f8e6fd20", - "shard": "000.db" - }, - { - "question": "who was Mieszko the first of poland?", - "target_title": "Mieszko I of Poland", - "target_root": "fee8e212dc5d6439d1b791eea1bc997d825a4b4abf3bfeeef2cfae57e948d077", - "shard": "000.db" - }, - { - "question": "who was Malcolm the first of scotland?", - "target_title": "Malcolm I of Scotland", - "target_root": "1f1983a45ed9b9cfa49425103fed17a1ca0e36284189525a31a9bc73a14d2181", - "shard": "000.db" - }, - { - "question": "who was Osman the second?", - "target_title": "Osman II", - "target_root": "71da21f6a76233de3ac45683f8d8b419d9962581cba0f2fb0cf81093b7e41b0a", - "shard": "000.db" - }, - { - "question": "who was Quake the second?", - "target_title": "Quake II", - "target_root": "86b1189f25d0fe612423ef88be44885da48f115c27b64024078c2fd257acb66a", - "shard": "000.db" - }, - { - "question": "who was Stephen the third?", - "target_title": "Stephen III", - "target_root": "07d31c9abc8c1e8626997b248c82592b09dff67e96069c31c90b95230407e4c2", - "shard": "000.db" - }, - { - "question": "who was Oscar the first of sweden?", - "target_title": "Oscar I of Sweden", - "target_root": "09ab96881d295f59dd8131a2d9d15bc2d633bd9e9606c1c27d7e1c7c889cfec0", - "shard": "000.db" - }, - { - "question": "who was Charles the fifteenth of sweden?", - "target_title": "Charles XV of Sweden", - "target_root": "381a3cc80ba62a7454ea87bfd56ddefb27229491e3926a88d84968fd791485bb", - "shard": "000.db" - }, - { - "question": "who was Sviatoslav the first of kiev?", - "target_title": "Sviatoslav I of Kiev", - "target_root": "99066a1e171e3a3924c115f6010a306743520730d9a0e0ed4f359c0e7239a7dd", - "shard": "000.db" - }, - { - "question": "who was Catherine the second of russia?", - "target_title": "Catherine II of Russia", - "target_root": "d7a6a6d1f63b0ddd4977b2c9203fdd6659b0fdcadbb43096910c028ea492a42b", - "shard": "000.db" - }, - { - "question": "who was Childeric the first?", - "target_title": "Childeric I", - "target_root": "a21db476b6e4ece9aa1cc7da6b68b199474f82c297eaa3304fd3713d0b7ec682", - "shard": "000.db" - }, - { - "question": "who was Rudolph the first of germany?", - "target_title": "Rudolph I of Germany", - "target_root": "7f882ac84cf32819f3af2dda446d18ad7cd818e94b201c56a3a89a0d78bf9161", - "shard": "000.db" - }, - { - "question": "who was Xerxes the second of persia?", - "target_title": "Xerxes II of Persia", - "target_root": "edafe8586c5666f8fb38a499ac73f0d954fba6c2d3b869610de16ba33bcc8a3f", - "shard": "000.db" - }, - { - "question": "who was Richard the second of england?", - "target_title": "Richard II of England", - "target_root": "d08b7f5eb56fb3352a77facfbc204b53b1a6cef4215d15ac7a2a97f3a30dda8d", - "shard": "000.db" - }, - { - "question": "who was Gustav the first of sweden?", - "target_title": "Gustav I of Sweden", - "target_root": "4ff24ede9c8fc1bd401972bef9a3a8e4a4614b8ad289027e9e51b00f16631a04", - "shard": "000.db" - }, - { - "question": "who was Photios the first of constantinople?", - "target_title": "Photios I of Constantinople", - "target_root": "1309569c1cb8f37ff5f7d2cfc00080efb5f6846e54b1a45c07357e1b0a2445f2", - "shard": "000.db" - }, - { - "question": "who was James the fifth of scotland?", - "target_title": "James V of Scotland", - "target_root": "6b0e3af072a252daaad838123f5b2c2db3b0ac91bd7833faf17ff4f0a488fbe0", - "shard": "000.db" - }, - { - "question": "who was Basarab the first of wallachia?", - "target_title": "Basarab I of Wallachia", - "target_root": "6f543c2b310d31ef08d43e4454a69eb2294ffc412df4cb26bafd92ea9841ca2e", - "shard": "000.db" } ] diff --git a/bench/recall_at_k.py b/bench/recall_at_k.py index 0c17b02..e263657 100644 --- a/bench/recall_at_k.py +++ b/bench/recall_at_k.py @@ -28,8 +28,13 @@ def _norm(t: str) -> str: return (t or "").replace("_", " ").strip().casefold() -def probe(item: dict, k: int) -> tuple[bool, int]: - """Return (target_in_topk, rank_or_-1). Retrieval only.""" +def probe(item: dict, k: int) -> int: + """Return the target's 0-based rank in retrieved sources, or -1 if + absent (retrieval only, no LLM). One retrieval → recall at ANY + k<=k is derivable from the rank (a too-lenient k hides a rank- + only lift; the 2026-05-18 accent-fold case — recall@8 flat but + rank-1 22->34. Report @1/@3/@k so rank-sensitive folds aren't + mis-judged).""" try: out = subprocess.run( [str(ARB), "--shards-dir", str(SHARDS), "query", "--dry-run", @@ -39,10 +44,10 @@ def probe(item: dict, k: int) -> tuple[bool, int]: ).stdout d = json.loads(out) except Exception: - return (False, -1) + return -1 tgt = _norm(item["target_title"]) titles = [_norm(s.get("title") or "") for s in (d.get("sources") or [])] - return (tgt in titles, titles.index(tgt) if tgt in titles else -1) + return titles.index(tgt) if tgt in titles else -1 def main() -> int: @@ -52,26 +57,23 @@ def main() -> int: ap.add_argument("--conc", type=int, default=4) a = ap.parse_args() items = json.loads(Path(a.map_json).read_text()) - hits = 0 - ranks: list[int] = [] - misses: list[str] = [] with cf.ThreadPoolExecutor(max_workers=a.conc) as ex: - for it, (ok, rank) in zip( - items, ex.map(lambda i: probe(i, a.k), items) - ): - if ok: - hits += 1 - ranks.append(rank) - else: - misses.append(f"{it['question']!r} -> {it['target_title']!r}") + ranks = list(ex.map(lambda i: probe(i, a.k), items)) n = len(items) - print(f"recall@{a.k}: {hits}/{n} = {hits/n:.0%} " - f"(deterministic, no LLM — the instrument)") - if ranks: - print(f" of the hits, mean rank: {sum(ranks)/len(ranks):.1f} " - f"(0=top); rank-1 count: {sum(1 for r in ranks if r == 0)}") - print(f" MISSES ({len(misses)}) — target article never surfaced:") - for m in misses[:25]: + paired = list(zip(items, ranks)) + print(f"n={n} (deterministic retrieval recall, no LLM — the instrument)") + for kk in sorted({1, 3, a.k}): + hit = sum(1 for r in ranks if 0 <= r < kk) + print(f" recall@{kk}: {hit}/{n} = {hit/n:.0%}") + found = [r for r in ranks if r >= 0] + if found: + print(f" surfaced-anywhere: {len(found)}/{n}; mean rank " + f"{sum(found)/len(found):.2f} (0=top); rank-1 " + f"{sum(1 for r in found if r == 0)}/{n}") + misses = [f"{it['question']!r} -> {it['target_title']!r}" + for it, r in paired if r < 0] + print(f" MISSES ({len(misses)}) — never surfaced:") + for m in misses[:20]: print(f" {m}") return 0 diff --git a/tests/test_accent_fold.py b/tests/test_accent_fold.py new file mode 100644 index 0000000..c634ae3 --- /dev/null +++ b/tests/test_accent_fold.py @@ -0,0 +1,83 @@ +"""Accent-fold: ASCII-folded variants for diacritic text, retrieval-side. + +Measured 2026-05-18 on the mined ground-truth fixture: recall@1 +55% -> 85% (+30pp), rank-1 22/40 -> 34/40. recall@8 was a near- +miss-revert artifact (95->98, noise) — a too-lenient k hid a +rank-only lift; @1/@3 is the resolution that drives primary-source +selection. Same additive+symmetric discipline as +`_hyphen_fold_variants` (#000007) / `_numeral_fold_variants`. + +Tested through the REAL path (FakeSource -> ingest -> query() -> +real `_Hit`), not a hand-built object — the discipline the reverted +disambiguation v1 violated. +""" +from __future__ import annotations + +from collections.abc import Iterator + +from arborist.qa.query import _accent_fold_variants, _title_query_tokens + + +def test_variant_is_additive_symmetric_and_noop_on_ascii(): + assert _accent_fold_variants("Béla Bartók") == {"bela", "bartok"} + assert _accent_fold_variants("André-Marie Ampère") >= {"andre", "ampere"} + # pure ASCII -> folds to itself -> empty -> zero effect (the + # additive-safety invariant that keeps non-accent queries intact) + assert _accent_fold_variants("what is the capital of france?") == set() + assert _accent_fold_variants("who painted the mona lisa?") == set() + + +def test_title_query_tokens_bridges_ascii_query_to_accented_title(): + q = _title_query_tokens("what is Bela Bartok?") + t = _title_query_tokens("Béla Bartók") + # the accented title otherwise fragments ("Béla"->"B","la") and + # never overlaps the ASCII form; the fold restores the bridge. + assert {"bela", "bartok"} <= (q & t) + # plain ASCII query unaffected + assert _title_query_tokens("who painted the mona lisa?") == { + "painted", "mona", "lisa" + } + + +def test_query_retrieves_accented_title_for_ascii_question(tmp_path): + from arborist.qa.client import StubClient + from arborist.document import Document + from arborist.ingest import ingest_source + from arborist.qa.query import DEFAULT_QUERY_POLICY, query + from arborist.source import Source + from arborist.store import connect + + class FakeSource(Source): + source_type = "test" + + def __init__(self, ds): + self.ds = ds + + def iter_documents(self) -> Iterator[Document]: + yield from self.ds + + shard = tmp_path / "shard.db" + c = connect(shard) + try: + ingest_source(c, FakeSource([ + Document(uri="t://1", + content="Béla Bartók was a Hungarian composer. " + "Béla Bartók pioneered ethnomusicology. " * 12, + source_type="test", title="Béla Bartók"), + Document(uri="t://2", + content="Unrelated article about gardening. " * 12, + source_type="test", title="Gardening"), + ])) + finally: + c.close() + r = query( + question="what is Bela Bartok?", + qa_db=tmp_path / "qa.db", + chat_client=StubClient(answer="Béla Bartók was a composer. [E1]\n"), + model_id="stub", + single_db=shard, + policy=dict(DEFAULT_QUERY_POLICY, answer_mode="claim_lattice_pointer"), + ) + titles = {(s.get("title") or "") for s in (r.get("sources") or [])} + assert r["status"] != "error" + assert "Béla Bartók" in titles # accented title surfaced from ASCII q