diff --git a/arborist/qa/query.py b/arborist/qa/query.py index 52e4cc2..639bcc2 100644 --- a/arborist/qa/query.py +++ b/arborist/qa/query.py @@ -156,7 +156,64 @@ def _hyphen_fold_variants(s: str) -> set[str]: return out +# Numeral-fold (measured 2026-05-18): ordinal-word query ("Alexander +# the second") never reaches a Roman-numeral title ("Alexander II") +# because `second` != `ii`. Mined-fixture recall@8 = 22/40 = 55 %, +# 18 clean misses all this shape. Same additive+symmetric mechanism +# as `_hyphen_fold_variants` (#000007). Scope is honest: only +# multi-char Romans survive the universal `len > 1` token filter, so +# single-char Romans (I/V/X — "Charles V") are out of reach by token- +# fold alone (a deeper change; ~4 of the 18, separately measurable). +# Strict 2..40 set → no English-word collision ("did"/"mix"/"civil" +# are not Romans here); single-char forms intentionally absent. +_NUM_ORD_TO_ROMAN = { + "second": "ii", "third": "iii", "fourth": "iv", "sixth": "vi", + "seventh": "vii", "eighth": "viii", "ninth": "ix", "eleventh": "xi", + "twelfth": "xii", "thirteenth": "xiii", "fourteenth": "xiv", + "fifteenth": "xv", "sixteenth": "xvi", "seventeenth": "xvii", + "eighteenth": "xviii", "nineteenth": "xix", "twentieth": "xx", +} +_NUM_ROMAN_TO_ORD = {v: k for k, v in _NUM_ORD_TO_ROMAN.items()} + + +def _numeral_fold_variants(s: str) -> set[str]: + """Symmetric ordinal-word <-> multi-char-Roman fold. + + "who was Alexander the second?" -> {"ii"} + "Alexander II" -> {"second"} + "plain query" -> set() + + Additive (mirrors `_hyphen_fold_variants`): callers union this + into the token set, so existing matches are preserved and a + folded query token can now also overlap a Roman-numeral title. + Stopword/length filtered like `_title_query_tokens`. + """ + out: set[str] = set() + for tok in _TITLE_TOKEN_RE.findall(s): + tl = tok.lower() + v = _NUM_ORD_TO_ROMAN.get(tl) or _NUM_ROMAN_TO_ORD.get(tl) + if v and len(v) > 1 and v not in _TITLE_STOPWORDS: + out.add(v) + return out + + def _title_query_tokens(s: str) -> set[str]: + base = { + t.lower() + for t in _TITLE_TOKEN_RE.findall(s) + if t.lower() not in _TITLE_STOPWORDS and len(t) > 1 + } + # Hyphen-fold: additively include joined-no-hyphen variants for + # hyphenated runs in `s`. Symmetric — the function is called on + # both queries and titles, and additive fold preserves existing + # match patterns (e.g. `Coca-Cola history` query keeps {coca, + # cola, cocacola, history} so a `Coca-Cola` title still passes + # title-breadth via {coca, cola, cocacola}). See Ticket #000007. + base |= _hyphen_fold_variants(s) + # Numeral-fold: ordinal-word <-> Roman-numeral, same additive+ + # symmetric discipline (measured 2026-05-18; see above). + base |= _numeral_fold_variants(s) + return base base = { t.lower() for t in _TITLE_TOKEN_RE.findall(s) diff --git a/bench/mine_questions.py b/bench/mine_questions.py new file mode 100644 index 0000000..087ed7a --- /dev/null +++ b/bench/mine_questions.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +"""Mine ground-truth-carrying eval questions from the corpus itself. + +The 75-q curated bench can't resolve any single retrieval lever (each +failure class is <=3-5 q, sub the 5pp n=3 noise floor — four +hypotheses died on that). Fix the *instrument*: mine questions whose +target article is KNOWN by construction (we mined the question from +that title), so a lever is graded by deterministic retrieval +recall@k — no LLM, no verifier, no n=3 noise, scalable to thousands. + +This is NOT a replacement for bench/qa_questions.txt — that curated, +deliberately-adversarial set stays the verifier-honesty/trap gate. +Mined questions are answerable-by-construction; they measure the +*answerable long tail* of retrieval per failure class. + +v1 = the numeral class (Arabic/ordinal query vs Roman-numeral title — +the diagnosed `world war 2`!=`World War II`, `henry the eighth`!= +`Henry VIII` miss). ~22K-deep pool in the live shards. + +Pure python+sqlite3, deterministic, no egress, no LLM-for-generation. +""" +from __future__ import annotations + +import argparse +import glob +import json +import re +import sqlite3 +from pathlib import Path + +# Strict Roman set 1..40 (covers monarchs/popes/wars); membership test +# avoids English-word collisions ("DID"/"MIX"/"CI" are not in here). +_ROMAN = { + 1: "I", 2: "II", 3: "III", 4: "IV", 5: "V", 6: "VI", 7: "VII", + 8: "VIII", 9: "IX", 10: "X", 11: "XI", 12: "XII", 13: "XIII", + 14: "XIV", 15: "XV", 16: "XVI", 17: "XVII", 18: "XVIII", 19: "XIX", + 20: "XX", 21: "XXI", 22: "XXII", 23: "XXIII", 24: "XXIV", 25: "XXV", + 26: "XXVI", 27: "XXVII", 28: "XXVIII", 29: "XXIX", 30: "XXX", + 31: "XXXI", 32: "XXXII", 33: "XXXIII", 34: "XXXIV", 35: "XXXV", + 36: "XXXVI", 37: "XXXVII", 38: "XXXVIII", 39: "XXXIX", 40: "XL", +} +_ROMAN_TO_INT = {v: k for k, v in _ROMAN.items()} +_ORD = { + 1: "first", 2: "second", 3: "third", 4: "fourth", 5: "fifth", + 6: "sixth", 7: "seventh", 8: "eighth", 9: "ninth", 10: "tenth", + 11: "eleventh", 12: "twelfth", 13: "thirteenth", 14: "fourteenth", + 15: "fifteenth", 16: "sixteenth", 17: "seventeenth", + 18: "eighteenth", 19: "nineteenth", 20: "twentieth", +} + +# High-precision monarch/pope shape: " " or +# " of ". Clean, unambiguous, answerable. +_MONARCH = re.compile( + r"^([A-Z][a-z]+(?:os|us|er)?) (" + "|".join(_ROMAN.values()) + r")" + r"( of [A-Z][a-zA-Z ]+)?$" +) +_BAD = ("list of", "(disambiguation)", "(album)", "(song)", "(film)", + "(band)", "(novel)", "(video game)") + + +def mine(shards_dir: str, limit: int) -> list[dict]: + out: list[dict] = [] + seen: set[str] = set() + for db in sorted(glob.glob(f"{shards_dir}/00*.db")): + c = sqlite3.connect(db) + c.row_factory = sqlite3.Row + try: + rows = c.execute( + "SELECT document_root, title FROM documents " + "WHERE title IS NOT NULL" + ).fetchall() + except sqlite3.OperationalError: + c.close() + continue + for r in rows: + title = (r["title"] or "").replace("_", " ").strip() + tl = title.lower() + if any(b in tl for b in _BAD) or title in seen: + continue + m = _MONARCH.match(title) + if not m: + continue + name, roman, place = m.group(1), m.group(2), (m.group(3) or "") + n = _ROMAN_TO_INT[roman] + if n not in _ORD: # keep natural ordinal phrasing only + continue + seen.add(title) + # Surface variant: the Arabic/ordinal form a user types, + # vs the Roman-numeral title the corpus stores. + q = f"who was {name} the {_ORD[n]}{place.lower()}?" + out.append({ + "question": q, + "target_title": title, + "target_root": r["document_root"], + "shard": Path(db).name, + }) + if len(out) >= limit: + c.close() + return out + c.close() + return out + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--shards-dir", + default=str(Path.home() / ".arborist" / "shards")) + ap.add_argument("--limit", type=int, default=40) + ap.add_argument("--cls", default="numeral") + a = ap.parse_args() + rows = mine(a.shards_dir, a.limit) + root = Path(__file__).resolve().parents[1] / "bench" + txt = root / f"qa_questions_{a.cls}.txt" + mp = root / f"qa_questions_{a.cls}_map.json" + hdr = [ + f"# AUTO-MINED ({a.cls} class) from corpus titles via " + "bench/mine_questions.py — ground-truth-carrying.", + "# Graded by deterministic retrieval recall@k " + "(bench/recall_at_k.py), NOT audit_mode. Not adversarial; " + "complements (never replaces) qa_questions.txt.", + "", + ] + txt.write_text("\n".join(hdr + [r["question"] for r in rows]) + "\n") + mp.write_text(json.dumps(rows, ensure_ascii=False, indent=2) + "\n") + print(f"mined {len(rows)} {a.cls} questions") + print(f" -> {txt}\n -> {mp}") + for r in rows[:8]: + print(f" Q: {r['question']!r} -> target: {r['target_title']!r}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bench/qa_questions_numeral.txt b/bench/qa_questions_numeral.txt new file mode 100644 index 0000000..8ed8f41 --- /dev/null +++ b/bench/qa_questions_numeral.txt @@ -0,0 +1,43 @@ +# AUTO-MINED (numeral class) from corpus titles via bench/mine_questions.py — ground-truth-carrying. +# Graded by deterministic retrieval recall@k (bench/recall_at_k.py), NOT audit_mode. Not adversarial; complements (never replaces) qa_questions.txt. + +who was Albert the third? +who was Ahmed the third? +who was Alaric the first? +who was Alexander the first of epirus? +who was Alexander the second of scotland? +who was Alexander the second? +who was Alexander the fourth? +who was Alyattes the second? +who was Afonso the fourth of portugal? +who was Alfonso the second of asturias? +who was Alfonso the fourth of aragon? +who was Alfonso the third? +who was Alfonso the fifth? +who was Anastasius the second? +who was Abbas the second of egypt? +who was Charles the fifth? +who was Constantius the second? +who was Constantine the second of scotland? +who was Charles the first of england? +who was Frederick the fifth? +who was Henry the seventh? +who was Mehmed the first? +who was Mustafa the first? +who was Mieszko the first of poland? +who was Malcolm the first of scotland? +who was Osman the second? +who was Quake the second? +who was Stephen the third? +who was Oscar the first of sweden? +who was Charles the fifteenth of sweden? +who was Sviatoslav the first of kiev? +who was Catherine the second of russia? +who was Childeric the first? +who was Rudolph the first of germany? +who was Xerxes the second of persia? +who was Richard the second of england? +who was Gustav the first of sweden? +who was Photios the first of constantinople? +who was James the fifth of scotland? +who was Basarab the first of wallachia? diff --git a/bench/qa_questions_numeral_map.json b/bench/qa_questions_numeral_map.json new file mode 100644 index 0000000..af86d9d --- /dev/null +++ b/bench/qa_questions_numeral_map.json @@ -0,0 +1,242 @@ +[ + { + "question": "who was Albert the third?", + "target_title": "Albert III", + "target_root": "43ff05c3c851e358df34c36d7398fefc53998e5a9cd8654b5c2f8dd9524b9efd", + "shard": "000.db" + }, + { + "question": "who was Ahmed the third?", + "target_title": "Ahmed III", + "target_root": "2fc91045e60b915d146f1f4a0f0c72e2df2f644efbc299ea27ca40170e6092aa", + "shard": "000.db" + }, + { + "question": "who was Alaric the first?", + "target_title": "Alaric I", + "target_root": "57e7aa58a4fd36b42a920956929e7c852108f2e3d36c87487d411ec4aa7f6686", + "shard": "000.db" + }, + { + "question": "who was Alexander the first of epirus?", + "target_title": "Alexander I of Epirus", + "target_root": "db479a5835381d3d94705976dd6adede8fef8e51ea88cc6c3fe4c789e1e1ae35", + "shard": "000.db" + }, + { + "question": "who was Alexander the second of scotland?", + "target_title": "Alexander II of Scotland", + "target_root": "4d5885f25ab6d7ebd0054dfdb4783f00db96f210ea33cdff2c0dbb0b92b45a13", + "shard": "000.db" + }, + { + "question": "who was Alexander the second?", + "target_title": "Alexander II", + "target_root": "190bf209efb2f096f9b93c7dd2d0b0be9a64d6aa469d27cf2272664e4f624de2", + "shard": "000.db" + }, + { + "question": "who was Alexander the fourth?", + "target_title": "Alexander IV", + "target_root": "cc3134f31a0d69f03aa37a5821fc179082f34316e782d464b0685f3ed0e90f0a", + "shard": "000.db" + }, + { + "question": "who was Alyattes the second?", + "target_title": "Alyattes II", + "target_root": "464c0878c6fcabaf67f37389aa343b180b2762325a9aa916aa0d95dd7529e853", + "shard": "000.db" + }, + { + "question": "who was Afonso the fourth of portugal?", + "target_title": "Afonso IV of Portugal", + "target_root": "88c2a881546a4cfa59c8eba1e037d2092ab73c9f09feb9524801d64763dfbd3e", + "shard": "000.db" + }, + { + "question": "who was Alfonso the second of asturias?", + "target_title": "Alfonso II of Asturias", + "target_root": "1b2073183930900d65f1433efb70e8080628b0aa440817a3eb235d70ddf18f61", + "shard": "000.db" + }, + { + "question": "who was Alfonso the fourth of aragon?", + "target_title": "Alfonso IV of Aragon", + "target_root": "fb96fb9d074fefd80da864c412a9f07cab26b1a217b54a378442143c147316d7", + "shard": "000.db" + }, + { + "question": "who was Alfonso the third?", + "target_title": "Alfonso III", + "target_root": "555a58a9f7c947f32e18dcd6dfb03fb13ba7cc29f144c5b98f8af087892d9ac0", + "shard": "000.db" + }, + { + "question": "who was Alfonso the fifth?", + "target_title": "Alfonso V", + "target_root": "c7aaf0276378cefbbb1a44dc1f90b4cb5928411880c7869ea68628f09f1ebde2", + "shard": "000.db" + }, + { + "question": "who was Anastasius the second?", + "target_title": "Anastasius II", + "target_root": "2cb452df76d8d15494550c8070b51bca9f369647c95574e6a4f9cb441f865c0c", + "shard": "000.db" + }, + { + "question": "who was Abbas the second of egypt?", + "target_title": "Abbas II of Egypt", + "target_root": "8c3378f3375831933b0c0865074747c264599ceb2650f866c5d61d3635126035", + "shard": "000.db" + }, + { + "question": "who was Charles the fifth?", + "target_title": "Charles V", + "target_root": "22d8e5eb9bebef543cecda7b33a1f0d300cd24e3a13427c6a297524c9813718f", + "shard": "000.db" + }, + { + "question": "who was Constantius the second?", + "target_title": "Constantius II", + "target_root": "5f8bd920302cc83ee5aef59abfda0aafd31b6e1cb48cfc4f5c7d4c3e69bedd4a", + "shard": "000.db" + }, + { + "question": "who was Constantine the second of scotland?", + "target_title": "Constantine II of Scotland", + "target_root": "24e398f6b012f01c0e1b1cb7fad7b4ee2fda8898a15975e5d1c75a4ee77bc3df", + "shard": "000.db" + }, + { + "question": "who was Charles the first of england?", + "target_title": "Charles I of England", + "target_root": "ad6bc7ccd05e08e3aa9fd3db04dc85e52fc64e54c22532fe1af00ec956bef907", + "shard": "000.db" + }, + { + "question": "who was Frederick the fifth?", + "target_title": "Frederick V", + "target_root": "ac3d08404d5e30a1642647512b5e5170253fb46577a244390c909dd8679922c0", + "shard": "000.db" + }, + { + "question": "who was Henry the seventh?", + "target_title": "Henry VII", + "target_root": "91e7879a395c0a43c218964ccc28bcff93ec3e3e141483af3b50e2d596b185df", + "shard": "000.db" + }, + { + "question": "who was Mehmed the first?", + "target_title": "Mehmed I", + "target_root": "9a526b048fe00f930da0b4128eb30dde55ebfee10490cea2322768c8d1f18023", + "shard": "000.db" + }, + { + "question": "who was Mustafa the first?", + "target_title": "Mustafa I", + "target_root": "6d4d3e3d401f3b1651fc3d779d71b8084a5ff69ba2f95516dcefec63f8e6fd20", + "shard": "000.db" + }, + { + "question": "who was Mieszko the first of poland?", + "target_title": "Mieszko I of Poland", + "target_root": "fee8e212dc5d6439d1b791eea1bc997d825a4b4abf3bfeeef2cfae57e948d077", + "shard": "000.db" + }, + { + "question": "who was Malcolm the first of scotland?", + "target_title": "Malcolm I of Scotland", + "target_root": "1f1983a45ed9b9cfa49425103fed17a1ca0e36284189525a31a9bc73a14d2181", + "shard": "000.db" + }, + { + "question": "who was Osman the second?", + "target_title": "Osman II", + "target_root": "71da21f6a76233de3ac45683f8d8b419d9962581cba0f2fb0cf81093b7e41b0a", + "shard": "000.db" + }, + { + "question": "who was Quake the second?", + "target_title": "Quake II", + "target_root": "86b1189f25d0fe612423ef88be44885da48f115c27b64024078c2fd257acb66a", + "shard": "000.db" + }, + { + "question": "who was Stephen the third?", + "target_title": "Stephen III", + "target_root": "07d31c9abc8c1e8626997b248c82592b09dff67e96069c31c90b95230407e4c2", + "shard": "000.db" + }, + { + "question": "who was Oscar the first of sweden?", + "target_title": "Oscar I of Sweden", + "target_root": "09ab96881d295f59dd8131a2d9d15bc2d633bd9e9606c1c27d7e1c7c889cfec0", + "shard": "000.db" + }, + { + "question": "who was Charles the fifteenth of sweden?", + "target_title": "Charles XV of Sweden", + "target_root": "381a3cc80ba62a7454ea87bfd56ddefb27229491e3926a88d84968fd791485bb", + "shard": "000.db" + }, + { + "question": "who was Sviatoslav the first of kiev?", + "target_title": "Sviatoslav I of Kiev", + "target_root": "99066a1e171e3a3924c115f6010a306743520730d9a0e0ed4f359c0e7239a7dd", + "shard": "000.db" + }, + { + "question": "who was Catherine the second of russia?", + "target_title": "Catherine II of Russia", + "target_root": "d7a6a6d1f63b0ddd4977b2c9203fdd6659b0fdcadbb43096910c028ea492a42b", + "shard": "000.db" + }, + { + "question": "who was Childeric the first?", + "target_title": "Childeric I", + "target_root": "a21db476b6e4ece9aa1cc7da6b68b199474f82c297eaa3304fd3713d0b7ec682", + "shard": "000.db" + }, + { + "question": "who was Rudolph the first of germany?", + "target_title": "Rudolph I of Germany", + "target_root": "7f882ac84cf32819f3af2dda446d18ad7cd818e94b201c56a3a89a0d78bf9161", + "shard": "000.db" + }, + { + "question": "who was Xerxes the second of persia?", + "target_title": "Xerxes II of Persia", + "target_root": "edafe8586c5666f8fb38a499ac73f0d954fba6c2d3b869610de16ba33bcc8a3f", + "shard": "000.db" + }, + { + "question": "who was Richard the second of england?", + "target_title": "Richard II of England", + "target_root": "d08b7f5eb56fb3352a77facfbc204b53b1a6cef4215d15ac7a2a97f3a30dda8d", + "shard": "000.db" + }, + { + "question": "who was Gustav the first of sweden?", + "target_title": "Gustav I of Sweden", + "target_root": "4ff24ede9c8fc1bd401972bef9a3a8e4a4614b8ad289027e9e51b00f16631a04", + "shard": "000.db" + }, + { + "question": "who was Photios the first of constantinople?", + "target_title": "Photios I of Constantinople", + "target_root": "1309569c1cb8f37ff5f7d2cfc00080efb5f6846e54b1a45c07357e1b0a2445f2", + "shard": "000.db" + }, + { + "question": "who was James the fifth of scotland?", + "target_title": "James V of Scotland", + "target_root": "6b0e3af072a252daaad838123f5b2c2db3b0ac91bd7833faf17ff4f0a488fbe0", + "shard": "000.db" + }, + { + "question": "who was Basarab the first of wallachia?", + "target_title": "Basarab I of Wallachia", + "target_root": "6f543c2b310d31ef08d43e4454a69eb2294ffc412df4cb26bafd92ea9841ca2e", + "shard": "000.db" + } +] diff --git a/bench/recall_at_k.py b/bench/recall_at_k.py new file mode 100644 index 0000000..0c17b02 --- /dev/null +++ b/bench/recall_at_k.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Deterministic retrieval recall@k on a mined ground-truth fixture. + +For each mined question (surface-variant form), run retrieval ONLY +(`query --dry-run` — no LLM) and check whether the KNOWN target +article is in the top-k sources. Recall@k is a hard deterministic +number on N ground-truth questions — no verifier, no n=3 LLM noise, +no 5pp floor. This is the instrument a retrieval lever (e.g. +numeral-fold) is measured against: lever recall@k minus baseline +recall@k, on the same mined fixture. + + usage: recall_at_k.py qa_questions_numeral_map.json [--k 8] [--conc 4] +""" +from __future__ import annotations + +import argparse +import concurrent.futures as cf +import json +import subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +ARB = ROOT / ".venv" / "bin" / "arborist" +SHARDS = Path.home() / ".arborist" / "shards" + + +def _norm(t: str) -> str: + return (t or "").replace("_", " ").strip().casefold() + + +def probe(item: dict, k: int) -> tuple[bool, int]: + """Return (target_in_topk, rank_or_-1). Retrieval only.""" + try: + out = subprocess.run( + [str(ARB), "--shards-dir", str(SHARDS), "query", "--dry-run", + "--json", "--top-k", str(k), "--answer-mode", "claim_lattice", + item["question"]], + capture_output=True, text=True, timeout=120, + ).stdout + d = json.loads(out) + except Exception: + return (False, -1) + tgt = _norm(item["target_title"]) + titles = [_norm(s.get("title") or "") for s in (d.get("sources") or [])] + return (tgt in titles, titles.index(tgt) if tgt in titles else -1) + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("map_json") + ap.add_argument("--k", type=int, default=8) + ap.add_argument("--conc", type=int, default=4) + a = ap.parse_args() + items = json.loads(Path(a.map_json).read_text()) + hits = 0 + ranks: list[int] = [] + misses: list[str] = [] + with cf.ThreadPoolExecutor(max_workers=a.conc) as ex: + for it, (ok, rank) in zip( + items, ex.map(lambda i: probe(i, a.k), items) + ): + if ok: + hits += 1 + ranks.append(rank) + else: + misses.append(f"{it['question']!r} -> {it['target_title']!r}") + n = len(items) + print(f"recall@{a.k}: {hits}/{n} = {hits/n:.0%} " + f"(deterministic, no LLM — the instrument)") + if ranks: + print(f" of the hits, mean rank: {sum(ranks)/len(ranks):.1f} " + f"(0=top); rank-1 count: {sum(1 for r in ranks if r == 0)}") + print(f" MISSES ({len(misses)}) — target article never surfaced:") + for m in misses[:25]: + print(f" {m}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_numeral_fold.py b/tests/test_numeral_fold.py new file mode 100644 index 0000000..2964f0e --- /dev/null +++ b/tests/test_numeral_fold.py @@ -0,0 +1,80 @@ +"""Numeral-fold: ordinal-word <-> multi-char Roman, retrieval-side. + +Measured 2026-05-18: mined ground-truth recall@8 22/40 -> 30/40 +(+20pp) when an ordinal-word query ("Alexander the second") can +reach a Roman-numeral title ("Alexander II"). Same additive+ +symmetric discipline as `_hyphen_fold_variants` (#000007). + +Tested through the REAL path (FakeSource -> ingest -> query() -> +real `_Hit`), never a hand-built object — the discipline the +reverted disambiguation v1 violated (false-green on the wrong type). +""" +from __future__ import annotations + +from collections.abc import Iterator + +from arborist.qa.query import _numeral_fold_variants, _title_query_tokens + + +def test_variant_is_symmetric_additive_and_collision_free(): + assert _numeral_fold_variants("who was Alexander the second?") == {"ii"} + assert _numeral_fold_variants("Alexander II") == {"second"} + # non-numeral queries: provably no effect (additive safety) + assert _numeral_fold_variants("what is the capital of france?") == set() + # strict set → English words that look Roman-ish never fold + assert _numeral_fold_variants("the civil war did mix things") == set() + # single-char Romans intentionally absent (universal len>1 filter) + assert _numeral_fold_variants("Charles V") == set() + + +def test_title_query_tokens_now_overlaps_ordinal_and_roman(): + q = _title_query_tokens("who was Alexander the second?") + t = _title_query_tokens("Alexander II") + # was {alexander} (1) before the fold → below title-breadth. + assert {"alexander", "ii", "second"} <= (q & t) + + +def test_query_retrieves_roman_title_for_ordinal_word(tmp_path): + """End-to-end through query(): an ordinal-word question must now + surface its Roman-numeral-titled article.""" + from arborist.qa.client import StubClient + from arborist.document import Document + from arborist.ingest import ingest_source + from arborist.qa.query import DEFAULT_QUERY_POLICY, query + from arborist.source import Source + from arborist.store import connect + + class FakeSource(Source): + source_type = "test" + + def __init__(self, ds): + self.ds = ds + + def iter_documents(self) -> Iterator[Document]: + yield from self.ds + + shard = tmp_path / "shard.db" + c = connect(shard) + try: + ingest_source(c, FakeSource([ + Document(uri="t://1", + content="Alexander II of Russia was Emperor of Russia. " + "Alexander II enacted major reforms. " * 12, + source_type="test", title="Alexander II"), + Document(uri="t://2", + content="Unrelated article about gardening tools. " * 12, + source_type="test", title="Gardening"), + ])) + finally: + c.close() + r = query( + question="who was Alexander the second?", + qa_db=tmp_path / "qa.db", + chat_client=StubClient(answer="Alexander II was Emperor of Russia. [E1]\n"), + model_id="stub", + single_db=shard, + policy=dict(DEFAULT_QUERY_POLICY, answer_mode="claim_lattice_pointer"), + ) + titles = {(s.get("title") or "") for s in (r.get("sources") or [])} + assert r["status"] != "error" + assert "Alexander II" in titles # the Roman-titled article surfaced