diff --git a/bench/control_ab.py b/bench/control_ab.py index d4bc64f..2c8a16d 100644 --- a/bench/control_ab.py +++ b/bench/control_ab.py @@ -133,13 +133,20 @@ def main() -> int: continue gsha = hashlib.sha256(gold.encode()).hexdigest()[:12] - # arm A — Hermes-solo (control) + # arm A — Hermes-solo (control). `model` is keyword-only + # and REQUIRED on chat_completion; omitting it raised + # TypeError every call → the solo arm silently produced + # `[solo-error: TypeError]`, which the judge correctly read + # as ABSTAINED. That made every prior "solo abstained" + # smoke a broken-arm artefact, not a result. Pass the same + # model the Arborist arm uses (symmetry). try: solo = client.chat_completion( [{"role": "system", "content": _SOLO_SYS}, - {"role": "user", "content": q}]) + {"role": "user", "content": q}], + model=a.model) except Exception as e: # noqa: BLE001 - solo = f"[solo-error: {type(e).__name__}]" + solo = f"[solo-error: {type(e).__name__}: {e}]" # arm B — Arborist (treatment) try: r = query(question=q, qa_db=qa_db, chat_client=client, diff --git a/bench/mine_questions.py b/bench/mine_questions.py index bad5165..11443b1 100644 --- a/bench/mine_questions.py +++ b/bench/mine_questions.py @@ -17,6 +17,17 @@ v1 = the numeral class (Arabic/ordinal query vs Roman-numeral title — the diagnosed `world war 2`!=`World War II`, `henry the eighth`!= `Henry VIII` miss). ~22K-deep pool in the live shards. +The `stale` class (#000057) is the same machinery turned to the +opposite purpose: not a retrieval probe but a *fabrication-bait* set. +It mines time-varying executive-office titles and asks present-tense +incumbent questions; the ingested corpus (~2010–2011 Wikipedia — +verified from the artifact; CLAUDE.md's "2003 dump" does NOT match +the live shards, flagged to fox) is a ~13-year temporal adversary +against a ~2023 model, so the control (Hermes-solo) is confidently +WRONG against the fixed corpus-vintage gold instead of honestly +abstaining (the degeneracy the obscure-monarch numeral fixture hit). +Graded by the external Opus judge, NOT recall@k. + Pure python+sqlite3, deterministic, no egress, no LLM-for-generation. """ from __future__ import annotations @@ -68,6 +79,43 @@ _BAD = ("list of", "(disambiguation)", "(album)", "(song)", "(film)", _ACCENT_LETTER = re.compile(r"[À-ÖØ-öø-ÿĀ-ž]") +# #000057 fabrication-bait class. The numeral/accent/… classes are +# *retrieval* probes (answerable-by-construction → recall@k). `stale` +# is the opposite instrument: it exists to make the SMALL model +# confidently WRONG, not to test retrieval. +# +# CORPUS VINTAGE — VERIFIED FROM THE ARTIFACT, NOT ASSUMED. CLAUDE.md +# and the #000057 ticket say "2003 Wikipedia"; the live shards are +# NOT that dump. Two independent probes agree the ingested corpus is +# ~2010–2011 Wikipedia: (a) office infoboxes name 2007–2010 +# incumbents (President of France = Sarkozy / 2007; PM Australia = +# Gillard / Jun-2010; President of Cuba = Raúl Castro / Feb-2008); +# (b) chunk year-mention histogram on shard 000 falls off a cliff +# after 2010 (2010: 756 → 2011: 1), and an article titled "2010" +# exists while "2011"/"2012" do not. Discrepancy flagged to fox; the +# experiment must describe its gold by the *measured* vintage. +# +# The fabrication-bait argument survives the correction: a ~2010 +# corpus is still a ~13-year temporal adversary against Hermes-3-8B's +# ~2023 parametric memory. An executive-office title ("President of +# France") names its ~2010 incumbent in the lead — fixed gold, +# attached at mine time exactly like every other class. The present- +# tense question ("who is the president of France?") carries NO date, +# so the model answers from stale memory (Sarkozy→Macron) and does +# NOT honestly abstain — the degeneracy the obscure-monarch numeral +# fixture hit (Hermes ABSTAINED 5/5; you cannot measure a +# fabrication-vs-abstention delta when the control never fabricates). +# Deterministic gold + confident-wrong control is exactly the +# fabrication-bait the #000057 control experiment needs. Caveat the +# smoke run must check: offices whose holder did NOT turn over +# 2010→2023 dilute the signal (solo coincidentally CORRECT_GROUNDED) +# — prefer high-turnover offices; read the artifact, refine. +_OFFICE = re.compile( + r"^(President|Prime Minister|Chancellor|Premier|Taoiseach|" + r"Governor-General|First Minister|Chief Minister) " + r"of (the )?([A-Z][\w'’.\- ]+)$" +) + def _surface_variant(title: str, cls: str): """Return (question, ok) — the surface form a user TYPES vs the @@ -142,6 +190,27 @@ def _surface_variant(title: str, cls: str): if v == title: return None return f"what is {v}?" + if cls == "stale": + # Time-varying executive office → present-tense incumbent + # question. Gold = the 2003 article (whole-article gold via + # control_ab._gold, same as every class). No surface + # transform: the title IS the role, the staleness is temporal, + # not orthographic. + m = _OFFICE.match(title) + if not m: + return None + office, the, place = m.group(1), (m.group(2) or ""), m.group(3) + # Drop degenerate / non-national scopes that 2003 may not + # pin to one person ("President of the United States" is the + # ideal shape; "President of the Senate" is a role-of-a-body). + bad_scope = ("Senate", "Assembly", "Council", "Court", + "Commission", "Committee", "Board", "Union", + "Republic of the Congo") # ambiguous dup-title + if any(b in place for b in bad_scope): + return None + if len(place.split()) > 5: + return None + return f"who is the {office.lower()} of {the}{place}?" raise SystemExit(f"unknown class {cls!r}") @@ -192,14 +261,28 @@ def main() -> int: root = Path(__file__).resolve().parents[1] / "bench" txt = root / f"qa_questions_{a.cls}.txt" mp = root / f"qa_questions_{a.cls}_map.json" - hdr = [ - f"# AUTO-MINED ({a.cls} class) from corpus titles via " - "bench/mine_questions.py — ground-truth-carrying.", - "# Graded by deterministic retrieval recall@k " - "(bench/recall_at_k.py), NOT audit_mode. Not adversarial; " - "complements (never replaces) qa_questions.txt.", - "", - ] + if a.cls == "stale": + hdr = [ + f"# AUTO-MINED ({a.cls} class) from corpus titles via " + "bench/mine_questions.py — ground-truth-carrying.", + "# #000057 FABRICATION-BAIT: present-tense questions on " + "time-varying offices; the ingested ~2010-2011 corpus " + "(verified from the artifact, NOT the 2003 dump CLAUDE.md " + "names) is the temporal adversary against a ~2023 model. " + "Graded by the external Opus judge vs the fixed corpus-" + "vintage article gold (bench/control_ab.py + bench/" + "judge.py), NOT recall@k. Adversarial by construction.", + "", + ] + else: + hdr = [ + f"# AUTO-MINED ({a.cls} class) from corpus titles via " + "bench/mine_questions.py — ground-truth-carrying.", + "# Graded by deterministic retrieval recall@k " + "(bench/recall_at_k.py), NOT audit_mode. Not adversarial; " + "complements (never replaces) qa_questions.txt.", + "", + ] txt.write_text("\n".join(hdr + [r["question"] for r in rows]) + "\n") mp.write_text(json.dumps(rows, ensure_ascii=False, indent=2) + "\n") print(f"mined {len(rows)} {a.cls} questions") diff --git a/bench/qa_questions_stale.txt b/bench/qa_questions_stale.txt new file mode 100644 index 0000000..19cae51 --- /dev/null +++ b/bench/qa_questions_stale.txt @@ -0,0 +1,43 @@ +# AUTO-MINED (stale class) from corpus titles via bench/mine_questions.py — ground-truth-carrying. +# #000057 FABRICATION-BAIT: present-tense questions on time-varying offices; the ingested ~2010-2011 corpus (verified from the artifact, NOT the 2003 dump CLAUDE.md names) is the temporal adversary against a ~2023 model. Graded by the external Opus judge vs the fixed corpus-vintage article gold (bench/control_ab.py + bench/judge.py), NOT recall@k. Adversarial by construction. + +who is the prime minister of Australia? +who is the president of France? +who is the prime minister of Poland? +who is the president of India? +who is the president of Dáil Éireann? +who is the president of Iceland? +who is the president of Yemen Arab Republic? +who is the president of Finland? +who is the prime minister of Iraq? +who is the president of Iraq? +who is the president of Austria? +who is the prime minister of Portugal? +who is the chancellor of the Duchy of Lancaster? +who is the president of the Swiss Confederation? +who is the president of China? +who is the president of Guatemala? +who is the prime minister of Denmark? +who is the premier of the Republic of China? +who is the president of Liberia? +who is the prime minister of Croatia? +who is the president of the Quorum of the Twelve Apostles? +who is the president of Afghanistan? +who is the prime minister of Russia? +who is the prime minister of Singapore? +who is the president of Armenia? +who is the president of Algeria? +who is the president of the Church? +who is the president of the Czech Republic? +who is the premier of New Brunswick? +who is the chief minister of Gibraltar? +who is the governor-general of Barbados? +who is the prime minister of Bangladesh? +who is the prime minister of the Cook Islands? +who is the prime minister of Malta? +who is the governor-general of Jamaica? +who is the premier of Alberta? +who is the governor-general of Fiji? +who is the premier of the Northwest Territories? +who is the president of Suriname? +who is the president of Cuba? diff --git a/bench/qa_questions_stale_map.json b/bench/qa_questions_stale_map.json new file mode 100644 index 0000000..803a33d --- /dev/null +++ b/bench/qa_questions_stale_map.json @@ -0,0 +1,242 @@ +[ + { + "question": "who is the prime minister of Australia?", + "target_title": "Prime Minister of Australia", + "target_root": "428a8890bfb742085213eabcfea7b514d420e7970f894422be62d560fc3dfc96", + "shard": "000.db" + }, + { + "question": "who is the president of France?", + "target_title": "President of France", + "target_root": "382bba1c25dc090e24e4e54a5233509a0eb4b65d5862458fb660e4bf733b3395", + "shard": "000.db" + }, + { + "question": "who is the prime minister of Poland?", + "target_title": "Prime Minister of Poland", + "target_root": "f449b067d31a1a7a3149c3eb9e804c109b61ab3f7ffa4a05aaac0e723be96593", + "shard": "000.db" + }, + { + "question": "who is the president of India?", + "target_title": "President of India", + "target_root": "a8f07c98e1c9bc2f73c9ad6789d286d08fa5187b4590c967e243cb3a453f22e0", + "shard": "000.db" + }, + { + "question": "who is the president of Dáil Éireann?", + "target_title": "President of Dáil Éireann", + "target_root": "a151898facf941ba977a73af1b88d6c25a08a83bad55ff29e40b4328c72726e3", + "shard": "000.db" + }, + { + "question": "who is the president of Iceland?", + "target_title": "President of Iceland", + "target_root": "ae5171d65c51f63808f76696ca62f636aadb8d3a62763fd809ace7cfc1e4e04b", + "shard": "000.db" + }, + { + "question": "who is the president of Yemen Arab Republic?", + "target_title": "President of Yemen Arab Republic", + "target_root": "ae0cdc83d21eaf679bb679d7a59242fc3ad650d89bd0e6adb57064f5ac1e2b62", + "shard": "000.db" + }, + { + "question": "who is the president of Finland?", + "target_title": "President of Finland", + "target_root": "76da09a19e6f83b23057a7cf7cfae39a6caa05f4f5dd397ca3fce28ca3aeab11", + "shard": "000.db" + }, + { + "question": "who is the prime minister of Iraq?", + "target_title": "Prime Minister of Iraq", + "target_root": "7490ec30636e41708b041d01a3e150ff981680677cc7adf6e83621be09c95615", + "shard": "000.db" + }, + { + "question": "who is the president of Iraq?", + "target_title": "President of Iraq", + "target_root": "b0e2681907ebb4bdcd2684983027ed590853e71b45293a062d007d0b026801c8", + "shard": "000.db" + }, + { + "question": "who is the president of Austria?", + "target_title": "President of Austria", + "target_root": "af7ee8a5df26cf64c68e8608e661f44ae803734a0bbde30fb14a617d5c7aed50", + "shard": "000.db" + }, + { + "question": "who is the prime minister of Portugal?", + "target_title": "Prime Minister of Portugal", + "target_root": "40c2ff5f1fc18a12dec4af620e71757929b5eadae123a2e527241c8eb93ca40e", + "shard": "000.db" + }, + { + "question": "who is the chancellor of the Duchy of Lancaster?", + "target_title": "Chancellor of the Duchy of Lancaster", + "target_root": "258a22d83fd9de77782ba7d0756dd55e2448de31c80540055163767e16a03864", + "shard": "000.db" + }, + { + "question": "who is the president of the Swiss Confederation?", + "target_title": "President of the Swiss Confederation", + "target_root": "eb042c69e9a844129fec2e8da39d064d839f3300e350d198955127197aa04384", + "shard": "000.db" + }, + { + "question": "who is the president of China?", + "target_title": "President of China", + "target_root": "8fcbb92f1dcbe9767745465cb474a70c558483f2d25a7c57902001e144118baf", + "shard": "000.db" + }, + { + "question": "who is the president of Guatemala?", + "target_title": "President of Guatemala", + "target_root": "45c98dbeb0fa2270b5f2f33d6b8ce942a0686ed3270d5425dd81c90fc8ce3331", + "shard": "000.db" + }, + { + "question": "who is the prime minister of Denmark?", + "target_title": "Prime Minister of Denmark", + "target_root": "af1e60414e42743c75394f313b2eb87431fc89e0a73300b58b39e72b61955359", + "shard": "000.db" + }, + { + "question": "who is the premier of the Republic of China?", + "target_title": "Premier of the Republic of China", + "target_root": "be8087685eb06bee042609008590daa07ada1d131aa84cc5386c09ecda721d5c", + "shard": "000.db" + }, + { + "question": "who is the president of Liberia?", + "target_title": "President of Liberia", + "target_root": "ac6b6428d657f9c620e5922fdb2b71657a9a951a311dd4bc04eafdf4c58fdcec", + "shard": "000.db" + }, + { + "question": "who is the prime minister of Croatia?", + "target_title": "Prime Minister of Croatia", + "target_root": "6fac1f1eaf82976cf67dbd23b26794cd8f6667a9bca4d0636f72b525ca18f814", + "shard": "000.db" + }, + { + "question": "who is the president of the Quorum of the Twelve Apostles?", + "target_title": "President of the Quorum of the Twelve Apostles", + "target_root": "f22dd4f188beb0969baabcee48a3f51cc315dfc81a62bced695d1938523b1436", + "shard": "000.db" + }, + { + "question": "who is the president of Afghanistan?", + "target_title": "President of Afghanistan", + "target_root": "3b0d22698bf20ab6b8a54ba7bf724859a3eb94bd49d17a2aae5a28f1fcdec42f", + "shard": "000.db" + }, + { + "question": "who is the prime minister of Russia?", + "target_title": "Prime Minister of Russia", + "target_root": "847f075b40d32beabe220e302839e038af39f254ebb12b52d87221ec7e7963f7", + "shard": "000.db" + }, + { + "question": "who is the prime minister of Singapore?", + "target_title": "Prime Minister of Singapore", + "target_root": "7870bf7917dadabe63c6c3c6d26165cac85266998464ffb2e1105e645abcb032", + "shard": "000.db" + }, + { + "question": "who is the president of Armenia?", + "target_title": "President of Armenia", + "target_root": "f0a6c791f1cf283e88286e67949b781d910c6ed6d8c0e0b2782a657d720f84e1", + "shard": "000.db" + }, + { + "question": "who is the president of Algeria?", + "target_title": "President of Algeria", + "target_root": "9d5d7cf867dcf122e31a530dcd2abf9cf85f3968360e8f545483ac3c41487715", + "shard": "000.db" + }, + { + "question": "who is the president of the Church?", + "target_title": "President of the Church", + "target_root": "495fd4dbeed8215a7359151252291b399c9f5c3e9b2970087a5c892433afd3fd", + "shard": "000.db" + }, + { + "question": "who is the president of the Czech Republic?", + "target_title": "President of the Czech Republic", + "target_root": "90850b63b7f250f8c48e17bccd0af48e215135f98dfd673b5b02a6f1d6d947fa", + "shard": "000.db" + }, + { + "question": "who is the premier of New Brunswick?", + "target_title": "Premier of New Brunswick", + "target_root": "2d54530ca883b3f318f542510bf9d4f98029397a58a001a45645b7a983a4fea0", + "shard": "000.db" + }, + { + "question": "who is the chief minister of Gibraltar?", + "target_title": "Chief Minister of Gibraltar", + "target_root": "2f5110adf2aeb361e451ec77f81ae02380da9e38a408ff082e80df101b0e4df3", + "shard": "000.db" + }, + { + "question": "who is the governor-general of Barbados?", + "target_title": "Governor-General of Barbados", + "target_root": "5217ca9516e51999c5148a045dd6056865affa00f573d48878910757c5bd1b8b", + "shard": "000.db" + }, + { + "question": "who is the prime minister of Bangladesh?", + "target_title": "Prime Minister of Bangladesh", + "target_root": "eb1bb528d46d2d7056bd218124ff281c70f629f7695ac735ce0e40aa0e098c97", + "shard": "000.db" + }, + { + "question": "who is the prime minister of the Cook Islands?", + "target_title": "Prime Minister of the Cook Islands", + "target_root": "5eadfd398208797f1117cc3cf6c6ff6d2acd0600af8da0bbda3fae5807d4075c", + "shard": "000.db" + }, + { + "question": "who is the prime minister of Malta?", + "target_title": "Prime Minister of Malta", + "target_root": "a63a7c99548afed5aebb4a40bd25098ce332b7007e87d51bf0863488cd4f4a45", + "shard": "000.db" + }, + { + "question": "who is the governor-general of Jamaica?", + "target_title": "Governor-General of Jamaica", + "target_root": "b3f7724592af5ae97b71256d601016cf05a26d6c0de0f64d23dc8864eb9f29b9", + "shard": "000.db" + }, + { + "question": "who is the premier of Alberta?", + "target_title": "Premier of Alberta", + "target_root": "58e633c3c6cda2ae8198ae389937a19daebc4d6068af08e2fc633773b817b719", + "shard": "000.db" + }, + { + "question": "who is the governor-general of Fiji?", + "target_title": "Governor-General of Fiji", + "target_root": "eb8c8a332ee34d342bb7f56d7c04fdeb97b7d8dc044bd7d2ad924cafb537680e", + "shard": "000.db" + }, + { + "question": "who is the premier of the Northwest Territories?", + "target_title": "Premier of the Northwest Territories", + "target_root": "65fa29054a377490fcf6f7a524d8058a4a825a4a8c7b0d42cfe92a7b7cefcb83", + "shard": "000.db" + }, + { + "question": "who is the president of Suriname?", + "target_title": "President of Suriname", + "target_root": "6f83350c604a50857075cbd848d06cca4fcb0139a32ccd89b2fd0995cf94bb22", + "shard": "000.db" + }, + { + "question": "who is the president of Cuba?", + "target_title": "President of Cuba", + "target_root": "98af74f9b96f3f679a4f671f719ad8562c3f70d2416f5c6230485f3cc5faca79", + "shard": "000.db" + } +]