From e561dfc5995f2358d068f11eddf1c751b76e4dcb Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Thu, 30 Apr 2026 16:58:00 -0400 Subject: [PATCH] bench/qa: refresh question set; live test updates Mars adversarial MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Question-set churn (bench/qa_questions.txt): - narrow factoid: drop 'who founded apple computer?'; add 'a bridge between new london & groton?' (single-answer factoid with named-entity grounding via Gold Star Memorial Bridge) - broad descriptive: drop 'tell me about the apollo program' - entity list: drop 'list the founders of microsoft'; add 'name simpsons family members including pets?', 'list of obelisk in connecticut' - leading/adversarial: add 'whales are endangered from over hunting' (declarative-shape, tests verifier under premise-seeking phrasing) - out-of-corpus: rephrase Mars no-such-entity question to 'who is a benevolent dictator for life for mars?' — sharper adversarial shape since BDFL is a real OSS-governance term the model is tempted to project onto Mars - minor edits: 'GNU linux', 'C programming language', 'method man', 'cold fusion breakthrough' phrasing tests/test_qa_quality_live.py: rename test_mars_prime_minister_* to test_mars_benevolent_dictator_*; refresh refutation markers to match the new BDFL phrasing; expand to cover 'no one' / 'no person' negations the model uses on this shape. --- bench/qa_questions.txt | 20 +++++++++++--------- tests/test_qa_quality_live.py | 25 ++++++++++++++++--------- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/bench/qa_questions.txt b/bench/qa_questions.txt index 189e678..895ffe8 100644 --- a/bench/qa_questions.txt +++ b/bench/qa_questions.txt @@ -6,27 +6,28 @@ # failure-mode shapes the verifier needs to handle. # narrow factoid — well-anchored single-fact questions -who founded apple computer? what is the capital of france? -who wrote the linux kernel? -when was the python programming language created? +who wrote GNU linux? +when was a programming language named Python created? who painted the mona lisa? +a bridge between new london & groton? # broad descriptive — encyclopedic shape, prone to mode-collapse tell me about connecticut -tell me about the python programming language -tell me about charles babbage -tell me about the apollo program +tell me about the C programming language +tell me about method man? # entity list — invites lazy-anchor on a magnet chunk what dinosaurs were in the first jurassic park film? who are the members of the beatles? name simpsons family members including pets? +list of obelisk in connecticut # relationship / multi-fact who is supermans girlfriend? who is bilbo baggins's nephew? -what is the relationship between linux and unix? +what is the relationship between linux & unix? +who is veronica ballestrini & what month was she born? # comparison — multi-entity, prone to attribution drift what's the difference between linux and bsd? @@ -38,7 +39,8 @@ who invented the doppler effect? # leading / adversarial — model should refuse over-claim isn't it true that the great wall of china is visible from space? +whales are endangered from over hunting # out-of-corpus — should land UNGROUNDED honestly -who is the prime minister of mars? -what year did the cold fusion breakthrough happen? +who is a benevolent dictator for life for mars? +what year does our cold fusion breakthrough happen? diff --git a/tests/test_qa_quality_live.py b/tests/test_qa_quality_live.py index ec0ee64..0ed943d 100644 --- a/tests/test_qa_quality_live.py +++ b/tests/test_qa_quality_live.py @@ -270,26 +270,33 @@ def test_homer_simpson_boss_is_mr_burns(): # ---------------------------------------------------------------- honest refusal -def test_mars_prime_minister_refuses_or_ungrounded(): - """Mars has no prime minister. The system should either refuse - (answer contains "no" / "does not exist" / similar) or land - UNGROUNDED. A grounded affirmative claim of a Mars PM would be a - hallucination — fail the test.""" - r = _ask("who is the prime minister of mars?") +def test_mars_benevolent_dictator_refuses_or_ungrounded(): + """Mars has no benevolent dictator for life. The system should + either refuse (answer contains "no" / "does not exist" / similar) + or land UNGROUNDED. A grounded affirmative claim of a Mars BDFL + would be a hallucination — fail the test. + + Adversarial shape: the phrasing "benevolent dictator for life" is + a real term-of-art (open-source governance — Linus, Guido) so the + model is tempted to fluently project it onto Mars. The corpus has + no such anchor; STRICT/HYBRID with an affirmative answer = fail.""" + r = _ask("who is a benevolent dictator for life for mars?") audit = r.get("audit_mode") txt = _answer_lower(r) # UNGROUNDED is the honest outcome on a no-such-thing question. if audit == "UNGROUNDED": return # If grounded, the answer must include a refutation phrase. Bare - # affirmative claims of a Mars PM are hallucinations. + # affirmative claims of a Mars BDFL are hallucinations. refutation_markers = ( - "no prime minister", + "no benevolent dictator", "does not have", "is not a", "no government", "not a sovereign", "no such", + "no one", + "no person", ) assert any(m in txt for m in refutation_markers), \ - f"audit={audit} but answer asserts a Mars PM: {txt[:300]}" + f"audit={audit} but answer asserts a Mars BDFL: {txt[:300]}"