diff --git a/bench/qa_questions.txt b/bench/qa_questions.txt index 189e678..895ffe8 100644 --- a/bench/qa_questions.txt +++ b/bench/qa_questions.txt @@ -6,27 +6,28 @@ # failure-mode shapes the verifier needs to handle. # narrow factoid — well-anchored single-fact questions -who founded apple computer? what is the capital of france? -who wrote the linux kernel? -when was the python programming language created? +who wrote GNU linux? +when was a programming language named Python created? who painted the mona lisa? +a bridge between new london & groton? # broad descriptive — encyclopedic shape, prone to mode-collapse tell me about connecticut -tell me about the python programming language -tell me about charles babbage -tell me about the apollo program +tell me about the C programming language +tell me about method man? # entity list — invites lazy-anchor on a magnet chunk what dinosaurs were in the first jurassic park film? who are the members of the beatles? name simpsons family members including pets? +list of obelisk in connecticut # relationship / multi-fact who is supermans girlfriend? who is bilbo baggins's nephew? -what is the relationship between linux and unix? +what is the relationship between linux & unix? +who is veronica ballestrini & what month was she born? # comparison — multi-entity, prone to attribution drift what's the difference between linux and bsd? @@ -38,7 +39,8 @@ who invented the doppler effect? # leading / adversarial — model should refuse over-claim isn't it true that the great wall of china is visible from space? +whales are endangered from over hunting # out-of-corpus — should land UNGROUNDED honestly -who is the prime minister of mars? -what year did the cold fusion breakthrough happen? +who is a benevolent dictator for life for mars? +what year does our cold fusion breakthrough happen? diff --git a/tests/test_qa_quality_live.py b/tests/test_qa_quality_live.py index ec0ee64..0ed943d 100644 --- a/tests/test_qa_quality_live.py +++ b/tests/test_qa_quality_live.py @@ -270,26 +270,33 @@ def test_homer_simpson_boss_is_mr_burns(): # ---------------------------------------------------------------- honest refusal -def test_mars_prime_minister_refuses_or_ungrounded(): - """Mars has no prime minister. The system should either refuse - (answer contains "no" / "does not exist" / similar) or land - UNGROUNDED. A grounded affirmative claim of a Mars PM would be a - hallucination — fail the test.""" - r = _ask("who is the prime minister of mars?") +def test_mars_benevolent_dictator_refuses_or_ungrounded(): + """Mars has no benevolent dictator for life. The system should + either refuse (answer contains "no" / "does not exist" / similar) + or land UNGROUNDED. A grounded affirmative claim of a Mars BDFL + would be a hallucination — fail the test. + + Adversarial shape: the phrasing "benevolent dictator for life" is + a real term-of-art (open-source governance — Linus, Guido) so the + model is tempted to fluently project it onto Mars. The corpus has + no such anchor; STRICT/HYBRID with an affirmative answer = fail.""" + r = _ask("who is a benevolent dictator for life for mars?") audit = r.get("audit_mode") txt = _answer_lower(r) # UNGROUNDED is the honest outcome on a no-such-thing question. if audit == "UNGROUNDED": return # If grounded, the answer must include a refutation phrase. Bare - # affirmative claims of a Mars PM are hallucinations. + # affirmative claims of a Mars BDFL are hallucinations. refutation_markers = ( - "no prime minister", + "no benevolent dictator", "does not have", "is not a", "no government", "not a sovereign", "no such", + "no one", + "no person", ) assert any(m in txt for m in refutation_markers), \ - f"audit={audit} but answer asserts a Mars PM: {txt[:300]}" + f"audit={audit} but answer asserts a Mars BDFL: {txt[:300]}"