From b9fb377d8e837bc9ab831de78ae17594347a8676 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Thu, 30 Apr 2026 16:43:46 -0400 Subject: [PATCH] qa(live-fixtures): swap apple+microsoft for veronica ballestrini Removed: - test_apple_founders_grounded_and_names_steve_jobs - test_microsoft_founders_names_gates_and_allen Added: - test_veronica_ballestrini_singer_october_birth The veronica fixture is a stronger probe than the two it replaces: - entity-disambiguation among 8 same-first-name candidates (Landers, Guerin, Belmont, Wadley, Gamba, Doran, the disambiguation page, plus Ballestrini herself in the corpus) - multi-part question (who is she + when was she born) - niche personal-bio source (small article, harder to anchor than well-known commercial-tech bios) Asserts: grounded + answer mentions Ballestrini + at least one bio marker (country/singer/songwriter/Connecticut/Italian) + at least one birth marker (October or 1991). Lands STRICT 1/1 in JSON mode at the live test. 10/10 fixtures pass. --- tests/test_qa_quality_live.py | 48 ++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/tests/test_qa_quality_live.py b/tests/test_qa_quality_live.py index e61ba4d..ec0ee64 100644 --- a/tests/test_qa_quality_live.py +++ b/tests/test_qa_quality_live.py @@ -107,16 +107,6 @@ def _grounded(result: dict) -> bool: # ---------------------------------------------------------------- narrow factoid -def test_apple_founders_grounded_and_names_steve_jobs(): - """The Apple article is well-anchored corpus — every mode should - hit at least HYBRID and Jobs should appear in the rendered answer. - """ - r = _ask("who founded apple computer?") - assert _grounded(r), f"audit_mode={r.get('audit_mode')!r}" - assert "steve jobs" in _answer_lower(r) or "jobs" in _answer_lower(r), \ - f"answer missing 'Jobs': {_answer_lower(r)[:200]}" - - def test_france_capital_strict_and_names_paris(): """Capital-of-X is the textbook narrow-factoid; we expect STRICT and the literal city name to appear in the cited or rendered span.""" @@ -172,6 +162,36 @@ def test_red_fish_blue_fish_identifies_seuss_book(): # ---------------------------------------------------------------- entity disambiguation +def test_veronica_ballestrini_singer_october_birth(): + """Multi-part question against a niche personal-bio article. + Retrieval surfaces 8 different Veronicas (Landers, Guerin, + Belmont, Wadley, Gamba, Doran, the disambiguation page, and + Ballestrini herself). The model should anchor on Ballestrini's + article and answer both parts: who she is (country singer / + songwriter / Connecticut) and when she was born (October). + Tests entity-disambiguation among same-first-name candidates + plus multi-part-question handling on a small-bio source. + """ + r = _ask( + "who is veronica ballestrini when what month was she born?" + ) + assert _grounded(r), f"audit_mode={r.get('audit_mode')!r}" + txt = _answer_lower(r) + assert "ballestrini" in txt, \ + f"answer doesn't even mention 'Ballestrini': {txt[:300]}" + # The "who is" half — accept any of the canonical descriptors + # from her bio span. + bio_markers = ("country", "singer", "songwriter", "connecticut", "italian") + assert any(m in txt for m in bio_markers), \ + f"answer missing bio markers: {txt[:300]}" + # The "when was she born" half — accept "october" or the full + # date "october 29, 1991" or any 1991 mention as evidence the + # model picked up the birth date span. + birth_markers = ("october", "1991") + assert any(m in txt for m in birth_markers), \ + f"answer missing birth markers: {txt[:300]}" + + def test_lara_croft_mentions_tomb_raider(): """Lara Croft (correctly spelled) should pull the Tomb Raider article; the answer should mention either the franchise or her @@ -225,14 +245,6 @@ def test_ninja_turtles_lists_all_four_plus_splinter(): assert "splinter" in txt, f"missing master 'Splinter': {txt[:300]}" -def test_microsoft_founders_names_gates_and_allen(): - r = _ask("list the founders of microsoft") - assert _grounded(r), f"audit_mode={r.get('audit_mode')!r}" - txt = _answer_lower(r) - assert "gates" in txt, f"missing 'Gates': {txt[:300]}" - assert "allen" in txt, f"missing 'Allen': {txt[:300]}" - - def test_homer_simpson_boss_is_mr_burns(): """Cross-document relationship: Homer's boss is named in the Homer Simpson article (and several episode articles that reference the