From 1d7a9ee38dc7d5bc88ba61fa4013e97af36a96d8 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Thu, 30 Apr 2026 14:55:39 -0400 Subject: [PATCH] =?UTF-8?q?qa:=20live=20fixture=20=E2=80=94=20red=20fish?= =?UTF-8?q?=20blue=20fish=20=E2=86=92=20Dr.=20Seuss=20book?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds test_red_fish_blue_fish_identifies_seuss_book under the new "retrieval disambiguation" section. The query partially matches many noise titles (Red Dwarf, Blue whale, Toronto Blue Jays, Red Sea, Detroit Red Wings, marine aquarium fish list) so it stress- tests the retrieval+rerank pipeline's ability to land on the right article — `One Fish Two Fish Red Fish Blue Fish` by Dr. Seuss — despite the surface noise. Asserts the answer is grounded and contains a Seuss-specific marker (seuss, rhyming, children's book, or creatures). 10 tests collected total. --- tests/test_qa_quality_live.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_qa_quality_live.py b/tests/test_qa_quality_live.py index b633a31..57c0632 100644 --- a/tests/test_qa_quality_live.py +++ b/tests/test_qa_quality_live.py @@ -141,6 +141,24 @@ def test_mona_lisa_grounded_and_names_leonardo(): f"answer missing 'Leonardo': {_answer_lower(r)[:200]}" +# ---------------------------------------------------------------- retrieval disambiguation + + +def test_red_fish_blue_fish_identifies_seuss_book(): + """Stress-tests retrieval disambiguation: "red fish blue fish" partially + matches many noise titles (Red Dwarf, Blue whale, Toronto Blue Jays, + Red Sea, Detroit Red Wings, marine aquarium fish list...). The right + answer is the Dr. Seuss book ``One Fish Two Fish Red Fish Blue Fish``. + The model should anchor on the Seuss article and the answer should + contain a Seuss-specific token.""" + r = _ask("plot of red fish blue fish?") + assert _grounded(r), f"audit_mode={r.get('audit_mode')!r}" + txt = _answer_lower(r) + seuss_markers = ("seuss", "rhyming", "children's book", "creatures") + assert any(m in txt for m in seuss_markers), \ + f"answer missing Seuss-book markers: {txt[:300]}" + + # ---------------------------------------------------------------- entity disambiguation