diff --git a/tests/test_qa_quality_live.py b/tests/test_qa_quality_live.py index 038f58a..e6c5f7f 100644 --- a/tests/test_qa_quality_live.py +++ b/tests/test_qa_quality_live.py @@ -124,13 +124,49 @@ def test_linux_kernel_grounded_and_names_torvalds(): f"answer missing 'Torvalds': {_answer_lower(r)[:200]}" -def test_mona_lisa_grounded_and_names_leonardo(): +def test_mona_lisa_strict_and_names_leonardo(): + """Bench standout (2026-04-30, n=3): JSON mode hit S:3 H:0 U:0 + while quote landed 0S and pointer 0S. Tightened from _grounded + to STRICT to lock in the JSON-mode unique win as a regression + gate — drop in JSON-mode quality on this question reverts the + STRICT signal first.""" r = _ask("who painted the mona lisa?") - assert _grounded(r), f"audit_mode={r.get('audit_mode')!r}" + assert r.get("audit_mode") == "STRICT", \ + f"expected STRICT (bench 3/3 STRICT in JSON), got {r.get('audit_mode')}" assert "leonardo" in _answer_lower(r), \ f"answer missing 'Leonardo': {_answer_lower(r)[:200]}" +def test_supermans_girlfriend_strict_and_names_lois_lane(): + """Bench standout (2026-04-30, n=3): JSON mode hit S:3 H:0 U:0 + while pointer landed S:0 H:3 (every grounded answer demoted by + lazy-anchor or pointer-saturation). The relationship-shape + question 'who is X's girlfriend' is JSON's strong suit — atomic + claims, single-name target. Gate: STRICT + 'lois' or 'lane'.""" + r = _ask("who is supermans girlfriend?") + assert r.get("audit_mode") == "STRICT", \ + f"expected STRICT (bench 3/3 STRICT in JSON), got {r.get('audit_mode')}" + txt = _answer_lower(r) + assert "lois" in txt or "lane" in txt, \ + f"answer missing Lois/Lane: {txt[:200]}" + + +def test_cold_fusion_breakthrough_year_strict_and_names_1989(): + """Bench standout (2026-04-30, n=3): originally classified + out-of-corpus but JSON mode hit S:3 H:0 U:0 — the corpus has + Pons & Fleischmann's 1989 announcement covered. Quote mode + landed 0S 0H 3U (UNGROUNDED) so this fixture also catches + quote/JSON divergence. Gate: STRICT + a 1989-era marker + (year, Pons, Fleischmann, or Utah).""" + r = _ask("what year does our cold fusion breakthrough happen?") + assert r.get("audit_mode") == "STRICT", \ + f"expected STRICT (bench 3/3 STRICT in JSON), got {r.get('audit_mode')}" + txt = _answer_lower(r) + cold_fusion_markers = ("1989", "pons", "fleischmann", "utah") + assert any(m in txt for m in cold_fusion_markers), \ + f"answer missing 1989-era cold-fusion markers: {txt[:300]}" + + # ---------------------------------------------------------------- retrieval disambiguation