diff --git a/Makefile b/Makefile index f315977..6a00000 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ SEARCH_Q ?= computer ingest-xml-attached ingest-abstract \ ingest-grok ingest-grok-media \ ingest-self ingest-git ingest-hg \ - verify search stats test docs chain-check chain-check-shards \ + verify search stats test test-live docs chain-check chain-check-shards \ falsify burn burn-kindergarten inspect bootstrap-crawler test-crawler crawl-ingest \ recrawl-check bench-qa clean clean-db clean-data help @@ -188,6 +188,10 @@ bench-qa: bootstrap ## QA-quality sweep: questions × modes × N samples [BENCH_ --limit $(BENCH_QA_LIMIT) \ --n $(BENCH_QA_N) +test-live: bootstrap ## live QA quality tests against Hermes (gated; ~1 min) + ABORIST_LIVE_TESTS=1 ABORIST_LIVE_SHARDS_DIR=$(SHARDS_DIR) \ + .venv/bin/pytest tests/test_qa_quality_live.py -v + verify-shards: bootstrap ## cross-shard Merkle round-trip on a random sample $(ABORIST) --shards-dir $(SHARDS_DIR) verify -n $(VERIFY_N) diff --git a/tests/test_qa_quality_live.py b/tests/test_qa_quality_live.py new file mode 100644 index 0000000..b633a31 --- /dev/null +++ b/tests/test_qa_quality_live.py @@ -0,0 +1,227 @@ +"""Live QA-quality tests against the real Hermes endpoint. + +These tests are deliberately gated — they hit +``hermes.ai.unturf.com/v1`` and require a populated shard set under +``~/.aborist/shards`` (default) or the path passed via +``ABORIST_LIVE_SHARDS_DIR``. Default `make test` does NOT run these; +opt in via ``ABORIST_LIVE_TESTS=1`` (the ``make test-live`` target +sets it). + +Why this exists: + +The unit tests in ``test_qa.py`` / ``test_query.py`` use ``StubClient`` +to validate plumbing (cache lookups, document mapping, schema, audit +chain). They cannot tell you if the system answers ``"who founded +Apple Computer?"`` correctly — that's a model+retrieval+verifier +behavior, not a unit-tested code path. The QA-quality bench +(``make bench-qa``) measures aggregate verdict counts but doesn't +assert specific content. These live fixtures sit between those two +concerns: each test runs ONE known-good question, asserts the +audit_mode is at least HYBRID, and checks for entity-level content +the answer should mention. + +"Benchmax" rationale: the bench gives us a scoreboard +(STRICT/HYBRID/UNGROUNDED counts); these tests give us *gates*. When +a future change improves the system, the strict-rate climbs in the +bench AND every fixture here keeps passing or gets stricter +assertions. When a change regresses, the bench number falls AND +specific fixtures fail by name, telling you where the regression +landed. + +Every assertion accepts a tolerance band — Hermes is non-deterministic +at temperature > 0, so single-sample tests are inherently flaky on +boundary cases. The bar is "the right entity is present in the +answer", not "the answer is byte-identical to last run". When a +fixture starts failing intermittently, treat it as evidence of +quality drift on that question shape and tighten retrieval / verifier +upstream rather than loosen the test. +""" +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + + +_LIVE_OPT_IN = os.environ.get("ABORIST_LIVE_TESTS") == "1" +_SHARDS_DIR = Path( + os.environ.get("ABORIST_LIVE_SHARDS_DIR") + or (Path.home() / ".aborist" / "shards") +) +_QA_DB = _SHARDS_DIR / "qa.db" +_ENDPOINT = os.environ.get( + "ABORIST_LLM_ENDPOINT", "https://hermes.ai.unturf.com/v1" +) +_MODEL = os.environ.get( + "ABORIST_LLM_MODEL", + "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic", +) + + +pytestmark = pytest.mark.skipif( + not _LIVE_OPT_IN, + reason="live QA tests gated by ABORIST_LIVE_TESTS=1; " + "run via `make test-live` or set the env var", +) + + +def _ask(question: str, *, mode: str = "claim_lattice", top_k: int = 8) -> dict: + """One live query against the configured shard set + Hermes endpoint.""" + if not _SHARDS_DIR.exists(): + pytest.skip(f"shards dir not found: {_SHARDS_DIR}") + if not _QA_DB.parent.exists(): + pytest.skip(f"qa.db parent not found: {_QA_DB.parent}") + + # Lazy import so collecting the file doesn't pull qa modules in + # default-skip mode. + from aborist.qa.client import OpenAICompatibleClient + from aborist.qa.query import DEFAULT_QUERY_POLICY, query + + api_key = os.environ.get("ABORIST_LLM_API_KEY") + client = OpenAICompatibleClient(base_url=_ENDPOINT, api_key=api_key) + policy = dict(DEFAULT_QUERY_POLICY) + policy["answer_mode"] = mode + + return query( + question=question, + qa_db=_QA_DB, + chat_client=client, + model_id=_MODEL, + shards_dir=_SHARDS_DIR, + top_k=top_k, + policy=policy, + burn_existing=True, # forces fresh inference per test + ) + + +def _answer_lower(result: dict) -> str: + return (result.get("answer_text") or "").lower() + + +def _grounded(result: dict) -> bool: + """STRICT or HYBRID — at least one pointer/quote pair verified.""" + return result.get("audit_mode") in ("STRICT", "HYBRID") + + +# ---------------------------------------------------------------- narrow factoid + + +def test_apple_founders_grounded_and_names_steve_jobs(): + """The Apple article is well-anchored corpus — every mode should + hit at least HYBRID and Jobs should appear in the rendered answer. + """ + r = _ask("who founded apple computer?") + assert _grounded(r), f"audit_mode={r.get('audit_mode')!r}" + assert "steve jobs" in _answer_lower(r) or "jobs" in _answer_lower(r), \ + f"answer missing 'Jobs': {_answer_lower(r)[:200]}" + + +def test_france_capital_strict_and_names_paris(): + """Capital-of-X is the textbook narrow-factoid; we expect STRICT + and the literal city name to appear in the cited or rendered span.""" + r = _ask("what is the capital of france?") + assert r.get("audit_mode") == "STRICT", \ + f"expected STRICT, got {r.get('audit_mode')}" + assert "paris" in _answer_lower(r), \ + f"answer missing 'Paris': {_answer_lower(r)[:200]}" + + +def test_linux_kernel_grounded_and_names_torvalds(): + r = _ask("who wrote the linux kernel?") + assert _grounded(r), f"audit_mode={r.get('audit_mode')!r}" + assert "torvalds" in _answer_lower(r), \ + f"answer missing 'Torvalds': {_answer_lower(r)[:200]}" + + +def test_mona_lisa_grounded_and_names_leonardo(): + r = _ask("who painted the mona lisa?") + assert _grounded(r), f"audit_mode={r.get('audit_mode')!r}" + assert "leonardo" in _answer_lower(r), \ + f"answer missing 'Leonardo': {_answer_lower(r)[:200]}" + + +# ---------------------------------------------------------------- entity disambiguation + + +def test_lara_croft_mentions_tomb_raider(): + """Lara Croft (correctly spelled) should pull the Tomb Raider + article; the answer should mention either the franchise or her + role/profession.""" + r = _ask("who is lara croft?") + assert _grounded(r), f"audit_mode={r.get('audit_mode')!r}" + txt = _answer_lower(r) + assert ( + "tomb raider" in txt + or "archaeologist" in txt + or "video game" in txt + ), f"answer missing Tomb Raider context: {txt[:200]}" + + +def test_laura_croft_finds_at_least_one_real_entity(): + """`Laura Croft` (an extra `u`) is BOTH a real Playmate and a + common misspelling of `Lara Croft`. The system should land on at + least one real entity from the corpus, not hallucinate a third + person.""" + r = _ask("who is laura croft?") + assert _grounded(r), f"audit_mode={r.get('audit_mode')!r}" + txt = _answer_lower(r) + # One of: the actual Laura Croft (model) markers, or the Lara + # Croft (character) markers. Either grounding is honest. + real_entity_markers = ( + "playmate", "playboy", "model", # Laura Croft (Playmate) + "tomb raider", "archaeologist", # Lara Croft (character) + ) + assert any(m in txt for m in real_entity_markers), \ + f"answer doesn't ground on any known entity: {txt[:300]}" + + +# ---------------------------------------------------------------- multi-part list + + +def test_ninja_turtles_lists_all_four_plus_splinter(): + """Multi-part question: all four turtle names + the master's name + must appear in the answer. Tests that the system handles a + conjunctive `&` query and produces a complete list.""" + r = _ask("what are the names of the ninja turtles & their master name.") + assert _grounded(r), f"audit_mode={r.get('audit_mode')!r}" + txt = _answer_lower(r) + for turtle in ("leonardo", "donatello", "michelangelo", "raphael"): + assert turtle in txt, f"missing turtle '{turtle}': {txt[:300]}" + assert "splinter" in txt, f"missing master 'Splinter': {txt[:300]}" + + +def test_microsoft_founders_names_gates_and_allen(): + r = _ask("list the founders of microsoft") + assert _grounded(r), f"audit_mode={r.get('audit_mode')!r}" + txt = _answer_lower(r) + assert "gates" in txt, f"missing 'Gates': {txt[:300]}" + assert "allen" in txt, f"missing 'Allen': {txt[:300]}" + + +# ---------------------------------------------------------------- honest refusal + + +def test_mars_prime_minister_refuses_or_ungrounded(): + """Mars has no prime minister. The system should either refuse + (answer contains "no" / "does not exist" / similar) or land + UNGROUNDED. A grounded affirmative claim of a Mars PM would be a + hallucination — fail the test.""" + r = _ask("who is the prime minister of mars?") + audit = r.get("audit_mode") + txt = _answer_lower(r) + # UNGROUNDED is the honest outcome on a no-such-thing question. + if audit == "UNGROUNDED": + return + # If grounded, the answer must include a refutation phrase. Bare + # affirmative claims of a Mars PM are hallucinations. + refutation_markers = ( + "no prime minister", + "does not have", + "is not a", + "no government", + "not a sovereign", + "no such", + ) + assert any(m in txt for m in refutation_markers), \ + f"audit={audit} but answer asserts a Mars PM: {txt[:300]}"