arborist/tests/test_qa_quality_live.py

"""Live QA-quality tests against the real Hermes endpoint.

These tests are deliberately gated — they hit
``hermes.ai.unturf.com/v1`` and require a populated shard set under
``~/.aborist/shards`` (default) or the path passed via
``ABORIST_LIVE_SHARDS_DIR``. Default `make test` does NOT run these;
opt in via ``ABORIST_LIVE_TESTS=1`` (the ``make test-live`` target
sets it).

Why this exists:

The unit tests in ``test_qa.py`` / ``test_query.py`` use ``StubClient``
to validate plumbing (cache lookups, document mapping, schema, audit
chain). They cannot tell you if the system answers ``"who founded
Apple Computer?"`` correctly — that's a model+retrieval+verifier
behavior, not a unit-tested code path. The QA-quality bench
(``make bench-qa``) measures aggregate verdict counts but doesn't
assert specific content. These live fixtures sit between those two
concerns: each test runs ONE known-good question, asserts the
audit_mode is at least HYBRID, and checks for entity-level content
the answer should mention.

"Benchmax" rationale: the bench gives us a scoreboard
(STRICT/HYBRID/UNGROUNDED counts); these tests give us *gates*. When
a future change improves the system, the strict-rate climbs in the
bench AND every fixture here keeps passing or gets stricter
assertions. When a change regresses, the bench number falls AND
specific fixtures fail by name, telling you where the regression
landed.

Every assertion accepts a tolerance band — Hermes is non-deterministic
at temperature > 0, so single-sample tests are inherently flaky on
boundary cases. The bar is "the right entity is present in the
answer", not "the answer is byte-identical to last run". When a
fixture starts failing intermittently, treat it as evidence of
quality drift on that question shape and tighten retrieval / verifier
upstream rather than loosen the test.
"""
from __future__ import annotations

import os
from pathlib import Path

import pytest


_LIVE_OPT_IN = os.environ.get("ABORIST_LIVE_TESTS") == "1"
_SHARDS_DIR = Path(
    os.environ.get("ABORIST_LIVE_SHARDS_DIR")
    or (Path.home() / ".aborist" / "shards")
)
_QA_DB = _SHARDS_DIR / "qa.db"
_ENDPOINT = os.environ.get(
    "ABORIST_LLM_ENDPOINT", "https://hermes.ai.unturf.com/v1"
)
_MODEL = os.environ.get(
    "ABORIST_LLM_MODEL",
    "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic",
)


pytestmark = pytest.mark.skipif(
    not _LIVE_OPT_IN,
    reason="live QA tests gated by ABORIST_LIVE_TESTS=1; "
           "run via `make test-live` or set the env var",
)


def _ask(question: str, *, mode: str = "claim_lattice", top_k: int = 8) -> dict:
    """One live query against the configured shard set + Hermes endpoint."""
    if not _SHARDS_DIR.exists():
        pytest.skip(f"shards dir not found: {_SHARDS_DIR}")
    if not _QA_DB.parent.exists():
        pytest.skip(f"qa.db parent not found: {_QA_DB.parent}")

    # Lazy import so collecting the file doesn't pull qa modules in
    # default-skip mode.
    from aborist.qa.client import OpenAICompatibleClient
    from aborist.qa.query import DEFAULT_QUERY_POLICY, query

    api_key = os.environ.get("ABORIST_LLM_API_KEY")
    client = OpenAICompatibleClient(base_url=_ENDPOINT, api_key=api_key)
    policy = dict(DEFAULT_QUERY_POLICY)
    policy["answer_mode"] = mode

    return query(
        question=question,
        qa_db=_QA_DB,
        chat_client=client,
        model_id=_MODEL,
        shards_dir=_SHARDS_DIR,
        top_k=top_k,
        policy=policy,
        burn_existing=True,  # forces fresh inference per test
    )


def _answer_lower(result: dict) -> str:
    return (result.get("answer_text") or "").lower()


def _grounded(result: dict) -> bool:
    """STRICT or HYBRID — at least one pointer/quote pair verified."""
    return result.get("audit_mode") in ("STRICT", "HYBRID")


# ---------------------------------------------------------------- narrow factoid


def test_france_capital_strict_and_names_paris():
    """Capital-of-X is the textbook narrow-factoid; we expect STRICT
    and the literal city name to appear in the cited or rendered span."""
    r = _ask("what is the capital of france?")
    assert r.get("audit_mode") == "STRICT", \
        f"expected STRICT, got {r.get('audit_mode')}"
    assert "paris" in _answer_lower(r), \
        f"answer missing 'Paris': {_answer_lower(r)[:200]}"


def test_linux_kernel_grounded_and_names_torvalds():
    r = _ask("who wrote the linux kernel?")
    assert _grounded(r), f"audit_mode={r.get('audit_mode')!r}"
    assert "torvalds" in _answer_lower(r), \
        f"answer missing 'Torvalds': {_answer_lower(r)[:200]}"


def test_mona_lisa_strict_and_names_leonardo():
    """Bench standout (2026-04-30, n=3): JSON mode hit S:3 H:0 U:0
    while quote landed 0S and pointer 0S. Tightened from _grounded
    to STRICT to lock in the JSON-mode unique win as a regression
    gate — drop in JSON-mode quality on this question reverts the
    STRICT signal first."""
    r = _ask("who painted the mona lisa?")
    assert r.get("audit_mode") == "STRICT", \
        f"expected STRICT (bench 3/3 STRICT in JSON), got {r.get('audit_mode')}"
    assert "leonardo" in _answer_lower(r), \
        f"answer missing 'Leonardo': {_answer_lower(r)[:200]}"


def test_supermans_girlfriend_strict_and_names_lois_lane():
    """Bench standout (2026-04-30, n=3): JSON mode hit S:3 H:0 U:0
    while pointer landed S:0 H:3 (every grounded answer demoted by
    lazy-anchor or pointer-saturation). The relationship-shape
    question 'who is X's girlfriend' is JSON's strong suit — atomic
    claims, single-name target. Gate: STRICT + 'lois' or 'lane'."""
    r = _ask("who is supermans girlfriend?")
    assert r.get("audit_mode") == "STRICT", \
        f"expected STRICT (bench 3/3 STRICT in JSON), got {r.get('audit_mode')}"
    txt = _answer_lower(r)
    assert "lois" in txt or "lane" in txt, \
        f"answer missing Lois/Lane: {txt[:200]}"


def test_cold_fusion_breakthrough_year_strict_and_names_1989():
    """Bench standout (2026-04-30, n=3): originally classified
    out-of-corpus but JSON mode hit S:3 H:0 U:0 — the corpus has
    Pons & Fleischmann's 1989 announcement covered. Quote mode
    landed 0S 0H 3U (UNGROUNDED) so this fixture also catches
    quote/JSON divergence. Gate: STRICT + a 1989-era marker
    (year, Pons, Fleischmann, or Utah)."""
    r = _ask("what year does our cold fusion breakthrough happen?")
    assert r.get("audit_mode") == "STRICT", \
        f"expected STRICT (bench 3/3 STRICT in JSON), got {r.get('audit_mode')}"
    txt = _answer_lower(r)
    cold_fusion_markers = ("1989", "pons", "fleischmann", "utah")
    assert any(m in txt for m in cold_fusion_markers), \
        f"answer missing 1989-era cold-fusion markers: {txt[:300]}"


# ---------------------------------------------------------------- retrieval disambiguation


def test_red_fish_blue_fish_identifies_seuss_book():
    """Stress-tests retrieval disambiguation: "red fish blue fish" partially
    matches many noise titles (Red Dwarf, Blue whale, Toronto Blue Jays,
    Red Sea, Detroit Red Wings, marine aquarium fish list...). The right
    answer is the Dr. Seuss book ``One Fish Two Fish Red Fish Blue Fish``.
    The model should anchor on the Seuss article and the answer should
    contain a Seuss-specific token.

    Pinned to claim_lattice_pointer mode: JSON mode hits a token-
    budget runaway on "plot of X" prose-summary shapes for this
    question (~2/3 of samples produce malformed JSON with whitespace
    spam after the closing brace, blowing the lenient parser's
    bounds). Pointer mode reliably hits STRICT 1/1 at 4.5s. Until
    JSON-mode max_tokens / prompt discipline addresses the runaway,
    this fixture documents that prose-summary questions are
    pointer-mode's strength.
    """
    r = _ask("plot of red fish blue fish?", mode="claim_lattice_pointer")
    assert _grounded(r), f"audit_mode={r.get('audit_mode')!r}"
    txt = _answer_lower(r)
    seuss_markers = ("seuss", "rhyming", "children's book", "creatures")
    assert any(m in txt for m in seuss_markers), \
        f"answer missing Seuss-book markers: {txt[:300]}"


# ---------------------------------------------------------------- entity disambiguation


def test_veronica_ballestrini_singer_october_birth():
    """Multi-part question against a niche personal-bio article.
    Retrieval surfaces 8 different Veronicas (Landers, Guerin,
    Belmont, Wadley, Gamba, Doran, the disambiguation page, and
    Ballestrini herself). The model should anchor on Ballestrini's
    article and answer both parts: who she is (country singer /
    songwriter / Connecticut) and when she was born (October).
    Tests entity-disambiguation among same-first-name candidates
    plus multi-part-question handling on a small-bio source.
    """
    r = _ask(
        "who is veronica ballestrini when what month was she born?"
    )
    assert _grounded(r), f"audit_mode={r.get('audit_mode')!r}"
    txt = _answer_lower(r)
    assert "ballestrini" in txt, \
        f"answer doesn't even mention 'Ballestrini': {txt[:300]}"
    # The "who is" half — accept any of the canonical descriptors
    # from her bio span.
    bio_markers = ("country", "singer", "songwriter", "connecticut", "italian")
    assert any(m in txt for m in bio_markers), \
        f"answer missing bio markers: {txt[:300]}"
    # The "when was she born" half — accept "october" or the full
    # date "october 29, 1991" or any 1991 mention as evidence the
    # model picked up the birth date span.
    birth_markers = ("october", "1991")
    assert any(m in txt for m in birth_markers), \
        f"answer missing birth markers: {txt[:300]}"


def test_lara_croft_mentions_tomb_raider():
    """Lara Croft (correctly spelled) should pull the Tomb Raider
    article; the answer should mention either the franchise or her
    role/profession."""
    r = _ask("who is lara croft?")
    assert _grounded(r), f"audit_mode={r.get('audit_mode')!r}"
    txt = _answer_lower(r)
    assert (
        "tomb raider" in txt
        or "archaeologist" in txt
        or "video game" in txt
    ), f"answer missing Tomb Raider context: {txt[:200]}"


def test_laura_croft_finds_at_least_one_real_entity():
    """`Laura Croft` (an extra `u`) is BOTH a real Playmate and a
    common misspelling of `Lara Croft`. The system should land on at
    least one real entity from the corpus, not hallucinate a third
    person."""
    r = _ask("who is laura croft?")
    assert _grounded(r), f"audit_mode={r.get('audit_mode')!r}"
    txt = _answer_lower(r)
    # One of: the actual Laura Croft (model) markers, or the Lara
    # Croft (character) markers. Either grounding is honest.
    # Broad marker net to absorb Hermes single-sample variance — if
    # the system grounded at all, ANY of these will land. A failure
    # of all of them implies the answer didn't anchor on a real
    # corpus entity.
    real_entity_markers = (
        "playmate", "playboy", "magazine", "model",   # Playmate context
        "tomb raider", "archaeologist", "video game", # Character context
        "fictional", "character", "british",          # Generic fallback
    )
    assert "croft" in txt, f"answer doesn't even mention 'Croft': {txt[:300]}"
    assert any(m in txt for m in real_entity_markers), \
        f"answer doesn't ground on any known entity: {txt[:300]}"


# ---------------------------------------------------------------- multi-part list


def test_ninja_turtles_lists_all_four_plus_splinter():
    """Multi-part question: all four turtle names + the master's name
    must appear in the answer. Tests that the system handles a
    conjunctive `&` query and produces a complete list."""
    r = _ask("what are the names of the ninja turtles & their master name.")
    assert _grounded(r), f"audit_mode={r.get('audit_mode')!r}"
    txt = _answer_lower(r)
    for turtle in ("leonardo", "donatello", "michelangelo", "raphael"):
        assert turtle in txt, f"missing turtle '{turtle}': {txt[:300]}"
    assert "splinter" in txt, f"missing master 'Splinter': {txt[:300]}"


def test_homer_simpson_boss_is_mr_burns():
    """Cross-document relationship: Homer's boss is named in the Homer
    Simpson article (and several episode articles that reference the
    Springfield Nuclear Power Plant). The model needs to land on
    `Mr. Burns` and cite a chunk from one of the Simpsons articles.

    Used to be pinned to claim_lattice_pointer mode: pre-2026-04-30
    the JSON mode prompt presented content-addressed evidence_ids
    (e.g. ``Eed1b6e396``) and Hermes-3-8B was fabricating
    plausible-looking near-miss IDs (``E1b6e396``) → UNGROUNDED on
    factually correct answers. Now JSON mode uses the same short
    pointer IDs as the pointer variant (E1, E2, …) — the model
    can't fabricate ``E27`` if only ``E1``-``E10`` were shown — so the
    fixture runs against the default (JSON) mode and grounds
    cleanly.
    """
    r = _ask("who is homer simpson's boss?")
    assert _grounded(r), f"audit_mode={r.get('audit_mode')!r}"
    txt = _answer_lower(r)
    assert "burns" in txt, f"answer missing 'Burns': {txt[:300]}"


# ---------------------------------------------------------------- forensic / leading


def test_great_wall_elevation_north_side_higher():
    """Forensic-shape question: which side of the Great Wall has higher
    ground elevation? Expected answer: north (per geographic surveys).
    The corpus may not directly contain the elevation fact, in which
    case Hermes-3-8B can lazy-anchor a "north" claim onto any
    Great-Wall span by token coincidence. The system today returns
    STRICT 1/1 with the right answer ("northern side higher") cited
    to a chunk about "Outer China beyond the Great Wall" — coverage
    threshold passes via topical-token overlap (north / Great Wall /
    China) without the cited span actually containing elevation
    information.

    Until the verifier grows a semantic-frame check, this fixture
    gates on entity-presence only: the answer must mention "north"
    (or "northern" / "northward"). STRICT or HYBRID is fine; the
    model sometimes deflects to other Great-Wall facts.
    """
    r = _ask(
        "which side is the ground elevation highest throughout the "
        "span of the great wall of china, the north or south?"
    )
    txt = _answer_lower(r)
    # Either grounded with a directional answer, OR honestly UNGROUNDED.
    # The fail mode is: grounded WITHOUT mentioning a direction.
    if r.get("audit_mode") == "UNGROUNDED":
        return
    direction_markers = ("north", "northern", "northward")
    assert any(m in txt for m in direction_markers), (
        f"audit={r.get('audit_mode')} but answer doesn't name a "
        f"direction: {txt[:300]}"
    )


# ---------------------------------------------------------------- honest refusal


def test_mars_benevolent_dictator_refuses_or_ungrounded():
    """Mars has no benevolent dictator for life. The system has three
    acceptable behaviors on this adversarial-premise question:

    1. UNGROUNDED (refused outright — no grounded claim emitted)
    2. STRICT/HYBRID + mentions "mars" + includes a refutation marker
       (model explicitly refuted the false premise)
    3. STRICT/HYBRID + does not mention "mars" at all (deflection to
       an adjacent grounded fact — e.g. "Guido van Rossum is BDFL for
       Python" — model declined the false-premise question and
       answered a grounded related one instead)

    The hallucination case (FAIL) is: the answer mentions Mars AND
    affirmatively names a Mars BDFL with no refutation. Empirically
    seen 2026-04-30: claim_lattice mode deflected to Guido/Python
    (option 3) and the verifier returned STRICT correctly — claims
    grounded, premise sidestepped.

    Adversarial shape: "benevolent dictator for life" is a real
    term-of-art (open-source governance — Linus, Guido) so the model
    is tempted to fluently project it onto Mars. The corpus has no
    such anchor; STRICT/HYBRID with an affirmative Mars-BDFL claim
    = fail."""
    r = _ask("who is a benevolent dictator for life for mars?")
    audit = r.get("audit_mode")
    txt = _answer_lower(r)
    # 1) Outright refusal — honest UNGROUNDED.
    if audit == "UNGROUNDED":
        return
    # 3) Deflection — answer is about something else entirely, no
    # Mars mention. This is acceptable: the system grounded its
    # claims (STRICT/HYBRID) but didn't assert the false premise.
    if "mars" not in txt:
        return
    # 2) Explicit refutation — answer mentions Mars but rejects the
    # premise. Markers cover the language patterns we've seen the
    # 8B-class model use on similar no-such-thing questions.
    refutation_markers = (
        "no benevolent dictator",
        "does not have",
        "is not a",
        "no government",
        "not a sovereign",
        "no such",
        "no one",
        "no person",
        "no recognized",
        "no known",
        "fictional",  # accepts answers that frame Mars-BDFL as scifi
    )
    assert any(m in txt for m in refutation_markers), \
        f"audit={audit} but answer asserts a Mars BDFL: {txt[:300]}"