arborist/tests/test_qa_quality_live.py

"""Live QA-quality tests against the real Hermes endpoint.

These tests are deliberately gated — they hit
``hermes.ai.unturf.com/v1`` and require a populated shard set under
``~/.arborist/shards`` (default) or the path passed via
``ARBORIST_LIVE_SHARDS_DIR``. Default `make test` does NOT run these;
opt in via ``ARBORIST_LIVE_TESTS=1`` (the ``make test-live`` target
sets it).

Why this exists:

The unit tests in ``test_qa.py`` / ``test_query.py`` use ``StubClient``
to validate plumbing (cache lookups, document mapping, schema, audit
chain). They cannot tell you if the system answers ``"who founded
Apple Computer?"`` correctly — that's a model+retrieval+verifier
behavior, not a unit-tested code path. The QA-quality bench
(``make bench-qa``) measures aggregate verdict counts but doesn't
assert specific content. These live fixtures sit between those two
concerns: each test runs ONE known-good question, asserts the
audit_mode is at least HYBRID, and checks for entity-level content
the answer should mention.

"Benchmax" rationale: the bench gives us a scoreboard
(STRICT/HYBRID/UNGROUNDED counts); these tests give us *gates*. When
a future change improves the system, the strict-rate climbs in the
bench AND every fixture here keeps passing or gets stricter
assertions. When a change regresses, the bench number falls AND
specific fixtures fail by name, telling you where the regression
landed.

Every assertion accepts a tolerance band — Hermes is non-deterministic
at temperature > 0, so single-sample tests are inherently flaky on
boundary cases. The bar is "the right entity is present in the
answer", not "the answer is byte-identical to last run". When a
fixture starts failing intermittently, treat it as evidence of
quality drift on that question shape and tighten retrieval / verifier
upstream rather than loosen the test.
"""
from __future__ import annotations

import os
from pathlib import Path

import pytest


_LIVE_OPT_IN = os.environ.get("ARBORIST_LIVE_TESTS") == "1"
_SHARDS_DIR = Path(
    os.environ.get("ARBORIST_LIVE_SHARDS_DIR")
    or (Path.home() / ".arborist" / "shards")
)
_QA_DB = _SHARDS_DIR / "qa.db"
_ENDPOINT = os.environ.get(
    "ARBORIST_LLM_ENDPOINT", "https://hermes.ai.unturf.com/v1"
)
_MODEL = os.environ.get(
    "ARBORIST_LLM_MODEL",
    "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic",
)


pytestmark = pytest.mark.skipif(
    not _LIVE_OPT_IN,
    reason="live QA tests gated by ARBORIST_LIVE_TESTS=1; "
           "run via `make test-live` or set the env var",
)


def _ask(question: str, *, mode: str = "claim_lattice", top_k: int = 8) -> dict:
    """One live query against the configured shard set + Hermes endpoint."""
    if not _SHARDS_DIR.exists():
        pytest.skip(f"shards dir not found: {_SHARDS_DIR}")
    if not _QA_DB.parent.exists():
        pytest.skip(f"qa.db parent not found: {_QA_DB.parent}")

    # Lazy import so collecting the file doesn't pull qa modules in
    # default-skip mode.
    from arborist.qa.client import OpenAICompatibleClient
    from arborist.qa.query import DEFAULT_QUERY_POLICY, query

    api_key = os.environ.get("ARBORIST_LLM_API_KEY")
    client = OpenAICompatibleClient(base_url=_ENDPOINT, api_key=api_key)
    policy = dict(DEFAULT_QUERY_POLICY)
    policy["answer_mode"] = mode

    return query(
        question=question,
        qa_db=_QA_DB,
        chat_client=client,
        model_id=_MODEL,
        shards_dir=_SHARDS_DIR,
        top_k=top_k,
        policy=policy,
        burn_existing=True,  # forces fresh inference per test
    )


def _answer_lower(result: dict) -> str:
    return (result.get("answer_text") or "").lower()


def _grounded(result: dict) -> bool:
    """STRICT or HYBRID — at least one pointer/quote pair verified."""
    return result.get("audit_mode") in ("STRICT", "HYBRID")


# ---------------------------------------------------------------- narrow factoid


def test_france_capital_strict_and_names_paris():
    """Capital-of-X is the textbook narrow-factoid; we expect STRICT
    and the literal city name to appear in the cited or rendered span."""
    r = _ask("what is the capital of france?")
    assert r.get("audit_mode") == "STRICT", \
        f"expected STRICT, got {r.get('audit_mode')}"
    assert "paris" in _answer_lower(r), \
        f"answer missing 'Paris': {_answer_lower(r)[:200]}"


def test_linux_kernel_grounded_and_names_torvalds():
    r = _ask("who wrote the linux kernel?")
    assert _grounded(r), f"audit_mode={r.get('audit_mode')!r}"
    assert "torvalds" in _answer_lower(r), \
        f"answer missing 'Torvalds': {_answer_lower(r)[:200]}"


def test_mona_lisa_strict_and_names_leonardo():
    """Bench standout (2026-04-30, n=3): JSON mode hit S:3 H:0 U:0
    while quote landed 0S and pointer 0S. Tightened from _grounded
    to STRICT to lock in the JSON-mode unique win as a regression
    gate — drop in JSON-mode quality on this question reverts the
    STRICT signal first."""
    r = _ask("who painted the mona lisa?")
    assert r.get("audit_mode") == "STRICT", \
        f"expected STRICT (bench 3/3 STRICT in JSON), got {r.get('audit_mode')}"
    assert "leonardo" in _answer_lower(r), \
        f"answer missing 'Leonardo': {_answer_lower(r)[:200]}"


def test_supermans_girlfriend_strict_and_names_lois_lane():
    """Bench standout (2026-04-30, n=3): JSON mode hit S:3 H:0 U:0
    while pointer landed S:0 H:3 (every grounded answer demoted by
    lazy-anchor or pointer-saturation). The relationship-shape
    question 'who is X's girlfriend' is JSON's strong suit — atomic
    claims, single-name target. Gate: STRICT + 'lois' or 'lane'."""
    r = _ask("who is supermans girlfriend?")
    assert r.get("audit_mode") == "STRICT", \
        f"expected STRICT (bench 3/3 STRICT in JSON), got {r.get('audit_mode')}"
    txt = _answer_lower(r)
    assert "lois" in txt or "lane" in txt, \
        f"answer missing Lois/Lane: {txt[:200]}"


def test_cold_fusion_breakthrough_year_strict_and_names_1989():
    """Bench standout (2026-04-30, n=3): originally classified
    out-of-corpus but JSON mode hit S:3 H:0 U:0 — the corpus has
    Pons & Fleischmann's 1989 announcement covered. Quote mode
    landed 0S 0H 3U (UNGROUNDED) so this fixture also catches
    quote/JSON divergence. Gate: STRICT + a 1989-era marker
    (year, Pons, Fleischmann, or Utah)."""
    r = _ask("what year does our cold fusion breakthrough happen?")
    assert r.get("audit_mode") == "STRICT", \
        f"expected STRICT (bench 3/3 STRICT in JSON), got {r.get('audit_mode')}"
    txt = _answer_lower(r)
    cold_fusion_markers = ("1989", "pons", "fleischmann", "utah")
    assert any(m in txt for m in cold_fusion_markers), \
        f"answer missing 1989-era cold-fusion markers: {txt[:300]}"


def test_czechoslovakia_two_capitals():
    """Subtle-factoid: Czechoslovakia had no single capital — Prague
    served the Czech Republic and Bratislava served the Slovak
    Republic. Tests that the model surfaces the dual-capital
    historical fact rather than picking just Prague. Bench live
    (2026-05-01): STRICT 1/1, model wrote 'Czechoslovakia had no
    single capital city. It had two capitals, one in each of the
    two constituent republics: Prague in the Czech Republic and
    Bratislava in the Slovak Republic.' Gate: grounded + at least
    one of the two capital cities present."""
    r = _ask("what is the capital of czechoslovakia")
    assert _grounded(r), f"audit_mode={r.get('audit_mode')!r}"
    txt = _answer_lower(r)
    assert "prague" in txt or "bratislava" in txt, \
        f"answer missing capital city: {txt[:300]}"


# ---------------------------------------------------------------- retrieval disambiguation


def test_transcranial_jargon_hint_finds_tms_and_rtfmri():
    """Multi-domain query with a technical-jargon hint. The bare
    question 'what technology... reconstruct another's thoughts'
    lands HYBRID with an honest 'no technology available' refusal
    — the model can't find the right anchor in retrieval. Adding
    'transcranial knowledge acquisition' as a hint pulls the
    Neurotechnology article into top-K and the model anchors
    cleanly on transcranial magnetic stimulation (TMS) +
    real-time functional MRI. Tests that domain-jargon in the
    question shape disambiguates retrieval to the right corpus
    region. Bench live (2026-05-01): STRICT 1/1."""
    r = _ask(
        "what technology are currently or soon available which "
        "may enable one person to reconstruct and understand some "
        "or a portion of another persons thoughts or ideas without "
        "speaking or sign language. transcranial knowledge acquisition"
    )
    assert _grounded(r), f"audit_mode={r.get('audit_mode')!r}"
    txt = _answer_lower(r)
    tech_markers = (
        "transcranial", "tms", "fmri", "magnetic stimulation",
        "neurotechnology", "brain",
    )
    assert any(m in txt for m in tech_markers), \
        f"answer missing neurotechnology markers: {txt[:300]}"


def test_spin_glass_explanation_grounded():
    """Niche physics topic (single Wikipedia article anchor). Tests
    that retrieval surfaces the actual ``Spin glass`` article above
    the noise of unrelated `glass` articles (Stained glass, Tiffany
    glass, Studio glass, Glass harmonica, etc.). Bench live
    (2026-05-01): STRICT 2/2, model cited the Spin glass article
    spans for both claims (the definition + the Sherrington-
    Kirkpatrick model)."""
    r = _ask("explain spin glass?")
    assert _grounded(r), f"audit_mode={r.get('audit_mode')!r}"
    txt = _answer_lower(r)
    physics_markers = (
        "ferromagnetic", "antiferromagnetic", "magnetic",
        "frustrated", "disorder", "ising", "sherrington",
    )
    assert any(m in txt for m in physics_markers), \
        f"answer missing spin-glass physics markers: {txt[:300]}"


def test_innis_and_gunn_brewery_grounded_despite_typo():
    """Misspelled brand name — question writes 'innus and gunn beer'
    when the corpus article is 'Innis & Gunn'. Tests retrieval
    robustness to common typos / phonetic spellings. The stem-aware
    title matching + multi-token-match purity rerank (landed
    2026-05-01) make this work even when one query token is a
    misspelling. Bench live (2026-04-30): STRICT 1/1, model anchored
    cleanly on the Innis & Gunn brewery article."""
    r = _ask("innus and gunn beer")
    assert _grounded(r), f"audit_mode={r.get('audit_mode')!r}"
    txt = _answer_lower(r)
    brand_markers = ("innis", "gunn", "edinburgh", "scotland", "brewery", "brewing")
    assert any(m in txt for m in brand_markers), \
        f"answer missing brewery markers: {txt[:300]}"


def test_red_fish_blue_fish_identifies_seuss_book():
    """Noise-resistance probe: "red fish blue fish" partially matches
    many noise titles (Red Dwarf, Blue whale, Toronto Blue Jays,
    Red Sea, Detroit Red Wings, marine aquarium fish list, Detroit
    Red Wings, Blue Velvet). The right answer is the Dr. Seuss book
    ``One Fish Two Fish Red Fish Blue Fish``.

    Three contracts gated:

    1. **Content correctness**: answer contains a Seuss-specific
       marker (seuss / rhyming / children's book / creatures).
    2. **Primary anchoring**: retrieval purity sidecar reports
       primary_answer_source at rank 1 AND that source was used
       (`primary_used == True`).
    3. **Noise resistance**: zero noise sources contributed to a
       verified citation (`noise_sources_used == 0`). The model
       saw the distractors but didn't anchor on them.

    Mode pin: claim_lattice_pointer. JSON mode hits a token-budget
    runaway on "plot of X" prose-summary shapes (~2/3 of samples
    produce malformed JSON with whitespace spam after the closing
    brace). Pointer mode reliably hits STRICT 1/1 at 4.5s.
    """
    r = _ask("plot of red fish blue fish?", mode="claim_lattice_pointer")
    txt = _answer_lower(r)
    seuss_markers = ("seuss", "rhyming", "children's book", "creatures")
    has_seuss_markers = any(m in txt for m in seuss_markers)

    # The fixture accepts two outcomes — both prove retrieval landed
    # on the right article:
    #   1. GROUNDED + Seuss markers in answer (model anchored cleanly)
    #   2. UNGROUNDED + Seuss markers in answer (model recognized
    #      the right source but declined to commit a plot claim —
    #      retrieval still found Seuss, model was just pedantic about
    #      "red fish blue fish" not being the canonical title)
    # Failure mode: any audit_mode where the answer DOESN'T mention
    # Seuss at all → retrieval landed somewhere wrong.
    assert has_seuss_markers, (
        f"audit={r.get('audit_mode')} answer missing Seuss markers; "
        f"retrieval probably misfired: {txt[:300]}"
    )

    # Noise-resistance contract — only meaningful when grounded
    # (claim_lattice modes carry the retrieval-purity sidecar; for
    # UNGROUNDED runs no claims exist so noise_sources_used is
    # vacuously 0). Gate when grounded.
    purity = r.get("retrieval_purity")
    if _grounded(r) and purity:
        assert purity.get("primary_rank") == 1, (
            f"primary_answer_source not at rank 1: {purity}"
        )
        assert purity.get("primary_used") is True, (
            f"primary_answer_source surfaced but not cited: {purity}"
        )
        assert purity.get("noise_sources_used", 0) == 0, (
            f"model anchored on noise sources: {purity}"
        )


# ---------------------------------------------------------------- entity disambiguation


def test_veronica_ballestrini_singer_october_birth():
    """Multi-part question against a niche personal-bio article.
    Retrieval surfaces 8 different Veronicas (Landers, Guerin,
    Belmont, Wadley, Gamba, Doran, the disambiguation page, and
    Ballestrini herself). The model should anchor on Ballestrini's
    article and answer both parts: who she is (country singer /
    songwriter / Connecticut) and when she was born (October).
    Tests entity-disambiguation among same-first-name candidates
    plus multi-part-question handling on a small-bio source.
    """
    r = _ask(
        "who is veronica ballestrini when what month was she born?"
    )
    assert _grounded(r), f"audit_mode={r.get('audit_mode')!r}"
    txt = _answer_lower(r)
    assert "ballestrini" in txt, \
        f"answer doesn't even mention 'Ballestrini': {txt[:300]}"
    # The "who is" half — accept any of the canonical descriptors
    # from her bio span.
    bio_markers = ("country", "singer", "songwriter", "connecticut", "italian")
    assert any(m in txt for m in bio_markers), \
        f"answer missing bio markers: {txt[:300]}"
    # The "when was she born" half — accept "october" or the full
    # date "october 29, 1991" or any 1991 mention as evidence the
    # model picked up the birth date span.
    birth_markers = ("october", "1991")
    assert any(m in txt for m in birth_markers), \
        f"answer missing birth markers: {txt[:300]}"


def test_lara_croft_mentions_tomb_raider():
    """Lara Croft (correctly spelled) should pull the Tomb Raider
    article; the answer should mention either the franchise or her
    role/profession."""
    r = _ask("who is lara croft?")
    assert _grounded(r), f"audit_mode={r.get('audit_mode')!r}"
    txt = _answer_lower(r)
    assert (
        "tomb raider" in txt
        or "archaeologist" in txt
        or "video game" in txt
    ), f"answer missing Tomb Raider context: {txt[:200]}"


def test_laura_croft_finds_at_least_one_real_entity():
    """`Laura Croft` (an extra `u`) is BOTH a real Playmate and a
    common misspelling of `Lara Croft`. The system should land on at
    least one real entity from the corpus, not hallucinate a third
    person."""
    r = _ask("who is laura croft?")
    assert _grounded(r), f"audit_mode={r.get('audit_mode')!r}"
    txt = _answer_lower(r)
    # One of: the actual Laura Croft (model) markers, or the Lara
    # Croft (character) markers. Either grounding is honest.
    # Broad marker net to absorb Hermes single-sample variance — if
    # the system grounded at all, ANY of these will land. A failure
    # of all of them implies the answer didn't anchor on a real
    # corpus entity.
    real_entity_markers = (
        "playmate", "playboy", "magazine", "model",   # Playmate context
        "tomb raider", "archaeologist", "video game", # Character context
        "fictional", "character", "british",          # Generic fallback
    )
    assert "croft" in txt, f"answer doesn't even mention 'Croft': {txt[:300]}"
    assert any(m in txt for m in real_entity_markers), \
        f"answer doesn't ground on any known entity: {txt[:300]}"


# ---------------------------------------------------------------- multi-part list


def test_ninja_turtles_lists_all_four_plus_splinter():
    """Multi-part question: all four turtle names + the master's name
    must appear in the answer. Tests that the system handles a
    conjunctive `&` query and produces a complete list."""
    r = _ask("what are the names of the ninja turtles & their master name.")
    assert _grounded(r), f"audit_mode={r.get('audit_mode')!r}"
    txt = _answer_lower(r)
    for turtle in ("leonardo", "donatello", "michelangelo", "raphael"):
        assert turtle in txt, f"missing turtle '{turtle}': {txt[:300]}"
    assert "splinter" in txt, f"missing master 'Splinter': {txt[:300]}"


def test_peter_pans_lost_boys_lists_all_named_members():
    """Stress-tests retrieval disambiguation across 8 Peter-Pan-and-
    Lost-Boys-titled candidates (Lost Boys article, The Lost Boys
    1987 film, The Lost Boys docudrama, Peter and Wendy, Peter Pan
    main, Peter and the Starcatchers, Lost Girls, Peter Pan 1953
    film) PLUS multi-entity list grounding. Bench standout
    (2026-04-30): JSON mode hit STRICT 1/1 anchoring on a single
    Lost-Boys span listing all five named members + their costumes.
    Gate: grounded + ≥4 of the 5 named members present (Slightly,
    Cubby, Nibs, Tootles, Twins) — Hermes occasionally drops one
    name, accept that as long as the list is mostly intact."""
    r = _ask("what are the names of peter pans lost boys?")
    assert _grounded(r), f"audit_mode={r.get('audit_mode')!r}"
    txt = _answer_lower(r)
    members = ("slightly", "cubby", "nibs", "tootles", "twin")
    present = [m for m in members if m in txt]
    assert len(present) >= 4, (
        f"only {len(present)}/5 lost-boy names present "
        f"({present}): {txt[:300]}"
    )


def test_homer_simpson_boss_is_mr_burns():
    """Cross-document relationship: Homer's boss is named in the Homer
    Simpson article (and several episode articles that reference the
    Springfield Nuclear Power Plant). The model needs to land on
    `Mr. Burns` and cite a chunk from one of the Simpsons articles.

    Used to be pinned to claim_lattice_pointer mode: pre-2026-04-30
    the JSON mode prompt presented content-addressed evidence_ids
    (e.g. ``Eed1b6e396``) and Hermes-3-8B was fabricating
    plausible-looking near-miss IDs (``E1b6e396``) → UNGROUNDED on
    factually correct answers. Now JSON mode uses the same short
    pointer IDs as the pointer variant (E1, E2, …) — the model
    can't fabricate ``E27`` if only ``E1``-``E10`` were shown — so the
    fixture runs against the default (JSON) mode and grounds
    cleanly.
    """
    r = _ask("who is homer simpson's boss?")
    assert _grounded(r), f"audit_mode={r.get('audit_mode')!r}"
    txt = _answer_lower(r)
    assert "burns" in txt, f"answer missing 'Burns': {txt[:300]}"


# ---------------------------------------------------------------- forensic / leading


def test_great_wall_elevation_north_side_higher():
    """Forensic-shape question: which side of the Great Wall has higher
    ground elevation? Expected answer: north (per geographic surveys).
    The corpus may not directly contain the elevation fact, in which
    case Hermes-3-8B can lazy-anchor a "north" claim onto any
    Great-Wall span by token coincidence. The system today returns
    STRICT 1/1 with the right answer ("northern side higher") cited
    to a chunk about "Outer China beyond the Great Wall" — coverage
    threshold passes via topical-token overlap (north / Great Wall /
    China) without the cited span actually containing elevation
    information.

    Until the verifier grows a semantic-frame check, this fixture
    gates on entity-presence only: the answer must mention "north"
    (or "northern" / "northward"). STRICT or HYBRID is fine; the
    model sometimes deflects to other Great-Wall facts.
    """
    r = _ask(
        "which side is the ground elevation highest throughout the "
        "span of the great wall of china, the north or south?"
    )
    txt = _answer_lower(r)
    # Either grounded with a directional answer, OR honestly UNGROUNDED.
    # The fail mode is: grounded WITHOUT mentioning a direction.
    if r.get("audit_mode") == "UNGROUNDED":
        return
    direction_markers = ("north", "northern", "northward")
    assert any(m in txt for m in direction_markers), (
        f"audit={r.get('audit_mode')} but answer doesn't name a "
        f"direction: {txt[:300]}"
    )


# ---------------------------------------------------------------- literary reference
# Question pair from B.A. Feldman — tests whether the model
# distinguishes between a leading-question premise (treated as a
# real-world geopolitical claim) versus a recognized literary
# reference (Orwell's 1984). Without an Orwell hint, the model
# should refuse the "always at war" premise. With a "do you
# understand the reference" hint, the model should anchor on the
# Nineteen Eighty-Four article and explain the in-novel context.


def test_oceania_war_eastasia_without_context_refuses_real_world_premise():
    """Question from B.A. Feldman — leading-premise probe.

    Without an Orwell hint, the model should NOT affirm 'Oceania has
    always been at war with East Asia' as if it were a real-world
    geopolitical fact. Acceptable behaviors:

    1. Honest refusal — answer mentions 'no evidence', 'does not
       state', or similar refutation marker.
    2. Unprompted reference recognition — answer mentions Orwell /
       1984 / Nineteen Eighty-Four / dystop* on its own.

    Failure mode: answer affirms the premise as a real conflict
    between real Oceania (Pacific island region) and real East Asia.
    Bench live (2026-04-30): HYBRID 2/16, model wrote 'evidence does
    not directly state that Oceania has always been at war with East
    Asia' — clean honest refusal."""
    r = _ask("has oceania always been at war with east asia")
    txt = _answer_lower(r)
    refusal_markers = (
        "does not", "no evidence", "no mention",
        "not directly", "no record", "not state",
        "no information",
    )
    orwell_markers = ("orwell", "1984", "nineteen eighty", "dystop")
    refuses = any(m in txt for m in refusal_markers)
    recognizes_reference = any(m in txt for m in orwell_markers)
    assert refuses or recognizes_reference, (
        f"model affirmed 'always at war' premise as real-world fact "
        f"without recognizing the 1984 reference: {txt[:300]}"
    )


def test_oceania_war_eastasia_with_reference_hint_recognizes_orwell():
    """Question from B.A. Feldman — reference-recognition probe.

    When the question hints 'do you understand what this reference',
    the model should anchor on Nineteen Eighty-Four and explain the
    Orwell context (Hate Week, historical revisionism, etc.).

    Two acceptable outcomes — both prove retrieval landed on
    Nineteen Eighty-Four:
      1. GROUNDED + Orwell markers in answer (model anchored cleanly)
      2. UNGROUNDED + Orwell markers in answer (model recognized
         the reference but the cited evidence chunk didn't pass the
         coverage threshold — Hermes can write a richer Orwell
         essay than the cited span supports word-for-word)

    Failure mode: any audit_mode where the answer DOESN'T contain
    Orwell / 1984 / Winston / Big Brother / Hate Week markers —
    that's retrieval failing to find the right article entirely.
    """
    r = _ask(
        "has oceania always been at war with east asia? "
        "do you undrestand what this reference"
    )
    txt = _answer_lower(r)
    orwell_markers = (
        "orwell", "1984", "nineteen eighty",
        "dystop", "winston", "big brother",
        "hate week", "ingsoc", "totalitarian",
        "ministry of truth", "eurasia", "eastasia",
    )
    assert any(m in txt for m in orwell_markers), (
        f"audit={r.get('audit_mode')} answer missing Orwell/1984 "
        f"markers: {txt[:300]}"
    )


# ---------------------------------------------------------------- honest refusal


def test_mars_benevolent_dictator_refuses_or_ungrounded():
    """Mars has no benevolent dictator for life. The system has three
    acceptable behaviors on this adversarial-premise question:

    1. UNGROUNDED (refused outright — no grounded claim emitted)
    2. STRICT/HYBRID + mentions "mars" + includes a refutation marker
       (model explicitly refuted the false premise)
    3. STRICT/HYBRID + does not mention "mars" at all (deflection to
       an adjacent grounded fact — e.g. "Guido van Rossum is BDFL for
       Python" — model declined the false-premise question and
       answered a grounded related one instead)

    The hallucination case (FAIL) is: the answer mentions Mars AND
    affirmatively names a Mars BDFL with no refutation. Empirically
    seen 2026-04-30: claim_lattice mode deflected to Guido/Python
    (option 3) and the verifier returned STRICT correctly — claims
    grounded, premise sidestepped.

    Adversarial shape: "benevolent dictator for life" is a real
    term-of-art (open-source governance — Linus, Guido) so the model
    is tempted to fluently project it onto Mars. The corpus has no
    such anchor; STRICT/HYBRID with an affirmative Mars-BDFL claim
    = fail."""
    r = _ask("who is a benevolent dictator for life for mars?")
    audit = r.get("audit_mode")
    txt = _answer_lower(r)
    # 1) Outright refusal — honest UNGROUNDED.
    if audit == "UNGROUNDED":
        return
    # 3) Deflection — answer is about something else entirely, no
    # Mars mention. This is acceptable: the system grounded its
    # claims (STRICT/HYBRID) but didn't assert the false premise.
    if "mars" not in txt:
        return
    # 2) Explicit refutation — answer mentions Mars but rejects the
    # premise. Markers cover the language patterns we've seen the
    # 8B-class model use on similar no-such-thing questions.
    refutation_markers = (
        "no benevolent dictator",
        "does not have",
        "is not a",
        "no government",
        "not a sovereign",
        "no such",
        "no one",
        "no person",
        "no recognized",
        "no known",
        "fictional",  # accepts answers that frame Mars-BDFL as scifi
    )
    assert any(m in txt for m in refutation_markers), \
        f"audit={audit} but answer asserts a Mars BDFL: {txt[:300]}"


def test_boss_baby_post_2010_correctly_ungrounded():
    """The Boss Baby is a 2017 DreamWorks film + a 2010 Marla Frazee
    children's book. The 2010-11 Wikipedia corpus has neither in its
    typical 4-shard slice — the film didn't exist yet, the book had
    just been published & had no Wikipedia article. This is the
    canonical out-of-corpus / training-prior case.

    Honest expected outcome: UNGROUNDED. The model emits training-prior
    info about the film/book (it knows; the model trained on
    post-2017 web), but no evidence chunk verifies the claim, so the
    verifier returns 0/N verified pairs and audit_mode=UNGROUNDED.

    Live observed (2026-05-02 fox bench):
      - 0/2 verified, audit_mode UNGROUNDED
      - Top-K filled with adjacent-token noise (Death of Baby P,
        Cake Boss, Beanie Baby, Baby Huey, etc.)
      - Model emitted "Boss Baby is a 2017 American computer-animated
        comedy film..." as the unverified claim.

    This test gates that behavior. Failure modes the test catches:

    1. STRICT/HYBRID with Boss-Baby content → false-positive grounding
       (model claim verified against unrelated chunk that happened to
        share content tokens — the kind of failure warrant-lite was
        built to catch).
    2. STRICT/HYBRID without Boss-Baby content (a deflection like the
       Mars-BDFL Guido pattern) is acceptable — the model declined the
       false-corpus question and answered something else grounded.

    Acceptable behaviors:
      A. UNGROUNDED (verifier honestly admits no grounding)
      B. STRICT/HYBRID without "boss baby" anywhere in the answer
         (deflection — model answered an adjacent grounded question)
    """
    r = _ask("what is boss baby?")
    audit = r.get("audit_mode")
    txt = _answer_lower(r)

    # Path A: honest UNGROUNDED — corpus-truth wins over model prior.
    if audit == "UNGROUNDED":
        return

    # Path B: deflection — model declined the false-corpus question
    # & answered something else (e.g. a Boss Hogg / Big Boss / Hugo Boss
    # adjacent grounded fact). No "boss baby" in answer means the
    # model didn't assert the post-2010 content.
    if "boss baby" not in txt:
        return

    # Anything else is a false-positive grounding: model named the
    # 2017 film or 2010 book AND the verifier said it was grounded.
    # That's the warrant-failure-class this test guards against.
    pytest.fail(
        f"audit={audit} affirmed Boss Baby content with grounding — "
        f"the corpus has no Boss Baby coverage at this snapshot. "
        f"Either the verifier accepted a citation that doesn't actually "
        f"name Boss Baby (warrant gap), or the corpus has been re-ingested "
        f"with newer content (update test). Answer head: {txt[:300]}"
    )


# ----------------------------------------------------- river/rainforest/deflection family
#
# Three queries from the same topical area exercising different
# verifier paths. Together they gate the river↔rainforest topic-shift
# class fox surfaced 2026-05-02:


def _violation_kinds(result: dict) -> set[str]:
    """Extract violation kinds from a query() result. Empty if cache-hit
    (the providence_cache schema doesn't restore violations) — tests
    that run with BURN=1 / cache_miss_then_written see them."""
    return {v.get("kind") for v in (result.get("violations") or []) if v.get("kind")}


def test_amazon_river_burning_who_deflects_or_grounds_river():
    """Question: "who burns the amazon river?"

    The corpus has Deforestation-of-the-Amazon-Rainforest content but
    no "amazon river burning" content (rivers don't burn). Two
    acceptable behaviors:

    1. The model deflects to rainforest deforestation (saw this live
       2026-05-02: cited Deforestation-of-the-Amazon-Rainforest E10).
       In that case the answer omits "river" — DEFLECTION_DETECTED
       soft-demote violation should fire & the verdict should not
       be EVIDENCE-WARRANTED (it should drop to ANCHOR-WARRANTED via
       the soft-demote path).
    2. The model grounds with "river" actually in the answer (e.g.
       refutes "rivers don't burn" + cites real rainforest content).
       Then EVIDENCE-WARRANTED is fine.

    Failure: EVIDENCE-WARRANTED (no DEFLECTION_DETECTED) AND no
    "river" in the answer — the silent topic shift fox flagged.
    """
    r = _ask("who burns the amazon river?")
    audit = r.get("audit_mode")
    txt = _answer_lower(r)
    kinds = _violation_kinds(r)

    # Path 1: deflection caught → soft-demote violation present.
    if "DEFLECTION_DETECTED" in kinds:
        return
    # Path 1 alt: model refused honestly → UNGROUNDED.
    if audit == "UNGROUNDED":
        return
    # Path 2: model kept the river framing → "river" in answer.
    if "river" in txt:
        return

    pytest.fail(
        f"audit={audit} grounded an answer about Amazon Rainforest "
        f"deforestation for a question about the Amazon RIVER without "
        f"naming 'river' — the silent topic shift the deflection "
        f"check is supposed to catch. violations={sorted(kinds)} "
        f"answer head: {txt[:300]}"
    )


def test_amazon_river_burning_culture_refused_or_ungrounded():
    """Question: "what culture burns the amazon river?"

    Adversarial-impossible premise (rivers don't burn). Mirrors the
    Mars-BDFL pattern — model should refuse OR the verifier should
    return UNGROUNDED.

    Live observed (2026-05-02): UNGROUNDED 0/1, model wrote "The
    Amazon River is not burned by any culture..."

    Failure: STRICT/HYBRID with an affirmative claim that some culture
    burns the river.
    """
    r = _ask("what culture burns the amazon river?")
    audit = r.get("audit_mode")
    txt = _answer_lower(r)
    kinds = _violation_kinds(r)

    # 1) Honest UNGROUNDED — verifier refused to ground a corpus-impossible claim.
    if audit == "UNGROUNDED":
        return
    # 2) Deflection caught — model may have said something tangential
    # that triggered the deflection signal.
    if "DEFLECTION_DETECTED" in kinds:
        return
    # 3) Refutation — model named the false premise.
    refutation_markers = (
        "not burned",
        "do not burn",
        "rivers do not",
        "river is not",
        "not a culture",
        "no culture",
        "natural river",
        "is not the same",
    )
    if any(m in txt for m in refutation_markers):
        return

    pytest.fail(
        f"audit={audit} affirmed a 'culture burns the amazon river' "
        f"claim with grounding — adversarial-impossible premise must "
        f"refuse, deflect, or refute. violations={sorted(kinds)} "
        f"answer head: {txt[:300]}"
    )


def test_amazon_rainforest_burning_culture_grounds_or_deflection_caught():
    """Question: "what culture burns the amazon rain forest?"

    The corpus has rainforest deforestation content but no specific
    "culture" framing (the Wikipedia article frames it as "human
    settlement / agricultural practices / soy farmers"). Two
    acceptable behaviors:

    1. Model emits the deforestation-driver framing without naming
       a culture. Subject-anchor "forest" is present in the answer
       (in "deforestation"). Deflection sidecar may or may not
       trigger — answer doesn't echo "culture" but the heuristic
       uses the LAST content token of the question. If
       DEFLECTION_DETECTED fires, it's a soft-demote and ladder
       drops to ANCHOR-WARRANTED.
    2. Model names a culture/people group (e.g. cattle ranchers,
       soy farmers) and grounds in cited spans. EVIDENCE-WARRANTED
       is fine.

    Failure: STRICT/HYBRID with a claim affirmatively naming a
    culture that doesn't appear in any cited span. The warrant
    layer (proper-noun anchor) should already catch this.
    """
    r = _ask("what culture burns the amazon rain forest?")
    audit = r.get("audit_mode")
    txt = _answer_lower(r)
    kinds = _violation_kinds(r)

    # Any of: ungrounded, warrant-missing, title-mismatch, deflection-detected
    # → the verifier caught a structural problem.
    SOFT_KINDS = {
        "DEFLECTION_DETECTED",
        "WARRANT_MISSING",
        "TITLE_MISMATCH",
        "LAZY_ANCHOR_DEMOTED",
    }
    if audit == "UNGROUNDED":
        return
    if kinds & SOFT_KINDS:
        return
    # Otherwise we expect the answer to reference rainforest content
    # honestly — at minimum the topic words.
    if "amazon" in txt and ("forest" in txt or "rainforest" in txt):
        return

    pytest.fail(
        f"audit={audit} grounded a 'culture burns rainforest' claim "
        f"with no Amazon/rainforest content and no soft-demote "
        f"violation — neither honest grounding nor a caught structural "
        f"problem. violations={sorted(kinds)} answer head: {txt[:300]}"
    )