arborist/tests/test_inspect.py

"""Sidecar diagnostic for unverified spans.

`aborist inspect --cache-key X` reads a providence_cache record, pulls
the same source chunks the verifier saw, and classifies each
unverified_quote into one of: verbatim_in_base, verbatim_in_raw_only,
trailing_artifact, paraphrase, partial_paraphrase, no_overlap.

These tests pin the classifier on synthetic contexts so the diagnoses
are deterministic. End-to-end coverage of `inspect_cache_key` (DB
plumbing) goes through a small fixture record below.
"""

from __future__ import annotations

import json
import time
from pathlib import Path

import pytest

from aborist.qa.inspect import _classify_span, _normalize, inspect_cache_key
from aborist.store import append_audit, connect, transaction


# ---------------------------------------------------------------------------
# _classify_span — pure function, no DB
# ---------------------------------------------------------------------------


def _norm(s: str) -> str:
    return _normalize(s)


def test_classify_verbatim_in_base():
    """If the verifier flagged a span but it IS in the base context, the
    diagnosis surfaces that as a likely verifier or canonicalization
    bug. Deliberately distinct label so an operator notices."""
    span = "the quick brown fox"
    base = "lorem ipsum the quick brown fox jumps over"
    raw = base
    out = _classify_span(span, _norm(base), _norm(raw))
    assert out["diagnosis"] == "verbatim_in_base"


def test_classify_verbatim_in_raw_only():
    """Span matches raw wikitext but base form differs — wikitext-strip
    edge case worth flagging separately."""
    span = "[[Cloud Strife]] is the protagonist"
    base = "Cloud Strife is the protagonist"  # wikitext stripped
    raw = "[[Cloud Strife]] is the protagonist"
    out = _classify_span(span, _norm(base), _norm(raw))
    assert out["diagnosis"] == "verbatim_in_raw_only"


def test_classify_trailing_artifact_citation_appended():
    """The Pikachu case — model appended `(Source: ...)` to a verbatim
    sentence; verifier flagged the whole thing. inspect should isolate
    the artifact."""
    base = (
        "Pikachu can store electricity in its cheeks and release it in "
        "lightning-based attacks. Pikachu evolves from Pichu."
    )
    span = (
        "Pikachu can store electricity in its cheeks and release it in "
        "lightning-based attacks. (Source: https://en.wikipedia.org/wiki/Pikachu)"
    )
    out = _classify_span(span, _norm(base), _norm(base))
    assert out["diagnosis"] == "trailing_artifact"
    assert out["matched_prefix_chars"] >= 60
    assert "Source:" in out["trailing_artifact"]


def test_classify_paraphrase_high_token_coverage():
    """Tokens all present, sequence different — model rewrote the source."""
    base = (
        "Pikachu is a Pokémon species in the Pokémon franchise developed by "
        "Game Freak and Nintendo. Pokémon games feature numerous creatures."
    )
    span = "Pikachu is a species of Pokémon creatures from the Pokémon franchise"
    out = _classify_span(span, _norm(base), _norm(base))
    assert out["diagnosis"] == "paraphrase"
    assert out["token_coverage"] >= 0.85


def test_classify_partial_paraphrase_some_tokens_missing():
    """Some content overlap but a meaningful token isn't there at all —
    likely model invention mixed with corpus tokens."""
    base = "Pikachu is a Pokémon. Pikachu battles other creatures."
    span = "Pikachu invented the lightning attack in Tokyo on Tuesday"
    out = _classify_span(span, _norm(base), _norm(base))
    assert out["diagnosis"] in ("partial_paraphrase", "no_overlap")
    if out["diagnosis"] == "partial_paraphrase":
        assert "missing_tokens" in out


def test_classify_no_overlap_full_invention():
    """Almost no content tokens shared with the corpus — pure model output."""
    base = "Pikachu is a Pokémon species."
    span = "The Roman Senate convened in 49 BC to debate Caesar's rebellion"
    out = _classify_span(span, _norm(base), _norm(base))
    assert out["diagnosis"] == "no_overlap"


# ---------------------------------------------------------------------------
# inspect_cache_key — end-to-end with a tiny fixture
# ---------------------------------------------------------------------------


def _seed_record(qa_db: Path, *, cache_key: str, sources: list[dict],
                 unverified: list[str]) -> None:
    """Insert a minimal providence_cache row with a merkle_proof that
    points at the given sources. Caller has already populated each
    shard with the actual document + chunks."""
    conn = connect(qa_db)
    try:
        with transaction(conn):
            event_hash = append_audit(
                conn,
                event_type="providence_query",
                subject_root=cache_key,
                body={"cache_key": cache_key},
            )
            conn.execute(
                "INSERT INTO providence_cache "
                "(cache_key, source_root, document_uri, question_hash, "
                " question_text, answer_text, merkle_proof, "
                " model_profile_hash, conversation_hash, "
                " governance_policy_hash, schema_version, "
                " canonicalization_version, chunking_version, "
                " falsification_state, chain, audit_event_hash, "
                " created_at, hit_count, audit_mode, n_quotes, "
                " n_verified, unverified_quotes, verifier_method) "
                "VALUES (?, ?, 'corpus://multi', ?, ?, ?, ?, ?, ?, ?, "
                " ?, ?, ?, 'live', 'private', ?, ?, 0, 'HYBRID', ?, ?, ?, 'span')",
                (
                    cache_key,
                    "00" * 32,
                    "qh",
                    "what is foo?",
                    "answer here",
                    json.dumps({"sources": sources}, ensure_ascii=False),
                    "mh",
                    "ch",
                    "gh",
                    "v9.8.0",
                    "norm-v1",
                    "tok-512-v1",
                    event_hash,
                    int(time.time()),
                    len(unverified) + 1,  # n_quotes (one verified plus N unverified)
                    1,                     # n_verified
                    json.dumps(unverified, ensure_ascii=False),
                ),
            )
    finally:
        conn.close()


def _seed_doc(shard: Path, *, document_root: str, document_uri: str,
              chunk_text: str) -> None:
    """Insert a documents row + one hot chunk with the given text."""
    conn = connect(shard)
    try:
        from aborist.compress import pack_chunk

        with transaction(conn):
            conn.execute(
                "INSERT INTO documents "
                "(document_root, document_uri, source_type, kind, "
                " compression_depth, chunking_version, "
                " canonicalization_version, schema_version, ingest_ts, hit_count) "
                "VALUES (?, ?, 'html', 'surface', 0, 'tok-512-v1', "
                " 'norm-v1', 'v9.8.0', ?, 0)",
                (document_root, document_uri, int(time.time())),
            )
            conn.execute(
                "INSERT INTO chunks (document_root, idx, leaf_hash, content, tier) "
                "VALUES (?, 0, ?, ?, 'hot')",
                (document_root, "ff" * 32, pack_chunk(chunk_text)),
            )
    finally:
        conn.close()


def test_inspect_cache_key_classifies_each_unverified(tmp_path):
    qa_db = tmp_path / "qa.db"
    shard = tmp_path / "001.db"
    DOC = "ab" * 32
    _seed_doc(
        shard,
        document_root=DOC,
        document_uri="https://example.com/x",
        chunk_text=(
            "Pikachu can store electricity in its cheeks. "
            "Pikachu evolves from Pichu."
        ),
    )
    _seed_record(
        qa_db,
        cache_key="01" * 32,
        sources=[{
            "document_root": DOC,
            "document_uri": "https://example.com/x",
            "title": "X",
            "shard": shard.name,
            "chunk_idx": 0,
        }],
        unverified=[
            "Pikachu can store electricity in its cheeks.",  # verbatim
            "Pikachu fought in the 1988 Olympics in Seoul",   # invention
        ],
    )

    result = inspect_cache_key(
        "01" * 32, qa_db=qa_db, shards_dir=tmp_path
    )
    assert result["status"] == "ok"
    assert result["record"]["cache_key"] == "01" * 32
    assert len(result["sources"]) == 1
    assert result["sources"][0]["chunk_count"] == 1
    assert len(result["unverified"]) == 2
    diags = [d["diagnosis"] for d in result["unverified"]]
    assert "verbatim_in_base" in diags
    # The Olympics span should be flagged as no_overlap or partial_paraphrase
    # depending on token coincidences — both honest classifications.
    assert any(d in ("no_overlap", "partial_paraphrase") for d in diags)


def test_inspect_cache_key_unknown_returns_not_found(tmp_path):
    qa_db = tmp_path / "qa.db"
    # touch DB so connect() initialises schema
    connect(qa_db).close()
    result = inspect_cache_key(
        "00" * 32, qa_db=qa_db, shards_dir=tmp_path
    )
    assert result["status"] == "not_found"


def test_inspect_does_not_mutate_state(tmp_path):
    """Sidecar invariant: inspect must not write to providence_cache,
    audit_events, or anything else. Pin so a future refactor can't
    silently add side effects."""
    qa_db = tmp_path / "qa.db"
    shard = tmp_path / "001.db"
    DOC = "cd" * 32
    _seed_doc(
        shard, document_root=DOC, document_uri="https://example.com/y",
        chunk_text="hello world",
    )
    _seed_record(
        qa_db, cache_key="02" * 32,
        sources=[{
            "document_root": DOC, "document_uri": "https://example.com/y",
            "title": "Y", "shard": shard.name, "chunk_idx": 0,
        }],
        unverified=["something the model said"],
    )

    conn = connect(qa_db)
    try:
        before_audit = conn.execute("SELECT COUNT(*) FROM audit_events").fetchone()[0]
        before_prov = conn.execute("SELECT COUNT(*) FROM providence_cache").fetchone()[0]
    finally:
        conn.close()

    inspect_cache_key("02" * 32, qa_db=qa_db, shards_dir=tmp_path)

    conn = connect(qa_db)
    try:
        after_audit = conn.execute("SELECT COUNT(*) FROM audit_events").fetchone()[0]
        after_prov = conn.execute("SELECT COUNT(*) FROM providence_cache").fetchone()[0]
    finally:
        conn.close()
    assert after_audit == before_audit
    assert after_prov == before_prov