arborist/tests/test_witness_to_5f.py

"""Tests for the witness-divergence → 5F Falsification extractor.

Two streams: the extractor logic itself + the audit-event write
in query.py that feeds it.
"""

from __future__ import annotations

import json
from pathlib import Path

import pytest

from arborist.qa.client import StubClient
from arborist.qa.query import DEFAULT_QUERY_POLICY, query
from arborist.store import connect


# ----- audit-event write (#000028 follow-up) -----------------------------


def test_witness_event_written_to_audit_chain(tmp_path: Path):
    """When witness fires, query() appends a
    providence_canonical_witness audit event with the question +
    canonical + LLM raw answer + agreement label."""
    qa_db = tmp_path / "qa.db"
    policy = dict(DEFAULT_QUERY_POLICY)
    policy["canonical_witness_enabled"] = True
    client = StubClient(answer="3/10")
    query(
        question="0.1 + 0.2", qa_db=qa_db,
        chat_client=client, model_id="stub", policy=policy,
    )
    conn = connect(qa_db)
    try:
        rows = conn.execute(
            "SELECT body FROM audit_events "
            "WHERE event_type = 'providence_canonical_witness'"
        ).fetchall()
    finally:
        conn.close()
    assert len(rows) == 1
    body = json.loads(rows[0]["body"])
    assert body["pi_star_ref"] == "arithmetic@v1"
    assert body["question_text"] == "0.1 + 0.2"
    assert body["canonical_answer_text"] == "3/10"
    assert body["llm_raw_text"] == "3/10"
    # Agreement: KERNEL-LLM-AGREE (no cache yet on first call).
    assert body["agreement_label"] == "KERNEL-LLM-AGREE"


def test_witness_event_records_divergence(tmp_path: Path):
    """LLM emits a wrong canonical-shape answer; the audit event
    captures the divergence label + raw text for the extractor."""
    qa_db = tmp_path / "qa.db"
    policy = dict(DEFAULT_QUERY_POLICY)
    policy["canonical_witness_enabled"] = True
    client = StubClient(answer="0.4")  # 0.4 → 2/5 ≠ kernel's 3/10
    query(
        question="0.1 + 0.2", qa_db=qa_db,
        chat_client=client, model_id="stub", policy=policy,
    )
    conn = connect(qa_db)
    try:
        body = json.loads(conn.execute(
            "SELECT body FROM audit_events "
            "WHERE event_type = 'providence_canonical_witness' "
            "ORDER BY seq DESC LIMIT 1"
        ).fetchone()["body"])
    finally:
        conn.close()
    assert body["agreement_label"] == "KERNEL-LLM-DIVERGED"
    assert body["llm_raw_text"] == "0.4"
    assert body["canonical_answer_text"] == "3/10"


def test_witness_no_event_when_disabled(tmp_path: Path):
    """Default-off witness path → no providence_canonical_witness
    event landed."""
    qa_db = tmp_path / "qa.db"
    client = StubClient(answer="<should-not-be-called>")
    query(
        question="0.1 + 0.2", qa_db=qa_db,
        chat_client=client, model_id="stub",
    )
    conn = connect(qa_db)
    try:
        n = conn.execute(
            "SELECT COUNT(*) FROM audit_events "
            "WHERE event_type = 'providence_canonical_witness'"
        ).fetchone()[0]
    finally:
        conn.close()
    assert n == 0


# ----- extractor (witness_to_5f.py) --------------------------------------


def _seed_witness_events(qa_db: Path, events: list[dict]) -> None:
    """Helper: fire `query` with synthetic LLM answers to build a
    qa.db carrying a known set of witness audit events."""
    policy = dict(DEFAULT_QUERY_POLICY)
    policy["canonical_witness_enabled"] = True
    for ev in events:
        client = StubClient(answer=ev["llm_answer"])
        query(
            question=ev["question"], qa_db=qa_db,
            chat_client=client, model_id="stub", policy=policy,
        )


def test_extract_divergence_fixtures_basic(tmp_path: Path):
    from bench.scripts.witness_to_5f import extract_divergence_fixtures
    qa_db = tmp_path / "qa.db"
    # Seed: 1 agreement, 2 divergences. Only the divergences come back.
    _seed_witness_events(qa_db, [
        {"question": "0.1 + 0.2", "llm_answer": "3/10"},   # agree
        {"question": "1 + 1",     "llm_answer": "3"},       # diverged
        {"question": "2 + 2",     "llm_answer": "5"},       # diverged
    ])
    fixtures = extract_divergence_fixtures(qa_db)
    assert len(fixtures) == 2
    questions = sorted(
        f["context"].split("\nquestion=")[1] for f in fixtures
    )
    assert questions == ["1 + 1", "2 + 2"]


def test_extract_skips_unparseable_llm_text(tmp_path: Path):
    """LLM emits prose rather than canonical-shape answer; that's
    KERNEL-ONLY (LLM modality absent), not divergence."""
    from bench.scripts.witness_to_5f import extract_divergence_fixtures
    qa_db = tmp_path / "qa.db"
    _seed_witness_events(qa_db, [
        {"question": "0.1 + 0.2", "llm_answer": "the answer is around 0.3"},
    ])
    fixtures = extract_divergence_fixtures(qa_db)
    # KERNEL-ONLY isn't in _DIVERGENCE_LABELS — skipped.
    assert len(fixtures) == 0


def test_extract_fixture_schema_matches_5f_falsification(tmp_path: Path):
    """The emitted fixture dict has the same fields the existing
    5F-falsification-live runner expects."""
    from bench.scripts.witness_to_5f import extract_divergence_fixtures
    qa_db = tmp_path / "qa.db"
    _seed_witness_events(qa_db, [
        {"question": "1 + 1", "llm_answer": "3"},
    ])
    fixtures = extract_divergence_fixtures(qa_db)
    assert len(fixtures) == 1
    fx = fixtures[0]
    # Required by the 5F-falsification schema.
    for required in (
        "id", "battery", "sub_battery", "version", "carrier",
        "domain", "pi_star_ref", "answer_text", "context",
        "expected_reason", "verifier_method_root", "expected",
    ):
        assert required in fx, f"missing field {required!r}"
    assert fx["battery"] == "5f"
    assert fx["sub_battery"] == "falsification"
    assert fx["version"] == "v1"
    assert fx["carrier"] == "providence_record"
    assert fx["expected_reason"] == "UNGROUNDED"
    assert fx["expected"] == "pass"
    # Witness traceability metadata stays out of the runner's path
    # (underscored field) but available for inspection.
    assert "_witness_meta" in fx
    assert fx["_witness_meta"]["agreement_label"] == "KERNEL-LLM-DIVERGED"


def test_extract_idempotent_on_same_db(tmp_path: Path):
    """Running the extractor twice against the same qa.db produces
    byte-equal fixtures (sorted by audit-event seq)."""
    from bench.scripts.witness_to_5f import (
        extract_divergence_fixtures,
    )
    qa_db = tmp_path / "qa.db"
    _seed_witness_events(qa_db, [
        {"question": "1 + 1", "llm_answer": "3"},
        {"question": "2 + 2", "llm_answer": "5"},
    ])
    a = extract_divergence_fixtures(qa_db)
    b = extract_divergence_fixtures(qa_db)
    assert a == b


def test_write_fixtures_creates_5f_jsonl_meta(tmp_path: Path):
    """The output JSONL starts with a meta header naming the
    battery + sub_battery + task_count, matching the rest of the
    bench/fixtures/5f/*-v1.jsonl files."""
    from bench.scripts.witness_to_5f import (
        extract_divergence_fixtures, write_fixtures,
    )
    qa_db = tmp_path / "qa.db"
    out = tmp_path / "out.jsonl"
    _seed_witness_events(qa_db, [
        {"question": "1 + 1", "llm_answer": "3"},
    ])
    fixtures = extract_divergence_fixtures(qa_db)
    write_fixtures(out, fixtures)
    lines = out.read_text(encoding="utf-8").splitlines()
    meta = json.loads(lines[0])["_meta"]
    assert meta["battery"] == "5f"
    assert meta["sub_battery"] == "falsification"
    assert meta["task_count"] == 1
    # Subsequent lines are valid fixture JSONs.
    fx = json.loads(lines[1])
    assert fx["id"].startswith("5f-fal-witness-")