arborist/tests/test_bench_qa_sweep.py

"""Bench-side aggregation tests for capacity metrics + deflection rate.

The bench harness in ``bench/qa_sweep.py`` is the analytics surface for
QA-quality work — it consumes per-row ``prompt_chars_*`` / ``answer_chars``
fields from the query result and renders a markdown summary including
strict-rate-by-size buckets. These tests pin the aggregation behavior on
synthetic rows so the rendering doesn't drift silently.
"""
from __future__ import annotations

import importlib.util
import sys
from pathlib import Path

import pytest


@pytest.fixture(scope="module")
def qa_sweep():
    """Import bench/qa_sweep.py as a module despite living outside the
    aborist package. Avoids polluting the package or requiring a
    pip-install of the bench harness."""
    bench_path = Path(__file__).parent.parent / "bench" / "qa_sweep.py"
    spec = importlib.util.spec_from_file_location("qa_sweep_under_test", bench_path)
    mod = importlib.util.module_from_spec(spec)
    sys.modules[spec.name] = mod
    spec.loader.exec_module(mod)
    return mod


def _row(**overrides) -> dict:
    """Synthetic bench-row fixture with sensible defaults for every
    field the summarizer/renderer reads."""
    base = {
        "question": "q",
        "answer_mode": "claim_lattice",
        "status": "cache_miss_then_written",
        "audit_mode": "STRICT",
        "n_quotes": 1,
        "n_verified": 1,
        "ratio": 1.0,
        "verifier_method": "claim_lattice",
        "lookup_path": "miss",
        "failure_stage": None,
        "lazy_anchor_ratio": None,
        "pointer_id_distribution": None,
        "cache_key": "abc123",
        "n_sources": 1,
        "elapsed_s": 1.0,
        "deflection_kind": "on_topic",
        "subject_anchor": "x",
        "subject_in_answer": True,
        "prompt_chars_total": 5000,
        "prompt_chars_evidence": 4000,
        "prompt_chars_system": 800,
        "prompt_chars_question": 30,
        "answer_chars": 100,
        "directive_compliance": {
            "D2_pointer_clauses": True,
            "D3_cti_substrate_ready": True,
            "D4_evidence_map_bound": True,
            "D6_warrant_fired": True,
            "D7_honest_label": True,
        },
        "error": None,
    }
    base.update(overrides)
    return base


def test_summarize_counts_verdicts_by_mode(qa_sweep):
    rows = [
        _row(answer_mode="quote", audit_mode="STRICT"),
        _row(answer_mode="quote", audit_mode="HYBRID"),
        _row(answer_mode="claim_lattice", audit_mode="STRICT"),
        _row(answer_mode="claim_lattice", audit_mode="UNGROUNDED"),
    ]
    summary = qa_sweep._summarize(rows)
    assert summary["quote"]["STRICT"] == 1
    assert summary["quote"]["HYBRID"] == 1
    assert summary["claim_lattice"]["STRICT"] == 1
    assert summary["claim_lattice"]["UNGROUNDED"] == 1


def test_summarize_counts_deflections_only_on_grounded_rows(qa_sweep):
    """A deflection_kind=='deflection' on UNGROUNDED isn't a real
    topic shift (no answer to deflect with). Only STRICT/HYBRID rows
    with deflection_kind='deflection' should count toward the
    deflections column."""
    rows = [
        _row(answer_mode="claim_lattice", audit_mode="STRICT", deflection_kind="deflection"),
        _row(answer_mode="claim_lattice", audit_mode="HYBRID", deflection_kind="deflection"),
        _row(answer_mode="claim_lattice", audit_mode="UNGROUNDED", deflection_kind="deflection"),
        _row(answer_mode="claim_lattice", audit_mode="STRICT", deflection_kind="on_topic"),
    ]
    summary = qa_sweep._summarize(rows)
    # 2 grounded deflections (STRICT + HYBRID); UNGROUNDED + on_topic don't count.
    assert summary["claim_lattice"]["deflections"] == 2


def test_render_markdown_includes_summary_table_with_capacity_metrics(qa_sweep):
    rows = [
        _row(answer_mode="claim_lattice", audit_mode="STRICT", prompt_chars_total=5000),
        _row(answer_mode="claim_lattice", audit_mode="HYBRID", prompt_chars_total=10000),
    ]
    summary = qa_sweep._summarize(rows)
    md = qa_sweep._render_markdown(
        rows, summary, "2026-05-01T00-00-00Z",
        ["claim_lattice"], ["q"], n_samples=2,
    )
    # Headline table has the deflections column (added before capacity work).
    assert "deflections" in md
    assert "strict-rate" in md
    # Strict-rate-by-prompt-size section exists.
    assert "strict-rate by prompt size" in md
    # Buckets are present in the markdown body.
    assert "<8KB" in md
    assert "8-16KB" in md


def test_render_markdown_buckets_strict_rate_by_size(qa_sweep):
    """Two STRICT rows in <8KB bucket + two UNGROUNDED rows in
    8-16KB bucket → <8KB strict-rate=1.0, 8-16KB strict-rate=0.0."""
    rows = [
        _row(answer_mode="claim_lattice", audit_mode="STRICT", prompt_chars_total=4000),
        _row(answer_mode="claim_lattice", audit_mode="STRICT", prompt_chars_total=5000),
        _row(answer_mode="claim_lattice", audit_mode="UNGROUNDED", prompt_chars_total=10000),
        _row(answer_mode="claim_lattice", audit_mode="UNGROUNDED", prompt_chars_total=12000),
    ]
    summary = qa_sweep._summarize(rows)
    md = qa_sweep._render_markdown(
        rows, summary, "2026-05-01T00-00-00Z",
        ["claim_lattice"], ["q"], n_samples=4,
    )
    # Find the size-bucket section; verify both buckets show their
    # expected strict-rate. Markdown table cell format is `| value |`.
    bucket_section = md.split("strict-rate by prompt size")[1]
    # <8KB row: 2 runs, 2 STRICT, rate 1.00
    lines_8kb = [
        L for L in bucket_section.splitlines()
        if "<8KB" in L and "claim_lattice" in L
    ]
    assert lines_8kb, f"missing <8KB row in:\n{bucket_section[:600]}"
    assert " 1.00 " in lines_8kb[0]
    # 8-16KB row: 2 runs, 0 STRICT, rate 0.00
    lines_16kb = [
        L for L in bucket_section.splitlines()
        if "8-16KB" in L and "claim_lattice" in L
    ]
    assert lines_16kb, f"missing 8-16KB row in:\n{bucket_section[:600]}"
    assert " 0.00 " in lines_16kb[0]


def test_render_markdown_recommended_context_budget_section(qa_sweep):
    """Bench surfaces a per-mode peak-strict-rate bucket as the
    'recommended context budget' — operator-readable, not auto-
    applied. Minimum sample size of 5 runs/bucket; smaller samples
    fall through as 'insufficient samples'."""
    # 6 STRICT in <8KB bucket + 6 UNGROUNDED in 16-32KB bucket
    # → <8KB has strict-rate 1.0, 16-32KB has 0.0; <8KB recommended.
    rows = []
    for _ in range(6):
        rows.append(_row(
            answer_mode="claim_lattice",
            audit_mode="STRICT",
            prompt_chars_total=4000,
        ))
    for _ in range(6):
        rows.append(_row(
            answer_mode="claim_lattice",
            audit_mode="UNGROUNDED",
            prompt_chars_total=20000,
        ))
    summary = qa_sweep._summarize(rows)
    md = qa_sweep._render_markdown(
        rows, summary, "2026-05-02T00-00-00Z",
        ["claim_lattice"], ["q"], n_samples=12,
    )
    assert "recommended context budget (learned from this bench)" in md
    rec_section = md.split("recommended context budget")[1]
    # Peak bucket is <8KB at 100%.
    assert "<8KB" in rec_section
    assert "1.00" in rec_section
    # 16-32KB bucket (0% strict) does NOT show as the peak.
    rec_table_lines = [
        L for L in rec_section.splitlines()
        if "claim_lattice" in L and "<8KB" in L
    ]
    assert rec_table_lines, f"missing recommended-bucket row:\n{rec_section[:500]}"


def test_render_markdown_recommended_budget_skips_undersampled(qa_sweep):
    """Buckets with fewer than 5 runs don't qualify as
    'recommended' — the recommendation needs statistical weight."""
    # 3 STRICT in <8KB (under sample-size floor) + 3 STRICT in
    # 16-32KB (also under floor). All buckets have <5 runs →
    # 'insufficient samples' fallback.
    rows = [
        _row(answer_mode="claim_lattice", audit_mode="STRICT", prompt_chars_total=4000),
        _row(answer_mode="claim_lattice", audit_mode="STRICT", prompt_chars_total=4000),
        _row(answer_mode="claim_lattice", audit_mode="STRICT", prompt_chars_total=4000),
        _row(answer_mode="claim_lattice", audit_mode="UNGROUNDED", prompt_chars_total=20000),
    ]
    summary = qa_sweep._summarize(rows)
    md = qa_sweep._render_markdown(
        rows, summary, "2026-05-02T00-00-00Z",
        ["claim_lattice"], ["q"], n_samples=4,
    )
    rec_section = md.split("recommended context budget")[1]
    assert "insufficient samples" in rec_section


def test_render_markdown_log_scale_buckets_cover_giant_context(qa_sweep):
    """Log-scale buckets reach 1M+ chars so the same bench harness
    covers 8B-class models (Hermes 82K context) through 1M-context
    models (Gemini / Claude). A 500K-char prompt lands in the
    512K-1M bucket without code change."""
    rows = []
    for _ in range(6):
        rows.append(_row(
            answer_mode="claim_lattice",
            audit_mode="STRICT",
            prompt_chars_total=600_000,  # 600KB → 512K-1M bucket
        ))
    summary = qa_sweep._summarize(rows)
    md = qa_sweep._render_markdown(
        rows, summary, "2026-05-02T00-00-00Z",
        ["claim_lattice"], ["q"], n_samples=6,
    )
    assert "512K-1M" in md
    rec_section = md.split("recommended context budget")[1]
    # 600K-char prompts land in the giant bucket; recommendation
    # should pick it.
    assert "512K-1M" in rec_section


def test_render_markdown_skips_empty_buckets(qa_sweep):
    """Bucket with zero runs in a mode shouldn't render an empty row
    — the table should only include cells with data."""
    rows = [
        _row(answer_mode="claim_lattice", audit_mode="STRICT", prompt_chars_total=5000),
    ]
    summary = qa_sweep._summarize(rows)
    md = qa_sweep._render_markdown(
        rows, summary, "2026-05-01T00-00-00Z",
        ["claim_lattice"], ["q"], n_samples=1,
    )
    bucket_section = md.split("strict-rate by prompt size")[1]
    # Only <8KB has data; >=64KB / 32-64KB / 16-32KB / 8-16KB rows
    # should be absent.
    for missing_label in (">=64KB", "32-64KB", "16-32KB", "8-16KB"):
        assert missing_label not in bucket_section, (
            f"empty bucket {missing_label} leaked into rendering:\n"
            f"{bucket_section[:600]}"
        )


# ---------------------------------------------------------------------------
# directive coverage (seven-point program)
# ---------------------------------------------------------------------------


def test_summarize_aggregates_directive_pass_counts_per_mode(qa_sweep):
    """Per-row directive_compliance booleans aggregate into per-mode
    pass counts. A 2-row sample with one D2 fail should report 1/2."""
    rows = [
        _row(
            answer_mode="claim_lattice",
            directive_compliance={
                "D2_pointer_clauses": True,
                "D3_cti_substrate_ready": True,
                "D4_evidence_map_bound": True,
                "D6_warrant_fired": True,
                "D7_honest_label": True,
            },
        ),
        _row(
            answer_mode="claim_lattice",
            directive_compliance={
                "D2_pointer_clauses": False,    # the failing row
                "D3_cti_substrate_ready": True,
                "D4_evidence_map_bound": True,
                "D6_warrant_fired": True,
                "D7_honest_label": True,
            },
        ),
    ]
    summary = qa_sweep._summarize(rows)
    dp = summary["claim_lattice"]["directive_pass"]
    assert dp["D2_pointer_clauses"] == 1
    assert dp["D3_cti_substrate_ready"] == 2
    assert dp["D4_evidence_map_bound"] == 2
    assert dp["D6_warrant_fired"] == 2
    assert dp["D7_honest_label"] == 2


def test_render_markdown_directive_coverage_section(qa_sweep):
    """The markdown summary includes a 'directive coverage' table
    showing per-mode pass counts per directive."""
    rows = [_row(answer_mode="claim_lattice")]
    summary = qa_sweep._summarize(rows)
    md = qa_sweep._render_markdown(
        rows, summary, "2026-05-01T00-00-00Z",
        ["claim_lattice"], ["q"], n_samples=1,
    )
    assert "directive coverage (seven-point program)" in md
    # Header row contains the directive abbreviations.
    assert "D2 pointer" in md
    assert "D3 cti-ready" in md
    assert "D4 ev-map bound" in md
    assert "D6 warrant" in md
    assert "D7 honest label" in md
    # Body cell shows fraction (1/1) for the synthetic row.
    coverage_section = md.split("directive coverage")[1]
    # Per-mode row with 100% coverage on each directive.
    assert "1/1 (100%)" in coverage_section


def test_directive_compliance_helper_marks_quote_mode_d2_false(qa_sweep):
    """Quote-mode rows fail D2 by definition (D2 demands lattice
    answer mode)."""
    result = {
        "audit_mode": "STRICT",
        "verifier_method": "quote",
        "run_dag_root": None,  # quote mode predates the run-DAG split
    }
    dc = qa_sweep._directive_compliance(
        answer_mode="quote",
        result=result,
        err=None,
    )
    assert dc["D2_pointer_clauses"] is False
    assert dc["D3_cti_substrate_ready"] is False  # not lattice mode
    assert dc["D4_evidence_map_bound"] is False   # no run_dag_root
    assert dc["D6_warrant_fired"] is False
    # D7: quote-mode STRICT keeps the STRICT label; the four-rung
    # ladder (POINTER-LINKED / ANCHOR-WARRANTED / EVIDENCE-WARRANTED)
    # applies to lattice modes only. Quote mode passes D7 vacuously.
    assert dc["D7_honest_label"] is True


def test_directive_compliance_helper_marks_lattice_mode_d2_true(qa_sweep):
    """Lattice-mode rows pass D2."""
    result = {
        "audit_mode": "STRICT",
        "verifier_method": "claim_lattice",
        "run_dag_root": "abc" * 21,
    }
    dc = qa_sweep._directive_compliance(
        answer_mode="claim_lattice",
        result=result,
        err=None,
    )
    assert dc["D2_pointer_clauses"] is True
    assert dc["D3_cti_substrate_ready"] is True
    assert dc["D4_evidence_map_bound"] is True
    assert dc["D6_warrant_fired"] is True
    assert dc["D7_honest_label"] is True


def test_directive_compliance_returns_empty_on_error_row(qa_sweep):
    """Error rows don't carry directive signal — no per-row check
    can succeed when the run threw."""
    dc = qa_sweep._directive_compliance(
        answer_mode="claim_lattice",
        result={},
        err="some error",
    )
    assert dc == {}