bench: per-row directive_compliance + 'directive coverage' summary section

Wires the seven-point program (docs/seven-point-program.md) into
the bench harness as a per-mode coverage table.

Per-row computation in _directive_compliance(answer_mode, result, err):
  D2_pointer_clauses        — answer_mode is lattice variant
  D3_cti_substrate_ready    — lattice mode + run_dag_root populated
                              (full coverage pending ticket #000002)
  D4_evidence_map_bound     — run_dag_root populated
                              (retrieval-plan binding pending #000001)
  D6_warrant_fired          — warrant ran (lattice mode today;
                              per-shape gating pending #000003)
  D7_honest_label           — audit_mode in canonical enum
                              (renderer transformation pinned in
                              test_cli_render.py)

D1 (no LLM in verifier), D5 (verifier_method enum), D8 (test-pinning
discipline) are global properties of the substrate and don't appear
per-row — they get tracked once in CLAUDE.md / the program doc.

_summarize aggregates per-row booleans into per-mode pass counts.
_render_markdown emits a 'directive coverage (seven-point program)'
section with per-mode per-directive coverage as 'count/N (pct%)'.

Discipline rationale per CLAUDE.md's five-step algorithm step 5:
the bench harness IS the automation substrate, so it must enforce
the directives before any feature gets layered on top. Bench scores
that climb without directive coverage climbing are graveyard-digging
(step 4: don't dig the grave faster).

5 new bench tests cover: per-mode pass aggregation, the markdown
section presence and content, the helper's behavior on quote mode
(D2 fail by design), lattice mode (all directives pass), and error
rows (empty dict — no signal). Full suite 669 passed.
This commit is contained in:
russell@unturf.com 2026-05-01 16:20:25 -04:00
parent b5f7c93fca
commit acd1f9ca84
No known key found for this signature in database
2 changed files with 225 additions and 0 deletions

View file

@ -137,10 +137,68 @@ def _run_one(
"prompt_chars_system": (result.get("prompt_chars") or {}).get("system_prompt", 0),
"prompt_chars_question": (result.get("prompt_chars") or {}).get("user_question", 0),
"answer_chars": result.get("answer_chars", 0),
# Seven-point program directive compliance — see
# docs/seven-point-program.md. Per-row booleans where the
# directive is observable from the result; aggregate
# coverage shows up in the markdown summary's
# "directive coverage" section. Directives that are global
# properties of the substrate (D1, D5, D8) don't appear
# per-row.
"directive_compliance": _directive_compliance(answer_mode, result, err),
"error": err,
}
def _directive_compliance(
answer_mode: str, result: dict, err: str | None
) -> dict:
"""Compute per-row pass/fail for the seven-point directives whose
pinning is observable from a single bench row.
Returns a dict mapping directive id -> bool (True = pass, False =
fail/pending). Some directives are system-global (D1 verifier
never calls LLM, D5 verifier_method enum, D8 discipline) and don't
appear per-row track those once in the summary header.
"""
if err or result.get("status") == "error":
return {}
audit = result.get("audit_mode")
method = result.get("verifier_method") or ""
is_lattice = answer_mode in ("claim_lattice_pointer", "claim_lattice")
return {
# D2: lattice modes emit pointer clauses (claim_lattice_pointer
# or claim_lattice JSON variant). Quote-mode rows count as
# n/a for D2 — they predate the directive.
"D2_pointer_clauses": is_lattice,
# D3: build CTI internally — proxy is "phrase route had a
# chance to fire" (lattice mode + run_dag_root populated).
# Module L (the answer-side multi-frame compilation) lands
# via ticket #000002; until then this is a structural
# readiness check, not full coverage.
"D3_cti_substrate_ready": (
is_lattice and bool(result.get("run_dag_root"))
),
# D4: evidence_map_root + run_dag_root present in the
# 9-stage CTI run-DAG. Retrieval-plan-hash binding pending
# via ticket #000001.
"D4_evidence_map_bound": bool(result.get("run_dag_root")),
# D6: warrant ran (relation/date anchor classes — today).
# Generalization to entity-list / count / why-cause shapes
# pending via ticket #000003. Mark True only when the
# verifier verdict shows the warrant was checked (today,
# any lattice-mode row gets the warrant pass; future
# per-shape gating sharpens this).
"D6_warrant_fired": is_lattice,
# D7: schema audit_mode is in the canonical enum. Renderer
# then maps lattice-mode STRICT/HYBRID → EVIDENCE-LINKED at
# display time (pinned by tests/test_cli_render.py). Per-row
# signal here just confirms the row's audit token is valid —
# the renderer side is a deterministic transformation tested
# separately, not something each bench row can re-verify.
"D7_honest_label": audit in ("STRICT", "HYBRID", "UNGROUNDED"),
}
def _summarize(rows: list[dict]) -> dict:
"""Group rows by mode → STRICT/HYBRID/UNGROUNDED totals + means."""
by_mode: dict[str, dict] = {}
@ -155,6 +213,16 @@ def _summarize(rows: list[dict]) -> dict:
"ratio_sum": 0.0,
"latency_sum": 0.0,
"deflections": 0,
# Per-directive pass counts (seven-point program). Init
# all known directive ids so absent rows report 0/N
# rather than missing-key.
"directive_pass": {
"D2_pointer_clauses": 0,
"D3_cti_substrate_ready": 0,
"D4_evidence_map_bound": 0,
"D6_warrant_fired": 0,
"D7_honest_label": 0,
},
})
b["n"] += 1
if r["error"] or r["status"] == "error":
@ -170,6 +238,11 @@ def _summarize(rows: list[dict]) -> dict:
"STRICT", "HYBRID"
):
b["deflections"] += 1
# Directive compliance — sum the per-row booleans into
# per-mode pass counts.
for did, ok in (r.get("directive_compliance") or {}).items():
if did in b["directive_pass"] and ok:
b["directive_pass"][did] += 1
return by_mode
@ -236,6 +309,37 @@ def _render_markdown(
f"{deflections}/{b['n']} |"
)
lines.append("")
lines.append("## directive coverage (seven-point program)")
lines.append("")
lines.append(
"Per-mode pass-rate for the directives whose pinning is observable "
"from a single bench row. See `docs/seven-point-program.md` for "
"the full directive list. D1 (no LLM in verifier), D5 (verifier_method "
"enum), and D8 (test-pinning discipline) are global properties of "
"the substrate and don't appear per-row. ½ in the doc means some "
"scope is implemented; bench numbers reflect the implemented portion."
)
lines.append("")
lines.append("| mode | D2 pointer | D3 cti-ready | D4 ev-map bound | D6 warrant | D7 honest label |")
lines.append("|------|-----------|--------------|-----------------|-----------|----------------|")
for mode in modes:
b = summary.get(mode)
if not b:
continue
n = b["n"] or 1
dp = b["directive_pass"]
cells = []
for did in (
"D2_pointer_clauses",
"D3_cti_substrate_ready",
"D4_evidence_map_bound",
"D6_warrant_fired",
"D7_honest_label",
):
count = dp.get(did, 0)
cells.append(f"{count}/{n} ({count / n:.0%})")
lines.append(f"| {mode} | " + " | ".join(cells) + " |")
lines.append("")
lines.append("## strict-rate by prompt size")
lines.append("")
lines.append(

View file

@ -55,6 +55,13 @@ def _row(**overrides) -> dict:
"prompt_chars_system": 800,
"prompt_chars_question": 30,
"answer_chars": 100,
"directive_compliance": {
"D2_pointer_clauses": True,
"D3_cti_substrate_ready": True,
"D4_evidence_map_bound": True,
"D6_warrant_fired": True,
"D7_honest_label": True,
},
"error": None,
}
base.update(overrides)
@ -163,3 +170,117 @@ def test_render_markdown_skips_empty_buckets(qa_sweep):
f"empty bucket {missing_label} leaked into rendering:\n"
f"{bucket_section[:600]}"
)
# ---------------------------------------------------------------------------
# directive coverage (seven-point program)
# ---------------------------------------------------------------------------
def test_summarize_aggregates_directive_pass_counts_per_mode(qa_sweep):
"""Per-row directive_compliance booleans aggregate into per-mode
pass counts. A 2-row sample with one D2 fail should report 1/2."""
rows = [
_row(
answer_mode="claim_lattice",
directive_compliance={
"D2_pointer_clauses": True,
"D3_cti_substrate_ready": True,
"D4_evidence_map_bound": True,
"D6_warrant_fired": True,
"D7_honest_label": True,
},
),
_row(
answer_mode="claim_lattice",
directive_compliance={
"D2_pointer_clauses": False, # the failing row
"D3_cti_substrate_ready": True,
"D4_evidence_map_bound": True,
"D6_warrant_fired": True,
"D7_honest_label": True,
},
),
]
summary = qa_sweep._summarize(rows)
dp = summary["claim_lattice"]["directive_pass"]
assert dp["D2_pointer_clauses"] == 1
assert dp["D3_cti_substrate_ready"] == 2
assert dp["D4_evidence_map_bound"] == 2
assert dp["D6_warrant_fired"] == 2
assert dp["D7_honest_label"] == 2
def test_render_markdown_directive_coverage_section(qa_sweep):
"""The markdown summary includes a 'directive coverage' table
showing per-mode pass counts per directive."""
rows = [_row(answer_mode="claim_lattice")]
summary = qa_sweep._summarize(rows)
md = qa_sweep._render_markdown(
rows, summary, "2026-05-01T00-00-00Z",
["claim_lattice"], ["q"], n_samples=1,
)
assert "directive coverage (seven-point program)" in md
# Header row contains the directive abbreviations.
assert "D2 pointer" in md
assert "D3 cti-ready" in md
assert "D4 ev-map bound" in md
assert "D6 warrant" in md
assert "D7 honest label" in md
# Body cell shows fraction (1/1) for the synthetic row.
coverage_section = md.split("directive coverage")[1]
# Per-mode row with 100% coverage on each directive.
assert "1/1 (100%)" in coverage_section
def test_directive_compliance_helper_marks_quote_mode_d2_false(qa_sweep):
"""Quote-mode rows fail D2 by definition (D2 demands lattice
answer mode)."""
result = {
"audit_mode": "STRICT",
"verifier_method": "quote",
"run_dag_root": None, # quote mode predates the run-DAG split
}
dc = qa_sweep._directive_compliance(
answer_mode="quote",
result=result,
err=None,
)
assert dc["D2_pointer_clauses"] is False
assert dc["D3_cti_substrate_ready"] is False # not lattice mode
assert dc["D4_evidence_map_bound"] is False # no run_dag_root
assert dc["D6_warrant_fired"] is False
# D7: quote-mode STRICT keeps the STRICT label; the EVIDENCE-LINKED
# relabel applies to lattice modes only. Quote mode passes D7
# vacuously.
assert dc["D7_honest_label"] is True
def test_directive_compliance_helper_marks_lattice_mode_d2_true(qa_sweep):
"""Lattice-mode rows pass D2."""
result = {
"audit_mode": "STRICT",
"verifier_method": "claim_lattice",
"run_dag_root": "abc" * 21,
}
dc = qa_sweep._directive_compliance(
answer_mode="claim_lattice",
result=result,
err=None,
)
assert dc["D2_pointer_clauses"] is True
assert dc["D3_cti_substrate_ready"] is True
assert dc["D4_evidence_map_bound"] is True
assert dc["D6_warrant_fired"] is True
assert dc["D7_honest_label"] is True
def test_directive_compliance_returns_empty_on_error_row(qa_sweep):
"""Error rows don't carry directive signal — no per-row check
can succeed when the run threw."""
dc = qa_sweep._directive_compliance(
answer_mode="claim_lattice",
result={},
err="some error",
)
assert dc == {}