bench: per-row directive_compliance + 'directive coverage' summary section
Wires the seven-point program (docs/seven-point-program.md) into
the bench harness as a per-mode coverage table.
Per-row computation in _directive_compliance(answer_mode, result, err):
D2_pointer_clauses — answer_mode is lattice variant
D3_cti_substrate_ready — lattice mode + run_dag_root populated
(full coverage pending ticket #000002)
D4_evidence_map_bound — run_dag_root populated
(retrieval-plan binding pending #000001)
D6_warrant_fired — warrant ran (lattice mode today;
per-shape gating pending #000003)
D7_honest_label — audit_mode in canonical enum
(renderer transformation pinned in
test_cli_render.py)
D1 (no LLM in verifier), D5 (verifier_method enum), D8 (test-pinning
discipline) are global properties of the substrate and don't appear
per-row — they get tracked once in CLAUDE.md / the program doc.
_summarize aggregates per-row booleans into per-mode pass counts.
_render_markdown emits a 'directive coverage (seven-point program)'
section with per-mode per-directive coverage as 'count/N (pct%)'.
Discipline rationale per CLAUDE.md's five-step algorithm step 5:
the bench harness IS the automation substrate, so it must enforce
the directives before any feature gets layered on top. Bench scores
that climb without directive coverage climbing are graveyard-digging
(step 4: don't dig the grave faster).
5 new bench tests cover: per-mode pass aggregation, the markdown
section presence and content, the helper's behavior on quote mode
(D2 fail by design), lattice mode (all directives pass), and error
rows (empty dict — no signal). Full suite 669 passed.
This commit is contained in:
parent
b5f7c93fca
commit
acd1f9ca84
2 changed files with 225 additions and 0 deletions
|
|
@ -137,10 +137,68 @@ def _run_one(
|
|||
"prompt_chars_system": (result.get("prompt_chars") or {}).get("system_prompt", 0),
|
||||
"prompt_chars_question": (result.get("prompt_chars") or {}).get("user_question", 0),
|
||||
"answer_chars": result.get("answer_chars", 0),
|
||||
# Seven-point program directive compliance — see
|
||||
# docs/seven-point-program.md. Per-row booleans where the
|
||||
# directive is observable from the result; aggregate
|
||||
# coverage shows up in the markdown summary's
|
||||
# "directive coverage" section. Directives that are global
|
||||
# properties of the substrate (D1, D5, D8) don't appear
|
||||
# per-row.
|
||||
"directive_compliance": _directive_compliance(answer_mode, result, err),
|
||||
"error": err,
|
||||
}
|
||||
|
||||
|
||||
def _directive_compliance(
|
||||
answer_mode: str, result: dict, err: str | None
|
||||
) -> dict:
|
||||
"""Compute per-row pass/fail for the seven-point directives whose
|
||||
pinning is observable from a single bench row.
|
||||
|
||||
Returns a dict mapping directive id -> bool (True = pass, False =
|
||||
fail/pending). Some directives are system-global (D1 verifier
|
||||
never calls LLM, D5 verifier_method enum, D8 discipline) and don't
|
||||
appear per-row — track those once in the summary header.
|
||||
"""
|
||||
if err or result.get("status") == "error":
|
||||
return {}
|
||||
audit = result.get("audit_mode")
|
||||
method = result.get("verifier_method") or ""
|
||||
is_lattice = answer_mode in ("claim_lattice_pointer", "claim_lattice")
|
||||
return {
|
||||
# D2: lattice modes emit pointer clauses (claim_lattice_pointer
|
||||
# or claim_lattice JSON variant). Quote-mode rows count as
|
||||
# n/a for D2 — they predate the directive.
|
||||
"D2_pointer_clauses": is_lattice,
|
||||
# D3: build CTI internally — proxy is "phrase route had a
|
||||
# chance to fire" (lattice mode + run_dag_root populated).
|
||||
# Module L (the answer-side multi-frame compilation) lands
|
||||
# via ticket #000002; until then this is a structural
|
||||
# readiness check, not full coverage.
|
||||
"D3_cti_substrate_ready": (
|
||||
is_lattice and bool(result.get("run_dag_root"))
|
||||
),
|
||||
# D4: evidence_map_root + run_dag_root present in the
|
||||
# 9-stage CTI run-DAG. Retrieval-plan-hash binding pending
|
||||
# via ticket #000001.
|
||||
"D4_evidence_map_bound": bool(result.get("run_dag_root")),
|
||||
# D6: warrant ran (relation/date anchor classes — today).
|
||||
# Generalization to entity-list / count / why-cause shapes
|
||||
# pending via ticket #000003. Mark True only when the
|
||||
# verifier verdict shows the warrant was checked (today,
|
||||
# any lattice-mode row gets the warrant pass; future
|
||||
# per-shape gating sharpens this).
|
||||
"D6_warrant_fired": is_lattice,
|
||||
# D7: schema audit_mode is in the canonical enum. Renderer
|
||||
# then maps lattice-mode STRICT/HYBRID → EVIDENCE-LINKED at
|
||||
# display time (pinned by tests/test_cli_render.py). Per-row
|
||||
# signal here just confirms the row's audit token is valid —
|
||||
# the renderer side is a deterministic transformation tested
|
||||
# separately, not something each bench row can re-verify.
|
||||
"D7_honest_label": audit in ("STRICT", "HYBRID", "UNGROUNDED"),
|
||||
}
|
||||
|
||||
|
||||
def _summarize(rows: list[dict]) -> dict:
|
||||
"""Group rows by mode → STRICT/HYBRID/UNGROUNDED totals + means."""
|
||||
by_mode: dict[str, dict] = {}
|
||||
|
|
@ -155,6 +213,16 @@ def _summarize(rows: list[dict]) -> dict:
|
|||
"ratio_sum": 0.0,
|
||||
"latency_sum": 0.0,
|
||||
"deflections": 0,
|
||||
# Per-directive pass counts (seven-point program). Init
|
||||
# all known directive ids so absent rows report 0/N
|
||||
# rather than missing-key.
|
||||
"directive_pass": {
|
||||
"D2_pointer_clauses": 0,
|
||||
"D3_cti_substrate_ready": 0,
|
||||
"D4_evidence_map_bound": 0,
|
||||
"D6_warrant_fired": 0,
|
||||
"D7_honest_label": 0,
|
||||
},
|
||||
})
|
||||
b["n"] += 1
|
||||
if r["error"] or r["status"] == "error":
|
||||
|
|
@ -170,6 +238,11 @@ def _summarize(rows: list[dict]) -> dict:
|
|||
"STRICT", "HYBRID"
|
||||
):
|
||||
b["deflections"] += 1
|
||||
# Directive compliance — sum the per-row booleans into
|
||||
# per-mode pass counts.
|
||||
for did, ok in (r.get("directive_compliance") or {}).items():
|
||||
if did in b["directive_pass"] and ok:
|
||||
b["directive_pass"][did] += 1
|
||||
return by_mode
|
||||
|
||||
|
||||
|
|
@ -236,6 +309,37 @@ def _render_markdown(
|
|||
f"{deflections}/{b['n']} |"
|
||||
)
|
||||
lines.append("")
|
||||
lines.append("## directive coverage (seven-point program)")
|
||||
lines.append("")
|
||||
lines.append(
|
||||
"Per-mode pass-rate for the directives whose pinning is observable "
|
||||
"from a single bench row. See `docs/seven-point-program.md` for "
|
||||
"the full directive list. D1 (no LLM in verifier), D5 (verifier_method "
|
||||
"enum), and D8 (test-pinning discipline) are global properties of "
|
||||
"the substrate and don't appear per-row. ½ in the doc means some "
|
||||
"scope is implemented; bench numbers reflect the implemented portion."
|
||||
)
|
||||
lines.append("")
|
||||
lines.append("| mode | D2 pointer | D3 cti-ready | D4 ev-map bound | D6 warrant | D7 honest label |")
|
||||
lines.append("|------|-----------|--------------|-----------------|-----------|----------------|")
|
||||
for mode in modes:
|
||||
b = summary.get(mode)
|
||||
if not b:
|
||||
continue
|
||||
n = b["n"] or 1
|
||||
dp = b["directive_pass"]
|
||||
cells = []
|
||||
for did in (
|
||||
"D2_pointer_clauses",
|
||||
"D3_cti_substrate_ready",
|
||||
"D4_evidence_map_bound",
|
||||
"D6_warrant_fired",
|
||||
"D7_honest_label",
|
||||
):
|
||||
count = dp.get(did, 0)
|
||||
cells.append(f"{count}/{n} ({count / n:.0%})")
|
||||
lines.append(f"| {mode} | " + " | ".join(cells) + " |")
|
||||
lines.append("")
|
||||
lines.append("## strict-rate by prompt size")
|
||||
lines.append("")
|
||||
lines.append(
|
||||
|
|
|
|||
|
|
@ -55,6 +55,13 @@ def _row(**overrides) -> dict:
|
|||
"prompt_chars_system": 800,
|
||||
"prompt_chars_question": 30,
|
||||
"answer_chars": 100,
|
||||
"directive_compliance": {
|
||||
"D2_pointer_clauses": True,
|
||||
"D3_cti_substrate_ready": True,
|
||||
"D4_evidence_map_bound": True,
|
||||
"D6_warrant_fired": True,
|
||||
"D7_honest_label": True,
|
||||
},
|
||||
"error": None,
|
||||
}
|
||||
base.update(overrides)
|
||||
|
|
@ -163,3 +170,117 @@ def test_render_markdown_skips_empty_buckets(qa_sweep):
|
|||
f"empty bucket {missing_label} leaked into rendering:\n"
|
||||
f"{bucket_section[:600]}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# directive coverage (seven-point program)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_summarize_aggregates_directive_pass_counts_per_mode(qa_sweep):
|
||||
"""Per-row directive_compliance booleans aggregate into per-mode
|
||||
pass counts. A 2-row sample with one D2 fail should report 1/2."""
|
||||
rows = [
|
||||
_row(
|
||||
answer_mode="claim_lattice",
|
||||
directive_compliance={
|
||||
"D2_pointer_clauses": True,
|
||||
"D3_cti_substrate_ready": True,
|
||||
"D4_evidence_map_bound": True,
|
||||
"D6_warrant_fired": True,
|
||||
"D7_honest_label": True,
|
||||
},
|
||||
),
|
||||
_row(
|
||||
answer_mode="claim_lattice",
|
||||
directive_compliance={
|
||||
"D2_pointer_clauses": False, # the failing row
|
||||
"D3_cti_substrate_ready": True,
|
||||
"D4_evidence_map_bound": True,
|
||||
"D6_warrant_fired": True,
|
||||
"D7_honest_label": True,
|
||||
},
|
||||
),
|
||||
]
|
||||
summary = qa_sweep._summarize(rows)
|
||||
dp = summary["claim_lattice"]["directive_pass"]
|
||||
assert dp["D2_pointer_clauses"] == 1
|
||||
assert dp["D3_cti_substrate_ready"] == 2
|
||||
assert dp["D4_evidence_map_bound"] == 2
|
||||
assert dp["D6_warrant_fired"] == 2
|
||||
assert dp["D7_honest_label"] == 2
|
||||
|
||||
|
||||
def test_render_markdown_directive_coverage_section(qa_sweep):
|
||||
"""The markdown summary includes a 'directive coverage' table
|
||||
showing per-mode pass counts per directive."""
|
||||
rows = [_row(answer_mode="claim_lattice")]
|
||||
summary = qa_sweep._summarize(rows)
|
||||
md = qa_sweep._render_markdown(
|
||||
rows, summary, "2026-05-01T00-00-00Z",
|
||||
["claim_lattice"], ["q"], n_samples=1,
|
||||
)
|
||||
assert "directive coverage (seven-point program)" in md
|
||||
# Header row contains the directive abbreviations.
|
||||
assert "D2 pointer" in md
|
||||
assert "D3 cti-ready" in md
|
||||
assert "D4 ev-map bound" in md
|
||||
assert "D6 warrant" in md
|
||||
assert "D7 honest label" in md
|
||||
# Body cell shows fraction (1/1) for the synthetic row.
|
||||
coverage_section = md.split("directive coverage")[1]
|
||||
# Per-mode row with 100% coverage on each directive.
|
||||
assert "1/1 (100%)" in coverage_section
|
||||
|
||||
|
||||
def test_directive_compliance_helper_marks_quote_mode_d2_false(qa_sweep):
|
||||
"""Quote-mode rows fail D2 by definition (D2 demands lattice
|
||||
answer mode)."""
|
||||
result = {
|
||||
"audit_mode": "STRICT",
|
||||
"verifier_method": "quote",
|
||||
"run_dag_root": None, # quote mode predates the run-DAG split
|
||||
}
|
||||
dc = qa_sweep._directive_compliance(
|
||||
answer_mode="quote",
|
||||
result=result,
|
||||
err=None,
|
||||
)
|
||||
assert dc["D2_pointer_clauses"] is False
|
||||
assert dc["D3_cti_substrate_ready"] is False # not lattice mode
|
||||
assert dc["D4_evidence_map_bound"] is False # no run_dag_root
|
||||
assert dc["D6_warrant_fired"] is False
|
||||
# D7: quote-mode STRICT keeps the STRICT label; the EVIDENCE-LINKED
|
||||
# relabel applies to lattice modes only. Quote mode passes D7
|
||||
# vacuously.
|
||||
assert dc["D7_honest_label"] is True
|
||||
|
||||
|
||||
def test_directive_compliance_helper_marks_lattice_mode_d2_true(qa_sweep):
|
||||
"""Lattice-mode rows pass D2."""
|
||||
result = {
|
||||
"audit_mode": "STRICT",
|
||||
"verifier_method": "claim_lattice",
|
||||
"run_dag_root": "abc" * 21,
|
||||
}
|
||||
dc = qa_sweep._directive_compliance(
|
||||
answer_mode="claim_lattice",
|
||||
result=result,
|
||||
err=None,
|
||||
)
|
||||
assert dc["D2_pointer_clauses"] is True
|
||||
assert dc["D3_cti_substrate_ready"] is True
|
||||
assert dc["D4_evidence_map_bound"] is True
|
||||
assert dc["D6_warrant_fired"] is True
|
||||
assert dc["D7_honest_label"] is True
|
||||
|
||||
|
||||
def test_directive_compliance_returns_empty_on_error_row(qa_sweep):
|
||||
"""Error rows don't carry directive signal — no per-row check
|
||||
can succeed when the run threw."""
|
||||
dc = qa_sweep._directive_compliance(
|
||||
answer_mode="claim_lattice",
|
||||
result={},
|
||||
err="some error",
|
||||
)
|
||||
assert dc == {}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue