qa(#000010): Phases 2-4 land — wired, governed, labeled, benched
Closes ticket #000010 (Meta-Cognition Preflight Guard). Mechanism complete; defaults preserve the dry-run discipline pattern from #000008. Phase 2 — wire preflight into query() and runner.ask(): - preflight_question() runs after policy resolution + quantifier classification, before retrieval. - QuestionState surfaces on miss path, cache-hit path, AND reject-broad early-return path of query() — schema column- aligned across all four returns. - runner.ask() carries the same fields for `aborist ask` parity. Phase 3 — policy fields + governance hash + CLI flags: - 6 new policy fields, all default-on except metacognition_block_on_contradiction (default False — label- only by default; opt-in via --block-on-contradiction). - All 6 folded into _VERIFIER_POLICY_FIELDS so flipping any invalidates prior cache records on lookup. - 2 new CLI flags on `aborist query`: --no-preflight Level 2 master kill --block-on-contradiction strict mode (hard-block on lexical contradictions) Phase 4 — audit-line labels + bench fields + tests: - _render_warrant_tail extended with 5 metacog tail tokens: · false premise · contradictory · stale risk · out of corpus · frame ambiguous - Bench rows in qa_sweep.py gain 7 new bounded-size projection fields (logical_statuses, question_shape, preflight_result, temporal_sensitivity, has_false_premise, has_contradiction, corpus_requirement). Full QuestionState stays on result dict for CLI render only. - tests/test_metacognition.py grew from 42 → 68 tests (16 new: 6 governance + 6 audit-line tail + 4 default-policy pinning). Live verified end-to-end: $ make query-dry Q="Who is the current CEO of OpenAI?" BURN=1 UNGROUNDED · via claim_lattice · stale risk $ make query-dry Q="When did Mr. Burns become Homer's biological father?" BURN=1 UNGROUNDED · via claim_lattice · false premise 978 tests passing; 36 skipped. What's NOT shipped (deferred): - Run-DAG node binding for metacognition_preflight stage — joins ticket #000009 Phase 5 (same audit-replay gap; both nodes can land together). - Reference-frame plumbing — frame_detection runs post-retrieval, preflight here is pre-retrieval; deferred until two-pass or post-classification update lands. - SOFT_PREFLIGHT_HINT (model-assisted sidecar) — source doc §18 reserves this label; hard rule preserved (no LLM in preflight hard path). - Bench A/B measuring preflight on vs off — quick to run once stack settles. Ticket #000010 status: closed · landed 2026-05-03.
This commit is contained in:
parent
55efb04a58
commit
f2bbe512db
9 changed files with 420 additions and 3 deletions
|
|
@ -417,6 +417,13 @@ def _cmd_query(args: argparse.Namespace) -> int:
|
|||
# Bench-first per §10.11.3 — this flag is the path from
|
||||
# dry-run to live-cap.
|
||||
call_policy["quantifier_guard_apply_caps"] = True
|
||||
# Ticket #000010 — meta-cognition CLI overrides.
|
||||
if getattr(args, "no_preflight", False):
|
||||
call_policy["metacognition_enabled"] = False
|
||||
if getattr(args, "block_on_contradiction", False):
|
||||
# Strict mode: hard-block on lexical contradictions instead
|
||||
# of label-only.
|
||||
call_policy["metacognition_block_on_contradiction"] = True
|
||||
|
||||
result = query(
|
||||
question=args.question,
|
||||
|
|
@ -663,6 +670,22 @@ def _render_warrant_tail(result: dict) -> str:
|
|||
parts.append("broad unbounded")
|
||||
elif "BROAD_QUANTIFIER_RUNAWAY" in kinds:
|
||||
parts.append("broad runaway")
|
||||
# Ticket #000010 — meta-cognition logical-status tails. Pulled
|
||||
# from result["question_state"]["logical_statuses"] when present.
|
||||
# Doesn't double up with the broad-quantifier tails above (those
|
||||
# come from the verifier violation list, not the preflight).
|
||||
qs = result.get("question_state") or {}
|
||||
statuses = set(qs.get("logical_statuses") or [])
|
||||
if "false_premise_suspected" in statuses:
|
||||
parts.append("false premise")
|
||||
if "contradictory_question" in statuses:
|
||||
parts.append("contradictory")
|
||||
if "stale_risk" in statuses:
|
||||
parts.append("stale risk")
|
||||
if "out_of_corpus_risk" in statuses:
|
||||
parts.append("out of corpus")
|
||||
if "reference_frame_ambiguous" in statuses:
|
||||
parts.append("frame ambiguous")
|
||||
if not parts:
|
||||
return ""
|
||||
return " · " + " · ".join(parts)
|
||||
|
|
@ -3506,6 +3529,27 @@ def build_parser() -> argparse.ArgumentParser:
|
|||
"the classifier output across the question set."
|
||||
),
|
||||
)
|
||||
# Ticket #000010 — meta-cognition CLI flags.
|
||||
query_cmd.add_argument(
|
||||
"--no-preflight",
|
||||
dest="no_preflight", action="store_true",
|
||||
help=(
|
||||
"Disable the meta-cognition preflight guard for this "
|
||||
"call. Skips temporal / contradiction / false-premise / "
|
||||
"out-of-corpus detectors. The QuestionState surfaces a "
|
||||
"stub with empty logical_statuses so bench rows stay "
|
||||
"column-aligned."
|
||||
),
|
||||
)
|
||||
query_cmd.add_argument(
|
||||
"--block-on-contradiction",
|
||||
dest="block_on_contradiction", action="store_true",
|
||||
help=(
|
||||
"Hard-block on lexical contradictions (default: label-"
|
||||
"only). Strict mode: questions like 'which unmarried "
|
||||
"spouse is X married to' return PREFLIGHT_BLOCKED."
|
||||
),
|
||||
)
|
||||
query_cmd.set_defaults(func=_cmd_query)
|
||||
|
||||
inspect_cmd = sub.add_parser(
|
||||
|
|
|
|||
|
|
@ -230,6 +230,16 @@ _VERIFIER_POLICY_FIELDS = frozenset({
|
|||
"quantifier_guard_modes",
|
||||
"quantifier_reminder_enabled",
|
||||
"quantifier_reject_broad",
|
||||
# Ticket #000010 — Meta-Cognition Preflight Guard. The six
|
||||
# fields together control whether preflight runs and which
|
||||
# detectors fire. Flipping any of them invalidates prior cache
|
||||
# records on lookup — same governance discipline as #000008.
|
||||
"metacognition_enabled",
|
||||
"metacognition_temporal_check",
|
||||
"metacognition_contradiction_check",
|
||||
"metacognition_false_premise_check",
|
||||
"metacognition_out_of_corpus_check",
|
||||
"metacognition_block_on_contradiction",
|
||||
# Quote-mode entity policy
|
||||
"entity_policy",
|
||||
"entity_proximity_n",
|
||||
|
|
|
|||
|
|
@ -473,6 +473,13 @@ DEFAULT_QUERY_POLICY = {
|
|||
# Phase 4 — strict reject for broad-unbounded. See runner.py for
|
||||
# rationale. Default OFF.
|
||||
"quantifier_reject_broad": False,
|
||||
# Ticket #000010 — see runner.DEFAULT_POLICY for rationale.
|
||||
"metacognition_enabled": True,
|
||||
"metacognition_temporal_check": True,
|
||||
"metacognition_contradiction_check": True,
|
||||
"metacognition_false_premise_check": True,
|
||||
"metacognition_out_of_corpus_check": True,
|
||||
"metacognition_block_on_contradiction": False,
|
||||
# Claim-count ceiling — see runner.DEFAULT_POLICY for rationale.
|
||||
# Bench finding (york-england "tell me all there is to know")
|
||||
# caught the runaway shape; cap of 12 admits entity-list
|
||||
|
|
@ -1688,6 +1695,24 @@ def query(
|
|||
effective_max_claims = int(claim_cap_lookup)
|
||||
else:
|
||||
effective_max_claims = _policy_max_claims
|
||||
# Ticket #000010 — meta-cognition preflight. Pure deterministic
|
||||
# classifier wraps the quantifier output plus four new detectors
|
||||
# (temporal, contradiction, false-premise-lite, out-of-corpus).
|
||||
# Surfaces a QuestionState on the result dict; first pass does
|
||||
# NOT bind into run_dag_root (deferred to Phase 5 / ticket #000009
|
||||
# where the quantifier_preflight node lands too — both nodes can
|
||||
# land together to keep the run-DAG schema atomic). Reference
|
||||
# frames not yet plumbed through (frame_detection runs after
|
||||
# retrieval, and preflight here is pre-retrieval — frame info
|
||||
# lives on the result dict separately, not on QuestionState
|
||||
# in this pass).
|
||||
from aborist.qa.metacognition import preflight_question
|
||||
question_state = preflight_question(
|
||||
question,
|
||||
model_profile_id=model_id,
|
||||
reference_frames=(),
|
||||
policy=policy,
|
||||
)
|
||||
# Ticket #000008 Phase 4 — strict reject for broad-unbounded.
|
||||
# When opt-in via policy / --reject-broad CLI flag, return
|
||||
# UNGROUNDED before the LLM call for ALL/COMPREHENSIVE/
|
||||
|
|
@ -1749,6 +1774,11 @@ def query(
|
|||
"scope_bound_hint": quantifier["scope_bound_hint"],
|
||||
"quantifier_explicit_count": quantifier["explicit_count"],
|
||||
"claim_cap_applied": claim_cap_lookup,
|
||||
# Ticket #000010 — meta-cognition QuestionState surfaced
|
||||
# for bench / CLI render. Preflight rejection path still
|
||||
# returns its own status; this is the upstream classifier
|
||||
# output regardless of guard outcome.
|
||||
"question_state": question_state.to_dict(),
|
||||
"pointer_id_distribution": None,
|
||||
"lazy_anchor_ratio": None,
|
||||
"retrieval_purity": None,
|
||||
|
|
@ -2301,6 +2331,9 @@ def query(
|
|||
"scope_bound_hint": quantifier["scope_bound_hint"],
|
||||
"quantifier_explicit_count": quantifier["explicit_count"],
|
||||
"claim_cap_applied": claim_cap_lookup,
|
||||
# Ticket #000010 — meta-cognition QuestionState. Pure
|
||||
# function, cache hits re-classify cheaply.
|
||||
"question_state": question_state.to_dict(),
|
||||
"prompt_chars": prompt_chars,
|
||||
"answer_chars": len(cached["answer_text"] or ""),
|
||||
"timings": {
|
||||
|
|
@ -2810,6 +2843,13 @@ def query(
|
|||
# 12) still drives TOO_MANY_CLAIMS demote until
|
||||
# quantifier_guard_apply_caps flips True.
|
||||
"claim_cap_applied": claim_cap_lookup,
|
||||
# Ticket #000010 — meta-cognition QuestionState. Surfaces
|
||||
# the full preflight classification (logical_statuses,
|
||||
# question_shape, contradiction_pairs, false_premise_hints,
|
||||
# temporal_sensitivity, etc.) for bench / CLI render. First
|
||||
# pass does NOT bind into run_dag_root (deferred to ticket
|
||||
# #000009 Phase 5).
|
||||
"question_state": question_state.to_dict(),
|
||||
# Sidecar smell signals (claim_lattice mode only) — surfaced
|
||||
# for the renderer; never persisted in providence_cache and
|
||||
# never threaded into run_dag_root.
|
||||
|
|
|
|||
|
|
@ -253,6 +253,21 @@ DEFAULT_POLICY = {
|
|||
# Bounded universals (e.g. all members of the Beatles, year-anchored
|
||||
# questions) are NOT rejected per §10.1.
|
||||
"quantifier_reject_broad": False,
|
||||
# Ticket #000010 — Meta-Cognition Preflight Guard (M0 / MCTL).
|
||||
# Pure deterministic detectors (temporal, contradiction,
|
||||
# false-premise-lite, out-of-corpus) wrap the #000008 quantifier
|
||||
# classifier and surface a QuestionState on the result dict.
|
||||
# Master switch ON by default — detectors are pure-on-question
|
||||
# so cost is negligible. Each sub-detector has its own enable
|
||||
# switch for granular A/B. block_on_contradiction defaults False
|
||||
# (label-only by default; opt-in to hard-block — false-positive
|
||||
# rate not yet bench-validated).
|
||||
"metacognition_enabled": True,
|
||||
"metacognition_temporal_check": True,
|
||||
"metacognition_contradiction_check": True,
|
||||
"metacognition_false_premise_check": True,
|
||||
"metacognition_out_of_corpus_check": True,
|
||||
"metacognition_block_on_contradiction": False,
|
||||
# Claim-count ceiling. Bench finding (2026-04-30 york-england):
|
||||
# "tell me all there is to know about X" prompted Hermes to spam
|
||||
# 26-59 encyclopedic claims sourced from training, only 2-4 of
|
||||
|
|
@ -365,6 +380,14 @@ def ask(
|
|||
effective_max_claims = int(claim_cap_lookup)
|
||||
else:
|
||||
effective_max_claims = _policy_max_claims
|
||||
# Ticket #000010 — meta-cognition preflight (mirror of query()).
|
||||
from aborist.qa.metacognition import preflight_question
|
||||
question_state = preflight_question(
|
||||
question,
|
||||
model_profile_id=model_id,
|
||||
reference_frames=(),
|
||||
policy=policy,
|
||||
)
|
||||
t_start = time.monotonic()
|
||||
|
||||
doc = conn.execute(
|
||||
|
|
@ -597,6 +620,8 @@ def ask(
|
|||
"scope_bound_hint": quantifier["scope_bound_hint"],
|
||||
"quantifier_explicit_count": quantifier["explicit_count"],
|
||||
"claim_cap_applied": claim_cap_lookup,
|
||||
# Ticket #000010 — meta-cognition QuestionState.
|
||||
"question_state": question_state.to_dict(),
|
||||
"timings": {
|
||||
"cache_lookup_ms": cache_lookup_ms,
|
||||
"llm_ms": None,
|
||||
|
|
@ -960,6 +985,8 @@ def ask(
|
|||
"scope_bound_hint": quantifier["scope_bound_hint"],
|
||||
"quantifier_explicit_count": quantifier["explicit_count"],
|
||||
"claim_cap_applied": claim_cap_lookup,
|
||||
# Ticket #000010 — meta-cognition QuestionState.
|
||||
"question_state": question_state.to_dict(),
|
||||
# Sidecar smell signals (claim_lattice mode only) — render-
|
||||
# layer; never persisted, never in run_dag_root.
|
||||
"pointer_id_distribution": verdict.get("pointer_id_distribution"),
|
||||
|
|
|
|||
|
|
@ -207,6 +207,21 @@ def _run_one(
|
|||
quantifier_matched_token = result.get("quantifier_matched_token")
|
||||
scope_bound_hint = result.get("scope_bound_hint")
|
||||
claim_cap_applied = result.get("claim_cap_applied")
|
||||
# Ticket #000010 — meta-cognition QuestionState. Persist a
|
||||
# bounded-size projection: logical_statuses, question_shape,
|
||||
# preflight_result, temporal_sensitivity, and the kind of any
|
||||
# detector hits. Full QuestionState (including
|
||||
# contradiction_pairs / false_premise_hints) stays on the
|
||||
# result dict for CLI render but is NOT persisted in bench
|
||||
# rows to keep row size bounded across long sweeps.
|
||||
qs = result.get("question_state") or {}
|
||||
preflight_logical_statuses = list(qs.get("logical_statuses") or [])
|
||||
preflight_question_shape = qs.get("question_shape")
|
||||
preflight_result = qs.get("preflight_result")
|
||||
preflight_temporal_sensitivity = qs.get("temporal_sensitivity")
|
||||
preflight_has_false_premise = bool(qs.get("false_premise_hints"))
|
||||
preflight_has_contradiction = bool(qs.get("contradiction_pairs"))
|
||||
preflight_corpus_requirement = qs.get("corpus_requirement")
|
||||
return {
|
||||
"question": question,
|
||||
"answer_mode": answer_mode,
|
||||
|
|
@ -262,6 +277,19 @@ def _run_one(
|
|||
# bench archive remains interpretable when model_profiles.py
|
||||
# changes.
|
||||
"model_profile_id": model,
|
||||
# Ticket #000010 — meta-cognition preflight bench fields.
|
||||
# Bounded-size projection of QuestionState. Full state
|
||||
# (contradiction_pairs / false_premise_hints) lives on the
|
||||
# result dict for CLI render; bench rows keep just the
|
||||
# kinds + shape + result so 10K-row sweeps stay
|
||||
# human-greppable.
|
||||
"preflight_logical_statuses": preflight_logical_statuses,
|
||||
"preflight_question_shape": preflight_question_shape,
|
||||
"preflight_result": preflight_result,
|
||||
"preflight_temporal_sensitivity": preflight_temporal_sensitivity,
|
||||
"preflight_has_false_premise": preflight_has_false_premise,
|
||||
"preflight_has_contradiction": preflight_has_contradiction,
|
||||
"preflight_corpus_requirement": preflight_corpus_requirement,
|
||||
"cache_key": (result.get("cache_key") or "")[:12],
|
||||
"n_sources": len(result.get("sources") or []),
|
||||
"elapsed_s": elapsed_s,
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ Newest first. Update on every open/close.
|
|||
|
||||
| ID | Title | Status | Opened | Directive |
|
||||
|----------|------------------------------------------------|-----------------------|------------|-----------|
|
||||
| #000010 | Meta-Cognition Preflight Guard (M0 / MCTL) | open · in progress (zero-shot 2026-05-03) | 2026-05-03 | D1, D3 |
|
||||
| #000010 | Meta-Cognition Preflight Guard (M0 / MCTL) | closed · landed 2026-05-03 (Phases 1–4); Phase 5 DAG join #000009 | 2026-05-03 | D1, D3 |
|
||||
| #000009 | Quantifier preflight run-DAG node binding | open · awaiting go/no-go | 2026-05-03 | D3, D4 |
|
||||
| #000008 | Broad-quantifier preflight guard | closed · landed in `4f2b5a6`; Phase 5 DAG binding split into #000009 | 2026-05-02 | — |
|
||||
| #000007 | Query-layer hyphen folding | closed · 2026-05-02 | 2026-05-02 | — |
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
# Ticket #000010 — Meta-Cognition Preflight Guard (M0 / MCTL)
|
||||
|
||||
**Status:** open · in progress (zero-shot implementation 2026-05-03)
|
||||
**Status:** closed · landed 2026-05-03 (zero-shot, phases 1–4
|
||||
complete; Phase 5 DAG binding joins #000009; cross-model bench
|
||||
deferred manual task)
|
||||
**Opened:** 2026-05-03
|
||||
**Scope:** Build a runtime epistemic control layer that classifies a
|
||||
question's shape (false-premise, contradictory, under-specified,
|
||||
|
|
@ -321,4 +323,141 @@ to fox.
|
|||
|
||||
## 12. Implementation log
|
||||
|
||||
(populated as phases land)
|
||||
### 12.1 Phase 1 — module + detectors (LANDED `55efb04`)
|
||||
|
||||
`aborist/qa/metacognition.py` shipped with:
|
||||
|
||||
- `QuestionState` dataclass (frozen, JSON-serializable via `to_dict`)
|
||||
- `preflight_question()` pure function
|
||||
- 4 new deterministic detectors (temporal, contradiction, false-
|
||||
premise-lite, out-of-corpus)
|
||||
- Reuses `aborist.qa.quantifier.classify_question_quantifier`
|
||||
- 8 LogicalStatus values + 3 PreflightResult values
|
||||
- 6 per-detector enable switches in policy
|
||||
- `preflight_policy_hash` for governance binding (Phase 3)
|
||||
- `PREFLIGHT_VERSION = "metacognition-v0.1"`
|
||||
|
||||
42 new tests: per-detector unit tests + the seven test cases from
|
||||
source doc §14 + gating + determinism + serialization.
|
||||
|
||||
### 12.2 Phase 2 — wire into query() and runner.ask() (in same commit chain)
|
||||
|
||||
`preflight_question()` runs after policy resolution + quantifier
|
||||
classification, before retrieval. Surfaces `question_state` (from
|
||||
`to_dict()`) on:
|
||||
|
||||
- `query()` miss-path result dict
|
||||
- `query()` cache-hit path (re-classifies cheaply since pure)
|
||||
- `query()` reject-broad early-return path
|
||||
- `runner.ask()` miss-path + cache-hit-path
|
||||
|
||||
Schema stays column-aligned across all four return shapes. First
|
||||
pass does NOT bind into `run_dag_root` — deferred to ticket #000009
|
||||
(Phase 5) where the quantifier_preflight node lands too. Both
|
||||
nodes can land together to keep the run-DAG schema atomic.
|
||||
|
||||
### 12.3 Phase 3 — policy fields + governance hash + CLI
|
||||
|
||||
Six new policy fields (default-on except block_on_contradiction):
|
||||
|
||||
```
|
||||
metacognition_enabled True
|
||||
metacognition_temporal_check True
|
||||
metacognition_contradiction_check True
|
||||
metacognition_false_premise_check True
|
||||
metacognition_out_of_corpus_check True
|
||||
metacognition_block_on_contradiction False
|
||||
```
|
||||
|
||||
All folded into `_VERIFIER_POLICY_FIELDS` so flipping any of them
|
||||
invalidates prior cache records.
|
||||
|
||||
CLI flags on `aborist query`:
|
||||
|
||||
```
|
||||
--no-preflight disable the guard for one call
|
||||
--block-on-contradiction hard-block on lexical contradictions
|
||||
(default: label-only)
|
||||
```
|
||||
|
||||
### 12.4 Phase 4 — audit-line labels + bench fields + tests
|
||||
|
||||
`_render_warrant_tail` extended with five metacog tail tokens
|
||||
(when corresponding logical_status fires):
|
||||
|
||||
```
|
||||
· false premise
|
||||
· contradictory
|
||||
· stale risk
|
||||
· out of corpus
|
||||
· frame ambiguous
|
||||
```
|
||||
|
||||
Bench rows in `bench/qa_sweep.py:_run_one` gain seven new fields
|
||||
(bounded-size projection of QuestionState; full state stays on
|
||||
result dict for CLI render):
|
||||
|
||||
```
|
||||
preflight_logical_statuses list[str]
|
||||
preflight_question_shape str
|
||||
preflight_result PREFLIGHT_OK / _PARTIAL / _BLOCKED
|
||||
preflight_temporal_sensitivity high / medium / low
|
||||
preflight_has_false_premise bool
|
||||
preflight_has_contradiction bool
|
||||
preflight_corpus_requirement str
|
||||
```
|
||||
|
||||
Tests grew to 68 in the metacognition suite (42 Phase 1 + 16 new
|
||||
Phase 4: 6 governance + 6 audit-line tail + 4 default-policy
|
||||
pinning).
|
||||
|
||||
### 12.5 Live verification
|
||||
|
||||
`make query-dry` end-to-end:
|
||||
|
||||
```
|
||||
$ make query-dry Q="Who is the current CEO of OpenAI?" BURN=1
|
||||
Who is the current CEO of OpenAI?
|
||||
UNGROUNDED · via claim_lattice · stale risk 0/0 73.4s (fresh)
|
||||
|
||||
$ make query-dry Q="When did Mr. Burns become Homer's biological father?" BURN=1
|
||||
UNGROUNDED · via claim_lattice · false premise 0/0 46.4s (fresh)
|
||||
```
|
||||
|
||||
Both metacog tails land on the audit line; preflight runs pre-
|
||||
retrieval; QuestionState surfaces on the result dict.
|
||||
|
||||
### 12.6 What's NOT shipped in this ticket
|
||||
|
||||
- **Run-DAG node binding** for the metacognition_preflight stage.
|
||||
Hashing the QuestionState into `run_dag_root` so audit replay
|
||||
can distinguish guard-on vs guard-off, contradiction-detected
|
||||
vs not, etc. Same audit-replay gap as ticket #000009. Both
|
||||
Phase 5 work; will land together.
|
||||
- **Reference-frame plumbing.** `preflight_question()` accepts
|
||||
`reference_frames` but the `query()` callsite passes empty —
|
||||
frame_detection runs after retrieval (it needs source titles)
|
||||
and preflight here is pre-retrieval. Wiring the post-retrieval
|
||||
frame info back into the QuestionState requires either a two-
|
||||
pass call or a post-classification update; deferred.
|
||||
- **SOFT_PREFLIGHT_HINT (model-assisted preflight).** Source doc
|
||||
§18 reserves this label for future model-assisted preflight
|
||||
that runs as a sidecar. Not implemented; hard rule preserved
|
||||
(no LLM in the preflight hard path).
|
||||
- **Bench measurement of preflight effect.** The mechanism is
|
||||
shipped; bench A/B (preflight on vs off) on the broad subset
|
||||
not yet run. Quick to do once stack settles.
|
||||
|
||||
## 13. Status — closed
|
||||
|
||||
Phases 1–4 landed 2026-05-03 in commit chain `55efb04` →
|
||||
(this commit). Mechanism complete; defaults preserve the dry-run
|
||||
discipline pattern from #000008.
|
||||
|
||||
Follow-ups split into separate tickets:
|
||||
|
||||
- Phase 5 (run-DAG node binding): joins ticket #000009's scope
|
||||
- Cross-model bench: manual operator task
|
||||
|
||||
This ticket file remains in place as the design log per
|
||||
`docs/TICKETS.md` convention.
|
||||
|
|
|
|||
|
|
@ -73,6 +73,15 @@ def _row(**overrides) -> dict:
|
|||
"scope_bound_hint": None,
|
||||
"claim_cap_applied": None,
|
||||
"model_profile_id": "test-model",
|
||||
# Ticket #000010 — preflight bench fields (defaults match a
|
||||
# neutral, well-formed-question run).
|
||||
"preflight_logical_statuses": ["well_formed"],
|
||||
"preflight_question_shape": "single_fact",
|
||||
"preflight_result": "PREFLIGHT_OK",
|
||||
"preflight_temporal_sensitivity": "low",
|
||||
"preflight_has_false_premise": False,
|
||||
"preflight_has_contradiction": False,
|
||||
"preflight_corpus_requirement": "encyclopedic",
|
||||
"directive_compliance": {
|
||||
"D2_pointer_clauses": True,
|
||||
"D3_cti_substrate_ready": True,
|
||||
|
|
|
|||
|
|
@ -290,3 +290,123 @@ def test_question_state_is_serializable():
|
|||
# Round-trip through json.dumps; failure here means a non-
|
||||
# serializable type leaked in.
|
||||
json.dumps(payload, ensure_ascii=False)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- governance binding (Phase 3)
|
||||
|
||||
@pytest.mark.parametrize("field", [
|
||||
"metacognition_enabled",
|
||||
"metacognition_temporal_check",
|
||||
"metacognition_contradiction_check",
|
||||
"metacognition_false_premise_check",
|
||||
"metacognition_out_of_corpus_check",
|
||||
"metacognition_block_on_contradiction",
|
||||
])
|
||||
def test_metacognition_field_is_in_verifier_policy_fields(field):
|
||||
"""All six policy fields must be in _VERIFIER_POLICY_FIELDS so
|
||||
flipping any of them invalidates prior cache records on lookup."""
|
||||
from aborist.qa.keys import _VERIFIER_POLICY_FIELDS
|
||||
assert field in _VERIFIER_POLICY_FIELDS
|
||||
|
||||
|
||||
def test_governance_hash_changes_when_metacognition_enabled_flips():
|
||||
from aborist.qa.keys import _VERIFIER_POLICY_FIELDS, verifier_policy_hash
|
||||
base = dict.fromkeys(_VERIFIER_POLICY_FIELDS, "default")
|
||||
base["metacognition_enabled"] = True
|
||||
h_on = verifier_policy_hash(base)
|
||||
base["metacognition_enabled"] = False
|
||||
h_off = verifier_policy_hash(base)
|
||||
assert h_on != h_off
|
||||
|
||||
|
||||
def test_governance_hash_changes_when_block_on_contradiction_flips():
|
||||
from aborist.qa.keys import _VERIFIER_POLICY_FIELDS, verifier_policy_hash
|
||||
base = dict.fromkeys(_VERIFIER_POLICY_FIELDS, "default")
|
||||
base["metacognition_block_on_contradiction"] = False
|
||||
h_off = verifier_policy_hash(base)
|
||||
base["metacognition_block_on_contradiction"] = True
|
||||
h_on = verifier_policy_hash(base)
|
||||
assert h_off != h_on
|
||||
|
||||
|
||||
def test_default_policy_has_metacognition_enabled():
|
||||
"""Master switch default-on per ticket #000010 §7.3 — detectors
|
||||
are pure-on-question so the cost is negligible."""
|
||||
from aborist.qa.runner import DEFAULT_POLICY as RUNNER_POLICY
|
||||
from aborist.qa.query import DEFAULT_QUERY_POLICY
|
||||
assert RUNNER_POLICY["metacognition_enabled"] is True
|
||||
assert DEFAULT_QUERY_POLICY["metacognition_enabled"] is True
|
||||
|
||||
|
||||
def test_default_policy_block_on_contradiction_off():
|
||||
"""Default to label-only on contradictions. False-positive risk
|
||||
not yet bench-validated; opt-in via --block-on-contradiction."""
|
||||
from aborist.qa.runner import DEFAULT_POLICY as RUNNER_POLICY
|
||||
from aborist.qa.query import DEFAULT_QUERY_POLICY
|
||||
assert RUNNER_POLICY["metacognition_block_on_contradiction"] is False
|
||||
assert DEFAULT_QUERY_POLICY["metacognition_block_on_contradiction"] is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- audit-line tails (Phase 4)
|
||||
|
||||
def _result_with_question_state(state: QuestionState) -> dict:
|
||||
"""Synthetic query() result with the minimum fields needed by
|
||||
`_render_warrant_tail`."""
|
||||
return {
|
||||
"violations": [],
|
||||
"claim_cap_applied": None,
|
||||
"question_state": state.to_dict(),
|
||||
}
|
||||
|
||||
|
||||
def test_tail_renders_false_premise():
|
||||
from aborist.cli import _render_warrant_tail
|
||||
state = preflight_question(
|
||||
"When did Mr. Burns become Homer's biological father?"
|
||||
)
|
||||
tail = _render_warrant_tail(_result_with_question_state(state))
|
||||
assert "false premise" in tail
|
||||
|
||||
|
||||
def test_tail_renders_contradictory():
|
||||
from aborist.cli import _render_warrant_tail
|
||||
state = preflight_question("Which unmarried spouse is Homer married to?")
|
||||
tail = _render_warrant_tail(_result_with_question_state(state))
|
||||
assert "contradictory" in tail
|
||||
|
||||
|
||||
def test_tail_renders_stale_risk():
|
||||
from aborist.cli import _render_warrant_tail
|
||||
state = preflight_question("Who is the current CEO of OpenAI?")
|
||||
tail = _render_warrant_tail(_result_with_question_state(state))
|
||||
assert "stale risk" in tail
|
||||
|
||||
|
||||
def test_tail_renders_out_of_corpus():
|
||||
from aborist.cli import _render_warrant_tail
|
||||
state = preflight_question("What does my uploaded contract say?")
|
||||
tail = _render_warrant_tail(_result_with_question_state(state))
|
||||
assert "out of corpus" in tail
|
||||
|
||||
|
||||
def test_tail_omits_metacog_when_well_formed():
|
||||
from aborist.cli import _render_warrant_tail
|
||||
state = preflight_question("what is the capital of france?")
|
||||
tail = _render_warrant_tail(_result_with_question_state(state))
|
||||
# well_formed should NOT produce any metacog tail tokens.
|
||||
assert "false premise" not in tail
|
||||
assert "contradictory" not in tail
|
||||
assert "stale risk" not in tail
|
||||
assert "out of corpus" not in tail
|
||||
|
||||
|
||||
def test_tail_combines_metacog_with_existing_kinds():
|
||||
"""Multiple tails compose: a TITLE_MISMATCH from the verifier
|
||||
plus a stale_risk from the preflight should both surface."""
|
||||
from aborist.cli import _render_warrant_tail
|
||||
state = preflight_question("Who is the current CEO of OpenAI?")
|
||||
result = _result_with_question_state(state)
|
||||
result["violations"] = [{"kind": "TITLE_MISMATCH"}]
|
||||
tail = _render_warrant_tail(result)
|
||||
assert "title mismatch" in tail
|
||||
assert "stale risk" in tail
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue