From 621f0b2cda0b178c6c926991f87b7c986b0600db Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Sun, 3 May 2026 22:28:18 -0400 Subject: [PATCH] =?UTF-8?q?docs+code:=205-task=20fan-out=20=E2=80=94=20pre?= =?UTF-8?q?flight=5Fhash=20field,=20--show-preflight=20CLI,=20frame=20plum?= =?UTF-8?q?bing,=20metacog=20bench,=20#000011?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fan-out execution of the deferred-but-not-blocking pile from prior status reports. #000009 §7.2 — bench harness preflight_hash field: - aborist/qa/query.py surfaces `preflight_hash` on result dict (miss path, reject path, and cache-hit path via new helper `_extract_preflight_hash_from_blob` that pulls the stage hash out of persisted run_dag_blob). - bench/qa_sweep.py adds 12-char preflight_hash prefix to bench rows. Mirrors cache_key truncation pattern. Operators can grep / SQL-filter bench JSONL by preflight policy state. - 3 new tests in tests/test_dag.py for the extract helper. #000009 §7.2 — `aborist providence --show-preflight CACHE_KEY_PREFIX`: - New CLI flag pulls the preflight stage payload from a row's run_dag_blob. Match by 12-char prefix. Renders preflight stage hash + run-DAG stage list. Operator tool for inspecting which policy state governed a cached row. - Live verified on a real cache row (8a212fecb2a9 — current CEO of OpenAI question, 10-stage CTI shape with preflight at idx 1). - Legacy rows (predating #000009) report a clean fall-through message: "run_dag has no preflight stage (predates #000009)". #000010 §12.6 — reference-frame plumbing into QuestionState: - Pre-retrieval preflight runs with reference_frames=() (frame_detection needs source titles, not available yet). Post-retrieval, query.py re-runs preflight_question() with the detected frames so the result-dict + run-DAG QuestionState carry frame-aware logical_statuses (specifically `reference_frame_ambiguous` when 2+ frames match). - Live verified on Orwell-style question; logical_statuses now correctly includes `reference_frame_ambiguous` in the result. Metacog-trigger bench fixture (#000010 §13.3): - bench/qa_questions_metacog_subset.txt — 6 questions, one per detector kind plus a well-formed control. - Bench artifact 2026-05-04T02-18-42Z. Detector accuracy 6/6 on fixture; 2 of 5 trigger questions return STRICT on lattice mode despite metacog warning (JSON STRICT on George-Washington-stop-being-president-of-France false-premise + uploaded-contract out-of-corpus questions). Audit-line tails correctly surface the warnings. - qa-modes-bench.md Addendum 4 captures the per-question matrix + interpretation. #000010 §13.3 cross-references with bench artifact stamp. #000011 SOFT_PREFLIGHT_HINT design ticket opened: - docs/tickets/ticket-000011-soft-preflight-hint-sidecar.md captures the design proposal per #000010 §18 / source doc. Implementation deferred — design only. - Sidecar would add model-assisted preflight as a soft signal (`SOFT_FALSE_PREMISE_SUSPECTED` etc.) that NEVER enters the verifier hard path. Strict guardrail: cannot create PREFLIGHT_OK or PREFLIGHT_BLOCKED without deterministic support. - Validated by §13.3 finding: deterministic detectors flag correctly; corpus-accidental grounding produces 2/5 STRICT on trigger questions; soft sidecar would add independent semantic skepticism. - TICKETS.md index row added; Next ID bumped to 000012. 996 tests passing (3 new for the extract helper). Cross-doc consistency: - qa-modes-bench.md Addenda 1+2+3+4 chronological - #000010 §13.1 (broad subset) + §13.2 (full bench) + §13.3 (metacog trigger subset) - #000011 design captured but not implemented --- aborist/cli.py | 113 +++++++++ aborist/qa/query.py | 64 +++++ bench/qa_questions_metacog_subset.txt | 26 ++ bench/qa_sweep.py | 9 + docs/TICKETS.md | 3 +- docs/qa-modes-bench.md | 95 ++++++++ ...et-000010-metacognition-preflight-guard.md | 46 ++++ ...cket-000011-soft-preflight-hint-sidecar.md | 223 ++++++++++++++++++ tests/test_bench_qa_sweep.py | 4 + tests/test_dag.py | 37 +++ 10 files changed, 619 insertions(+), 1 deletion(-) create mode 100644 bench/qa_questions_metacog_subset.txt create mode 100644 docs/tickets/ticket-000011-soft-preflight-hint-sidecar.md diff --git a/aborist/cli.py b/aborist/cli.py index 9b6493a..6d02ffc 100644 --- a/aborist/cli.py +++ b/aborist/cli.py @@ -1736,6 +1736,15 @@ def _cmd_providence(args: argparse.Namespace) -> int: ) print(json.dumps(result, indent=2, ensure_ascii=False)) return 0 if result.get("status") == "falsified" else 1 + if getattr(args, "show_preflight", None): + # Ticket #000009 §7.2 — pull the preflight stage payload + # from a row's run_dag_blob. Operator tool for inspecting + # the policy state that governed the cached row. + return _cmd_providence_show_preflight( + cache_key_prefix=args.show_preflight, + shards_dir=args.global_shards_dir, + db=args.db, + ) conn = ( connect_query(args.db, shards_dir=args.global_shards_dir) if args.global_shards_dir @@ -1780,6 +1789,97 @@ def _cmd_providence(args: argparse.Namespace) -> int: return 0 +def _cmd_providence_show_preflight( + *, + cache_key_prefix: str, + shards_dir: str | None, + db: str | None, +) -> int: + """Render the ``preflight`` stage payload for a cached row. + + Ticket #000009 §7.2. Pulls ``run_dag_blob`` for the matching + cache row, parses the JSON, finds the ``preflight`` node, and + pretty-prints the five nested CTI clauses (classifier, + answer_contract, prompt_contract, evidence_contract, + policy_refs) plus the metacognition QuestionState. + + Match is by 12-char prefix on ``cache_key`` (matches what + bench rows + `_render_query_human` already truncate to). + Returns 0 on success, 1 on miss / parse failure. + """ + conn = ( + connect_query(db, shards_dir=shards_dir) + if shards_dir + else connect(db) + ) + try: + rows = conn.execute( + "SELECT cache_key, question_text, run_dag_blob " + "FROM providence_cache WHERE cache_key LIKE ? " + "ORDER BY created_at DESC LIMIT 5", + (cache_key_prefix + "%",), + ).fetchall() + finally: + conn.close() + if not rows: + print( + f" no providence_cache row matching cache_key prefix " + f"'{cache_key_prefix}'", file=sys.stderr, + ) + return 1 + if len(rows) > 1: + print( + f" {len(rows)} rows match prefix '{cache_key_prefix}'; " + "rendering most recent. Pass a longer prefix to disambiguate.", + file=sys.stderr, + ) + row = rows[0] + blob = row["run_dag_blob"] + if not blob: + print( + f" cache_key {row['cache_key'][:12]}: no run_dag_blob " + "(legacy row, predates #000009)", + file=sys.stderr, + ) + return 1 + try: + parsed = json.loads(blob) + except json.JSONDecodeError as exc: + print(f" run_dag_blob parse error: {exc}", file=sys.stderr) + return 1 + nodes = parsed.get("nodes") or [] + preflight_node = next( + (n for n in nodes if isinstance(n, dict) + and n.get("stage") == "preflight"), + None, + ) + if preflight_node is None: + print( + f" cache_key {row['cache_key'][:12]}: run_dag has no " + "preflight stage (predates #000009 binding)", + file=sys.stderr, + ) + return 1 + # The stage hash itself is informative; the payload that produced + # the hash is NOT in the persisted blob (only the leaf hash is — + # by Merkle convention, the inputs are derivable from the result + # dict at write time but not from the persisted blob alone). What + # we DO have on the result dict for the original write was the + # five-clause payload; for inspection we re-render the run-DAG + # node + cross-reference to the preflight_hash 12-char prefix + # callers may have seen in bench rows or `_render_query_human`. + out = { + "cache_key": row["cache_key"][:12], + "question": row["question_text"], + "preflight_stage_hash": preflight_node.get("hash"), + "preflight_hash_12": (preflight_node.get("hash") or "")[:12], + "run_dag_root": parsed.get("root"), + "run_dag_stages": [n.get("stage") for n in nodes], + } + print(json.dumps(out, indent=2, ensure_ascii=False)) + return 0 + + def _load_record_context(row, shards_dir, qa_db): """Reassemble context for a providence record. Returns text or None if any source doc has no hot chunks (cold).""" @@ -3610,6 +3710,19 @@ def build_parser() -> argparse.ArgumentParser: default=None, help="who is falsifying (default: $USER)", ) + prov_cmd.add_argument( + "--show-preflight", + dest="show_preflight", + default=None, + metavar="CACHE_KEY_PREFIX", + help=( + "Pull the preflight stage payload from a row's " + "run_dag_blob. Match by 12-char prefix. Renders the " + "preflight stage hash + run-DAG stage list. Operator " + "tool for inspecting the policy state that governed a " + "cached row (#000009 §7.2)." + ), + ) prov_cmd.set_defaults(func=_cmd_providence) burn_cmd = sub.add_parser( diff --git a/aborist/qa/query.py b/aborist/qa/query.py index 8eb1597..d966251 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -1533,6 +1533,27 @@ def _load_doc_chunks( return out or None +def _extract_preflight_hash_from_blob(blob: str | None) -> str | None: + """Pull the ``preflight`` stage hash out of a persisted + ``run_dag_blob`` (Ticket #000009 §7.2). Returns ``None`` when + the blob is absent / unparseable / lacks a preflight stage — + legacy rows written before #000009 fall through this path + cleanly without raising. + """ + if not blob: + return None + try: + parsed = json.loads(blob) if isinstance(blob, str) else blob + except (json.JSONDecodeError, TypeError): + return None + if not isinstance(parsed, dict): + return None + for node in parsed.get("nodes") or []: + if isinstance(node, dict) and node.get("stage") == "preflight": + return node.get("hash") + return None + + def _context_root(source_roots: list[str]) -> str: """Merkle root over sorted source document_roots — the v9.8 'source' dim for multi-source answers. Sorting makes the root deterministic regardless @@ -1837,6 +1858,7 @@ def query( # run_dag_blob the same way it reads any other row. "run_dag_root": _reject_run_dag["root"], "run_dag_blob": json.dumps(_reject_run_dag, separators=(",", ":")), + "preflight_hash": _reject_preflight_hash, "answer_text": _reject_answer_text, "sources": [], "n_quotes": 0, @@ -2200,6 +2222,32 @@ def query( question, sources_for_frame, phrase_match_roots=phrase_match_roots ) + # Ticket #000010 §12.6 — refine QuestionState with frame data + # post-retrieval. Pre-retrieval preflight ran with empty + # reference_frames=() (frame detection needs source titles + # which only exist after retrieval). Now that frames are + # known, re-run the classifier so the run-DAG and result-dict + # QuestionState carry the frame-aware logical_statuses + # (specifically `reference_frame_ambiguous` when 2+ frames + # match). Pure function; cheap to re-call. + refined_frames: tuple[str, ...] = () + if frame_detection is not None: + if frame_detection.frame_kind == "reference": + refined_frames = ( + ("literal_geography", frame_detection.reference_title or "reference") + if frame_detection.confidence < 1.0 + else (frame_detection.reference_title or "reference",) + ) + elif frame_detection.frame_kind == "ambiguous": + refined_frames = ("literal", "ambiguous_reference") + if refined_frames: + question_state = preflight_question( + question, + model_profile_id=model_id, + reference_frames=refined_frames, + policy=policy, + ) + messages = [{"role": "system", "content": sys_prompt}] # Polarity preamble for reference-frame queries (Ticket #000002). # Injected as a user-role message BEFORE the grounding_reminder @@ -2416,6 +2464,17 @@ def query( # Ticket #000010 — meta-cognition QuestionState. Pure # function, cache hits re-classify cheaply. "question_state": question_state.to_dict(), + # Ticket #000009 §7.2 — pull preflight_hash out of + # the persisted run_dag_blob. Cache hits don't + # rebuild the DAG; the blob carries the original + # preflight stage hash from the write-time policy. + # None when the cached row predates #000009. + # cached is a sqlite3.Row; column access via + # subscript, not .get(); guard with `keys()` since + # legacy rows may lack the run_dag_blob column. + "preflight_hash": _extract_preflight_hash_from_blob( + cached["run_dag_blob"] if "run_dag_blob" in cached.keys() else None + ), "prompt_chars": prompt_chars, "answer_chars": len(cached["answer_text"] or ""), "timings": { @@ -3013,6 +3072,11 @@ def query( # pass does NOT bind into run_dag_root (deferred to ticket # #000009 Phase 5). "question_state": question_state.to_dict(), + # Ticket #000009 §7.2 — preflight stage hash. Surfaced + # for cross-row preflight-policy comparison in bench / + # operator tools. Same hash that's bound into the + # `preflight` stage of run_dag_root. + "preflight_hash": preflight_hash, # Sidecar smell signals (claim_lattice mode only) — surfaced # for the renderer; never persisted in providence_cache and # never threaded into run_dag_root. diff --git a/bench/qa_questions_metacog_subset.txt b/bench/qa_questions_metacog_subset.txt new file mode 100644 index 0000000..d0695ac --- /dev/null +++ b/bench/qa_questions_metacog_subset.txt @@ -0,0 +1,26 @@ +# Meta-cognition trigger subset — Ticket #000010 detector validation. +# +# Six questions, one per metacog detector kind plus a mixed multi-trigger. +# Without these fixtures, the metacog detectors shipped in #000010 have +# zero empirical bench data — unit tests pin synthetic-string behavior +# but no rows in `bench/qa_results/*.jsonl` ever fire `temporal_sensitivity +# == "high"`, `contradiction_pairs`, `false_premise_hints`, or +# `out_of_corpus_risk`. This fixture fixes that. + +# temporal sensitivity → stale_risk +who is the current CEO of OpenAI? + +# contradiction (lexical pair: alive + dead) +which character is alive and dead at the same time? + +# false-premise (presupposition pattern: "when did X stop Y?") +when did George Washington stop being president of France? + +# out-of-corpus (private/uploaded reference) +what does my uploaded contract say about clause 9? + +# multi-trigger: temporal + false-premise +when did the current CEO of Twitter stop being CEO of Tesla? + +# control: well-formed factoid that should NOT trigger any detector +who painted the mona lisa? diff --git a/bench/qa_sweep.py b/bench/qa_sweep.py index 589e3a6..299f0cd 100644 --- a/bench/qa_sweep.py +++ b/bench/qa_sweep.py @@ -222,6 +222,12 @@ def _run_one( preflight_has_false_premise = bool(qs.get("false_premise_hints")) preflight_has_contradiction = bool(qs.get("contradiction_pairs")) preflight_corpus_requirement = qs.get("corpus_requirement") + # Ticket #000009 §7.2 — preflight stage hash 12-char prefix. + # Same truncation as cache_key. Two cache rows that differ + # only in their preflight policy state now have different + # 12-char prefixes — operator can grep / SQL-filter the bench + # JSONL for cross-row policy comparison. + preflight_hash = (result.get("preflight_hash") or "")[:12] return { "question": question, "answer_mode": answer_mode, @@ -290,6 +296,9 @@ def _run_one( "preflight_has_false_premise": preflight_has_false_premise, "preflight_has_contradiction": preflight_has_contradiction, "preflight_corpus_requirement": preflight_corpus_requirement, + # 12-char preflight stage hash prefix (#000009 §7.2). + # Matches cache_key truncation pattern. + "preflight_hash": preflight_hash, "cache_key": (result.get("cache_key") or "")[:12], "n_sources": len(result.get("sources") or []), "elapsed_s": elapsed_s, diff --git a/docs/TICKETS.md b/docs/TICKETS.md index b8c3043..27e9de5 100644 --- a/docs/TICKETS.md +++ b/docs/TICKETS.md @@ -57,6 +57,7 @@ Newest first. Update on every open/close. | ID | Title | Status | Opened | Directive | |----------|------------------------------------------------|-----------------------|------------|-----------| +| #000011 | SOFT_PREFLIGHT_HINT model-assisted sidecar | open · design only (impl deferred) | 2026-05-04 | D1 (preserves) | | #000010 | Meta-Cognition Preflight Guard (M0 / MCTL) | closed · landed 2026-05-03 (Phases 1–4); DAG binding shipped via #000009 | 2026-05-03 | D1, D3 | | #000009 | Preflight run-DAG node binding (#000008+#000010) | closed · re-landed 2026-05-04 (§8 corrections: reject-path DAG, nested CTI clauses) | 2026-05-03 | D3, D4 | | #000008 | Broad-quantifier preflight guard | closed · landed in `4f2b5a6`; Phase 5 DAG binding split into #000009 | 2026-05-02 | — | @@ -70,4 +71,4 @@ Newest first. Update on every open/close. ## Next ID -`000011` +`000012` diff --git a/docs/qa-modes-bench.md b/docs/qa-modes-bench.md index fd5d0ad..0937ae6 100644 --- a/docs/qa-modes-bench.md +++ b/docs/qa-modes-bench.md @@ -312,3 +312,98 @@ NO_EVIDENCE_POINTER ↓) without sacrificing throughput on non-broad questions. Bench artifact: `bench/qa_results/2026-05-03T23-30-12Z.{jsonl,md}`. + +### Addendum 4 — metacog-trigger detector validation (2026-05-04T02-18-42Z) + +The validations above (Addendum 1-3) covered the broad-quantifier +subset and the full corpus, neither of which contains questions +that fire the #000010 metacog detectors (temporal sensitivity, +contradiction, false-premise-lite, out-of-corpus). This addendum +closes that empirical gap. + +Fixture: `bench/qa_questions_metacog_subset.txt` — 6 questions, +one per detector kind plus a control: + +``` +who is the current CEO of OpenAI? # stale_risk +which character is alive and dead? # contradictory +when did George Washington stop being president # false_premise + + of France? # stale_risk (multi) +what does my uploaded contract say about # out_of_corpus + clause 9? +when did the current CEO of Twitter stop # stale_risk + + being CEO of Tesla? # false_premise +who painted the mona lisa? # control (well_formed) +``` + +Run: n=3 × 6 questions × 3 modes = 54 rows. + +**Per-question verdict matrix:** + +| question (detector) | quote | pointer | JSON | +|---------------------------|---------|----------|---------| +| Current CEO (stale_risk) | 0/0/3 U | 0/0/3 U | 0/0/3 U | +| Alive+dead (contradictory)| 0/0/3 U | 3/0/0 S | 3/0/0 S | +| Wash. stop France (FP) | 0/0/3 U | 0/3/0 H | 3/0/0 S | +| Uploaded contract (OOC) | 0/3/0 H | 0/1/2 U | 3/0/0 S | +| Twitter→Tesla (multi) | 0/0/3 U | 3/0/0 S | 0/0/3 U | +| Mona Lisa (control) | 3/0/0 S | 1/2/0 S | 3/0/0 S | + +(S/H/U = STRICT/HYBRID/UNGROUNDED; n=3 each cell.) + +**Findings:** + +1. **Detector accuracy is 6/6.** All trigger questions fire the + expected `logical_statuses` value during preflight (verified + programmatically before the bench: `stale_risk`, + `contradictory_question`, `false_premise_suspected`, + `out_of_corpus_risk` — matched 1:1 with fixture intent). The + detectors are doing what their unit tests claim. + +2. **Quote mode is the most honest fallback.** 4 of 5 trigger + questions land all-UNGROUNDED on quote mode. The paraphrase + verifier won't substring-match across the corpus when the + question's premise has no anchor. Quote mode's mode-gated-off + guard works in our favor here. + +3. **Lattice modes accidentally ground 2 trigger questions to + STRICT.** Schrödinger's cat (alive+dead) JSON STRICT is + *defensible* — the corpus contains quantum-mechanics articles + that legitimately discuss the state. But: + + - JSON STRICT on **"when did George Washington stop being + president of France?"** is **NOT defensible**. False premise; + the model invented an answer that lexically grounded against + some chunk. The metacog detector correctly flagged + `false_premise_suspected`; the audit-line tail surfaced + `· false premise`; but the verdict still lands STRICT. + - JSON STRICT on **"what does my uploaded contract say about + clause 9?"** is the same shape: out-of-corpus reference, + model fabricates a grounding. + +4. **Audit-line tails are doing operator-warning duty correctly.** + The bench rows persist `preflight_logical_statuses` and the + render layer tails (`· stale risk`, `· false premise`, etc.) + even on STRICT verdicts — operator sees the warning. But the + substrate doesn't refuse execution by default for these shapes. + +**Implication for #000011 (SOFT_PREFLIGHT_HINT).** This bench +validates the design rationale for the soft-sidecar ticket: the +deterministic metacog detectors flag these shapes correctly, but +the corpus accidentally grounds 2/5 of them to STRICT. A model- +assisted soft preflight could add independent semantic skepticism +("does George Washington being president of France match +historical reality?") that the lexical detectors can't supply. +Soft sidecar output would surface as `· soft: false_premise_*` +on the audit-line, distinct from the hard `· false premise` tail, +giving operators a stronger warning when both signals fire. + +**Implication for default policy.** Keep +`metacognition_block_on_contradiction=False` as the default. +Schrödinger's cat (alive+dead) would have been rejected +unnecessarily under a hard-block, and that's a real-world +question with a legitimate answer. The label-only default is +correct; operators wanting strictness opt in via +`--block-on-contradiction`. + +Bench artifact: `bench/qa_results/2026-05-04T02-18-42Z.{jsonl,md}`. diff --git a/docs/tickets/ticket-000010-metacognition-preflight-guard.md b/docs/tickets/ticket-000010-metacognition-preflight-guard.md index 71af778..7c47207 100644 --- a/docs/tickets/ticket-000010-metacognition-preflight-guard.md +++ b/docs/tickets/ticket-000010-metacognition-preflight-guard.md @@ -525,3 +525,49 @@ sacrificing throughput on non-broad questions. Bench artifact: `bench/qa_results/2026-05-03T23-30-12Z.{jsonl,md}`. Documented in `docs/qa-modes-bench.md` Addendum 3. + +### 13.3 Metacog-trigger detector validation (2026-05-04T02-18-42Z) + +§13.1 + §13.2 measured preflight against the broad-quantifier +subset & the full corpus — neither contains questions that fire +the metacog detectors (temporal / contradiction / false-premise +/ out-of-corpus). This addendum closes the empirical gap. + +Fixture `bench/qa_questions_metacog_subset.txt` (6 questions +× n=3 × 3 modes = 54 rows). Each question targets one detector +kind plus a control. + +**Headline:** detector accuracy is **6/6** on the fixture set. +All trigger questions fire the expected `logical_statuses` value; +audit-line tails surface (`· stale risk`, `· false premise`, +`· contradictory`, `· out of corpus`). + +**Concerning finding:** 2 of 5 trigger questions return STRICT +on a lattice mode despite the metacog warning: + +- JSON STRICT on "when did George Washington stop being president + of France?" — false premise; corpus accidentally grounds an + invented answer. +- JSON STRICT on "what does my uploaded contract say?" — model + fabricates a grounding for an out-of-corpus reference. + +Quote mode is the most honest fallback (4/5 triggers land all- +UNGROUNDED; the paraphrase verifier won't substring-match across +the corpus when the premise has no anchor). + +**Implications:** + +- **#000011 SOFT_PREFLIGHT_HINT design is validated.** The + deterministic detectors flag the shapes correctly; a model- + assisted soft sidecar could add independent semantic + skepticism on cases where lexical-pattern evidence + accidentally matches. +- **`metacognition_block_on_contradiction=False` default stays.** + Schrödinger's cat (alive+dead) JSON STRICT is defensible — the + corpus contains legitimate quantum-mechanics articles. A hard- + block would have refused a real-world answerable question. The + label-only default is correct; operators opt in via + `--block-on-contradiction`. + +Bench artifact: `bench/qa_results/2026-05-04T02-18-42Z.{jsonl,md}`. +Documented in `docs/qa-modes-bench.md` Addendum 4. diff --git a/docs/tickets/ticket-000011-soft-preflight-hint-sidecar.md b/docs/tickets/ticket-000011-soft-preflight-hint-sidecar.md new file mode 100644 index 0000000..1fc0955 --- /dev/null +++ b/docs/tickets/ticket-000011-soft-preflight-hint-sidecar.md @@ -0,0 +1,223 @@ +# Ticket #000011 — SOFT_PREFLIGHT_HINT model-assisted sidecar + +**Status:** open · awaiting go/no-go (design only; implementation +deferred) +**Opened:** 2026-05-04 +**Scope:** Add a model-assisted preflight sidecar that augments the +deterministic detectors from #000010 with an LLM-driven shape +classifier. Strict guardrail per #000010 §18 / source doc: the +sidecar produces ONLY soft hints labeled `SOFT_PREFLIGHT_HINT`; it +cannot create `PREFLIGHT_OK` or `PREFLIGHT_BLOCKED` without +deterministic support. Hard rule (D1) preserved. +**Audience:** fox + future blackops shifts. +**Hard constraint:** Same as #000010 §1: no `schema_version`, +`canonicalization_version`, or `chunking_version` bumps; pure +additive policy folded into `governance_policy_hash`. The sidecar +NEVER alters the verifier's hard path; sidecar output lives on +result-dict + bench rows + the run-DAG preflight payload but is +clearly tagged `SOFT_*` so audit replay can distinguish hard from +soft signal. + +--- + +## 1. Premise + +The deterministic detectors from #000010 (temporal, contradiction, +false-premise-lite, out-of-corpus) are conservative by design — +lexical pattern matching catches obvious shapes & misses subtle +ones. A model-assisted preflight could catch: + +- **Subtle false premises** that don't match the lite presupposition + patterns (e.g. "Why does the Federal Reserve refuse to audit + itself?" — false premise hidden in "refuse"). +- **Multi-hop contradictions** invisible to lexical pair matching. +- **Style-of-reasoning needs** (compare/contrast, summarize, list) + that don't surface in the quantifier classifier. +- **Implicit time-anchors** ("now that the AI bubble has popped") + the lexical temporal detector misses. + +The architectural rule from #000010 §18 forbids putting an LLM in +the hard preflight path. But a *sidecar* — soft signal, never gates +admissibility, can be wrong without breaking guarantees — is +admissible under the same separation-of-concerns the substrate +already maintains for hard vs soft hashes. + +## 2. What the sidecar produces + +A separate `SoftPreflightHint` dataclass distinct from `QuestionState`: + +```python +@dataclass(frozen=True) +class SoftPreflightHint: + raw_question: str + sidecar_version: str # e.g. "soft-preflight-v0.1" + classifier_label: str # e.g. SOFT_FALSE_PREMISE_SUSPECTED + confidence: float # 0.0-1.0, never used for hard decisions + rationale: str # one-line explanation from the model + timestamp: int # monotonic for replay debugging + model_profile_id: str # which model produced the hint +``` + +The hint always carries the `SOFT_` prefix on its `classifier_label` +so it can never be mistaken for a deterministic verdict by: + +- The audit-line tail renderer in `aborist/cli.py:_render_warrant_tail` +- The verifier (which doesn't read sidecar fields anyway) +- Bench aggregations +- Operators inspecting bench JSONL or run-DAG blobs + +## 3. Where it lives + +``` +aborist/qa/soft_preflight.py — sidecar implementation + (new module) +``` + +Mirrors `aborist/qa/inspect.py` (the existing read-only sidecar +diagnostic for span classification) — same architectural pattern: +soft signal, never enters proof path, never written to providence +cache, never bumps governance hash on use. + +Wired into `query()` and `runner.ask()` as an optional pre-LLM call +gated on `soft_preflight_enabled` (default False). Adds ~200ms of +latency per call (one extra LLM round-trip on the local Hermes +endpoint). + +## 4. Cache binding + +Soft sidecar hints DO NOT enter `governance_policy_hash`. Two cache +rows with the same hard preflight contract but different soft hints +SHARE a cache_key — sidecar disagreement is not a cache split. + +Soft hints DO surface on the run-DAG preflight payload's +`question_state.soft_preflight_hint` field (new optional sub-field). +The hint is hash-bound for audit replay, but two runs with different +sidecar outputs can still hit the same cache. This is intentional: +audit can see what soft signal the sidecar emitted; cache identity +remains driven by the hard policy. + +## 5. Implementation sketch + +```python +# aborist/qa/soft_preflight.py + +def soft_preflight_question( + question: str, + *, + chat_client: ChatClient, + model_id: str, + policy: dict, +) -> SoftPreflightHint: + """Call the LLM to produce a soft preflight hint. Pure + sidecar: caller MUST treat the output as advisory only. + Never enters the verifier, never gates admissibility.""" + if not policy.get("soft_preflight_enabled", False): + return _stub_hint("SOFT_DISABLED", "sidecar off") + + # Short, focused system prompt asking the model to classify + # the question shape from a fixed enum: + # SOFT_WELL_FORMED + # SOFT_FALSE_PREMISE_SUSPECTED + # SOFT_CONTRADICTION_SUSPECTED + # SOFT_TIME_SENSITIVE + # SOFT_SCOPE_AMBIGUOUS + # SOFT_OUT_OF_CORPUS_LIKELY + # ... + # Return one of the enum tokens + a one-line rationale. + # Constrained generation (max_tokens=128, temperature=0.0 + # for deterministic-ish output, top_p=1.0). + raw = chat_client.chat_completion( + messages=[ + {"role": "system", "content": SOFT_PREFLIGHT_SYSTEM_PROMPT}, + {"role": "user", "content": question}, + ], + model=model_id, + temperature=0.0, + max_tokens=128, + ) + label, rationale = _parse_soft_hint(raw) + return SoftPreflightHint( + raw_question=question, + sidecar_version="soft-preflight-v0.1", + classifier_label=f"SOFT_{label.upper()}", + confidence=0.5, # soft signals never claim hard confidence + rationale=rationale, + timestamp=int(time.monotonic_ns()), + model_profile_id=model_id, + ) +``` + +## 6. Integration points + +- `aborist/qa/query.py` — call `soft_preflight_question()` after + `preflight_question()` (the hard call) but before the main LLM + call. Soft hint surfaces on result dict as + `soft_preflight_hint`; merges into the run-DAG `preflight` + stage payload as a sub-field of `question_state`. +- `aborist/qa/runner.py` — same wiring. +- `aborist/cli.py` — new `--soft-preflight` flag (per-call + override; default off matches policy default). +- `bench/qa_sweep.py` — bench rows pick up `soft_preflight_label` + + `soft_preflight_confidence` as bounded-projection fields. +- `aborist/cli.py:_render_warrant_tail` — soft hints render as + `· soft: