diff --git a/Makefile b/Makefile index 37db34c..9b21b1f 100644 --- a/Makefile +++ b/Makefile @@ -166,17 +166,26 @@ QUERY_TOP_K ?= 8 # ANSWER_MODE=quote for the legacy substring verifier. ANSWER_MODE= # (empty) defers to DEFAULT_QUERY_POLICY. ANSWER_MODE ?= claim_lattice -query: bootstrap ## ask the corpus a question [JSON=1 BURN=1 REPAIR=1 REPROMPTS=N K="extra retrieval keywords" ANSWER_MODE=claim_lattice|claim_lattice_pointer|quote]; JSON by default +# BROAD=1 → flip on the Ticket #000008 quantifier-cap apply-gate for +# this call. Default-off per §10.11.3 dry-run discipline; operator +# opts in here for broad-quantifier shapes (winners-of-all, +# tell-me-everything-about-X). Pairs cleanly with the broad- +# quantifier reminder which is default-on for lattice modes. +# Bench (#000008 §12.10): cap-on JSON wins +14pp on STRICT-rate. +# REJECT_BROAD=1 → strict reject for ALL/COMPREHENSIVE/OPEN_REQUEST +# unbounded shapes; returns UNGROUNDED before the LLM call. +# ALLOW_BROAD=1 → emergent search; classifier on, caps off. +query: bootstrap ## ask the corpus a question [JSON=1 BURN=1 REPAIR=1 REPROMPTS=N K="extra retrieval keywords" ANSWER_MODE=claim_lattice|claim_lattice_pointer|quote BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1]; JSON by default @if [ -z "$$Q" ] && [ -z "$(Q)" ]; then \ - echo "usage: make query Q=\"your question\" [JSON=1 BURN=1 REPAIR=1 REPROMPTS=N K=\"extra retrieval keywords\" ANSWER_MODE=claim_lattice|claim_lattice_pointer|quote]"; exit 2; \ + echo "usage: make query Q=\"your question\" [JSON=1 BURN=1 REPAIR=1 REPROMPTS=N K=\"extra retrieval keywords\" ANSWER_MODE=claim_lattice|claim_lattice_pointer|quote BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1]"; exit 2; \ fi - $(ABORIST) --shards-dir $(SHARDS_DIR) query --top-k $(QUERY_TOP_K) $(if $(JSON),--json,) $(if $(BURN),--burn,) $(if $(REPAIR),--repair,) $(if $(REPROMPTS),--repair-reprompts $(REPROMPTS),) $(if $(ANSWER_MODE),--answer-mode $(ANSWER_MODE),) $(if $(K),--retrieval-keywords "$(K)",) "$(Q)" + $(ABORIST) --shards-dir $(SHARDS_DIR) query --top-k $(QUERY_TOP_K) $(if $(JSON),--json,) $(if $(BURN),--burn,) $(if $(REPAIR),--repair,) $(if $(REPROMPTS),--repair-reprompts $(REPROMPTS),) $(if $(ANSWER_MODE),--answer-mode $(ANSWER_MODE),) $(if $(K),--retrieval-keywords "$(K)",) $(if $(BROAD),--apply-quantifier-caps,) $(if $(REJECT_BROAD),--reject-broad,) $(if $(ALLOW_BROAD),--allow-broad,) "$(Q)" -query-dry: bootstrap ## like 'make query' but skip the LLM call (dry-run) [JSON=1 BURN=1 ANSWER_MODE=...] +query-dry: bootstrap ## like 'make query' but skip the LLM call (dry-run) [JSON=1 BURN=1 ANSWER_MODE=... BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1] @if [ -z "$$Q" ] && [ -z "$(Q)" ]; then \ - echo "usage: make query-dry Q=\"your question\" [JSON=1 BURN=1 ANSWER_MODE=claim_lattice|claim_lattice_pointer|quote]"; exit 2; \ + echo "usage: make query-dry Q=\"your question\" [JSON=1 BURN=1 ANSWER_MODE=claim_lattice|claim_lattice_pointer|quote BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1]"; exit 2; \ fi - $(ABORIST) --shards-dir $(SHARDS_DIR) query --top-k $(QUERY_TOP_K) --dry-run $(if $(JSON),--json,) $(if $(BURN),--burn,) $(if $(ANSWER_MODE),--answer-mode $(ANSWER_MODE),) "$(Q)" + $(ABORIST) --shards-dir $(SHARDS_DIR) query --top-k $(QUERY_TOP_K) --dry-run $(if $(JSON),--json,) $(if $(BURN),--burn,) $(if $(ANSWER_MODE),--answer-mode $(ANSWER_MODE),) $(if $(BROAD),--apply-quantifier-caps,) $(if $(REJECT_BROAD),--reject-broad,) $(if $(ALLOW_BROAD),--allow-broad,) "$(Q)" BENCH_QA_QUESTIONS ?= bench/qa_questions.txt BENCH_QA_OUT ?= bench/qa_results diff --git a/docs/TICKETS.md b/docs/TICKETS.md index 13f9683..e423930 100644 --- a/docs/TICKETS.md +++ b/docs/TICKETS.md @@ -57,7 +57,8 @@ Newest first. Update on every open/close. | ID | Title | Status | Opened | Directive | |----------|------------------------------------------------|-----------------------|------------|-----------| -| #000008 | Broad-quantifier preflight guard | open · phases 0–4 landed; defaults flipped (Option A); Phase 5 DAG-binding pending | 2026-05-02 | — | +| #000009 | Quantifier preflight run-DAG node binding | open · awaiting go/no-go | 2026-05-03 | D3, D4 | +| #000008 | Broad-quantifier preflight guard | closed · landed in `4f2b5a6`; Phase 5 DAG binding split into #000009 | 2026-05-02 | — | | #000007 | Query-layer hyphen folding | closed · 2026-05-02 | 2026-05-02 | — | | #000006 | Bench-emergent findings (rolling research log) | open · rolling | 2026-05-02 | — | | #000005 | Label ladder migration (POINTER-LINKED → …) | closed · 2026-05-02 | 2026-05-01 | D7 | @@ -68,4 +69,4 @@ Newest first. Update on every open/close. ## Next ID -`000009` +`000010` diff --git a/docs/tickets/ticket-000008-broad-quantifier-preflight-guard.md b/docs/tickets/ticket-000008-broad-quantifier-preflight-guard.md index 2e6d530..f89c08d 100644 --- a/docs/tickets/ticket-000008-broad-quantifier-preflight-guard.md +++ b/docs/tickets/ticket-000008-broad-quantifier-preflight-guard.md @@ -446,14 +446,29 @@ enhancement, not blocking. ## 8. Status -**All four phases landed 2026-05-03. Mechanism complete. Defaults -flipped per §12.11 after the four-cell A/B + n=5 verification.** -Pure-Option-A landing: reminder default-on for lattice modes, cap -default operator-opt-in with JSON-only allowlist. Pointer-mode -cap is filtered out (wasted on the 0/45 STRICT floor). Bench-side -mechanism + measurement complete; only Phase 5 (run-DAG node -binding) and cross-model Qwen/GPT-4 verification remain as -follow-ups. +**Closed · landed in `4f2b5a6` (2026-05-03).** + +All four phases landed; defaults flipped per §12.11 after the +four-cell A/B + n=5 verification. Pure-Option-A landing: reminder +default-on for lattice modes, cap default operator-opt-in with +JSON-only allowlist. Pointer-mode cap is filtered out (wasted on +the 0/45 STRICT floor). Bench-side mechanism + measurement +complete. + +Follow-ups split into separate tickets per the design log +convention: + +- **Ticket #000009** — quantifier-preflight run-DAG node binding + (Phase 5 / §9.5 Merkle-AGI-DAG framing). Required to close the + audit-replay gap surfaced in §11.11 (currently quantifier output + is on result dict but NOT in `run_dag_root` hash). Substantial + separate work; cleaner as its own ticket. +- **Cross-model Qwen / GPT-4 bench** — manual operator task. + Out-of-automated-scope; not blocking #000008's closure. + +This ticket file remains in place as the design log per +`docs/TICKETS.md` convention ("Do not delete tickets; they are +the design log"). Implementation commit chain (all on `origin/main`): diff --git a/docs/tickets/ticket-000009-quantifier-preflight-dag-binding.md b/docs/tickets/ticket-000009-quantifier-preflight-dag-binding.md new file mode 100644 index 0000000..aec7e57 --- /dev/null +++ b/docs/tickets/ticket-000009-quantifier-preflight-dag-binding.md @@ -0,0 +1,215 @@ +# Ticket #000009 — Quantifier preflight run-DAG node binding + +**Status:** open · awaiting go/no-go +**Opened:** 2026-05-03 +**Scope:** Bind the broad-quantifier preflight contract from +ticket #000008 into the per-run Merkle-DAG so the classifier output ++ cap-application decision are hash-bound rather than just +surface-level result-dict fields. +**Audience:** fox + future blackops shifts. +**Hard constraint:** No `schema_version`, `canonicalization_version`, +or `chunking_version` bumps. Same constraint #000008 §1 carried. +The new DAG node is part of the per-run `run_dag_blob` payload, not +a column on `providence_cache`. Adding the node bumps a derived +field (`run_dag_root` SHA), which is allowed — that's already +recomputed every run. + +--- + +## 1. Problem statement + +#000008 landed Phases 0–4 in commit `4f2b5a6`. The preflight +classifier output (`quantifier_intensity`, `scope_bound_hint`, +`claim_cap_applied`) and the policy decision (cap applied or not, +reminder injected or not) appear on the `query()` / `runner.ask()` +result dict and on bench rows. + +**They do NOT appear in `run_dag_root`.** + +That means an audit replay of a `providence_cache` row CANNOT +distinguish: + +``` +question Q + classifier=ALL + cap=8-applied + reminder=injected +question Q + classifier=ALL + cap=12-applied + reminder=injected +question Q + classifier=ALL + cap=8-applied + reminder=skipped +question Q + classifier=null (guard off entirely) +``` + +All four would produce the same `run_dag_root` because the run +DAG is unchanged by quantifier-guard activity. Same model output, +same verifier verdict, same hash — but causally different +behavior. + +The §9.5 Merkle-AGI-DAG framing (#000008) explicitly calls for the +preflight decision to be hash-bound: + +> The node hash should include: stage, question_hash, +> model_profile_hash, governance_policy_hash, classifier_version, +> intensity, matched_token, explicit_count, claim_cap, allow_broad, +> reject_broad, reminder_injected. + +That binding is what closes the audit-replay gap. + +## 2. Scope + +What this ticket covers: + +- Extend the per-run DAG with a `quantifier_preflight` node between + `question` and `retrieval_plan` stages. +- Three-clause CTI contract from #000008 §10.4 — `quantifier`, + `answer_contract`, `evidence_contract` — bound as sub-fields of + the new node OR as three separate nodes (design TBD). +- Hash binding: classifier output + policy decision contribute to + `run_dag_root` so audit replay can distinguish guard-on vs + guard-off, cap-applied vs not, reminder-injected vs skipped. + +What this ticket does NOT cover: + +- Verifier semantics. The preflight node carries policy state into + the audit chain; the verifier still operates on the same inputs. +- Schema bumps. Per #000008 §1 hard constraint, no + `schema_version`/`canonicalization_version`/`chunking_version` + changes. +- Cross-model bench (Qwen / GPT-4) — that's a separate manual task + per #000008 §11.11. + +## 3. Design sketch + +### 3.1 New stage between question and retrieval_plan + +Current run-DAG order (claim_lattice modes): + +``` +question → retrieval_plan → retrieval_result → evidence_map → +prompt → raw_model_output → parsed_claim_lattice → pointer_verify +→ format_check → anchor_warrant → final_label +``` + +Proposed: + +``` +question → quantifier_preflight → retrieval_plan → ... +``` + +`quantifier_preflight` is hash-bound and feeds into `retrieval_plan` +the same way `question` does today (parent → child via +`HashCombine`). + +### 3.2 Node payload + +Single-node form (proposal): + +```json +{ + "stage": "quantifier_preflight", + "classifier_version": "quantifier-v0.1", + "intensity": "ALL", + "matched_token": "all", + "explicit_count": null, + "scope_bound_hint": "unbounded", + "is_broad": true, + "operational_shape": "universal_enumeration", + "guard_enabled": true, + "mode_gated": true, + "apply_caps_active": false, + "claim_cap_resolved": 8, + "claim_cap_applied": null, + "reminder_enabled": true, + "reminder_injected": true, + "reject_broad_active": false +} +``` + +Three-clause form (per #000008 §10.4): + +```json +{ + "stage": "quantifier_preflight", + "classifier_version": "quantifier-v0.1", + "quantifier_clause": { "intensity": "ALL", ... }, + "answer_contract": { "max_claims": 8, ... }, + "evidence_contract": { "max_evidence_ids_exposed": 8, ... } +} +``` + +Three-clause form is more extensible (each clause can grow +independently) but adds one indirection layer. Single-node form +is denser and simpler. **Recommendation: single node initially; +split if a future clause needs independent hashing.** + +### 3.3 Where the node lives in code + +- New helper in `aborist/qa/dag.py` (alongside existing per-stage + helpers): `_quantifier_preflight_node(quantifier, policy, + effective_max_claims, reminder_injected)`. +- Wire into `query()` at the same point quantifier classification + runs (after policy resolution, before `t_start`). +- Wire into `runner.ask()` symmetrically. +- The node hash contributes to `run_dag_root` automatically via + the existing run-DAG framing. + +### 3.4 Audit replay implications + +After this lands, an audit reading a `providence_cache` row can: + +``` +hash(quantifier_preflight node) = h_pre +→ pin: classifier output, cap decision, reminder decision + +If h_pre changes between two cache_keys for the same question, +that row reflects a different preflight policy. + +Allows for: regression bisection ("which day did the cap default +flip break our STRICT-rate?"), policy A/B reconstruction +("show me all rows where cap was applied vs not"), cross-model +diff ("hermes vs qwen on the same question with the same +preflight contract"). +``` + +## 4. Implementation sketch + +``` +1. New function in aborist/qa/dag.py: + def quantifier_preflight_node(quantifier, policy, ...) +2. Wire it into query() between policy resolution and t_start. +3. Wire it into runner.ask() symmetrically. +4. Run-DAG ordering: insert as child of `question` node, parent + of `retrieval_plan` node. +5. Update tests/test_dag.py with a fixture asserting the node + appears with the expected payload + position. +6. Update docs/cti-architecture.md to reflect the new stage. +7. Bench cycle to confirm no run_dag_root collisions across + guard-on vs guard-off cells. +``` + +Estimated cost: 3-4 hours including tests + bench verification. + +## 5. Why now + +#000008 §11.11 explicitly listed this as Phase 5 follow-up. +Closing #000008 with the preflight mechanism shipped but the +audit-replay gap still open creates a "shipped but +non-auditable" state — the seven-point program D3 directive +("build CTI internally") is half-honored: the contract exists +in code, but isn't bound into the audit chain. + +This ticket closes the loop. + +## 6. Open questions + +- Single node vs three-clause split (§3.2)? Default: single, split + later if needed. +- Should the node hash include the model_profile_id verbatim, or + just rely on the existing `model_profile_hash`? Default: rely on + existing — it's already in the cache_key derivation. +- Bench cell needed before merging? Default: yes — confirm + run_dag_root bumps when a guard knob flips, AND that + governance_policy_hash continues to invalidate cache rows + appropriately. + +## 7. Status + +Open · awaiting go/no-go. Mechanism is well-understood from +#000008 §9.5; this ticket is the audit-binding cleanup, not new +behavior.