#000037 follow-through: persist falsification proposals + harvest live stream
Two coupled changes that wire the §13 Step 11 proposal stream from in-memory-and-discarded to persisted-and-harvestable. (A) emit_controller_events now writes a 4th event kind controller_falsification_proposal, one row per decision.falsification_proposals entry. Body carries branch_id + witness_divergence + reason; label column carries the reason for terminal-table inspection. Deterministic ordering by (branch_id, witness_divergence) so canonical-body hashes are stable. Idempotent under the existing UNIQUE (event_kind, body_hash) constraint. The audit-chain semantics remain unchanged (still a sibling table; no event_hash preimage entry). The CLI inspector's --kind choices gain the new event kind so operators can filter for it directly. (B) bench/scripts/harvest_falsification_proposals.py gains a third source bucket CONTROLLER_PROPOSAL alongside the existing HYBRID + UNGROUNDED providence_cache buckets. Reads controller_falsification_proposal rows from controller_events, extracts the 16-char cache_key prefix from branch_id (qa:<prefix> pattern from the QA-runner advisory), joins back to providence_cache for fixture enrichment (answer_text + audit_mode + verifier_method), and tags _harvest_meta.harvested_from = "controller_events" so the two source paths stay distinguishable in the fixture pack. Dedup against the providence_cache buckets by fixture id. Today this typically yields 0 new fixtures because (i) no live QA has fired since Phase 2 wiring landed, and (ii) the QA-runner single-branch advisory's proposals overlap providence_cache content the harvester already finds. Real net value comes from Phase 3 (#000045) sweep emissions, which will produce multi-branch chunk proposals that providence_cache rows can't predict. Tests: 3 new in tests/test_prometheus_audit.py (proposal-row emission, no-proposal no-row, idempotency); 1 new in tests/test_bench_batteries.py (synthetic qa.db with both providence_cache + controller_events rows; asserts the controller_events bucket surfaces a fixture invisible to the divergence-thresholded providence_cache buckets). The pre-existing harvested-pack-runs-clean test relaxes its harvested_from pin from "providence_cache"-only to {"providence_cache", "controller_events"}.
This commit is contained in:
parent
af27533fa3
commit
70c2184b01
5 changed files with 359 additions and 23 deletions
|
|
@ -693,7 +693,13 @@ def test_5f_falsification_harvested_pack_runs_clean():
|
|||
row = json.loads(line)
|
||||
meta = row.get("_harvest_meta", {})
|
||||
assert meta.get("source_ticket") == "#000037 §13 step 11"
|
||||
assert meta.get("harvested_from") == "providence_cache"
|
||||
# Two harvest sources now: providence_cache directly
|
||||
# (Phase 1 buckets) and controller_events stream (live
|
||||
# Phase 2 advisory writes).
|
||||
assert meta.get("harvested_from") in (
|
||||
"providence_cache",
|
||||
"controller_events",
|
||||
)
|
||||
assert isinstance(meta.get("witness_divergence"), (int, float))
|
||||
assert meta["witness_divergence"] >= 0.5
|
||||
assert meta.get("audit_mode_at_harvest") in ("HYBRID", "UNGROUNDED")
|
||||
|
|
@ -722,6 +728,86 @@ def test_5f_falsification_harvested_pack_widens_motif_coverage():
|
|||
assert "UNGROUNDED" in motifs
|
||||
|
||||
|
||||
def test_harvest_includes_controller_events_bucket(tmp_path):
|
||||
"""The harvester pulls a third bucket from the live
|
||||
controller_events stream (event_kind =
|
||||
controller_falsification_proposal) in addition to the two
|
||||
providence_cache buckets. Builds a synthetic qa.db with both
|
||||
sources and asserts the controller_events-derived fixture appears
|
||||
with the right provenance."""
|
||||
import sqlite3 as _sqlite3
|
||||
from bench.scripts.harvest_falsification_proposals import harvest
|
||||
|
||||
qa_db = tmp_path / "qa.db"
|
||||
conn = _sqlite3.connect(qa_db)
|
||||
conn.row_factory = _sqlite3.Row
|
||||
# Minimal providence_cache schema for the harvest path.
|
||||
conn.executescript(
|
||||
"CREATE TABLE providence_cache ("
|
||||
" cache_key TEXT PRIMARY KEY, audit_mode TEXT, n_quotes INTEGER,"
|
||||
" n_verified INTEGER, unverified_quotes TEXT, verifier_method TEXT,"
|
||||
" falsification_state TEXT, answer_text TEXT);"
|
||||
"CREATE TABLE controller_events ("
|
||||
" event_id INTEGER PRIMARY KEY AUTOINCREMENT,"
|
||||
" organism_root TEXT NOT NULL, branch_id TEXT, event_kind TEXT NOT NULL,"
|
||||
" label TEXT, entropy REAL, difficulty REAL, allocation REAL,"
|
||||
" body_blob TEXT NOT NULL, body_hash TEXT NOT NULL,"
|
||||
" recorded_at INTEGER NOT NULL,"
|
||||
" UNIQUE (event_kind, body_hash));"
|
||||
)
|
||||
# Providence row whose cache_key matches a future
|
||||
# controller_events branch_id prefix. Divergence < 0.5 so the row
|
||||
# is INVISIBLE to the providence buckets — it surfaces ONLY via the
|
||||
# controller_events stream.
|
||||
conn.execute(
|
||||
"INSERT INTO providence_cache VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(
|
||||
"ce-only-cache-key-12345678abcdef",
|
||||
"HYBRID",
|
||||
10, 9,
|
||||
json.dumps(["one quote"]), # 1/10 = 0.1 < threshold
|
||||
"claim_lattice",
|
||||
"live",
|
||||
"Synthetic answer text from a controller-events row.",
|
||||
),
|
||||
)
|
||||
# Matching controller_falsification_proposal row. Divergence here
|
||||
# is the high signal that surfaced the proposal at controller-decide
|
||||
# time (sweep mode would commonly push such rows).
|
||||
conn.execute(
|
||||
"INSERT INTO controller_events VALUES "
|
||||
"(NULL, ?, ?, ?, ?, NULL, NULL, NULL, ?, ?, ?)",
|
||||
(
|
||||
"qa:ce-only-cache-key-12345678abcdef",
|
||||
"qa:ce-only-cache-key", # 16-char prefix of the cache_key
|
||||
"controller_falsification_proposal",
|
||||
"WITNESS_DIVERGENCE_EXCEEDS_THRESHOLD",
|
||||
json.dumps({
|
||||
"kind": "controller_falsification_proposal",
|
||||
"organism_root": "qa:ce-only-cache-key-12345678abcdef",
|
||||
"branch_id": "qa:ce-only-cache-key",
|
||||
"witness_divergence": 0.8,
|
||||
"reason": "WITNESS_DIVERGENCE_EXCEEDS_THRESHOLD",
|
||||
}),
|
||||
"deadbeef" * 8, # placeholder body_hash for the test
|
||||
1700000000,
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
fixtures = harvest(qa_db, threshold=0.5, sample_per_bucket=20)
|
||||
ce_fixtures = [
|
||||
fx for fx in fixtures
|
||||
if fx["_harvest_meta"]["harvested_from"] == "controller_events"
|
||||
]
|
||||
assert len(ce_fixtures) == 1
|
||||
fx = ce_fixtures[0]
|
||||
assert fx["_harvest_meta"]["controller_organism_root"].startswith("qa:")
|
||||
assert fx["_harvest_meta"]["witness_divergence"] == 0.8
|
||||
assert fx["id"] == "5f-fal-harvested-ce-only-cache-ke"
|
||||
|
||||
|
||||
def test_5f_falsification_live_helper_calls_real_verifier():
|
||||
"""UNGROUNDED + STRICT_SPAN signals come straight from verify_quotes."""
|
||||
from bench.batteries.b_5f import _live_falsification_violations
|
||||
|
|
|
|||
|
|
@ -52,6 +52,14 @@ from arborist.substrate.prometheus_audit import (
|
|||
# landing.
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StubProposal:
|
||||
organism_root: str
|
||||
branch_id: str
|
||||
witness_divergence: float
|
||||
reason: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StubDecision:
|
||||
selected_branch_id: str | None
|
||||
|
|
@ -63,6 +71,7 @@ class StubDecision:
|
|||
notes: tuple
|
||||
memory_proposals: tuple = ()
|
||||
selfmodel_proposals: tuple = ()
|
||||
falsification_proposals: tuple = ()
|
||||
advisory_events: tuple = ()
|
||||
|
||||
|
||||
|
|
@ -388,6 +397,97 @@ def test_query_by_kind_returns_rows(shard):
|
|||
assert n_alloc == 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------
|
||||
# §13 Step 11 proposal persistence (4th event kind)
|
||||
# ---------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_emit_persists_falsification_proposal_rows(shard):
|
||||
"""A decision carrying §13 Step 11 falsification proposals writes
|
||||
one ``controller_falsification_proposal`` row per proposal."""
|
||||
conn, _ = shard
|
||||
dec = _stub_decision(
|
||||
falsification_proposals=(
|
||||
StubProposal(
|
||||
organism_root="qa:abcd1234abcd1234abcd",
|
||||
branch_id="qa:abcd1234abcd1234",
|
||||
witness_divergence=0.66,
|
||||
reason="WITNESS_DIVERGENCE_EXCEEDS_THRESHOLD",
|
||||
),
|
||||
StubProposal(
|
||||
organism_root="qa:abcd1234abcd1234abcd",
|
||||
branch_id="qa:other-branch",
|
||||
witness_divergence=0.75,
|
||||
reason="WITNESS_DIVERGENCE_EXCEEDS_THRESHOLD",
|
||||
),
|
||||
),
|
||||
)
|
||||
with transaction(conn):
|
||||
emit_controller_events(conn, dec, "qa:abcd1234abcd1234abcd")
|
||||
|
||||
rows = conn.execute(
|
||||
"SELECT branch_id, label FROM controller_events "
|
||||
"WHERE event_kind='controller_falsification_proposal' "
|
||||
"ORDER BY branch_id"
|
||||
).fetchall()
|
||||
assert len(rows) == 2
|
||||
# Sorted-by-branch_id ordering pin matches emit_controller_events'
|
||||
# canonical-body discipline.
|
||||
assert rows[0]["branch_id"] == "qa:abcd1234abcd1234"
|
||||
assert rows[1]["branch_id"] == "qa:other-branch"
|
||||
# ``label`` carries the proposal reason for terminal-table inspection.
|
||||
assert rows[0]["label"] == "WITNESS_DIVERGENCE_EXCEEDS_THRESHOLD"
|
||||
|
||||
|
||||
def test_emit_no_proposals_writes_no_proposal_rows(shard):
|
||||
"""Empty falsification_proposals tuple → no proposal rows. The
|
||||
other three event kinds are unaffected."""
|
||||
conn, _ = shard
|
||||
dec = _stub_decision(falsification_proposals=())
|
||||
with transaction(conn):
|
||||
emit_controller_events(conn, dec, "qa:no-proposals")
|
||||
n = conn.execute(
|
||||
"SELECT COUNT(*) FROM controller_events "
|
||||
"WHERE event_kind='controller_falsification_proposal'"
|
||||
).fetchone()[0]
|
||||
assert n == 0
|
||||
# Other rows still present.
|
||||
n_other = conn.execute(
|
||||
"SELECT COUNT(*) FROM controller_events "
|
||||
"WHERE event_kind != 'controller_falsification_proposal'"
|
||||
).fetchone()[0]
|
||||
assert n_other >= 3 # decision + difficulty + ≥1 allocation
|
||||
|
||||
|
||||
def test_emit_proposal_rows_idempotent(shard):
|
||||
"""Re-emitting the same decision (same proposals) is a no-op under
|
||||
the UNIQUE (event_kind, body_hash) constraint."""
|
||||
conn, _ = shard
|
||||
proposals = (
|
||||
StubProposal(
|
||||
organism_root="qa:idem-root",
|
||||
branch_id="qa:idem-branch",
|
||||
witness_divergence=0.55,
|
||||
reason="WITNESS_DIVERGENCE_EXCEEDS_THRESHOLD",
|
||||
),
|
||||
)
|
||||
dec = _stub_decision(falsification_proposals=proposals)
|
||||
with transaction(conn):
|
||||
emit_controller_events(conn, dec, "qa:idem-root")
|
||||
first_n = conn.execute(
|
||||
"SELECT COUNT(*) FROM controller_events "
|
||||
"WHERE event_kind='controller_falsification_proposal'"
|
||||
).fetchone()[0]
|
||||
with transaction(conn):
|
||||
emit_controller_events(conn, dec, "qa:idem-root")
|
||||
second_n = conn.execute(
|
||||
"SELECT COUNT(*) FROM controller_events "
|
||||
"WHERE event_kind='controller_falsification_proposal'"
|
||||
).fetchone()[0]
|
||||
assert first_n == 1
|
||||
assert second_n == first_n
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------
|
||||
# Phase 2 integration: arborist.qa.runner._emit_qa_controller_advisory
|
||||
# wires real QA verdicts → controller_events rows
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue