diff --git a/aborist/qa/query.py b/aborist/qa/query.py index 790b324..90e0526 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -109,6 +109,7 @@ _TITLE_STOPWORDS = frozenset( it we they me him her us them do does did have has had can could should would will may might tell show describe explain summarize say give list find make please + all there know everything anything something """.split() ) diff --git a/aborist/search/fts5.py b/aborist/search/fts5.py index 9fb7b6d..c27c48d 100644 --- a/aborist/search/fts5.py +++ b/aborist/search/fts5.py @@ -26,6 +26,7 @@ _FTS5_STOPWORDS = frozenset( do does did have has had can could should would will may might tell show describe explain summarize say give list find make please + all there know everything anything something """.split() ) diff --git a/bench/qa_sweep.py b/bench/qa_sweep.py index 0970a2e..381e19b 100644 --- a/bench/qa_sweep.py +++ b/bench/qa_sweep.py @@ -100,6 +100,14 @@ def _run_one( err = f"{type(e).__name__}: {e}" elapsed_s = round(time.monotonic() - t0, 2) + # Deflection sidecar — sidecar-only signal, never feeds the + # verifier. Surfaces the Mars-BDFL pattern (STRICT but answer + # never mentions the question's subject) at bench-aggregate + # scale so a creeping "model deflects rather than refuses" + # regression is legible across runs. + from aborist.qa.inspect import diagnose_deflection + deflection = diagnose_deflection(question, result.get("answer_text") or "") + audit_mode = result.get("audit_mode") return { "question": question, @@ -117,6 +125,9 @@ def _run_one( "cache_key": (result.get("cache_key") or "")[:12], "n_sources": len(result.get("sources") or []), "elapsed_s": elapsed_s, + "deflection_kind": deflection["kind"], + "subject_anchor": deflection["subject_anchor"], + "subject_in_answer": deflection["subject_in_answer"], "error": err, } @@ -134,6 +145,7 @@ def _summarize(rows: list[dict]) -> dict: "errors": 0, "ratio_sum": 0.0, "latency_sum": 0.0, + "deflections": 0, }) b["n"] += 1 if r["error"] or r["status"] == "error": @@ -142,6 +154,13 @@ def _summarize(rows: list[dict]) -> dict: b[r["audit_mode"]] += 1 b["ratio_sum"] += r["ratio"] or 0.0 b["latency_sum"] += r["elapsed_s"] or 0.0 + # Deflection rate: STRICT/HYBRID with subject-anchor missing + # from answer. UNGROUNDED + deflection isn't interesting (no + # answer to deflect with), and no_question_tokens is vacuous. + if r.get("deflection_kind") == "deflection" and r["audit_mode"] in ( + "STRICT", "HYBRID" + ): + b["deflections"] += 1 return by_mode @@ -190,8 +209,8 @@ def _render_markdown( lines.append("") lines.append("## summary") lines.append("") - lines.append("| mode | runs | STRICT | HYBRID | UNGROUNDED | err | strict-rate | mean ratio | mean latency |") - lines.append("|------|------|--------|--------|------------|-----|-------------|-----------|--------------|") + lines.append("| mode | runs | STRICT | HYBRID | UNGROUNDED | err | strict-rate | mean ratio | mean latency | deflections |") + lines.append("|------|------|--------|--------|------------|-----|-------------|-----------|--------------|-------------|") for mode in modes: b = summary.get(mode) if not b: @@ -200,10 +219,12 @@ def _render_markdown( mean_ratio = b["ratio_sum"] / n mean_lat = b["latency_sum"] / n strict_rate = b["STRICT"] / n + deflections = b.get("deflections", 0) lines.append( f"| {mode} | {b['n']} | {b['STRICT']} | {b['HYBRID']} | " f"{b['UNGROUNDED']} | {b['errors']} | " - f"{strict_rate:.2f} | {mean_ratio:.3f} | {mean_lat:.1f}s |" + f"{strict_rate:.2f} | {mean_ratio:.3f} | {mean_lat:.1f}s | " + f"{deflections}/{b['n']} |" ) lines.append("") lines.append("## per question")