From ecaf69e503c6e791f38511726ff7668026318007 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Thu, 30 Apr 2026 20:15:25 -0400 Subject: [PATCH] qa: strip 'all/there/know/everything' from retrieval queries; bench gains deflection rate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two paired changes: (1) Retrieval template-phrase stopwords. Bench finding 2026-04-30: 'tell me all there is to know about york england?' missed the en.wikipedia.org/wiki/York article entirely — corpus has it but the FTS5 AND-mode MATCH expanded to "all" AND "there" AND "know" AND "york" AND "england", which filters to only docs containing every template word. Articles like "Charles I of England" happen to mention "all" + "there" + "know" + "england" enough to win; the bare "York" stub doesn't, so it falls out of the top-K. Added `all there know everything anything something` to both _FTS5_STOPWORDS (search/fts5.py) and _TITLE_STOPWORDS (qa/query.py) so they stay in sync. After the change "York" article ranks #1 on both 'tell me all there is to know about' and 'tell me everything about' phrasings. These are retrieval-time stopwords only — they don't enter the cache_key or governance_policy_hash (cache_key hashes the original question text via question_hash). Existing cached records remain valid; new queries get better retrieval. (2) bench/qa_sweep.py: per-row deflection_kind + subject_anchor + subject_in_answer (from qa.inspect.diagnose_deflection); summary table grows a 'deflections' column tracking STRICT/HYBRID rows where the question's subject anchor is missing from the answer. Surfaces the Mars-BDFL pattern at aggregate scale. --- aborist/qa/query.py | 1 + aborist/search/fts5.py | 1 + bench/qa_sweep.py | 27 ++++++++++++++++++++++++--- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/aborist/qa/query.py b/aborist/qa/query.py index 790b324..90e0526 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -109,6 +109,7 @@ _TITLE_STOPWORDS = frozenset( it we they me him her us them do does did have has had can could should would will may might tell show describe explain summarize say give list find make please + all there know everything anything something """.split() ) diff --git a/aborist/search/fts5.py b/aborist/search/fts5.py index 9fb7b6d..c27c48d 100644 --- a/aborist/search/fts5.py +++ b/aborist/search/fts5.py @@ -26,6 +26,7 @@ _FTS5_STOPWORDS = frozenset( do does did have has had can could should would will may might tell show describe explain summarize say give list find make please + all there know everything anything something """.split() ) diff --git a/bench/qa_sweep.py b/bench/qa_sweep.py index 0970a2e..381e19b 100644 --- a/bench/qa_sweep.py +++ b/bench/qa_sweep.py @@ -100,6 +100,14 @@ def _run_one( err = f"{type(e).__name__}: {e}" elapsed_s = round(time.monotonic() - t0, 2) + # Deflection sidecar — sidecar-only signal, never feeds the + # verifier. Surfaces the Mars-BDFL pattern (STRICT but answer + # never mentions the question's subject) at bench-aggregate + # scale so a creeping "model deflects rather than refuses" + # regression is legible across runs. + from aborist.qa.inspect import diagnose_deflection + deflection = diagnose_deflection(question, result.get("answer_text") or "") + audit_mode = result.get("audit_mode") return { "question": question, @@ -117,6 +125,9 @@ def _run_one( "cache_key": (result.get("cache_key") or "")[:12], "n_sources": len(result.get("sources") or []), "elapsed_s": elapsed_s, + "deflection_kind": deflection["kind"], + "subject_anchor": deflection["subject_anchor"], + "subject_in_answer": deflection["subject_in_answer"], "error": err, } @@ -134,6 +145,7 @@ def _summarize(rows: list[dict]) -> dict: "errors": 0, "ratio_sum": 0.0, "latency_sum": 0.0, + "deflections": 0, }) b["n"] += 1 if r["error"] or r["status"] == "error": @@ -142,6 +154,13 @@ def _summarize(rows: list[dict]) -> dict: b[r["audit_mode"]] += 1 b["ratio_sum"] += r["ratio"] or 0.0 b["latency_sum"] += r["elapsed_s"] or 0.0 + # Deflection rate: STRICT/HYBRID with subject-anchor missing + # from answer. UNGROUNDED + deflection isn't interesting (no + # answer to deflect with), and no_question_tokens is vacuous. + if r.get("deflection_kind") == "deflection" and r["audit_mode"] in ( + "STRICT", "HYBRID" + ): + b["deflections"] += 1 return by_mode @@ -190,8 +209,8 @@ def _render_markdown( lines.append("") lines.append("## summary") lines.append("") - lines.append("| mode | runs | STRICT | HYBRID | UNGROUNDED | err | strict-rate | mean ratio | mean latency |") - lines.append("|------|------|--------|--------|------------|-----|-------------|-----------|--------------|") + lines.append("| mode | runs | STRICT | HYBRID | UNGROUNDED | err | strict-rate | mean ratio | mean latency | deflections |") + lines.append("|------|------|--------|--------|------------|-----|-------------|-----------|--------------|-------------|") for mode in modes: b = summary.get(mode) if not b: @@ -200,10 +219,12 @@ def _render_markdown( mean_ratio = b["ratio_sum"] / n mean_lat = b["latency_sum"] / n strict_rate = b["STRICT"] / n + deflections = b.get("deflections", 0) lines.append( f"| {mode} | {b['n']} | {b['STRICT']} | {b['HYBRID']} | " f"{b['UNGROUNDED']} | {b['errors']} | " - f"{strict_rate:.2f} | {mean_ratio:.3f} | {mean_lat:.1f}s |" + f"{strict_rate:.2f} | {mean_ratio:.3f} | {mean_lat:.1f}s | " + f"{deflections}/{b['n']} |" ) lines.append("") lines.append("## per question")