qa: strip 'all/there/know/everything' from retrieval queries; bench gains deflection rate
Two paired changes: (1) Retrieval template-phrase stopwords. Bench finding 2026-04-30: 'tell me all there is to know about york england?' missed the en.wikipedia.org/wiki/York article entirely — corpus has it but the FTS5 AND-mode MATCH expanded to "all" AND "there" AND "know" AND "york" AND "england", which filters to only docs containing every template word. Articles like "Charles I of England" happen to mention "all" + "there" + "know" + "england" enough to win; the bare "York" stub doesn't, so it falls out of the top-K. Added `all there know everything anything something` to both _FTS5_STOPWORDS (search/fts5.py) and _TITLE_STOPWORDS (qa/query.py) so they stay in sync. After the change "York" article ranks #1 on both 'tell me all there is to know about' and 'tell me everything about' phrasings. These are retrieval-time stopwords only — they don't enter the cache_key or governance_policy_hash (cache_key hashes the original question text via question_hash). Existing cached records remain valid; new queries get better retrieval. (2) bench/qa_sweep.py: per-row deflection_kind + subject_anchor + subject_in_answer (from qa.inspect.diagnose_deflection); summary table grows a 'deflections' column tracking STRICT/HYBRID rows where the question's subject anchor is missing from the answer. Surfaces the Mars-BDFL pattern at aggregate scale.
This commit is contained in:
parent
49a85b97d6
commit
ecaf69e503
3 changed files with 26 additions and 3 deletions
|
|
@ -109,6 +109,7 @@ _TITLE_STOPWORDS = frozenset(
|
|||
it we they me him her us them do does did have has had can could should
|
||||
would will may might
|
||||
tell show describe explain summarize say give list find make please
|
||||
all there know everything anything something
|
||||
""".split()
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ _FTS5_STOPWORDS = frozenset(
|
|||
do does did have has had can could should would will may might
|
||||
tell show describe explain summarize say give list find make
|
||||
please
|
||||
all there know everything anything something
|
||||
""".split()
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -100,6 +100,14 @@ def _run_one(
|
|||
err = f"{type(e).__name__}: {e}"
|
||||
elapsed_s = round(time.monotonic() - t0, 2)
|
||||
|
||||
# Deflection sidecar — sidecar-only signal, never feeds the
|
||||
# verifier. Surfaces the Mars-BDFL pattern (STRICT but answer
|
||||
# never mentions the question's subject) at bench-aggregate
|
||||
# scale so a creeping "model deflects rather than refuses"
|
||||
# regression is legible across runs.
|
||||
from aborist.qa.inspect import diagnose_deflection
|
||||
deflection = diagnose_deflection(question, result.get("answer_text") or "")
|
||||
|
||||
audit_mode = result.get("audit_mode")
|
||||
return {
|
||||
"question": question,
|
||||
|
|
@ -117,6 +125,9 @@ def _run_one(
|
|||
"cache_key": (result.get("cache_key") or "")[:12],
|
||||
"n_sources": len(result.get("sources") or []),
|
||||
"elapsed_s": elapsed_s,
|
||||
"deflection_kind": deflection["kind"],
|
||||
"subject_anchor": deflection["subject_anchor"],
|
||||
"subject_in_answer": deflection["subject_in_answer"],
|
||||
"error": err,
|
||||
}
|
||||
|
||||
|
|
@ -134,6 +145,7 @@ def _summarize(rows: list[dict]) -> dict:
|
|||
"errors": 0,
|
||||
"ratio_sum": 0.0,
|
||||
"latency_sum": 0.0,
|
||||
"deflections": 0,
|
||||
})
|
||||
b["n"] += 1
|
||||
if r["error"] or r["status"] == "error":
|
||||
|
|
@ -142,6 +154,13 @@ def _summarize(rows: list[dict]) -> dict:
|
|||
b[r["audit_mode"]] += 1
|
||||
b["ratio_sum"] += r["ratio"] or 0.0
|
||||
b["latency_sum"] += r["elapsed_s"] or 0.0
|
||||
# Deflection rate: STRICT/HYBRID with subject-anchor missing
|
||||
# from answer. UNGROUNDED + deflection isn't interesting (no
|
||||
# answer to deflect with), and no_question_tokens is vacuous.
|
||||
if r.get("deflection_kind") == "deflection" and r["audit_mode"] in (
|
||||
"STRICT", "HYBRID"
|
||||
):
|
||||
b["deflections"] += 1
|
||||
return by_mode
|
||||
|
||||
|
||||
|
|
@ -190,8 +209,8 @@ def _render_markdown(
|
|||
lines.append("")
|
||||
lines.append("## summary")
|
||||
lines.append("")
|
||||
lines.append("| mode | runs | STRICT | HYBRID | UNGROUNDED | err | strict-rate | mean ratio | mean latency |")
|
||||
lines.append("|------|------|--------|--------|------------|-----|-------------|-----------|--------------|")
|
||||
lines.append("| mode | runs | STRICT | HYBRID | UNGROUNDED | err | strict-rate | mean ratio | mean latency | deflections |")
|
||||
lines.append("|------|------|--------|--------|------------|-----|-------------|-----------|--------------|-------------|")
|
||||
for mode in modes:
|
||||
b = summary.get(mode)
|
||||
if not b:
|
||||
|
|
@ -200,10 +219,12 @@ def _render_markdown(
|
|||
mean_ratio = b["ratio_sum"] / n
|
||||
mean_lat = b["latency_sum"] / n
|
||||
strict_rate = b["STRICT"] / n
|
||||
deflections = b.get("deflections", 0)
|
||||
lines.append(
|
||||
f"| {mode} | {b['n']} | {b['STRICT']} | {b['HYBRID']} | "
|
||||
f"{b['UNGROUNDED']} | {b['errors']} | "
|
||||
f"{strict_rate:.2f} | {mean_ratio:.3f} | {mean_lat:.1f}s |"
|
||||
f"{strict_rate:.2f} | {mean_ratio:.3f} | {mean_lat:.1f}s | "
|
||||
f"{deflections}/{b['n']} |"
|
||||
)
|
||||
lines.append("")
|
||||
lines.append("## per question")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue