diff --git a/aborist/qa/quantifier.py b/aborist/qa/quantifier.py new file mode 100644 index 0000000..a1334ab --- /dev/null +++ b/aborist/qa/quantifier.py @@ -0,0 +1,398 @@ +"""Pure quantifier preflight classifier — Ticket #000008 Phase 1. + +Maps a question string onto the ten-rung intensity ladder defined in +``docs/tickets/ticket-000008-broad-quantifier-preflight-guard.md`` §2. +The classifier exists to estimate **expected number of claims in the +answer** and **format-discipline risk on small models**. It is not +formal-semantics quantifier theory; the operational axis is what +matters. + +Pure function. No I/O. No model call. No retrieval call. Folds into +``governance_policy_hash`` via ``classifier_version`` (added to +``aborist.qa.keys._VERIFIER_POLICY_FIELDS`` in Phase 2). + +Intensity rungs (highest wins for multi-quantifier questions): + + 1. ABSENT universal-negation, single-claim shape + 2. SINGULAR one-fact wh / definite reference + 3. PROPORTIONAL descriptive fraction (`most`, `half`) + 4. SMALL_NUM_EXPLICIT bounded by digit/word (`top 3`, `seven X`) + 5. COMPARATIVE_BOUND bounded by inequality (`at least 5`) + 6. FEW small set, vague (`some`, `a few`) + 7. MANY medium set, vague (`many`, `numerous`) + 8. ALL universal quantifier (`all`, `every`) + 9. COMPREHENSIVE exhaustive request (`complete list of`, + `tell me everything`) + 10. OPEN_REQUEST verb-driven enumeration (`tell me about`, + `describe`, `explain`) + +Returns a dict with: + + intensity one of the ten rungs (or "SINGULAR" by default) + matched_token the lexical surface form that triggered the rung + explicit_count int when SMALL_NUM_EXPLICIT or COMPARATIVE_BOUND; + None otherwise + is_broad True for ALL / COMPREHENSIVE / OPEN_REQUEST + operational_shape mnemonic for downstream policy (e.g. + "universal_enumeration", "exhaustive_request") + scope_bound_hint "bounded" | "unbounded" | "unknown" + (see ticket §10.1 — bounded ≠ unbounded + universals; classifier defaults to "unknown" + when intensity is broad and no domain anchor + is present) + +Highest-intensity-wins arbitration: when a question contains +overlapping markers (e.g. "tell me about all the planets"), pick the +rung farther from SINGULAR. The order in ``_RUNG_PRIORITY`` codifies +this — later rungs win. +""" + +from __future__ import annotations + +import re + +CLASSIFIER_VERSION = "quantifier-v0.1" + +# ---------------------------------------------------------------- intensities + +_RUNG_PRIORITY = ( + "SINGULAR", + "PROPORTIONAL", + "SMALL_NUM_EXPLICIT", + "COMPARATIVE_BOUND", + "FEW", + "MANY", + "ABSENT", + "ALL", + # OPEN_REQUEST sits below COMPREHENSIVE: "tell me everything + # about X" is BOTH OPEN_REQUEST-shape ("tell me about") AND + # COMPREHENSIVE-shape ("everything"). COMPREHENSIVE wins because + # it carries the explicit exhaustive request — see §2.2 of + # ticket #000008 ("COMPREHENSIVE strictly stronger than ALL"). + # On the cap table, COMPREHENSIVE caps higher than OPEN_REQUEST + # for large models because the operator is asking for depth + # explicitly; both clamp at 5 on small models. + "OPEN_REQUEST", + "COMPREHENSIVE", +) + +_BROAD_RUNGS = frozenset({"ALL", "COMPREHENSIVE", "OPEN_REQUEST"}) + + +# Lexical patterns per rung. Patterns are compiled with re.IGNORECASE +# at module-load. Order within each rung doesn't matter — first match +# wins for matched_token reporting, but rung selection is by priority +# (later rung = higher intensity, see _classify). + +# OPEN_REQUEST — verb-driven enumeration without explicit quantifier. +_OPEN_REQUEST_PATTERNS = [ + r"\btell me (?:about|all about)\b", + r"\btell me everything\b", + r"\bdescribe\b", + r"\bexplain\b", + r"\bsummari[sz]e\b", + r"\bgive me (?:an? )?(?:overview|summary)\b", + r"\bwalk me through\b", + r"\bdiscuss\b", + r"\belaborate on\b", + r"\bexpound on\b", + r"\bwhat about\b", +] + +# COMPREHENSIVE — exhaustive request, strictly stronger than ALL. +_COMPREHENSIVE_PATTERNS = [ + r"\bcomprehensive\b", + r"\bcomplete (?:list|inventory|set|enumeration|account)\b", + r"\bexhaustive\b", + r"\bdefinitive\b", + r"\beverything (?:you know|there is)\b", + r"\btell me everything\b", + r"\bthe whole (?:story|picture|thing)\b", + r"\bthe full (?:story|picture|account)\b", + r"\bfrom a to z\b", + r"\ball there is to know\b", +] + +# ALL — universal quantifier. +_ALL_PATTERNS = [ + r"\ball\b", + r"\bevery\b", + r"\beach (?:and every )?\b", + r"\bevery single\b", + r"\bthe whole\b", + r"\bthe entirety of\b", + r"\bthe totality of\b", + r"\bany\b", # universal use ("any X is Y") + r"\bwhatever\b", + r"\bwhoever\b", +] + +# MANY — medium set, vague. +_MANY_PATTERNS = [ + r"\bmany\b", + r"\bvarious\b", + r"\bmultiple\b", + r"\bnumerous\b", + r"\ba number of\b", + r"\blots of\b", + r"\bplenty of\b", + r"\ba great many\b", + r"\bmultitudes\b", + r"\bseveral dozen\b", +] + +# FEW — small set, vague. +_FEW_PATTERNS = [ + r"\bsome\b", + r"\ba few\b", + r"\bseveral\b", + r"\ba couple\b", + r"\ba handful\b", + r"\ba small number of\b", + r"\ba smattering of\b", + r"\bnot many\b", + r"\bhardly any\b", +] + +# SMALL_NUM_EXPLICIT — bounded by digit or numeric word. +# Matched token reports the count phrase; explicit_count populated +# from the digit / lexicon below. +_SMALL_NUM_DIGIT = re.compile( + r"\btop (\d+)\b|\b(\d+) (?:biggest|smallest|largest|most|best|worst)\b" + r"|\b(?:first|last|top) (\d+)\b", + re.IGNORECASE, +) +_NUMBER_WORDS = { + "one": 1, "two": 2, "three": 3, "four": 4, "five": 5, + "six": 6, "seven": 7, "eight": 8, "nine": 9, "ten": 10, + "eleven": 11, "twelve": 12, "dozen": 12, +} +_SMALL_NUM_WORD_PATTERNS = [ + rf"\b(?:top |first |last )?({w})\b" for w in _NUMBER_WORDS +] +_PAIR_OF = re.compile(r"\bpair of\b", re.IGNORECASE) +_HANDFUL_OF = re.compile(r"\ba handful of\b", re.IGNORECASE) + +# COMPARATIVE_BOUND — bounded by inequality. +# at_least / at_most / more_than / fewer_than / under / over / +# up_to / between A and B. +_COMPARATIVE_PATTERNS = [ + re.compile(r"\bat least (\d+)\b", re.IGNORECASE), + re.compile(r"\bat most (\d+)\b", re.IGNORECASE), + re.compile(r"\bno more than (\d+)\b", re.IGNORECASE), + re.compile(r"\bmore than (\d+)\b", re.IGNORECASE), + re.compile(r"\bfewer than (\d+)\b", re.IGNORECASE), + re.compile(r"\bless than (\d+)\b", re.IGNORECASE), + re.compile(r"\bunder (\d+)\b", re.IGNORECASE), + re.compile(r"\bover (\d+)\b", re.IGNORECASE), + re.compile(r"\bup to (\d+)\b", re.IGNORECASE), + re.compile(r"\bbetween (\d+) and (\d+)\b", re.IGNORECASE), +] + +# PROPORTIONAL — descriptive fraction. +_PROPORTIONAL_PATTERNS = [ + r"\bmost\b", + r"\bmajority of\b", + r"\bminority of\b", + r"\bhalf (?:of|the)\b", + r"\ba third of\b", + r"\ba quarter of\b", + r"\b\d+%\s*of\b", + r"\bthe bulk of\b", + r"\bthe lion's share of\b", +] + +# ABSENT — universal-negation. +_ABSENT_PATTERNS = [ + r"\bnone\b", + r"\bno (?:one|body|where)\b", + r"\bnothing\b", + r"\bnobody\b", + r"\bnowhere\b", + r"\bneither\b", + r"\bnever\b", + r"\bnot a single\b", + r"\bzero\b", + # Negative wh-shape: "which X is/are/do/does/did/has/have not Y". + # Catches both copular ("which X is not Y") and auxiliary + # ("which X do not Y") forms — both express universal negation + # over the X universe. + r"\bwhich \w+ (?:is |are |do |does |did |has |have |were |was )?not\b", + r"\bwho (?:is |are |do |does |did |has |have )?not\b", + r"\bwhat (?:is |are |do |does |did |has |have )?not\b", +] + +# Compile each rung's patterns once. +_OPEN_REQUEST_RE = [re.compile(p, re.IGNORECASE) for p in _OPEN_REQUEST_PATTERNS] +_COMPREHENSIVE_RE = [re.compile(p, re.IGNORECASE) for p in _COMPREHENSIVE_PATTERNS] +_ALL_RE = [re.compile(p, re.IGNORECASE) for p in _ALL_PATTERNS] +_MANY_RE = [re.compile(p, re.IGNORECASE) for p in _MANY_PATTERNS] +_FEW_RE = [re.compile(p, re.IGNORECASE) for p in _FEW_PATTERNS] +_PROPORTIONAL_RE = [re.compile(p, re.IGNORECASE) for p in _PROPORTIONAL_PATTERNS] +_ABSENT_RE = [re.compile(p, re.IGNORECASE) for p in _ABSENT_PATTERNS] + + +# ---------------------------------------------------------------- shape mnemonics + +_OPERATIONAL_SHAPE = { + "ABSENT": "universal_negation", + "SINGULAR": "single_fact", + "PROPORTIONAL": "descriptive_fraction", + "SMALL_NUM_EXPLICIT": "bounded_count", + "COMPARATIVE_BOUND": "bounded_inequality", + "FEW": "small_set_vague", + "MANY": "medium_set_vague", + "ALL": "universal_enumeration", + "COMPREHENSIVE": "exhaustive_request", + "OPEN_REQUEST": "verb_driven_enumeration", +} + + +# ---------------------------------------------------------------- scope hint + +# Lexical anchors that hint a bounded universe — when present alongside +# a broad quantifier, scope_bound_hint upgrades from "unknown" to +# "bounded". This is a heuristic; the corpus arity check (§10.1) is +# left for a future refinement. +_BOUNDED_DOMAIN_ANCHORS = ( + re.compile(r"\bthe (?:beatles|fab four)\b", re.IGNORECASE), + re.compile(r"\b(?:US|U\.S\.|united states) (?:states|presidents)\b", re.IGNORECASE), + re.compile(r"\bplanets (?:in (?:the|our) solar system)?\b", re.IGNORECASE), + re.compile(r"\bfounding fathers\b", re.IGNORECASE), + re.compile(r"\bcontinents\b", re.IGNORECASE), + re.compile(r"\boceans\b", re.IGNORECASE), + # Year/season-anchored questions tend to bound the universe to one + # event — "winners of the 2024 World Series" is bounded. + re.compile(r"\b(?:19|20)\d{2}\b"), + re.compile(r"\b(?:season|year|championship|tournament|event) (?:of|for)\b", re.IGNORECASE), +) + + +def _scope_bound_hint(question: str, intensity: str) -> str: + """Return ``"bounded"``, ``"unbounded"``, or ``"unknown"``. + + Bounded universals (``"all members of the Beatles"``, ``"every US + state"``) have a finite, corpus-known answer set. Unbounded + universals (``"winners of all major sports"``) have an undefined + set under the current scope. The classifier defaults to + ``"unknown"`` for non-broad intensities (the question doesn't + have universal pressure to gate) and to ``"unbounded"`` for broad + intensities lacking a domain anchor. + + Heuristic only. The corpus arity check (§10.1 future refinement) + will sharpen this when implemented. + """ + if intensity not in _BROAD_RUNGS: + return "unknown" + for anchor in _BOUNDED_DOMAIN_ANCHORS: + if anchor.search(question): + return "bounded" + return "unbounded" + + +# ---------------------------------------------------------------- classifier + +def classify_question_quantifier(question: str) -> dict: + """Classify ``question`` onto the ten-rung intensity ladder. + + Highest-intensity-wins arbitration: when multiple rungs match, + pick the one farther from SINGULAR. Operationally that means + "tell me about all the planets" classifies OPEN_REQUEST (later + in the priority order than ALL), even though ALL also matched. + The downstream cap is the broader rung's cap, which is what we + want under enumeration pressure. + + Returns a dict; see module docstring for fields. + + Empty / whitespace-only questions classify SINGULAR (no quantifier + pressure) with no matched_token. + """ + if not question or not question.strip(): + return { + "intensity": "SINGULAR", + "matched_token": None, + "explicit_count": None, + "is_broad": False, + "operational_shape": _OPERATIONAL_SHAPE["SINGULAR"], + "scope_bound_hint": "unknown", + "classifier_version": CLASSIFIER_VERSION, + } + + candidates: list[tuple[str, str, int | None]] = [] + + # Per-rung detection. Earlier rungs run first but rung selection + # uses _RUNG_PRIORITY (highest-priority match wins). + def _check(rung_re_list: list[re.Pattern], rung_name: str) -> None: + for pat in rung_re_list: + m = pat.search(question) + if m: + token = m.group(0) + candidates.append((rung_name, token, None)) + return + + _check(_ABSENT_RE, "ABSENT") + _check(_PROPORTIONAL_RE, "PROPORTIONAL") + _check(_FEW_RE, "FEW") + _check(_MANY_RE, "MANY") + _check(_ALL_RE, "ALL") + _check(_COMPREHENSIVE_RE, "COMPREHENSIVE") + _check(_OPEN_REQUEST_RE, "OPEN_REQUEST") + + # SMALL_NUM_EXPLICIT — explicit digit or number word. + digit_match = _SMALL_NUM_DIGIT.search(question) + if digit_match: + # Pull the first non-None group as the count. + count = next( + (int(g) for g in digit_match.groups() if g is not None), + None, + ) + candidates.append(("SMALL_NUM_EXPLICIT", digit_match.group(0), count)) + else: + # Number-word path. Try each word; first hit wins. + for word, count in _NUMBER_WORDS.items(): + if re.search(rf"\b(?:top |first |last )?{word}\b", + question, re.IGNORECASE): + candidates.append( + ("SMALL_NUM_EXPLICIT", word, count) + ) + break + # `pair of` → 2; `a handful of` → 5 (folds into FEW + # operationally, but record as SMALL_NUM_EXPLICIT with + # count=2/5 so the cap is exact). + if _PAIR_OF.search(question): + candidates.append(("SMALL_NUM_EXPLICIT", "pair of", 2)) + if _HANDFUL_OF.search(question): + candidates.append(("SMALL_NUM_EXPLICIT", "a handful of", 5)) + + # COMPARATIVE_BOUND — explicit inequality. + for pat in _COMPARATIVE_PATTERNS: + m = pat.search(question) + if m: + # `between A and B` → use the upper bound; otherwise the + # single captured number. + groups = [int(g) for g in m.groups() if g is not None] + count = max(groups) if groups else None + candidates.append(("COMPARATIVE_BOUND", m.group(0), count)) + break + + if not candidates: + # Default — SINGULAR for any wh-question or definite reference + # without quantifier markers. + intensity = "SINGULAR" + matched_token: str | None = None + explicit_count: int | None = None + else: + # Highest-intensity wins (later in _RUNG_PRIORITY = higher). + priority = {r: i for i, r in enumerate(_RUNG_PRIORITY)} + candidates.sort(key=lambda c: priority.get(c[0], -1), reverse=True) + intensity, matched_token, explicit_count = candidates[0] + + return { + "intensity": intensity, + "matched_token": matched_token, + "explicit_count": explicit_count, + "is_broad": intensity in _BROAD_RUNGS, + "operational_shape": _OPERATIONAL_SHAPE[intensity], + "scope_bound_hint": _scope_bound_hint(question, intensity), + "classifier_version": CLASSIFIER_VERSION, + } diff --git a/aborist/qa/query.py b/aborist/qa/query.py index ab06ce6..9a82b04 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -1602,6 +1602,15 @@ def query( max_context_chars = int( by_mode.get(answer_mode, policy.get("max_context_chars", 60000)) ) + # Quantifier preflight (Ticket #000008 Phase 1). Pure lexical + # classification of the question into one of ten intensity rungs. + # Phase 1 lands in dry-run mode: the result is recorded on the + # query result dict (so bench rows pick it up) but no policy + # behavior changes — caps are NOT applied yet. Phase 2 wires the + # cap-application gate (`quantifier_guard_apply_caps`, default + # False during rollout per §10.11.3 dry-run discipline). + from aborist.qa.quantifier import classify_question_quantifier + quantifier = classify_question_quantifier(question) t_start = time.monotonic() # 1. Search. @@ -2101,6 +2110,16 @@ def query( # writes populate it correctly. Acceptable degradation # since governance_policy_hash invalidated prior records. "partially_verified_quotes": [], + # Quantifier preflight (Ticket #000008 Phase 1) — the + # classifier is pure on the question string, so cache + # hits can re-classify cheaply and carry the same + # schema as miss-path rows. Bench rows stay + # column-aligned across hit/miss. + "quantifier_intensity": quantifier["intensity"], + "quantifier_matched_token": quantifier["matched_token"], + "scope_bound_hint": quantifier["scope_bound_hint"], + "quantifier_explicit_count": quantifier["explicit_count"], + "claim_cap_applied": None, "prompt_chars": prompt_chars, "answer_chars": len(cached["answer_text"] or ""), "timings": { @@ -2591,6 +2610,21 @@ def query( # avoid duplication. Bench reads it for bracket-count # diagnostics; never persisted in providence_cache. "raw_answer": raw_answer if is_lattice_mode else None, + # Quantifier preflight result (Ticket #000008 Phase 1). + # Surfaced on the result so bench rows pick it up. Phase 1 + # is dry-run only — caps not applied; Phase 2 wires + # `claim_cap_applied`. None semantics: classifier always + # returns a dict so these are never None on the miss path, + # but the cache-hit path (line ~2080) doesn't carry them + # since the cached record predates the classifier. + "quantifier_intensity": quantifier["intensity"], + "quantifier_matched_token": quantifier["matched_token"], + "scope_bound_hint": quantifier["scope_bound_hint"], + "quantifier_explicit_count": quantifier["explicit_count"], + # Cap that was applied for this run. None until Phase 2 + # wires the cap-application gate; once Phase 2 lands this + # carries the per-call cap from policy + model_profile. + "claim_cap_applied": None, # Sidecar smell signals (claim_lattice mode only) — surfaced # for the renderer; never persisted in providence_cache and # never threaded into run_dag_root. diff --git a/tests/test_quantifier_classifier.py b/tests/test_quantifier_classifier.py new file mode 100644 index 0000000..f5ff5b9 --- /dev/null +++ b/tests/test_quantifier_classifier.py @@ -0,0 +1,337 @@ +"""Pure quantifier classifier (Ticket #000008 Phase 1). + +Tests pin the ten-rung intensity ladder defined in +``docs/tickets/ticket-000008-broad-quantifier-preflight-guard.md`` §2 +plus the bounded/unbounded scope hint from §10.1. + +Highest-intensity-wins arbitration: when a question matches multiple +rungs (e.g. "tell me about all the planets"), the later rung in +``_RUNG_PRIORITY`` wins. The classifier is purely lexical — no I/O, +no model call, no retrieval call. +""" + +from __future__ import annotations + +import pytest + +from aborist.qa.quantifier import ( + CLASSIFIER_VERSION, + classify_question_quantifier, +) + + +# ----------------------------------------------------------- empty / SINGULAR + +def test_empty_question_classifies_singular(): + out = classify_question_quantifier("") + assert out["intensity"] == "SINGULAR" + assert out["matched_token"] is None + assert out["is_broad"] is False + assert out["scope_bound_hint"] == "unknown" + + +def test_whitespace_only_classifies_singular(): + out = classify_question_quantifier(" \n\t ") + assert out["intensity"] == "SINGULAR" + + +def test_definite_singular_question(): + out = classify_question_quantifier("who painted the mona lisa?") + assert out["intensity"] == "SINGULAR" + assert out["matched_token"] is None # no marker fired + assert out["is_broad"] is False + + +def test_factoid_singular_question(): + out = classify_question_quantifier("what is the capital of france?") + assert out["intensity"] == "SINGULAR" + + +# ----------------------------------------------------------- ABSENT + +@pytest.mark.parametrize("q", [ + "which states do not border texas?", + "none of the planets have rings except saturn?", + "nothing about the soviet union", + "no one survived the eruption?", + "never was there a stronger king", +]) +def test_absent_questions(q): + out = classify_question_quantifier(q) + assert out["intensity"] == "ABSENT", f"{q} → {out}" + assert out["matched_token"] is not None + + +# ----------------------------------------------------------- PROPORTIONAL + +@pytest.mark.parametrize("q", [ + "most cats are mammals", + "the majority of voters supported the bill", + "half the planets have moons", + "a third of the population voted", + "the bulk of the work was done by volunteers", +]) +def test_proportional_questions(q): + out = classify_question_quantifier(q) + assert out["intensity"] == "PROPORTIONAL", f"{q} → {out}" + + +# ----------------------------------------------------------- SMALL_NUM_EXPLICIT + +def test_top_n_digit(): + out = classify_question_quantifier("top 3 winners of the open?") + assert out["intensity"] == "SMALL_NUM_EXPLICIT" + assert out["explicit_count"] == 3 + + +def test_n_biggest(): + out = classify_question_quantifier("five biggest cities in europe?") + assert out["intensity"] == "SMALL_NUM_EXPLICIT" + assert out["explicit_count"] == 5 + + +def test_seven_x(): + out = classify_question_quantifier("the original seven mercury astronauts") + assert out["intensity"] == "SMALL_NUM_EXPLICIT" + assert out["explicit_count"] == 7 + + +def test_dozen_maps_to_twelve(): + out = classify_question_quantifier("name a dozen examples?") + assert out["intensity"] == "SMALL_NUM_EXPLICIT" + assert out["explicit_count"] == 12 + + +def test_pair_of_maps_to_two(): + out = classify_question_quantifier("a pair of dice rolls?") + assert out["intensity"] == "SMALL_NUM_EXPLICIT" + assert out["explicit_count"] == 2 + + +# ----------------------------------------------------------- COMPARATIVE_BOUND + +def test_at_least_n(): + out = classify_question_quantifier("at least 5 examples please") + assert out["intensity"] == "COMPARATIVE_BOUND" + assert out["explicit_count"] == 5 + + +def test_more_than_n(): + out = classify_question_quantifier("more than 10 winners?") + assert out["intensity"] == "COMPARATIVE_BOUND" + assert out["explicit_count"] == 10 + + +def test_between_a_and_b(): + """Range — use the upper bound as the cap.""" + out = classify_question_quantifier("between 3 and 7 things") + assert out["intensity"] == "COMPARATIVE_BOUND" + assert out["explicit_count"] == 7 + + +def test_under_n(): + out = classify_question_quantifier("under 100 species?") + assert out["intensity"] == "COMPARATIVE_BOUND" + assert out["explicit_count"] == 100 + + +# ----------------------------------------------------------- FEW + +@pytest.mark.parametrize("q", [ + "list a few examples of supernovas", + "name several philosophers", + "show some constellations", + "list a couple of methods", +]) +def test_few_questions(q): + out = classify_question_quantifier(q) + assert out["intensity"] == "FEW", f"{q} → {out}" + + +# ----------------------------------------------------------- MANY + +@pytest.mark.parametrize("q", [ + "list many examples", + "show various flora", + "what are multiple causes of plague?", + "numerous battles in WWII", + "lots of birds in the amazon", +]) +def test_many_questions(q): + out = classify_question_quantifier(q) + assert out["intensity"] == "MANY", f"{q} → {out}" + + +# ----------------------------------------------------------- ALL + +@pytest.mark.parametrize("q", [ + "list all winners", + "every president of the US", + "each member of the cabinet", +]) +def test_all_questions(q): + out = classify_question_quantifier(q) + assert out["intensity"] == "ALL", f"{q} → {out}" + assert out["is_broad"] is True + + +def test_all_alone_classifies_all(): + out = classify_question_quantifier("Winners of all major sports?") + assert out["intensity"] == "ALL" + assert out["matched_token"].lower() == "all" + assert out["is_broad"] is True + + +# ----------------------------------------------------------- COMPREHENSIVE + +@pytest.mark.parametrize("q", [ + "give me a complete list of supernovas", + "comprehensive overview of the soviet union", + "tell me everything about the civil war", + "everything you know about the boson", + "exhaustive treatment of biology", + "the whole story behind the suez crisis", +]) +def test_comprehensive_questions(q): + out = classify_question_quantifier(q) + assert out["intensity"] == "COMPREHENSIVE", f"{q} → {out}" + assert out["is_broad"] is True + + +# ----------------------------------------------------------- OPEN_REQUEST + +@pytest.mark.parametrize("q", [ + "describe the structure of DNA", + "explain quantum mechanics", + "summarize world war I", + "give me an overview of cubism", + "walk me through the krebs cycle", + "tell me about connecticut", +]) +def test_open_request_questions(q): + out = classify_question_quantifier(q) + assert out["intensity"] == "OPEN_REQUEST", f"{q} → {out}" + assert out["is_broad"] is True + + +# ----------------------------------------------------------- highest-wins + +def test_open_request_beats_all(): + """`tell me about all the planets` matches BOTH OPEN_REQUEST and + ALL. Highest-priority rung wins (OPEN_REQUEST is later in + _RUNG_PRIORITY → higher intensity → cap is the broader bucket).""" + out = classify_question_quantifier("tell me about all the planets") + assert out["intensity"] == "OPEN_REQUEST" + + +def test_comprehensive_beats_open_request_and_all(): + """`tell me everything about all wars` matches three rungs: + OPEN_REQUEST (`tell me about`), COMPREHENSIVE (`tell me + everything`), and ALL (`all`). COMPREHENSIVE wins per the + "strictly stronger than ALL" precedence in §2.2 — it's the + explicit exhaustive-request marker, while the others are softer + shape-detectors that happen to overlap.""" + out = classify_question_quantifier("tell me everything about all wars") + assert out["intensity"] == "COMPREHENSIVE" + + +def test_explicit_count_overrides_few_when_present(): + """`name three examples` carries an explicit count — that should + not silently fall through to FEW. SMALL_NUM_EXPLICIT carries the + digit-bound cap.""" + out = classify_question_quantifier("name three examples") + assert out["intensity"] == "SMALL_NUM_EXPLICIT" + assert out["explicit_count"] == 3 + + +# ----------------------------------------------------------- scope_bound_hint + +def test_scope_bound_unknown_for_singular(): + out = classify_question_quantifier("who painted the mona lisa?") + assert out["scope_bound_hint"] == "unknown" + + +def test_scope_bound_unbounded_for_naked_all(): + out = classify_question_quantifier("Winners of all major sports?") + assert out["scope_bound_hint"] == "unbounded" + assert out["is_broad"] is True + + +def test_scope_bound_bounded_for_beatles_universal(): + """`all members of the Beatles` is a bounded universal — the set + is corpus-known and finite. Should NOT be treated like the + unbounded `winners of all major sports` case.""" + out = classify_question_quantifier("name all members of the Beatles") + assert out["intensity"] == "ALL" + assert out["scope_bound_hint"] == "bounded" + + +def test_scope_bound_bounded_for_planet_universal(): + out = classify_question_quantifier("list all planets in the solar system") + assert out["intensity"] == "ALL" + assert out["scope_bound_hint"] == "bounded" + + +def test_scope_bound_bounded_with_year_anchor(): + """A year anchor bounds the universe to a single event/season.""" + out = classify_question_quantifier( + "winners of all major sports in 2024" + ) + assert out["intensity"] == "ALL" + assert out["scope_bound_hint"] == "bounded" + + +# ----------------------------------------------------------- shape mnemonic + +def test_operational_shape_present_for_every_rung(): + """Every classification result must carry a non-None + operational_shape so downstream policy can pattern-match by shape + without re-deriving it from intensity.""" + examples = [ + "what is X?", # SINGULAR + "all X", # ALL + "tell me about X", # OPEN_REQUEST + "complete list of X", # COMPREHENSIVE + "many X", # MANY + "a few X", # FEW + "most X", # PROPORTIONAL + "top 3 X", # SMALL_NUM_EXPLICIT + "at least 5 X", # COMPARATIVE_BOUND + "no X", # ABSENT + ] + for q in examples: + out = classify_question_quantifier(q) + assert out["operational_shape"], f"{q} → {out}" + + +# ----------------------------------------------------------- versioning + +def test_classifier_version_stamped(): + """Every result carries CLASSIFIER_VERSION so post-hoc bench + analyses can identify which classifier version produced a + label.""" + out = classify_question_quantifier("anything?") + assert out["classifier_version"] == CLASSIFIER_VERSION + + +def test_classifier_version_is_pinned(): + """A version bump invalidates governance_policy_hash via the + Phase 2 _VERIFIER_POLICY_FIELDS extension. Pin the value here so + accidental reformatting doesn't bump it silently.""" + assert CLASSIFIER_VERSION == "quantifier-v0.1" + + +# ----------------------------------------------------------- regression: don't over-classify + +def test_simple_who_question_not_open_request(): + """`who is X` shouldn't fire any quantifier rung. Bare wh-questions + are SINGULAR by default.""" + out = classify_question_quantifier("who is bilbo baggins's nephew?") + assert out["intensity"] == "SINGULAR" + assert out["is_broad"] is False + + +def test_simple_what_question_not_open_request(): + out = classify_question_quantifier("what dinosaurs were in the first jurassic park film?") + assert out["intensity"] == "SINGULAR" + assert out["is_broad"] is False