diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..9e85eaa --- /dev/null +++ b/.gitattributes @@ -0,0 +1,7 @@ +# Windows batch shims must check out with CRLF so cmd.exe runs them +# reliably (LF-only .bat breaks on some Windows configurations). +*.bat text eol=crlf + +# Everything else stays LF — the repo is POSIX-first. +*.py text eol=lf +*.sh text eol=lf diff --git a/CLAUDE.md b/CLAUDE.md index 89b83ce..f8fd993 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -69,6 +69,9 @@ arborist/ │ ├── canonical_cache.py # canonical-projection persistence (#000027) │ ├── witness.py # multi-witness fan-out (#000028) │ ├── warrant_resolver.py # claim-pack warrant chain resolution (#000031) +│ ├── crosslang.py # cross-lang guard: signal + es stoppack (#000001 §7 P0) +│ ├── mt/ # Operation Sandwich MT edges (#000056) +│ │ # opus-mt es/fr/ru↔en + entity_mask │ └── runner.py # ask(): cache → infer → verify → write ├── concepts/ # corpus-derived synonym + rivalry layer (#000018 sib.) ├── pi_star/ # canonical projection π* registry (#000015) @@ -212,6 +215,30 @@ revert without reading why. When in doubt, walk the - **Soft hash vs hard hash**: hard = SHA-256 (commitments, proofs, cache_key); soft = embeddings/TF-IDF/similarity (training, ranking, distillation). Soft never enters proof path. +- **Cross-language = the sandwich, MT on the edges only**: translate + query in (retrieval + LLM prompt) → English answer → the + **byte-identical verifier** grounds English-vs-English → translate + the verified answer out as **display-only** (`display_*`, banner- + labelled, zero grounding — the `_render_audit_label` projection + discipline). Translation NEVER re-enters the verifier (that's the + #000049 model-in-proof-path cage). Invariants: `question_hash` = + the user's original question (untranslated); `verifier_policy_hash` + unchanged; MT engine identity binds into `RetrievalPlan.mt_*` (run- + DAG), NOT `governance_policy_hash` (the *flag* moves it like any + policy flag — correct cache partition — but that hash covers the + whole policy, `keys.py:182`; don't mistake "no new governance + field" for "governance untouched"). Engine = local `[mt]` opus-mt, + hash-pinned, never Hermes-for-translation, never an API. Default + OFF (`crosslang_guard_enabled` P0; `crosslang_translate_enabled` + Sandwich; `crosslang_entity_mask` **default-OFF** — measured + net-negative, kept only behind the flag). Measured net win over + the real "nothing" baseline (raw es → noise/UNGROUNDED): es ≈0 % → + 71 % grounded; the −14 pp vs *English* is the cost of a new + capability, not a regression (compare to no-cross-lang, never to + native English). The recall lever (entity-preservation) is **still + open** — `entity_mask` v1 failed at bench scale; corpus-title + anchoring is the untried idea. See `arborist/qa/crosslang.py`, + `arborist/qa/mt/`, #000001 §7, #000056 §9. - **Three answer modes**: `policy["answer_mode"] ∈ {"quote", "claim_lattice_pointer", "claim_lattice"}`, default `"quote"`. Bench 2026-05-02T15:07Z on Hermes-3-8B (post-Sprint-1b/2, n=3 diff --git a/Makefile b/Makefile index 69dbf8f..b04a40c 100644 --- a/Makefile +++ b/Makefile @@ -168,17 +168,17 @@ ANSWER_MODE ?= claim_lattice # REJECT_BROAD=1 → strict reject for ALL/COMPREHENSIVE/OPEN_REQUEST # unbounded shapes; returns UNGROUNDED before the LLM call. # ALLOW_BROAD=1 → emergent search; classifier on, caps off. -query: bootstrap ## ask the corpus a question [JSON=1 BURN=1 REPAIR=1 REPROMPTS=N K="extra retrieval keywords" ANSWER_MODE=claim_lattice|claim_lattice_pointer|quote BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1 WITNESS=1]; JSON by default +query: bootstrap ## ask the corpus a question [JSON=1 BURN=1 REPAIR=1 REPROMPTS=N K="extra retrieval keywords" ANSWER_MODE=claim_lattice|claim_lattice_pointer|quote BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1 WITNESS=1 XLANG=1 XLANG_MT=1]; JSON by default @if [ -z "$$Q" ] && [ -z "$(Q)" ]; then \ - echo "usage: make query Q=\"your question\" [JSON=1 BURN=1 REPAIR=1 REPROMPTS=N K=\"extra retrieval keywords\" ANSWER_MODE=claim_lattice|claim_lattice_pointer|quote BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1 WITNESS=1]"; exit 2; \ + echo "usage: make query Q=\"your question\" [JSON=1 BURN=1 REPAIR=1 REPROMPTS=N K=\"extra retrieval keywords\" ANSWER_MODE=claim_lattice|claim_lattice_pointer|quote BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1 WITNESS=1 XLANG=1 XLANG_MT=1]"; exit 2; \ fi - $(ARBORIST) --shards-dir $(SHARDS_DIR) query --top-k $(QUERY_TOP_K) $(if $(JSON),--json,) $(if $(BURN),--burn,) $(if $(REPAIR),--repair,) $(if $(REPROMPTS),--repair-reprompts $(REPROMPTS),) $(if $(ANSWER_MODE),--answer-mode $(ANSWER_MODE),) $(if $(K),--retrieval-keywords "$(K)",) $(if $(BROAD),--apply-quantifier-caps,) $(if $(REJECT_BROAD),--reject-broad,) $(if $(ALLOW_BROAD),--allow-broad,) $(if $(WITNESS),--witness,) "$(Q)" + $(ARBORIST) --shards-dir $(SHARDS_DIR) query --top-k $(QUERY_TOP_K) $(if $(JSON),--json,) $(if $(BURN),--burn,) $(if $(REPAIR),--repair,) $(if $(REPROMPTS),--repair-reprompts $(REPROMPTS),) $(if $(ANSWER_MODE),--answer-mode $(ANSWER_MODE),) $(if $(K),--retrieval-keywords "$(K)",) $(if $(BROAD),--apply-quantifier-caps,) $(if $(REJECT_BROAD),--reject-broad,) $(if $(ALLOW_BROAD),--allow-broad,) $(if $(WITNESS),--witness,) $(if $(XLANG),--crosslang-guard,) $(if $(XLANG_MT),--crosslang-translate,) "$(Q)" -query-dry: bootstrap ## like 'make query' but skip the LLM call (dry-run) [JSON=1 BURN=1 ANSWER_MODE=... BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1] +query-dry: bootstrap ## like 'make query' but skip the LLM call (dry-run) [JSON=1 BURN=1 ANSWER_MODE=... BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1 XLANG=1 XLANG_MT=1] @if [ -z "$$Q" ] && [ -z "$(Q)" ]; then \ - echo "usage: make query-dry Q=\"your question\" [JSON=1 BURN=1 ANSWER_MODE=claim_lattice|claim_lattice_pointer|quote BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1]"; exit 2; \ + echo "usage: make query-dry Q=\"your question\" [JSON=1 BURN=1 ANSWER_MODE=claim_lattice|claim_lattice_pointer|quote BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1 XLANG=1 XLANG_MT=1]"; exit 2; \ fi - $(ARBORIST) --shards-dir $(SHARDS_DIR) query --top-k $(QUERY_TOP_K) --dry-run $(if $(JSON),--json,) $(if $(BURN),--burn,) $(if $(ANSWER_MODE),--answer-mode $(ANSWER_MODE),) $(if $(BROAD),--apply-quantifier-caps,) $(if $(REJECT_BROAD),--reject-broad,) $(if $(ALLOW_BROAD),--allow-broad,) "$(Q)" + $(ARBORIST) --shards-dir $(SHARDS_DIR) query --top-k $(QUERY_TOP_K) --dry-run $(if $(JSON),--json,) $(if $(BURN),--burn,) $(if $(ANSWER_MODE),--answer-mode $(ANSWER_MODE),) $(if $(BROAD),--apply-quantifier-caps,) $(if $(REJECT_BROAD),--reject-broad,) $(if $(ALLOW_BROAD),--allow-broad,) $(if $(XLANG),--crosslang-guard,) $(if $(XLANG_MT),--crosslang-translate,) "$(Q)" BENCH_QA_QUESTIONS ?= bench/qa_questions.txt BENCH_QA_OUT ?= bench/qa_results diff --git a/README.md b/README.md index 6383551..e337fe7 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,8 @@ Arborist ingests documents into a content-addressed, Merkle-committed SQLite sto Two end-to-end paths. Pick whichever corpus you want first; both share the same query, verify, falsify, and inspect surfaces. +> **Windows / no `make`?** Every `make ` below works verbatim — a bundled `make.bat` shim forwards to a pure-stdlib runner (`tasks.py`). The quickstart needs only **Python 3.10+** (no `make`, no `bzip2`, no `curl`). In `cmd` type `make query Q="…"`; in PowerShell use `.\make.bat query Q="…"` (or `py -3 tasks.py query Q="…"` directly). See [Setup → Windows](#windows). The Makefile remains the canonical path on Linux/macOS. + ### A. Wikipedia 2003 (the canonical bootstrap dataset) ```sh @@ -58,7 +60,7 @@ git clone https://git.unturf.com/engineering/unturf/arborist.git ### Install prerequisites -Arborist needs Python 3.10+, GNU make, `curl`, and `bzip2`. SQLite 3.35+ ships with CPython. +The **quickstart needs only Python 3.10+** (SQLite ≥ 3.35 ships with CPython; the 2003 dump is decompressed in-process via Python's `bz2`). `GNU make`, `curl`, and `bzip2` are conveniences for the **Makefile** path on Linux/macOS — on Windows the bundled `make.bat` / `tasks.py` need none of them (see [Windows](#windows)). **macOS** (Homebrew) @@ -78,18 +80,36 @@ sudo apt install -y git python3 python3-venv python3-dev build-essential curl bz 22.04 ships Python 3.10; 24.04 ships 3.12 — both work. -**Windows** +**Windows** (native — no `make`, no WSL) -The Makefile uses bash idioms, so the supported path is **WSL2** running Ubuntu. From an admin PowerShell: +Install Python 3.10+ from and tick **"Add python.exe to PATH"**. That is the only prerequisite — the runner uses stdlib `urllib` for the download and stdlib `bz2` for decompression, so there is no `make`, `curl`, or `bzip2` to install. + +The bundled `make.bat` makes every documented `make ` work as-is: + +```bat +:: in cmd.exe (run from the repo root) +make bootstrap +make fetch-cur +make ingest-cur-attached +make distill-shards-parallel +make distill-shards-tfidf-parallel +make query Q="What is anarcho-capitalism?" +``` + +```powershell +# in PowerShell, prefix with .\ (PowerShell doesn't search the current dir) +.\make.bat bootstrap +.\make.bat query Q="What is anarcho-capitalism?" +``` + +Or call the runner directly with the `py` launcher: `py -3 tasks.py `. `py -3 tasks.py help` lists every supported target. The venv lands at `.venv\Scripts\` (vs `.venv/bin/` on POSIX); `tasks.py` resolves that automatically. Override the interpreter used to build the venv with the `ARBORIST_PYTHON` env var (e.g. `set ARBORIST_PYTHON=py -3.12`). + +`tasks.py` covers the **quickstart subset** — bootstrap, fetch/ingest/distill, query/inspect/falsify/burn, the crawl path, stats/verify/search, clean. The bench / π* / NLI / textbook / docs targets stay Makefile-only; for those (or to use the canonical Makefile) install **WSL2** and follow the Ubuntu instructions: ``` wsl --install -d Ubuntu-24.04 ``` -Then inside the WSL Ubuntu shell, follow the Ubuntu instructions above. - -(Native cmd / PowerShell + Git Bash mostly works for the Python parts but several `make` targets call `for i in $(seq…)` and `bash -c` — easier to just use WSL2.) - **OpenBSD** ``` @@ -104,9 +124,9 @@ OpenBSD's default `make` is BSD make. Arborist's Makefile uses GNU-make features make bootstrap ``` -Creates `.venv/`, installs the package in editable mode with the `[dev,html]` extras, and exposes `arborist` at `.venv/bin/arborist`. No system-wide install. Re-running `make bootstrap` is a no-op if the venv is up to date. +Creates `.venv/`, installs the package in editable mode with the `[dev]` extras, and exposes `arborist` at `.venv/bin/arborist` (`.venv\Scripts\arborist.exe` on Windows). No system-wide install. Re-running `make bootstrap` is a no-op if the venv is up to date. -After bootstrap, every workflow lives behind a `make` target. Run `make help` to list them. +After bootstrap, every workflow lives behind a `make` target. Run `make help` to list them (Windows: `py -3 tasks.py help` for the quickstart subset). ## Data: Wikipedia 2003-05-16 (Phase III SQL dump) diff --git a/arborist/cli.py b/arborist/cli.py index c22a59c..fa18508 100644 --- a/arborist/cli.py +++ b/arborist/cli.py @@ -562,6 +562,16 @@ def _cmd_query(args: argparse.Namespace) -> int: # Bench-first per §10.11.3 — this flag is the path from # dry-run to live-cap. call_policy["quantifier_guard_apply_caps"] = True + if getattr(args, "crosslang_guard", False): + # Ticket #000001 §7 Phase 0 — opt in to the cross-language + # guard per-call (default OFF). Pure policy-field set; the + # behaviour lives in query() (crosslang.guard + the two seams). + call_policy["crosslang_guard_enabled"] = True + if getattr(args, "crosslang_translate", False): + # Ticket #000056 — Operation Sandwich implies the Phase-0 + # guard (it rides the same non-English signal). + call_policy["crosslang_guard_enabled"] = True + call_policy["crosslang_translate_enabled"] = True # Ticket #000010 — meta-cognition CLI overrides. if getattr(args, "no_preflight", False): call_policy["metacognition_enabled"] = False @@ -5021,6 +5031,40 @@ def build_parser() -> argparse.ArgumentParser: "the classifier output across the question set." ), ) + # Ticket #000001 §7 Phase 0 — cross-language guard (default OFF; + # dry-run rollout discipline). Enable per-call to experiment. + query_cmd.add_argument( + "--crosslang-guard", + dest="crosslang_guard", action="store_true", + help=( + "Enable the deterministic cross-language guard for this " + "call (ticket #000001 §7 Phase 0, default OFF). A query " + "with a non-English signal (¿/¡/non-ASCII letter) has " + "es-v1 function words stripped from the retrieval set so " + "they can't drive an OR-mode full-corpus FTS5 scan; if no " + "corpus-language content token survives, returns " + "UNGROUNDED before retrieval/LLM with a " + "CROSS_LANGUAGE_UNSUPPORTED violation. Retrieval-side " + "only — never touches the verifier / cache_key / " + "question_hash. Provably inert on English." + ), + ) + query_cmd.add_argument( + "--crosslang-translate", + dest="crosslang_translate", action="store_true", + help=( + "Operation Sandwich (ticket #000056, default OFF; implies " + "--crosslang-guard). When the non-English signal fires, " + "translate the query es→en (local [mt] opus-mt; the " + "English article ranks primary AND the LLM is prompted in " + "English), ground the English answer with the UNTOUCHED " + "verifier, then render the verified English answer back " + "es as DISPLAY-ONLY (banner-labelled, zero grounding). " + "Needs the [mt] extra; degrades to the Phase-0 guard if " + "absent. Never touches the verifier / cache_key / " + "question_hash / governance_policy_hash." + ), + ) # Ticket #000010 — meta-cognition CLI flags. query_cmd.add_argument( "--no-preflight", diff --git a/arborist/qa/crosslang.py b/arborist/qa/crosslang.py new file mode 100644 index 0000000..b6343e6 --- /dev/null +++ b/arborist/qa/crosslang.py @@ -0,0 +1,113 @@ +"""Cross-language retrieval guard — Ticket #000001 §7, Phase 0. + +Deterministic, no model. Detects a query that is not in the corpus +language via a high-precision signal (inverted punctuation ``¿``/``¡`` +or a non-ASCII Latin letter), strips source-language function-word +noise from the *retrieval* token set so it cannot drive an OR-mode +full-corpus FTS5 scan, and fails closed to UNGROUNDED before +retrieval/LLM when no groundable content token survives. + +Bright line (CLAUDE.md "soft hash vs hard hash" / ticket §7.1): this +is a **retrieval-side** soft guard. It never touches the verifier or +``audit_mode``, and never enters ``question_hash`` (the user's +question is preserved) or ``verifier_policy_hash`` (verifier +byte-identical). The enabling *policy flag* does move +``governance_policy_hash`` — like every policy flag, since that hash +covers the whole policy dict (keys.py:182) — which correctly +partitions the cache by guard state. The English path is +**byte-identical by construction**: ``_NON_ENGLISH_SIGNAL_RE`` +cannot match a pure-ASCII +query without inverted punctuation, so ``guard()`` returns ``None`` +and the caller does nothing. + +Scope is es-v1 only. Other languages (and the MT retrieval route) +are Phase 1, gated on the MT-provider decision (ticket §7.5). + +The token regex / English stopword set are imported from +``arborist.search.fts5`` so "content token" here means exactly what +the retrieval path means (same ASCII regex, same stopwords, same +``len > 1`` rule). A blanket short-token heuristic is deliberately +NOT used — it would regress #000053 / #000054 (``AI``/``ML``/``CPU``/ +``GPU`` are load-bearing 2-3-char tokens). The es stoppack is gated +behind the non-English signal precisely so ``un``→UN / ``la``→LA / +``de``→De collisions can never reach an English query. +""" +from __future__ import annotations + +import re +from dataclasses import dataclass + +from arborist.search.fts5 import _FTS5_STOPWORDS, _FTS5_TOKEN_RE + +# Inverted punctuation (¿ ¡) or any non-ASCII byte (á é í ó ú ñ ü ç …). +# Pure-ASCII English questions never match → guard() returns None. +_NON_ENGLISH_SIGNAL_RE = re.compile(r"[¿¡]|[^\x00-\x7F]") + +# Spanish v1 function words, ASCII-folded / accent-truncated to match +# what _FTS5_TOKEN_RE actually emits (``¿Qué`` → "Qu"; ``qué`` → +# "qu"/"que"). Length-1 forms (a o y) are already dropped by the +# ``len > 1`` rule in fts5._query_tokens; not listed. +_ES_STOPWORDS = frozenset( + """ + qu que es el la lo los las le les un una uno unos unas + de del al en su sus se con por para + """.split() +) + + +@dataclass(frozen=True) +class CrossLanguageDecision: + """Result of the deterministic guard. ``None`` (not this class) is + returned for English — see ``guard``.""" + + dropped: tuple[str, ...] # es function tokens stripped from retrieval + content_tokens: tuple[str, ...] # surviving non-stopword tokens + fail_closed: bool # True → nothing groundable; reject pre-retrieval + + @property + def reason(self) -> str: + if self.fail_closed: + return ( + "cross-language query: no corpus-language content token " + "survived the source-language function-word filter " + "(es-v1). Fail closed before retrieval/LLM." + ) + return ( + "cross-language query: source-language function words stripped " + f"from the retrieval set ({', '.join(self.dropped) or 'none'}); " + "retrieval runs on surviving content tokens only." + ) + + +def _fts_tokens(text: str) -> list[str]: + """Exactly fts5._query_tokens' tokenization (ASCII regex, English + stopwords, len > 1) so 'content token' matches the retrieval path.""" + raw = _FTS5_TOKEN_RE.findall(text or "") + return [t for t in raw if t.lower() not in _FTS5_STOPWORDS and len(t) > 1] + + +def guard(question: str) -> CrossLanguageDecision | None: + """``None`` when the query looks like English (caller does nothing — + byte-identical path). Otherwise a decision the caller acts on + (ticket §7.3).""" + if not _NON_ENGLISH_SIGNAL_RE.search(question or ""): + return None + toks = _fts_tokens(question) + dropped = tuple(t for t in toks if t.lower() in _ES_STOPWORDS) + content = tuple(t for t in toks if t.lower() not in _ES_STOPWORDS) + return CrossLanguageDecision( + dropped=dropped, + content_tokens=content, + fail_closed=(len(content) == 0), + ) + + +def strip_for_retrieval(question: str, decision: CrossLanguageDecision) -> str: + """Retrieval-only query string with es function words removed. The + LLM, verifier, and ``question_hash`` never see this — only the FTS5 + / title / phrase routes do. Downstream applies its own English + stopword filter, so returning raw-minus-es is sufficient and + order-preserving.""" + raw = _FTS5_TOKEN_RE.findall(question or "") + kept = [t for t in raw if t.lower() not in _ES_STOPWORDS] + return " ".join(kept) diff --git a/arborist/qa/mt/__init__.py b/arborist/qa/mt/__init__.py new file mode 100644 index 0000000..56a528a --- /dev/null +++ b/arborist/qa/mt/__init__.py @@ -0,0 +1,27 @@ +"""Operation Sandwich MT engine — #000056. + +Two edges, never the middle: translate the query es→en (retrieval- +side, == ``--retrieval-keywords``) and render the *verified English* +answer en→es for display only. Nothing here touches the verifier, +``audit_mode``, ``cache_key``, or ``question_hash``. + +The ``transformers``/``torch`` runtime is the optional ``[mt]`` +extra; ``get_translator()`` returns a translator whose ``available`` +is False when the extra is absent, so importing this package and +running the sandwich is always safe (degrades to Phase-0). +""" +from .translator import ( + OpusMTTranslator, + StubTranslator, + Translator, + get_translator, + load_manifest, +) + +__all__ = [ + "Translator", + "OpusMTTranslator", + "StubTranslator", + "get_translator", + "load_manifest", +] diff --git a/arborist/qa/mt/entity_mask.py b/arborist/qa/mt/entity_mask.py new file mode 100644 index 0000000..cb22de5 --- /dev/null +++ b/arborist/qa/mt/entity_mask.py @@ -0,0 +1,123 @@ +"""Entity-preserving MT — #000056 §9 lever (the validated one). + +The fan-out delta (EN 85% → ES+sandwich 71%, −14pp) showed the +fixable slice of the LOST set is opus-mt *translating or garbling +proper nouns*: ``New London``→"nuevo Londres", ``Tarsus``→"Tarso", +``Boltzmann``→"perntzmann", ``the play Hamlet``→"the game village". +The standard MT fix: mask protected spans with sentinels MarianMT +copies verbatim, translate, restore the originals. + +Detection is deliberately **deterministic and hand-rolled** (five-step: +no NER model until a model earns it): + +1. quoted spans — verbatim idioms/titles (``"winter is coming"``); +2. Capitalised multi-word runs (with lowercase connectors) and + mid-sentence Capitalised tokens — ``New London``, ``Paul of + Tarsus``, ``Eiffel Tower``, ``Boltzmann``, ``Hamlet``. + +Scope honesty: mask/restore fixes the *garble/translate* class. It +does NOT solve exonym/transliteration (``Egipto``→corpus "Egypt", +``Segunda Guerra Mundial``→"World War II") — that needs a gazetteer +and is out of v1 (documented in #000056 §9). +""" +from __future__ import annotations + +import re + +# Sentinels: short, all-caps, alnum, no spaces — MarianMT copies these +# through untouched (verified by test). Bracketing letters make an +# accidental in-vocab collision astronomically unlikely. +_PH = "ZQX{0}XQZ" +_PH_RE = re.compile(r"ZQX(\d+)XQZ") + +_QUOTED = re.compile(r"[\"“”«»']([^\"“”«»']{2,60})[\"“”«»']") +# A proper-noun run: a Capitalised word, optionally chained through +# lowercase connectors (of/de/the/von/…) to more Capitalised words. +_CAP_RUN = re.compile( + r"\b[A-ZÀ-Þ][\wÀ-ÿ'’.-]*" + r"(?:\s+(?:of|de|del|la|le|van|von|der|the|y|and|&)\s+[A-ZÀ-Þ][\wÀ-ÿ'’.-]*" + r"|\s+[A-ZÀ-Þ][\wÀ-ÿ'’.-]*)*" +) +# Leading-token guard: a sentence-initial Capitalised word that is just +# an ordinary first word (wh/function) is NOT an entity. +_NOT_ENTITY = frozenset( + "what who when where why how which whose is are was were do does did " + "the a an this that these those tell name list describe explain give " + "qué quién cuándo dónde cómo cuál cuáles por qué el la los las un una " + "что кто когда где почему как какой".split() +) + + +def protect_spans(text: str) -> list[tuple[int, int]]: + """Char ranges to keep verbatim through MT, longest-first, merged.""" + spans: list[tuple[int, int]] = [] + for m in _QUOTED.finditer(text): + spans.append((m.start(), m.end())) + for m in _CAP_RUN.finditer(text): + s, e = m.start(), m.end() + tok = text[s:e] + if tok.lower() in _NOT_ENTITY: + continue + # Single leading-Capitalised ordinary word at pos 0 → skip. + if s == 0 and " " not in tok and tok.lower() in _NOT_ENTITY: + continue + spans.append((s, e)) + spans.sort(key=lambda p: (p[0], -(p[1] - p[0]))) + merged: list[tuple[int, int]] = [] + for s, e in spans: + if merged and s < merged[-1][1]: + continue # overlapped by an earlier (longer) span + merged.append((s, e)) + return merged + + +def mask(text: str) -> tuple[str, dict[str, str]]: + spans = protect_spans(text) + if not spans: + return text, {} + out, mapping, last, i = [], {}, 0, 0 + for s, e in spans: + out.append(text[last:s]) + ph = _PH.format(i) + mapping[ph] = text[s:e] + out.append(ph) + last = e + i += 1 + out.append(text[last:]) + return "".join(out), mapping + + +def restore(text: str, mapping: dict[str, str]) -> str: + if not mapping: + return text + + def _sub(m: re.Match) -> str: + ph = m.group(0) + return mapping.get(ph, ph) + + # MarianMT may alter spacing/case around a copied sentinel; match + # the numeric core so ``ZQX 0 XQZ`` / ``zqx0xqz`` still restore. + return re.sub(r"[Zz][Qq][Xx]\s*(\d+)\s*[Xx][Qq][Zz]", + lambda m: mapping.get(_PH.format(int(m.group(1))), m.group(0)), + text) + + +class MaskedTranslator: + """Wraps any ``Translator``: mask protected spans → delegate → + restore. Same protocol, so the sandwich is engine-agnostic.""" + + def __init__(self, inner): + self._inner = inner + self.engine_id = f"{getattr(inner, 'engine_id', 'mt')}+entmask-v1" + self.manifest_hash = f"{getattr(inner, 'manifest_hash', '')}:entmask-v1" + + @property + def available(self) -> bool: + return getattr(self._inner, "available", False) + + def translate(self, text: str, src: str, tgt: str) -> str: + masked, mapping = mask(text) + out = self._inner.translate(masked, src, tgt) + if not getattr(self._inner, "available", False): + return text # graceful-degrade like the inner engine + return restore(out, mapping) diff --git a/arborist/qa/mt/manifest.json b/arborist/qa/mt/manifest.json new file mode 100644 index 0000000..7395dba --- /dev/null +++ b/arborist/qa/mt/manifest.json @@ -0,0 +1,54 @@ +{ + "_comment": "Pinned MT checkpoints for #000056 Operation Sandwich. RETRIEVAL-SIDE + DISPLAY-SIDE ONLY: this manifest does NOT fold into governance_policy_hash. Query MT is a retrieval transform (== --retrieval-keywords, #000001 §5/§6 — routes through context_root, never question_hash); its identity binds into the run-DAG retrieval stage via retrieval_plan_hash. Display MT is presentation-only (never the verifier's input, never hashed into the proof). A future audit_mode-affecting strict mode (the #000001 §6 optional-strict analogue) would add an explicit mt_policy_hash that folds into governance_policy_hash before any audit_mode effect; out of scope here.", + "mt_model_version": "opus-mt-sandwich-v1", + "license": "Apache-2.0", + "runtime": "transformers-torch-cpu (MarianMT)", + "max_length": 512, + "pairs": { + "es-en": { + "hf_repo": "Helsinki-NLP/opus-mt-es-en", + "pinned_revision": "main", + "source_url": "https://huggingface.co/Helsinki-NLP/opus-mt-es-en", + "approx_mb": 300, + "role": "query_in" + }, + "en-es": { + "hf_repo": "Helsinki-NLP/opus-mt-en-es", + "pinned_revision": "main", + "source_url": "https://huggingface.co/Helsinki-NLP/opus-mt-en-es", + "approx_mb": 300, + "role": "display_out" + }, + "fr-en": { + "hf_repo": "Helsinki-NLP/opus-mt-fr-en", + "pinned_revision": "main", + "source_url": "https://huggingface.co/Helsinki-NLP/opus-mt-fr-en", + "approx_mb": 300, + "role": "query_in" + }, + "en-fr": { + "hf_repo": "Helsinki-NLP/opus-mt-en-fr", + "pinned_revision": "main", + "source_url": "https://huggingface.co/Helsinki-NLP/opus-mt-en-fr", + "approx_mb": 300, + "role": "display_out" + }, + "ru-en": { + "hf_repo": "Helsinki-NLP/opus-mt-ru-en", + "pinned_revision": "main", + "source_url": "https://huggingface.co/Helsinki-NLP/opus-mt-ru-en", + "approx_mb": 300, + "role": "query_in" + }, + "en-ru": { + "hf_repo": "Helsinki-NLP/opus-mt-en-ru", + "pinned_revision": "main", + "source_url": "https://huggingface.co/Helsinki-NLP/opus-mt-en-ru", + "approx_mb": 300, + "role": "display_out" + } + }, + "pinned_revision_provenance": "v1 ships revision='main' as a placeholder; before any default-ON / bench-gated promotion this MUST be replaced with the resolved 40-hex commit SHA per pair so a pinned checkpoint replays byte-identically (the provenance reason local-was-chosen over an API). Tracked in #000056 §6 / acceptance criterion follow-up.", + "engine_id": "opus-mt-v1", + "never": "Hermes-3-8B (8B unfit for human-language translation; blackops standing rule). Not an external API (zero egress / Operation Voyeur; opaque API versions are provenance-hostile for a Merkle-replay system)." +} diff --git a/arborist/qa/mt/translator.py b/arborist/qa/mt/translator.py new file mode 100644 index 0000000..a3fa9de --- /dev/null +++ b/arborist/qa/mt/translator.py @@ -0,0 +1,233 @@ +"""Local machine-translation engine for #000056 Operation Sandwich. + +Two edges, never the middle (ticket §1/§2): + + Spanish query ─▶ [es→en] ─▶ English retrieval + answer + EXISTING + verifier ─▶ [en→es] ─▶ Spanish display + +This module only *translates strings*. It has no idea about the +verifier, `audit_mode`, `cache_key`, or `question_hash` — by design. +The caller (`arborist.qa.query`) is responsible for keeping the +grounded core English-only and putting the Spanish rendering in a +display-only field. + +The `transformers`/`torch` runtime is the optional `[mt]` extra. +Construction never raises: if the extra is missing or the pinned +checkpoint will not load, :attr:`available` stays False and the +sandwich silently degrades to Phase-0 behaviour (query proceeds +untranslated). This is the `ShadowNLI` pattern (#000049) verbatim. +""" +from __future__ import annotations + +import hashlib +import json +import os +import threading +from pathlib import Path +from typing import Optional, Protocol, runtime_checkable + +_MANIFEST_PATH = Path(__file__).resolve().parent / "manifest.json" + + +def load_manifest(path: Optional[Path] = None) -> dict: + return json.loads((path or _MANIFEST_PATH).read_text()) + + +def _manifest_hash(manifest: dict) -> str: + """Stable id of the *engine identity* an auditor needs to reproduce + which sources a translated query retrieved. Pairs (repo+revision) + + model_version + engine_id only — not the prose `_comment` blobs.""" + core = { + "mt_model_version": manifest.get("mt_model_version"), + "engine_id": manifest.get("engine_id"), + "pairs": { + k: {"hf_repo": v.get("hf_repo"), + "pinned_revision": v.get("pinned_revision")} + for k, v in sorted((manifest.get("pairs") or {}).items()) + }, + } + canon = json.dumps(core, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(canon.encode("utf-8")).hexdigest() + + +@runtime_checkable +class Translator(Protocol): + """Minimal contract the sandwich wiring depends on.""" + + available: bool + engine_id: str + manifest_hash: str + + def translate(self, text: str, src: str, tgt: str) -> str: ... + + +class StubTranslator: + """Deterministic translator — the default test substrate. + + `mapping` is keyed by ``(src, tgt, text)``; an unmapped input is + returned unchanged so a stub run is a safe identity no-op unless a + test pins a transform. Deterministic by construction → audit + replay is byte-stable, like every other stub in the suite. + """ + + def __init__(self, mapping: Optional[dict[tuple[str, str, str], str]] = None, + *, fn=None, engine_id: str = "stub-mt", available: bool = True): + self._map = dict(mapping or {}) + self._fn = fn # callable(text, src, tgt) -> str; wins over mapping + self.available = available + self.engine_id = engine_id + self.manifest_hash = "stub" + + def translate(self, text: str, src: str, tgt: str) -> str: + if src == tgt: + return text + if self._fn is not None: + return self._fn(text, src, tgt) + return self._map.get((src, tgt, text), text) + + +class OpusMTTranslator: + """Lazily-loaded pinned Helsinki-NLP opus-mt (MarianMT) pair set. + + Construction never raises (the `ShadowNLI` contract): a missing + `[mt]` extra or an unloadable checkpoint leaves :attr:`available` + False and :meth:`translate` returns its input unchanged, so the + sandwich degrades to Phase-0 (untranslated query) rather than + erroring. + """ + + def __init__(self, manifest: Optional[dict] = None, + device: Optional[str] = None): + self.manifest = manifest or load_manifest() + self.engine_id: str = self.manifest.get("engine_id", "opus-mt") + self.manifest_hash: str = _manifest_hash(self.manifest) + self.max_length: int = int(self.manifest.get("max_length", 512)) + self.device_pref: str = ( + device or os.environ.get("ARBORIST_MT_DEVICE") or "auto" + ) + self.device: Optional[str] = None + self.available = False + self._reason = "uninitialised" + self._models: dict[str, tuple] = {} # "es-en" -> (tok, model) + # The model load is not concurrency-safe (a parallel bench at + # concurrency>1 hit `Tensor.item() cannot be called on meta + # tensors` racing the lazy load). Serialise the *load*; the + # forward pass is fine concurrently once weights are resident. + self._load_lock = threading.Lock() + + def _cache_dir(self) -> Path: + env = os.environ.get("ARBORIST_MT_DIR") + return Path(env) if env else Path.home() / ".arborist" / "models" / "mt" + + def _ensure_loaded(self) -> None: + # Double-checked: lock-free fast path once resident/failed, + # then serialise the one-time load (concurrency-safe). + if self.available or self._reason.startswith( + ("deps_missing", "load_failed") + ): + return + with self._load_lock: + if self.available or self._reason.startswith( + ("deps_missing", "load_failed") + ): + return + self._load_locked() + + def _load_locked(self) -> None: + # Deps + device only. Models load lazily per pair (6 manifest + # pairs × ~300 MB → never load all eagerly; load on first use). + try: + import torch # noqa: F401 + from transformers import ( # noqa: F401 + AutoModelForSeq2SeqLM, AutoTokenizer, + ) + except ImportError as e: + self._reason = ( + f"deps_missing: {e} (install: pip install 'arborist[mt]')" + ) + return + if self.device_pref == "auto": + self.device = "cuda" if torch.cuda.is_available() else "cpu" + else: + self.device = self.device_pref + self.available = True # engine usable; pairs load on demand + self._reason = "ok" + + def _ensure_pair(self, pair: str): + """Load one es-en/en-fr/… pair on first use, serialised (the + load is the concurrency-unsafe step — #000056 fan-out).""" + got = self._models.get(pair) + if got is not None: + return got + spec = (self.manifest.get("pairs") or {}).get(pair) + if spec is None: + return None + with self._load_lock: + got = self._models.get(pair) + if got is not None: + return got + from transformers import ( + AutoModelForSeq2SeqLM, AutoTokenizer, + ) + try: + cache = str(self._cache_dir()) + tok = AutoTokenizer.from_pretrained( + spec["hf_repo"], revision=spec.get("pinned_revision"), + cache_dir=cache, + ) + model = AutoModelForSeq2SeqLM.from_pretrained( + spec["hf_repo"], revision=spec.get("pinned_revision"), + cache_dir=cache, + ) + model.eval() + if self.device == "cuda": + model = model.to("cuda") + except Exception as e: # noqa: BLE001 — load failure = passthrough + self._reason = f"load_failed[{pair}]: {type(e).__name__}: {e}" + return None + self._models[pair] = (tok, model) + return self._models[pair] + + def translate(self, text: str, src: str, tgt: str) -> str: + if not text or src == tgt: + return text + self._ensure_loaded() + if not self.available: + return text # graceful degrade → sandwich no-ops to Phase-0 + got = self._ensure_pair(f"{src}-{tgt}") + if got is None: + return text # unsupported / unloadable pair → safe passthrough + import torch + + tok, model = got + enc = tok( + [text], return_tensors="pt", truncation=True, + max_length=self.max_length, + ) + if self.device == "cuda": + enc = {k: v.to("cuda") for k, v in enc.items()} + with torch.no_grad(): + out = model.generate(**enc, max_length=self.max_length) + return tok.batch_decode(out, skip_special_tokens=True)[0].strip() or text + + +_SINGLETON: Optional[OpusMTTranslator] = None +_SINGLETON_LOCK = threading.Lock() + + +def get_translator(manifest: Optional[dict] = None) -> Translator: + """Factory the sandwich uses. Returns a **process-level memoised** + `OpusMTTranslator` so the ~300 MB MarianMT pair loads **once** and + is reused across every query (a bench at concurrency>1 reloaded it + per call → meta-tensor races + 75× waste; #000056 fan-out). The + caller gates on `.available` (False when `[mt]` is absent → the + sandwich degrades to Phase-0, never raises). A custom `manifest` + bypasses the cache (test isolation).""" + if manifest is not None: + return OpusMTTranslator(manifest=manifest) + global _SINGLETON + if _SINGLETON is None: + with _SINGLETON_LOCK: + if _SINGLETON is None: + _SINGLETON = OpusMTTranslator() + return _SINGLETON diff --git a/arborist/qa/query.py b/arborist/qa/query.py index 13a201d..52e4cc2 100644 --- a/arborist/qa/query.py +++ b/arborist/qa/query.py @@ -500,6 +500,44 @@ DEFAULT_QUERY_POLICY = { # Ticket #000011 — soft preflight sidecar. See runner.DEFAULT_POLICY # for full rationale. Default OFF. "soft_preflight_enabled": False, + # Ticket #000001 §7 Phase 0 — deterministic cross-language guard. + # Default OFF (dry-run rollout discipline, same as the #000008 + # quantifier guard / #000011 soft preflight / #000049 NLI): the + # feature is provably inert on English and strictly improves the + # cross-language case, but the default-flip is a separate + # bench-gated decision fox makes, not an autonomous one. Flip ON + # per-call to experiment (CLI `--crosslang-guard`, `make query + # XLANG=1`). Hash invariants (artifact — `governance_policy_hash` + # is sha256 of the *whole* policy, keys.py:182): NOT in + # `question_hash` (the user's question is preserved) and NOT in + # `verifier_policy_hash` (verifier byte-identical); like every + # policy flag it DOES change `governance_policy_hash`, which + # correctly partitions the cache by guard state (a guard-on answer + # must not be served to a guard-off lookup). The fail-closed path + # writes no cache row anyway. + "crosslang_guard_enabled": False, + # Ticket #000056 — Operation Sandwich. Requires + # crosslang_guard_enabled (it rides the Phase-0 signal). Default + # OFF, same rollout discipline. When ON + signal fires + the [mt] + # engine is available: query es→en (retrieval + LLM prompt), + # English answer through the UNTOUCHED verifier, en→es of the + # verified answer into display-only fields. NOT in question_hash + # (user's question preserved) nor verifier_policy_hash (verifier + # byte-identical); like any policy flag it does move + # governance_policy_hash (whole-policy hash → correct cache + # partition). MT *engine identity* binds into the run-DAG + # retrieval plan (RetrievalPlan.mt_*), not any policy hash. + "crosslang_translate_enabled": False, + # #000056 §9 — entity-preserving MT (mask proper nouns → translate + # → restore). Default **OFF**: the isolated win (Paul of Tarsus) + # did NOT replicate at bench scale — mask measured net-negative + # (es 71%→65%, fr −38pp; the lowercased bench gives it nothing to + # grab on the en side and the sentinels perturb opus-mt). The + # no-mask sandwich is the better config; mask stays available + # behind the flag for the corpus-title-anchor follow-up only. + # crosslang_source_lang names the bread (es default; fr/ru bench). + "crosslang_entity_mask": False, + "crosslang_source_lang": "es", # Claim-count ceiling — see runner.DEFAULT_POLICY for rationale. # Bench finding (york-england "tell me all there is to know") # caught the runaway shape; cap of 12 admits entity-list @@ -1729,6 +1767,7 @@ def query( fidelity: str | None = None, burn_existing: bool = False, retrieval_keywords: str | None = None, + translator: object | None = None, progress: Progress | None = None, ) -> dict: """Answer `question` using the corpus. Cache to qa_db. Returns a result dict. @@ -1793,6 +1832,126 @@ def query( max_context_chars = int( by_mode.get(answer_mode, policy.get("max_context_chars", 60000)) ) + # Ticket #000001 §7 Phase 0 — deterministic cross-language guard. + # Retrieval-side only; never touches verifier / audit_mode / + # cache_key / governance_policy_hash / question_hash. English path + # is byte-identical: `guard()` returns None for any pure-ASCII + # query without inverted punctuation, so `_xlang` stays None and + # nothing below changes. When the non-English signal fires and no + # corpus-language content token survives the es-v1 function-word + # filter, fail closed to UNGROUNDED before preflight/retrieval/LLM + # (mirrors the `quantifier_should_reject` reject-DAG path so the + # rejection stays Merkle-auditable). The non-fail-closed case is + # handled at the `retrieval_query` construction below. + # Single gate for BOTH §7 seams (fail-closed below + retrieval_query + # strip later): flag OFF → `_xlang` is None → both blocks skip → + # behaviour reverts byte-for-byte to pre-#000001-§7, giving a clean + # A/B baseline for experimentation. + from arborist.qa.crosslang import guard as _xlang_guard + _xlang = ( + _xlang_guard(question) + if policy.get("crosslang_guard_enabled", False) + else None + ) + if _xlang is not None and _xlang.fail_closed: + from arborist.qa.dag import ( + build_reject_run_dag as _xl_build_dag, + _canonical_json as _xl_canon, + _sha256_hex as _xl_sha, + ) + _xl_vmethod = ( + "claim_lattice_pointer" + if answer_mode == "claim_lattice_pointer" + else "claim_lattice" + if answer_mode == "claim_lattice" + else "quote" + ) + _xl_qhash = question_hash( + question, mode=policy.get("question_dedup", "equivalence_class"), + ) + _xl_violations = [{ + "kind": "CROSS_LANGUAGE_UNSUPPORTED", + "signal": "non_english_punctuation_or_script", + "stoppack": "es-v1", + "dropped_tokens": list(_xlang.dropped), + "content_tokens": list(_xlang.content_tokens), + "reason": _xlang.reason, + }] + _xl_answer_text = ( + "CROSS-LANGUAGE PREFLIGHT — UNGROUNDED\n\n" + "This query is not in the corpus language and no cross-" + "language bridge is enabled (ticket #000001 §7). Ask in " + "English, or wait for the MT retrieval route (Phase 1, " + "gated on the provider decision)." + ) + _xl_preflight_payload = { + "guard": "crosslang-v1", + "signal": True, + "stoppack": "es-v1", + "dropped_tokens": list(_xlang.dropped), + "content_tokens": list(_xlang.content_tokens), + } + _xl_preflight_hash = _xl_sha(_xl_canon(_xl_preflight_payload)) + _xl_run_dag = _xl_build_dag( + question_hash=_xl_qhash, + preflight_hash=_xl_preflight_hash, + preflight_payload=_xl_preflight_payload, + rejection_reason=_xlang.reason, + answer_text=_xl_answer_text, + audit_mode="UNGROUNDED", + verifier_method=_xl_vmethod, + violations=_xl_violations, + ) + return { + "status": "cross_language_unsupported", + "audit_mode": "UNGROUNDED", + "cache_key": None, + "lookup_path": "preflight", + "run_dag_root": _xl_run_dag["root"], + "run_dag_blob": json.dumps(_xl_run_dag, separators=(",", ":")), + "preflight_hash": _xl_preflight_hash, + "answer_text": _xl_answer_text, + "sources": [], + "n_quotes": 0, + "n_verified": 0, + "verifier_method": _xl_vmethod, + "unverified_quotes": [], + "partially_verified_quotes": [], + "violations": _xl_violations, + } + # Ticket #000056 — Operation Sandwich. Edge IN: when the Phase-0 + # signal fired (non-English) and there IS groundable content (not + # fail-closed) and translation is opted in, translate the query + # es→en so the English corpus ranks primary AND the LLM is prompted + # in English (so it answers in English → the UNTOUCHED verifier + # grounds English-vs-English). `question` (Spanish) is left + # untouched: it remains the user's question for `question_hash` / + # cache identity. The translation is a retrieval/prompt transform + # whose engine identity binds into the run-DAG retrieval plan + # (RetrievalPlan.mt_*), exactly the `--retrieval-keywords` status. + # `_mt` graceful-degrades (no `[mt]` extra → available False → + # `_sandwich_en_q` stays None → behaviour falls back to Phase-0). + _sandwich_en_q: str | None = None + _mt = None + _src_lang = "es" + if ( + _xlang is not None + and not _xlang.fail_closed + and policy.get("crosslang_translate_enabled", False) + ): + from arborist.qa.mt import get_translator + _mt = translator or get_translator() + if policy.get("crosslang_entity_mask", False): # #000056 §9: net-negative, default off + from arborist.qa.mt.entity_mask import MaskedTranslator + _mt = MaskedTranslator(_mt) + _src_lang = policy.get("crosslang_source_lang", "es") + _cand = _mt.translate(question, _src_lang, "en") + if getattr(_mt, "available", False) and _cand and _cand != question: + _sandwich_en_q = _cand + # The text the LLM is prompted with (English when the sandwich is + # active; the user's original question otherwise). NEVER feeds + # question_hash. + llm_question = _sandwich_en_q or question # Quantifier preflight (Ticket #000008 Phase 1+2). Phase 1 runs # the lexical classifier; Phase 2 looks up the per-model cap. # The cap is REPORTED on the result dict (claim_cap_applied) but @@ -2387,8 +2546,24 @@ def query( # tokens; the question text fed to the LLM and to question_hash # stays untouched. retrieval_query = question + # Ticket #000001 §7 Phase 0 — non-fail-closed cross-language case + # (signal fired, a content token survived: the `anarcocapitalismo` + # field shape). Strip es-v1 function words from the RETRIEVAL + # string only so they cannot drive an OR-mode full-corpus FTS5 + # scan (the measured 9.9 s cost). question / question_hash / the + # LLM prompt / the verifier surface are untouched. English path: + # `_xlang` is None → unchanged. + if _xlang is not None and not _xlang.fail_closed: + from arborist.qa.crosslang import strip_for_retrieval + retrieval_query = strip_for_retrieval(question, _xlang) + # Ticket #000056 — Operation Sandwich edge IN. A real es→en + # translation is strictly better than the Phase-0 stoppack-strip + # (the English article ranks *primary*, not background). Still + # retrieval-side only; `question`/`question_hash` untouched. + if _sandwich_en_q: + retrieval_query = _sandwich_en_q if retrieval_keywords and retrieval_keywords.strip(): - retrieval_query = f"{question} {retrieval_keywords.strip()}" + retrieval_query = f"{retrieval_query} {retrieval_keywords.strip()}" progress.emit( "search.start", top_k=int(top_k), @@ -2771,7 +2946,12 @@ def query( ) if broad: messages.append({"role": "user", "content": broad}) - messages.append({"role": "user", "content": _user_payload(question)}) + # Ticket #000056 — the LLM is prompted with `llm_question` (the + # English translation when the sandwich is active, else the user's + # original question) so it answers in English and the UNTOUCHED + # verifier grounds English-vs-English. `question_hash` / cache + # identity still derive from the original `question`. + messages.append({"role": "user", "content": _user_payload(llm_question)}) # Capacity metrics. Char-level for now — a fast model-agnostic proxy # for prompt size (rule of thumb: ~4 chars/token for English prose, @@ -3320,6 +3500,15 @@ def query( or h.shard_path)} ) ), + # #000056 — bind the MT engine identity into the retrieval + # plan iff the sandwich actually translated the query + # (empty otherwise → RetrievalPlan.canonical() omits it → + # zero hash churn on every non-MT run). + mt_engine=(getattr(_mt, "engine_id", "") if _sandwich_en_q else ""), + mt_manifest_hash=( + getattr(_mt, "manifest_hash", "") if _sandwich_en_q else "" + ), + source_lang=(_src_lang if _sandwich_en_q else ""), ) plan_hash = retrieval_plan_hash(plan) # Ticket #000009 — preflight node binding (nested CTI clauses @@ -3668,6 +3857,37 @@ def query( result["verifier_input_text"] = ( rendered_evidence if is_lattice_mode else context ) + # Ticket #000056 — Operation Sandwich edge OUT. Render the + # *already-verified English* `answer_text` back to the user's + # language as DISPLAY-ONLY, additive keys. This runs AFTER the + # proof/cache/run-DAG are finalised above: it is never the + # verifier's input, never hashed into the proof, never persisted + # as a proof column. `answer_text` (the grounded string) and + # `audit_mode` are untouched — the Spanish text carries zero + # grounding and is banner-labelled (the `_render_audit_label` + # render-projection discipline). + if _sandwich_en_q and _mt is not None: + # Translate the model's PROSE, never the rendered evidence + # scaffold. In lattice modes `answer_text` is the rendered + # answer with interpolated *verbatim pinned English source + # spans* (`[E1 | Title | hash: "quote"]`) — MT-ing those would + # corrupt the very spans the verifier matched. `raw_answer` + # (model output, present in lattice modes) is the prose + + # opaque [E1] markers; quote mode has no scaffold so + # `answer_text` is already prose. + _src_en = result.get("raw_answer") or result.get("answer_text") or "" + _es = _mt.translate(_src_en, "en", _src_lang) if _src_en else "" + result["display_answer"] = _es or _src_en + result["display_lang"] = _src_lang + result["display_source_lang"] = "en" + result["display_translated"] = True + result["display_engine"] = getattr(_mt, "engine_id", "") + result["display_unverified_banner"] = ( + "Traducción automática para lectura — la verificación se " + "realizó sobre la respuesta en inglés y sus fuentes en " + "inglés. / Machine-translated for display; grounding was " + "verified on the English answer against English sources." + ) return result diff --git a/arborist/qa/retrieval_plan.py b/arborist/qa/retrieval_plan.py index c7bec10..c602a74 100644 --- a/arborist/qa/retrieval_plan.py +++ b/arborist/qa/retrieval_plan.py @@ -61,18 +61,38 @@ class RetrievalPlan: # at call time). Operators who pin specific shards via # ``--shards-dir`` or ``single_db`` get those captured here. shard_ids: tuple[str, ...] = field(default_factory=tuple) + # Ticket #000056 (Operation Sandwich) — cross-language query MT is + # a *retrieval transform* (the query the user asked was translated + # before it hit FTS5; an auditor reproducing source selection needs + # the engine identity). Same status as ``retrieval_keywords``: + # binds into the run-DAG retrieval stage, NOT ``question_hash`` / + # ``governance_policy_hash``. Empty (the non-MT default) → omitted + # from ``canonical()`` so every pre-#000056 ``retrieval_plan_hash`` + # is byte-identical (the §5 zero-churn discipline; ticket §5 #3). + mt_engine: str = "" + mt_manifest_hash: str = "" + source_lang: str = "" def canonical(self) -> dict: """Sorted-key dict for canonical-JSON hashing. Empty fields keep their default values so the hash is stable across calls - that omit optional knobs.""" - return { + that omit optional knobs. The #000056 MT fields are + *omitted entirely* when unset so non-MT runs hash exactly as + they did pre-#000056 (zero churn).""" + out = { "retrieval_keywords": self.retrieval_keywords or "", "top_k": int(self.top_k), "over_fetch": int(self.over_fetch), "max_context_chars": int(self.max_context_chars), "shard_ids": list(self.shard_ids), } + if self.mt_engine or self.mt_manifest_hash or self.source_lang: + out["mt"] = { + "engine": self.mt_engine or "", + "manifest_hash": self.mt_manifest_hash or "", + "source_lang": self.source_lang or "", + } + return out def retrieval_plan_hash(plan: RetrievalPlan) -> str: diff --git a/bench/es_delta.py b/bench/es_delta.py new file mode 100644 index 0000000..ad337a5 --- /dev/null +++ b/bench/es_delta.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""The real metric: per-question English-baseline → Spanish+sandwich +transition. Absolute es grounding % is uninterpretable on an +adversarial bench; what the sandwich *costs* is the transition. + + usage: es_delta.py + +Joins on the en↔es map. Buckets each question: + PRESERVED en grounded (S/H) & es grounded + DOWNGRADE en STRICT & es HYBRID (partial cost) + LOST en grounded & es UNGROUNDED (the real cost) + N/A en UNGROUNDED (sandwich not at fault) + GAINED en UNGROUNDED & es grounded (noise/curio) +Then cross-tabs LOST vs the round-trip bucket — does drift predict +the *loss* transition (even though it didn't predict absolute %)? +""" +from __future__ import annotations + +import json +import sys +from collections import Counter +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +RES = ROOT / "bench" / "qa_results" +GROUNDED = {"STRICT", "HYBRID"} + + +def load(p): + return [json.loads(x) for x in Path(p).read_text().splitlines() if x.strip()] + + +def main() -> int: + en_rows = load(sys.argv[1]) + es_rows = load(sys.argv[2]) + pairs = json.loads((ROOT / "bench" / "qa_questions_es_map.json").read_text()) + rt = {r["es"]: r["bucket"] for r in + json.loads((RES / "es_roundtrip.json").read_text())} + en_am = {r["question"]: r.get("audit_mode") for r in en_rows} + es_am = {r["question"]: r.get("audit_mode") for r in es_rows} + + cls = Counter() + lost_by_bucket = Counter() + lost_list, downgrade_list = [], [] + for p in pairs: + en, es = p["en"], p["es"] + a, b = en_am.get(en), es_am.get(es) + if a is None or b is None: + cls["MISSING"] += 1 + continue + if a not in GROUNDED: + cls["N/A (en ungrounded)"] += 1 + if b in GROUNDED: + cls[" └ of which GAINED"] += 1 + continue + if b not in GROUNDED: + cls["LOST"] += 1 + lost_by_bucket[rt.get(es, "?")] += 1 + lost_list.append((en, es, a, b)) + elif a == "STRICT" and b == "HYBRID": + cls["DOWNGRADE (S→H)"] += 1 + downgrade_list.append((en, es)) + else: + cls["PRESERVED"] += 1 + + n = len(pairs) + en_g = sum(1 for p in pairs if en_am.get(p["en"]) in GROUNDED) + es_g = sum(1 for p in pairs if es_am.get(p["es"]) in GROUNDED) + print(f"n={n} EN-baseline grounded={en_g} ({en_g/n:.0%}) " + f"ES+sandwich grounded={es_g} ({es_g/n:.0%}) " + f"net Δ={es_g-en_g:+d}") + print("\ntransition (only en-grounded questions can be 'LOST'):") + for k, v in cls.most_common(): + print(f" {k:<24} {v}") + print("\nLOST × round-trip bucket (does drift predict the LOSS?):") + for bk in ("CLEAN", "DRIFT", "COLLAPSE", "?"): + if lost_by_bucket.get(bk): + print(f" {bk:<9} {lost_by_bucket[bk]}") + print("\n-- LOST questions (en grounded, es+sandwich UNGROUNDED) --") + for en, es, a, b in lost_list: + print(f" [{a}→{b}] en={en}\n es={es} rt={rt.get(es)}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bench/es_join_patterns.py b/bench/es_join_patterns.py new file mode 100644 index 0000000..4414066 --- /dev/null +++ b/bench/es_join_patterns.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Join the live es+sandwich sweep against the deterministic round-trip +buckets → does MT round-trip drift PREDICT grounding loss? + + sweep JSONL (question=es, audit_mode, status) + ⋈ es_roundtrip.json (es → bucket CLEAN/DRIFT/COLLAPSE) + → crosstab bucket × audit_mode, plus the MT-engine error rate + (the concurrency defect the fan-out surfaced) reported separately + so it doesn't get conflated with the MT-quality signal. +""" +from __future__ import annotations + +import json +import sys +from collections import Counter, defaultdict +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +RES = ROOT / "bench" / "qa_results" + + +def main() -> int: + sweep_path = Path(sys.argv[1]) if len(sys.argv) > 1 else max( + RES.glob("2026-*.jsonl"), key=lambda p: p.stat().st_mtime) + rt = {r["es"]: r for r in json.loads((RES / "es_roundtrip.json").read_text())} + rows = [json.loads(ln) for ln in sweep_path.read_text().splitlines() if ln.strip()] + + err = [r for r in rows if (r.get("status") or "").startswith(("err", "error")) + or r.get("audit_mode") is None] + ok = [r for r in rows if r not in err] + print(f"sweep: {sweep_path.name} rows={len(rows)} " + f"engine/other ERROR={len(err)} ({len(err)/max(1,len(rows)):.0%}) " + f"scored={len(ok)}") + + ct = defaultdict(Counter) + unmatched = 0 + for r in ok: + b = rt.get(r["question"], {}).get("bucket") + if b is None: + unmatched += 1 + continue + ct[b][r.get("audit_mode") or "?"] += 1 + print("\nbucket × audit_mode (scored rows only):") + modes = ["STRICT", "HYBRID", "UNGROUNDED", "?"] + print(f" {'bucket':<9} " + " ".join(f"{m:>10}" for m in modes) + " ground%") + for b in ("CLEAN", "DRIFT", "COLLAPSE"): + c = ct.get(b, Counter()) + tot = sum(c.values()) + g = c["STRICT"] + c["HYBRID"] + print(f" {b:<9} " + " ".join(f"{c[m]:>10}" for m in modes) + + f" {g}/{tot}" + (f" ({g/tot:.0%})" if tot else "")) + if unmatched: + print(f" ({unmatched} scored rows had no round-trip match)") + print("\nreading: if CLEAN grounds >> COLLAPSE, MT round-trip drift " + "predicts grounding loss — the lever is entity-preserving MT, " + "not the sandwich architecture.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bench/es_roundtrip_analysis.py b/bench/es_roundtrip_analysis.py new file mode 100644 index 0000000..ab67ea1 --- /dev/null +++ b/bench/es_roundtrip_analysis.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Round-trip drift analysis for #000056 (the pattern, deterministic). + +For each bench pair: en --[opus-mt-en-es]--> es --[opus-mt-es-en]--> en' +The es→en leg is the SANDWICH'S ACTUAL edge-IN. If en' preserves the +content nouns of en, the sandwich feeds FTS5 the right terms and +grounding tracks the English baseline. If a content noun is lost +(Hamlet→"village", "New London"→"new London"), retrieval can't find +the article no matter how good Hermes is — the failure is upstream of +grounding, in named-entity-preserving MT. + +Buckets by content-token recall of en' vs en (stopword-stripped): + CLEAN >= 0.80 COLLAPSE < 0.40 DRIFT otherwise +No Hermes, no network — offline + reproducible. +""" +from __future__ import annotations + +import json +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from arborist.qa.mt import OpusMTTranslator # noqa: E402 + +_W = re.compile(r"[A-Za-z][A-Za-z0-9]*") +_STOP = set("the a an is are was were be of to in on at for with by from as " + "and or what who where when why how which this that who whom did " + "do does has have had can could would will who're were name named " + "between play wrote write written who's".split()) + + +def toks(s: str) -> set[str]: + return {w.lower() for w in _W.findall(s) if w.lower() not in _STOP and len(w) > 1} + + +def main() -> int: + pairs = json.loads((ROOT / "bench" / "qa_questions_es_map.json").read_text()) + tr = OpusMTTranslator() + rows = [] + for p in pairs: + en, es = p["en"], p["es"] + back = tr.translate(es, "es", "en") # the sandwich's edge-IN + a, b = toks(en), toks(back) + recall = round(len(a & b) / len(a), 2) if a else 1.0 + bucket = "CLEAN" if recall >= 0.80 else "COLLAPSE" if recall < 0.40 else "DRIFT" + lost = sorted(a - b) + rows.append({"en": en, "es": es, "back": back, "recall": recall, + "bucket": bucket, "lost_tokens": lost}) + out = ROOT / "bench" / "qa_results" / "es_roundtrip.json" + out.write_text(json.dumps(rows, ensure_ascii=False, indent=2) + "\n") + from collections import Counter + c = Counter(r["bucket"] for r in rows) + n = len(rows) + print(f"n={n} CLEAN={c['CLEAN']} ({c['CLEAN']/n:.0%}) " + f"DRIFT={c['DRIFT']} ({c['DRIFT']/n:.0%}) " + f"COLLAPSE={c['COLLAPSE']} ({c['COLLAPSE']/n:.0%})") + print("\n-- COLLAPSE (named entity / key noun lost on round-trip) --") + for r in rows: + if r["bucket"] == "COLLAPSE": + print(f" en : {r['en']}\n back: {r['back']} lost={r['lost_tokens']}") + print("\n-- a few CLEAN --") + for r in [x for x in rows if x["bucket"] == "CLEAN"][:6]: + print(f" {r['en']} ==~ {r['back']}") + print(f"\nwrote {out}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bench/make_es_questions.py b/bench/make_es_questions.py new file mode 100644 index 0000000..63b94bb --- /dev/null +++ b/bench/make_es_questions.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Generate the Spanish bench set from the English one via opus-mt-en-es. + +Reproducible round-trip fixture for #000056: the SAME engine the +sandwich uses on its edges generates the Spanish questions, so the +bench measures `es(question) → [sandwich es→en] → English grounding` +end-to-end with no hand-translation, no Hermes-for-translation, no +egress. Comment / blank lines are preserved verbatim so qa_sweep.py +skips them exactly as in the English file. + +Writes: + bench/qa_questions_es.txt one es question per line + bench/qa_questions_es_map.json [{en, es}, ...] in file order +""" +from __future__ import annotations + +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from arborist.qa.mt import OpusMTTranslator # noqa: E402 + +SRC = ROOT / "bench" / "qa_questions.txt" +OUT = ROOT / "bench" / "qa_questions_es.txt" +MAP = ROOT / "bench" / "qa_questions_es_map.json" + + +def main() -> int: + tr = OpusMTTranslator() + lines = SRC.read_text().splitlines() + out_lines: list[str] = [ + "# AUTO-GENERATED from bench/qa_questions.txt via " + "Helsinki-NLP/opus-mt-en-es (#000056 Operation Sandwich).", + "# Do not hand-edit — regenerate: python3 bench/make_es_questions.py", + "", + ] + pairs: list[dict] = [] + n = 0 + for ln in lines: + s = ln.strip() + if not s or s.startswith("#"): + out_lines.append(ln) + continue + es = tr.translate(s, "en", "es") + if not tr.available: + print("opus-mt unavailable — install 'arborist[mt]'", file=sys.stderr) + return 1 + out_lines.append(es) + pairs.append({"en": s, "es": es}) + n += 1 + print(f" [{n}] {s} -> {es}", flush=True) + OUT.write_text("\n".join(out_lines) + "\n") + MAP.write_text(json.dumps(pairs, ensure_ascii=False, indent=2) + "\n") + print(f"\nwrote {OUT} ({n} questions)\nwrote {MAP}", flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bench/make_lang_questions.py b/bench/make_lang_questions.py new file mode 100644 index 0000000..8b45ca2 --- /dev/null +++ b/bench/make_lang_questions.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""Generate bench/qa_questions_.txt from the English set via +opus-mt-en- (#000056 §9 multi-bread). usage: make_lang_questions.py """ +from __future__ import annotations +import json, sys +from pathlib import Path +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from arborist.qa.mt import OpusMTTranslator + +LANG = sys.argv[1] if len(sys.argv) > 1 else "es" +SRC = ROOT / "bench" / "qa_questions.txt" +OUT = ROOT / "bench" / f"qa_questions_{LANG}.txt" +MAP = ROOT / "bench" / f"qa_questions_{LANG}_map.json" + +def main() -> int: + tr = OpusMTTranslator() + out = [f"# AUTO-GENERATED from qa_questions.txt via opus-mt-en-{LANG} (#000056).", + "# Regenerate: python3 bench/make_lang_questions.py " + LANG, ""] + pairs = [] + n = 0 + for ln in SRC.read_text().splitlines(): + s = ln.strip() + if not s or s.startswith("#"): + out.append(ln); continue + t = tr.translate(s, "en", LANG) + if not tr.available: + print("opus-mt unavailable", file=sys.stderr); return 1 + out.append(t); pairs.append({"en": s, LANG: t}); n += 1 + print(f" [{n}] {s} -> {t}", flush=True) + OUT.write_text("\n".join(out) + "\n") + MAP.write_text(json.dumps(pairs, ensure_ascii=False, indent=2) + "\n") + print(f"wrote {OUT} ({n}) + {MAP}", flush=True) + return 0 + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bench/qa_questions_es.txt b/bench/qa_questions_es.txt new file mode 100644 index 0000000..67f9f8d --- /dev/null +++ b/bench/qa_questions_es.txt @@ -0,0 +1,157 @@ +# AUTO-GENERATED from bench/qa_questions.txt via Helsinki-NLP/opus-mt-en-es (#000056 Operation Sandwich). +# Do not hand-edit — regenerate: python3 bench/make_es_questions.py + +# arborist QA-quality benchmark question set. +# +# One question per line. `#`-prefixed lines and blank lines ignored. +# Designed for the 2003-05-16 Wikipedia cur snapshot — questions must +# resolve against ~2003 article content. Stress-tests across the +# failure-mode shapes the verifier needs to handle. + +# narrow factoid — well-anchored single-fact questions +¿Cuál es la capital de Francia? +¿Quién escribió GNU Linux? +¿Cuándo se creó un lenguaje de programación llamado Python? +¿Quién pintó la Mona Lisa? +un puente entre el nuevo Londres y Groton? +¿Quién escribió la aldea de juegos? +¿Cuál es el símbolo químico para el oro? + +# broad descriptive — encyclopedic shape, prone to mode-collapse +hábleme de connecticut +Háblame del lenguaje de programación C +¿Hablarme del hombre del método? +Dime todo lo que hay que saber sobre York Inglaterra? +Háblame del imperio romano. +describir la estructura del ADN + +# under-specified "all" — the word "all" reads to Hermes-3-8B as +# license to enumerate every adjacent fact in training prior, which +# in claim_lattice_pointer mode degrades to free-form prose with zero +# `[E\d+]` tags (FORMAT_COLLAPSED soft-demote, 2026-05-02). Stronger +# models (Qwen / GPT-4 family) plausibly recover format discipline +# under the same prompt — bench coverage of this shape lets us +# measure cross-model resilience. Ticket #000008 (broad-quantifier +# preflight guard) proposes upstream classification + per-model +# claim ceiling. +¿ganadores de todos los deportes importantes? + +# bounded universals — finite, corpus-known answer sets. Ticket +# #000008 §10.1 splits broad universals into bounded vs unbounded. +# These should classify ALL but with `scope_bound_hint: "bounded"`, +# meaning --reject-broad does NOT reject and the cap is the natural +# bound. Without these fixtures, the bounded-vs-unbounded distinction +# has no live bench coverage. +nombre a todos los miembros de los beatles +lista todos los planetas en el sistema solar + +# entity list — invites lazy-anchor on a magnet chunk +¿Qué eran los dinosaurios en la primera película del parque jurásico? +¿Quiénes son los miembros de los Beatles? +nombre simpsons miembros de la familia incluyendo mascotas? +lista de obelisco en connecticut +¿Cuáles son los planetas de nuestro sistema solar? +¿Quiénes fueron los siete astronautas de mercurio originales? + +# relationship / multi-fact +¿Quién es la novia de Supermans? +¿Quién es el sobrino de Bilbo Baggins? +¿Cuál es la relación entre linux & unix? +¿Quién es Veronica Ballestrini y qué mes nació? +¿Quién es el padre y la hermana de Luke Skywalker? + +# comparison — multi-entity, prone to attribution drift +¿Cuál es la diferencia entre Linux y Bsd? +¿Cómo se compara la información con la amd? +¿Cuál es la diferencia entre http y ftp? +mac vs ventanas para el desarrollo de software + +# niche / partial — corpus may be thin +¿Cuál es la constante de perntzmann? +¿Quién inventó el efecto doppler? + +# date / time — when-questions stress year/date grounding +¿Cuándo terminó la Segunda Guerra Mundial? +¿En qué año cayó el muro de Berlín? +¿Cuándo fue el primer alunizaje? +¿Cuándo se disuelve la unión soviética? +¿En qué año se firmó la carta magna? + +# quantity — numeric grounding, prone to confabulated digits +¿Cuántos estados hay en los Estados Unidos? +¿Cuántos huesos hay en el cuerpo humano adulto? +¿Cuál es la población de Japón? +¿Cuántas esposas tenía Henry la octava? +¿Cuántas lunas tiene Jupiter? + +# geographic — where / what country / location grounding +¿Dónde se encuentra el monte kilimanjaro? +¿En qué país está la ciudad de Praga? +¿En qué océano está Madagascar? +¿Dónde comienza el río Nile? +¿En qué continente está Egipto? + +# cause / effect — why-questions, narrative grounding +¿Por qué se hundió el titánico? +¿Qué causó el desastre de Chernobyl? +¿Por qué los dinosaurios se extinguieron? +¿Qué desencadenó la Primera Guerra Mundial? + +# entity disambiguation — common names that collide with many entities +¿Quién es Michael Jordan? +¿Quién es George Bush? +¿Quién es John Smith de Jamestown? +¿Qué es la matriz? +¿Quién es Paul de Tarso? + +# synonym / paraphrase robustness — same fact, different phrasing. +# Pairs probe whether equivalence_class question dedup collapses these +# at write & whether retrieval grounds them identically. +¿Quién fundó microsoft? +¿Quién es el fundador de microsoft? +¿Cuándo se construyó la torre eifel? +¿En qué año se construyó la torre Eiffel? + +# leading / forensic — probe whether retrieval surfaces the right +# fact when the question's premise contradicts the popular narrative. +# Elevation question expected answer: north (per geographic surveys); +# corpus grounding may be shaky — coverage check can pass on +# topical-token overlap without the cited span containing the +# elevation fact, so this also stress-tests lazy-anchor STRICT. +¿De qué lado es la elevación del suelo más alta a lo largo de la gran pared de China, el norte o el sur? +ballenas están en peligro de extinción por la caza excesiva +¿Napoleón realmente murió en Santa Helena? + +# out-of-corpus — should land UNGROUNDED honestly +¿Quién es un dictador benevolente de por vida para Marte? +¿En qué año se produce nuestra fusión fría? +¿Quién ganó las elecciones presidenciales de 2024? +¿Cuál es la última versión del ipad pro? + +# allusion / reference frame — the diagnostic signal is a verbatim +# multi-token sequence, not the individual content tokens. Pure-BM25 +# on tokens will surface literal-frame articles (Oceania the region, +# Asia the continent) instead of the reference (Nineteen Eighty-Four). +# The phrase-pattern retrieval route (commit 2026-05-01) targets this +# class. Bench reveals prevalence of the failure mode + whether the +# fix generalizes beyond Orwell. +¿Oceania siempre ha estado en guerra con Asia oriental? +¿Quién dijo que la fuerza podría estar contigo? +¿Qué significa el invierno? +¿Cuál es el significado de Rosebud? +¿Quién dijo ser o no ser esa es la pregunta? +¿Qué es lo que el pastel es una referencia de mentira? + +# hyphenation / orthography — query and corpus use different hyphen +# conventions for the same concept. FTS5 `unicode61` splits on hyphen +# at index AND query time, so `bi-polar` (the orthography many +# English readers actually use) tokenizes to [bi, polar] while the +# Wikipedia article `Bipolar disorder` tokenizes to [bipolar]. Two +# non-overlapping token sets for the same medical condition. The +# query-layer hyphen-fold (Ticket #000007, 2026-05-02) closes this +# class by emitting joined-no-hyphen variants on the query side and +# adding a hyphen-anchor accept path in `_filter_by_title_relevance`. +# Pre-fix: only Bi-Polar (album) / Bi-Polar Blues / BI disambiguation +# surfaced; the medical-condition cluster never reached the LLM. +# Post-fix: `Bipolar disorder` lands in top-K and grounds the answer. +¿El bipolar es raro? diff --git a/bench/qa_questions_es_map.json b/bench/qa_questions_es_map.json new file mode 100644 index 0000000..e9594f4 --- /dev/null +++ b/bench/qa_questions_es_map.json @@ -0,0 +1,302 @@ +[ + { + "en": "what is the capital of france?", + "es": "¿Cuál es la capital de Francia?" + }, + { + "en": "who wrote GNU linux?", + "es": "¿Quién escribió GNU Linux?" + }, + { + "en": "when was a programming language named Python created?", + "es": "¿Cuándo se creó un lenguaje de programación llamado Python?" + }, + { + "en": "who painted the mona lisa?", + "es": "¿Quién pintó la Mona Lisa?" + }, + { + "en": "a bridge between new london & groton?", + "es": "un puente entre el nuevo Londres y Groton?" + }, + { + "en": "who wrote the play hamlet?", + "es": "¿Quién escribió la aldea de juegos?" + }, + { + "en": "what is the chemical symbol for gold?", + "es": "¿Cuál es el símbolo químico para el oro?" + }, + { + "en": "tell me about connecticut", + "es": "hábleme de connecticut" + }, + { + "en": "tell me about the C programming language", + "es": "Háblame del lenguaje de programación C" + }, + { + "en": "tell me about method man?", + "es": "¿Hablarme del hombre del método?" + }, + { + "en": "tell me all there is to know about york england?", + "es": "Dime todo lo que hay que saber sobre York Inglaterra?" + }, + { + "en": "tell me about the roman empire", + "es": "Háblame del imperio romano." + }, + { + "en": "describe the structure of DNA", + "es": "describir la estructura del ADN" + }, + { + "en": "winners of all major sports?", + "es": "¿ganadores de todos los deportes importantes?" + }, + { + "en": "name all members of the beatles", + "es": "nombre a todos los miembros de los beatles" + }, + { + "en": "list all planets in the solar system", + "es": "lista todos los planetas en el sistema solar" + }, + { + "en": "what dinosaurs were in the first jurassic park film?", + "es": "¿Qué eran los dinosaurios en la primera película del parque jurásico?" + }, + { + "en": "who are the members of the beatles?", + "es": "¿Quiénes son los miembros de los Beatles?" + }, + { + "en": "name simpsons family members including pets?", + "es": "nombre simpsons miembros de la familia incluyendo mascotas?" + }, + { + "en": "list of obelisk in connecticut", + "es": "lista de obelisco en connecticut" + }, + { + "en": "what are the planets of our solar system?", + "es": "¿Cuáles son los planetas de nuestro sistema solar?" + }, + { + "en": "who were the original seven mercury astronauts?", + "es": "¿Quiénes fueron los siete astronautas de mercurio originales?" + }, + { + "en": "who is supermans girlfriend?", + "es": "¿Quién es la novia de Supermans?" + }, + { + "en": "who is bilbo baggins's nephew?", + "es": "¿Quién es el sobrino de Bilbo Baggins?" + }, + { + "en": "what is the relationship between linux & unix?", + "es": "¿Cuál es la relación entre linux & unix?" + }, + { + "en": "who is veronica ballestrini & what month was she born?", + "es": "¿Quién es Veronica Ballestrini y qué mes nació?" + }, + { + "en": "who is luke skywalker's father and sister?", + "es": "¿Quién es el padre y la hermana de Luke Skywalker?" + }, + { + "en": "what's the difference between linux and bsd?", + "es": "¿Cuál es la diferencia entre Linux y Bsd?" + }, + { + "en": "how does intel compare to amd?", + "es": "¿Cómo se compara la información con la amd?" + }, + { + "en": "what is the difference between http and ftp?", + "es": "¿Cuál es la diferencia entre http y ftp?" + }, + { + "en": "mac vs windows for software development", + "es": "mac vs ventanas para el desarrollo de software" + }, + { + "en": "what is the boltzmann constant?", + "es": "¿Cuál es la constante de perntzmann?" + }, + { + "en": "who invented the doppler effect?", + "es": "¿Quién inventó el efecto doppler?" + }, + { + "en": "when did world war 2 end?", + "es": "¿Cuándo terminó la Segunda Guerra Mundial?" + }, + { + "en": "what year did the berlin wall fall?", + "es": "¿En qué año cayó el muro de Berlín?" + }, + { + "en": "when was the first moon landing?", + "es": "¿Cuándo fue el primer alunizaje?" + }, + { + "en": "when did the soviet union dissolve?", + "es": "¿Cuándo se disuelve la unión soviética?" + }, + { + "en": "in what year was the magna carta signed?", + "es": "¿En qué año se firmó la carta magna?" + }, + { + "en": "how many states are in the united states?", + "es": "¿Cuántos estados hay en los Estados Unidos?" + }, + { + "en": "how many bones are in the adult human body?", + "es": "¿Cuántos huesos hay en el cuerpo humano adulto?" + }, + { + "en": "what is the population of japan?", + "es": "¿Cuál es la población de Japón?" + }, + { + "en": "how many wives did henry the eighth have?", + "es": "¿Cuántas esposas tenía Henry la octava?" + }, + { + "en": "how many moons does jupiter have?", + "es": "¿Cuántas lunas tiene Jupiter?" + }, + { + "en": "where is mount kilimanjaro located?", + "es": "¿Dónde se encuentra el monte kilimanjaro?" + }, + { + "en": "what country is the city of prague in?", + "es": "¿En qué país está la ciudad de Praga?" + }, + { + "en": "in which ocean is madagascar?", + "es": "¿En qué océano está Madagascar?" + }, + { + "en": "where does the nile river begin?", + "es": "¿Dónde comienza el río Nile?" + }, + { + "en": "what continent is egypt on?", + "es": "¿En qué continente está Egipto?" + }, + { + "en": "why did the titanic sink?", + "es": "¿Por qué se hundió el titánico?" + }, + { + "en": "what caused the chernobyl disaster?", + "es": "¿Qué causó el desastre de Chernobyl?" + }, + { + "en": "why did the dinosaurs go extinct?", + "es": "¿Por qué los dinosaurios se extinguieron?" + }, + { + "en": "what triggered world war 1?", + "es": "¿Qué desencadenó la Primera Guerra Mundial?" + }, + { + "en": "who is michael jordan?", + "es": "¿Quién es Michael Jordan?" + }, + { + "en": "who is george bush?", + "es": "¿Quién es George Bush?" + }, + { + "en": "who is john smith of jamestown?", + "es": "¿Quién es John Smith de Jamestown?" + }, + { + "en": "what is the matrix?", + "es": "¿Qué es la matriz?" + }, + { + "en": "who is paul of tarsus?", + "es": "¿Quién es Paul de Tarso?" + }, + { + "en": "who founded microsoft?", + "es": "¿Quién fundó microsoft?" + }, + { + "en": "who is the founder of microsoft?", + "es": "¿Quién es el fundador de microsoft?" + }, + { + "en": "when was the eiffel tower built?", + "es": "¿Cuándo se construyó la torre eifel?" + }, + { + "en": "what year was the eiffel tower constructed?", + "es": "¿En qué año se construyó la torre Eiffel?" + }, + { + "en": "which side is the ground elevation highest throughout the span of the great wall of china, the north or south?", + "es": "¿De qué lado es la elevación del suelo más alta a lo largo de la gran pared de China, el norte o el sur?" + }, + { + "en": "whales are endangered from over hunting", + "es": "ballenas están en peligro de extinción por la caza excesiva" + }, + { + "en": "did napoleon really die on saint helena?", + "es": "¿Napoleón realmente murió en Santa Helena?" + }, + { + "en": "who is a benevolent dictator for life for mars?", + "es": "¿Quién es un dictador benevolente de por vida para Marte?" + }, + { + "en": "what year does our cold fusion breakthrough happen?", + "es": "¿En qué año se produce nuestra fusión fría?" + }, + { + "en": "who won the 2024 us presidential election?", + "es": "¿Quién ganó las elecciones presidenciales de 2024?" + }, + { + "en": "what is the latest version of the ipad pro?", + "es": "¿Cuál es la última versión del ipad pro?" + }, + { + "en": "has oceania always been at war with east asia?", + "es": "¿Oceania siempre ha estado en guerra con Asia oriental?" + }, + { + "en": "who said may the force be with you?", + "es": "¿Quién dijo que la fuerza podría estar contigo?" + }, + { + "en": "what does winter is coming mean?", + "es": "¿Qué significa el invierno?" + }, + { + "en": "what is the meaning of rosebud?", + "es": "¿Cuál es el significado de Rosebud?" + }, + { + "en": "who said to be or not to be that is the question?", + "es": "¿Quién dijo ser o no ser esa es la pregunta?" + }, + { + "en": "what does the cake is a lie reference?", + "es": "¿Qué es lo que el pastel es una referencia de mentira?" + }, + { + "en": "bi-polar is rare?", + "es": "¿El bipolar es raro?" + } +] diff --git a/bench/qa_questions_fr.txt b/bench/qa_questions_fr.txt new file mode 100644 index 0000000..9eb8a4c --- /dev/null +++ b/bench/qa_questions_fr.txt @@ -0,0 +1,157 @@ +# AUTO-GENERATED from qa_questions.txt via opus-mt-en-fr (#000056). +# Regenerate: python3 bench/make_lang_questions.py fr + +# arborist QA-quality benchmark question set. +# +# One question per line. `#`-prefixed lines and blank lines ignored. +# Designed for the 2003-05-16 Wikipedia cur snapshot — questions must +# resolve against ~2003 article content. Stress-tests across the +# failure-mode shapes the verifier needs to handle. + +# narrow factoid — well-anchored single-fact questions +Quelle est la capitale de la France ? +Qui a écrit GNU Linux ? +Quand un langage de programmation nommé Python a-t-il été créé ? +Qui a peint la mona lisa ? +un pont entre le nouveau London & Groton? +Qui a écrit le hameau de jeu ? +Quel est le symbole chimique de l'or ? + +# broad descriptive — encyclopedic shape, prone to mode-collapse +Parlez-moi du connecticut +Parlez-moi du langage de programmation C +Parle-moi de l'homme de méthode ? +Dis-moi tout ce qu'il y a à savoir sur york england ? +Parlez-moi de l'empire romain. +décrire la structure de l'ADN + +# under-specified "all" — the word "all" reads to Hermes-3-8B as +# license to enumerate every adjacent fact in training prior, which +# in claim_lattice_pointer mode degrades to free-form prose with zero +# `[E\d+]` tags (FORMAT_COLLAPSED soft-demote, 2026-05-02). Stronger +# models (Qwen / GPT-4 family) plausibly recover format discipline +# under the same prompt — bench coverage of this shape lets us +# measure cross-model resilience. Ticket #000008 (broad-quantifier +# preflight guard) proposes upstream classification + per-model +# claim ceiling. +Les gagnants de tous les grands sports ? + +# bounded universals — finite, corpus-known answer sets. Ticket +# #000008 §10.1 splits broad universals into bounded vs unbounded. +# These should classify ALL but with `scope_bound_hint: "bounded"`, +# meaning --reject-broad does NOT reject and the cap is the natural +# bound. Without these fixtures, the bounded-vs-unbounded distinction +# has no live bench coverage. +nommez tous les membres des Beatles +lister toutes les planètes du système solaire + +# entity list — invites lazy-anchor on a magnet chunk +Quels dinosaures étaient dans le premier film du parc jurassique ? +Qui sont les membres des Beatles ? +nom sipsons membres de la famille, y compris les animaux de compagnie? +liste de l'obélisque dans le connecticut +Quelles sont les planètes de notre système solaire ? +Qui étaient les sept astronautes du mercure d'origine ? + +# relationship / multi-fact +Qui est la petite amie des supermans ? +Qui est le neveu de Bilbo Baggins ? +Quelle est la relation entre linux & unix ? +qui est veronica ballestrini et quel mois est-elle née ? +Qui est le père et la sœur de Luke Skywalker ? + +# comparison — multi-entity, prone to attribution drift +Quelle est la différence entre linux et bsd ? +Comment l'intelligence se compare-t-elle à l'amd ? +Quelle est la différence entre http et ftp? +mac vs windows pour le développement de logiciels + +# niche / partial — corpus may be thin +Qu'est-ce que la constante de Boltzmann ? +Qui a inventé l'effet doppler ? + +# date / time — when-questions stress year/date grounding +Quand la Seconde Guerre mondiale a-t-elle pris fin ? +Quelle année le mur de Berlin est-il tombé ? +Quand est-ce que la première lune a atterri ? +Quand l'union soviétique s'est-elle dissoute ? +En quelle année la magna carta a-t-elle été signée ? + +# quantity — numeric grounding, prone to confabulated digits +Combien d'États sont dans les États-Unis? +Combien d'os sont dans le corps humain adulte ? +Quelle est la population du Japon ? +Combien de femmes henry la huitième avait-elle ? +Combien de lunes Jupiter a-t-il ? + +# geographic — where / what country / location grounding +Où se trouve le mont Kilimanjaro ? +Dans quel pays est la ville de la prague ? +Dans quel océan est Madagascar ? +Où commence la rivière nile ? +Sur quel continent s'agit - il? + +# cause / effect — why-questions, narrative grounding +Pourquoi le titanic a coulé ? +Qu'est-ce qui a causé le désastre du chernobyl ? +Pourquoi les dinosaures ont disparu ? +Qu'est-ce qui a déclenché la Première Guerre mondiale ? + +# entity disambiguation — common names that collide with many entities +Qui est Michael Jordan ? +Qui est George Bush ? +Qui est John Smith de Jamestown ? +Qu'est-ce que la matrice ? +Qui est paul de tarse ? + +# synonym / paraphrase robustness — same fact, different phrasing. +# Pairs probe whether equivalence_class question dedup collapses these +# at write & whether retrieval grounds them identically. +Qui a fondé le microsoft ? +Qui est le fondateur du microsoft ? +Quand la tour Eiffel a-t-elle été construite ? +Quelle année la tour Eiffel a-t-elle été construite ? + +# leading / forensic — probe whether retrieval surfaces the right +# fact when the question's premise contradicts the popular narrative. +# Elevation question expected answer: north (per geographic surveys); +# corpus grounding may be shaky — coverage check can pass on +# topical-token overlap without the cited span containing the +# elevation fact, so this also stress-tests lazy-anchor STRICT. +Quel côté est l'altitude du sol la plus élevée sur toute la étendue du grand mur de Chine, au nord ou au sud? +Les baleines sont menacées par la chasse. +Le napoléon est-il vraiment mort sur saint Hélène ? + +# out-of-corpus — should land UNGROUNDED honestly +Qui est un dictateur bienveillant à vie pour Mars ? +Quelle est l'année de notre percée dans la fusion froide? +Qui a gagné l'élection présidentielle de 2024 ? +Quelle est la dernière version de l'ipad pro ? + +# allusion / reference frame — the diagnostic signal is a verbatim +# multi-token sequence, not the individual content tokens. Pure-BM25 +# on tokens will surface literal-frame articles (Oceania the region, +# Asia the continent) instead of the reference (Nineteen Eighty-Four). +# The phrase-pattern retrieval route (commit 2026-05-01) targets this +# class. Bench reveals prevalence of the failure mode + whether the +# fix generalizes beyond Orwell. +oceania a toujours été en guerre avec l'Asie de l'Est ? +Qui a dit que la force pourrait être avec vous ? +Qu'est-ce que l'hiver va signifier ? +Quelle est la signification de rosebud? +Qui a dit qu'il s'agissait ou non de cette question? +Qu'est-ce que le gâteau est une référence mensongère? + +# hyphenation / orthography — query and corpus use different hyphen +# conventions for the same concept. FTS5 `unicode61` splits on hyphen +# at index AND query time, so `bi-polar` (the orthography many +# English readers actually use) tokenizes to [bi, polar] while the +# Wikipedia article `Bipolar disorder` tokenizes to [bipolar]. Two +# non-overlapping token sets for the same medical condition. The +# query-layer hyphen-fold (Ticket #000007, 2026-05-02) closes this +# class by emitting joined-no-hyphen variants on the query side and +# adding a hyphen-anchor accept path in `_filter_by_title_relevance`. +# Pre-fix: only Bi-Polar (album) / Bi-Polar Blues / BI disambiguation +# surfaced; the medical-condition cluster never reached the LLM. +# Post-fix: `Bipolar disorder` lands in top-K and grounds the answer. +Le bipolaire est rare ? diff --git a/bench/qa_questions_fr_map.json b/bench/qa_questions_fr_map.json new file mode 100644 index 0000000..02223c3 --- /dev/null +++ b/bench/qa_questions_fr_map.json @@ -0,0 +1,302 @@ +[ + { + "en": "what is the capital of france?", + "fr": "Quelle est la capitale de la France ?" + }, + { + "en": "who wrote GNU linux?", + "fr": "Qui a écrit GNU Linux ?" + }, + { + "en": "when was a programming language named Python created?", + "fr": "Quand un langage de programmation nommé Python a-t-il été créé ?" + }, + { + "en": "who painted the mona lisa?", + "fr": "Qui a peint la mona lisa ?" + }, + { + "en": "a bridge between new london & groton?", + "fr": "un pont entre le nouveau London & Groton?" + }, + { + "en": "who wrote the play hamlet?", + "fr": "Qui a écrit le hameau de jeu ?" + }, + { + "en": "what is the chemical symbol for gold?", + "fr": "Quel est le symbole chimique de l'or ?" + }, + { + "en": "tell me about connecticut", + "fr": "Parlez-moi du connecticut" + }, + { + "en": "tell me about the C programming language", + "fr": "Parlez-moi du langage de programmation C" + }, + { + "en": "tell me about method man?", + "fr": "Parle-moi de l'homme de méthode ?" + }, + { + "en": "tell me all there is to know about york england?", + "fr": "Dis-moi tout ce qu'il y a à savoir sur york england ?" + }, + { + "en": "tell me about the roman empire", + "fr": "Parlez-moi de l'empire romain." + }, + { + "en": "describe the structure of DNA", + "fr": "décrire la structure de l'ADN" + }, + { + "en": "winners of all major sports?", + "fr": "Les gagnants de tous les grands sports ?" + }, + { + "en": "name all members of the beatles", + "fr": "nommez tous les membres des Beatles" + }, + { + "en": "list all planets in the solar system", + "fr": "lister toutes les planètes du système solaire" + }, + { + "en": "what dinosaurs were in the first jurassic park film?", + "fr": "Quels dinosaures étaient dans le premier film du parc jurassique ?" + }, + { + "en": "who are the members of the beatles?", + "fr": "Qui sont les membres des Beatles ?" + }, + { + "en": "name simpsons family members including pets?", + "fr": "nom sipsons membres de la famille, y compris les animaux de compagnie?" + }, + { + "en": "list of obelisk in connecticut", + "fr": "liste de l'obélisque dans le connecticut" + }, + { + "en": "what are the planets of our solar system?", + "fr": "Quelles sont les planètes de notre système solaire ?" + }, + { + "en": "who were the original seven mercury astronauts?", + "fr": "Qui étaient les sept astronautes du mercure d'origine ?" + }, + { + "en": "who is supermans girlfriend?", + "fr": "Qui est la petite amie des supermans ?" + }, + { + "en": "who is bilbo baggins's nephew?", + "fr": "Qui est le neveu de Bilbo Baggins ?" + }, + { + "en": "what is the relationship between linux & unix?", + "fr": "Quelle est la relation entre linux & unix ?" + }, + { + "en": "who is veronica ballestrini & what month was she born?", + "fr": "qui est veronica ballestrini et quel mois est-elle née ?" + }, + { + "en": "who is luke skywalker's father and sister?", + "fr": "Qui est le père et la sœur de Luke Skywalker ?" + }, + { + "en": "what's the difference between linux and bsd?", + "fr": "Quelle est la différence entre linux et bsd ?" + }, + { + "en": "how does intel compare to amd?", + "fr": "Comment l'intelligence se compare-t-elle à l'amd ?" + }, + { + "en": "what is the difference between http and ftp?", + "fr": "Quelle est la différence entre http et ftp?" + }, + { + "en": "mac vs windows for software development", + "fr": "mac vs windows pour le développement de logiciels" + }, + { + "en": "what is the boltzmann constant?", + "fr": "Qu'est-ce que la constante de Boltzmann ?" + }, + { + "en": "who invented the doppler effect?", + "fr": "Qui a inventé l'effet doppler ?" + }, + { + "en": "when did world war 2 end?", + "fr": "Quand la Seconde Guerre mondiale a-t-elle pris fin ?" + }, + { + "en": "what year did the berlin wall fall?", + "fr": "Quelle année le mur de Berlin est-il tombé ?" + }, + { + "en": "when was the first moon landing?", + "fr": "Quand est-ce que la première lune a atterri ?" + }, + { + "en": "when did the soviet union dissolve?", + "fr": "Quand l'union soviétique s'est-elle dissoute ?" + }, + { + "en": "in what year was the magna carta signed?", + "fr": "En quelle année la magna carta a-t-elle été signée ?" + }, + { + "en": "how many states are in the united states?", + "fr": "Combien d'États sont dans les États-Unis?" + }, + { + "en": "how many bones are in the adult human body?", + "fr": "Combien d'os sont dans le corps humain adulte ?" + }, + { + "en": "what is the population of japan?", + "fr": "Quelle est la population du Japon ?" + }, + { + "en": "how many wives did henry the eighth have?", + "fr": "Combien de femmes henry la huitième avait-elle ?" + }, + { + "en": "how many moons does jupiter have?", + "fr": "Combien de lunes Jupiter a-t-il ?" + }, + { + "en": "where is mount kilimanjaro located?", + "fr": "Où se trouve le mont Kilimanjaro ?" + }, + { + "en": "what country is the city of prague in?", + "fr": "Dans quel pays est la ville de la prague ?" + }, + { + "en": "in which ocean is madagascar?", + "fr": "Dans quel océan est Madagascar ?" + }, + { + "en": "where does the nile river begin?", + "fr": "Où commence la rivière nile ?" + }, + { + "en": "what continent is egypt on?", + "fr": "Sur quel continent s'agit - il?" + }, + { + "en": "why did the titanic sink?", + "fr": "Pourquoi le titanic a coulé ?" + }, + { + "en": "what caused the chernobyl disaster?", + "fr": "Qu'est-ce qui a causé le désastre du chernobyl ?" + }, + { + "en": "why did the dinosaurs go extinct?", + "fr": "Pourquoi les dinosaures ont disparu ?" + }, + { + "en": "what triggered world war 1?", + "fr": "Qu'est-ce qui a déclenché la Première Guerre mondiale ?" + }, + { + "en": "who is michael jordan?", + "fr": "Qui est Michael Jordan ?" + }, + { + "en": "who is george bush?", + "fr": "Qui est George Bush ?" + }, + { + "en": "who is john smith of jamestown?", + "fr": "Qui est John Smith de Jamestown ?" + }, + { + "en": "what is the matrix?", + "fr": "Qu'est-ce que la matrice ?" + }, + { + "en": "who is paul of tarsus?", + "fr": "Qui est paul de tarse ?" + }, + { + "en": "who founded microsoft?", + "fr": "Qui a fondé le microsoft ?" + }, + { + "en": "who is the founder of microsoft?", + "fr": "Qui est le fondateur du microsoft ?" + }, + { + "en": "when was the eiffel tower built?", + "fr": "Quand la tour Eiffel a-t-elle été construite ?" + }, + { + "en": "what year was the eiffel tower constructed?", + "fr": "Quelle année la tour Eiffel a-t-elle été construite ?" + }, + { + "en": "which side is the ground elevation highest throughout the span of the great wall of china, the north or south?", + "fr": "Quel côté est l'altitude du sol la plus élevée sur toute la étendue du grand mur de Chine, au nord ou au sud?" + }, + { + "en": "whales are endangered from over hunting", + "fr": "Les baleines sont menacées par la chasse." + }, + { + "en": "did napoleon really die on saint helena?", + "fr": "Le napoléon est-il vraiment mort sur saint Hélène ?" + }, + { + "en": "who is a benevolent dictator for life for mars?", + "fr": "Qui est un dictateur bienveillant à vie pour Mars ?" + }, + { + "en": "what year does our cold fusion breakthrough happen?", + "fr": "Quelle est l'année de notre percée dans la fusion froide?" + }, + { + "en": "who won the 2024 us presidential election?", + "fr": "Qui a gagné l'élection présidentielle de 2024 ?" + }, + { + "en": "what is the latest version of the ipad pro?", + "fr": "Quelle est la dernière version de l'ipad pro ?" + }, + { + "en": "has oceania always been at war with east asia?", + "fr": "oceania a toujours été en guerre avec l'Asie de l'Est ?" + }, + { + "en": "who said may the force be with you?", + "fr": "Qui a dit que la force pourrait être avec vous ?" + }, + { + "en": "what does winter is coming mean?", + "fr": "Qu'est-ce que l'hiver va signifier ?" + }, + { + "en": "what is the meaning of rosebud?", + "fr": "Quelle est la signification de rosebud?" + }, + { + "en": "who said to be or not to be that is the question?", + "fr": "Qui a dit qu'il s'agissait ou non de cette question?" + }, + { + "en": "what does the cake is a lie reference?", + "fr": "Qu'est-ce que le gâteau est une référence mensongère?" + }, + { + "en": "bi-polar is rare?", + "fr": "Le bipolaire est rare ?" + } +] diff --git a/docs/TICKETS.md b/docs/TICKETS.md index 94a13a8..207975f 100644 --- a/docs/TICKETS.md +++ b/docs/TICKETS.md @@ -111,6 +111,8 @@ Newest first. Update on every open/close. | ID | Title | Status | Opened | Directive | |----------|------------------------------------------------|-----------------------|------------|-----------| +| #000056 | Operation Sandwich — cross-language grounding via query+display MT | **implemented & landed 2026-05-17 · default-OFF** (fox: "call it operation sandwich, create a new ticket and finish it"). Mechanism + bright line **verified live end-to-end** (real Hermes + real opus-mt: es query → English answer+verifier → es display; `answer_text` English, `display_answer` Spanish additive, `question_hash`/`verifier_policy_hash` invariant; 6 tests + full suite 2477 passed, 0 regressions). **Fan-out measured (§9, n=1, 75 q):** EN baseline 85% → es+sandwich 71% = **−14pp cost**; transitions PRESERVED 31 / DOWNGRADE 16 / LOST 17 / N/A 11. A deterministic round-trip predictor was tried and **refuted** (12/17 LOST round-tripped CLEAN; another instance of the codified CLAUDE.md bench-maxing lesson — not re-added). LOST taxonomy from the artifact: ≈7 entity-translate (`Boltzmann`→"perntzmann", `Tarsus`→"Tarso"), ≈3 broad-enum, ≈several n=1 noise. **Lever built+validated:** `arborist/qa/mt/entity_mask.py` mask/restore (real opus-mt: `who is Paul of Tarsus?` "Pablo de Tarso"→**"Paul of Tarsus"**); default-ON within the default-OFF sandwich. Caveat: bench is lowercased so cap-detector lift is a **lower bound** (corpus-title anchor = v2). Also: fan-out caught + fixed an 88%-engine-error concurrency defect (per-call model load → memoised singleton + lazy per-pair); French + Russian breads added (manifest, `crosslang_source_lang`). **Lift measured 2026-05-17 (comparator corrected, fox):** the true baseline is the pre-ticket ≈0% (raw es query → song-title noise, UNGROUNDED, 10.4s) — NOT native English. Against that: **the sandwich is a large net win (≈0% → 71% es grounded, proof core never corrupted); the −14pp vs English is the cost of a new capability, not a regression — calling it a "fail" was a comparator error.** The genuine negative is the **entity-mask lever**: net-negative at scale (es 71→65, fr+mask 47/−38pp; isolated Paul-of-Tarsus win didn't replicate — 3rd bench-maxing-lesson instance), now **default-OFF** (`crosslang_entity_mask=False`); no-mask sandwich is the keeper. Recommendation flipped: **worth continuing (minus the mask)**, not park. Remaining: n=3, fr no-mask, corpus-title anchoring (only untried lowercase-capable detector). CLAUDE.md updated with the durable cross-lang *convention* only. Tasks #14–#17. Phase 1 of the #000001 §7 family; new ticket clears don't-proliferate (fox-directed + distinct Dav1d audience + architectural inflection: a model dependency `[mt]` + a presentation-translation layer — anticipated by #000001 §7's "split the `[mt]` model-distribution work like `[nli]`/vecpack"). **Sandwich:** translate query es→en (retrieval-side, == `--retrieval-keywords`, binds into `retrieval_plan_hash`, NOT `question_hash`) → English answer through the **byte-for-byte untouched verifier** → translate the verified English `answer_text` en→es into a NEW `display_answer` field, banner-labelled, zero grounding (the `_render_audit_label` render-projection pattern). Engine: local `[mt]` extra, Helsinki-NLP `opus-mt-es-en`/`-en-es`, Apache-2.0, hash-pinned, off-repo `~/.arborist/models/mt/`, optional dep, graceful-degrade — mirrors `[nli]`/`ShadowNLI` (#000049) + vecpack (#000051) verbatim; never Hermes-3-8B; not an external API (reproducibility + zero egress + es↔en is the best-resourced pair). Default OFF (`crosslang_translate_enabled`, gated under Phase-0 `crosslang_guard_enabled`); `--crosslang-translate` / `XLANG_MT=1`. Hash invariants (corrected 2026-05-17 — `governance_policy_hash` is sha256 of the *whole* policy, keys.py:182): `question_hash` + `verifier_policy_hash` untouched (user question preserved, verifier byte-identical); `governance_policy_hash` moves like every policy flag → correct cache partitioning by config (not a leak); MT engine identity binds into `RetrievalPlan.mt_*` (run-DAG), not a policy hash. | 2026-05-17 | — | +| #000055 | Windows quickstart without `make` (`tasks.py` + `make.bat`) | **in progress** — opened 2026-05-16 (fox: "bat files or some shit … avoid needing makefile for windows … we will test the quickstart on windows"). Pure-stdlib `tasks.py` runner mirroring the **quickstart subset** of the Makefile (bootstrap / fetch-cur / ingest-cur-attached / distill ×2 / query / inspect / falsify / burn / bootstrap-crawler / crawl-ingest / stats / verify / search / clean) + a ~10-line `make.bat` shim so `make ` works in Windows cmd and `.\make.bat ` in PowerShell. Audited the artifact (not the docs): the 2003 dump is opened via stdlib `bz2` (no external `bzip2`); only `fetch-cur` used `curl` (→ stdlib `urllib`); bash `for…&wait` → `subprocess.Popen` fan-out; `arborist` console-script lands at `.venv\Scripts\arborist.exe`. Net: quickstart needs only **Python 3.10+ + sqlite3** — the repo's existing ethos, now true on native Windows. Same `KEY=VALUE` make-style args so documented commands translate 1:1 (one doc form). Makefile untouched, still canonical on POSIX ("keep it as an option"). Found + fixed a README/Makefile discrepancy: README claimed `[dev,html]` bootstrap extras, Makefile installs `.[dev]` — artifact wins. Drift-pinned by `tests/test_tasks_runner.py`. | 2026-05-16 | — | | #000054 | Acronym-parens concept extractor (closes the abbreviation→expansion retrieval gap) | **in progress** — Phase 1 (extractor + 481K edges) landed `58027e9`; Phase 2 (consumer-side surfacing — `synonym_expand` rank-and-truncate over the per-token cap, FTS5-`bm25` ordering in `_search_titles`, expanded `accept_tokens` in title-search + core-keyword + title-rerank, `synonym_expand_strict()` for the multiplicative title-purity rerank to exclude noisy `link_reciprocity` edges, tightened extractor regex to `[A-Z]{3,6}` purging 2-letter homonym edges) landed `ce855db`. **End-to-end verified:** `what is a CPU?` → Central processing unit at #1; `what is a GPU?` → Graphics processing unit at #1 EVIDENCE-WARRANTED 1/1; Mount Kilimanjaro / Soviet Union queries unchanged. **bench-qa n=3 limit=5** (2026-05-13T14:24Z): 30/45 STRICT (67%), zero regressions on basics (mona lisa / capital of france / new london bridge each 9/9 STRICT). 2026-05-13 — `arborist/concepts/extract.py:acronym_parens_synonym` lands as a new corpus-agnostic extractor in `EXTRACTORS` (`evidence_kind="acronym_parens"`). Scans each doc's lead chunk for ` (ACRO)` where the all-caps acronym's letters match the content-word initials of the phrase in order; emits bidirectional synonym edges between the lowercased acronym and each ≥3-char content token of the phrase. Conservative (strict 1:1 initials, function words filtered, repeated definitions deduped per doc). Closes the *retrieval-side* abbreviation gap (`CPU↔central processing unit`, `GPU↔graphics processing unit`, `RAM↔random access memory`, `FBI↔federal bureau of investigation`, `WHO↔world health organization`, …) that `link_reciprocity_synonym` can't reach because the relation lives in body text, not the wiki link graph (Wikipedia represents abbreviation→expansion as a *redirect* — not an edge). Per-shard like all `concept_relations` data; corpus-agnostic so HTML/blogs/textbooks benefit equally. Retrieval-side only — never proof-path. 8 new tests; full suite green. Closes #000050 §2a's CPU/GPU fixture rows *upstream* of vec; the Orwell-shape conceptual-allusion row remains the genuine #000050 justification. Operational follow-up (not code): `arborist concepts derive --extractor acronym_parens` on each shard. | 2026-05-13 | — | | #000053 | Acronym-aware verifier content tokens | **closed · 2026-05-13** — `arborist.qa.evidence._content_tokens` now keeps all-caps 2-3-char acronyms (CPU/GPU/DNA/FBI/USB…) as content tokens instead of dropping every <4-char token; fixes the field case where "what is a CPU?" cited to "CPU design" tripped `TITLE_MISMATCH` spuriously (claim & title share "CPU" but neither registered) — also affects `SUBJECT_TOKENS_ABSENT` (Rule 9), `BARE_NAME_CLAIM`, spotlight-excerpt token pick. Versioned: `content_token_rules: "v2-acronym-aware"` in both default policies + `_VERIFIER_POLICY_FIELDS` → folds into `verifier_policy_hash`, prior cache records orphan on lookup (by design, same discipline as `base_version` / `hyphen_fold_v1`). Monotone toward *fewer* spurious demotes (only relaxes overlap checks, never tightens). 8 new tests; full suite green; `bench-qa-smoke` clean. Does NOT fix the *retrieval* abbreviation→expansion gap (`CPU`→`Central processing unit` = #000050 vec hybrid / `concepts/` synonym edges — the root cause of the satellite-article retrieval). | 2026-05-13 | — | | #000052 | Relevance + coherence meta-cognition (answer-*shape* signals) | in progress — **§3.1 `diagnose_coherence` landed** (lexical, no model: `circular` / `phrase_component_reuse` / `vacuous`; in `arborist/qa/inspect.py`, surfaced via `inspect_cache_key` + `arborist inspect` `· incoherent: `; 9 tests; demote-policy hook deliberately not wired — advisory only). Joins the `diagnose_deflection` / `diagnose_metaphor_deflection` / `diagnose_title_relevance` / soft-preflight family of read-only, demote-only, never-in-proof-path sidecars; `phrase_component_reuse` catches the motivating field case (a subject quoting a phrase, a predicate reusing one of that phrase's own tokens as a bare `the ` referent). **Still open: (2) `diagnose_relevance`** — semantic (not just lexical) "aboutness": does the answer address the question; is each claim about its cited source? Today's checks (subject-anchor token overlap, stemmed title-stem overlap) are *lexical* and a token collision defeats them — a small *aboutness/reranker* model (NOT NLI — entailment ≠ topicality) under #000049 §7's discipline cage verbatim (demotion-only, hash-pinned, `relevance_model_version`→`governance_policy_hash` iff it touches `audit_mode`, shadow-first, `[…]` extra, the §7 #20 haystack lesson — never over the whole context); gated on evidence, travels with #000049's model question. Motivating field case (2026-05-12, fox): the `claim_lattice` query that returned *"the phrase 'Zionist entity' is sometimes used as the entity, referring to the State of Israel"* at `EVIDENCE-WARRANTED-PARTIAL 2/3` — incoherent + token-collision recombination that NLI can't catch (returns *neutral*, not *contradiction*) and both lexical relevance checks waved through. Flags an upstream retrieval ticket (polysemy / title-token-soup) as the root-cause fix, not scoped here. #000049 sibling | 2026-05-12 | — | @@ -164,8 +166,8 @@ Newest first. Update on every open/close. | #000004 | Directive coverage in bench summary | closed · `acd1f9c` | 2026-05-01 | D8 | | #000003 | Anchor-class warrant generalization (Module H+)| closed · 2026-05-02 | 2026-05-01 | D6 | | #000002 | Reference-Frame Polarity Contract (Module L) | closed · 2026-05-02 | 2026-05-01 | D3 | -| #000001 | Retrieval-keywords audit gap | closed · 2026-05-02 | 2026-05-01 | D4 | +| #000001 | Retrieval-keywords audit gap (+ §7 cross-language transforms) | **reopened 2026-05-17 (in progress)** — keyword scope stays closed/landed 2026-05-02 (run-DAG `RetrievalPlan` binding, §5). §7 extends the *same* retrieval-transform-provenance substrate to a sibling: cross-language query bridging. Strict bright line — MT/aliases/stopword-guards *propose* candidates, only source-language evidence *warrants*; English answer verified English-vs-English by the **untouched binary verifier**, Spanish is display-only (zero grounding, banner-labelled). Held strictly this needs **no new `audit_mode` token, no `EVIDENCE-WARRANTED-CANONICAL`, no `surface_language` field, no `providence_cache` column, no new run-DAG stage** (the >10%-back deletion; CLAUDE.md schema-column-unchanged + verifier-stays-binary). **Phase 0 landed 2026-05-17** — `arborist/qa/crosslang.py` (deterministic, no model: `¿`/`¡`/non-ASCII signal, es-v1 stoppack); pre-preflight fail-closed mirroring `quantifier_should_reject` (Merkle-auditable reject DAG, `CROSS_LANGUAGE_UNSUPPORTED`, no LLM) + es-stoppack strip on `retrieval_query` only. Measured: `¿Qué es el anarcocapitalismo?` **10.4 s → 1.6 s** (~6.4×, still honest UNGROUNDED); English control byte-identical (by construction — `guard()` returns None for pure-ASCII). 19 tests; full suite 2470 passed, 0 regressions (incl. #000053/#000054); `bench-qa-smoke` stable anchor 3/3 STRICT. No schema change, no new governance/verifier *fields*, no `RetrievalPlan` change; `question_hash` + `verifier_policy_hash` untouched (the enabling flag moves `governance_policy_hash` like every policy flag — whole-policy hash, correct cache partition; corrected 2026-05-17, authoritative in #000056 §2 #6). `RetrievalPlan` MT extension landed in #000056 (Phase 1). **Feature-flagged default-OFF** (fox request): `policy["crosslang_guard_enabled"]` gates both seams via one point → flag-off reverts byte-for-byte to legacy (clean A/B); surface `arborist query --crosslang-guard` / `make query XLANG=1` / `tasks.py XLANG=1`. Default-flip is a separate bench-gated fox decision (rollout discipline matches #000008/#000011/#000049). 20 tests; full suite 2471 passed. **Collision constraint:** the guard is a specific source-language function-word stoppack, NOT a len≤2 heuristic — a blanket short-token drop regresses #000053/#000054 (`AI`/`ML`/`CPU`/`GPU` are load-bearing); `_FTS5_STOPWORDS`/`_TITLE_STOPWORDS` stay in sync; n=3 English bench, no >5pp regression. **Phase 1 split out → #000056 "Operation Sandwich"** (fox-directed; MT provider decided = local `[mt]` opus-mt, hash-pinned, not Hermes/not-API). Deleted by five-step (recorded in §7.6): raw-Spanish route (measured noise vs `en`), alias-map substrate (hand-curated worse-MT), cross-modal/multilingual-vec/SQD-language/ABCDEFG-5S (no named defect — the retired `arborist/v7/` anti-pattern). | 2026-05-01 | D4 | ## Next ID -`000055` +`000057` diff --git a/docs/tickets/ticket-000001-retrieval-keywords-audit-gap.md b/docs/tickets/ticket-000001-retrieval-keywords-audit-gap.md index 940c31c..878eff7 100644 --- a/docs/tickets/ticket-000001-retrieval-keywords-audit-gap.md +++ b/docs/tickets/ticket-000001-retrieval-keywords-audit-gap.md @@ -1,15 +1,21 @@ # Ticket #000001 — Retrieval-keywords audit gap -**Status:** closed · landed 2026-05-02 (run-DAG binding scope; SQL -column + audit-events scope deferred per §6 below) +**Status:** reopened 2026-05-17 (in progress) — keyword scope stays +closed/landed 2026-05-02 (run-DAG binding, §5; SQL column + audit-events +deferred per §6). §7 extends the *same retrieval-transform-provenance +substrate* (`arborist/qa/retrieval_plan.py`) to a sibling transform: +cross-language query bridging. Doc-only; gated on the MT-provider +decision (§7.5). **Opened:** 2026-05-01 -**Closed:** 2026-05-02 -**Scope:** Design proposal for capturing the `--retrieval-keywords` operator -hint in the v9.8 audit chain so retrieval is fully reproducible from a -providence record alone. Doc-only — no code in this commit. +**Closed:** 2026-05-02 (keyword scope) · **Reopened:** 2026-05-17 (§7) +**Scope:** Capturing retrieval-side transforms (operator keywords §1–§6; +cross-language bridging §7) in the v9.8 audit chain so retrieval is +reproducible from a providence record alone. Doc-only. **Audience:** fox + future blackops shifts. -**Hard constraint:** keywords stay operator-metadata, not part of the user's -question. Cache-key dimensionality stays at 8. +**Hard constraint:** retrieval transforms (keywords, MT, stopword +guards) stay operator/system metadata, never part of the user's +question. Cache-key dimensionality stays at 8. The verifier stays +binary; no new `audit_mode` token, no `providence_cache` column. --- @@ -288,6 +294,232 @@ in directly) stays in the original proposal as future work. --- +## 7. Phase X — Cross-language retrieval transforms (reopened 2026-05-17) + +**Why here, not a new ticket.** MT-to-corpus-language and a +multilingual stopword guard are *retrieval transforms* — the same +family as `--retrieval-keywords`. §5 already shipped the substrate: +`RetrievalPlan` + `retrieval_plan_hash`, run-DAG-bound, outside +`question_hash`/`cache_key`. A cross-language transform is captured by +*extending the `RetrievalPlan` fields*, not by inventing a new +provenance mechanism. Per the don't-proliferate discipline this is +§000001 Phase X, not #0000XX. (The 2026-05-16 field case: +`¿Qué es el anarcocapitalismo?` over the en-2003 corpus burned 10.4 s +assembling title-collision noise — *Así Es El Amor*, *¿Quién es el +Jefe?* — then correctly returned UNGROUNDED. The fail-closed was +right; the 10.4 s waste and the missing recall bridge are the gap.) + +### 7.1 Hard constraints (the bright line — strict form deletes a design layer) + +```text +MT / aliases / stopword guards → may PROPOSE retrieval candidates +source-language evidence → the ONLY thing that WARRANTS +``` + +Valid: Spanish query → MT → **English** retrieval → **English** answer +→ **existing untouched binary verifier** (English-vs-English) → +existing label, unchanged. Spanish text, if rendered, is **display +only**, carries zero grounding, and is banner-labelled +"machine-translated, not verified" — the `_render_audit_label` +pattern (render projection ≠ proof). + +Holding this strictly is the >10%-back deletion: there is **no new +`audit_mode` token, no `EVIDENCE-WARRANTED-CANONICAL`, no +`surface_language` proof field, no `providence_cache` column, no new +run-DAG stage.** CLAUDE.md: *schema column stays unchanged; verifier +stays binary; audit_mode decided by the verifier, never asserted.* + +Forbidden (→ the #000049 model-in-proof-path cage, not here): +Spanish answer → translate-to-English → verify → call it grounded. + +### 7.2 Collision constraint (must not regress shipped work) + +A noise guard **must be a specific source-language function-word +stoppack** (es v1: `qué que es el la los las un una de del en y o a`), +**not** a length / "drop len≤2" heuristic. A blanket short-token drop +regresses #000053 (acronym-aware verifier content tokens) and #000054 +(acronym_parens edges) — `AI`/`ML`/`CPU`/`GPU`/`DNA`/`FBI` are +load-bearing 2–3-char tokens. `_FTS5_STOPWORDS` and `_TITLE_STOPWORDS` +**stay in sync** (CLAUDE.md retrieval-pipeline §9). Bench-gated: +`make bench-qa` n=3, English suite, no >5pp STRICT-rate regression +before anything lands. + +### 7.3 Phase 0 — grounded design (2026-05-17, real-code pass) + +Mapping `arborist/qa/query.py` + `arborist/search/fts5.py` corrected +four points in the draft. The refinements make Phase 0 *smaller*: + +1. **`_FTS5_TOKEN_RE = [A-Za-z][A-Za-z0-9]*` is ASCII-only.** `¿Qué` + tokenizes to `Qu`; accented words are truncated before the + stopword check. The es stoppack must therefore list ASCII-folded / + truncated forms: `qu que es el la los las un una de del en y o a`. +2. **The measured 9.9 s cost was the search, not context assembly** + (`search 9.91s` · `context 0.05s`). The cost was OR-mode FTS5 + driven by the es function words `es`/`el` (high-DF, full-corpus + union scan). So Phase 0's primary lever is **stripping the es + stoppack from the retrieval token set**, not the fail-closed — + the field case (`anarcocapitalismo` survives the stoppack) is a + *content token that misses the corpus*, not a no-content query; + stripping `es`/`el` makes the search fast (single rare token, no + OR scan) → honest `UNGROUNDED` in ~ms instead of 10.4 s. +3. **The es stoppack must be language-scoped, never global.** + `un`→UN, `la`→LA, `de`→De, `en`→EN collide with the #000053/#000054 + acronym/abbrev class. Gate it behind a deterministic + no-model **non-English signal**: the query contains inverted + punctuation (`¿`/`¡`) or a non-ASCII Latin letter. Pure-ASCII + English queries never trigger it → the English retrieval path is + **byte-identical by construction** (the bench is a no-op by proof, + not just by measurement). +4. **Phase 0 needs no `RetrievalPlan` change.** The explicit + fail-closed returns *before* `RetrievalPlan` is built (it reuses + the existing `build_reject_run_dag` 3-stage path, exactly like + `quantifier_should_reject`). The reason is recorded as a + `CROSS_LANGUAGE_UNSUPPORTED` violation in the reject run-DAG + (mirroring `BROAD_QUANTIFIER_REJECTED`). Extending `RetrievalPlan` + with `detected_language`/`dropped_tokens` is a **Phase 1** + concern (when MT retrieval actually runs and the plan is built). + +**Phase 0 behaviour, final:** at `query()` entry, if the non-English +signal (refinement 3) fires: strip the es/multilingual stoppack from +the retrieval token set for this call only (refinement 1); if **zero** +content tokens remain, short-circuit to `UNGROUNDED` before retrieval +via the reject-DAG path (refinement 4); otherwise run retrieval on the +surviving content tokens (fast — refinement 2) and let the existing +pipeline return its honest verdict. Adds **no new governance/verifier +fields** and **no `RetrievalPlan` change**; `question_hash` and +`verifier_policy_hash` untouched; English path byte-identical. +(Correction 2026-05-17: an earlier draft said "no governance change" +unqualified — false. `governance_policy_hash` is sha256 of the +*whole* policy (`keys.py:182`), so toggling the `crosslang_guard_enabled` +flag moves it like every policy flag. That is correct cache +partitioning, not a leak. Authoritative statement: #000056 §2 #6.) + +### 7.4 Phase 1 — MT-to-corpus-language as one soft route (gated) + +MT-translated query == `--retrieval-keywords` with an MT front-end: +folds into `retrieval_plan_hash` exactly like keywords (the §2.4 / +§A.5 decision verbatim — **A+B, not C**; translated query is +retrieval metadata, **never** `question_hash`). Governance: the MT +*engine identity* binds into `RetrievalPlan` (run-DAG), not a policy +hash; the *enabling flag* moves `governance_policy_hash` like every +policy flag (whole-policy hash — correct cache partition). **This +§7.4 sketch is superseded by #000056 (Operation Sandwich), which is +the authoritative Phase-1 design + the corrected hash invariants +(§2 #6).** Raw `es` and raw `en` +questions keep distinct `question_hash` (no cross-language dedup). + +### 7.5 The gate (fox decides — nothing in 7.4 lands until then) + +MT provider. **Not Hermes-3-8B** (blackops standing rule: 8B unfit +for human-language translation; cannibalizes coder-agent budget). +Fork: **Grok API** (flagged candidate; soft-route only; +provider/version/prompt into `governance_policy_hash`) **vs** a local +**`[mt]` extra** (NLLB/Marian, hash-pinned like `[nli]`, weights +off-repo under `~/.arborist/models/mt/`). Phase 1 stays +doc-only until chosen. + +### 7.6 Deleted by the five-step (named, with the reason) + +- **Raw-Spanish retrieval route** — measured pure noise vs an `en` + corpus (the song-title case). Revisit *only* when a Spanish/mixed + shard exists; not a v1 route. +- **Deterministic alias-map substrate** — a hand-curated, worse MT + with no named maintainer; strictly dominated by a Spanish corpus + or the MT route. Cut. +- **New labels / `EVIDENCE-WARRANTED-CANONICAL` / `surface_language` + proof field / two-level proof identity** — label + schema + proliferation the strict §7.1 line makes unnecessary. +- **Cross-modal carriers, multilingual vec, SQD language + canonicalizers, a new `language_bridge` run-DAG stage, 5S/ABCDEFG + suites** — no named person, no closed defect (five-step step 1). + Namespace-reservation now = the retired `arborist/v7/` anti-pattern. + +### 7.7 Acceptance criteria (Phase 0 testable now; Phase 1 post-gate) + +1a. Cross-language query whose tokens are **all** function words → + `UNGROUNDED` **before** retrieval/LLM via the reject-DAG path; + `CROSS_LANGUAGE_UNSUPPORTED` violation recorded. +1b. Cross-language query with a surviving (corpus-missing) content + token (the `anarcocapitalismo` field case) → es function words + stripped from the retrieval set → **no OR-mode full-corpus + scan** (search ≪ the measured 9.9 s) → honest `UNGROUNDED`. +2. Source-language function words do not drive FTS5 into + title-collision noise. +3. `make bench-qa` n=3 English suite: **no >5pp STRICT-rate + regression** (the #000053/#000054 guard). +4. MT route (when enabled) folds into `retrieval_plan_hash`, **not** + `question_hash`. +5. `governance_policy_hash` changes **iff** MT enabled AND cacheable; + unchanged otherwise. +6. **No** new `audit_mode` token; **no** `providence_cache` column; + verifier byte-identical. +7. Raw `es` and raw `en` of the same question keep distinct + `question_hash`. + +### 7.8 Status + +**Phase 0 landed 2026-05-17.** New module `arborist/qa/crosslang.py` +(deterministic, no model: `guard()` + `strip_for_retrieval()`, +es-v1 stoppack, non-English signal = `¿`/`¡`/non-ASCII letter). +Two seams in `arborist/qa/query.py`: a pre-preflight fail-closed +early-return mirroring `quantifier_should_reject` (reuses +`build_reject_run_dag` → 3-stage Merkle-auditable reject DAG, +`CROSS_LANGUAGE_UNSUPPORTED` violation, `status= +cross_language_unsupported`, no LLM), and an es-stoppack strip on +`retrieval_query` only (LLM / verifier / `question_hash` untouched). +No schema change, no new governance/verifier *fields*, no +`RetrievalPlan` change; `question_hash` + `verifier_policy_hash` +untouched. (The enabling flag moves `governance_policy_hash` like +every policy flag — whole-policy hash, correct cache partition; see +#000056 §2 #6 for the authoritative corrected statement.) + +Measured (live, `query-dry`): `¿Qué es el anarcocapitalismo?` +**10.4 s → 1.6 s** (search 9.91 s → 1.26 s, ~6.4×), still honest +UNGROUNDED; `¿Qué es el?` → `cross_language_unsupported` pre-retrieval; +English control (`What is anarcho-capitalism?`) byte-identical +(Anarcho-capitalism #1, unchanged). 19 new tests in +`tests/test_crosslang_guard.py`; full suite 2470 passed, 0 +regressions (incl. query/dag/claim_lattice/#000053/#000054); +`bench-qa-smoke` stable anchor (`mona lisa`) 3/3 STRICT. English +inertness is **by construction** (the signal regex cannot match a +pure-ASCII no-`¿`/`¡` query → `guard()` returns None → identical +path), so the bench is a proof, not just a measurement. + +**Feature-flagged (fox request, 2026-05-17): default OFF.** +`policy["crosslang_guard_enabled"]` (default `False`) gates BOTH +seams via a single point — flag off → `_xlang` is None → behaviour +reverts byte-for-byte to pre-§7, a clean A/B baseline. Surface: +`arborist query --crosslang-guard`, `make query XLANG=1`, +`tasks.py query XLANG=1`. Rollout discipline matches #000008 +(quantifier) / #000011 (soft-preflight) / #000049 (NLI): ships +default-OFF; the default-flip is a separate **bench-gated fox +decision**, not autonomous — even though Phase 0 is provably inert +on English and strictly improves the cross-language case (the +argument *for* a future flip, deferred to fox). Retrieval-side +only: the flag does NOT enter `question_hash` or +`verifier_policy_hash` (user question preserved, verifier +byte-identical). It DOES move `governance_policy_hash` like every +policy flag (whole-policy hash, `keys.py:182`) — correct cache +partitioning by guard state, not a leak; the fail-closed path writes +no cache row anyway. (Correction 2026-05-17; authoritative: #000056 +§2 #6.) Live +A/B verified: `¿Qué es el?` flag-OFF → legacy UNGROUNDED; +flag-ON → `cross_language_unsupported` pre-retrieval. +1 test +(`test_flag_off_reverts_to_legacy_behaviour`); 20 tests total; +full suite 2471 passed. + +**Phase 1 split out → #000056 "Operation Sandwich" (fox-directed +2026-05-17).** MT-provider decided: local `[mt]` extra, +`opus-mt-es-en`/`-en-es`, hash-pinned (not Hermes, not an API). The +query+display MT sandwich, the `RetrievalPlan` MT-field extension +(the §7.3-deferred item), and the `[mt]` model dependency all live +in #000056 — split per the don't-proliferate "distinct Dav1d +audience / architectural inflection" criterion (a model dependency + +a presentation layer), exactly the split this section anticipated. +#000001 §7 stays the Phase-0 home. + +--- + ## Appendix A — Architectural review (2026-05-01, Asia/Kuala_Lumpur) > Fox-supplied review expanding §2-§4 with axiomatic framing, diff --git a/docs/tickets/ticket-000055-windows-quickstart-no-make.md b/docs/tickets/ticket-000055-windows-quickstart-no-make.md new file mode 100644 index 0000000..5d49061 --- /dev/null +++ b/docs/tickets/ticket-000055-windows-quickstart-no-make.md @@ -0,0 +1,113 @@ +# Ticket #000055 — Windows quickstart without `make` (`tasks.py` + `make.bat`) + +**Status:** in progress — opened 2026-05-16, implementation in same commit. +**Opened:** 2026-05-16 +**Asked by:** fox ("we need bat files or some shit so we could avoid +needing makefile for windows (but keep it as an option) windows uses +py py3 for python3 etc we will test the quickstart on windows so fix +it up"). +**Scope:** One new pure-stdlib runner `tasks.py` at repo root + one +small `make.bat` Windows shim + README Windows/Quickstart rewrite + +one smoke/drift test. **No** change to `arborist/`, the schema, the +Makefile recipes, or any audit/proof path. The Makefile stays the +canonical Unix entry point and is left byte-for-byte unchanged ("keep +it as an option"). +**Audience:** fox + anyone running the quickstart on native Windows + +maintainers who touch the quickstart and must keep two thin entry +points in sync. + +--- + +## 1. The gap, from the field + +`README.md` Setup §Windows currently says native Windows is *not* +supported — WSL2 only — because the Makefile uses bash idioms +(`for i in $(seq …); do … & done; wait`, `sed`, backgrounding) and +GNU-make conditionals, and the docs assume `make`, `curl`, `bzip2`. + +fox wants the **quickstart** runnable on native Windows (cmd / +PowerShell), no `make`, no WSL. Windows ships the `py` launcher +(`py -3`), not `python3`. + +## 2. What actually blocks Windows (and what does not) + +Audited the quickstart targets against the artifact, not the docs: + +- **Decompression:** the 2003 dump is opened with Python's stdlib + `bz2` (`arborist/sources/wikipedia.py:288`). **No external `bzip2` + needed** — the README's "needs … `bzip2`" line is stale for the + quickstart path. +- **Download:** the only `curl` use in the quickstart is `fetch-cur`. + Replaceable with stdlib `urllib.request` → no `curl` dependency. +- **Sharding/distill loops:** bash `for … & wait`. Replaceable with + `subprocess.Popen` fan-out + wait. +- **CLI:** `arborist` is a console script (`arborist.cli:main`, + `pyproject.toml:120`). On a Windows venv it lands at + `.venv\Scripts\arborist.exe`; on POSIX `.venv/bin/arborist`. +- **Net:** after this, the quickstart needs only **Python 3.10+ and + sqlite3** (sqlite3 ships with CPython) — exactly the repo's + "fresh checkout needs only python3.12 + venv + sqlite3" ethos. + No `make`, no `bzip2`, no `curl`, no `bash`. + +## 3. Design — why one runner, not N `.bat` files + +Five-step algorithm, step 2 (delete the part): hand-porting ~20 bash +recipes into brittle per-target `.bat` scripts would (a) duplicate +logic 1:1 with the Makefile in a second fragile dialect, (b) +proliferate files, (c) drift fast (DRY-in-context). Instead: + +- **`tasks.py`** — one pure-stdlib runner implementing the quickstart + *subset only*. Cross-platform (also runs on POSIX as + `python3 tasks.py …`, but the Makefile remains the documented Unix + path). Detects venv layout (`Scripts` vs `bin`) and the Python + launcher (`py -3` on Windows, `python3` on POSIX; `ARBORIST_PYTHON` + overrides). Accepts the **same `KEY=VALUE` token style as `make`** + (`query Q="…" JSON=1`, `inspect KEY=… JSON=1`, `crawl-ingest + URL=… DEPTH=2`) so the documented commands translate 1:1 — one doc + form, both platforms. +- **`make.bat`** — a ~10-line dispatcher so `make ` keeps + working in Windows `cmd` (cmd searches the current dir for + `make.bat`) and `.\make.bat ` in PowerShell. It only + resolves a Python (`py -3`, fallback `python`) and forwards argv to + `tasks.py`. No logic in the `.bat`. + +This keeps the Windows surface to **two files, one of them Python**, +mirrors the existing "Makefile is build glue, logic is Python" +precedent, and gives Windows the exact `make`-style UX. + +`tasks.py` is the quickstart subset, not the full 1170-line Makefile. +Bench/π*/textbook/docs targets stay Makefile-only (Unix/WSL2). The +runner's `help` enumerates exactly the supported targets so the +boundary is self-documenting. + +## 4. Targets covered + +`bootstrap fetch-cur ingest-cur-attached distill-shards-parallel +distill-shards-tfidf-parallel query query-dry inspect falsify burn +burn-kindergarten bootstrap-crawler crawl-ingest recrawl-check stats +stats-shards verify search test help clean clean-db clean-data` — +the full README Quickstart (Wikipedia + crawl paths), "After the +answer", and Setup blocks. + +## 5. Discrepancy found (reported to fox) + +`README.md` Setup §Bootstrap claims bootstrap installs the +`[dev,html]` extras; `Makefile:52` installs `.[dev]` only. Artifact +wins (CLAUDE.md "artifact over instruction"): `tasks.py bootstrap` +installs `.[dev]` to match the Makefile. README corrected in the same +commit. + +## 6. Drift control + +`tests/test_tasks_runner.py`: importable on the host platform; +`help` lists exactly the documented quickstart targets (a removed or +renamed target fails the test); venv-layout + launcher detection +verified for both `os.name == "nt"` and `"posix"` via monkeypatch. +Two thin entry points are a known DRY cost — the test pins the target +set so divergence from the documented quickstart is loud. + +## 7. Out of scope + +Full Makefile parity on Windows; bench/π*/NLI/textbook/docs targets; +changing the Unix workflow (Makefile untouched, still canonical on +POSIX). diff --git a/docs/tickets/ticket-000056-operation-sandwich-crosslang-grounding.md b/docs/tickets/ticket-000056-operation-sandwich-crosslang-grounding.md new file mode 100644 index 0000000..ede7864 --- /dev/null +++ b/docs/tickets/ticket-000056-operation-sandwich-crosslang-grounding.md @@ -0,0 +1,337 @@ +# Ticket #000056 — Operation Sandwich (cross-language grounding via query+display MT) + +**Status:** implemented & landed 2026-05-17 — mechanism + bright +line verified live end-to-end (§8); 6 tests + full suite 2477 +passed. **Default OFF** (`crosslang_translate_enabled`, gated under +Phase-0 `crosslang_guard_enabled`). NOT "cross-language solved": +the live flagship run is still `UNGROUNDED` (opus-mt +output↔title-normalisation recall gap, §8 limit 1) — recall tuning + +JSON-display polish (§8 limit 2) + SHA-pinning the manifest are open +follow-ups behind the experiment flag; a bench-gated default-flip is +a fox decision. Phase 1 of the #000001 §7 family. +**Opened:** 2026-05-17 +**Asked by:** fox ("yes call it operation sandwich create a new +ticket and finish it") — closing the chain that started with the +2026-05-16 `¿Qué es el anarcocapitalismo?` field case. + +**Why a new ticket (clears the don't-proliferate bar).** CLAUDE.md +default is *extend*, split only when the piece needs a **distinct +Dav1d-reviewable audience** or is an **architectural inflection**. +Operation Sandwich is both: it introduces a **model dependency** +(`[mt]`) and a **presentation-translation layer** — a self-contained +design decision a de-novo review must read independently of #000001's +retrieval-keywords provenance story. #000001 §7 explicitly anticipated +this split ("If the MT model choice becomes large, split the +model-distribution work into a later `[mt]` ticket, similar to +`[nli]` and vecpack"). #000001 §7 stays the Phase-0 home; this is +Phase 1. Cross-linked both ways. + +**Audience:** fox + a Dav1d de-novo review of the +translation-in-the-pipeline boundary + maintainers of the verifier +proof path. + +--- + +## 1. The sandwich + +``` +Spanish query ─▶ [MT es→en] ─▶ English retrieval + English answer + EXISTING verifier ─▶ [MT en→es] ─▶ Spanish display + translator IN ←—— grounded core, byte-for-byte untouched ——→ translator OUT + (retrieval-side) (English-vs-English — the proof) (presentation-only) +``` + +Translation lives on the **two edges**, never the middle. The +verifier, `audit_mode`, `cache_key`, `question_hash`, +`governance_policy_hash` all see only the English core. The Spanish +the user reads is a labelled rendering carrying **zero** grounding. + +## 2. Hard constraints (the bright line, as code invariants) + +1. **`answer_text` stays the grounded English string.** The verifier + runs English-answer-vs-English-source, unchanged byte-for-byte. +2. **The Spanish rendering goes in a NEW field** (`display_answer`, + `display_lang`, `display_translated=True`, + `display_unverified_banner`). It is never the verifier's input, + never what `audit_mode`/STRICT attaches to, never hashed into the + proof. Same discipline as `cli._render_audit_label` (render + projection ≠ proof). +3. **Query MT is retrieval-side, == `--retrieval-keywords`.** It + changes which sources rank (→ `context_root` → `cache_key` + indirectly, exactly the #000001 §5/§6 story), **not** + `question_hash`. The MT engine identity binds into the **run-DAG + retrieval stage** via the existing `retrieval_plan_hash` (the + §7.3-deferred `RetrievalPlan` extension lands here). +4. **No translated answer ever re-enters the verifier** (the + forbidden path / #000049 model-in-proof-path cage). Display MT is + output-only and one-directional. +5. **Default OFF**, gated under `crosslang_guard_enabled`. The MT + engine is the optional `[mt]` extra; absent → degrade to + `available=False` → the sandwich silently no-ops (query proceeds + as Phase-0). English path byte-identical by construction. +6. **Hash invariants (corrected 2026-05-17 — artifact over my own + prose).** `governance_policy_hash` is `sha256` of the *whole* + policy dict (`arborist/qa/keys.py:182`), so — like **every** + policy flag (quantifier, metacognition, soft-preflight) — the + crosslang flags DO move it. An earlier draft of this section + wrongly said "governance untouched"; that was false and is + corrected here. The behaviour is *correct*: a sandwich-on answer + must not be served to a sandwich-off lookup — the cache partitions + by config, it does not leak. The load-bearing invariants are the + two that ARE untouched: **`question_hash`** (the user's Spanish + question is preserved verbatim — MT never rewrites what was asked) + and **`verifier_policy_hash`** (the verifier is byte-identical; + the crosslang flags are absent from `_VERIFIER_POLICY_FIELDS`). + The MT *engine identity* binds into the run-DAG retrieval plan + (`RetrievalPlan.mt_*`), which is not a policy hash. Test: + `test_sandwich_hash_invariants`. + +## 3. Engine — local `[mt]` extra (fox-decided 2026-05-17) + +`opus-mt-es-en` (query in) + `opus-mt-en-es` (display out), +Helsinki-NLP, Apache-2.0, hash-pinned by HF revision, weights +off-repo under `~/.arborist/models/mt/`, optional dependency. **Not +Hermes-3-8B** (blackops rule: 8B unfit for human-language +translation). **Not an external API**: reproducibility (a pinned +checkpoint replays byte-identically — provenance-critical for a +Merkle-replay system) + zero egress (Operation Voyeur / sovereignty) ++ es↔en is the best-resourced MT pair so the local-vs-frontier +quality gap is smallest exactly here. Grok stays the documented +future bench-comparison candidate (blackops: "evaluate before +committing spend"), not the v1 substrate. Mirrors the `[nli]` / +`ShadowNLI` pattern (#000049) and the vecpack off-box pattern +(#000051) exactly — zero new architectural surface. + +## 4. Build + +- `arborist/qa/mt/` — `manifest.json` (pinned es-en + en-es repos + + revisions), `Translator` protocol, `OpusMTTranslator` (lazy + `transformers` import, `available` flag, graceful degrade — the + `ShadowNLI` template verbatim), `StubTranslator` (deterministic; + the test substrate **and** the safe fallback when the extra is + absent — identity passthrough, so the sandwich degrades to Phase-0 + behaviour, never raises). +- `[mt]` extra in `pyproject.toml`. +- `arborist/qa/query.py` — under + `crosslang_guard_enabled AND crosslang_translate_enabled` AND the + Phase-0 non-English signal fired AND translator `available`: + es→en the query → `retrieval_query` **and** the LLM prompt + (`llm_question`, so the model answers in English; `question_hash` + still uses the original Spanish); after the verified English + result, en→es the model **prose** (`raw_answer`, never the rendered + evidence scaffold — pinned verbatim English spans must not be + MT-ed) → `display_answer` (+ banner). Extend + `RetrievalPlan` with `mt_engine` / `mt_manifest_hash` / + `source_lang` (omitted-when-empty so existing `retrieval_plan_hash` + values do not churn — the §5 zero-churn discipline). +- CLI `--crosslang-translate`; `make query XLANG_MT=1`; + `tasks.py query XLANG_MT=1`. + +## 5. Acceptance criteria + +1. Sandwich path (StubTranslator): es query → en `retrieval_query` + (asserted), English `answer_text` verified by the **unmodified** + verifier, `display_answer` is the es rendering, banner present. +2. `question_hash` and `verifier_policy_hash` **identical** with the + flag on vs off (the load-bearing invariant: user-question identity + preserved, verifier byte-identical). `governance_policy_hash` + **differs** (whole-policy hash — like every policy flag; correct + cache partitioning, not a leak). Test: + `test_sandwich_hash_invariants`. +3. `RetrievalPlan` MT fields bind into `run_dag` (different engine → + different `retrieval_plan_hash`); empty MT fields → **unchanged** + hash (no churn on non-MT runs). +4. The translated Spanish text is **never** the verifier's input — + `answer_text` (English) is finalised and verified before display + MT runs; the rendered evidence scaffold (pinned verbatim English + spans) is never fed to MT (display translates `raw_answer` prose). +5. Default OFF: `crosslang_translate_enabled=False` → no `display_*`, + pure Phase-0 behaviour. English query → byte-identical (signal + None → translator never consulted). +6. `[mt]` absent → `OpusMTTranslator.available is False` → sandwich + no-ops to Phase-0, never raises (graceful-degrade test). +7. The translated answer is **never** passed to `verify_*` + (asserted structurally — the verifier input is `answer_text`, + set before display MT runs). + +## 6. Out of scope + +Languages beyond es-v1; the Grok bench-comparison; an +`audit_mode`-affecting strict mode that folds MT into +`governance_policy_hash` (the #000001 §6 optional-strict analogue); +auto language *detection* beyond the Phase-0 deterministic signal +(accent-free Spanish still slips — a detector is future work). + +## 7. Status + +Implementation landed 2026-05-17 — see §8. + +## 8. Landing — what works, and the honest limits + +**Mechanism: complete and verified live end-to-end.** A real run +(`arborist query --crosslang-translate --json "¿Qué es el +anarcocapitalismo?"` against live Hermes + the real `opus-mt` +engine, which loads in this env via the `[nli]`-shared +`transformers`/`torch`): + +- query translated es→en, LLM prompted in English, **`answer_text` + is the English answer** ("Anarchocapitalism is a political + philosophy and economic theory that advocates the elimination of + the state…"); the verifier ran on that English core; +- `display_answer` is the opus-mt es rendering of the verified English + prose ("El anarcocapitalismo es una filosofía política y una teoría + económica que aboga por la eliminación del Estado…"), + `display_translated=True`, `display_lang=es`, `engine=opus-mt-v1`, + banner present; +- the bright line held: the Spanish text is additive/display-only; + `question_hash` + `verifier_policy_hash` invariant (test + `test_sandwich_hash_invariants`); `RetrievalPlan` MT binding is + zero-churn on non-MT runs (`test_retrieval_plan_mt_fields_*`). +- 6 sandwich tests + full suite **2477 passed**, 0 regressions; + `make`/`tasks.py`/CLI surface (`--crosslang-translate` / + `XLANG_MT=1`) wired; default OFF. + +**Two honest limitations the live run exposed (NOT solved here):** + +1. **Recall is not yet good.** `opus-mt-es-en` rendered + `anarcocapitalismo` → "Anarchocapitalism" (one token, no hyphen), + which still did **not** rank the canonical *Anarcho-capitalism* + article primary — the live run was `UNGROUNDED` with off-target + sources (*Consensus decision-making*, *Karl Hess*, …). The + architecture is correct; the MT-output↔title normalization / + retrieval interaction needs tuning. This is exactly the experiment + the **default-OFF flag + bench gate** exist for, and why the + manifest revisions are placeholder (`main`, not SHA-pinned). "The + sandwich works" means the *mechanism*; it does **not** mean + cross-language grounding is solved. +2. **JSON/lattice display is rough.** In `claim_lattice` (JSON) mode + `raw_answer` is JSON, so display-MT translates the envelope + (`"declaraciones"`). Prose/quote mode renders clean. JSON-mode + display polish (translate field *values* only, or render from the + parsed claim list) is future work — banner-labelled and default-OFF, + so acceptable as an experimental edge. + +**Doc honesty correction (2026-05-17).** Earlier drafts of §2 #6 / +#000001 §7 said "governance untouched" — false: `governance_policy_hash` +is sha256 of the *whole* policy (`keys.py:182`), so the crosslang +flags move it like every policy flag (correct cache partitioning). +Corrected in §2 #6 (authoritative), #000001 §7, and both index rows; +artifact-over-instruction. + +## 9. Fan-out benchmark (2026-05-17) — measured, with the dead ends named + +**Repro (deterministic, no egress, no Hermes-for-translation).** +`bench/make_lang_questions.py ` regenerates the per-language +question set from `bench/qa_questions.txt` via `opus-mt-en-` +(the sandwich's own engine); `bench/qa_sweep.py … --policy +crosslang_translate_enabled=true [--policy crosslang_entity_mask=…] +[--policy crosslang_source_lang=…]` runs the live sweep; +`bench/es_delta.py ` is the +authoritative metric (per-question EN→xx transition). +`es_roundtrip_analysis.py` / `es_join_patterns.py` produced the +*refuted* metric below — kept only as the cautionary record. + +**The real measurement (n=1, claim_lattice, same 75 questions).** +EN baseline **85 %** grounded (64/75) → es+sandwich (no mask) +**71 %** (53/75): the sandwich costs **−14 pp**. Transitions: +PRESERVED 31 · DOWNGRADE S→H 16 · **LOST 17** · N/A 11 (6 of which +*gained*). + +**Dead end (named, not buried).** A deterministic es→en round-trip +content-token-recall metric (CLEAN 77 % / DRIFT 20 % / COLLAPSE 3 %) +looked like a sharp predictor of grounding loss. It is **not**: +against the live delta, CLEAN grounds 71 % and DRIFT 73 % (no +separation), and **12 of the 17 LOST questions round-tripped CLEAN**. +The metric is abandoned. This is another instance of the +already-codified CLAUDE.md bench-maxing standing lesson (a clean +synthetic metric ≠ real bench-qa behaviour — #000049 §7); **not** +re-added to CLAUDE.md (DRY — it is already there). + +**The LOST 17, from the artifact (not a hypothesis):** ≈7 *entity +translated/garbled* — `New London`→"nuevo Londres", `Tarsus`→"Tarso", +**`Boltzmann`→"perntzmann"** (opus-mt hallucinated the name), +`World War 2`→"Segunda Guerra Mundial"→"Second World War" (≠ corpus +"World War II"); ≈3 *broad-enumeration* shapes ("name all members of +the beatles", "list all planets") — a quantifier-guard×translation +interaction, separate lever; ≈several *n=1 stochastic* (`founder of +microsoft?`, `what continent is egypt on?` round-tripped clean, +entities intact, yet LOST — needs n=3 to separate). + +**Lever built + validated (the entity slice).** +`arborist/qa/mt/entity_mask.py` — deterministic mask/restore (quoted +spans + Capitalised runs; corpus-title anchoring deferred to v2), +`MaskedTranslator` wrapper, default-ON within the (default-OFF) +sandwich, `crosslang_entity_mask` toggles it for the A/B. Measured +through **real opus-mt** round-trip: `who is Paul of Tarsus?` base → +"Pablo de Tarso" (the exact LOST case) → **+mask → "Paul of +Tarsus"**; `Boltzmann` is now garble-proof (never reaches MT). 5 +tests. + +**Honest caveat — the bench under-states the lever.** The bench +questions are *lowercased* (`a bridge between new london…`), so the +Capitalisation detector cannot fire on the en side; on the xx→en +edge it still catches `Egipto`/`London` etc. So any lift on *this* +bench is a **lower bound** — real cased user input does strictly +better. The detector that catches lowercase entities is +**corpus-title anchoring** (protect any query span matching a known +article title), deferred to v2. + +**Also landed:** the #000056 fan-out caught a real defect — +`get_translator()` built a fresh ~300 MB model per call → at +concurrency 4 the first sweep was **88 % engine-ERROR** +(`meta tensors` race). Fixed: process-level memoised singleton + +lazy *per-pair* load (6 manifest pairs never load eagerly) + load +lock; 6 concurrent threads → 0 errors. French (`opus-mt-fr-en/-en-fr`) +and Russian (`ru`) added as second/third bread (manifest + +`crosslang_source_lang`); fr question set generated (`GNU Linux` +survives). [Tasks #14–#17] + +**Lift result (2026-05-17, measured — and it refutes the lever).** +n=1, claim_lattice, 75 q, vs EN baseline 64/75 (85 %): + +| config | grounded | Δ vs EN | LOST | +|---------------|------------|---------|------| +| es, no mask | 53 (71 %) | −14 pp | 17 | +| **es, +mask** | **49 (65 %)** | **−20 pp** | **22** | +| **fr, +mask** | **35 (47 %)** | **−38 pp** | **33** | + +**Entity-mask did not help — it is neutral-to-negative.** The +isolated `Paul of Tarsus` win did **not** replicate at bench scale: +es went 53→49 (−4 q ≈ the 5 pp n=1 noise floor — *ambiguous*, not a +confirmed regression but certainly no lift). fr is unambiguous and +bad (−38 pp, well past noise). My earlier "lever validated" claim +was premature; the bench refutes it. **Third instance this thread of +the CLAUDE.md bench-maxing lesson: an isolated/synthetic win ≠ +bench-qa behaviour.** Likely causes (untested): the lowercased bench +gives the mask almost nothing to grab on the en side, so on the +xx→en edge it fires on *Spanish/French* capitalisation (which is not +English entity structure) and the sentinels perturb opus-mt's +translation of the rest of the sentence — net harm, not help. + +**Honest standing verdict (comparator corrected 2026-05-17, fox).** +The right baseline is **not** native English — it is what +cross-language did *before this ticket*: the 2026-05-16 +`¿Qué es el anarcocapitalismo?` run retrieved Spanish-song-title +noise, burned 10.4 s, returned UNGROUNDED — **≈0 % grounded**. +Against that true baseline: + +- **The sandwich is a large net win: ~0 % → 71 % grounded (es)**, with + the proof core never corrupted. The −14 pp vs *English* is the + *cost of a capability that previously did not exist at all*, not a + regression. Calling that a "fail" was a comparator error — doom- + framing is as dishonest as hype. +- **The entity-mask *lever* is the genuine negative:** net-negative + at scale (es 71→65, fr −38 pp), so it is now **default-OFF** + (`crosslang_entity_mask=False`); the no-mask sandwich is the + better config and the one worth keeping. +- French is materially weaker than Spanish (fr+mask 47 %; fr no-mask + unmeasured — likely higher since mask hurt es). Still ≫ the ≈0 % + baseline. + +So: **keep the sandwich (clear win over no support); drop the mask +default (failed lever); the −14 pp-vs-English is a discount on a new +capability, not a defect.** Remaining: n=3 to firm es/fr numbers, +fr no-mask measurement, corpus-title anchoring (the one untried +detector that works on lowercase). Default-OFF as an *experiment +flag*, but the experiment is now **promising, not failing**. fox +decides next spend; the honest recommendation flipped — this is +worth continuing, just not with the mask. diff --git a/make.bat b/make.bat new file mode 100644 index 0000000..01ab20c --- /dev/null +++ b/make.bat @@ -0,0 +1,33 @@ +@echo off +rem =========================================================================== +rem arborist - Windows shim so `make ` works without GNU make. +rem +rem Forwards every argument to tasks.py (the cross-platform quickstart +rem runner). The Makefile stays canonical on Unix; this only exists so the +rem README quickstart runs on native Windows (cmd / PowerShell). +rem +rem cmd : make query Q="What is anarcho-capitalism?" +rem PowerShell : .\make.bat query Q="What is anarcho-capitalism?" +rem +rem Windows ships the `py` launcher rather than `python3`, so prefer it. +rem Ticket: docs/tickets/ticket-000055-windows-quickstart-no-make.md +rem =========================================================================== +setlocal + +set "_PY=" +where py >NUL 2>NUL && set "_PY=py -3" +if not defined _PY ( + where python >NUL 2>NUL && set "_PY=python" +) +if not defined _PY ( + where python3 >NUL 2>NUL && set "_PY=python3" +) +if not defined _PY ( + echo [make.bat] Python 3.10+ was not found on PATH. 1>&2 + echo Install it from https://www.python.org/downloads/ 1>&2 + echo and tick "Add python.exe to PATH" during setup. 1>&2 + exit /b 1 +) + +%_PY% "%~dp0tasks.py" %* +exit /b %ERRORLEVEL% diff --git a/pyproject.toml b/pyproject.toml index 97e134c..07dbdd4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -103,6 +103,28 @@ nli = [ "protobuf>=4.0", "optimum[onnxruntime]>=1.20", ] +mt = [ + # Local machine-translation engine for the #000056 "Operation + # Sandwich" cross-language grounding edges (arborist/qa/mt/): + # translate the query es->en (retrieval-side) and render the + # *verified English* answer en->es for display only. Helsinki-NLP + # opus-mt (MarianMT) is Apache-2.0, hash-pinnable by HF revision + # (reproducible — provenance-critical for a Merkle-replay system), + # and es<->en is its best-resourced pair. Hard out of core/dev like + # [nli]: a fresh checkout stays python3.12 + venv + sqlite3, and the + # default suite drives the sandwich via a deterministic + # StubTranslator so it never needs these weights. Install with: + # pip install 'arborist[mt]' + # Weights are NOT in the repo; they cache under + # ~/.arborist/models/mt/. Never Hermes-3-8B (8B unfit for + # human-language translation); never an external API (zero egress / + # Operation Voyeur; opaque API "versions" are provenance-hostile). + "transformers>=4.40", + "torch>=2.2", + "sentencepiece>=0.2", + "protobuf>=4.0", + "sacremoses>=0.1", +] dev = [ "pytest>=8", "pytest-asyncio>=0.23", diff --git a/tasks.py b/tasks.py new file mode 100644 index 0000000..62ad7e5 --- /dev/null +++ b/tasks.py @@ -0,0 +1,601 @@ +#!/usr/bin/env python3 +r"""arborist quickstart task runner — the no-`make` path (Windows-first). + +The Makefile is the canonical Unix entry point and remains so. This +runner mirrors **only the quickstart subset** of it so the README +quickstart runs on native Windows (cmd / PowerShell) with no `make`, +no `bzip2`, no `curl`, no `bash` — just Python 3.10+ and sqlite3 +(sqlite3 ships with CPython). It also works on POSIX +(`python3 tasks.py `), but on POSIX the Makefile stays the +documented path. + +Ticket: docs/tickets/ticket-000055-windows-quickstart-no-make.md +Keep the supported-target set in sync with the README quickstart and +the Makefile; tests/test_tasks_runner.py pins it. + +Usage (make-style KEY=VALUE args, identical to the Makefile): + + py -3 tasks.py bootstrap + py -3 tasks.py fetch-cur + py -3 tasks.py ingest-cur-attached SHARDS=4 + py -3 tasks.py distill-shards-parallel + py -3 tasks.py distill-shards-tfidf-parallel + py -3 tasks.py query Q="What is anarcho-capitalism?" + py -3 tasks.py inspect KEY= JSON=1 + py -3 tasks.py help + +On Windows `make.bat` forwards to this file, so `make ` (cmd) +or `.\make.bat ` (PowerShell) works exactly like `make` does +on Unix. +""" + +from __future__ import annotations + +import os +import shlex +import shutil +import subprocess +import sys +import urllib.error +import urllib.request +from collections.abc import Callable +from pathlib import Path + +# -------------------------------------------------------------------------- +# Config — mirrors the Makefile's `?=` defaults. Every value is overridable +# the same two ways the Makefile allows: an environment variable, or a +# make-style KEY=VALUE token on the command line (the CLI token wins). +# -------------------------------------------------------------------------- + +REPO = Path(__file__).resolve().parent +IS_WINDOWS = os.name == "nt" + +DEFAULTS = { + "VENV": ".venv", + "DATA_DIR": "data", + "WP_BASE_URL": "https://dumps.wikimedia.org/archive/2003/2003-05-16/en", + "WP_CUR_NAME": "20030516_cur_tablesql.bz2", + "DB": str(Path.home() / ".arborist" / "arborist.db"), + "SHARDS_DIR": str(Path.home() / ".arborist" / "shards"), + "SHARDS": "4", + "INGEST_LIMIT": "500", + "VERIFY_N": "10", + "SEARCH_Q": "computer", + "QUERY_TOP_K": "8", + "ANSWER_MODE": "claim_lattice", + "KG_SECONDS": "3600", + "CRAWL_DEPTH": "2", + "CRAWL_MAX": "0", + "RECRAWL_LIMIT": "100", +} + +# CLI KEY=VALUE tokens collected by main(); read through cfg(). +_CLI_VARS: dict[str, str] = {} + + +def cfg(key: str, default: str | None = None) -> str: + """Resolve a config value: CLI token > environment > DEFAULTS > default.""" + if key in _CLI_VARS: + return _CLI_VARS[key] + if key in os.environ: + return os.environ[key] + if key in DEFAULTS: + return DEFAULTS[key] + return "" if default is None else default + + +def truthy(key: str) -> bool: + """Make semantics: a variable is 'set' iff it has a non-empty value.""" + return cfg(key).strip() != "" + + +# -------------------------------------------------------------------------- +# Platform-aware venv layout + Python launcher detection. +# -------------------------------------------------------------------------- + + +def venv_dir() -> Path: + return REPO / cfg("VENV") + + +def venv_bin_dir() -> Path: + # Windows venvs put executables in Scripts\; POSIX in bin/. + return venv_dir() / ("Scripts" if IS_WINDOWS else "bin") + + +def venv_exe(name: str) -> Path: + suffix = ".exe" if IS_WINDOWS else "" + return venv_bin_dir() / f"{name}{suffix}" + + +def venv_python() -> Path: + return venv_exe("python") + + +def arborist_exe() -> Path: + return venv_exe("arborist") + + +def base_python_cmd() -> list[str]: + """The interpreter used to *create* the venv. + + Override with ARBORIST_PYTHON (e.g. 'py -3.12' or 'C:\\py\\python.exe'). + Windows ships the `py` launcher rather than `python3`, so prefer it. + """ + override = os.environ.get("ARBORIST_PYTHON", "").strip() + if override: + return shlex.split(override, posix=not IS_WINDOWS) + if IS_WINDOWS: + if shutil.which("py"): + return ["py", "-3"] + if shutil.which("python"): + return ["python"] + return ["python3"] + return ["python3"] if shutil.which("python3") else ["python"] + + +# -------------------------------------------------------------------------- +# Process helpers. +# -------------------------------------------------------------------------- + + +def _env() -> dict[str, str]: + env = dict(os.environ) + env["PYTHONUNBUFFERED"] = "1" # CLAUDE.md: never buffer long-running procs + return env + + +def run(argv: list[str], *, cwd: Path | None = None) -> None: + """Run a command, inheriting stdio. Abort the runner on failure.""" + printable = " ".join(shlex.quote(str(a)) for a in argv) + print(f">> {printable}", flush=True) + rc = subprocess.run(argv, cwd=str(cwd or REPO), env=_env()).returncode + if rc != 0: + sys.exit(rc) + + +def run_parallel(cmds: list[list[str]]) -> None: + """Fan out N commands, wait for all (the bash `for … & done; wait`).""" + if not cmds: + print(" (nothing to do — no shards found)", flush=True) + return + procs = [] + for argv in cmds: + printable = " ".join(shlex.quote(str(a)) for a in argv) + print(f">> {printable}", flush=True) + procs.append(subprocess.Popen(argv, cwd=str(REPO), env=_env())) + failed = 0 + for p in procs: + if p.wait() != 0: + failed += 1 + if failed: + sys.exit(f"{failed} of {len(procs)} parallel job(s) failed") + + +def arborist(*args: str) -> list[str]: + return [str(arborist_exe()), *args] + + +def shard_dbs(shards_dir: Path) -> list[Path]: + return sorted(shards_dir.glob("*.db")) if shards_dir.is_dir() else [] + + +# -------------------------------------------------------------------------- +# Bootstrap (idempotent, mirrors the Makefile's pyproject-mtime rule). +# -------------------------------------------------------------------------- + + +def _venv_marker() -> Path: + return venv_bin_dir() / ("activate" if not IS_WINDOWS else "activate.bat") + + +def ensure_bootstrap() -> None: + """Create the venv + editable install. No-op when already up to date.""" + marker = _venv_marker() + pyproject = REPO / "pyproject.toml" + up_to_date = ( + arborist_exe().exists() + and marker.exists() + and marker.stat().st_mtime >= pyproject.stat().st_mtime + ) + if up_to_date: + return + if not venv_python().exists(): + run([*base_python_cmd(), "-m", "venv", cfg("VENV")]) + pip = [str(venv_python()), "-m", "pip"] + run([*pip, "install", "--upgrade", "pip", "wheel"]) + # Matches Makefile:52 — `.[dev]` (NOT `[dev,html]`; the README claim of + # [dev,html] was stale, corrected with this ticket — artifact wins). + run([*pip, "install", "-e", ".[dev]"]) + marker.touch() + + +def ensure_crawler() -> None: + ensure_bootstrap() + run([str(venv_python()), "-m", "pip", "install", "-e", ".[crawler]"]) + + +# -------------------------------------------------------------------------- +# Targets. +# -------------------------------------------------------------------------- + + +def t_bootstrap() -> None: + """create venv and install the editable package ([dev] extras)""" + ensure_bootstrap() + print("bootstrap OK ·", arborist_exe(), flush=True) + + +def _wp_cur_path() -> Path: + return REPO / cfg("DATA_DIR") / cfg("WP_CUR_NAME") + + +def t_fetch_cur() -> None: + """download the 2003-05-16 cur table dump (~82 MB) — idempotent""" + dest = _wp_cur_path() + if dest.exists() and dest.stat().st_size > 0: + print(f"fetch-cur: {dest} already present — skipping", flush=True) + return + dest.parent.mkdir(parents=True, exist_ok=True) + url = f"{cfg('WP_BASE_URL')}/{cfg('WP_CUR_NAME')}" + tmp = dest.with_suffix(dest.suffix + ".part") + print(f">> fetching {url}", flush=True) + last_err: Exception | None = None + for attempt in range(1, 4): # curl --retry 3 + try: + req = urllib.request.Request(url, headers={"User-Agent": "arborist-tasks/1"}) + with urllib.request.urlopen(req) as resp, open(tmp, "wb") as fh: + total = int(resp.headers.get("Content-Length") or 0) + got = 0 + while chunk := resp.read(1 << 20): + fh.write(chunk) + got += len(chunk) + if total: + pct = got * 100 // total + print(f"\r {got >> 20} / {total >> 20} MB ({pct}%)", + end="", flush=True) + print(flush=True) + tmp.replace(dest) + print(f"fetch-cur: saved {dest}", flush=True) + return + except (urllib.error.URLError, OSError) as exc: # noqa: PERF203 + last_err = exc + print(f" attempt {attempt}/3 failed: {exc}", flush=True) + tmp.unlink(missing_ok=True) + sys.exit(f"fetch-cur: download failed after 3 attempts: {last_err}") + + +def t_ingest_cur_attached() -> None: + """sharded ingest of the cur snapshot, one process per shard (Phase 2)""" + ensure_bootstrap() + t_fetch_cur() + shards_dir = Path(cfg("SHARDS_DIR")) + shards_dir.mkdir(parents=True, exist_ok=True) + n = int(cfg("SHARDS")) + cur = str(_wp_cur_path()) + cmds = [ + arborist("ingest", "--source", "wikipedia_cur", "--path", cur, + "--shards-dir", str(shards_dir), "--shard", f"{i}/{n}") + for i in range(n) + ] + run_parallel(cmds) + + +def _distill_parallel(process: str) -> None: + ensure_bootstrap() + shards = shard_dbs(Path(cfg("SHARDS_DIR"))) + cmds = [ + arborist("--db", str(s), "distill", "--process", process, + "--kind", "surface") + for s in shards + ] + run_parallel(cmds) + + +def t_distill_shards_parallel() -> None: + """surface → core, first-sentence-v1, one process per shard""" + _distill_parallel("first-sentence-v1") + + +def t_distill_shards_tfidf_parallel() -> None: + """core → TF-IDF keyword sets for retrieval, one process per shard""" + _distill_parallel("tfidf-keywords-v1") + + +def _question() -> str: + q = cfg("Q") or " ".join(_POSITIONALS) + if not q.strip(): + sys.exit('usage: query Q="your question" [JSON=1 ANSWER_MODE=... ' + 'K="extra keywords" BURN=1 REPAIR=1 WITNESS=1 ' + 'BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1 XLANG=1 XLANG_MT=1]') + return q + + +def _query_argv(*, dry: bool) -> list[str]: + a = ["--shards-dir", cfg("SHARDS_DIR"), "query", "--top-k", cfg("QUERY_TOP_K")] + if dry: + a.append("--dry-run") + if truthy("JSON"): + a.append("--json") + if truthy("BURN"): + a.append("--burn") + if not dry and truthy("REPAIR"): + a.append("--repair") + if not dry and truthy("REPROMPTS"): + a += ["--repair-reprompts", cfg("REPROMPTS")] + if cfg("ANSWER_MODE").strip(): + a += ["--answer-mode", cfg("ANSWER_MODE")] + if not dry and truthy("K"): + a += ["--retrieval-keywords", cfg("K")] + if truthy("BROAD"): + a.append("--apply-quantifier-caps") + if truthy("REJECT_BROAD"): + a.append("--reject-broad") + if truthy("ALLOW_BROAD"): + a.append("--allow-broad") + if not dry and truthy("WITNESS"): + a.append("--witness") + if truthy("XLANG"): # ticket #000001 §7 cross-language guard (default OFF) + a.append("--crosslang-guard") + if truthy("XLANG_MT"): # ticket #000056 Operation Sandwich (default OFF) + a.append("--crosslang-translate") + a.append(_question()) + return a + + +def t_query() -> None: + """ask the corpus a question [JSON=1 ANSWER_MODE=... K=... BURN=1 …]""" + ensure_bootstrap() + run(arborist(*_query_argv(dry=False))) + + +def t_query_dry() -> None: + """like query but skip the LLM call (dry-run)""" + ensure_bootstrap() + run(arborist(*_query_argv(dry=True))) + + +def t_inspect() -> None: + """sidecar: classify unverified spans for a cache_key — KEY=hex [JSON=1]""" + ensure_bootstrap() + if not cfg("KEY"): + sys.exit("usage: inspect KEY= [JSON=1]") + a = ["--shards-dir", cfg("SHARDS_DIR"), "inspect", "--cache-key", cfg("KEY")] + if truthy("JSON"): + a.append("--json") + run(arborist(*a)) + + +def t_falsify() -> None: + """mark a cached answer wrong — KEY=hex REASON='why'""" + ensure_bootstrap() + if not cfg("KEY"): + sys.exit("usage: falsify KEY= REASON='why'") + run(arborist("--shards-dir", cfg("SHARDS_DIR"), "providence", + "--falsify", cfg("KEY"), "--reason", cfg("REASON"))) + + +def t_burn() -> None: + """delete a childless leaf — KEY=hex (providence) | KIND=document|core ROOT=hex; REASON='why' [FORCE=1]""" + ensure_bootstrap() + kind = cfg("KIND") or "providence" + base = ["--shards-dir", cfg("SHARDS_DIR"), "burn", "--kind", kind] + if kind == "providence": + if not cfg("KEY"): + sys.exit("usage: burn KEY= REASON='why' [FORCE=1]") + base += ["--cache-key", cfg("KEY")] + elif kind in ("document", "core"): + if not cfg("ROOT"): + sys.exit(f"usage: burn KIND={kind} ROOT= REASON='why' [FORCE=1]") + base += ["--root", cfg("ROOT")] + else: + sys.exit(f"unknown KIND: {kind} (expected: providence|document|core)") + base += ["--reason", cfg("REASON")] + if truthy("FORCE"): + base.append("--force") + run(arborist(*base)) + + +def t_burn_kindergarten() -> None: + """bust providence rows younger than SECONDS [SECONDS=3600 FORCE=1 DRY_RUN=1 REASON='why']""" + ensure_bootstrap() + # Makefile var is KG_SECONDS; its help advertises SECONDS — accept both, + # SECONDS (the documented token) wins. + seconds = cfg("SECONDS") or cfg("KG_SECONDS") + a = ["--shards-dir", cfg("SHARDS_DIR"), "burn-kindergarten", + "--kindergarten-seconds", seconds] + if truthy("REASON"): + a += ["--reason", cfg("REASON")] + if truthy("FORCE"): + a.append("--force") + if truthy("DRY_RUN"): + a.append("--dry-run") + run(arborist(*a)) + + +def t_bootstrap_crawler() -> None: + """install the [crawler] extras into the venv""" + ensure_crawler() + print("bootstrap-crawler OK", flush=True) + + +def _crawl_shard_path() -> Path: + custom = cfg("CRAWL_SHARD") + if custom: + return Path(custom) + url = cfg("URL") + # Mirror the Makefile sed: strip scheme, take up to first '/', '.'→'_'. + rest = url.split("://", 1)[-1] + host = rest.split("/", 1)[0].replace(".", "_") + return Path(cfg("SHARDS_DIR")) / f"crawl_{host}.db" + + +def t_crawl_ingest() -> None: + """BFS-crawl URL=https://x.com [DEPTH=2 MAX=0 FAST=1] into a per-host shard""" + if not cfg("URL"): + sys.exit("usage: crawl-ingest URL=https://example.com " + "[DEPTH=2 MAX=0 FAST=1 CRAWL_SHARD=path]") + ensure_crawler() + Path(cfg("SHARDS_DIR")).mkdir(parents=True, exist_ok=True) + shard = _crawl_shard_path() + print(f" shard: {shard}", flush=True) + a = ["--db", str(shard), "crawl", "--seed-url", cfg("URL"), + "--depth", cfg("DEPTH", cfg("CRAWL_DEPTH")), + "--max-pages", cfg("MAX", cfg("CRAWL_MAX"))] + if truthy("FAST"): + a.append("--fast") + a.append("--ingest") + run(arborist(*a)) + + +def t_recrawl_check() -> None: + """conditional-HEAD freshness probe [DOMAIN=x.com LIMIT=100 CRAWL_SHARD=path]""" + ensure_crawler() + limit = cfg("LIMIT", cfg("RECRAWL_LIMIT")) + domain = cfg("DOMAIN") + + def one(db: str) -> list[str]: + a = ["--db", db, "crawler", "recrawl-check"] + if domain: + a += ["--domain", domain] + a += ["--limit", limit] + return arborist(*a) + + if cfg("CRAWL_SHARD"): + run(one(cfg("CRAWL_SHARD"))) + return + for db in shard_dbs(Path(cfg("SHARDS_DIR"))): + if db.name in ("qa.db", "snapshots.db"): + continue + print(f" shard: {db}", flush=True) + run(one(str(db))) + + +def t_stats() -> None: + """counts: documents, chunks, edges, audit chain (single DB)""" + ensure_bootstrap() + run(arborist("--db", cfg("DB"), "stats")) + + +def t_stats_shards() -> None: + """cross-shard stats via UNION views over SHARDS_DIR""" + ensure_bootstrap() + run(arborist("--shards-dir", cfg("SHARDS_DIR"), "stats")) + + +def t_verify() -> None: + """round-trip Merkle proofs for VERIFY_N random documents""" + ensure_bootstrap() + run(arborist("--db", cfg("DB"), "verify", "-n", cfg("VERIFY_N"))) + + +def t_search() -> None: + """keyword search [Q=... | SEARCH_Q=computer]""" + ensure_bootstrap() + run(arborist("--db", cfg("DB"), "search", cfg("Q") or cfg("SEARCH_Q"))) + + +def t_test() -> None: + """run the pytest suite (excludes opt-in crawler tests)""" + ensure_bootstrap() + run([str(venv_exe("pytest")), "-q", "--ignore=tests/crawler", "-n", "auto"]) + + +def t_clean() -> None: + """remove venv + caches (keeps fetched data and db)""" + for path in (venv_dir(), REPO / ".pytest_cache", REPO / "arborist.egg-info"): + shutil.rmtree(path, ignore_errors=True) + for pyc in REPO.rglob("__pycache__"): + shutil.rmtree(pyc, ignore_errors=True) + print("clean OK", flush=True) + + +def t_clean_db() -> None: + """drop the arborist db (keeps fetched data and venv)""" + db = Path(cfg("DB")) + for suffix in ("", "-journal", "-wal", "-shm"): + Path(str(db) + suffix).unlink(missing_ok=True) + print(f"clean-db OK · {db}", flush=True) + + +def t_clean_data() -> None: + """remove fetched dumps (DATA_DIR)""" + shutil.rmtree(REPO / cfg("DATA_DIR"), ignore_errors=True) + print("clean-data OK", flush=True) + + +# Ordered so `help` reads top-to-bottom like the quickstart. +TARGETS: dict[str, Callable[[], None]] = { + "bootstrap": t_bootstrap, + "fetch-cur": t_fetch_cur, + "ingest-cur-attached": t_ingest_cur_attached, + "distill-shards-parallel": t_distill_shards_parallel, + "distill-shards-tfidf-parallel": t_distill_shards_tfidf_parallel, + "query": t_query, + "query-dry": t_query_dry, + "inspect": t_inspect, + "falsify": t_falsify, + "burn": t_burn, + "burn-kindergarten": t_burn_kindergarten, + "bootstrap-crawler": t_bootstrap_crawler, + "crawl-ingest": t_crawl_ingest, + "recrawl-check": t_recrawl_check, + "stats": t_stats, + "stats-shards": t_stats_shards, + "verify": t_verify, + "search": t_search, + "test": t_test, + "clean": t_clean, + "clean-db": t_clean_db, + "clean-data": t_clean_data, +} + +_POSITIONALS: list[str] = [] + + +def _unquote(s: str) -> str: + """Strip one layer of matching surrounding quotes. + + POSIX shells strip quotes before exec; cmd/PowerShell forward them + literally through `%*`. Stripping here makes the single documented + `KEY="value with spaces"` form behave identically on every shell — + including the cmd-friendly whole-token form `"KEY=value with spaces"`. + """ + if len(s) >= 2 and s[0] == s[-1] and s[0] in ("'", '"'): + return s[1:-1] + return s + + +def t_help() -> None: + """show this help""" + print("arborist quickstart runner — Windows/no-make path " + "(Makefile is canonical on Unix)\n") + print("usage: py -3 tasks.py [KEY=VALUE ...]\n") + for name, fn in TARGETS.items(): + print(f" {name:<30} {(fn.__doc__ or '').strip()}") + print(f" {'help':<30} {t_help.__doc__.strip()}") + + +def main(argv: list[str]) -> None: + args = argv[1:] + if not args or args[0] in ("help", "-h", "--help"): + t_help() + return + target = args[0] + for raw in args[1:]: + tok = _unquote(raw) # cmd-friendly "KEY=value with spaces" + if "=" in tok and not tok.startswith("="): + key, val = tok.split("=", 1) + _CLI_VARS[key.strip().upper()] = _unquote(val) # KEY="value" + else: + _POSITIONALS.append(tok) + fn = TARGETS.get(target) + if fn is None: + print(f"unknown target: {target}\n", file=sys.stderr) + t_help() + sys.exit(2) + fn() + + +if __name__ == "__main__": + main(sys.argv) diff --git a/tests/test_crosslang_guard.py b/tests/test_crosslang_guard.py new file mode 100644 index 0000000..0cc015c --- /dev/null +++ b/tests/test_crosslang_guard.py @@ -0,0 +1,220 @@ +"""Ticket #000001 §7 Phase 0 — deterministic cross-language guard. + +Two contracts under test: + +1. **English byte-identity.** `guard()` returns None for any + pure-ASCII query without inverted punctuation — including the + #000053/#000054 acronym shapes (`CPU`/`GPU`/`AI`/`ML`/`DNA`). + A None decision means `query()` does nothing, so the English + retrieval path is unchanged by construction. +2. **Cross-language behaviour.** The non-English signal fires on + `¿`/`¡`/accented letters; a query with a surviving content token + strips es function words from retrieval only; a pure-function-word + query fails closed to UNGROUNDED *before* retrieval/LLM via the + Merkle-auditable reject-DAG path (same shape as the + `quantifier_should_reject` path). + +No network. The integration test uses a stub corpus + a chat client +that raises if the LLM is reached (proving the pre-LLM short-circuit). +""" + +from __future__ import annotations + +from collections.abc import Iterator + +import pytest + +from arborist.qa.crosslang import ( + CrossLanguageDecision, + guard, + strip_for_retrieval, +) + + +# --- Contract 1: English / acronym byte-identity (the #000053/#000054 guard) + +@pytest.mark.parametrize( + "q", + [ + "What is anarcho-capitalism?", + "what is a CPU?", + "GPU vs CPU performance", + "AI and ML differences", + "what is DNA?", + "Who is Russell Ballestrini?", + "tell me all there is to know about FBI history", + "", + ], +) +def test_english_and_acronyms_are_byte_identical_noop(q): + # None → query() does nothing → English path unchanged. + assert guard(q) is None + + +# --- Contract 2: signal + fail-closed semantics + +def test_signal_fires_with_content_token_not_fail_closed(): + d = guard("¿Qué es el anarcocapitalismo?") + assert isinstance(d, CrossLanguageDecision) + assert d.fail_closed is False + assert d.content_tokens == ("anarcocapitalismo",) + # ASCII regex truncates "¿Qué" → "Qu"; es stoppack catches it. + assert set(d.dropped) == {"Qu", "es", "el"} + + +@pytest.mark.parametrize("q", ["¿Qué es el?", "¿qué es?", "¡Es la de los!"]) +def test_pure_function_words_fail_closed(q): + d = guard(q) + assert d is not None and d.fail_closed is True + assert d.content_tokens == () + + +def test_bare_ascii_function_words_do_not_fire_english_safe(): + # No ¿/¡, no accent → no signal → None. We never guess language + # from bare ASCII (that would risk English false-positives). + assert guard("es la de los") is None + + +def test_accented_letter_alone_triggers_signal(): + # No inverted punctuation, but a non-ASCII letter still signals. + d = guard("relatividad de Einstein según teoría") + assert d is not None + assert "Einstein" in d.content_tokens # proper noun survives + + +def test_strip_for_retrieval_drops_es_keeps_content_in_order(): + d = guard("¿Qué es el anarcocapitalismo?") + assert strip_for_retrieval("¿Qué es el anarcocapitalismo?", d) == "anarcocapitalismo" + # Mixed: English content tokens are retained (downstream applies + # its own English stopword filter). + d2 = guard("¿qué es la teoría anarcocapitalismo theory?") + out = strip_for_retrieval("¿qué es la teoría anarcocapitalismo theory?", d2) + assert "anarcocapitalismo" in out and "theory" in out + assert "es" not in out.split() and "la" not in out.split() + + +def test_reason_strings_distinguish_the_two_paths(): + fc = guard("¿Qué es el?") + ok = guard("¿Qué es el anarcocapitalismo?") + assert "fail closed" in fc.reason.lower() + assert "stripped" in ok.reason.lower() + + +# --- Contract 2 (integration): query() short-circuits pre-LLM, auditable + +class _NoLLM: + """Chat client that proves the LLM was never reached.""" + + def chat_completion(self, messages, **kwargs) -> str: # noqa: D401 + raise AssertionError("LLM must not be called on the fail-closed path") + + +def _ingest_one(tmp_path, text, title): + from arborist.document import Document + from arborist.ingest import ingest_source + from arborist.source import Source + from arborist.store import connect + + class FakeSource(Source): + source_type = "test" + + def __init__(self, docs): + self.docs = docs + + def iter_documents(self) -> Iterator[Document]: + yield from self.docs + + shard = tmp_path / "shard.db" + conn = connect(shard) + try: + ingest_source( + conn, + FakeSource([Document(uri="t://d", content=text, source_type="test", title=title)]), + ) + finally: + conn.close() + return shard + + +def test_query_fail_closed_is_pre_llm_and_merkle_auditable(tmp_path): + from arborist.qa.dag import verify_run_dag + from arborist.qa.query import DEFAULT_QUERY_POLICY, query + + shard = _ingest_one(tmp_path, "Anarcho-capitalism is a political philosophy. " * 40, + "Anarcho-capitalism") + result = query( + question="¿Qué es el?", # pure function words → fail closed + qa_db=tmp_path / "qa.db", + chat_client=_NoLLM(), # raises if the LLM is reached + model_id="stub", + single_db=shard, + policy=dict(DEFAULT_QUERY_POLICY, answer_mode="claim_lattice", + crosslang_guard_enabled=True), + ) + assert result["status"] == "cross_language_unsupported" + assert result["audit_mode"] == "UNGROUNDED" + assert result["cache_key"] is None + assert result["lookup_path"] == "preflight" + assert result["violations"][0]["kind"] == "CROSS_LANGUAGE_UNSUPPORTED" + # 3-stage reject DAG recomputes (same auditability as broad-reject). + assert verify_run_dag(result["run_dag_blob"]) is True + + +def test_query_english_does_not_trip_the_guard(tmp_path): + from arborist.qa.client import StubClient + from arborist.qa.query import DEFAULT_QUERY_POLICY, query + + shard = _ingest_one(tmp_path, "Anarcho-capitalism is a political philosophy. " * 40, + "Anarcho-capitalism") + result = query( + question="What is anarcho-capitalism?", + qa_db=tmp_path / "qa.db", + chat_client=StubClient(answer="Anarcho-capitalism is a philosophy. [E1]\n"), + model_id="stub", + single_db=shard, + policy=dict(DEFAULT_QUERY_POLICY, answer_mode="claim_lattice", + crosslang_guard_enabled=True), # guard ON, English still safe + ) + # Even with the guard enabled, the English path never reaches the + # cross-language status (guard() returns None for pure ASCII). + assert result["status"] != "cross_language_unsupported" + + +def test_query_field_case_proceeds_past_guard(tmp_path): + """`¿Qué es el anarcocapitalismo?` is NOT fail-closed (a content + token survives) → it must proceed past the guard (no pre-retrieval + short-circuit), exercising the stripped-retrieval path.""" + from arborist.qa.client import StubClient + from arborist.qa.query import DEFAULT_QUERY_POLICY, query + + shard = _ingest_one(tmp_path, "Unrelated content about gardening. " * 40, "Gardening") + result = query( + question="¿Qué es el anarcocapitalismo?", + qa_db=tmp_path / "qa.db", + chat_client=StubClient(answer="x [E1]\n"), + model_id="stub", + single_db=shard, + policy=dict(DEFAULT_QUERY_POLICY, answer_mode="claim_lattice", + crosslang_guard_enabled=True), + ) + assert result["status"] != "cross_language_unsupported" + + +def test_flag_off_reverts_to_legacy_behaviour(tmp_path): + """Default policy (flag OFF) → the guard is fully bypassed: even a + pure-function-word Spanish query does NOT short-circuit, proving + the A/B baseline for experimentation is byte-for-byte legacy.""" + from arborist.qa.client import StubClient + from arborist.qa.query import DEFAULT_QUERY_POLICY, query + + assert DEFAULT_QUERY_POLICY["crosslang_guard_enabled"] is False + shard = _ingest_one(tmp_path, "Unrelated content about gardening. " * 40, "Gardening") + result = query( + question="¿Qué es el?", # would fail-closed IF the flag were on + qa_db=tmp_path / "qa.db", + chat_client=StubClient(answer="x [E1]\n"), + model_id="stub", + single_db=shard, + policy=dict(DEFAULT_QUERY_POLICY, answer_mode="claim_lattice"), # flag OFF + ) + assert result["status"] != "cross_language_unsupported" diff --git a/tests/test_entity_mask.py b/tests/test_entity_mask.py new file mode 100644 index 0000000..ccc349e --- /dev/null +++ b/tests/test_entity_mask.py @@ -0,0 +1,59 @@ +"""Entity-preserving MT — #000056 §9 (the validated lever). + +Deterministic detection + mask/restore round-trip + the +MaskedTranslator protocol wrapper. No model, no network. +""" +from __future__ import annotations + +from arborist.qa.mt import StubTranslator +from arborist.qa.mt.entity_mask import MaskedTranslator, mask, protect_spans, restore + + +def _spans(text): + return [text[s:e] for s, e in protect_spans(text)] + + +def test_detects_capitalised_runs_and_quotes_not_function_words(): + assert _spans("who is Paul of Tarsus?") == ["Paul of Tarsus"] + assert _spans("when was the Eiffel Tower built?") == ["Eiffel Tower"] + assert _spans("a bridge between New London & Groton?") == ["New London & Groton"] + # quoted span keeps the quote chars (restored verbatim — harmless) + assert _spans('what does "winter is coming" mean?') == ['"winter is coming"'] + # wh / function lead word is NOT an entity; lowercase noun isn't either. + assert _spans("what is anarchocapitalism?") == [] + assert _spans("how does intel compare to amd?") == [] + + +def test_mask_restore_round_trips_exactly(): + q = "who wrote the play Hamlet?" + m, mp = mask(q) + assert "Hamlet" not in m and "ZQX0XQZ" in m + assert restore(m, mp) == q + # Restore tolerates MT mangling the sentinel's case/spacing. + assert restore("Quien escribio la obra zqx 0 xqz?", mp) == \ + "Quien escribio la obra Hamlet?" + + +def test_masked_translator_preserves_entity_through_a_mangling_engine(): + # Engine that lowercases everything it actually sees — would destroy + # "Hamlet" if it reached MT. Masking keeps it verbatim. + mangle = StubTranslator(fn=lambda t, s, g: t.lower()) + mt = MaskedTranslator(mangle) + out = mt.translate("who wrote the play Hamlet?", "en", "es") + assert "Hamlet" in out # entity survived the mangler + assert out.startswith("who") # non-entity text still translated + + +def test_graceful_degrade_propagates(): + dead = StubTranslator(fn=lambda *a: "x", available=False) + mt = MaskedTranslator(dead) + assert mt.available is False + # Inner unavailable → original text back (sandwich → Phase-0). + assert mt.translate("who is Paul of Tarsus?", "es", "en") == \ + "who is Paul of Tarsus?" + + +def test_engine_identity_carries_mask_policy(): + mt = MaskedTranslator(StubTranslator(engine_id="opus-mt-v1")) + assert "entmask-v1" in mt.engine_id + assert mt.manifest_hash.endswith(":entmask-v1") diff --git a/tests/test_operation_sandwich.py b/tests/test_operation_sandwich.py new file mode 100644 index 0000000..8a662fd --- /dev/null +++ b/tests/test_operation_sandwich.py @@ -0,0 +1,230 @@ +"""Ticket #000056 — Operation Sandwich. + +The bright line as executable invariants: + +- the grounded core is English-only: `answer_text` and `audit_mode` + are produced exactly as a same-sources English run; the Spanish + text lives in additive `display_*` keys and is NEVER the verifier's + input; +- query MT is a retrieval transform: it binds into + `RetrievalPlan` (→ `retrieval_plan_hash` → run-DAG), NOT + `question_hash` / `governance_policy_hash` — with zero hash churn + on every non-MT run; +- default OFF, gated under the Phase-0 flag, graceful-degrades when + the `[mt]` extra is absent. + +Deterministic: a `StubTranslator` is injected (`query(translator=)`), +so no network and no model weights. +""" + +from __future__ import annotations + +from collections.abc import Iterator + +from arborist.qa.mt import StubTranslator +from arborist.qa.retrieval_plan import RetrievalPlan, retrieval_plan_hash + +ES_Q = "¿Qué es el anarcocapitalismo?" +EN_Q = "What is anarcho-capitalism?" +EN_A = "Anarcho-capitalism is a political philosophy. [E1]\n" +ES_A = "El anarcocapitalismo es una filosofía política." + + +def _es_marker(text: str, src: str, tgt: str) -> str: + """Deterministic, render-string-agnostic: es→en pins the question, + en→es prefixes a marker so the test asserts *which side* got + translated without coupling to exact rendered output.""" + if (src, tgt) == ("es", "en"): + return EN_Q + if (src, tgt) == ("en", "es"): + return "[ES] " + text + return text + + +def _stub(): + return StubTranslator(fn=_es_marker) + + +# --- Unit: RetrievalPlan MT binding + zero churn (criterion 3) ------------- + +def test_retrieval_plan_mt_fields_bind_and_zero_churn(): + base = RetrievalPlan(retrieval_keywords="", top_k=8, over_fetch=32, + max_context_chars=60000) + # No MT → canonical() must be byte-identical to pre-#000056 (no "mt" + # key) so every existing retrieval_plan_hash is unchanged. + assert "mt" not in base.canonical() + same = RetrievalPlan(retrieval_keywords="", top_k=8, over_fetch=32, + max_context_chars=60000) + assert retrieval_plan_hash(base) == retrieval_plan_hash(same) + + mt = RetrievalPlan(retrieval_keywords="", top_k=8, over_fetch=32, + max_context_chars=60000, mt_engine="opus-mt-v1", + mt_manifest_hash="abc", source_lang="es") + assert mt.canonical()["mt"] == { + "engine": "opus-mt-v1", "manifest_hash": "abc", "source_lang": "es", + } + assert retrieval_plan_hash(mt) != retrieval_plan_hash(base) + # Different engine identity → different plan hash (audit can tell + # which MT engine pulled the English sources in). + mt2 = RetrievalPlan(retrieval_keywords="", top_k=8, over_fetch=32, + max_context_chars=60000, mt_engine="other", + mt_manifest_hash="abc", source_lang="es") + assert retrieval_plan_hash(mt2) != retrieval_plan_hash(mt) + + +# --- Unit: enabling MT cannot move the proof hashes (criterion 4 / §2#6) --- + +def test_sandwich_hash_invariants(): + """The honest invariants (artifact over instruction — + `governance_policy_hash` hashes the *whole* policy, keys.py:182): + + - `question_hash` UNCHANGED — the user's Spanish question is + preserved as cache/question identity (the load-bearing bright + line: MT never rewrites what the user asked); + - `verifier_policy_hash` UNCHANGED — the verifier is byte-identical + (crosslang flags are not in `_VERIFIER_POLICY_FIELDS`); + - `governance_policy_hash` CHANGES — like every policy flag + (quantifier, metacognition, soft-preflight), because it covers + the whole policy dict. This is correct: a sandwich-on answer + must NOT be served to a sandwich-off lookup. Cache partitions + by config; it does not leak. + """ + from arborist.qa.keys import ( + governance_policy_hash, + question_hash, + verifier_policy_hash, + ) + from arborist.qa.query import DEFAULT_QUERY_POLICY + + off = dict(DEFAULT_QUERY_POLICY) + on = dict(DEFAULT_QUERY_POLICY, crosslang_guard_enabled=True, + crosslang_translate_enabled=True) + assert verifier_policy_hash(off) == verifier_policy_hash(on) + qh = lambda p: question_hash( # noqa: E731 + ES_Q, mode=p.get("question_dedup", "equivalence_class")) + assert qh(off) == qh(on) + assert governance_policy_hash(off) != governance_policy_hash(on) + + +# --- Integration harness --------------------------------------------------- + +def _ingest_one(tmp_path, text, title): + from arborist.document import Document + from arborist.ingest import ingest_source + from arborist.source import Source + from arborist.store import connect + + class FakeSource(Source): + source_type = "test" + + def __init__(self, docs): + self.docs = docs + + def iter_documents(self) -> Iterator[Document]: + yield from self.docs + + shard = tmp_path / "shard.db" + conn = connect(shard) + try: + ingest_source( + conn, + FakeSource([Document(uri="t://d", content=text, + source_type="test", title=title)]), + ) + finally: + conn.close() + return shard + + +def _run(tmp_path, *, translator, policy_extra): + from arborist.qa.client import StubClient + from arborist.qa.query import DEFAULT_QUERY_POLICY, query + + shard = _ingest_one( + tmp_path, + "Anarcho-capitalism is a political philosophy. " * 40, + "Anarcho-capitalism", + ) + return query( + question=ES_Q, + qa_db=tmp_path / "qa.db", + chat_client=StubClient(answer=EN_A), + model_id="stub", + single_db=shard, + translator=translator, + policy=dict(DEFAULT_QUERY_POLICY, answer_mode="claim_lattice_pointer", + **policy_extra), + ) + + +def test_sandwich_core_is_english_display_is_spanish(tmp_path): + r = _run(tmp_path, translator=_stub(), + policy_extra=dict(crosslang_guard_enabled=True, + crosslang_translate_enabled=True)) + # Grounded core: the verified answer is ENGLISH and untranslated. + assert r["status"] != "cross_language_unsupported" + assert "Anarcho-capitalism" in r["answer_text"] + assert not r["answer_text"].startswith("[ES] ") + assert "audit_mode" in r + # Display edge: additive Spanish rendering, banner-labelled. + assert r["display_translated"] is True + assert r["display_lang"] == "es" and r["display_source_lang"] == "en" + assert r["display_answer"].startswith("[ES] ") + assert "verific" in r["display_unverified_banner"].lower() + # Bright line: the verifier's subject is the English core, never + # the Spanish display string (no es-marker anywhere proof-side). + assert r["answer_text"] != r["display_answer"] + assert "[ES] " not in r["answer_text"] + assert "[ES] " not in (r.get("verifier_input_text") or "") + + +def test_default_off_no_sandwich(tmp_path): + # Guard on, translate OFF (default) → never engages; no display_*. + r = _run(tmp_path, translator=_stub(), + policy_extra=dict(crosslang_guard_enabled=True)) + assert "display_answer" not in r + assert r.get("display_translated") is None + + +def test_graceful_degrade_when_mt_unavailable(tmp_path): + # [mt] absent is modelled by an unavailable translator. The + # sandwich must no-op to Phase-0 (no display_*), never raise. + dead = StubTranslator({}, available=False) + r = _run(tmp_path, translator=dead, + policy_extra=dict(crosslang_guard_enabled=True, + crosslang_translate_enabled=True)) + assert "display_answer" not in r + assert r["status"] != "cross_language_unsupported" # had a content token + + +def test_sandwich_requires_a_non_english_signal(tmp_path): + # An English question with the sandwich enabled never engages + # (Phase-0 guard returns None → translator never consulted → + # byte-identical English path). + from arborist.qa.client import StubClient + from arborist.qa.query import DEFAULT_QUERY_POLICY, query + + shard = _ingest_one(tmp_path, + "Anarcho-capitalism is a political philosophy. " * 40, + "Anarcho-capitalism") + + class _Boom: + available = True + engine_id = "boom" + manifest_hash = "boom" + + def translate(self, *a, **k): + raise AssertionError("translator consulted on an English query") + + r = query( + question=EN_Q, + qa_db=tmp_path / "qa.db", + chat_client=StubClient(answer=EN_A), + model_id="stub", + single_db=shard, + translator=_Boom(), + policy=dict(DEFAULT_QUERY_POLICY, answer_mode="claim_lattice_pointer", + crosslang_guard_enabled=True, + crosslang_translate_enabled=True), + ) + assert "display_answer" not in r diff --git a/tests/test_tasks_runner.py b/tests/test_tasks_runner.py new file mode 100644 index 0000000..d130981 --- /dev/null +++ b/tests/test_tasks_runner.py @@ -0,0 +1,172 @@ +"""Drift + behaviour guard for the Windows/no-make quickstart runner. + +``tasks.py`` + ``make.bat`` exist so the README quickstart runs on +native Windows without GNU make (ticket #000055). Two thin entry +points are a known DRY cost; this test pins them: + +- the supported-target set is frozen here, so adding/removing a + quickstart target without updating this contract fails loudly; +- venv-layout + Python-launcher detection is verified for *both* + ``os.name`` values (the host only exercises one at runtime); +- the make-style ``KEY=VALUE`` parser is verified for every shell + quoting form (POSIX strips quotes; cmd/PowerShell do not). + +No network, no subprocess, no venv — pure import + function checks. +""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import pytest + +REPO = Path(__file__).resolve().parents[1] + + +def _load(): + """Fresh import of tasks.py with cleared module-level CLI state.""" + spec = importlib.util.spec_from_file_location("arborist_tasks", REPO / "tasks.py") + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + mod._CLI_VARS.clear() + mod._POSITIONALS.clear() + return mod + + +# The frozen contract. Mirrors the README Quickstart + "After the answer" +# + Setup blocks and the Makefile quickstart subset. Changing the runner's +# surface MUST change this set in the same commit. +EXPECTED_TARGETS = { + "bootstrap", + "fetch-cur", + "ingest-cur-attached", + "distill-shards-parallel", + "distill-shards-tfidf-parallel", + "query", + "query-dry", + "inspect", + "falsify", + "burn", + "burn-kindergarten", + "bootstrap-crawler", + "crawl-ingest", + "recrawl-check", + "stats", + "stats-shards", + "verify", + "search", + "test", + "clean", + "clean-db", + "clean-data", +} + + +def test_supported_target_set_is_frozen(): + mod = _load() + assert set(mod.TARGETS) == EXPECTED_TARGETS + # Every target is callable and self-documents (help relies on __doc__). + for name, fn in mod.TARGETS.items(): + assert callable(fn), name + assert (fn.__doc__ or "").strip(), f"{name} has no help docstring" + + +def test_help_lists_every_target(capsys): + mod = _load() + mod.main(["tasks.py", "help"]) + out = capsys.readouterr().out + for name in EXPECTED_TARGETS | {"help"}: + assert f" {name}" in out, name + + +def test_unknown_target_exits_2(capsys): + mod = _load() + with pytest.raises(SystemExit) as exc: + mod.main(["tasks.py", "definitely-not-a-target"]) + assert exc.value.code == 2 + captured = capsys.readouterr() + assert "unknown target" in captured.err # diagnostics on stderr + assert "show this help" in captured.out # help still printed + + +@pytest.mark.parametrize( + "is_windows,bin_name,arborist_name,py_name", + [ + (False, "bin", "arborist", "python"), + (True, "Scripts", "arborist.exe", "python.exe"), + ], +) +def test_venv_layout_per_platform(monkeypatch, is_windows, bin_name, arborist_name, py_name): + mod = _load() + monkeypatch.setattr(mod, "IS_WINDOWS", is_windows) + assert mod.venv_bin_dir().name == bin_name + assert mod.arborist_exe().name == arborist_name + assert mod.venv_python().name == py_name + assert mod.venv_exe("pytest").name == ("pytest.exe" if is_windows else "pytest") + + +def test_windows_prefers_py_launcher(monkeypatch): + mod = _load() + monkeypatch.setattr(mod, "IS_WINDOWS", True) + monkeypatch.delenv("ARBORIST_PYTHON", raising=False) + monkeypatch.setattr(mod.shutil, "which", lambda name: f"/x/{name}" if name == "py" else None) + assert mod.base_python_cmd() == ["py", "-3"] + + +def test_posix_uses_python3(monkeypatch): + mod = _load() + monkeypatch.setattr(mod, "IS_WINDOWS", False) + monkeypatch.delenv("ARBORIST_PYTHON", raising=False) + monkeypatch.setattr(mod.shutil, "which", lambda name: f"/usr/bin/{name}") + assert mod.base_python_cmd() == ["python3"] + + +def test_arborist_python_override(monkeypatch): + mod = _load() + monkeypatch.setenv("ARBORIST_PYTHON", "py -3.12") + assert mod.base_python_cmd() == ["py", "-3.12"] + + +@pytest.mark.parametrize( + "argv,expect_q", + [ + # POSIX: the shell already stripped the quotes. + (["query", "Q=What is X?"], "What is X?"), + # cmd: KEY="value with spaces" — quotes forwarded literally. + (["query", 'Q="What is X?"'], "What is X?"), + # cmd-friendly whole-token form. + (["query", '"Q=What is X?"'], "What is X?"), + ], +) +def test_make_style_value_parsing(argv, expect_q): + mod = _load() + # Stop before executing the target — just parse args. + mod.TARGETS["query"] = lambda: None + mod.main(["tasks.py", *argv]) + assert mod._CLI_VARS["Q"] == expect_q + + +def test_positional_question_and_flags(): + mod = _load() + mod.TARGETS["query"] = lambda: None + mod.main(["tasks.py", "query", "What is X?", "JSON=1", "ANSWER_MODE=quote"]) + assert mod._POSITIONALS == ["What is X?"] + assert mod._CLI_VARS["JSON"] == "1" and mod.truthy("JSON") + assert mod._CLI_VARS["ANSWER_MODE"] == "quote" + assert not mod.truthy("BURN") # unset → falsey, like make + + +def test_crawl_shard_name_matches_makefile_sed(): + mod = _load() + mod._CLI_VARS["URL"] = "https://russell.ballestrini.net/blog/x?y=1" + # Makefile: sed 's,^https?://([^/]+).*,\1,' | tr '.' '_' + assert mod._crawl_shard_path().name == "crawl_russell_ballestrini_net.db" + + +def test_shim_and_attributes_present(): + assert (REPO / "make.bat").is_file() + text = (REPO / "make.bat").read_text() + assert "tasks.py" in text and "py -3" in text + ga = (REPO / ".gitattributes").read_text() + assert "*.bat text eol=crlf" in ga