feat: cross-language Q&A (Operation Sandwich) + Windows quickstart — all default-OFF
Three workstreams, full suite 2482 passed, experimental paths default-OFF. #000055 — Windows quickstart without make tasks.py (pure-stdlib runner) + make.bat shim + .gitattributes; README Windows section rewritten. Quickstart needs only Python 3.10+ (no make/bzip2/curl/bash). Mirrors the Makefile quickstart subset; drift-pinned by tests/test_tasks_runner.py. #000001 §7 Phase 0 — deterministic cross-language guard arborist/qa/crosslang.py: non-English signal (¿/¡/non-ASCII) + an es function-word stoppack. Fail-closed to UNGROUNDED before retrieval/LLM (mirrors the quantifier reject-DAG) when no content token survives, else strips es stopwords from the retrieval query only. English path byte-identical by construction. Default OFF (crosslang_guard_enabled). Measured: the anarcocapitalismo field case 10.4s -> 1.6s. #000056 — Operation Sandwich (cross-language grounding) arborist/qa/mt/: opus-mt es/fr/ru<->en, lazy per-pair memoised singleton (fixes the 88%-engine-error concurrency defect), manifest-pinned, [mt] extra; entity_mask wrapper. Sandwich = translate query in (retrieval + LLM prompt) -> English answer -> UNTOUCHED verifier grounds English-vs-English -> translate the verified answer out as display-only (banner-labelled, zero grounding). question_hash + verifier_policy_hash invariant; MT engine identity binds into RetrievalPlan, not governance. CLI --crosslang-translate / make XLANG_MT=1. Default OFF; entity_mask default OFF (measured net-negative at bench scale). Fan-out bench (bench/*.py): Spanish ~0% -> 71% grounded vs the real no-support baseline; the round-trip predictor was tried and refuted; the entity-mask lever failed at scale (corpus-title anchoring untried). CLAUDE.md: cross-language bright-line convention + module map. Pre-existing modified diagram files are intentionally excluded.
This commit is contained in:
parent
b711215f11
commit
2c98fc964e
32 changed files with 4173 additions and 29 deletions
61
bench/es_join_patterns.py
Normal file
61
bench/es_join_patterns.py
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Join the live es+sandwich sweep against the deterministic round-trip
|
||||
buckets → does MT round-trip drift PREDICT grounding loss?
|
||||
|
||||
sweep JSONL (question=es, audit_mode, status)
|
||||
⋈ es_roundtrip.json (es → bucket CLEAN/DRIFT/COLLAPSE)
|
||||
→ crosstab bucket × audit_mode, plus the MT-engine error rate
|
||||
(the concurrency defect the fan-out surfaced) reported separately
|
||||
so it doesn't get conflated with the MT-quality signal.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
RES = ROOT / "bench" / "qa_results"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
sweep_path = Path(sys.argv[1]) if len(sys.argv) > 1 else max(
|
||||
RES.glob("2026-*.jsonl"), key=lambda p: p.stat().st_mtime)
|
||||
rt = {r["es"]: r for r in json.loads((RES / "es_roundtrip.json").read_text())}
|
||||
rows = [json.loads(ln) for ln in sweep_path.read_text().splitlines() if ln.strip()]
|
||||
|
||||
err = [r for r in rows if (r.get("status") or "").startswith(("err", "error"))
|
||||
or r.get("audit_mode") is None]
|
||||
ok = [r for r in rows if r not in err]
|
||||
print(f"sweep: {sweep_path.name} rows={len(rows)} "
|
||||
f"engine/other ERROR={len(err)} ({len(err)/max(1,len(rows)):.0%}) "
|
||||
f"scored={len(ok)}")
|
||||
|
||||
ct = defaultdict(Counter)
|
||||
unmatched = 0
|
||||
for r in ok:
|
||||
b = rt.get(r["question"], {}).get("bucket")
|
||||
if b is None:
|
||||
unmatched += 1
|
||||
continue
|
||||
ct[b][r.get("audit_mode") or "?"] += 1
|
||||
print("\nbucket × audit_mode (scored rows only):")
|
||||
modes = ["STRICT", "HYBRID", "UNGROUNDED", "?"]
|
||||
print(f" {'bucket':<9} " + " ".join(f"{m:>10}" for m in modes) + " ground%")
|
||||
for b in ("CLEAN", "DRIFT", "COLLAPSE"):
|
||||
c = ct.get(b, Counter())
|
||||
tot = sum(c.values())
|
||||
g = c["STRICT"] + c["HYBRID"]
|
||||
print(f" {b:<9} " + " ".join(f"{c[m]:>10}" for m in modes)
|
||||
+ f" {g}/{tot}" + (f" ({g/tot:.0%})" if tot else ""))
|
||||
if unmatched:
|
||||
print(f" ({unmatched} scored rows had no round-trip match)")
|
||||
print("\nreading: if CLEAN grounds >> COLLAPSE, MT round-trip drift "
|
||||
"predicts grounding loss — the lever is entity-preserving MT, "
|
||||
"not the sandwich architecture.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Loading…
Add table
Add a link
Reference in a new issue