diff --git a/bench/qa_sweep.py b/bench/qa_sweep.py index 01fd276..f1fd518 100644 --- a/bench/qa_sweep.py +++ b/bench/qa_sweep.py @@ -330,15 +330,25 @@ def _run_one( # #000049 Phase 2 — when ARBORIST_NLI_SHADOW is set, query() surfaces # the verifier-input text; carry it (+ the answer) into the row so a # downstream `nli_shadow_sweep.py --input ` can measure - # the would-demote rate on real traffic (§7 #12 gate item 4). Off by - # default — these two fields can be large; never persisted otherwise. + # the would-demote rate on bench-qa traffic (§7 #12 gate item 4). Off + # by default — these two fields can be large; never persisted + # otherwise. Scrub lone surrogates first: real Wikipedia context + # occasionally carries U+D800–U+DFFF code points (mangled source + # encoding) that `json.dumps(..., ensure_ascii=False)` then refuses + # to UTF-8-encode — a measurement field, U+FFFD is fine. vit = result.get("verifier_input_text") if vit is not None: - row["answer_text"] = result.get("answer_text") - row["context"] = vit + row["answer_text"] = _scrub_surrogates(result.get("answer_text")) + row["context"] = _scrub_surrogates(vit) return row +def _scrub_surrogates(s): + if not isinstance(s, str): + return s + return s.encode("utf-8", "replace").decode("utf-8") + + def _directive_compliance( answer_mode: str, result: dict, err: str | None ) -> dict: