From 42a0f933ff3e63df718a4f8ba4f48c4eb2fa0c01 Mon Sep 17 00:00:00 2001
From: "russell@unturf.com" <russell@unturf.com>
Date: Tue, 12 May 2026 17:04:33 -0400
Subject: [PATCH] bench/qa_sweep: scrub lone surrogates from the NLI-shadow
 answer_text/context fields before json.dumps
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

ARBORIST_NLI_SHADOW=1 carries the raw verifier-input text into bench
rows; real Wikipedia context occasionally has U+D800–U+DFFF code points
(mangled source encoding) that json.dumps(..., ensure_ascii=False) then
refuses to UTF-8-encode → the run died at row 224/225. Scrub via
encode('utf-8','replace').decode() — U+FFFD is fine for a measurement
field. Only the two new shadow fields are touched.
---
 bench/qa_sweep.py | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/bench/qa_sweep.py b/bench/qa_sweep.py
index 01fd276..f1fd518 100644
--- a/bench/qa_sweep.py
+++ b/bench/qa_sweep.py
@@ -330,15 +330,25 @@ def _run_one(
     # #000049 Phase 2 — when ARBORIST_NLI_SHADOW is set, query() surfaces
     # the verifier-input text; carry it (+ the answer) into the row so a
     # downstream `nli_shadow_sweep.py --input <this.jsonl>` can measure
-    # the would-demote rate on real traffic (§7 #12 gate item 4). Off by
-    # default — these two fields can be large; never persisted otherwise.
+    # the would-demote rate on bench-qa traffic (§7 #12 gate item 4). Off
+    # by default — these two fields can be large; never persisted
+    # otherwise. Scrub lone surrogates first: real Wikipedia context
+    # occasionally carries U+D800–U+DFFF code points (mangled source
+    # encoding) that `json.dumps(..., ensure_ascii=False)` then refuses
+    # to UTF-8-encode — a measurement field, U+FFFD is fine.
     vit = result.get("verifier_input_text")
     if vit is not None:
-        row["answer_text"] = result.get("answer_text")
-        row["context"] = vit
+        row["answer_text"] = _scrub_surrogates(result.get("answer_text"))
+        row["context"] = _scrub_surrogates(vit)
     return row
 
 
+def _scrub_surrogates(s):
+    if not isinstance(s, str):
+        return s
+    return s.encode("utf-8", "replace").decode("utf-8")
+
+
 def _directive_compliance(
     answer_mode: str, result: dict, err: str | None
 ) -> dict: