diff --git a/aborist/qa/client.py b/aborist/qa/client.py index 26d0a97..dc7b4b9 100644 --- a/aborist/qa/client.py +++ b/aborist/qa/client.py @@ -114,6 +114,22 @@ class OpenAICompatibleClient: self.close() return False + @staticmethod + def _scrub_surrogates(s: str) -> str: + """Replace lone UTF-16 surrogates with U+FFFD. + + Wikipedia chunks (and other ingested text) occasionally + contain unpaired surrogates from the ingest of invalid-UTF-8 + source. httpx's json= path does ``.encode('utf-8')`` on the + serialized request body, which raises UnicodeEncodeError on + any lone surrogate. Sanitize incoming message content here + so the outbound HTTP request always serializes cleanly. We + round-trip through WTF-8 (surrogatepass) bytes, then decode + as standard UTF-8 with replacement — invalid sequences + become U+FFFD (REPLACEMENT CHARACTER). + """ + return s.encode("utf-8", errors="surrogatepass").decode("utf-8", errors="replace") + def chat_completion( self, messages: list[dict], @@ -130,6 +146,18 @@ class OpenAICompatibleClient: client = self._http # persistent connection pool from __init__ headers = {"Content-Type": "application/json"} + # Sanitize message content for httpx's json-encode path. + messages = [ + { + **m, + "content": ( + self._scrub_surrogates(m["content"]) + if isinstance(m.get("content"), str) + else m.get("content") + ), + } + for m in messages + ] if self.api_key: headers["Authorization"] = f"Bearer {self.api_key}" payload = {