From 975dceadedebdd9bb43c4328f49a4a95db4ed1be Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Sat, 30 May 2026 17:48:38 -0400 Subject: [PATCH] wallet: accent-fold tokens + post-RRF title-relevance filter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bugs surfaced by 'what are the 3 starter pokemon in pokemon red?' that returned UNGROUNDED with Russell Ballestrini's blog cited above every Pokémon Wikipedia article. Bug 1: ASCII-only word regex split 'Pokémon' on 'é' into ['Pok', 'mon'], so the term 'pokemon' never landed in the sidecar dict and the title-boost never matched 'Pokémon Red and Blue' against a 'pokemon' query token. Fix: NFKD-fold accents before tokenizing (both sides — build + query — agree). Mirrored in _to_fts5 too so the bucket-direct FTS5 path stays consistent (FTS5's unicode61 tokenizer already folds, so the sanitizer was the only place that needed the fix). Bug 2: Multi-shard RRF treated a rank-1 hit in a 223-doc personal blog identically to a rank-1 hit in a 1M-doc shard. Russell Ballestrini's blog incidentally contains 'red' or 'starter' somewhere, so its FTS5 returned the root page at rank 1; RRF tied with Pokémon Red and Blue (also rank 1 in genesis shard 0) and won by insertion order. Fix: post-merge title-relevance filter (mirrors local query.py's search.title_filter) — drop hits whose titles share zero stemmed+folded tokens with the query before RRF combines them. Russell Ballestrini blog title {russell, ballestrini} overlaps neither {starter, pokemon, red} → dropped. Live demo (cloud-query 'what are the 3 starter pokemon in pokemon red?'): before: UNGROUNDED 0/1 — Russell Ballestrini #1, no useful evidence after : EVIDENCE-WARRANTED-PARTIAL 3/6 — 'Bulbasaur, Charmander, and Squirtle' cited to Pokémon Red and Blue article --- arborist/wallet/bucket.py | 48 ++++++++++++++++++++++++++++++++++---- arborist/wallet/sidecar.py | 27 ++++++++++++++++++--- 2 files changed, 67 insertions(+), 8 deletions(-) diff --git a/arborist/wallet/bucket.py b/arborist/wallet/bucket.py index fc278be..98a2c44 100644 --- a/arborist/wallet/bucket.py +++ b/arborist/wallet/bucket.py @@ -94,12 +94,23 @@ def _to_fts5(query: str) -> str: stopwords, OR's the remaining content tokens. Returns an empty string when nothing survives — the caller treats that as "no hits." + NFKD-folds accents first so a user typing ``pokemon`` matches + ``Pokémon`` body terms (FTS5's default tokenizer is "unicode61" + which folds the same way, so the bucket-direct path agreed before + this fix; the sidecar path's ASCII-only word regex did not). + Examples: - "who developed virt-back?" -> "virt OR back" - "what is anarchism" -> "anarchism" - "merkle tree" -> "merkle OR tree" + "who developed virt-back?" -> "virt OR back" + "what is anarchism" -> "anarchism" + "merkle tree" -> "merkle OR tree" + "pokemon red" -> "pokemon OR red" """ - tokens = [t.lower() for t in _FTS5_WORD_RE.findall(query)] + import unicodedata + folded = "".join( + c for c in unicodedata.normalize("NFKD", query) + if not unicodedata.combining(c) + ) + tokens = [t.lower() for t in _FTS5_WORD_RE.findall(folded)] keep = [t for t in tokens if t and t not in _FTS5_STOPWORDS and len(t) > 1] return " OR ".join(keep) @@ -851,9 +862,16 @@ class MultiShardSidecarCorpus: the sum of reciprocal ranks across shards. Standard k=60 is TREC-recommended. - score_rrf(d) = sum over shards of 1 / (rrf_k + rank(d, shard)) + Post-merge title-relevance filter: drop hits whose titles + share zero query tokens with the query. Mirrors local + query.py's ``search.title_filter`` step — without it, + small shards (e.g. a 200-doc personal blog) hand back rank-1 + hits for any matched token, ties with rank-1 from a 1M-doc + shard under RRF, and ranks above topically correct docs. With + the filter, off-topic docs are dropped before the merge. """ from concurrent.futures import ThreadPoolExecutor + from arborist.wallet.sidecar import fold_accents, tokenize_text def _one(item): sh_url, sh_client, _ = item @@ -874,6 +892,26 @@ class MultiShardSidecarCorpus: for sh_url, hits in ex.map(_one, triples): per_shard.append((sh_url, hits)) + # Title-relevance filter: drop hits whose title shares zero + # query tokens. tokenize_text already folds accents + drops + # stopwords, so "Pokémon Red and Blue" → {pokemon, red, blue} + # overlaps query {starter, pokemon, red}; Russell Ballestrini + # blog root → {russell, ballestrini} overlaps NEITHER → dropped. + # Falls open (no filter) if the query has no content tokens + # after sanitization. + query_tokens = set(tokenize_text(query)) + if query_tokens: + def _title_relevant(h: dict) -> bool: + title = h.get("title") or "" + if not title: + return False + title_tokens = set(tokenize_text(title)) + return bool(query_tokens & title_tokens) + per_shard = [ + (sh_url, [h for h in hits if _title_relevant(h)]) + for sh_url, hits in per_shard + ] + # RRF merge: keyed by document_root so duplicates across shards # (rare but possible — e.g. a doc replicated for redundancy) # combine. diff --git a/arborist/wallet/sidecar.py b/arborist/wallet/sidecar.py index dce5459..f991218 100644 --- a/arborist/wallet/sidecar.py +++ b/arborist/wallet/sidecar.py @@ -76,6 +76,8 @@ from dataclasses import dataclass from pathlib import Path from typing import Iterable, Sequence +import unicodedata + from arborist.compress import unpack_chunk @@ -91,10 +93,27 @@ _WORD_RE = re.compile(r"[A-Za-z0-9_]+") from arborist.wallet.bucket import _FTS5_STOPWORDS as STOPWORDS +def fold_accents(text: str) -> str: + """NFKD-normalize + strip combining marks → ASCII-equivalent. + + Critical for retrieval: a user typing 'pokemon' should match + Wikipedia titles like 'Pokémon Red and Blue', and our ASCII-only + word regex was splitting 'Pokémon' into ['Pok', 'mon'] (because é + is not in [A-Za-z]), so the term 'pokemon' never landed in the + dictionary. NFKD folds 'é'→'e' before tokenization fires. + """ + return "".join( + c for c in unicodedata.normalize("NFKD", text) + if not unicodedata.combining(c) + ) + + def tokenize_text(text: str) -> list[str]: - """Lowercase word tokens, drop stopwords + 1-char tokens.""" + """Lowercase word tokens, drop stopwords + 1-char tokens. + Accent-fold first so 'pokémon' indexes/queries as 'pokemon'.""" + folded = fold_accents(text) return [ - t.lower() for t in _WORD_RE.findall(text) + t.lower() for t in _WORD_RE.findall(folded) if t.lower() not in STOPWORDS and len(t) > 1 ] @@ -502,7 +521,9 @@ class SidecarReader: return t query_stems = {_t_stem(t) for t in terms} for did in scores: - title = self._doc_title(did).lower() + # Same accent-fold + tokenize on the title side so + # 'pokemon' (query) overlaps 'Pokémon' (title). + title = fold_accents(self._doc_title(did).lower()) title_tokens = {_t_stem(t) for t in _WORD_RE.findall(title)} overlap = len(query_stems & title_tokens) if overlap: