diff --git a/arborist/wallet/bucket.py b/arborist/wallet/bucket.py index fc278be..98a2c44 100644 --- a/arborist/wallet/bucket.py +++ b/arborist/wallet/bucket.py @@ -94,12 +94,23 @@ def _to_fts5(query: str) -> str: stopwords, OR's the remaining content tokens. Returns an empty string when nothing survives — the caller treats that as "no hits." + NFKD-folds accents first so a user typing ``pokemon`` matches + ``Pokémon`` body terms (FTS5's default tokenizer is "unicode61" + which folds the same way, so the bucket-direct path agreed before + this fix; the sidecar path's ASCII-only word regex did not). + Examples: - "who developed virt-back?" -> "virt OR back" - "what is anarchism" -> "anarchism" - "merkle tree" -> "merkle OR tree" + "who developed virt-back?" -> "virt OR back" + "what is anarchism" -> "anarchism" + "merkle tree" -> "merkle OR tree" + "pokemon red" -> "pokemon OR red" """ - tokens = [t.lower() for t in _FTS5_WORD_RE.findall(query)] + import unicodedata + folded = "".join( + c for c in unicodedata.normalize("NFKD", query) + if not unicodedata.combining(c) + ) + tokens = [t.lower() for t in _FTS5_WORD_RE.findall(folded)] keep = [t for t in tokens if t and t not in _FTS5_STOPWORDS and len(t) > 1] return " OR ".join(keep) @@ -851,9 +862,16 @@ class MultiShardSidecarCorpus: the sum of reciprocal ranks across shards. Standard k=60 is TREC-recommended. - score_rrf(d) = sum over shards of 1 / (rrf_k + rank(d, shard)) + Post-merge title-relevance filter: drop hits whose titles + share zero query tokens with the query. Mirrors local + query.py's ``search.title_filter`` step — without it, + small shards (e.g. a 200-doc personal blog) hand back rank-1 + hits for any matched token, ties with rank-1 from a 1M-doc + shard under RRF, and ranks above topically correct docs. With + the filter, off-topic docs are dropped before the merge. """ from concurrent.futures import ThreadPoolExecutor + from arborist.wallet.sidecar import fold_accents, tokenize_text def _one(item): sh_url, sh_client, _ = item @@ -874,6 +892,26 @@ class MultiShardSidecarCorpus: for sh_url, hits in ex.map(_one, triples): per_shard.append((sh_url, hits)) + # Title-relevance filter: drop hits whose title shares zero + # query tokens. tokenize_text already folds accents + drops + # stopwords, so "Pokémon Red and Blue" → {pokemon, red, blue} + # overlaps query {starter, pokemon, red}; Russell Ballestrini + # blog root → {russell, ballestrini} overlaps NEITHER → dropped. + # Falls open (no filter) if the query has no content tokens + # after sanitization. + query_tokens = set(tokenize_text(query)) + if query_tokens: + def _title_relevant(h: dict) -> bool: + title = h.get("title") or "" + if not title: + return False + title_tokens = set(tokenize_text(title)) + return bool(query_tokens & title_tokens) + per_shard = [ + (sh_url, [h for h in hits if _title_relevant(h)]) + for sh_url, hits in per_shard + ] + # RRF merge: keyed by document_root so duplicates across shards # (rare but possible — e.g. a doc replicated for redundancy) # combine. diff --git a/arborist/wallet/sidecar.py b/arborist/wallet/sidecar.py index dce5459..f991218 100644 --- a/arborist/wallet/sidecar.py +++ b/arborist/wallet/sidecar.py @@ -76,6 +76,8 @@ from dataclasses import dataclass from pathlib import Path from typing import Iterable, Sequence +import unicodedata + from arborist.compress import unpack_chunk @@ -91,10 +93,27 @@ _WORD_RE = re.compile(r"[A-Za-z0-9_]+") from arborist.wallet.bucket import _FTS5_STOPWORDS as STOPWORDS +def fold_accents(text: str) -> str: + """NFKD-normalize + strip combining marks → ASCII-equivalent. + + Critical for retrieval: a user typing 'pokemon' should match + Wikipedia titles like 'Pokémon Red and Blue', and our ASCII-only + word regex was splitting 'Pokémon' into ['Pok', 'mon'] (because é + is not in [A-Za-z]), so the term 'pokemon' never landed in the + dictionary. NFKD folds 'é'→'e' before tokenization fires. + """ + return "".join( + c for c in unicodedata.normalize("NFKD", text) + if not unicodedata.combining(c) + ) + + def tokenize_text(text: str) -> list[str]: - """Lowercase word tokens, drop stopwords + 1-char tokens.""" + """Lowercase word tokens, drop stopwords + 1-char tokens. + Accent-fold first so 'pokémon' indexes/queries as 'pokemon'.""" + folded = fold_accents(text) return [ - t.lower() for t in _WORD_RE.findall(text) + t.lower() for t in _WORD_RE.findall(folded) if t.lower() not in STOPWORDS and len(t) > 1 ] @@ -502,7 +521,9 @@ class SidecarReader: return t query_stems = {_t_stem(t) for t in terms} for did in scores: - title = self._doc_title(did).lower() + # Same accent-fold + tokenize on the title side so + # 'pokemon' (query) overlaps 'Pokémon' (title). + title = fold_accents(self._doc_title(did).lower()) title_tokens = {_t_stem(t) for t in _WORD_RE.findall(title)} overlap = len(query_stems & title_tokens) if overlap: