From 00200b8f55638b649d8702a92fb306a19ebc8aa8 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Sat, 30 May 2026 17:10:17 -0400 Subject: [PATCH] sidecar: title-token boost so primary-source articles rank above incidentals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pure-BM25 ranking penalizes long main articles ('Homer Simpson' Wikipedia article) vs short episode articles with the same query tokens — length-normalization is doing what BM25 was designed to do but the result mis-ranks the primary source. Local query.py runs a SEPARATE title-LIKE retrieval route that surfaces title-match docs unconditionally; sidecar didn't, so the Homer Simpson article landed rank 4 behind 'Bart vs. the Space Mutants' and similar incidentals. Fix: at query time, after BM25 scoring, add a fixed bonus per query- term overlap with the doc title (already stored in the sidecar doc table — no rebuild needed). Both sides stemmed with the same possessive+plural normalization as _claim_title_overlap so 'simpsons' (query) overlaps 'simpson' (title). Default title_boost=8.0 is enough to lift the right primary source above incidental short-doc matches without overwhelming BM25 elsewhere. Live demo (cloud-ask 'who is homer simpsons boss?'): before: 'Hank Scorpio is Homer Simpson's boss' (cited episode article where Scorpio appears; primary 'Homer Simpson' article ranked rank 4 in its shard, never reached the merged top-K) after : 'Homer Simpson's boss is Charles Montgomery Burns.' (cited E1=Homer Simpson + E3=Homer's Odyssey episode) EVIDENCE-WARRANTED-PARTIAL · via claim_lattice 1/2 26.6s --- arborist/wallet/sidecar.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/arborist/wallet/sidecar.py b/arborist/wallet/sidecar.py index 7cb1d40..6a15e40 100644 --- a/arborist/wallet/sidecar.py +++ b/arborist/wallet/sidecar.py @@ -420,6 +420,7 @@ class SidecarReader: mode: str = "or", # or | and k1: float = 1.5, b: float = 0.75, + title_boost: float = 8.0, ) -> list[SidecarHit]: """Tokenize ``query`` (matching tokenize_text), look each term up in the dictionary, score with BM25, return top ``limit`` hits. @@ -481,12 +482,48 @@ class SidecarReader: tf_norm = tf * (k1 + 1.0) / denom if denom > 0 else 0.0 scores[did] = scores.get(did, 0.0) + idf * tf_norm + # Title-token boost: docs whose TITLE contains a query term get + # a fixed bonus per overlap. BM25-only ranking penalizes long + # main articles ("Homer Simpson" Wikipedia article) vs short + # episode articles even when the main article is the right + # primary source. The local query.py compensates via a + # separate title-LIKE retrieval route; sidecar emulates with + # a render-time boost using the title we already store in the + # doc table. Stems both sides (possessive + plural collapse — + # matches the title-mismatch verifier's _stem helper) so + # "simpsons" (query) overlaps "simpson" (title). title_boost=0 + # disables; default 8 is enough to lift a #3-by-BM25 long-doc + # match above #1-by-BM25 short-doc matches on the homer query. + if title_boost > 0 and scores: + def _t_stem(t: str) -> str: + t = t.replace("'", "").replace("’", "") + if len(t) > 4 and t.endswith("s") and not t.endswith("ss"): + return t[:-1] + return t + query_stems = {_t_stem(t) for t in terms} + for did in scores: + title = self._doc_title(did).lower() + title_tokens = {_t_stem(t) for t in _WORD_RE.findall(title)} + overlap = len(query_stems & title_tokens) + if overlap: + scores[did] += title_boost * overlap + ranked = sorted(scores.items(), key=lambda x: -x[1]) out: list[SidecarHit] = [] for did, score in ranked[:limit]: out.append(self._doc_record(did, score=score)) return out + def _doc_title(self, doc_id: int) -> str: + """Read just the title field from the doc table (no allocation + of the rest of the doc record).""" + off = self.docs_offset + self._doc_offsets[doc_id] + off += 32 # document_root + (uri_len,) = struct.unpack_from(" int: """Read doc_len from the doc table (varint after title).""" cached = getattr(self, "_doc_len_cache", None)