diff --git a/arborist/wallet/sidecar.py b/arborist/wallet/sidecar.py index 7cb1d40..6a15e40 100644 --- a/arborist/wallet/sidecar.py +++ b/arborist/wallet/sidecar.py @@ -420,6 +420,7 @@ class SidecarReader: mode: str = "or", # or | and k1: float = 1.5, b: float = 0.75, + title_boost: float = 8.0, ) -> list[SidecarHit]: """Tokenize ``query`` (matching tokenize_text), look each term up in the dictionary, score with BM25, return top ``limit`` hits. @@ -481,12 +482,48 @@ class SidecarReader: tf_norm = tf * (k1 + 1.0) / denom if denom > 0 else 0.0 scores[did] = scores.get(did, 0.0) + idf * tf_norm + # Title-token boost: docs whose TITLE contains a query term get + # a fixed bonus per overlap. BM25-only ranking penalizes long + # main articles ("Homer Simpson" Wikipedia article) vs short + # episode articles even when the main article is the right + # primary source. The local query.py compensates via a + # separate title-LIKE retrieval route; sidecar emulates with + # a render-time boost using the title we already store in the + # doc table. Stems both sides (possessive + plural collapse — + # matches the title-mismatch verifier's _stem helper) so + # "simpsons" (query) overlaps "simpson" (title). title_boost=0 + # disables; default 8 is enough to lift a #3-by-BM25 long-doc + # match above #1-by-BM25 short-doc matches on the homer query. + if title_boost > 0 and scores: + def _t_stem(t: str) -> str: + t = t.replace("'", "").replace("’", "") + if len(t) > 4 and t.endswith("s") and not t.endswith("ss"): + return t[:-1] + return t + query_stems = {_t_stem(t) for t in terms} + for did in scores: + title = self._doc_title(did).lower() + title_tokens = {_t_stem(t) for t in _WORD_RE.findall(title)} + overlap = len(query_stems & title_tokens) + if overlap: + scores[did] += title_boost * overlap + ranked = sorted(scores.items(), key=lambda x: -x[1]) out: list[SidecarHit] = [] for did, score in ranked[:limit]: out.append(self._doc_record(did, score=score)) return out + def _doc_title(self, doc_id: int) -> str: + """Read just the title field from the doc table (no allocation + of the rest of the doc record).""" + off = self.docs_offset + self._doc_offsets[doc_id] + off += 32 # document_root + (uri_len,) = struct.unpack_from(" int: """Read doc_len from the doc table (varint after title).""" cached = getattr(self, "_doc_len_cache", None)