sidecar: numeral fold (7↔VII) + keep numeric single-char tokens

A query like 'when was final fantasy 7 created?' was scoring every
'Final Fantasy ___' article identically because:
  1. '7' got stripped by the length>1 filter, so the query reduced
     to {final, fantasy}
  2. Even surviving, '7' literally doesn't appear in the corpus when
     it spells 'VII'

Two fixes:
  * tokenize_text now keeps single-char numeric tokens (so '7' survives
    sanitization). Stopword + len filter still applies to letters.
  * numeral_expand() pairs 1↔I, 2↔II, ..., 20↔XX. Applied in two places:
      - search() expands the query term list before dict lookup, so '7'
        finds 'vii' postings (no rebuild needed — chunks tokenized to
        'vii' at build time).
      - title-boost computes overlap on the expanded set, so 'Final
        Fantasy VII' titles win the +24 boost vs +16 for siblings.

Live demo (cloud-query 'when was final fantasy 7 created?' LLM=qwen):
  before: UNGROUNDED 0/1 — top sources Final Fantasy X-2, Tactics,
          Character design (FF VII article rank 5, never made top-4)
  after : EVIDENCE-WARRANTED 2/2 — 'Final Fantasy VII was originally
          released on January 31, 1997.' + 'Development began in 1994.'
          cited E1 = Final Fantasy VII (#1 in merged top-4)
This commit is contained in:
russell@unturf.com 2026-05-30 20:45:37 -04:00
parent 289631bd0d
commit 1d5d3accb3
No known key found for this signature in database

View file

@ -108,14 +108,68 @@ def fold_accents(text: str) -> str:
)
# Roman numeral <-> Arabic digit equivalence table for 1..20.
# A user typing 'final fantasy 7' should overlap a title 'Final
# Fantasy VII'. This is the same gap local query.py closes with its
# numeral-fold step. We only need 1..20 — beyond that, titles use
# Arabic digits in practice.
_NUMERAL_PAIRS: dict[str, str] = {
"1": "i", "i": "1",
"2": "ii", "ii": "2",
"3": "iii", "iii": "3",
"4": "iv", "iv": "4",
"5": "v", "v": "5",
"6": "vi", "vi": "6",
"7": "vii", "vii": "7",
"8": "viii","viii": "8",
"9": "ix", "ix": "9",
"10": "x", "x": "10",
"11": "xi", "xi": "11",
"12": "xii","xii": "12",
"13": "xiii","xiii": "13",
"14": "xiv","xiv": "14",
"15": "xv", "xv": "15",
"16": "xvi","xvi": "16",
"17": "xvii","xvii": "17",
"18": "xviii","xviii": "18",
"19": "xix","xix": "19",
"20": "xx", "xx": "20",
}
def numeral_expand(tokens) -> set:
"""Return a set with each token's numeral-equivalent form added.
'final fantasy 7' tokens → {final, fantasy, 7, vii}
'Final Fantasy VII' tokens → {final, fantasy, vii, 7}
Overlap of the two sets is 4 (or 3 if the duplicate folds), vs 2
without the fold — enough to lift the right title above siblings
that share only 'final' + 'fantasy'.
"""
out = set(tokens)
for t in tokens:
alt = _NUMERAL_PAIRS.get(t)
if alt is not None:
out.add(alt)
return out
def tokenize_text(text: str) -> list[str]:
"""Lowercase word tokens, drop stopwords + 1-char tokens.
Accent-fold first so 'pokémon' indexes/queries as 'pokemon'."""
"""Lowercase word tokens, drop stopwords + 1-char tokens (except
numeric singletons like '7' which carry meaning — 'Final Fantasy 7'
needs '7' to survive sanitization so the numeral-fold can later
pair it with 'VII'). Accent-fold first so 'pokémon' indexes/queries
as 'pokemon'."""
folded = fold_accents(text)
return [
t.lower() for t in _WORD_RE.findall(folded)
if t.lower() not in STOPWORDS and len(t) > 1
]
out: list[str] = []
for raw in _WORD_RE.findall(folded):
t = raw.lower()
if t in STOPWORDS:
continue
if len(t) <= 1 and not t.isdigit():
continue
out.append(t)
return out
# ---------------------------------------------------------------------------
@ -458,6 +512,12 @@ class SidecarReader:
terms = tokenize_text(query)
if not terms:
return []
# Numeral expand so 'final fantasy 7' query also looks up 'vii'
# in the dict (built from chunk text where 'Final Fantasy VII'
# tokenizes to ['final', 'fantasy', 'vii']). The expansion is
# additive — original tokens stay, equivalents joined as OR
# so docs matching either form contribute to BM25.
terms = sorted(numeral_expand(terms))
N = self.docs_count
if not hasattr(self, "_avg_doc_len"):
# Compute once + cache; needs the full doc_lens. Lazy because
@ -519,12 +579,15 @@ class SidecarReader:
if len(t) > 4 and t.endswith("s") and not t.endswith("ss"):
return t[:-1]
return t
query_stems = {_t_stem(t) for t in terms}
query_stems = numeral_expand({_t_stem(t) for t in terms})
for did in scores:
# Same accent-fold + tokenize on the title side so
# 'pokemon' (query) overlaps 'Pokémon' (title).
# Same accent-fold + tokenize + numeral-expand on the
# title side so 'pokemon'/'pokémon' AND '7'/'VII' both
# overlap correctly.
title = fold_accents(self._doc_title(did).lower())
title_tokens = {_t_stem(t) for t in _WORD_RE.findall(title)}
title_tokens = numeral_expand(
{_t_stem(t) for t in _WORD_RE.findall(title)}
)
overlap = len(query_stems & title_tokens)
if overlap:
scores[did] += title_boost * overlap