diff --git a/aborist/qa/query.py b/aborist/qa/query.py index 77570fc..9202121 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -579,15 +579,41 @@ def _search_titles(conn, qtokens: list[str], limit: int) -> list[tuple]: """SQL LIKE over documents.title — finds the HTTP article that FTS5 misses because list-pages with many URLs have higher 'http' term frequency than the actual protocol article. Returns rows shaped - to match the FTS5 hit tuple.""" + to match the FTS5 hit tuple. + + Bug fix 2026-05-02 (fox case: ``"what date did back to the + future come out?"``): the previous SQL had no ``ORDER BY``, so + SQLite returned rows in arbitrary internal order and ``LIMIT`` + truncated before the actual title-token-overlap winners landed. + The film article ``"Back to the Future"`` (title contains + ``back`` AND ``future``) lost the LIMIT race to substring-match + junk like ``"Out (poker)"`` (matches ``out`` substring), + ``"Aberdeen, South Dakota"`` (``south`` contains ``out``), + ``"Backplane"`` (``back`` substring), etc. + + Fix: score each row by how many query tokens its title contains + (substring-level — the post-filter in the caller does + word-boundary verification), and ``ORDER BY`` that score + descending with title length ascending as tiebreaker (shorter + title = higher title purity per char). The legitimate + multi-token matches surface above single-substring junk before + LIMIT truncates. + """ if not qtokens: return [] clauses = " OR ".join(["LOWER(title) LIKE ?"] * len(qtokens)) - params = [f"%{t.lower()}%" for t in qtokens] - params.append(limit) + score_expr = " + ".join( + ["CASE WHEN LOWER(title) LIKE ? THEN 1 ELSE 0 END"] * len(qtokens) + ) + likes = [f"%{t.lower()}%" for t in qtokens] + # Two passes of params: once for the score-CASE-WHEN block, + # once for the WHERE clause. + params = likes + likes + [limit] rows = conn.execute( - f"SELECT document_root, document_uri, title FROM documents " - f"WHERE {clauses} LIMIT ?", + f"SELECT document_root, document_uri, title, " + f"({score_expr}) AS title_score FROM documents " + f"WHERE {clauses} " + f"ORDER BY title_score DESC, LENGTH(title) ASC LIMIT ?", params, ).fetchall() return rows @@ -918,9 +944,29 @@ def _search_corpus( # CPU?". Now that contribution is the same scale as FTS5 body # BM25, which lets Pentium_4 (high body relevance, no title # overlap) win on its actual topical fit. + # Word-boundary overlap check (was substring-match, + # which falsely passed `"out" in "south"`, + # `"date" in "candidate"`, `"come" in "outcome"`, etc.). + # 2026-05-02 case fox surfaced: "what date did back to + # the future come out?" — the substring check admitted + # "Aberdeen, South Dakota" (south contains "out"), + # "Backplane" (back), "Outline of biology" (out), and + # consumed the over_fetch budget so the legitimate + # `Back to the Future` film article never made the + # rerank cut. Tokenizing the title via + # `_title_query_tokens` + stem-aware comparison keeps + # the title-LIKE pass returning only docs whose title + # has an actual matching word. + accept_stems = { + _stem_token_for_match(t) for t in accept_tokens + } for r in _search_titles(conn, list(accept_tokens), over_fetch): - title_lower = (r["title"] or "").lower().replace("_", " ") - overlap = sum(1 for t in accept_tokens if t in title_lower) + title_norm = (r["title"] or "").replace("_", " ") + title_stems = { + _stem_token_for_match(t) + for t in _title_query_tokens(title_norm) + } + overlap = len(accept_stems & title_stems) if overlap == 0: continue title_score = overlap * 10.0