From 0192255f7d73774dc7db4dbcff7f9638c939c39a Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Mon, 27 Apr 2026 13:39:29 -0400 Subject: [PATCH] three relevance signals: title + TF-IDF cores + body density MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "permacomputer" query fell into title-relevance's empty-fallback because "permacomputer" appears nowhere in any title. Fallback was top-1 by raw BM25, getting one tangential source. Same problem for any user-coined term: title-LIKE alone can't find it. _filter_by_title_relevance now grows three accept paths: 1. Title-token overlap (existing). 2. TF-IDF core keyword overlap. A doc's distinctive low-frequency terms get distilled into its tfidf-keywords-v1 core. A query for a rare term ("permacomputer", "unturf") hits docs whose core lists it as a keyword even when no title does. 3. Body density. Docs that mention a query token >= 3 times pass regardless of title or core. Cheap proxy for "actually about it." Empty-all-three fallback widens from top-1 to top-N (default 5) — the LLM gets enough context to honestly say "I don't know" rather than fabricating from a single tangential source. Implementation: - _docs_with_core_keyword_match: SQL join chunks -> documents -> derivations finds source docs whose TF-IDF core content matches a token. Folded into the per-shard search loop alongside FTS5 + title-LIKE. - _body_density_passes: counts case-insensitive occurrences in concatenated chunk content. Threshold 3. - _search_corpus stashes core_match_roots + root_to_shard on the returned hits list as side-channel info for _rerank. Makefile: distill-shards-tfidf-parallel for the per-shard TF-IDF run. Demo on the live cluster (after TF-IDF distill of 95k cur surface docs): Q "tell me about permacomputer ?" sources: MOAD Cheat Sheet + Unturf AI Infrastructure Expansion answer: synthesized from both, bounds coverage honestly Q "what is unturf about ?" sources: Unturf Trademark + Unturf AI Infrastructure Expansion answer: "grassroots, community-driven initiative ... ethical AI hardware, software, and networks" Both queries now find user-coined terms via TF-IDF cores even though the terms never appear in any title. 79 tests passing. --- Makefile | 8 ++ aborist/qa/query.py | 181 +++++++++++++++++++++++++++++++++++++++----- 2 files changed, 172 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index c34468b..9b9e30c 100644 --- a/Makefile +++ b/Makefile @@ -151,6 +151,14 @@ distill-shards-parallel: bootstrap ## one distill process per shard (parallel) $(ABORIST) --db $$shard distill --process first-sentence-v1 --kind surface & \ done; wait +# TF-IDF cores serve as enriched titles for retrieval — distinctive +# low-frequency terms that surface in body text get promoted into +# something queryable without a real title match. +distill-shards-tfidf-parallel: bootstrap ## TF-IDF cores per shard, in parallel + @for shard in $(SHARDS_DIR)/*.db; do \ + $(ABORIST) --db $$shard distill --process tfidf-keywords-v1 --kind surface & \ + done; wait + ingest: ingest-cur ## default ingest = cur (use ingest-old or *-parallel for full) # Grok account-export ETL. diff --git a/aborist/qa/query.py b/aborist/qa/query.py index acaafb4..eee1fa5 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -106,32 +106,57 @@ def _rerank_by_title( return hits -def _filter_by_title_relevance(hits: list, question: str) -> list: - """Concept-aware title filter: - - synonym expansion: query 'Athlon' also matches AMD-titled docs - - rivalry exclusion: query 'AMD' alone excludes Intel-titled docs; - 'AMD vs Intel' suppresses the exclusion (compare phrasing) - - hard filter: only keep title-matchers when matchers exist +def _filter_by_title_relevance( + hits: list, + question: str, + *, + core_match_roots: set[str] | None = None, + body_density_check: callable | None = None, + fallback_top_n: int = 5, +) -> list: + """Concept-aware relevance filter with three accept paths: - No matchers (corpus has nothing topical) → keep the top one so the - LLM sees a source and can honestly say "I don't know." + 1. Title-token overlap (after synonym expansion). Strongest signal. + 2. TF-IDF core keyword overlap — `core_match_roots` is a precomputed + set of source document_roots whose TF-IDF cores contain query + tokens. Closes the gap for neologisms like "permacomputer" that + never appear in titles but are distinctive enough to be TF-IDF + keywords of conversation bodies. + 3. Body density — docs mentioning the query token >= N times pass + even without title or core match. Cheap proxy for "actually about + the topic." `body_density_check(hit)` returns bool. + + Rivalry exclusion (Intel-titled docs in AMD queries) still applies on + every accept path. + + If all three accept paths together produce nothing, fall back to the + top `fallback_top_n` body-BM25 hits — the LLM gets enough context to + say "I don't know" rather than fabricating from a single tangential + source. """ qtokens = _title_query_tokens(question) if not qtokens: return hits accept = synonym_expand(qtokens) exclude = rivalry_excluded(qtokens, compare_phrasing=has_compare_phrasing(question)) - kept = [] + core_roots = core_match_roots or set() + + kept: list = [] for h in hits: - if not h.title: - continue - ttokens = _title_query_tokens(h.title.replace("_", " ")) + ttokens = _title_query_tokens(h.title.replace("_", " ")) if h.title else set() if exclude & ttokens: continue # rivalry: opposing-side title, drop it if accept & ttokens: kept.append(h) + continue + if h.document_root in core_roots: + kept.append(h) + continue + if body_density_check is not None and body_density_check(h): + kept.append(h) + continue if not kept: - return hits[:1] if hits else [] + return hits[: max(1, fallback_top_n)] if hits else [] return kept @@ -190,6 +215,68 @@ def _search_titles(conn, qtokens: list[str], limit: int) -> list[tuple]: return rows +def _docs_with_core_keyword_match( + conn, qtokens: list[str], limit: int +) -> list[tuple]: + """Find SOURCE docs whose TF-IDF core keywords contain a query token. + + TF-IDF cores act as enriched titles: a doc's distinctive low-frequency + terms get distilled into the core's content as comma-separated keywords. + A query for "permacomputer" (a neologism that never makes it into a + title) can match the TF-IDF core of a Grok conversation that mentioned + it, because permacomputer is a rare term that ranks high under TF-IDF. + + Returns rows of (document_root, document_uri, title) for the SOURCE docs + (not the cores) — the source is what gets fed to the LLM as context. + """ + if not qtokens: + return [] + clauses = " OR ".join(["LOWER(c.content) LIKE ?"] * len(qtokens)) + params = [f"%{t.lower()}%" for t in qtokens] + params.append(limit) + rows = conn.execute( + f""" + SELECT DISTINCT src.document_root, src.document_uri, src.title + FROM chunks c + JOIN documents core ON core.document_root = c.document_root + JOIN derivations der ON der.core_root = core.document_root + JOIN documents src ON src.document_root = der.src_root + WHERE core.source_type LIKE 'core:tfidf-%' + AND ({clauses}) + LIMIT ? + """, + params, + ).fetchall() + return rows + + +def _body_density_passes( + conn, document_root: str, qtokens: set[str], min_mentions: int = 3 +) -> bool: + """Body-token relevance: docs that mention a query token >= N times pass. + + Cheap proxy for "this doc is actually about the topic" when title + doesn't help. Counts case-insensitive substring matches in concatenated + chunk content. min_mentions=3 skips docs that mention the term in + passing without being topical. + """ + if not qtokens: + return False + rows = conn.execute( + "SELECT content FROM chunks WHERE document_root = ? AND content IS NOT NULL", + (document_root,), + ).fetchall() + if not rows: + return False + body = " ".join(r["content"] for r in rows).lower() + total_mentions = 0 + for t in qtokens: + total_mentions += body.count(t.lower()) + if total_mentions >= min_mentions: + return True + return False + + def _search_corpus( shards_dir: Path | None, single_db: Path | None, @@ -219,6 +306,13 @@ def _search_corpus( paths = [] raw: list[tuple] = [] + # Roots whose TF-IDF cores contain a query token — collected across + # shards. Used downstream by _filter_by_title_relevance as accept-path 2. + core_match_roots: set[str] = set() + # Per-shard mapping of doc root -> shard path, for body-density lookups + # in accept-path 3. Lets us reach back to the source shard cheaply. + root_to_shard: dict[str, str] = {} + for p in paths: conn = connect(p) try: @@ -234,6 +328,7 @@ def _search_corpus( str(p.resolve()), ) ) + root_to_shard[h.document_root] = str(p.resolve()) # Parallel title search using synonym-expanded tokens. for r in _search_titles(conn, list(accept_tokens), over_fetch): title_lower = (r["title"] or "").lower().replace("_", " ") @@ -251,6 +346,27 @@ def _search_corpus( str(p.resolve()), ) ) + root_to_shard[r["document_root"]] = str(p.resolve()) + # Core-keyword search: docs whose TF-IDF core keywords match. + # The query token doesn't need to be in title or even in body — + # being a TF-IDF keyword of the doc's core is enough signal. + for r in _docs_with_core_keyword_match( + conn, list(accept_tokens), over_fetch + ): + core_match_roots.add(r["document_root"]) + # Add as a hit too — give it a moderate score below title + # exact-match but above raw body BM25. + raw.append( + ( + 40.0, + r["document_root"], + r["document_uri"], + r["title"], + 0, + str(p.resolve()), + ) + ) + root_to_shard[r["document_root"]] = str(p.resolve()) finally: conn.close() @@ -274,9 +390,20 @@ def _search_corpus( return out -def _rerank(hits: list[_Hit], question: str) -> list[_Hit]: - """Filter out off-topic titles, then boost matches by overlap count.""" - hits = _filter_by_title_relevance(hits, question) +def _rerank( + hits: list[_Hit], + question: str, + *, + core_match_roots: set[str] | None = None, + body_density_check: callable | None = None, +) -> list[_Hit]: + """Filter out off-topic, then boost title-token matches by overlap count.""" + hits = _filter_by_title_relevance( + hits, + question, + core_match_roots=core_match_roots, + body_density_check=body_density_check, + ) return _rerank_by_title(hits, question) @@ -342,7 +469,27 @@ def query( "total_ms": _ms_since(t_start), }, } - hits = _rerank(hits, question) + core_match_roots = getattr(hits, "_core_match_roots", set()) + root_to_shard = getattr(hits, "_root_to_shard", {}) + qtokens_lower = {t.lower() for t in _title_query_tokens(question)} + + def _body_density_check(h) -> bool: + # Lazy per-hit check: open the shard, count token mentions in this doc. + sp = root_to_shard.get(h.document_root) or h.shard_path + if not sp: + return False + c = connect(sp) + try: + return _body_density_passes(c, h.document_root, qtokens_lower) + finally: + c.close() + + hits = _rerank( + hits, + question, + core_match_roots=core_match_roots, + body_density_check=_body_density_check, + ) search_ms = _ms_since(t_phase) # 2. Pull doc texts within budget.