concept overlay: synonym expansion + rivalry exclusion

A small knowledge-graph layer that does two distinct jobs:

1. SYNONYM_GROUPS broaden retrieval. A query mentioning "Athlon"
   now also matches AMD-titled docs because Athlon IS an AMD product.
   Groups currently cover AMD-family, Intel-family, HTTP family,
   FTP, Mac, Windows, Linux. Easy to extend.

2. RIVALRIES narrow retrieval. The pair (AMD-group, Intel-group)
   means: if the query mentions one side and not the other, drop
   docs whose titles contain the OTHER side's tokens. So a "fastest
   AMD CPU" question never gets Pentium_4 in the context — even if
   FTS5 BM25 ranks it high — because Pentium is in the Intel group
   and Intel isn't in the query.

   COMPARE_WORDS ("vs", "versus", "compare", "between", ...) suppress
   the exclusion. "compare AMD vs Intel" keeps both sides. "what is
   the fastest AMD CPU?" does not.

Demos against the 128k Wikipedia 2003-05-16 cur shards:

  Q "fastest AMD CPU"   → 8 sources, all AMD/CPU titled, NO Intel
                          Answer: "Athlon XP 3200+" (real AMD chip,
                          grounded in the AMD article)

  Q "compare AMD vs Intel"
                        → 8 sources, mix of Intel_8028x, Intel_8048x,
                          AMD_Duron — both sides preserved

  Q "Athlon processor"  → AMD, AMD_Duron, AMD_5x86, Athlon all
                          surfaced via synonym expansion

Phase 1 implementation hand-curates the groups; Phase 2 idea is to
derive them from Wikipedia's link/category graph (dense bidirectional
clusters → synonym groups; same-category-without-cross-links →
rivalry candidates).

66 tests passing.

aborist/qa/concepts.py

@@ -0,0 +1,109 @@
+"""Concept overlay: synonyms (broaden retrieval) + rivalries (narrow it).
+
+A small knowledge-graph layer over the corpus. Two structures:
+
+- SYNONYM_GROUPS — sets of tokens that retrieve interchangeably. Querying
+  for `athlon` should also pull `AMD`-titled docs because Athlon IS an
+  AMD product. Groups are unordered and case-insensitive at compare time.
+
+- RIVALRIES — pairs of group-indices that compete. If the query mentions
+  one side and not the other, docs whose titles contain the OTHER side's
+  tokens get filtered out (no Intel pages poisoning AMD answers). If the
+  query mentions BOTH sides — "AMD vs Intel", "compare AMD and Intel" —
+  no filtering: the user wants both sides.
+
+Phase 1: hand-curated. Phase 2 idea: derive from Wikipedia's category
+graph or from "See also" sections (articles that link bidirectionally
+in dense clusters → synonym group; articles in the same category that
+DON'T cross-link → potential rivalries).
+"""
+
+from __future__ import annotations
+
+
+# Each group is a frozenset of lowercased tokens. Title-token overlap with
+# any element promotes the doc as topically relevant.
+SYNONYM_GROUPS: list[frozenset[str]] = [
+    # AMD CPU family
+    frozenset({
+        "amd", "athlon", "duron", "opteron", "ryzen", "epyc",
+        "thunderbird", "palomino", "thoroughbred", "barton",
+        "k6", "k7", "k8", "k10",
+        "5x86",  # AMD 5x86
+    }),
+    # Intel CPU family
+    frozenset({
+        "intel", "pentium", "celeron", "xeon", "itanium",
+        "i7", "i5", "i3", "i9",
+        "core2", "coreduo", "skylake", "haswell", "ivy",
+        "8086", "80286", "80386", "80486",
+    }),
+    # HTTP / web protocol family
+    frozenset({"http", "https", "hypertext", "rfc2068", "rfc2616"}),
+    # FTP / file-transfer protocol family (rivalry candidate vs HTTP for some queries)
+    frozenset({"ftp", "sftp", "ftps"}),
+    # Mac vs Windows family
+    frozenset({"macintosh", "mac", "macos", "osx", "apple"}),
+    frozenset({"windows", "microsoft", "win32", "winnt", "win9x"}),
+    # Linux family
+    frozenset({"linux", "gnu", "ubuntu", "debian", "fedora", "redhat", "kernel"}),
+]
+
+
+# Pairs of SYNONYM_GROUPS indices that compete. Bidirectional.
+RIVALRIES: list[tuple[int, int]] = [
+    (0, 1),  # AMD ↔ Intel
+    (4, 5),  # Mac ↔ Windows
+]
+
+
+# Tokens that, if present in the query, mean "the user wants both sides
+# of any rivalry shown" — comparative phrasing. When ANY of these appears,
+# rivalry exclusion is suppressed.
+COMPARE_WORDS: frozenset[str] = frozenset({
+    "vs", "versus", "compare", "compared", "comparison", "compares",
+    "between", "difference", "differences", "or", "either",
+})
+
+
+def synonym_expand(tokens: set[str]) -> set[str]:
+    """Add all synonym-group members for any token that hits a group."""
+    expanded = set(tokens)
+    for t in tokens:
+        for group in SYNONYM_GROUPS:
+            if t in group:
+                expanded |= group
+                break
+    return expanded
+
+
+def rivalry_excluded(tokens: set[str], compare_phrasing: bool = False) -> set[str]:
+    """Return tokens whose presence in a doc title means EXCLUDE that doc.
+
+    Logic: for each rivalry pair (A, B), if exactly ONE side is present
+    in the query AND no compare-phrasing was detected, exclude the OTHER
+    side's tokens. If both sides are present, or if the user used
+    comparison language, no exclusion (they wanted both).
+    """
+    if compare_phrasing:
+        return set()
+    excluded: set[str] = set()
+    for a_idx, b_idx in RIVALRIES:
+        a = SYNONYM_GROUPS[a_idx]
+        b = SYNONYM_GROUPS[b_idx]
+        a_in = bool(tokens & a)
+        b_in = bool(tokens & b)
+        if a_in and not b_in:
+            excluded |= b
+        elif b_in and not a_in:
+            excluded |= a
+    return excluded
+
+
+def has_compare_phrasing(question: str) -> bool:
+    """True if the question contains comparison language."""
+    lower = question.lower()
+    # Word-boundary check via simple split on non-alpha
+    import re
+    words = set(re.findall(r"[a-z]+", lower))
+    return bool(words & COMPARE_WORDS)

aborist/qa/query.py

@@ -40,6 +40,11 @@ from aborist import (
 )
 from aborist.merkle import MerkleTree
 from aborist.qa.client import ChatClient
+from aborist.qa.concepts import (
+    has_compare_phrasing,
+    rivalry_excluded,
+    synonym_expand,
+)
 from aborist.qa.keys import (
     cache_key,
     conversation_hash,
@@ -96,25 +101,29 @@ def _rerank_by_title(
 
 
 def _filter_by_title_relevance(hits: list, question: str) -> list:
-    """Hard-filter: when ANY hit's title overlaps the query, drop the rest.
+    """Concept-aware title filter:
+    - synonym expansion: query 'Athlon' also matches AMD-titled docs
+    - rivalry exclusion: query 'AMD' alone excludes Intel-titled docs;
+      'AMD vs Intel' suppresses the exclusion (compare phrasing)
+    - hard filter: only keep title-matchers when matchers exist
 
-    Reranking by score wasn't strong enough — Pentium_4 with score 7 still
+    No matchers (corpus has nothing topical) → keep the top one so the
-    ended up in the top-K context window for "fastest AMD CPU" and Hermes
+    LLM sees a source and can honestly say "I don't know."
-    conflated it as the AMD answer. Fix: if we have ANY title-matchers, use
-    only those. Off-topic articles never enter the context.
-
-    If zero hits match by title (corpus has nothing topical), keep the top
-    one so the LLM sees a single source and can honestly say "I don't know
-    based on these sources." Empty context produces fabrication.
     """
     qtokens = _title_query_tokens(question)
     if not qtokens:
         return hits
-    kept = [
+    accept = synonym_expand(qtokens)
-        h for h in hits
+    exclude = rivalry_excluded(qtokens, compare_phrasing=has_compare_phrasing(question))
-        if h.title
+    kept = []
-        and (qtokens & _title_query_tokens(h.title.replace("_", " ")))
+    for h in hits:
-    ]
+        if not h.title:
+            continue
+        ttokens = _title_query_tokens(h.title.replace("_", " "))
+        if exclude & ttokens:
+            continue  # rivalry: opposing-side title, drop it
+        if accept & ttokens:
+            kept.append(h)
     if not kept:
         return hits[:1] if hits else []
     return kept
@@ -180,6 +189,8 @@ def _search_corpus(
     out-ranks FTS5 body hits so the actual topic article rises to the top.
     """
     qtokens = _title_query_tokens(question)
+    # Synonym expansion: a query for "athlon" also fetches AMD-titled docs.
+    accept_tokens = synonym_expand(qtokens)
     paths: list[Path]
     if shards_dir is not None:
         paths = discover_shards(shards_dir)
@@ -204,12 +215,10 @@ def _search_corpus(
                         str(p.resolve()),
                     )
                 )
-            # Parallel title search. Score is overlap-weighted and
+            # Parallel title search using synonym-expanded tokens.
-            # baseline-elevated so an exact title match (HTTP, Stoicism)
+            for r in _search_titles(conn, list(accept_tokens), over_fetch):
-            # comes above body BM25.
-            for r in _search_titles(conn, qtokens, over_fetch):
                 title_lower = (r["title"] or "").lower().replace("_", " ")
-                overlap = sum(1 for t in qtokens if t.lower() in title_lower)
+                overlap = sum(1 for t in accept_tokens if t in title_lower)
                 if overlap == 0:
                     continue
                 title_score = 50.0 + overlap * 10.0

tests/test_concepts.py

@@ -0,0 +1,72 @@
+"""Concept overlay: synonym expansion + rivalry exclusion."""
+
+from __future__ import annotations
+
+from aborist.qa.concepts import (
+    has_compare_phrasing,
+    rivalry_excluded,
+    synonym_expand,
+)
+
+
+def test_synonym_expand_amd_pulls_athlon_and_back():
+    expanded = synonym_expand({"athlon"})
+    assert "amd" in expanded
+    assert "duron" in expanded
+    assert "thunderbird" in expanded
+
+    expanded = synonym_expand({"amd"})
+    assert "athlon" in expanded
+
+
+def test_synonym_expand_unrelated_token_unchanged():
+    expanded = synonym_expand({"banana"})
+    assert expanded == {"banana"}
+
+
+def test_synonym_expand_doesnt_cross_groups():
+    """AMD and Intel are in different groups — expanding one shouldn't
+    pull in the other."""
+    expanded = synonym_expand({"amd"})
+    assert "intel" not in expanded
+    assert "pentium" not in expanded
+
+
+def test_rivalry_excluded_amd_query_drops_intel():
+    excluded = rivalry_excluded({"amd"}, compare_phrasing=False)
+    assert "intel" in excluded
+    assert "pentium" in excluded
+
+
+def test_rivalry_excluded_intel_query_drops_amd():
+    excluded = rivalry_excluded({"intel"}, compare_phrasing=False)
+    assert "amd" in excluded
+    assert "athlon" in excluded
+
+
+def test_rivalry_excluded_both_sides_no_exclusion():
+    excluded = rivalry_excluded({"amd", "intel"}, compare_phrasing=False)
+    assert excluded == set()
+
+
+def test_rivalry_excluded_compare_phrasing_suppresses():
+    excluded = rivalry_excluded({"amd"}, compare_phrasing=True)
+    assert excluded == set()
+
+
+def test_compare_phrasing_detection():
+    assert has_compare_phrasing("compare AMD and Intel")
+    assert has_compare_phrasing("AMD vs Intel")
+    assert has_compare_phrasing("difference between Mac and Windows")
+    assert not has_compare_phrasing("what is the fastest AMD CPU?")
+    assert not has_compare_phrasing("tell me about Athlon")
+
+
+def test_mac_windows_rivalry():
+    """Independent rivalry pair — Mac vs Windows."""
+    excluded = rivalry_excluded({"macintosh"}, compare_phrasing=False)
+    assert "windows" in excluded
+
+    excluded = rivalry_excluded({"windows"}, compare_phrasing=False)
+    assert "macintosh" in excluded
+    assert "macos" in excluded