russell@unturf.com
c6182ae54e
A small knowledge-graph layer that does two distinct jobs:
1. SYNONYM_GROUPS broaden retrieval. A query mentioning "Athlon"
now also matches AMD-titled docs because Athlon IS an AMD product.
Groups currently cover AMD-family, Intel-family, HTTP family,
FTP, Mac, Windows, Linux. Easy to extend.
2. RIVALRIES narrow retrieval. The pair (AMD-group, Intel-group)
means: if the query mentions one side and not the other, drop
docs whose titles contain the OTHER side's tokens. So a "fastest
AMD CPU" question never gets Pentium_4 in the context — even if
FTS5 BM25 ranks it high — because Pentium is in the Intel group
and Intel isn't in the query.
COMPARE_WORDS ("vs", "versus", "compare", "between", ...) suppress
the exclusion. "compare AMD vs Intel" keeps both sides. "what is
the fastest AMD CPU?" does not.
Demos against the 128k Wikipedia 2003-05-16 cur shards:
Q "fastest AMD CPU" → 8 sources, all AMD/CPU titled, NO Intel
Answer: "Athlon XP 3200+" (real AMD chip,
grounded in the AMD article)
Q "compare AMD vs Intel"
→ 8 sources, mix of Intel_8028x, Intel_8048x,
AMD_Duron — both sides preserved
Q "Athlon processor" → AMD, AMD_Duron, AMD_5x86, Athlon all
surfaced via synonym expansion
Phase 1 implementation hand-curates the groups; Phase 2 idea is to
derive them from Wikipedia's link/category graph (dense bidirectional
clusters → synonym groups; same-category-without-cross-links →
rivalry candidates).
66 tests passing.
109 lines
4 KiB
Python
109 lines
4 KiB
Python
"""Concept overlay: synonyms (broaden retrieval) + rivalries (narrow it).
|
|
|
|
A small knowledge-graph layer over the corpus. Two structures:
|
|
|
|
- SYNONYM_GROUPS — sets of tokens that retrieve interchangeably. Querying
|
|
for `athlon` should also pull `AMD`-titled docs because Athlon IS an
|
|
AMD product. Groups are unordered and case-insensitive at compare time.
|
|
|
|
- RIVALRIES — pairs of group-indices that compete. If the query mentions
|
|
one side and not the other, docs whose titles contain the OTHER side's
|
|
tokens get filtered out (no Intel pages poisoning AMD answers). If the
|
|
query mentions BOTH sides — "AMD vs Intel", "compare AMD and Intel" —
|
|
no filtering: the user wants both sides.
|
|
|
|
Phase 1: hand-curated. Phase 2 idea: derive from Wikipedia's category
|
|
graph or from "See also" sections (articles that link bidirectionally
|
|
in dense clusters → synonym group; articles in the same category that
|
|
DON'T cross-link → potential rivalries).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
|
|
# Each group is a frozenset of lowercased tokens. Title-token overlap with
|
|
# any element promotes the doc as topically relevant.
|
|
SYNONYM_GROUPS: list[frozenset[str]] = [
|
|
# AMD CPU family
|
|
frozenset({
|
|
"amd", "athlon", "duron", "opteron", "ryzen", "epyc",
|
|
"thunderbird", "palomino", "thoroughbred", "barton",
|
|
"k6", "k7", "k8", "k10",
|
|
"5x86", # AMD 5x86
|
|
}),
|
|
# Intel CPU family
|
|
frozenset({
|
|
"intel", "pentium", "celeron", "xeon", "itanium",
|
|
"i7", "i5", "i3", "i9",
|
|
"core2", "coreduo", "skylake", "haswell", "ivy",
|
|
"8086", "80286", "80386", "80486",
|
|
}),
|
|
# HTTP / web protocol family
|
|
frozenset({"http", "https", "hypertext", "rfc2068", "rfc2616"}),
|
|
# FTP / file-transfer protocol family (rivalry candidate vs HTTP for some queries)
|
|
frozenset({"ftp", "sftp", "ftps"}),
|
|
# Mac vs Windows family
|
|
frozenset({"macintosh", "mac", "macos", "osx", "apple"}),
|
|
frozenset({"windows", "microsoft", "win32", "winnt", "win9x"}),
|
|
# Linux family
|
|
frozenset({"linux", "gnu", "ubuntu", "debian", "fedora", "redhat", "kernel"}),
|
|
]
|
|
|
|
|
|
# Pairs of SYNONYM_GROUPS indices that compete. Bidirectional.
|
|
RIVALRIES: list[tuple[int, int]] = [
|
|
(0, 1), # AMD ↔ Intel
|
|
(4, 5), # Mac ↔ Windows
|
|
]
|
|
|
|
|
|
# Tokens that, if present in the query, mean "the user wants both sides
|
|
# of any rivalry shown" — comparative phrasing. When ANY of these appears,
|
|
# rivalry exclusion is suppressed.
|
|
COMPARE_WORDS: frozenset[str] = frozenset({
|
|
"vs", "versus", "compare", "compared", "comparison", "compares",
|
|
"between", "difference", "differences", "or", "either",
|
|
})
|
|
|
|
|
|
def synonym_expand(tokens: set[str]) -> set[str]:
|
|
"""Add all synonym-group members for any token that hits a group."""
|
|
expanded = set(tokens)
|
|
for t in tokens:
|
|
for group in SYNONYM_GROUPS:
|
|
if t in group:
|
|
expanded |= group
|
|
break
|
|
return expanded
|
|
|
|
|
|
def rivalry_excluded(tokens: set[str], compare_phrasing: bool = False) -> set[str]:
|
|
"""Return tokens whose presence in a doc title means EXCLUDE that doc.
|
|
|
|
Logic: for each rivalry pair (A, B), if exactly ONE side is present
|
|
in the query AND no compare-phrasing was detected, exclude the OTHER
|
|
side's tokens. If both sides are present, or if the user used
|
|
comparison language, no exclusion (they wanted both).
|
|
"""
|
|
if compare_phrasing:
|
|
return set()
|
|
excluded: set[str] = set()
|
|
for a_idx, b_idx in RIVALRIES:
|
|
a = SYNONYM_GROUPS[a_idx]
|
|
b = SYNONYM_GROUPS[b_idx]
|
|
a_in = bool(tokens & a)
|
|
b_in = bool(tokens & b)
|
|
if a_in and not b_in:
|
|
excluded |= b
|
|
elif b_in and not a_in:
|
|
excluded |= a
|
|
return excluded
|
|
|
|
|
|
def has_compare_phrasing(question: str) -> bool:
|
|
"""True if the question contains comparison language."""
|
|
lower = question.lower()
|
|
# Word-boundary check via simple split on non-alpha
|
|
import re
|
|
words = set(re.findall(r"[a-z]+", lower))
|
|
return bool(words & COMPARE_WORDS)
|