russell@unturf.com
c6182ae54e
A small knowledge-graph layer that does two distinct jobs:
1. SYNONYM_GROUPS broaden retrieval. A query mentioning "Athlon"
now also matches AMD-titled docs because Athlon IS an AMD product.
Groups currently cover AMD-family, Intel-family, HTTP family,
FTP, Mac, Windows, Linux. Easy to extend.
2. RIVALRIES narrow retrieval. The pair (AMD-group, Intel-group)
means: if the query mentions one side and not the other, drop
docs whose titles contain the OTHER side's tokens. So a "fastest
AMD CPU" question never gets Pentium_4 in the context — even if
FTS5 BM25 ranks it high — because Pentium is in the Intel group
and Intel isn't in the query.
COMPARE_WORDS ("vs", "versus", "compare", "between", ...) suppress
the exclusion. "compare AMD vs Intel" keeps both sides. "what is
the fastest AMD CPU?" does not.
Demos against the 128k Wikipedia 2003-05-16 cur shards:
Q "fastest AMD CPU" → 8 sources, all AMD/CPU titled, NO Intel
Answer: "Athlon XP 3200+" (real AMD chip,
grounded in the AMD article)
Q "compare AMD vs Intel"
→ 8 sources, mix of Intel_8028x, Intel_8048x,
AMD_Duron — both sides preserved
Q "Athlon processor" → AMD, AMD_Duron, AMD_5x86, Athlon all
surfaced via synonym expansion
Phase 1 implementation hand-curates the groups; Phase 2 idea is to
derive them from Wikipedia's link/category graph (dense bidirectional
clusters → synonym groups; same-category-without-cross-links →
rivalry candidates).
66 tests passing.
72 lines
2.2 KiB
Python
72 lines
2.2 KiB
Python
"""Concept overlay: synonym expansion + rivalry exclusion."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from aborist.qa.concepts import (
|
|
has_compare_phrasing,
|
|
rivalry_excluded,
|
|
synonym_expand,
|
|
)
|
|
|
|
|
|
def test_synonym_expand_amd_pulls_athlon_and_back():
|
|
expanded = synonym_expand({"athlon"})
|
|
assert "amd" in expanded
|
|
assert "duron" in expanded
|
|
assert "thunderbird" in expanded
|
|
|
|
expanded = synonym_expand({"amd"})
|
|
assert "athlon" in expanded
|
|
|
|
|
|
def test_synonym_expand_unrelated_token_unchanged():
|
|
expanded = synonym_expand({"banana"})
|
|
assert expanded == {"banana"}
|
|
|
|
|
|
def test_synonym_expand_doesnt_cross_groups():
|
|
"""AMD and Intel are in different groups — expanding one shouldn't
|
|
pull in the other."""
|
|
expanded = synonym_expand({"amd"})
|
|
assert "intel" not in expanded
|
|
assert "pentium" not in expanded
|
|
|
|
|
|
def test_rivalry_excluded_amd_query_drops_intel():
|
|
excluded = rivalry_excluded({"amd"}, compare_phrasing=False)
|
|
assert "intel" in excluded
|
|
assert "pentium" in excluded
|
|
|
|
|
|
def test_rivalry_excluded_intel_query_drops_amd():
|
|
excluded = rivalry_excluded({"intel"}, compare_phrasing=False)
|
|
assert "amd" in excluded
|
|
assert "athlon" in excluded
|
|
|
|
|
|
def test_rivalry_excluded_both_sides_no_exclusion():
|
|
excluded = rivalry_excluded({"amd", "intel"}, compare_phrasing=False)
|
|
assert excluded == set()
|
|
|
|
|
|
def test_rivalry_excluded_compare_phrasing_suppresses():
|
|
excluded = rivalry_excluded({"amd"}, compare_phrasing=True)
|
|
assert excluded == set()
|
|
|
|
|
|
def test_compare_phrasing_detection():
|
|
assert has_compare_phrasing("compare AMD and Intel")
|
|
assert has_compare_phrasing("AMD vs Intel")
|
|
assert has_compare_phrasing("difference between Mac and Windows")
|
|
assert not has_compare_phrasing("what is the fastest AMD CPU?")
|
|
assert not has_compare_phrasing("tell me about Athlon")
|
|
|
|
|
|
def test_mac_windows_rivalry():
|
|
"""Independent rivalry pair — Mac vs Windows."""
|
|
excluded = rivalry_excluded({"macintosh"}, compare_phrasing=False)
|
|
assert "windows" in excluded
|
|
|
|
excluded = rivalry_excluded({"windows"}, compare_phrasing=False)
|
|
assert "macintosh" in excluded
|
|
assert "macos" in excluded
|