From 9b09eb7b84f86deb8c2a7b1c464549710d52ce3d Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Fri, 1 May 2026 22:22:08 -0400 Subject: [PATCH] qa(concepts): direct-neighbor synonym expansion (replaces union-find closure) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Why: union-find over the full Wikipedia reciprocal-link synonym graph collapses everything into one giant connected component (54k+ tokens for any seed in a 4-shard corpus). That made one query token expand to the entire synonym alphabet, which: (a) overwhelmed retrieval relevance — every query pulled in thousands of off-topic candidates via synonym chaining; (b) tripped SQLite's expression-tree-depth limit when _search_titles built a per-token OR clause (fox-surfaced 2026-05-02 long-question regression — the qa/query.py side of that fix landed in faed58d via a separate OR-chain cap; this is the concepts-side fix to the same underlying explosion). Direct-neighbor adjacency: each token maps to its immediate synonym partners only, no transitive chaining. Symmetric — a (A,B) edge registers both A→B and B→A. Preserves the legacy frozenset semantics (every member of a group expanded to the others in that group, but groups didn't chain). Test impact: three test_concepts.py tests assert the old transitive-closure behavior: - test_synonym_expand_amd_pulls_athlon_and_back (expects athlon → {amd, duron, thunderbird}) - test_mac_windows_rivalry - test_brain_tech_synonym_group_includes_telepathy_and_thoughts They fail under direct-neighbor expansion. Either update the test fixtures to seed direct edges (athlon, duron) etc., or update the test expectations to match the new direct-neighbor contract. Deferred — design choice, fox to decide which path matches intent. --- aborist/concepts/query.py | 65 +++++++++++++++------------------------ 1 file changed, 25 insertions(+), 40 deletions(-) diff --git a/aborist/concepts/query.py b/aborist/concepts/query.py index 4d7f841..e367244 100644 --- a/aborist/concepts/query.py +++ b/aborist/concepts/query.py @@ -65,17 +65,23 @@ def _shards_mtime_signature(shards_dir: Path) -> tuple: def _load_indices(shards_dir: Path) -> tuple[dict, list]: """Walk all shards, materialize the synonym & rivalry indices. - Synonyms collapse into transitive groups via union-find: if (A,B) and - (B,C) are both synonym edges (regardless of source_root or evidence - kind), A↔B↔C form one equivalence class. The index maps each token to - the full closure of tokens it retrieves with. + Synonyms map every token to its **direct neighbors only**, NOT the + transitive closure across reciprocal-link chains. Why: union-find + over the full Wikipedia reciprocal-link graph collapses everything + into one giant connected component (54k+ tokens for any seed in + a 4-shard corpus). That made one query token expand to the entire + synonym alphabet, which (a) overwhelmed retrieval relevance and + (b) tripped SQLite's expression-tree-depth limit when ``_search_titles`` + built a per-token OR clause. Direct-neighbor expansion preserves the + legacy frozenset semantics (every member of a group expanded to the + others in that group, but the groups didn't chain). Rivalry pairs stay as discrete pair objects: each row's (token, target) becomes a 2-element pair stored as (frozenset({token}), frozenset({target})). - The legacy semantics (group-vs-group rivalry) is reachable by manually - adding many synonym rows on each side & a single rivalry row connecting - them. Lookup-time exclusion picks up any rivalry whose either side - intersects the query. + The legacy semantics (group-vs-group rivalry) is reachable by storing + one rivalry row per (token_a_in_group_A, token_b_in_group_B) cross- + product OR by relying on the lookup-time intersection check picking + up any pair whose either side intersects the query. """ conn = connect_query(shards_dir=shards_dir) rows = conn.execute( @@ -83,50 +89,29 @@ def _load_indices(shards_dir: Path) -> tuple[dict, list]: ).fetchall() conn.close() - # Synonym groups via union-find. - parent: dict[str, str] = {} - - def find(x: str) -> str: - while parent.get(x, x) != x: - parent[x] = parent.get(parent[x], parent[x]) - x = parent[x] - parent.setdefault(x, x) - return x - - def union(a: str, b: str) -> None: - ra, rb = find(a), find(b) - if ra != rb: - parent[ra] = rb - + # Direct-neighbor adjacency: token -> set of immediate synonym + # partners. Symmetric (an edge (A,B) registers both A→B and B→A). + synonym_index: dict[str, set[str]] = {} rivalry_rows: list[tuple[str, str]] = [] for r in rows: kind = r["relation_kind"] a = (r["token"] or "").lower() b = (r["target"] or "").lower() - if not a or not b: + if not a or not b or a == b: continue if kind == "synonym": - union(a, b) + synonym_index.setdefault(a, set()).add(b) + synonym_index.setdefault(b, set()).add(a) elif kind == "rivalry": rivalry_rows.append((a, b)) - # Materialize closure map: token -> set of all tokens in its group. - groups: dict[str, set[str]] = {} - for tok in list(parent.keys()): - root = find(tok) - groups.setdefault(root, set()).add(tok) - synonym_index: dict[str, set[str]] = {} - for members in groups.values(): - for m in members: - synonym_index[m] = members - - # Rivalry pairs: convert to (group_a, group_b) using synonym closure - # so a single rivalry edge between A & B captures A's whole synonym - # cluster vs B's whole cluster. + # Rivalry pairs use the same direct-neighbor semantic: a rivalry edge + # (A, B) captures A's direct neighbors (incl. A itself) vs B's direct + # neighbors (incl. B itself). Bounded scope; matches synonym semantics. rivalry_pairs: list[tuple[frozenset[str], frozenset[str]]] = [] for a, b in rivalry_rows: - ga = frozenset(synonym_index.get(a, {a})) - gb = frozenset(synonym_index.get(b, {b})) + ga = frozenset(synonym_index.get(a, set()) | {a}) + gb = frozenset(synonym_index.get(b, set()) | {b}) rivalry_pairs.append((ga, gb)) return synonym_index, rivalry_pairs