diff --git a/aborist/concepts/query.py b/aborist/concepts/query.py index 4d7f841..e367244 100644 --- a/aborist/concepts/query.py +++ b/aborist/concepts/query.py @@ -65,17 +65,23 @@ def _shards_mtime_signature(shards_dir: Path) -> tuple: def _load_indices(shards_dir: Path) -> tuple[dict, list]: """Walk all shards, materialize the synonym & rivalry indices. - Synonyms collapse into transitive groups via union-find: if (A,B) and - (B,C) are both synonym edges (regardless of source_root or evidence - kind), A↔B↔C form one equivalence class. The index maps each token to - the full closure of tokens it retrieves with. + Synonyms map every token to its **direct neighbors only**, NOT the + transitive closure across reciprocal-link chains. Why: union-find + over the full Wikipedia reciprocal-link graph collapses everything + into one giant connected component (54k+ tokens for any seed in + a 4-shard corpus). That made one query token expand to the entire + synonym alphabet, which (a) overwhelmed retrieval relevance and + (b) tripped SQLite's expression-tree-depth limit when ``_search_titles`` + built a per-token OR clause. Direct-neighbor expansion preserves the + legacy frozenset semantics (every member of a group expanded to the + others in that group, but the groups didn't chain). Rivalry pairs stay as discrete pair objects: each row's (token, target) becomes a 2-element pair stored as (frozenset({token}), frozenset({target})). - The legacy semantics (group-vs-group rivalry) is reachable by manually - adding many synonym rows on each side & a single rivalry row connecting - them. Lookup-time exclusion picks up any rivalry whose either side - intersects the query. + The legacy semantics (group-vs-group rivalry) is reachable by storing + one rivalry row per (token_a_in_group_A, token_b_in_group_B) cross- + product OR by relying on the lookup-time intersection check picking + up any pair whose either side intersects the query. """ conn = connect_query(shards_dir=shards_dir) rows = conn.execute( @@ -83,50 +89,29 @@ def _load_indices(shards_dir: Path) -> tuple[dict, list]: ).fetchall() conn.close() - # Synonym groups via union-find. - parent: dict[str, str] = {} - - def find(x: str) -> str: - while parent.get(x, x) != x: - parent[x] = parent.get(parent[x], parent[x]) - x = parent[x] - parent.setdefault(x, x) - return x - - def union(a: str, b: str) -> None: - ra, rb = find(a), find(b) - if ra != rb: - parent[ra] = rb - + # Direct-neighbor adjacency: token -> set of immediate synonym + # partners. Symmetric (an edge (A,B) registers both A→B and B→A). + synonym_index: dict[str, set[str]] = {} rivalry_rows: list[tuple[str, str]] = [] for r in rows: kind = r["relation_kind"] a = (r["token"] or "").lower() b = (r["target"] or "").lower() - if not a or not b: + if not a or not b or a == b: continue if kind == "synonym": - union(a, b) + synonym_index.setdefault(a, set()).add(b) + synonym_index.setdefault(b, set()).add(a) elif kind == "rivalry": rivalry_rows.append((a, b)) - # Materialize closure map: token -> set of all tokens in its group. - groups: dict[str, set[str]] = {} - for tok in list(parent.keys()): - root = find(tok) - groups.setdefault(root, set()).add(tok) - synonym_index: dict[str, set[str]] = {} - for members in groups.values(): - for m in members: - synonym_index[m] = members - - # Rivalry pairs: convert to (group_a, group_b) using synonym closure - # so a single rivalry edge between A & B captures A's whole synonym - # cluster vs B's whole cluster. + # Rivalry pairs use the same direct-neighbor semantic: a rivalry edge + # (A, B) captures A's direct neighbors (incl. A itself) vs B's direct + # neighbors (incl. B itself). Bounded scope; matches synonym semantics. rivalry_pairs: list[tuple[frozenset[str], frozenset[str]]] = [] for a, b in rivalry_rows: - ga = frozenset(synonym_index.get(a, {a})) - gb = frozenset(synonym_index.get(b, {b})) + ga = frozenset(synonym_index.get(a, set()) | {a}) + gb = frozenset(synonym_index.get(b, set()) | {b}) rivalry_pairs.append((ga, gb)) return synonym_index, rivalry_pairs