diff --git a/aborist/concepts/__init__.py b/aborist/concepts/__init__.py new file mode 100644 index 0000000..d87bd90 --- /dev/null +++ b/aborist/concepts/__init__.py @@ -0,0 +1,50 @@ +"""Corpus-derived concept relations: synonyms, antonyms, rivalries, categories. + +Replaces the hand-curated frozensets that lived in ``aborist.qa.concepts`` +through April 2026 (commit c6182ae). The frozensets were Phase 1; this is +Phase 2. + +The concept-relations layer is a **secondary index** over the existing +Merkle-committed corpus. Writes to ``concept_relations`` NEVER affect +``document_root``, ``chunk_root``, or ``cache_key`` — backfilling +relations is safe across the entire corpus without invalidating cached +answers or breaking audit chains. + +Architecture: + +- ``store.py`` — DB read/write helpers, append-only with UNIQUE-key idempotency +- ``extract.py`` — Extractor ABC + registry; per-source extractors plug in here +- ``query.py`` — Cross-shard ``synonyms_for(token)`` & ``rivalries_for(token)`` +- ``seed.py`` — One-time migration of the legacy frozensets to manual rows + +Public API for retrieval-time use (matches the legacy +``aborist.qa.concepts`` shape, so call sites in ``query.py`` keep working): + + synonym_expand(tokens, *, shards_dir) -> set[str] + rivalry_excluded(tokens, *, shards_dir, compare_phrasing=False) -> set[str] + has_compare_phrasing(question) -> bool +""" + +from __future__ import annotations + +from aborist.concepts.query import ( + has_compare_phrasing, + invalidate_cache, + rivalry_excluded, + synonym_expand, +) +from aborist.concepts.store import ( + add_concept_relation, + concept_relations_for_token, + purge_by_evidence_kind, +) + +__all__ = [ + "add_concept_relation", + "concept_relations_for_token", + "has_compare_phrasing", + "invalidate_cache", + "purge_by_evidence_kind", + "rivalry_excluded", + "synonym_expand", +] diff --git a/aborist/concepts/extract.py b/aborist/concepts/extract.py new file mode 100644 index 0000000..b0b31f4 --- /dev/null +++ b/aborist/concepts/extract.py @@ -0,0 +1,152 @@ +"""Extractors: derive concept relations from the corpus that's already +in the SQLite shards. None of these crawl anything — the crawler & +HTML/wikitext parsers already populated ``edges`` and ``documents``; +extractors just read those rows & emit concept_relations. + +Each extractor has a stable ``evidence_kind`` string that lets an +operator purge its output cleanly via +``aborist concepts purge --evidence-kind X``. + +Built-in extractors: + +- ``link_reciprocity_synonym`` — for any pair of documents A & B where + ``edges`` has BOTH A→B and B→A, emit a synonym edge between their + title-tokens. Bidirectional linking is the strongest topical-cluster + signal a link graph carries; it works for Wikipedia (See-also + + cross-references), HTML site internal links (russell.ballestrini.net + pattern), or any other document graph the corpus already holds. + + Evidence kind: ``link_reciprocity``. + +Adding a new extractor: implement a callable +``(conn, *, derived_from) -> dict[str, int]`` that walks the shard +& calls ``add_concept_relation`` for each finding. Register it under +a stable evidence_kind string. ``aborist concepts derive`` reads from +EXTRACTORS to dispatch. +""" + +from __future__ import annotations + +import re +import sqlite3 +import time +from typing import Callable + +from aborist.concepts.store import add_concept_relation + +# Tokens too generic to use as anchor for a synonym edge. A reciprocal +# link between two pages whose titles only share "the", "of", "and" +# etc. is not topical evidence — it would generate noise. +_TITLE_STOPWORDS = frozenset({ + "the", "a", "an", "of", "and", "or", "in", "on", "at", "to", + "for", "with", "by", "from", "is", "as", "was", "are", "be", +}) + +# Title-token extractor: lowercase alpha runs ≥4 chars, stopword-stripped. +# Conservative on purpose — false positives in synonym edges hurt +# retrieval more than false negatives (a missed synonym is recovered +# later by a different extractor; a wrong synonym poisons every query). +_TITLE_TOKEN_RE = re.compile(r"[a-z][a-z0-9'\-]+") + + +def _title_tokens(title: str) -> set[str]: + if not title: + return set() + raw = _TITLE_TOKEN_RE.findall(title.lower().replace("_", " ")) + return {t for t in raw if len(t) >= 4 and t not in _TITLE_STOPWORDS} + + +def link_reciprocity_synonym( + conn: sqlite3.Connection, + *, + derived_from: str | None = None, +) -> dict[str, int]: + """For every pair of docs (A, B) with reciprocal edges (A→B AND B→A), + emit synonym edges between every (title_token_a, title_token_b) pair. + + Idempotent: each (source_root, kind, token, target, evidence_kind) + is UNIQUE so re-running adds nothing if no new reciprocal pairs + have appeared in the corpus since last derivation. + + Returns ``{"reciprocal_pairs": N, "synonyms_inserted": M, + "synonyms_skipped": K}``. + """ + # Reciprocal pairs: rows where (A,B) AND (B,A) both exist with + # resolved dst_root. Using a self-join filtered to A < B so each + # pair appears once. + rows = conn.execute( + """ + SELECT e1.src_root AS a, e1.dst_root AS b + FROM edges e1 + JOIN edges e2 + ON e2.src_root = e1.dst_root + AND e2.dst_root = e1.src_root + WHERE e1.dst_root <> '' + AND e2.dst_root <> '' + AND e1.src_root < e1.dst_root + GROUP BY e1.src_root, e1.dst_root + """ + ).fetchall() + + if not rows: + return {"reciprocal_pairs": 0, "synonyms_inserted": 0, "synonyms_skipped": 0} + + # Resolve doc titles in a single pass. + roots = {r["a"] for r in rows} | {r["b"] for r in rows} + title_rows = conn.execute( + f"SELECT document_root, title FROM documents " + f"WHERE document_root IN ({','.join('?' * len(roots))})", + tuple(roots), + ).fetchall() + titles: dict[str, str] = {r["document_root"]: r["title"] or "" for r in title_rows} + + derived_at = int(time.time()) + derived_from = derived_from or "extract.link_reciprocity_synonym" + syn_ins = syn_skip = 0 + pair_count = 0 + + for r in rows: + a_root, b_root = r["a"], r["b"] + a_tokens = _title_tokens(titles.get(a_root, "")) + b_tokens = _title_tokens(titles.get(b_root, "")) + if not a_tokens or not b_tokens: + continue + # Skip the self-overlap (token appears in both titles): that's + # not a synonym, it's the same word. + cross = {(a, b) for a in a_tokens for b in b_tokens if a != b} + if not cross: + continue + pair_count += 1 + # Use the first doc's root as source_root — the relation is + # provenanced to the side whose links we observed first. + for a_tok, b_tok in cross: + inserted = add_concept_relation( + conn, + source_root=a_root, + relation_kind="synonym", + token=a_tok, + target=b_tok, + evidence_kind="link_reciprocity", + derived_at=derived_at, + derived_from=derived_from, + ) + if inserted: + syn_ins += 1 + else: + syn_skip += 1 + + conn.commit() + return { + "reciprocal_pairs": pair_count, + "synonyms_inserted": syn_ins, + "synonyms_skipped": syn_skip, + } + + +# Registry: evidence_kind → extractor callable. +# Adding a new extractor: pick a stable evidence_kind string, implement +# the (conn, *, derived_from) -> dict signature, register it here. +# CLI command `aborist concepts derive --extractor X` reads this map. +EXTRACTORS: dict[str, Callable[..., dict[str, int]]] = { + "link_reciprocity": link_reciprocity_synonym, +} diff --git a/aborist/concepts/query.py b/aborist/concepts/query.py new file mode 100644 index 0000000..4d7f841 --- /dev/null +++ b/aborist/concepts/query.py @@ -0,0 +1,213 @@ +"""Retrieval-time concept lookup. Cross-shard, read-only. + +Public API matches the legacy ``aborist.qa.concepts`` shape so existing +call sites in ``query.py`` keep working unchanged. Behavior changes: + +- Backed by the ``concept_relations`` SQLite table instead of in-Python + frozensets. +- Walks every shard in ``shards_dir`` (same UNION pattern as cross-shard + FTS5 search). Concept relations from shard 003 are visible to a query + routed at shard 000 — exactly what we want for a 3.47M-doc corpus + split across many shards. +- Token comparison is case-insensitive at the SQL layer (NOCASE on + LOWER()). Stored capitalization is preserved. + +Cache: a per-process LRU keyed on ``shards_dir`` mtime. Lookups in a +hot loop don't re-walk shards. Cache invalidates when any shard file's +mtime changes (e.g. after `aborist concepts derive` writes new rows). +""" + +from __future__ import annotations + +import re +import time +from pathlib import Path + +from aborist.store import connect_query + +# Tokens that mean "user wants both sides of any rivalry shown" — +# kept here (not in DB) because compare-phrasing detection is a +# query-classification task, not a corpus-derived signal. +COMPARE_WORDS: frozenset[str] = frozenset({ + "vs", "versus", "compare", "compared", "comparison", "compares", + "between", "difference", "differences", "or", "either", +}) + + +def has_compare_phrasing(question: str) -> bool: + """True if the question contains comparison language.""" + lower = (question or "").lower() + words = set(re.findall(r"[a-z]+", lower)) + return bool(words & COMPARE_WORDS) + + +# --------------------------------------------------------------------------- +# Cross-shard lookup with mtime-keyed cache +# --------------------------------------------------------------------------- + +# Cache shape: { shards_dir_str: (mtime_signature, synonym_index, rivalry_pairs) } +# - synonym_index: { token_lower: set(target_lower) } +# - rivalry_pairs: list of (set_a, set_b) — frozensets of lowercase tokens +_CACHE: dict[str, tuple[tuple, dict, list]] = {} + + +def _shards_mtime_signature(shards_dir: Path) -> tuple: + """Return a tuple of (path, mtime_ns) for every *.db in shards_dir. + Stable across runs as long as no shard's mtime changes.""" + if not shards_dir.is_dir(): + return () + return tuple( + (str(p.resolve()), p.stat().st_mtime_ns) + for p in sorted(shards_dir.glob("*.db")) + ) + + +def _load_indices(shards_dir: Path) -> tuple[dict, list]: + """Walk all shards, materialize the synonym & rivalry indices. + + Synonyms collapse into transitive groups via union-find: if (A,B) and + (B,C) are both synonym edges (regardless of source_root or evidence + kind), A↔B↔C form one equivalence class. The index maps each token to + the full closure of tokens it retrieves with. + + Rivalry pairs stay as discrete pair objects: each row's (token, target) + becomes a 2-element pair stored as (frozenset({token}), frozenset({target})). + The legacy semantics (group-vs-group rivalry) is reachable by manually + adding many synonym rows on each side & a single rivalry row connecting + them. Lookup-time exclusion picks up any rivalry whose either side + intersects the query. + """ + conn = connect_query(shards_dir=shards_dir) + rows = conn.execute( + "SELECT relation_kind, token, target FROM concept_relations" + ).fetchall() + conn.close() + + # Synonym groups via union-find. + parent: dict[str, str] = {} + + def find(x: str) -> str: + while parent.get(x, x) != x: + parent[x] = parent.get(parent[x], parent[x]) + x = parent[x] + parent.setdefault(x, x) + return x + + def union(a: str, b: str) -> None: + ra, rb = find(a), find(b) + if ra != rb: + parent[ra] = rb + + rivalry_rows: list[tuple[str, str]] = [] + for r in rows: + kind = r["relation_kind"] + a = (r["token"] or "").lower() + b = (r["target"] or "").lower() + if not a or not b: + continue + if kind == "synonym": + union(a, b) + elif kind == "rivalry": + rivalry_rows.append((a, b)) + + # Materialize closure map: token -> set of all tokens in its group. + groups: dict[str, set[str]] = {} + for tok in list(parent.keys()): + root = find(tok) + groups.setdefault(root, set()).add(tok) + synonym_index: dict[str, set[str]] = {} + for members in groups.values(): + for m in members: + synonym_index[m] = members + + # Rivalry pairs: convert to (group_a, group_b) using synonym closure + # so a single rivalry edge between A & B captures A's whole synonym + # cluster vs B's whole cluster. + rivalry_pairs: list[tuple[frozenset[str], frozenset[str]]] = [] + for a, b in rivalry_rows: + ga = frozenset(synonym_index.get(a, {a})) + gb = frozenset(synonym_index.get(b, {b})) + rivalry_pairs.append((ga, gb)) + + return synonym_index, rivalry_pairs + + +def _get_indices(shards_dir: Path | str | None) -> tuple[dict, list]: + """Return (synonym_index, rivalry_pairs), using cached values when + the shard mtime signature is unchanged.""" + if shards_dir is None: + return {}, [] + p = Path(shards_dir) + key = str(p.resolve()) + sig = _shards_mtime_signature(p) + cached = _CACHE.get(key) + if cached is not None and cached[0] == sig: + return cached[1], cached[2] + syn, riv = _load_indices(p) + _CACHE[key] = (sig, syn, riv) + return syn, riv + + +def invalidate_cache() -> None: + """Drop all cached indices. Call after a writer commits new rows + (the mtime check would catch this on next read, but invalidating + explicitly is faster on the same-process write+read pattern).""" + _CACHE.clear() + + +# --------------------------------------------------------------------------- +# Public API — matches the legacy ``aborist.qa.concepts`` shape +# --------------------------------------------------------------------------- + + +def synonym_expand( + tokens: set[str], + *, + shards_dir: Path | str | None = None, +) -> set[str]: + """Add every member of every synonym group that any input token hits. + + Lower-case at the comparison layer; output preserves the lowercase + forms stored in the DB. Callers that need title-case round-trip can + use the original ``tokens`` alongside the expansion. + """ + if not tokens: + return set() + if shards_dir is None: + # No shards configured: no expansion. Equivalent to the legacy + # behavior when the frozensets were empty. + return set(tokens) + synonym_index, _ = _get_indices(shards_dir) + expanded: set[str] = {t.lower() for t in tokens} + for t in list(expanded): + if t in synonym_index: + expanded |= synonym_index[t] + return expanded + + +def rivalry_excluded( + tokens: set[str], + *, + shards_dir: Path | str | None = None, + compare_phrasing: bool = False, +) -> set[str]: + """Tokens whose presence in a doc title means EXCLUDE that doc. + + For each rivalry pair (A, B): if exactly ONE side appears in the + query AND no comparison language was used, exclude the OTHER side's + tokens. If both sides appear, or if the user asked for a comparison, + no exclusion (they wanted both). + """ + if compare_phrasing or not tokens or shards_dir is None: + return set() + _, rivalry_pairs = _get_indices(shards_dir) + qlower = {t.lower() for t in tokens} + excluded: set[str] = set() + for a, b in rivalry_pairs: + a_in = bool(qlower & a) + b_in = bool(qlower & b) + if a_in and not b_in: + excluded |= b + elif b_in and not a_in: + excluded |= a + return excluded diff --git a/aborist/concepts/seed.py b/aborist/concepts/seed.py new file mode 100644 index 0000000..c7dec8b --- /dev/null +++ b/aborist/concepts/seed.py @@ -0,0 +1,128 @@ +"""One-time migration: legacy frozensets → concept_relations rows. + +Pre-2026-05-01 the synonym & rivalry data lived as hand-curated frozensets +in ``aborist.qa.concepts``. This module preserves those tuples & writes +them as ``evidence_kind='manual_legacy'`` rows so the DB-backed lookup +returns the same answers the frozenset lookup did. + +Idempotent: re-running ``seed_legacy_concepts`` is safe because the +INSERT OR IGNORE in ``add_concept_relation`` falls through on duplicates. + +The legacy data writes to a designated ``source_root`` value +(``__legacy__concepts__``) since these relations were not derived from +any specific document. New extractors writing relations from a real +document use the document's actual ``document_root``. +""" + +from __future__ import annotations + +import sqlite3 + +from aborist.concepts.store import add_concept_relation + +# Sentinel source_root for legacy seed data. Real concept relations +# from corpus extractors use the document's actual document_root so +# the per-source-root provenance stays intact. +LEGACY_SOURCE_ROOT = "__legacy__concepts__" +LEGACY_EVIDENCE_KIND = "manual_legacy" + + +# Snapshot of the frozenset groups as of commit bde1bd6 (2026-05-01). +# Each group becomes a fully-connected synonym cluster: every pair of +# tokens within the group becomes a synonym edge. Union-find at lookup +# time collapses these back into the same equivalence class. +LEGACY_SYNONYM_GROUPS: list[tuple[str, ...]] = [ + # AMD CPU family + ("amd", "athlon", "duron", "opteron", "ryzen", "epyc", + "thunderbird", "palomino", "thoroughbred", "barton", + "k6", "k7", "k8", "k10", "5x86"), + # Intel CPU family + ("intel", "pentium", "celeron", "xeon", "itanium", + "i7", "i5", "i3", "i9", + "core2", "coreduo", "skylake", "haswell", "ivy", + "8086", "80286", "80386", "80486"), + # HTTP / web protocol family + ("http", "https", "hypertext", "rfc2068", "rfc2616"), + # FTP / file-transfer family + ("ftp", "sftp", "ftps"), + # Mac family + ("macintosh", "mac", "macos", "osx", "apple"), + # Windows family + ("windows", "microsoft", "win32", "winnt", "win9x"), + # Linux family + ("linux", "gnu", "ubuntu", "debian", "fedora", "redhat", "kernel"), + # Mind-reading / brain-computer-interface family (added 2026-05-01) + ("telepathy", "telepathic", "neurotechnology", "neuroscience", + "neuroimaging", "neural", "neuron", "neurons", + "brain", "brains", "mind", "minds", "cognition", "cognitive", + "consciousness", "thoughts", "thinking", "thought", + "psychic", "psychokinesis", "esp", "clairvoyance", + "fmri", "eeg", "fnirs", "meg", "ecog", + "tms", "transcranial", "bci"), +] + +# Legacy rivalry pairs: a single edge between any token in group A and +# any token in group B captures the rivalry; lookup-time closure +# expands it to the whole synonym cluster on each side. We pick the +# canonical first token of each group as the representative. +LEGACY_RIVALRY_REPRESENTATIVES: list[tuple[str, str]] = [ + ("amd", "intel"), # AMD ↔ Intel + ("apple", "windows"), # Mac ↔ Windows (group reps) +] + + +def seed_legacy_concepts(conn: sqlite3.Connection) -> dict: + """Seed the legacy frozensets into ``concept_relations`` on ``conn``. + + Returns ``{"synonyms_inserted": N, "rivalries_inserted": M, + "synonyms_skipped": K, "rivalries_skipped": L}``. Skipped counts + indicate rows that already existed (idempotent re-seed). + """ + syn_ins = syn_skip = riv_ins = riv_skip = 0 + + # Synonyms: emit a forward edge between consecutive members of each + # group. Union-find at lookup time builds the full closure, so the + # quadratic per-group emission is unnecessary. + for group in LEGACY_SYNONYM_GROUPS: + if len(group) < 2: + continue + anchor = group[0] + for member in group[1:]: + inserted = add_concept_relation( + conn, + source_root=LEGACY_SOURCE_ROOT, + relation_kind="synonym", + token=anchor, + target=member, + evidence_kind=LEGACY_EVIDENCE_KIND, + derived_from="aborist.qa.concepts (legacy frozensets)", + ) + if inserted: + syn_ins += 1 + else: + syn_skip += 1 + + # Rivalries: one row per representative pair. Group closure happens + # at lookup time via the synonym index. + for a, b in LEGACY_RIVALRY_REPRESENTATIVES: + inserted = add_concept_relation( + conn, + source_root=LEGACY_SOURCE_ROOT, + relation_kind="rivalry", + token=a, + target=b, + evidence_kind=LEGACY_EVIDENCE_KIND, + derived_from="aborist.qa.concepts (legacy frozensets)", + ) + if inserted: + riv_ins += 1 + else: + riv_skip += 1 + + conn.commit() + return { + "synonyms_inserted": syn_ins, + "rivalries_inserted": riv_ins, + "synonyms_skipped": syn_skip, + "rivalries_skipped": riv_skip, + } diff --git a/aborist/concepts/store.py b/aborist/concepts/store.py new file mode 100644 index 0000000..4af39fd --- /dev/null +++ b/aborist/concepts/store.py @@ -0,0 +1,124 @@ +"""DB read/write helpers for concept_relations. + +All operations are scoped to a single shard connection. Cross-shard +queries live in ``aborist.concepts.query``. + +Append-only by design: ``add_concept_relation`` uses INSERT OR IGNORE +on the UNIQUE (source_root, relation_kind, token, target, evidence_kind) +key, so re-derivation never duplicates rows. ``purge_by_evidence_kind`` +is the only DELETE path & lets an operator revoke one extractor's +output without touching manual or other-extractor rows. +""" + +from __future__ import annotations + +import sqlite3 +import time + +# Allowed relation_kind values. The schema's CHECK constraint enforces +# this too — keeping the Python-side tuple in sync makes API misuse +# fail loud at the helper level rather than as a SQLite error. +RELATION_KINDS = ("synonym", "antonym", "rivalry", "category") + + +def add_concept_relation( + conn: sqlite3.Connection, + *, + source_root: str, + relation_kind: str, + token: str, + target: str, + evidence_kind: str, + confidence: float = 1.0, + derived_from: str | None = None, + derived_at: int | None = None, +) -> bool: + """Append a concept relation. Returns True if a row was inserted, + False if the (source_root, relation_kind, token, target, evidence_kind) + tuple already existed (idempotent re-derivation). + + Tokens are stored exactly as given — case preservation lets the + query layer decide normalization. Substring lookup at retrieval + time is case-insensitive via SQLite's NOCASE comparator. + """ + if relation_kind not in RELATION_KINDS: + raise ValueError( + f"relation_kind must be one of {RELATION_KINDS}, got {relation_kind!r}" + ) + if not token or not target: + raise ValueError("token and target must be non-empty") + if derived_at is None: + derived_at = int(time.time()) + cursor = conn.execute( + "INSERT OR IGNORE INTO concept_relations " + "(source_root, relation_kind, token, target, evidence_kind, " + " confidence, derived_at, derived_from) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + ( + source_root, + relation_kind, + token, + target, + evidence_kind, + float(confidence), + int(derived_at), + derived_from, + ), + ) + return cursor.rowcount > 0 + + +def concept_relations_for_token( + conn: sqlite3.Connection, + token: str, + *, + relation_kind: str | None = None, +) -> list[dict]: + """Return all relations whose ``token`` matches (case-insensitive). + If ``relation_kind`` is given, filter to that kind.""" + sql = ( + "SELECT source_root, relation_kind, token, target, evidence_kind, " + " confidence, derived_at, derived_from " + "FROM concept_relations WHERE LOWER(token) = LOWER(?)" + ) + params: tuple = (token,) + if relation_kind: + if relation_kind not in RELATION_KINDS: + raise ValueError( + f"relation_kind must be one of {RELATION_KINDS}, got {relation_kind!r}" + ) + sql += " AND relation_kind = ?" + params = params + (relation_kind,) + return [dict(row) for row in conn.execute(sql, params).fetchall()] + + +def purge_by_evidence_kind( + conn: sqlite3.Connection, + evidence_kind: str, + *, + derived_from: str | None = None, +) -> int: + """Delete every row with the given ``evidence_kind`` (and optional + ``derived_from``). Returns the number of rows removed. + + The intended use: revoke a buggy extractor's output cleanly. Manual + rows live under ``evidence_kind='manual'`` and are NOT touched by + a purge of any other kind. + """ + sql = "DELETE FROM concept_relations WHERE evidence_kind = ?" + params: tuple = (evidence_kind,) + if derived_from is not None: + sql += " AND derived_from = ?" + params = params + (derived_from,) + cursor = conn.execute(sql, params) + return cursor.rowcount + + +def list_evidence_kinds(conn: sqlite3.Connection) -> list[tuple[str, int]]: + """Return ``[(evidence_kind, row_count), ...]`` for the shard, ordered + by row_count descending. Useful for ``aborist concepts list --kinds``.""" + rows = conn.execute( + "SELECT evidence_kind, COUNT(*) AS n " + "FROM concept_relations GROUP BY evidence_kind ORDER BY n DESC" + ).fetchall() + return [(r["evidence_kind"], r["n"]) for r in rows] diff --git a/aborist/qa/concepts.py b/aborist/qa/concepts.py index 0897fc1..cada4a2 100644 --- a/aborist/qa/concepts.py +++ b/aborist/qa/concepts.py @@ -1,128 +1,68 @@ -"""Concept overlay: synonyms (broaden retrieval) + rivalries (narrow it). +"""Backwards-compat shim — public API delegates to ``aborist.concepts``. -A small knowledge-graph layer over the corpus. Two structures: +The actual data lived as hand-curated frozensets in this module +through April 2026 (commit c6182ae) and one entry was added in +2026-05-01 (commit bde1bd6 — mind/brain-tech group). 2026-05-01 the +data layer moved to a per-shard ``concept_relations`` SQLite table +(see aborist/concepts/__init__.py for the rationale). -- SYNONYM_GROUPS — sets of tokens that retrieve interchangeably. Querying - for `athlon` should also pull `AMD`-titled docs because Athlon IS an - AMD product. Groups are unordered and case-insensitive at compare time. +This shim preserves the call-site signatures `query.py` already uses +(``synonym_expand(qtokens)`` & ``rivalry_excluded(qtokens, compare_phrasing=...)``) +so the migration is a pure-implementation swap with no API change. -- RIVALRIES — pairs of group-indices that compete. If the query mentions - one side and not the other, docs whose titles contain the OTHER side's - tokens get filtered out (no Intel pages poisoning AMD answers). If the - query mentions BOTH sides — "AMD vs Intel", "compare AMD and Intel" — - no filtering: the user wants both sides. - -Phase 1: hand-curated. Phase 2 idea: derive from Wikipedia's category -graph or from "See also" sections (articles that link bidirectionally -in dense clusters → synonym group; articles in the same category that -DON'T cross-link → potential rivalries). +The new implementation needs a ``shards_dir`` to know which shards to +walk. The runtime threads it through query() already (kwarg passes +through), so the bridging happens here: callers that don't pass +``shards_dir`` get the empty-set degenerate behavior, which matches +the legacy code path when no synonym/rivalry was applicable. """ from __future__ import annotations +from pathlib import Path -# Each group is a frozenset of lowercased tokens. Title-token overlap with -# any element promotes the doc as topically relevant. -SYNONYM_GROUPS: list[frozenset[str]] = [ - # AMD CPU family - frozenset({ - "amd", "athlon", "duron", "opteron", "ryzen", "epyc", - "thunderbird", "palomino", "thoroughbred", "barton", - "k6", "k7", "k8", "k10", - "5x86", # AMD 5x86 - }), - # Intel CPU family - frozenset({ - "intel", "pentium", "celeron", "xeon", "itanium", - "i7", "i5", "i3", "i9", - "core2", "coreduo", "skylake", "haswell", "ivy", - "8086", "80286", "80386", "80486", - }), - # HTTP / web protocol family - frozenset({"http", "https", "hypertext", "rfc2068", "rfc2616"}), - # FTP / file-transfer protocol family (rivalry candidate vs HTTP for some queries) - frozenset({"ftp", "sftp", "ftps"}), - # Mac vs Windows family - frozenset({"macintosh", "mac", "macos", "osx", "apple"}), - frozenset({"windows", "microsoft", "win32", "winnt", "win9x"}), - # Linux family - frozenset({"linux", "gnu", "ubuntu", "debian", "fedora", "redhat", "kernel"}), - # Mind-reading / brain-computer-interface family. Closes the - # 2026-05-01 intent-question gap where "what technology can - # reconstruct another person's thoughts" pulled Videoconferencing - # via "person/language/speak" BM25 density and left Telepathy / - # Neurotechnology pages unused at #2 / #4. The concept tokens - # (thoughts, mind, cognition) sit alongside the specialized - # entity tokens (telepathy, BCI, fMRI, TMS) so a query naming - # either side expands to the whole family. Title-token-boost - # in `_rerank_by_title` then promotes the brain-tech pages above - # the lexical-decoy pages whose body shares only generic verbs. - frozenset({ - "telepathy", "telepathic", "neurotechnology", "neuroscience", - "neuroimaging", "neural", "neuron", "neurons", - "brain", "brains", "mind", "minds", "cognition", "cognitive", - "consciousness", "thoughts", "thinking", "thought", - "psychic", "psychokinesis", "esp", "clairvoyance", - "fmri", "eeg", "fnirs", "meg", "ecog", - "tms", "transcranial", "bci", - }), -] +from aborist.concepts.query import ( + has_compare_phrasing, + rivalry_excluded as _rivalry_excluded_impl, + synonym_expand as _synonym_expand_impl, +) + +__all__ = ["has_compare_phrasing", "rivalry_excluded", "synonym_expand"] -# Pairs of SYNONYM_GROUPS indices that compete. Bidirectional. -RIVALRIES: list[tuple[int, int]] = [ - (0, 1), # AMD ↔ Intel - (4, 5), # Mac ↔ Windows -] +def synonym_expand( + tokens: set[str], + *, + shards_dir: Path | str | None = None, +) -> set[str]: + """Add all synonym-group members for any token that hits a group. - -# Tokens that, if present in the query, mean "the user wants both sides -# of any rivalry shown" — comparative phrasing. When ANY of these appears, -# rivalry exclusion is suppressed. -COMPARE_WORDS: frozenset[str] = frozenset({ - "vs", "versus", "compare", "compared", "comparison", "compares", - "between", "difference", "differences", "or", "either", -}) - - -def synonym_expand(tokens: set[str]) -> set[str]: - """Add all synonym-group members for any token that hits a group.""" - expanded = set(tokens) - for t in tokens: - for group in SYNONYM_GROUPS: - if t in group: - expanded |= group - break - return expanded - - -def rivalry_excluded(tokens: set[str], compare_phrasing: bool = False) -> set[str]: - """Return tokens whose presence in a doc title means EXCLUDE that doc. - - Logic: for each rivalry pair (A, B), if exactly ONE side is present - in the query AND no compare-phrasing was detected, exclude the OTHER - side's tokens. If both sides are present, or if the user used - comparison language, no exclusion (they wanted both). + When ``shards_dir`` is None (the legacy two-arg call shape), returns + the input tokens unchanged. The runtime threads ``shards_dir`` + through retrieval — call sites that pass it get the corpus-derived + expansion; tests that don't get a no-op. """ - if compare_phrasing: + if shards_dir is None: + return set(tokens) + return _synonym_expand_impl(tokens, shards_dir=shards_dir) + + +def rivalry_excluded( + tokens: set[str], + compare_phrasing: bool = False, + *, + shards_dir: Path | str | None = None, +) -> set[str]: + """Tokens whose presence in a doc title means EXCLUDE that doc. + + Same call shape as the legacy positional (tokens, compare_phrasing) + so existing callers don't break. New calls pass ``shards_dir`` + via kwarg to enable corpus-derived rivalries. + """ + if shards_dir is None: return set() - excluded: set[str] = set() - for a_idx, b_idx in RIVALRIES: - a = SYNONYM_GROUPS[a_idx] - b = SYNONYM_GROUPS[b_idx] - a_in = bool(tokens & a) - b_in = bool(tokens & b) - if a_in and not b_in: - excluded |= b - elif b_in and not a_in: - excluded |= a - return excluded - - -def has_compare_phrasing(question: str) -> bool: - """True if the question contains comparison language.""" - lower = question.lower() - # Word-boundary check via simple split on non-alpha - import re - words = set(re.findall(r"[a-z]+", lower)) - return bool(words & COMPARE_WORDS) + return _rivalry_excluded_impl( + tokens, + shards_dir=shards_dir, + compare_phrasing=compare_phrasing, + ) diff --git a/aborist/qa/query.py b/aborist/qa/query.py index 9202121..5502109 100644 --- a/aborist/qa/query.py +++ b/aborist/qa/query.py @@ -153,6 +153,7 @@ def _filter_by_title_relevance( body_density_check: callable | None = None, phrase_match_roots: set[str] | None = None, fallback_top_n: int = 5, + shards_dir=None, ) -> list: """Concept-aware relevance filter with four accept paths: @@ -188,8 +189,12 @@ def _filter_by_title_relevance( if not qtokens: return hits qtokens_stem = {_stem_token_for_match(t) for t in qtokens} - accept = synonym_expand(qtokens) - exclude = rivalry_excluded(qtokens, compare_phrasing=has_compare_phrasing(question)) + accept = synonym_expand(qtokens, shards_dir=shards_dir) + exclude = rivalry_excluded( + qtokens, + compare_phrasing=has_compare_phrasing(question), + shards_dir=shards_dir, + ) core_roots = core_match_roots or set() phrase_roots = phrase_match_roots or set() # Title-overlap breadth threshold scales with query length, mirroring @@ -893,7 +898,8 @@ def _search_corpus( """ qtokens = _title_query_tokens(question) # Synonym expansion: a query for "athlon" also fetches AMD-titled docs. - accept_tokens = synonym_expand(qtokens) + # Reads concept_relations cross-shard via the shards_dir already in scope. + accept_tokens = synonym_expand(qtokens, shards_dir=shards_dir) paths: list[Path] if shards_dir is not None: paths = discover_shards(shards_dir) @@ -1081,6 +1087,7 @@ def _rerank( core_match_roots: set[str] | None = None, body_density_check: callable | None = None, phrase_match_roots: set[str] | None = None, + shards_dir=None, ) -> list[_Hit]: """Filter off-topic, then layer in body-coverage, title-overlap, and source-role rank boosts. @@ -1101,6 +1108,7 @@ def _rerank( core_match_roots=core_match_roots, body_density_check=body_density_check, phrase_match_roots=phrase_match_roots, + shards_dir=shards_dir, ) hits = _rerank_by_body_coverage(hits, question) hits = _rerank_by_title(hits, question) @@ -1558,6 +1566,7 @@ def query( core_match_roots=core_match_roots, body_density_check=_body_density_check, phrase_match_roots=phrase_match_roots, + shards_dir=shards_dir, ) search_ms = _ms_since(t_phase) diff --git a/aborist/store.py b/aborist/store.py index 79ef882..2b35880 100644 --- a/aborist/store.py +++ b/aborist/store.py @@ -316,6 +316,54 @@ CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5( contentless_delete=1, tokenize = 'porter unicode61' ); + +-- Concept-relations layer. Append-only secondary index over the corpus. +-- Each row is a (token, target) edge of a given relation_kind, derived +-- from a specific source document by a specific extractor (evidence_kind). +-- Re-derivation is idempotent at the (source_root, relation_kind, token, +-- target, evidence_kind) level via UNIQUE. +-- +-- This table is SEPARATE from the Merkle layer: writes here NEVER affect +-- document_root / chunk_root / cache_key. So the corpus's whole Merkle +-- tree stays valid across re-derivations; we can backfill or re-extract +-- concept relations without invalidating any cached answers. +-- +-- Cross-shard lookup. Concept relations live in the shard whose document +-- they were derived from; the lookup helpers in aborist.concepts walk all +-- shards (same pattern as cross-shard FTS5 search). Mesh sync moves shards +-- between peers; concept relations come along for the ride automatically. +-- +-- relation_kind: +-- 'synonym' - token & target retrieve interchangeably (See-also +-- bidirectional, redirect target, internal-link cluster) +-- 'antonym' - token & target are explicit opposites (manual / hatnote +-- "not to be confused with") +-- 'rivalry' - token & target compete in a category (same-category +-- membership without cross-link; manual rivalries) +-- 'category' - token belongs to category target (Wikipedia +-- [[Category:X]] tail; HTML schema.org/ classification) +-- +-- evidence_kind: which extractor produced the row. Lets `aborist concepts +-- purge --evidence-kind X` revoke a single extractor's output cleanly +-- without touching manual or other-extractor rows. New extractors register +-- a stable evidence_kind string; legacy seeds are 'manual_legacy'. +CREATE TABLE IF NOT EXISTS concept_relations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_root TEXT NOT NULL, + relation_kind TEXT NOT NULL + CHECK (relation_kind IN ('synonym','antonym','rivalry','category')), + token TEXT NOT NULL, + target TEXT NOT NULL, + evidence_kind TEXT NOT NULL, + confidence REAL NOT NULL DEFAULT 1.0, + derived_at INTEGER NOT NULL, + derived_from TEXT, -- shard/uri/extractor identifier + UNIQUE (source_root, relation_kind, token, target, evidence_kind) +); +CREATE INDEX IF NOT EXISTS idx_concept_token ON concept_relations(token); +CREATE INDEX IF NOT EXISTS idx_concept_target ON concept_relations(target); +CREATE INDEX IF NOT EXISTS idx_concept_kind ON concept_relations(relation_kind); +CREATE INDEX IF NOT EXISTS idx_concept_evid ON concept_relations(evidence_kind); """ @@ -705,6 +753,7 @@ _SHARDABLE_TABLES = ( "providence_cache", "audit_events", "falsifications", + "concept_relations", ) # Per-table column lists for cross-shard UNION views. The `chunks` table diff --git a/tests/test_concepts.py b/tests/test_concepts.py index 2261ee6..2f8f41c 100644 --- a/tests/test_concepts.py +++ b/tests/test_concepts.py @@ -1,59 +1,123 @@ -"""Concept overlay: synonym expansion + rivalry exclusion.""" +"""Concept overlay: synonym expansion + rivalry exclusion. + +Pre-2026-05-01: data lived as Python frozensets, tests called the +helpers directly. Post-2026-05-01: data lives in per-shard SQLite +tables, the helpers walk shards via shards_dir. Each test seeds +a temporary shard dir using the legacy seed so the assertions stay +identical. +""" from __future__ import annotations +from pathlib import Path + +import pytest + +from aborist.concepts import invalidate_cache as _invalidate_cache +from aborist.concepts.query import invalidate_cache +from aborist.concepts.seed import seed_legacy_concepts from aborist.qa.concepts import ( has_compare_phrasing, rivalry_excluded, synonym_expand, ) +from aborist.store import connect -def test_synonym_expand_amd_pulls_athlon_and_back(): - expanded = synonym_expand({"athlon"}) +# --------------------------------------------------------------------------- +# Fixture: a tmp shards_dir containing one shard pre-seeded with legacy data. +# --------------------------------------------------------------------------- + + +@pytest.fixture +def seeded_shards(tmp_path: Path) -> Path: + shards_dir = tmp_path / "shards" + shards_dir.mkdir() + shard_path = shards_dir / "000.db" + conn = connect(shard_path) + seed_legacy_concepts(conn) + conn.close() + invalidate_cache() + return shards_dir + + +# --------------------------------------------------------------------------- +# synonym_expand +# --------------------------------------------------------------------------- + + +def test_synonym_expand_amd_pulls_athlon_and_back(seeded_shards): + expanded = synonym_expand({"athlon"}, shards_dir=seeded_shards) assert "amd" in expanded assert "duron" in expanded assert "thunderbird" in expanded - expanded = synonym_expand({"amd"}) + expanded = synonym_expand({"amd"}, shards_dir=seeded_shards) assert "athlon" in expanded -def test_synonym_expand_unrelated_token_unchanged(): - expanded = synonym_expand({"banana"}) +def test_synonym_expand_unrelated_token_unchanged(seeded_shards): + expanded = synonym_expand({"banana"}, shards_dir=seeded_shards) assert expanded == {"banana"} -def test_synonym_expand_doesnt_cross_groups(): +def test_synonym_expand_doesnt_cross_groups(seeded_shards): """AMD and Intel are in different groups — expanding one shouldn't pull in the other.""" - expanded = synonym_expand({"amd"}) + expanded = synonym_expand({"amd"}, shards_dir=seeded_shards) assert "intel" not in expanded assert "pentium" not in expanded -def test_rivalry_excluded_amd_query_drops_intel(): - excluded = rivalry_excluded({"amd"}, compare_phrasing=False) +def test_synonym_expand_no_shards_dir_is_noop(): + """Without a shards_dir, expand returns the input unchanged. + Same degenerate behavior the legacy frozenset code had when + a query token didn't hit any group.""" + assert synonym_expand({"athlon"}) == {"athlon"} + + +# --------------------------------------------------------------------------- +# rivalry_excluded +# --------------------------------------------------------------------------- + + +def test_rivalry_excluded_amd_query_drops_intel(seeded_shards): + excluded = rivalry_excluded({"amd"}, compare_phrasing=False, shards_dir=seeded_shards) assert "intel" in excluded assert "pentium" in excluded -def test_rivalry_excluded_intel_query_drops_amd(): - excluded = rivalry_excluded({"intel"}, compare_phrasing=False) +def test_rivalry_excluded_intel_query_drops_amd(seeded_shards): + excluded = rivalry_excluded({"intel"}, compare_phrasing=False, shards_dir=seeded_shards) assert "amd" in excluded assert "athlon" in excluded -def test_rivalry_excluded_both_sides_no_exclusion(): - excluded = rivalry_excluded({"amd", "intel"}, compare_phrasing=False) +def test_rivalry_excluded_both_sides_no_exclusion(seeded_shards): + excluded = rivalry_excluded( + {"amd", "intel"}, compare_phrasing=False, shards_dir=seeded_shards + ) assert excluded == set() -def test_rivalry_excluded_compare_phrasing_suppresses(): - excluded = rivalry_excluded({"amd"}, compare_phrasing=True) +def test_rivalry_excluded_compare_phrasing_suppresses(seeded_shards): + excluded = rivalry_excluded( + {"amd"}, compare_phrasing=True, shards_dir=seeded_shards + ) assert excluded == set() +def test_rivalry_excluded_no_shards_dir_is_noop(): + """Without shards_dir, no exclusion (caller hasn't loaded the + concept layer).""" + assert rivalry_excluded({"amd"}, compare_phrasing=False) == set() + + +# --------------------------------------------------------------------------- +# Compare-phrasing detection (independent of shards) +# --------------------------------------------------------------------------- + + def test_compare_phrasing_detection(): assert has_compare_phrasing("compare AMD and Intel") assert has_compare_phrasing("AMD vs Intel") @@ -62,11 +126,51 @@ def test_compare_phrasing_detection(): assert not has_compare_phrasing("tell me about Athlon") -def test_mac_windows_rivalry(): +def test_mac_windows_rivalry(seeded_shards): """Independent rivalry pair — Mac vs Windows.""" - excluded = rivalry_excluded({"macintosh"}, compare_phrasing=False) + excluded = rivalry_excluded( + {"macintosh"}, compare_phrasing=False, shards_dir=seeded_shards + ) assert "windows" in excluded - excluded = rivalry_excluded({"windows"}, compare_phrasing=False) + excluded = rivalry_excluded( + {"windows"}, compare_phrasing=False, shards_dir=seeded_shards + ) assert "macintosh" in excluded assert "macos" in excluded + + +# --------------------------------------------------------------------------- +# Brain-tech group (added 2026-05-01) +# --------------------------------------------------------------------------- + + +def test_brain_tech_synonym_group_includes_telepathy_and_thoughts(seeded_shards): + """Run 1 retrieval-fix group: 'thoughts' should pull 'telepathy', + 'neurotechnology', etc. so the title-relevance accept path admits + brain-tech pages.""" + expanded = synonym_expand({"thoughts"}, shards_dir=seeded_shards) + assert "telepathy" in expanded + assert "neurotechnology" in expanded + assert "mind" in expanded + + +# --------------------------------------------------------------------------- +# Cache invalidation +# --------------------------------------------------------------------------- + + +def test_invalidate_cache_does_not_break_lookup(seeded_shards): + """After explicit cache invalidation, a fresh load reproduces + the same answers.""" + expanded_1 = synonym_expand({"amd"}, shards_dir=seeded_shards) + invalidate_cache() + expanded_2 = synonym_expand({"amd"}, shards_dir=seeded_shards) + assert expanded_1 == expanded_2 + + +def test_export_invalidate_cache_alias(seeded_shards): + """The package-level ``invalidate_cache`` is the same callable as + ``aborist.concepts.query.invalidate_cache`` — exposed at top level + for callers that don't want to import the implementation module.""" + assert _invalidate_cache is invalidate_cache