qa(concepts): per-token IDF for cap-time ranking — replaces alphabetical truncation
Phase 1 of the holistic-tuning pair fox requested 2026-05-02. Replaces
the dumb alphabetical-truncation heuristic in synonym_expand's cap
path with corpus-derived IDF ranking — rare topical synonyms win the
truncation, common-corpus noise drops.
(1) New per-shard table: concept_token_idf (token, doc_freq, total_docs,
derived_at). Indexed on doc_freq for ORDER BY ranking. Folds into
the cross-shard UNION view via _SHARDABLE_TABLES.
(2) Extractor: backfill_token_idf (registered as evidence_kind
"token_idf"). Reads chunk-frequency from FTS5's fts5vocab virtual
table, bounded by the union of token + target columns in
concept_relations (only synonym tokens get an IDF row, not the 6M
full corpus vocabulary). Idempotent INSERT OR REPLACE.
(3) Query layer: _load_token_idf sums per-token doc_freq across all
shards (cross-shard ranking). _get_indices return tuple grew from
(manual, derived, rivalry) → (manual, derived, rivalry, idf). Cache
LRU keyed on shards_dir mtime so the IDF lookup is per-process-once.
(4) synonym_expand cap-time truncation: when expanded set exceeds
MAX_TOTAL_TOKENS, sort neighbors by (doc_freq ASC, token ASC) so
rare tokens come first. Tokens missing from concept_token_idf get
a sentinel high-rarity score (treats unknown as hapax — fail-safe
when the IDF backfill hasn't run yet).
Live verified on the brain-tech 19-token query:
Pre-IDF expansion (alphabetical 50): bumper, buster, channel,
convention, dave, duck, esp, field, glen, mind, minimum, ...
Post-IDF expansion (rare-first 50): neuroimaging, neurons,
neuroscience, neurotechnology, psychokinesis, telepathic,
telepathy, thinking, tms, transcranial — all the brain-tech
terms that lost the alphabetical race in pre-IDF now win.
Backfill cost: ~50s wall-clock across 4 wiki shards (~22k tokens
indexed per shard). Fts5vocab over chunks_fts is the right
infrastructure — no full corpus scan needed since we filter by the
concept_relations token set. Storage: ~1 MB per shard for the IDF
table. Stays well inside the ~90 MB/shard concept-layer budget fox
set earlier.
Tests: 641 passed (no regression). Live fixtures (boss baby,
amazon river/rainforest) all pass — ranking doesn't break gating.
This commit is contained in:
parent
8f4cb470aa
commit
b7f087f26c
3 changed files with 161 additions and 13 deletions
|
|
@ -143,10 +143,105 @@ def link_reciprocity_synonym(
|
|||
}
|
||||
|
||||
|
||||
def backfill_token_idf(
|
||||
conn: sqlite3.Connection,
|
||||
*,
|
||||
derived_from: str | None = None,
|
||||
) -> dict[str, int]:
|
||||
"""Populate ``concept_token_idf`` with chunk-frequency for every
|
||||
token that appears in ``concept_relations`` (as token OR target).
|
||||
|
||||
Uses FTS5's ``fts5vocab`` virtual table to read per-term doc
|
||||
frequencies directly from the chunks_fts index — no full table
|
||||
scan. Bounded by the size of the concept_relations token set
|
||||
(~10-50k unique tokens per shard, much smaller than the 6M-term
|
||||
full vocab).
|
||||
|
||||
The output drives ``synonym_expand``'s cap-time ranking: when the
|
||||
expanded set exceeds ``MAX_TOTAL_TOKENS``, neighbors with lower
|
||||
doc_freq (rarer in corpus = more topical) win the truncation.
|
||||
Replaces the prior alphabetical-truncation heuristic.
|
||||
|
||||
Idempotent: ``INSERT OR REPLACE`` overwrites any existing row, so
|
||||
re-running on a re-ingested shard refreshes the counts.
|
||||
|
||||
Returns ``{"tokens_indexed": N, "total_docs": M, "elapsed_ms": K}``.
|
||||
"""
|
||||
derived_at = int(time.time())
|
||||
derived_from = derived_from or "extract.backfill_token_idf"
|
||||
t0 = time.time()
|
||||
|
||||
# Total docs for IDF normalization.
|
||||
total_docs = conn.execute("SELECT COUNT(*) FROM documents").fetchone()[0]
|
||||
if total_docs == 0:
|
||||
return {"tokens_indexed": 0, "total_docs": 0, "elapsed_ms": 0}
|
||||
|
||||
# Materialize the union of tokens & targets from concept_relations.
|
||||
# Lowercase already at concept_relations write time so direct match.
|
||||
unique_tokens = set(
|
||||
r[0]
|
||||
for r in conn.execute(
|
||||
"SELECT DISTINCT token FROM concept_relations "
|
||||
"UNION SELECT DISTINCT target FROM concept_relations"
|
||||
).fetchall()
|
||||
)
|
||||
if not unique_tokens:
|
||||
return {"tokens_indexed": 0, "total_docs": total_docs, "elapsed_ms": 0}
|
||||
|
||||
# FTS5 vocab table — virtual; cheap to create on the fly.
|
||||
conn.execute(
|
||||
"CREATE VIRTUAL TABLE IF NOT EXISTS _tmp_chunks_vocab "
|
||||
"USING fts5vocab(chunks_fts, 'col')"
|
||||
)
|
||||
try:
|
||||
# Bulk-fetch doc-freq for every token in concept_relations.
|
||||
# The IN clause with a temp set is the cleanest path; vocab
|
||||
# row count is bounded by |unique_tokens| not the full 6M.
|
||||
placeholders = ",".join("?" for _ in unique_tokens)
|
||||
rows = conn.execute(
|
||||
f"SELECT term, doc FROM _tmp_chunks_vocab WHERE term IN ({placeholders})",
|
||||
tuple(unique_tokens),
|
||||
).fetchall()
|
||||
finally:
|
||||
conn.execute("DROP TABLE _tmp_chunks_vocab")
|
||||
|
||||
# Insert/upsert. Tokens NOT found in vocab (concept-relations row
|
||||
# added but no chunk contains the term) get doc_freq=0 so the
|
||||
# ranking still has a row to look up; default-rank-as-rare-but-
|
||||
# untrusted falls naturally out of the math (we treat doc_freq=0
|
||||
# as "highest rarity" — same as a hapax).
|
||||
conn.executemany(
|
||||
"INSERT OR REPLACE INTO concept_token_idf "
|
||||
"(token, doc_freq, total_docs, derived_at) VALUES (?, ?, ?, ?)",
|
||||
[(term, freq, total_docs, derived_at) for term, freq in rows],
|
||||
)
|
||||
indexed_terms = {r[0] for r in rows}
|
||||
missing_terms = unique_tokens - indexed_terms
|
||||
if missing_terms:
|
||||
conn.executemany(
|
||||
"INSERT OR REPLACE INTO concept_token_idf "
|
||||
"(token, doc_freq, total_docs, derived_at) VALUES (?, 0, ?, ?)",
|
||||
[(t, total_docs, derived_at) for t in missing_terms],
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
elapsed_ms = int((time.time() - t0) * 1000)
|
||||
return {
|
||||
"tokens_indexed": len(rows) + len(missing_terms),
|
||||
"tokens_with_chunk_hits": len(rows),
|
||||
"total_docs": total_docs,
|
||||
"elapsed_ms": elapsed_ms,
|
||||
}
|
||||
|
||||
|
||||
# Registry: evidence_kind → extractor callable.
|
||||
# Adding a new extractor: pick a stable evidence_kind string, implement
|
||||
# the (conn, *, derived_from) -> dict signature, register it here.
|
||||
# CLI command `aborist concepts derive --extractor X` reads this map.
|
||||
EXTRACTORS: dict[str, Callable[..., dict[str, int]]] = {
|
||||
"link_reciprocity": link_reciprocity_synonym,
|
||||
# Not a relation extractor — populates concept_token_idf for IDF
|
||||
# ranking at synonym_expand cap-time. Run AFTER any synonym
|
||||
# extractor since it indexes the union of token + target columns.
|
||||
"token_idf": backfill_token_idf,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -45,11 +45,12 @@ def has_compare_phrasing(question: str) -> bool:
|
|||
# Cross-shard lookup with mtime-keyed cache
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Cache shape: { shards_dir_str: (mtime_sig, manual_index, derived_index, rivalry_pairs) }
|
||||
# Cache shape: { shards_dir_str: (mtime_sig, manual_index, derived_index, rivalry_pairs, token_idf) }
|
||||
# - manual_index: curated synonym edges; always expanded
|
||||
# - derived_index: corpus-derived synonym edges; expansion subject to per-token cap
|
||||
# - rivalry_pairs: list of (set_a, set_b) — frozensets of lowercase tokens
|
||||
_CACHE: dict[str, tuple[tuple, dict, dict, list]] = {}
|
||||
# - token_idf: { token_lower: doc_freq } summed across shards, used for cap-time ranking
|
||||
_CACHE: dict[str, tuple[tuple, dict, dict, list, dict]] = {}
|
||||
|
||||
|
||||
def _shards_mtime_signature(shards_dir: Path) -> tuple:
|
||||
|
|
@ -63,6 +64,23 @@ def _shards_mtime_signature(shards_dir: Path) -> tuple:
|
|||
)
|
||||
|
||||
|
||||
def _load_token_idf(shards_dir: Path) -> dict[str, int]:
|
||||
"""Sum per-token doc_freq across all shards.
|
||||
|
||||
The concept_token_idf table is per-shard (the `chunks_fts` index
|
||||
it derives from is per-shard), so cross-shard ranking aggregates
|
||||
via SUM over the UNION view exposed by connect_query. Tokens not
|
||||
in any shard's table get NO entry; callers default to "treat as
|
||||
rare" via dict.get() with a high-rarity default.
|
||||
"""
|
||||
conn = connect_query(shards_dir=shards_dir)
|
||||
rows = conn.execute(
|
||||
"SELECT token, SUM(doc_freq) AS df FROM concept_token_idf GROUP BY token"
|
||||
).fetchall()
|
||||
conn.close()
|
||||
return {(r["token"] or "").lower(): int(r["df"] or 0) for r in rows}
|
||||
|
||||
|
||||
def _load_indices(shards_dir: Path) -> tuple[dict, dict, list]:
|
||||
"""Walk all shards, materialize the synonym & rivalry indices.
|
||||
|
||||
|
|
@ -133,20 +151,21 @@ def _load_indices(shards_dir: Path) -> tuple[dict, dict, list]:
|
|||
return manual_index, derived_index, rivalry_pairs
|
||||
|
||||
|
||||
def _get_indices(shards_dir: Path | str | None) -> tuple[dict, dict, list]:
|
||||
"""Return (manual_index, derived_index, rivalry_pairs), using
|
||||
cached values when the shard mtime signature is unchanged."""
|
||||
def _get_indices(shards_dir: Path | str | None) -> tuple[dict, dict, list, dict]:
|
||||
"""Return (manual_index, derived_index, rivalry_pairs, token_idf),
|
||||
using cached values when the shard mtime signature is unchanged."""
|
||||
if shards_dir is None:
|
||||
return {}, {}, []
|
||||
return {}, {}, [], {}
|
||||
p = Path(shards_dir)
|
||||
key = str(p.resolve())
|
||||
sig = _shards_mtime_signature(p)
|
||||
cached = _CACHE.get(key)
|
||||
if cached is not None and cached[0] == sig:
|
||||
return cached[1], cached[2], cached[3]
|
||||
return cached[1], cached[2], cached[3], cached[4]
|
||||
manual, derived, riv = _load_indices(p)
|
||||
_CACHE[key] = (sig, manual, derived, riv)
|
||||
return manual, derived, riv
|
||||
idf = _load_token_idf(p)
|
||||
_CACHE[key] = (sig, manual, derived, riv, idf)
|
||||
return manual, derived, riv, idf
|
||||
|
||||
|
||||
def invalidate_cache() -> None:
|
||||
|
|
@ -212,7 +231,7 @@ def synonym_expand(
|
|||
return set()
|
||||
if shards_dir is None:
|
||||
return set(tokens)
|
||||
manual_index, derived_index, _ = _get_indices(shards_dir)
|
||||
manual_index, derived_index, _, token_idf = _get_indices(shards_dir)
|
||||
qlower = {t.lower() for t in tokens}
|
||||
expanded: set[str] = set(qlower)
|
||||
# Manual (curated) synonyms always expand: brain-tech / AMD-family /
|
||||
|
|
@ -233,9 +252,23 @@ def synonym_expand(
|
|||
continue
|
||||
expanded |= neighbors
|
||||
if len(expanded) > max_total:
|
||||
ordered_neighbors = sorted(expanded - qlower)
|
||||
# IDF-rank the neighbors (rarer = more topical = keep first).
|
||||
# Tokens absent from concept_token_idf get a sentinel high-
|
||||
# rarity score so a hapax doesn't lose to a known-common
|
||||
# token at the truncation boundary. When the IDF table is
|
||||
# empty (backfill not run yet), ranking degenerates to
|
||||
# alphabetical (fallback compatibility).
|
||||
neighbors_only = expanded - qlower
|
||||
# Lower doc_freq → rarer → higher rank → kept first.
|
||||
# `total_docs + 1` sentinel pushes "missing" tokens to the
|
||||
# rarest-bucket so they tie-break before the most common.
|
||||
SENTINEL_HIGH_RARITY = 0
|
||||
ranked = sorted(
|
||||
neighbors_only,
|
||||
key=lambda t: (token_idf.get(t, SENTINEL_HIGH_RARITY), t),
|
||||
)
|
||||
budget = max(0, max_total - len(qlower))
|
||||
expanded = qlower | set(ordered_neighbors[:budget])
|
||||
expanded = qlower | set(ranked[:budget])
|
||||
return expanded
|
||||
|
||||
|
||||
|
|
@ -254,7 +287,7 @@ def rivalry_excluded(
|
|||
"""
|
||||
if compare_phrasing or not tokens or shards_dir is None:
|
||||
return set()
|
||||
_, _, rivalry_pairs = _get_indices(shards_dir)
|
||||
_, _, rivalry_pairs, _ = _get_indices(shards_dir)
|
||||
qlower = {t.lower() for t in tokens}
|
||||
excluded: set[str] = set()
|
||||
for a, b in rivalry_pairs:
|
||||
|
|
|
|||
|
|
@ -364,6 +364,25 @@ CREATE INDEX IF NOT EXISTS idx_concept_token ON concept_relations(token);
|
|||
CREATE INDEX IF NOT EXISTS idx_concept_target ON concept_relations(target);
|
||||
CREATE INDEX IF NOT EXISTS idx_concept_kind ON concept_relations(relation_kind);
|
||||
CREATE INDEX IF NOT EXISTS idx_concept_evid ON concept_relations(evidence_kind);
|
||||
|
||||
-- Per-token corpus document-frequency (for IDF ranking at synonym
|
||||
-- expansion cap-time). Computed once at backfill via fts5vocab over
|
||||
-- chunks_fts. Only tokens that appear in concept_relations get a row;
|
||||
-- the synonym layer is the consumer & it ranks expansion by 1/log(doc_freq)
|
||||
-- when the cap saturates so common-corpus words drop before rare topical
|
||||
-- ones.
|
||||
--
|
||||
-- doc_freq is FTS5-chunk-level (number of chunks containing the term);
|
||||
-- adequate proxy for true doc-level since chunks are sized 512 tokens
|
||||
-- and a doc rarely has the same term in only one chunk. Cross-shard
|
||||
-- ranking sums doc_freq across all shards' rows.
|
||||
CREATE TABLE IF NOT EXISTS concept_token_idf (
|
||||
token TEXT PRIMARY KEY,
|
||||
doc_freq INTEGER NOT NULL,
|
||||
total_docs INTEGER NOT NULL,
|
||||
derived_at INTEGER NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_token_idf_freq ON concept_token_idf(doc_freq);
|
||||
"""
|
||||
|
||||
|
||||
|
|
@ -754,6 +773,7 @@ _SHARDABLE_TABLES = (
|
|||
"audit_events",
|
||||
"falsifications",
|
||||
"concept_relations",
|
||||
"concept_token_idf",
|
||||
)
|
||||
|
||||
# Per-table column lists for cross-shard UNION views. The `chunks` table
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue