From b64395de0c7de3fb1a8ccabb9870743fa6535387 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Sun, 10 May 2026 12:47:58 -0400 Subject: [PATCH] tests/concepts/extract: 20 tests for synonym/idf/fts5 extractors (was zero coverage) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit arborist/concepts/extract.py — concept-relation write-side (extractors that derive concept_relations rows from the corpus state). 288 LOC, no direct tests despite test_concepts.py exercising the read-side end-to-end. Coverage: _title_tokens (pure private helper) - empty string → empty set - stopword strip (the/a/an/of/and/...) - underscore→space (Wikipedia title style: 'New_York_City') - length≥4 floor (3-letter words = noisy anchors) - lowercase output regardless of input case - hyphenated tokens kept as one (state-of-the-art) - dedupe via set return type - skip pure-digit tokens (regex requires [a-z] start) EXTRACTORS registry contract - 3 documented keys present (link_reciprocity / token_idf / documents_fts) - all values callable - dispatch table maps to actual implementations (identity check) backfill_documents_fts on synthetic mini-shard - indexes titles → returns rows_indexed count - idempotent (DELETE+INSERT pattern) - skips NULL titles (FTS5 row absent) - default + explicit derived_from kwarg paths link_reciprocity_synonym - empty edges → zero pairs - one-way edge → no reciprocal pair - reciprocal pair with no token overlap → cross-product synonyms - idempotency (concept_relations PK uniqueness) - self-overlap token excluded from cross product --- tests/test_concepts_extract.py | 278 +++++++++++++++++++++++++++++++++ 1 file changed, 278 insertions(+) create mode 100644 tests/test_concepts_extract.py diff --git a/tests/test_concepts_extract.py b/tests/test_concepts_extract.py new file mode 100644 index 0000000..8ceaede --- /dev/null +++ b/tests/test_concepts_extract.py @@ -0,0 +1,278 @@ +"""Tests for ``arborist.concepts.extract`` — concept-relation +extractors that derive synonyms / IDF / FTS5-titles from corpus +state. Module had zero direct tests despite being the write-side +of every concept-relations row the existing test_concepts.py +exercises on the read-side. + +Coverage: + - _title_tokens (pure private helper) — stopword strip, + underscore→space, length≥4 filter, lowercase, hyphen-handling + - EXTRACTORS registry contract — keys + signature + - backfill_documents_fts on a synthetic mini-shard + - link_reciprocity_synonym idempotency on a 2-doc reciprocal pair + +The extractors operate on real shard schema; we build a minimal +in-memory schema that matches store.py's contract just enough to +exercise the code paths. +""" + +from __future__ import annotations + +import sqlite3 + +import pytest + +from arborist.concepts.extract import ( + EXTRACTORS, + _title_tokens, + backfill_documents_fts, + backfill_token_idf, + link_reciprocity_synonym, +) +from arborist.store import SCHEMA_SQL + + +# --- _title_tokens (pure) ------------------------------------------ + + +def test_title_tokens_empty_string(): + assert _title_tokens("") == set() + + +def test_title_tokens_strips_stopwords(): + """Documented stopwords (the/a/an/of/and/...) drop out.""" + out = _title_tokens("the history of computing") + assert "the" not in out # stopword + assert "history" in out + assert "computing" in out + # "of" stopword + length<4 anyway + assert "of" not in out + + +def test_title_tokens_underscore_to_space(): + """Wikipedia title style 'Foo_Bar' → split on underscore.""" + out = _title_tokens("New_York_City") + assert "york" in out + assert "city" in out + + +def test_title_tokens_length_four_floor(): + """Tokens length<4 dropped (3-letter words = noisy anchors).""" + out = _title_tokens("the cat ran") + assert "cat" not in out # 3 chars + assert "ran" not in out # 3 chars + + +def test_title_tokens_lowercase(): + """Output always lowercase regardless of input case.""" + out = _title_tokens("Quantum CHROMODYNAMICS") + assert "quantum" in out + assert "chromodynamics" in out + + +def test_title_tokens_handles_hyphenated_token(): + """Title-token regex `[a-z][a-z0-9'\\-]+` keeps hyphens inside + a token. 'state-of-the-art' → 'state-of-the-art' as one token + (length>=4).""" + out = _title_tokens("state-of-the-art technology") + # The regex starts on a letter then captures alphanum/apos/hyphen. + # 'state-of-the-art' becomes one token. + assert any("-" in t for t in out) + assert "technology" in out + + +def test_title_tokens_dedupe(): + """Output is a set — duplicates collapse.""" + out = _title_tokens("Quantum quantum mechanics") + quantum_count = sum(1 for t in out if t == "quantum") + assert quantum_count == 1 + + +def test_title_tokens_skips_pure_digits(): + """Token must START with [a-z] per the regex; 1984 → no match.""" + out = _title_tokens("1984 novel") + assert "1984" not in out + assert "novel" in out + + +# --- EXTRACTORS registry -------------------------------------------- + + +def test_extractors_registry_has_documented_keys(): + """Three extractors documented in the module-level docstring.""" + assert "link_reciprocity" in EXTRACTORS + assert "token_idf" in EXTRACTORS + assert "documents_fts" in EXTRACTORS + + +def test_extractors_registry_callables(): + """Each registry value is callable.""" + for name, fn in EXTRACTORS.items(): + assert callable(fn), f"EXTRACTORS[{name!r}] is not callable" + + +def test_extractors_registry_link_reciprocity_routes_to_function(): + """The dispatch table maps to the actual implementations.""" + assert EXTRACTORS["link_reciprocity"] is link_reciprocity_synonym + assert EXTRACTORS["token_idf"] is backfill_token_idf + assert EXTRACTORS["documents_fts"] is backfill_documents_fts + + +# --- backfill_documents_fts ---------------------------------------- + + +@pytest.fixture +def empty_shard(tmp_path): + """Fresh SQLite with arborist's full schema applied.""" + db_path = tmp_path / "shard.db" + conn = sqlite3.connect(str(db_path)) + conn.row_factory = sqlite3.Row + conn.executescript(SCHEMA_SQL) + yield conn + conn.close() + + +def _insert_doc(conn, root: str, uri: str, title: str): + conn.execute( + "INSERT INTO documents " + "(document_root, document_uri, source_type, kind, " + " compression_depth, title, chunking_version, " + " canonicalization_version, schema_version, ingest_ts) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + (root, uri, "html", "surface", 0, title, + "tok-512-v1", "norm-v1", "v9.8.0", 0), + ) + + +def test_backfill_documents_fts_indexes_titles(empty_shard): + _insert_doc(empty_shard, "a" * 64, "uri-a", "Apple Inc") + _insert_doc(empty_shard, "b" * 64, "uri-b", "Banana fruit") + _insert_doc(empty_shard, "c" * 64, "uri-c", "Cherry tree") + empty_shard.commit() + + result = backfill_documents_fts(empty_shard) + assert result["rows_indexed"] == 3 + assert result["elapsed_ms"] >= 0 + + +def test_backfill_documents_fts_idempotent(empty_shard): + """Re-running on the same shard yields identical row count + (DELETE + INSERT inside).""" + _insert_doc(empty_shard, "a" * 64, "uri-a", "First") + _insert_doc(empty_shard, "b" * 64, "uri-b", "Second") + empty_shard.commit() + + r1 = backfill_documents_fts(empty_shard) + r2 = backfill_documents_fts(empty_shard) + assert r1["rows_indexed"] == r2["rows_indexed"] == 2 + + +def test_backfill_documents_fts_skips_null_titles(empty_shard): + """Documents with NULL title are not indexed (no FTS5 row).""" + _insert_doc(empty_shard, "a" * 64, "uri-a", "Has Title") + # Insert a row with no title. + empty_shard.execute( + "INSERT INTO documents " + "(document_root, document_uri, source_type, kind, " + " compression_depth, title, chunking_version, " + " canonicalization_version, schema_version, ingest_ts) " + "VALUES (?, ?, ?, ?, ?, NULL, ?, ?, ?, ?)", + ("b" * 64, "uri-b", "html", "surface", 0, + "tok-512-v1", "norm-v1", "v9.8.0", 0), + ) + empty_shard.commit() + result = backfill_documents_fts(empty_shard) + assert result["rows_indexed"] == 1 # only the titled doc + + +def test_backfill_documents_fts_default_derived_from_doesnt_break(empty_shard): + """Default derived_from kwarg path works (None → derived from + the function name internally).""" + _insert_doc(empty_shard, "a" * 64, "uri-a", "Test") + empty_shard.commit() + # Default kwarg path + r1 = backfill_documents_fts(empty_shard) + # Explicit kwarg path + r2 = backfill_documents_fts(empty_shard, derived_from="custom-extractor") + assert r1["rows_indexed"] == r2["rows_indexed"] + + +# --- link_reciprocity_synonym -------------------------------------- + + +def _insert_edge(conn, src: str, dst: str, dst_root: str): + conn.execute( + "INSERT INTO edges " + "(src_root, edge_type, dst_root, dst_uri, anchor) " + "VALUES (?, 'related_doc', ?, ?, '')", + (src, dst_root, dst), + ) + + +def test_link_reciprocity_no_edges_returns_zero(empty_shard): + result = link_reciprocity_synonym(empty_shard) + assert result["reciprocal_pairs"] == 0 + assert result["synonyms_inserted"] == 0 + + +def test_link_reciprocity_one_way_edge_no_pair(empty_shard): + """Edge A→B without reverse B→A → no reciprocal pair → no + synonym.""" + a, b = "a" * 64, "b" * 64 + _insert_doc(empty_shard, a, "uri-a", "Apple Inc") + _insert_doc(empty_shard, b, "uri-b", "Banana fruit") + _insert_edge(empty_shard, a, "uri-b", b) # only A→B + empty_shard.commit() + result = link_reciprocity_synonym(empty_shard) + assert result["reciprocal_pairs"] == 0 + + +def test_link_reciprocity_pair_creates_synonyms(empty_shard): + """A↔B with title tokens → synonym edges.""" + a, b = "a" * 64, "b" * 64 + _insert_doc(empty_shard, a, "uri-a", "Apple computing") + _insert_doc(empty_shard, b, "uri-b", "Banana technology") + _insert_edge(empty_shard, a, "uri-b", b) + _insert_edge(empty_shard, b, "uri-a", a) + empty_shard.commit() + result = link_reciprocity_synonym(empty_shard) + assert result["reciprocal_pairs"] == 1 + # Cross-product of 2 tokens × 2 tokens, minus self-overlap. + # apple/computing × banana/technology — no overlap → 4 pairs. + assert result["synonyms_inserted"] == 4 + + +def test_link_reciprocity_idempotent(empty_shard): + """Re-running adds no new rows (concept_relations PK enforces + uniqueness).""" + a, b = "a" * 64, "b" * 64 + _insert_doc(empty_shard, a, "uri-a", "Apple computing") + _insert_doc(empty_shard, b, "uri-b", "Banana technology") + _insert_edge(empty_shard, a, "uri-b", b) + _insert_edge(empty_shard, b, "uri-a", a) + empty_shard.commit() + r1 = link_reciprocity_synonym(empty_shard) + r2 = link_reciprocity_synonym(empty_shard) + assert r1["synonyms_inserted"] >= 1 + assert r2["synonyms_skipped"] == r1["synonyms_inserted"] + assert r2["synonyms_inserted"] == 0 + + +def test_link_reciprocity_skips_self_overlap_token(empty_shard): + """If both titles share a token (e.g. 'computing' in both), + that's not a synonym (same word) — skipped from the cross + product.""" + a, b = "a" * 64, "b" * 64 + _insert_doc(empty_shard, a, "uri-a", "Apple computing") + _insert_doc(empty_shard, b, "uri-b", "Banana computing") + _insert_edge(empty_shard, a, "uri-b", b) + _insert_edge(empty_shard, b, "uri-a", a) + empty_shard.commit() + result = link_reciprocity_synonym(empty_shard) + # 4 cross pairs minus the (computing, computing) self-overlap = 3 + # apple→banana, apple→computing, banana→computing, computing→apple, + # computing→banana. Self-pair (computing, computing) excluded. + # Implementation: cross = {(a,b) for a in A for b in B if a != b} + # A = {apple, computing}, B = {banana, computing} + # → (apple, banana), (apple, computing), (computing, banana) = 3 + assert result["synonyms_inserted"] == 3