diff --git a/arborist/concepts/extract.py b/arborist/concepts/extract.py index bea9927..ea56699 100644 --- a/arborist/concepts/extract.py +++ b/arborist/concepts/extract.py @@ -143,6 +143,161 @@ def link_reciprocity_synonym( } +# Acronym-parens extractor (#000054) — closes the abbreviation→expansion +# retrieval gap (CPU→Central Processing Unit, GPU→Graphics Processing +# Unit, RAM→Random Access Memory, FBI→Federal Bureau of Investigation, +# WHO→World Health Organization, …) that ``link_reciprocity_synonym`` +# structurally can't reach because the relation lives in body text, not +# the link graph. Corpus-agnostic — works on any source whose ``chunks`` +# carry prose (Wikipedia, HTML, textbooks, blogs). The lead chunk +# (idx=0) is enough: encyclopedic and reference-shape writing introduces +# its abbreviation in the first paragraph by convention. + +# Strict 1:1 acronym matching: every all-caps letter must correspond, +# in order, to the first letter of one content word in the phrase +# (function words filtered). Drops HTTP-shape acronyms where letters +# land mid-word — the catch isn't worth the false-positive risk. +_ACRONYM_PHRASE_RE = re.compile( + r"\b([A-Za-z][A-Za-z\-]+(?:\s+[A-Za-z][A-Za-z\-]+){1,6})\s*" + r"\(\s*([A-Z]{2,6})\s*\)" +) + +# Function words that appear inside acronym expansions but never +# contribute an initial ("Federal Bureau of Investigation" → FBI, not +# FBoI). Conservative — a missed match (DOA's "On") is cheaper than a +# false positive. +_ACRONYM_SKIPWORDS = frozenset({ + "of", "the", "and", "or", "for", "on", "in", "to", "a", "an", + "by", "at", "as", "is", "are", "was", "be", +}) + + +def _acronym_content_atoms(phrase: str) -> list[str]: + """Split the phrase into atomic content words for acronym matching. + Whitespace + hyphens split; ``_ACRONYM_SKIPWORDS`` filtered.""" + atoms: list[str] = [] + for raw in phrase.split(): + for piece in raw.split("-"): + piece = piece.strip() + if piece and piece.lower() not in _ACRONYM_SKIPWORDS: + atoms.append(piece) + return atoms + + +def _acronym_matches_phrase(phrase: str, acronym: str) -> list[str] | None: + """Strict 1:1 match. Returns the list of content atoms whose + initials form the acronym in order, or None on miss.""" + atoms = _acronym_content_atoms(phrase) + if len(atoms) != len(acronym): + return None + for letter, atom in zip(acronym, atoms): + if not atom or atom[0].lower() != letter.lower(): + return None + return atoms + + +def acronym_parens_synonym( + conn: sqlite3.Connection, + *, + derived_from: str | None = None, +) -> dict[str, int]: + """For every doc whose lead chunk contains the pattern + `` (ACRO)`` where ACRO's letters match the + content-word initials of the phrase in order, emit synonym edges + between the (lowercased) acronym and each ≥3-char content token + of the phrase, bidirectionally. + + Why this complements ``link_reciprocity_synonym``: Wikipedia + represents most abbreviation→expansion relations as **redirects** + (CPU → Central processing unit), which the ingest does not record + as edges. The relation IS in the lead-paragraph text of every + article that uses the term, though, in the convention + "Central processing unit (CPU)". Lexical + corpus-agnostic, so + HTML/blogs/textbooks benefit the same way as Wikipedia. + + Idempotent like ``link_reciprocity_synonym``. Evidence kind: + ``acronym_parens``. + + Returns + ------- + ``{"docs_scanned": N, "pairs_found": M, "synonyms_inserted": …, + "synonyms_skipped": …}``. + """ + from arborist.compress import unpack_chunk + + derived_at = int(time.time()) + derived_from = derived_from or "extract.acronym_parens_synonym" + syn_ins = syn_skip = 0 + docs_scanned = 0 + pairs_found = 0 + # (document_root, acronym, phrase) dedupe — a doc may repeat the + # parenthetical pattern; one edge set per (doc, acronym) is plenty. + seen: set[tuple[str, str, str]] = set() + + rows = conn.execute( + """ + SELECT d.document_root, d.title, c.content + FROM documents d + JOIN chunks c + ON c.document_root = d.document_root + AND c.idx = 0 + """ + ).fetchall() + + for r in rows: + docs_scanned += 1 + body = unpack_chunk(r["content"]) or "" + # Lead window only: encyclopedic intros (and reference prose + # generally) introduce their abbreviation in the first + # paragraph by convention. Bounded scan keeps this cheap on + # long articles. Title is NOT prepended — it gets pulled into + # the phrase capture by `\s+` (e.g. title "RAM" + body + # "Random Access Memory (RAM)" → bogus 4-word phrase). The + # convention "Phrase (ACRO)" lives in body text by design. + text = body[:4000] + for m in _ACRONYM_PHRASE_RE.finditer(text): + phrase = m.group(1).strip() + acronym = m.group(2).strip() + atoms = _acronym_matches_phrase(phrase, acronym) + if not atoms: + continue + acro_lo = acronym.lower() + phrase_key = " ".join(a.lower() for a in atoms) + key = (r["document_root"], acro_lo, phrase_key) + if key in seen: + continue + seen.add(key) + pairs_found += 1 + for atom in atoms: + tok_lo = atom.lower() + if tok_lo == acro_lo or len(tok_lo) < 3: + continue + # bidirectional: acro ↔ word + for token, target in ((acro_lo, tok_lo), (tok_lo, acro_lo)): + inserted = add_concept_relation( + conn, + source_root=r["document_root"], + relation_kind="synonym", + token=token, + target=target, + evidence_kind="acronym_parens", + derived_at=derived_at, + derived_from=derived_from, + ) + if inserted: + syn_ins += 1 + else: + syn_skip += 1 + + conn.commit() + return { + "docs_scanned": docs_scanned, + "pairs_found": pairs_found, + "synonyms_inserted": syn_ins, + "synonyms_skipped": syn_skip, + } + + def backfill_token_idf( conn: sqlite3.Connection, *, @@ -276,6 +431,12 @@ def backfill_documents_fts( EXTRACTORS: dict[str, Callable[..., dict[str, int]]] = { "link_reciprocity": link_reciprocity_synonym, + # Acronym-parens text-pattern extractor (#000054). Closes the + # abbreviation→expansion retrieval gap (CPU↔Central Processing Unit, + # GPU↔Graphics Processing Unit, RAM↔Random Access Memory, …) that + # link_reciprocity can't reach because the relation lives in body + # text, not the link graph. Corpus-agnostic. + "acronym_parens": acronym_parens_synonym, # Not a relation extractor — populates concept_token_idf for IDF # ranking at synonym_expand cap-time. Run AFTER any synonym # extractor since it indexes the union of token + target columns. diff --git a/docs/TICKETS.md b/docs/TICKETS.md index 2aff28b..a30718d 100644 --- a/docs/TICKETS.md +++ b/docs/TICKETS.md @@ -101,6 +101,7 @@ Newest first. Update on every open/close. | ID | Title | Status | Opened | Directive | |----------|------------------------------------------------|-----------------------|------------|-----------| +| #000054 | Acronym-parens concept extractor (closes the abbreviation→expansion retrieval gap) | **closed · 2026-05-13** — `arborist/concepts/extract.py:acronym_parens_synonym` lands as a new corpus-agnostic extractor in `EXTRACTORS` (`evidence_kind="acronym_parens"`). Scans each doc's lead chunk for ` (ACRO)` where the all-caps acronym's letters match the content-word initials of the phrase in order; emits bidirectional synonym edges between the lowercased acronym and each ≥3-char content token of the phrase. Conservative (strict 1:1 initials, function words filtered, repeated definitions deduped per doc). Closes the *retrieval-side* abbreviation gap (`CPU↔central processing unit`, `GPU↔graphics processing unit`, `RAM↔random access memory`, `FBI↔federal bureau of investigation`, `WHO↔world health organization`, …) that `link_reciprocity_synonym` can't reach because the relation lives in body text, not the wiki link graph (Wikipedia represents abbreviation→expansion as a *redirect* — not an edge). Per-shard like all `concept_relations` data; corpus-agnostic so HTML/blogs/textbooks benefit equally. Retrieval-side only — never proof-path. 8 new tests; full suite green. Closes #000050 §2a's CPU/GPU fixture rows *upstream* of vec; the Orwell-shape conceptual-allusion row remains the genuine #000050 justification. Operational follow-up (not code): `arborist concepts derive --extractor acronym_parens` on each shard. | 2026-05-13 | — | | #000053 | Acronym-aware verifier content tokens | **closed · 2026-05-13** — `arborist.qa.evidence._content_tokens` now keeps all-caps 2-3-char acronyms (CPU/GPU/DNA/FBI/USB…) as content tokens instead of dropping every <4-char token; fixes the field case where "what is a CPU?" cited to "CPU design" tripped `TITLE_MISMATCH` spuriously (claim & title share "CPU" but neither registered) — also affects `SUBJECT_TOKENS_ABSENT` (Rule 9), `BARE_NAME_CLAIM`, spotlight-excerpt token pick. Versioned: `content_token_rules: "v2-acronym-aware"` in both default policies + `_VERIFIER_POLICY_FIELDS` → folds into `verifier_policy_hash`, prior cache records orphan on lookup (by design, same discipline as `base_version` / `hyphen_fold_v1`). Monotone toward *fewer* spurious demotes (only relaxes overlap checks, never tightens). 8 new tests; full suite green; `bench-qa-smoke` clean. Does NOT fix the *retrieval* abbreviation→expansion gap (`CPU`→`Central processing unit` = #000050 vec hybrid / `concepts/` synonym edges — the root cause of the satellite-article retrieval). | 2026-05-13 | — | | #000052 | Relevance + coherence meta-cognition (answer-*shape* signals) | in progress — **§3.1 `diagnose_coherence` landed** (lexical, no model: `circular` / `phrase_component_reuse` / `vacuous`; in `arborist/qa/inspect.py`, surfaced via `inspect_cache_key` + `arborist inspect` `· incoherent: `; 9 tests; demote-policy hook deliberately not wired — advisory only). Joins the `diagnose_deflection` / `diagnose_metaphor_deflection` / `diagnose_title_relevance` / soft-preflight family of read-only, demote-only, never-in-proof-path sidecars; `phrase_component_reuse` catches the motivating field case (a subject quoting a phrase, a predicate reusing one of that phrase's own tokens as a bare `the ` referent). **Still open: (2) `diagnose_relevance`** — semantic (not just lexical) "aboutness": does the answer address the question; is each claim about its cited source? Today's checks (subject-anchor token overlap, stemmed title-stem overlap) are *lexical* and a token collision defeats them — a small *aboutness/reranker* model (NOT NLI — entailment ≠ topicality) under #000049 §7's discipline cage verbatim (demotion-only, hash-pinned, `relevance_model_version`→`governance_policy_hash` iff it touches `audit_mode`, shadow-first, `[…]` extra, the §7 #20 haystack lesson — never over the whole context); gated on evidence, travels with #000049's model question. Motivating field case (2026-05-12, fox): the `claim_lattice` query that returned *"the phrase 'Zionist entity' is sometimes used as the entity, referring to the State of Israel"* at `EVIDENCE-WARRANTED-PARTIAL 2/3` — incoherent + token-collision recombination that NLI can't catch (returns *neutral*, not *contradiction*) and both lexical relevance checks waved through. Flags an upstream retrieval ticket (polysemy / title-token-soup) as the root-cause fix, not scoped here. #000049 sibling | 2026-05-12 | — | | #000051 | Federated vecpack distribution (gossip the embedding backfill) | open · awaiting go/no-go · doc-only scaffold. Makes `chunk_vecs` a distributable artifact: backfill once on any CPU box (cloud / Prometheus-Σ sweep — #000037 §3.1), publish a **vecpack** `(shard_root, vec_backend_version, [(leaf_hash, embedding_blob)…])` over the mesh wire layer, every peer pulls + bulk-loads (sub-ms/chunk on the receiver — the laptop never runs the transformer). Keyed on `leaf_hash` (portable) not `chunk_id` (shard-local). Vecpacks are **soft data** — embeddings are `UNGROUNDED`, never proof path — so a cheap structural sanity gate (chunk exists locally w/ matching leaf_hash, right blob length for (dim,quant), finite norm, backend_version matches) suffices, no Merkle-proof-grade verification needed. Supplies #000050's prereq #1 ("a vecpack exists & is imported on the bench box", not "fox embedded the corpus locally"). GPU producer (the fast path): bge-small-en-v1.5 batched on a CUDA box (4090) ≈ 10³–10⁴ chunks/s → full 6.24M-chunk corpus in *minutes*, not days — drop a CUDA `Embedder` into `default_embedder()`; CUDA stack lives only on the producer box, never in arborist's `python+sqlite3` core. The mechanism behind whitepaper §1's "the embedding pass runs off the device". #000039 / #000050 sibling | 2026-05-12 | — | @@ -157,4 +158,4 @@ Newest first. Update on every open/close. ## Next ID -`000054` +`000055` diff --git a/docs/tickets/ticket-000054-acronym-parens-concept-extractor.md b/docs/tickets/ticket-000054-acronym-parens-concept-extractor.md new file mode 100644 index 0000000..d6052b0 --- /dev/null +++ b/docs/tickets/ticket-000054-acronym-parens-concept-extractor.md @@ -0,0 +1,151 @@ +# Ticket #000054 — Acronym-parens concept extractor (closing the abbreviation→expansion retrieval gap) + +**Status:** closed · 2026-05-13 — `arborist/concepts/extract.py:acronym_parens_synonym` lands as a new corpus-agnostic extractor in the existing `EXTRACTORS` registry (`evidence_kind = "acronym_parens"`). Scans the lead chunk of every document for the pattern ` (ACRO)` where the all-caps parenthesized acronym's letters match the content-word initials of the phrase in order; emits bidirectional synonym edges between the (lowercased) acronym and each ≥3-char content token of the phrase. Conservative: strict 1:1 initial match, function words filtered, repeated definitions deduped per doc. 8 new tests in `tests/test_concepts_extract.py` (28 → 36); full suite green. Closes the *retrieval-side* abbreviation gap (`CPU↔central processing unit`, `GPU↔graphics processing unit`, `RAM↔random access memory`, `FBI↔federal bureau of investigation`, `WHO↔world health organization`, …) that `link_reciprocity_synonym` structurally can't reach because the relation lives in body text, not the wiki link graph (Wikipedia represents abbreviation→expansion as a *redirect*, which the ingest does not record as an edge). Per-shard like all `concept_relations` data; corpus-agnostic so HTML / blogs / textbooks benefit the same way as Wikipedia. Complements #000050 (vec hybrid) without overlap — the abbreviation cases the vec layer would otherwise have to carry are now closable cheaply; the Orwell-shape *conceptual* allusion remains a vec-only case. +**Opened:** 2026-05-13 +**Scope:** One new extractor in `arborist/concepts/extract.py` + registry entry + tests. No schema change, no proof-path change (synonym edges are a *retrieval-side* soft signal — they reshape which candidates `qa/query.py` considers but never enter `audit_mode` / `cache_key` / `audit_event_hash`). +**Audience:** fox + anyone maintaining the retrieval pipeline + future shifts that wonder "why didn't `CPU` find `Central processing unit`?" +**Hard constraint:** synonym edges are **retrieval-side, never proof-path** — same discipline as `link_reciprocity_synonym` (CLAUDE.md "soft hash vs hard hash"). The extractor is idempotent (`INSERT OR IGNORE` via `add_concept_relation`) and writes only `concept_relations` rows. No verifier change, no `governance_policy_hash` bump, no cache invalidation. + +--- + +## 1. The gap, from the field + +`make query Q="what is a CPU?"` retrieves "CPU design", "CPU socket", +"CPU time", "CPU cache", "CPU-Z", "CPU multiplier", "CPU +(disambiguation)" — every "CPU *" satellite — but **not** "Central +processing unit", the canonical definitional article. Same shape for +"what is a GPU?" → satellites only, never "Graphics processing +unit". The four FTS5 routes (body BM25, title-LIKE, title-token, +phrase-pattern) all key on the query token `CPU`; the canonical +article's title shares *zero tokens* with that query (its title is +"Central processing unit"), so it never ranks. Body-BM25 *could* +surface it on raw frequency, but the satellite articles where `CPU` is +in *both* title and body outrank it on title-boost. (2026-05-13 field +case, fox.) + +This is exactly what `concepts/synonym_expand` exists to fix: a query +token gets expanded to a set including its synonyms, FTS5 then +retrieves on the union. But the existing extractor +(`link_reciprocity_synonym`) finds synonyms only from *reciprocal +wikilinks* — and Wikipedia represents abbreviation→expansion as a +**one-way redirect** (`CPU → Central processing unit`), which the +ingest does not record as an edge. So `concepts/` never learned the +relation. + +It is, however, in *body text* of every article that uses the term, by +near-universal convention: "Central processing unit (CPU) is the most +important component of a computer." That's the signal the new +extractor reads. + +## 2. The extractor + +`arborist/concepts/extract.py:acronym_parens_synonym`. Scans +`chunks.content` at `idx = 0` (the lead chunk; encyclopedic and +reference prose introduces its abbreviation in the first paragraph by +convention; bounded to the first 4000 chars for cheapness). Pattern: + +``` + (){1..6} ( ) +``` + +For each candidate `(phrase, ACRO)` pair: split the phrase into atomic +words (whitespace + hyphens both split), drop function words (`of`, +`the`, `and`, `or`, `for`, `on`, `in`, `to`, `a`, `an`, `by`, `at`, +`as`, `is`, `are`, `was`, `be`), then require **strict 1:1 match** — +`len(atoms) == len(ACRO)` and each acronym letter equals the first +letter of the corresponding atom (case-insensitive). On match, emit a +bidirectional synonym edge between the lowercased acronym and each +≥3-char content token of the phrase, anchored to the doc's +`document_root` and tagged `evidence_kind = "acronym_parens"`. + +Deliberately conservative: + +- **Strict 1:1.** `Hypertext Transfer Protocol (HTTP)` (3 atoms, 4 + letters) is *rejected* — the catch isn't worth the false-positive + risk that comes with allowing letters to land mid-word. Same for + `USA`-shape acronyms where letters skip atoms. +- **Function-word filter for atoms only.** "Federal Bureau of + Investigation (FBI)" → atoms `[Federal, Bureau, Investigation]`, FBI + matches (length 3 to 3). Atom-skipwords don't appear as edges either + (no `of ↔ fbi`). +- **Per-doc dedupe.** Same `(phrase, ACRO)` repeated in the same doc + produces one edge set, not N. +- **≥3-char target floor.** "We" or 2-char particles never become a + target. + +Registry: `EXTRACTORS["acronym_parens"] = acronym_parens_synonym`. + +## 3. Why this complements #000050 (and didn't get folded into it) + +#000050 (vec RRF hybrid) is the right tool for *genuine conceptual* +allusion — the Orwell→Eastasia case, where the query and target share +no semantic structure beyond meaning. Abbreviation→expansion is a +*different* shape: the relation is explicit in body text under a +universal convention, lexical and deterministic. A vec embedder +*could* close it (CPU and "Central processing unit" are semantically +adjacent), but at ~10–100× the ingest cost and a transformer in the +query path. The cheap, lexical, per-shard `concept_relations` table is +the right home; vec stays the answer to the cases lexical structurally +can't reach. #000050's §2a fixture set names the CPU/GPU cases — this +ticket closes them upstream; the bench should record *which* fix +closes each row, and the Orwell case is the one that genuinely +*requires* the vec layer. + +## 4. Out of scope + +- Backfilling the existing shards (`arborist concepts derive + --extractor acronym_parens` on each `~/.arborist/shards/*.db` — + operational, not code; one-shot, idempotent, cheap). +- Permissive acronym matching (HTTP-shape, letters mid-word) — would + fire #000054.1 if the false-positive rate of the strict matcher + proves limiting in practice. +- Cross-shard synonym indices (per-shard is the existing discipline; + cross-shard requires schema work that isn't justified yet — the + query-time expansion already merges across shards by lookup). +- Wiring an actual `arborist concepts derive` CLI subcommand (the + docstrings reference one but the surface isn't wired in `cli.py` + today; extractors are called programmatically. Worth its own ticket + if operations grows past one-off scripting.) + +## 5. Acceptance criteria + +1. `acronym_parens_synonym` lands in `arborist/concepts/extract.py` + with a stable `evidence_kind = "acronym_parens"` and is registered + in `EXTRACTORS`. +2. The CPU case (`"central processing unit (CPU)"` in body) emits + bidirectional edges `cpu ↔ central`, `cpu ↔ processing`, + `cpu ↔ unit` and is idempotent on re-run. +3. False-positive rejection: `Hypertext Transfer Protocol (HTTP)` + (mismatched length) and `Apple Banana Carrot (XYZ)` (mismatched + initials) emit zero edges. +4. Function-word filter: `Federal Bureau of Investigation (FBI)` emits + `fbi ↔ federal/bureau/investigation`; `of` never appears as a + target. +5. Per-doc dedupe: the same `(phrase, ACRO)` repeated in one doc emits + one edge set. +6. 8 new tests in `tests/test_concepts_extract.py` (covering CPU / + idempotency / function-word filter / length mismatch / initial + mismatch / hyphenated words / repeat dedupe / registry presence); + full suite green. + +## 6. References + +- `arborist/concepts/extract.py` — sibling `link_reciprocity_synonym` + is the design precedent (per-shard, evidence-kind-tagged, idempotent, + same return-dict shape). +- `arborist/concepts/store.py:add_concept_relation` — the write API. + `RELATION_KINDS = ("synonym", "antonym", "rivalry", "category")`. +- `arborist/qa/concepts.py` + `arborist/concepts/query.py` — the + read-side: `synonym_expand(qtokens, shards_dir=…)` walks every + shard's `concept_relations` and unions the matches into the query + token set before FTS5 retrieval runs. +- #000050 §2a (semantic-allusion fixture set) — CPU/GPU named there + as the two abbreviation rows; this extractor closes them *upstream* + of vec, leaving the Orwell row as the genuine vec justification. +- #000053 — verifier acronym-aware `_content_tokens`; orthogonal + (verifier-side, not retrieval-side) but in the same field-case + thread (a CPU/GPU query exposed *both* gaps; closing one didn't + close the other). +- CLAUDE.md "Retrieval pipeline" §5 (`concepts/` rivalry + + synonym layer) / "soft hash vs hard hash" — the discipline this + extractor obeys. diff --git a/tests/test_concepts_extract.py b/tests/test_concepts_extract.py index 8ceaede..35eb6b0 100644 --- a/tests/test_concepts_extract.py +++ b/tests/test_concepts_extract.py @@ -276,3 +276,148 @@ def test_link_reciprocity_skips_self_overlap_token(empty_shard): # A = {apple, computing}, B = {banana, computing} # → (apple, banana), (apple, computing), (computing, banana) = 3 assert result["synonyms_inserted"] == 3 + + +# --- acronym_parens_synonym (#000054) ------------------------------- + + +def _insert_lead_chunk(conn, doc_root: str, text: str): + """Insert a single idx=0 chunk for ``doc_root`` carrying ``text`` + in the schema-expected packed form.""" + from arborist.compress import pack_chunk + content = pack_chunk(text) + conn.execute( + "INSERT INTO chunks " + "(document_root, idx, leaf_hash, content, tier) " + "VALUES (?, ?, ?, ?, ?)", + (doc_root, 0, "0" * 64, content, "hot"), + ) + + +def _doc_with_lead(conn, root: str, uri: str, title: str, body: str): + _insert_doc(conn, root, uri, title) + _insert_lead_chunk(conn, root, body) + + +def test_acronym_parens_emits_bidirectional_synonym_on_cpu_case(empty_shard): + from arborist.concepts.extract import acronym_parens_synonym + _doc_with_lead( + empty_shard, + "a" * 64, "uri-a", "Central processing unit", + "A central processing unit (CPU) is the most important component " + "of a computer.", + ) + empty_shard.commit() + r = acronym_parens_synonym(empty_shard) + assert r["pairs_found"] == 1 + # 3 content words (central, processing, unit) × 2 directions = 6 edges + assert r["synonyms_inserted"] == 6 + rows = empty_shard.execute( + "SELECT token, target FROM concept_relations " + "WHERE evidence_kind = 'acronym_parens' ORDER BY token, target" + ).fetchall() + pairs = {(r["token"], r["target"]) for r in rows} + for word in ("central", "processing", "unit"): + assert ("cpu", word) in pairs + assert (word, "cpu") in pairs + + +def test_acronym_parens_idempotent(empty_shard): + from arborist.concepts.extract import acronym_parens_synonym + _doc_with_lead( + empty_shard, "a" * 64, "uri-a", "RAM", + "Random Access Memory (RAM) is volatile.", + ) + empty_shard.commit() + r1 = acronym_parens_synonym(empty_shard) + r2 = acronym_parens_synonym(empty_shard) + assert r1["synonyms_inserted"] >= 1 + assert r2["synonyms_inserted"] == 0 + assert r2["synonyms_skipped"] == r1["synonyms_inserted"] + + +def test_acronym_parens_filters_function_words_in_phrase(empty_shard): + """'Federal Bureau of Investigation (FBI)' — 'of' is dropped before + initial matching, leaving F/B/I to match Federal/Bureau/Investigation.""" + from arborist.concepts.extract import acronym_parens_synonym + _doc_with_lead( + empty_shard, "a" * 64, "uri-a", "FBI", + "The Federal Bureau of Investigation (FBI) is a US agency.", + ) + empty_shard.commit() + r = acronym_parens_synonym(empty_shard) + assert r["pairs_found"] == 1 + rows = empty_shard.execute( + "SELECT token, target FROM concept_relations " + "WHERE token = 'fbi'" + ).fetchall() + targets = {row["target"] for row in rows} + assert {"federal", "bureau", "investigation"} <= targets + assert "of" not in targets # function word — never an edge + + +def test_acronym_parens_rejects_mismatched_initials(empty_shard): + """'Hypertext Transfer Protocol (HTTP)' — 4-letter acronym but only + 3 content words → strict 1:1 fails → no edge emitted.""" + from arborist.concepts.extract import acronym_parens_synonym + _doc_with_lead( + empty_shard, "a" * 64, "uri-a", "HTTP", + "Hypertext Transfer Protocol (HTTP) is a network protocol.", + ) + empty_shard.commit() + r = acronym_parens_synonym(empty_shard) + assert r["pairs_found"] == 0 + assert r["synonyms_inserted"] == 0 + + +def test_acronym_parens_rejects_initials_mismatch_at_position(empty_shard): + """'Apple Banana Carrot (XYZ)' — letters don't match in order → no edge.""" + from arborist.concepts.extract import acronym_parens_synonym + _doc_with_lead( + empty_shard, "a" * 64, "uri-a", "Misleading", + "Apple Banana Carrot (XYZ) is a fake expansion.", + ) + empty_shard.commit() + r = acronym_parens_synonym(empty_shard) + assert r["pairs_found"] == 0 + + +def test_acronym_parens_handles_hyphenated_words(empty_shard): + """'Read-only memory (ROM)' — 'Read-only' splits into Read+only; + 'only' is a function-style word but NOT in skipwords, so atoms = + [Read, only, memory] → ROM (R/O/M) → match.""" + from arborist.concepts.extract import acronym_parens_synonym + _doc_with_lead( + empty_shard, "a" * 64, "uri-a", "ROM", + "Read-only memory (ROM) cannot be written.", + ) + empty_shard.commit() + r = acronym_parens_synonym(empty_shard) + assert r["pairs_found"] == 1 + rows = empty_shard.execute( + "SELECT token, target FROM concept_relations WHERE token = 'rom'" + ).fetchall() + targets = {row["target"] for row in rows} + assert {"read", "memory"} <= targets # "only" is 4 chars, kept; "read" 4 chars + # 'only' is included as a target since len >= 3 + assert "only" in targets + + +def test_acronym_parens_dedupes_repeated_definition_in_same_doc(empty_shard): + """Same parenthetical pattern repeated → one edge set per doc.""" + from arborist.concepts.extract import acronym_parens_synonym + _doc_with_lead( + empty_shard, "a" * 64, "uri-a", "CPU", + "A central processing unit (CPU) is the brain. " + "The central processing unit (CPU) executes instructions.", + ) + empty_shard.commit() + r = acronym_parens_synonym(empty_shard) + assert r["pairs_found"] == 1 + assert r["synonyms_inserted"] == 6 # 3 words × 2 directions + + +def test_acronym_parens_in_extractors_registry(): + from arborist.concepts.extract import EXTRACTORS + assert "acronym_parens" in EXTRACTORS + assert callable(EXTRACTORS["acronym_parens"])