From 2c9d00ac95383f35fb18ec83c270168e075f1542 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Thu, 7 May 2026 09:24:46 -0400 Subject: [PATCH] modified: CLAUDE.md modified: Makefile modified: README.md deleted: aborist/search/__init__.py renamed: aborist/__init__.py -> arborist/__init__.py renamed: aborist/cli.py -> arborist/cli.py renamed: aborist/compress.py -> arborist/compress.py renamed: aborist/concepts/__init__.py -> arborist/concepts/__init__.py renamed: aborist/concepts/extract.py -> arborist/concepts/extract.py renamed: aborist/concepts/query.py -> arborist/concepts/query.py renamed: aborist/concepts/seed.py -> arborist/concepts/seed.py renamed: aborist/concepts/store.py -> arborist/concepts/store.py renamed: aborist/distill/__init__.py -> arborist/distill/__init__.py renamed: aborist/distill/base.py -> arborist/distill/base.py renamed: aborist/distill/first_sentence.py -> arborist/distill/first_sentence.py renamed: aborist/distill/runner.py -> arborist/distill/runner.py renamed: aborist/distill/tfidf.py -> arborist/distill/tfidf.py renamed: aborist/document.py -> arborist/document.py renamed: aborist/evict.py -> arborist/evict.py renamed: aborist/ingest.py -> arborist/ingest.py renamed: aborist/journal.py -> arborist/journal.py renamed: aborist/merkle.py -> arborist/merkle.py renamed: aborist/mesh/__init__.py -> arborist/mesh/__init__.py renamed: aborist/mesh/crypto.py -> arborist/mesh/crypto.py renamed: aborist/mesh/members.py -> arborist/mesh/members.py renamed: aborist/mesh/state.py -> arborist/mesh/state.py renamed: aborist/mesh/wire.py -> arborist/mesh/wire.py renamed: aborist/progress.py -> arborist/progress.py renamed: aborist/qa/__init__.py -> arborist/qa/__init__.py renamed: aborist/qa/client.py -> arborist/qa/client.py renamed: aborist/qa/concepts.py -> arborist/qa/concepts.py renamed: aborist/qa/dag.py -> arborist/qa/dag.py renamed: aborist/qa/evidence.py -> arborist/qa/evidence.py renamed: aborist/qa/frame.py -> arborist/qa/frame.py renamed: aborist/qa/inspect.py -> arborist/qa/inspect.py renamed: aborist/qa/keys.py -> arborist/qa/keys.py renamed: aborist/qa/metacognition.py -> arborist/qa/metacognition.py renamed: aborist/qa/model_profiles.py -> arborist/qa/model_profiles.py renamed: aborist/qa/parse_claims.py -> arborist/qa/parse_claims.py renamed: aborist/qa/prompts.py -> arborist/qa/prompts.py renamed: aborist/qa/quantifier.py -> arborist/qa/quantifier.py renamed: aborist/qa/quantifier_reminder.py -> arborist/qa/quantifier_reminder.py renamed: aborist/qa/query.py -> arborist/qa/query.py renamed: aborist/qa/repair.py -> arborist/qa/repair.py renamed: aborist/qa/retrieval_plan.py -> arborist/qa/retrieval_plan.py renamed: aborist/qa/runner.py -> arborist/qa/runner.py renamed: aborist/qa/soft_preflight.py -> arborist/qa/soft_preflight.py renamed: aborist/qa/verify.py -> arborist/qa/verify.py renamed: aborist/qa/warrant.py -> arborist/qa/warrant.py new file: arborist/search/__init__.py renamed: aborist/search/base.py -> arborist/search/base.py renamed: aborist/search/fts5.py -> arborist/search/fts5.py renamed: aborist/snapshot.py -> arborist/snapshot.py renamed: aborist/source.py -> arborist/source.py renamed: aborist/sources/__init__.py -> arborist/sources/__init__.py renamed: aborist/sources/crawler/__init__.py -> arborist/sources/crawler/__init__.py renamed: aborist/sources/crawler/async_web_fetcher.py -> arborist/sources/crawler/async_web_fetcher.py renamed: aborist/sources/crawler/bridge.py -> arborist/sources/crawler/bridge.py renamed: aborist/sources/crawler/web_fetch.py -> arborist/sources/crawler/web_fetch.py renamed: aborist/sources/grok.py -> arborist/sources/grok.py renamed: aborist/sources/html_page.py -> arborist/sources/html_page.py renamed: aborist/sources/providence.py -> arborist/sources/providence.py renamed: aborist/sources/vcs.py -> arborist/sources/vcs.py renamed: aborist/sources/wikipedia.py -> arborist/sources/wikipedia.py renamed: aborist/sources/wikipedia_xml.py -> arborist/sources/wikipedia_xml.py renamed: aborist/store.py -> arborist/store.py renamed: aborist/wikitext.py -> arborist/wikitext.py modified: pyproject.toml --- CLAUDE.md | 60 +++--- Makefile | 90 ++++---- README.md | 48 ++--- aborist/search/__init__.py | 6 - {aborist => arborist}/__init__.py | 2 +- {aborist => arborist}/cli.py | 198 +++++++++--------- {aborist => arborist}/compress.py | 0 {aborist => arborist}/concepts/__init__.py | 8 +- {aborist => arborist}/concepts/extract.py | 8 +- {aborist => arborist}/concepts/query.py | 8 +- {aborist => arborist}/concepts/seed.py | 8 +- {aborist => arborist}/concepts/store.py | 4 +- {aborist => arborist}/distill/__init__.py | 6 +- {aborist => arborist}/distill/base.py | 2 +- .../distill/first_sentence.py | 4 +- {aborist => arborist}/distill/runner.py | 12 +- {aborist => arborist}/distill/tfidf.py | 4 +- {aborist => arborist}/document.py | 0 {aborist => arborist}/evict.py | 10 +- {aborist => arborist}/ingest.py | 18 +- {aborist => arborist}/journal.py | 52 ++--- {aborist => arborist}/merkle.py | 0 {aborist => arborist}/mesh/__init__.py | 12 +- {aborist => arborist}/mesh/crypto.py | 2 +- {aborist => arborist}/mesh/members.py | 2 +- {aborist => arborist}/mesh/state.py | 8 +- {aborist => arborist}/mesh/wire.py | 16 +- {aborist => arborist}/progress.py | 0 {aborist => arborist}/qa/__init__.py | 8 +- {aborist => arborist}/qa/client.py | 0 {aborist => arborist}/qa/concepts.py | 6 +- {aborist => arborist}/qa/dag.py | 8 +- {aborist => arborist}/qa/evidence.py | 4 +- {aborist => arborist}/qa/frame.py | 0 {aborist => arborist}/qa/inspect.py | 20 +- {aborist => arborist}/qa/keys.py | 6 +- {aborist => arborist}/qa/metacognition.py | 8 +- {aborist => arborist}/qa/model_profiles.py | 6 +- {aborist => arborist}/qa/parse_claims.py | 0 {aborist => arborist}/qa/prompts.py | 6 +- {aborist => arborist}/qa/quantifier.py | 2 +- .../qa/quantifier_reminder.py | 0 {aborist => arborist}/qa/query.py | 74 +++---- {aborist => arborist}/qa/repair.py | 2 +- {aborist => arborist}/qa/retrieval_plan.py | 4 +- {aborist => arborist}/qa/runner.py | 68 +++--- {aborist => arborist}/qa/soft_preflight.py | 0 {aborist => arborist}/qa/verify.py | 42 ++-- {aborist => arborist}/qa/warrant.py | 0 arborist/search/__init__.py | 6 + {aborist => arborist}/search/base.py | 2 +- {aborist => arborist}/search/fts5.py | 4 +- {aborist => arborist}/snapshot.py | 6 +- {aborist => arborist}/source.py | 4 +- {aborist => arborist}/sources/__init__.py | 12 +- .../sources/crawler/__init__.py | 18 +- .../sources/crawler/async_web_fetcher.py | 6 +- .../sources/crawler/bridge.py | 20 +- .../sources/crawler/web_fetch.py | 10 +- {aborist => arborist}/sources/grok.py | 4 +- {aborist => arborist}/sources/html_page.py | 10 +- {aborist => arborist}/sources/providence.py | 10 +- {aborist => arborist}/sources/vcs.py | 6 +- {aborist => arborist}/sources/wikipedia.py | 4 +- .../sources/wikipedia_xml.py | 8 +- {aborist => arborist}/store.py | 10 +- {aborist => arborist}/wikitext.py | 8 +- pyproject.toml | 16 +- 68 files changed, 508 insertions(+), 508 deletions(-) delete mode 100644 aborist/search/__init__.py rename {aborist => arborist}/__init__.py (76%) rename {aborist => arborist}/cli.py (96%) rename {aborist => arborist}/compress.py (100%) rename {aborist => arborist}/concepts/__init__.py (86%) rename {aborist => arborist}/concepts/extract.py (97%) rename {aborist => arborist}/concepts/query.py (97%) rename {aborist => arborist}/concepts/seed.py (94%) rename {aborist => arborist}/concepts/store.py (97%) rename {aborist => arborist}/distill/__init__.py (70%) rename {aborist => arborist}/distill/base.py (96%) rename {aborist => arborist}/distill/first_sentence.py (95%) rename {aborist => arborist}/distill/runner.py (97%) rename {aborist => arborist}/distill/tfidf.py (96%) rename {aborist => arborist}/document.py (100%) rename {aborist => arborist}/evict.py (96%) rename {aborist => arborist}/ingest.py (97%) rename {aborist => arborist}/journal.py (85%) rename {aborist => arborist}/merkle.py (100%) rename {aborist => arborist}/mesh/__init__.py (82%) rename {aborist => arborist}/mesh/crypto.py (99%) rename {aborist => arborist}/mesh/members.py (99%) rename {aborist => arborist}/mesh/state.py (98%) rename {aborist => arborist}/mesh/wire.py (98%) rename {aborist => arborist}/progress.py (100%) rename {aborist => arborist}/qa/__init__.py (68%) rename {aborist => arborist}/qa/client.py (100%) rename {aborist => arborist}/qa/concepts.py (92%) rename {aborist => arborist}/qa/dag.py (98%) rename {aborist => arborist}/qa/evidence.py (99%) rename {aborist => arborist}/qa/frame.py (100%) rename {aborist => arborist}/qa/inspect.py (98%) rename {aborist => arborist}/qa/keys.py (98%) rename {aborist => arborist}/qa/metacognition.py (98%) rename {aborist => arborist}/qa/model_profiles.py (97%) rename {aborist => arborist}/qa/parse_claims.py (100%) rename {aborist => arborist}/qa/prompts.py (97%) rename {aborist => arborist}/qa/quantifier.py (99%) rename {aborist => arborist}/qa/quantifier_reminder.py (100%) rename {aborist => arborist}/qa/query.py (98%) rename {aborist => arborist}/qa/repair.py (99%) rename {aborist => arborist}/qa/retrieval_plan.py (96%) rename {aborist => arborist}/qa/runner.py (96%) rename {aborist => arborist}/qa/soft_preflight.py (100%) rename {aborist => arborist}/qa/verify.py (98%) rename {aborist => arborist}/qa/warrant.py (100%) create mode 100644 arborist/search/__init__.py rename {aborist => arborist}/search/base.py (99%) rename {aborist => arborist}/search/fts5.py (98%) rename {aborist => arborist}/snapshot.py (97%) rename {aborist => arborist}/source.py (81%) rename {aborist => arborist}/sources/__init__.py (66%) rename {aborist => arborist}/sources/crawler/__init__.py (83%) rename {aborist => arborist}/sources/crawler/async_web_fetcher.py (99%) rename {aborist => arborist}/sources/crawler/bridge.py (96%) rename {aborist => arborist}/sources/crawler/web_fetch.py (98%) rename {aborist => arborist}/sources/grok.py (99%) rename {aborist => arborist}/sources/html_page.py (94%) rename {aborist => arborist}/sources/providence.py (94%) rename {aborist => arborist}/sources/vcs.py (98%) rename {aborist => arborist}/sources/wikipedia.py (99%) rename {aborist => arborist}/sources/wikipedia_xml.py (98%) rename {aborist => arborist}/store.py (99%) rename {aborist => arborist}/wikitext.py (95%) diff --git a/CLAUDE.md b/CLAUDE.md index 0f2ff5b..cc48c14 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,11 +1,11 @@ -# Agent Blackops — aborist +# Agent Blackops — arborist This repo is operated by **agent blackops** for fox/timehexon on the unsandbox / unturf / permacomputer platform. Identity shard: `~/git/unsandbox.com/blackops/BLACKOPS.md`. -## What aborist is +## What arborist is A content-addressed, Merkle-committed document store. Implements the runtime spec from **Merkle Providence Reverse RAG** (April 2026 @@ -19,13 +19,13 @@ Three layers stacked on one SQLite file: per-chunk inclusion proofs in `derivations.proof_blob`. Recursive. 3. **Providence cache** — Q&A records keyed on the v9.8 8-dim invariant. Each record carries `audit_mode` (STRICT / HYBRID / - UNGROUNDED) decided by the verifier in `aborist/qa/verify.py`. + UNGROUNDED) decided by the verifier in `arborist/qa/verify.py`. For claim-lattice modes the renderer maps that token to a four- rung ladder (POINTER-LINKED → ANCHOR-WARRANTED → EVIDENCE-WARRANTED; ENTAILMENT-VERIFIED reserved); UNGROUNDED below all rungs. The schema column stays unchanged — programmatic callers see the trichotomy, human-facing surfaces see the ladder. See - `aborist/cli.py:_render_audit_label`. + `arborist/cli.py:_render_audit_label`. ## Source papers @@ -35,13 +35,13 @@ Three layers stacked on one SQLite file: encoding A1, public quantization A2, collision-resistant hash A3, theorems T1–T5). - `~/git/proxy.unturf.com/pkg/verified/merkle.go` — fox's existing - Go merkle. Aborist Python ports its conventions exactly. + Go merkle. Arborist Python ports its conventions exactly. - `~/git/proxy.unturf.com/docs/merkle-tree.md` — convention reference. ## Architecture ``` -aborist/ +arborist/ ├── merkle.py # Python port of proxy.unturf.com Go conventions ├── store.py # v9.8 SQLite schema, audit-chain helpers ├── document.py # Document, Edge, Chunker (TokenChunker default) @@ -99,7 +99,7 @@ chain audit. Chain breaks are the loudest possible signal. Cache lookups filter on `state='live'`. Drift → `stale`. - **Audit chain**: every state-changing op writes one row in `audit_events` with `event_hash = sha256(prev || canonical(body))`. - Verified by `make chain-check-shards`. Use `aborist.store.append_audit` + Verified by `make chain-check-shards`. Use `arborist.store.append_audit` — never insert into `audit_events` directly. - **Cores never evict.** `evict_to_cold` only touches `kind='surface'`. - **Idempotent re-ingest.** Same content → same `document_root` → @@ -115,7 +115,7 @@ revert without reading why. When in doubt, walk the - **Merkle conventions**: non-commutative `HashCombine` prefix `0x03`, leaves `0x00`, odd-element rule = self-duplicate (NOT zero-pad). `MerkleProof.siblings` carries `is_left` flag — never sort lexically. - See `aborist/merkle.py`. + See `arborist/merkle.py`. - **Versioned defaults**: `tok-512-v1` (chunker), `norm-v1` (canonicalization), `v9.8.0` (schema), `wikitext-base-v1` (prose). Changing any default stales every prior cache record. Add a new @@ -123,7 +123,7 @@ revert without reading why. When in doubt, walk the - **`question_hash` is dedup-mode-aware** (`strict` | `equivalence_class`); folds into `governance_policy_hash`. JIT `fidelity` parameter on `query()`/`ask()` decouples lookup tolerance - from write policy. See `aborist/qa/keys.py`. + from write policy. See `arborist/qa/keys.py`. - **`audit_mode` is decided by the verifier, never asserted.** Four layered strategies tried in order, first to find evidence classifies: **quote** (sequential pair-matching, NOT regex — @@ -132,15 +132,15 @@ revert without reading why. When in doubt, walk the `entity_policy ∈ {strict, hybrid, drop, proximity}`), **paraphrase** (token-coverage, prose-shaped only; `verifier_method='paraphrase'`). Trichotomy: STRICT = every unit verifies, HYBRID = mixed, - UNGROUNDED = none. Never overclaim. See `aborist/qa/verify.py`. + UNGROUNDED = none. Never overclaim. See `arborist/qa/verify.py`. - **Verifier stays binary; falsifications carry soft signal.** No per-quote diagnosis fields on hard verifier output. Sidecars - (`aborist.qa.inspect.diagnose_*`, `aborist inspect --cache-key X`) + (`arborist.qa.inspect.diagnose_*`, `arborist inspect --cache-key X`) classify unverified spans, deflection, title-relevance — never write to `providence_cache` or `audit_events`. - **Trailing-citation strip**: `_strip_trailing_citation` peels one trailing parenthetical at end-of-span (gated on a citation cue or - URL) before substring testing. See `aborist/qa/verify.py`. + URL) before substring testing. See `arborist/qa/verify.py`. - **Soft hash vs hard hash**: hard = SHA-256 (commitments, proofs, cache_key); soft = embeddings/TF-IDF/similarity (training, ranking, distillation). Soft never enters proof path. @@ -155,7 +155,7 @@ revert without reading why. When in doubt, walk the (D2/D3/D4/D6/D7) on lattice modes. Both lattice modes share `verifier_method="claim_lattice"`; `answer_mode` on the run-DAG + `json_fixups` disambiguate. Each mode folds into - `governance_policy_hash`. See `aborist/qa/verify.py`, + `governance_policy_hash`. See `arborist/qa/verify.py`, `docs/qa-modes-bench.md`. - **Four-rung ladder (lattice-mode display layer)**: POINTER-LINKED → ANCHOR-WARRANTED → EVIDENCE-WARRANTED → (ENTAILMENT-VERIFIED @@ -168,23 +168,23 @@ revert without reading why. When in doubt, walk the verify against pinned spans, not synthesis). Pure render-layer — cache_key, governance_policy_hash, & all programmatic callers see the underlying audit_mode unchanged. See - `aborist/cli.py:_render_audit_label`. + `arborist/cli.py:_render_audit_label`. - **Claim-lattice-pointer mode (G0 / CTI)**: runtime mints `pointer_id` (E1, E2, … — what the model sees) and content-addressed `evidence_id` (what the cache & run-DAG store). Renderer interpolates literal source spans via `_spotlight_excerpt`. Synthetic-elision-by- construction-impossible — model never types the quote string. 9-stage - run-DAG. See `aborist/qa/evidence.py`, `docs/cti-architecture.md`. + run-DAG. See `arborist/qa/evidence.py`, `docs/cti-architecture.md`. - **Claim-count ceiling (`TOO_MANY_CLAIMS`)**: default 12 per answer. Catches "tell me all there is to know about X" runaway. Demotes STRICT → HYBRID without truncating. Folds into - `governance_policy_hash`. See `aborist/qa/verify.py`. + `governance_policy_hash`. See `arborist/qa/verify.py`. - **Broad-quantifier preflight guard (Ticket #000008)**: pure lexical - classifier (`aborist/qa/quantifier.py`) maps a question onto a + classifier (`arborist/qa/quantifier.py`) maps a question onto a 10-rung intensity ladder (ABSENT → SINGULAR → … → ALL → - COMPREHENSIVE). Per-model `aborist/qa/model_profiles.py` PROFILES + COMPREHENSIVE). Per-model `arborist/qa/model_profiles.py` PROFILES dict picks a per-call claim-cap from the (intensity, model) pair; - `aborist/qa/quantifier_reminder.py` synthesizes a one-line user-turn + `arborist/qa/quantifier_reminder.py` synthesizes a one-line user-turn reminder for broad questions. Six-level disable hierarchy (per-test, per-call CLI, per-phase policy, per-mode, per-model, master-via-governance-hash). 7 policy fields fold into @@ -192,7 +192,7 @@ revert without reading why. When in doubt, walk the records. Defaults preserve dry-run discipline: `quantifier_guard_apply_caps=False`, `quantifier_reminder_enabled=False`, - `quantifier_reject_broad=False`. CLI flags on `aborist query`: + `quantifier_reject_broad=False`. CLI flags on `arborist query`: `--no-quantifier-guard`, `--allow-broad`, `--reject-broad`, `--apply-quantifier-caps`. Bench A/B (2026-05-03, n=3 × 9 broad questions × 3 modes): reminder default-on supported (FORMAT_COLLAPSED @@ -201,7 +201,7 @@ revert without reading why. When in doubt, walk the best on pointer mean ratio (0.684) but not strictly best on JSON STRICT-rate. See `docs/tickets/ticket-000008-broad-quantifier-preflight-guard.md` §12 for the four-cell A/B data. -- **Wikitext base prose**: `aborist/wikitext.py:to_base()` runs +- **Wikitext base prose**: `arborist/wikitext.py:to_base()` runs before the LLM call AND inside `verify_quotes` so model and verifier see the same prose. Optional dep — graceful fallback when `mwparserfromhell` is missing. @@ -209,9 +209,9 @@ revert without reading why. When in doubt, walk the detects topic-shift via subject-anchor heuristic (LAST content token in question must appear in answer). Suppressed for date / count / cause shapes ("when", "why", "how many"). See - `aborist/qa/inspect.py`. + `arborist/qa/inspect.py`. - **Title-relevance hard check (Rule 8)**: `_claim_title_overlap` - in `aborist/qa/verify.py`. For each claim that resolved, at least + in `arborist/qa/verify.py`. For each claim that resolved, at least one cited evidence's source title must share ≥1 stemmed content token with the claim text. When NO cited title overlaps, record a `TITLE_MISMATCH` violation & demote `STRICT → HYBRID`. Catches @@ -224,7 +224,7 @@ revert without reading why. When in doubt, walk the label alongside `· warrant missing`. - **Title-relevance sidecar (legacy diagnostic)**: `diagnose_title_relevance(claim, cited_titles)` in - `aborist.qa.inspect` returns the same signal in dict form for + `arborist.qa.inspect` returns the same signal in dict form for per-cache-key inspection. Sidecar; never enters proof path. Pre-dates the Rule 8 promotion (2026-05-02). @@ -232,11 +232,11 @@ revert without reading why. When in doubt, walk the - LLM: `https://hermes.ai.unturf.com/v1` (Hermes-3 Llama-3.1-8B-FP8- Dynamic on vLLM, 82K ctx, no auth). `uncloseai.com` is marketing - only. Override via `--endpoint` or `ABORIST_LLM_ENDPOINT`. + only. Override via `--endpoint` or `ARBORIST_LLM_ENDPOINT`. - Wikipedia dumps: `https://dumps.wikimedia.org/archive/2003/2003-05-16/en/`. `robots.txt` returned 404 → no rules. -## Retrieval pipeline (`aborist/qa/query.py`) +## Retrieval pipeline (`arborist/qa/query.py`) Multi-stage. Each stage exists because something earlier wasn't enough; revert at your peril. Order: @@ -252,7 +252,7 @@ enough; revert at your peril. Order: 4. **`_filter_by_title_relevance` — four accept paths**: title-token overlap, TF-IDF core match, body density, **phrase match** (accept-path 4 lets phrase-route hits with no title overlap survive). -5. **Rivalry exclusion + synonym expansion** (`aborist/concepts/`) — +5. **Rivalry exclusion + synonym expansion** (`arborist/concepts/`) — Intel-titled docs drop from AMD queries; reverse holds. Backed by the per-shard `concept_relations` SQLite table (corpus-derived, not hand-curated). 1.6% storage tax measured at backfill on 6 GB @@ -276,7 +276,7 @@ question. Provenance gap on this is tracked in ## Hot path / gotchas -- **Hand-rolled wikitext parser** (`aborist/sources/wikipedia.py`): +- **Hand-rolled wikitext parser** (`arborist/sources/wikipedia.py`): char-position state machine, escape-aware, 4× faster than char- by-char loops via `str.find` + slicing. cProfile any change. - **`PRAGMA synchronous=NORMAL`** per-connection in `store.connect()`. @@ -380,8 +380,8 @@ git log --oneline -5 git status make test make chain-check-shards # 0 per shard = intact -.venv/bin/aborist --shards-dir ~/.aborist/shards stats -.venv/bin/aborist --shards-dir ~/.aborist/shards analyze --gravity-top 5 +.venv/bin/arborist --shards-dir ~/.arborist/shards stats +.venv/bin/arborist --shards-dir ~/.arborist/shards analyze --gravity-top 5 ``` Then ask fox what the mission is. diff --git a/Makefile b/Makefile index 8e330b8..86aa3f4 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -# aborist — Makefile entry points +# arborist — Makefile entry points # Every workflow lives behind a `make` target. Bare python commands are not # the user interface. @@ -7,7 +7,7 @@ PYTHON ?= python3 VENV ?= .venv PIP := $(VENV)/bin/pip PY := $(VENV)/bin/python -ABORIST := $(VENV)/bin/aborist +ARBORIST := $(VENV)/bin/arborist # Data + DB DATA_DIR ?= data @@ -18,7 +18,7 @@ WP_OLD_2 := $(DATA_DIR)/old_tablesqlbz2.2 WP_OLD := $(DATA_DIR)/20030516_old_tablesql.bz2 # Back-compat alias (older callers used WP_DUMP for the cur snapshot). WP_DUMP := $(WP_CUR) -DB ?= $(HOME)/.aborist/aborist.db +DB ?= $(HOME)/.arborist/arborist.db # Smoke-test caps so make all stays fast INGEST_LIMIT ?= 500 @@ -75,53 +75,53 @@ fetch-old: $(WP_OLD) ## download old (revision history) parts and concatenate (~ fetch: fetch-cur fetch-old ## download all 3 files (cur + old.1 + old.2 + concat) ingest-cur: bootstrap fetch-cur ## ingest INGEST_LIMIT cur articles - $(ABORIST) --db $(DB) ingest --source wikipedia_cur --path $(WP_CUR) --limit $(INGEST_LIMIT) + $(ARBORIST) --db $(DB) ingest --source wikipedia_cur --path $(WP_CUR) --limit $(INGEST_LIMIT) ingest-old: bootstrap fetch-old ## ingest INGEST_LIMIT old (history) revisions - $(ABORIST) --db $(DB) ingest --source wikipedia_old --path $(WP_OLD) --limit $(INGEST_LIMIT) + $(ARBORIST) --db $(DB) ingest --source wikipedia_old --path $(WP_OLD) --limit $(INGEST_LIMIT) # Phase 2: attach-forever sharding. Each shard owns its own SQLite file — -# no WAL contention. Reads via `aborist --shards-dir ` attach +# no WAL contention. Reads via `arborist --shards-dir ` attach # all shards as UNION views. "Merge cost" = 0. SHARDS ?= 4 -SHARDS_DIR ?= $(HOME)/.aborist/shards +SHARDS_DIR ?= $(HOME)/.arborist/shards ingest-cur-attached: bootstrap fetch-cur ## sharded ingest, no WAL contention (Phase 2) @mkdir -p $(SHARDS_DIR) @for i in $$(seq 0 $$(($(SHARDS) - 1))); do \ - $(ABORIST) ingest --source wikipedia_cur --path $(WP_CUR) \ + $(ARBORIST) ingest --source wikipedia_cur --path $(WP_CUR) \ --shards-dir $(SHARDS_DIR) --shard $$i/$(SHARDS) & \ done; wait ingest-old-attached: bootstrap fetch-old ## sharded ingest of old history (Phase 2) @mkdir -p $(SHARDS_DIR) @for i in $$(seq 0 $$(($(SHARDS) - 1))); do \ - $(ABORIST) ingest --source wikipedia_old --path $(WP_OLD) \ + $(ARBORIST) ingest --source wikipedia_old --path $(WP_OLD) \ --shards-dir $(SHARDS_DIR) --shard $$i/$(SHARDS) & \ done; wait stats-shards: bootstrap ## cross-shard stats via UNION views over $(SHARDS_DIR) - $(ABORIST) --shards-dir $(SHARDS_DIR) stats + $(ARBORIST) --shards-dir $(SHARDS_DIR) stats ACTIVITY_LIMIT ?= 10 activity: bootstrap ## recent Q&A + freshly cached docs (agent timeline) - $(ABORIST) --shards-dir $(SHARDS_DIR) activity --limit $(ACTIVITY_LIMIT) + $(ARBORIST) --shards-dir $(SHARDS_DIR) activity --limit $(ACTIVITY_LIMIT) inspect: bootstrap ## sidecar diagnose unverified spans for a cache_key: make inspect KEY=hex [JSON=1] @if [ -z "$(KEY)" ]; then echo "usage: make inspect KEY= [JSON=1]" >&2; exit 2; fi - $(ABORIST) --shards-dir $(SHARDS_DIR) inspect --cache-key $(KEY) $(if $(JSON),--json,) + $(ARBORIST) --shards-dir $(SHARDS_DIR) inspect --cache-key $(KEY) $(if $(JSON),--json,) falsify: bootstrap ## mark a cached answer wrong: make falsify KEY=hex REASON='why' @if [ -z "$(KEY)" ]; then echo "usage: make falsify KEY= REASON='why'" >&2; exit 2; fi - $(ABORIST) --shards-dir $(SHARDS_DIR) providence --falsify $(KEY) --reason "$(REASON)" + $(ARBORIST) --shards-dir $(SHARDS_DIR) providence --falsify $(KEY) --reason "$(REASON)" burn: bootstrap ## delete a leaf with no children. providence: KEY=; document/core: KIND=document|core ROOT=. REASON='why' [FORCE=1] @kind="$${KIND:-providence}"; \ if [ "$$kind" = "providence" ]; then \ if [ -z "$(KEY)" ]; then echo "usage: make burn KEY= REASON='why' [FORCE=1]" >&2; exit 2; fi; \ - $(ABORIST) --shards-dir $(SHARDS_DIR) burn --kind providence --cache-key $(KEY) --reason "$(REASON)" $(if $(FORCE),--force,); \ + $(ARBORIST) --shards-dir $(SHARDS_DIR) burn --kind providence --cache-key $(KEY) --reason "$(REASON)" $(if $(FORCE),--force,); \ elif [ "$$kind" = "document" ] || [ "$$kind" = "core" ]; then \ if [ -z "$(ROOT)" ]; then echo "usage: make burn KIND=$$kind ROOT= REASON='why' [FORCE=1]" >&2; exit 2; fi; \ - $(ABORIST) --shards-dir $(SHARDS_DIR) burn --kind $$kind --root $(ROOT) --reason "$(REASON)" $(if $(FORCE),--force,); \ + $(ARBORIST) --shards-dir $(SHARDS_DIR) burn --kind $$kind --root $(ROOT) --reason "$(REASON)" $(if $(FORCE),--force,); \ else \ echo "unknown KIND: $$kind (expected: providence|document|core)" >&2; exit 2; \ fi @@ -132,7 +132,7 @@ burn: bootstrap ## delete a leaf with no children. providence: KEY=; # retrieval/verifier tunings — wipe recent test runs in one shot. KG_SECONDS ?= 3600 burn-kindergarten: bootstrap ## bust providence rows < SECONDS old [SECONDS=3600 FORCE=1 DRY_RUN=1 REASON='why'] - $(ABORIST) --shards-dir $(SHARDS_DIR) burn-kindergarten \ + $(ARBORIST) --shards-dir $(SHARDS_DIR) burn-kindergarten \ --kindergarten-seconds $(KG_SECONDS) \ $(if $(REASON),--reason "$(REASON)",) \ $(if $(FORCE),--force,) \ @@ -165,13 +165,13 @@ query: bootstrap ## ask the corpus a question [JSON=1 BURN=1 REPAIR=1 REPROMPTS= @if [ -z "$$Q" ] && [ -z "$(Q)" ]; then \ echo "usage: make query Q=\"your question\" [JSON=1 BURN=1 REPAIR=1 REPROMPTS=N K=\"extra retrieval keywords\" ANSWER_MODE=claim_lattice|claim_lattice_pointer|quote BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1]"; exit 2; \ fi - $(ABORIST) --shards-dir $(SHARDS_DIR) query --top-k $(QUERY_TOP_K) $(if $(JSON),--json,) $(if $(BURN),--burn,) $(if $(REPAIR),--repair,) $(if $(REPROMPTS),--repair-reprompts $(REPROMPTS),) $(if $(ANSWER_MODE),--answer-mode $(ANSWER_MODE),) $(if $(K),--retrieval-keywords "$(K)",) $(if $(BROAD),--apply-quantifier-caps,) $(if $(REJECT_BROAD),--reject-broad,) $(if $(ALLOW_BROAD),--allow-broad,) "$(Q)" + $(ARBORIST) --shards-dir $(SHARDS_DIR) query --top-k $(QUERY_TOP_K) $(if $(JSON),--json,) $(if $(BURN),--burn,) $(if $(REPAIR),--repair,) $(if $(REPROMPTS),--repair-reprompts $(REPROMPTS),) $(if $(ANSWER_MODE),--answer-mode $(ANSWER_MODE),) $(if $(K),--retrieval-keywords "$(K)",) $(if $(BROAD),--apply-quantifier-caps,) $(if $(REJECT_BROAD),--reject-broad,) $(if $(ALLOW_BROAD),--allow-broad,) "$(Q)" query-dry: bootstrap ## like 'make query' but skip the LLM call (dry-run) [JSON=1 BURN=1 ANSWER_MODE=... BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1] @if [ -z "$$Q" ] && [ -z "$(Q)" ]; then \ echo "usage: make query-dry Q=\"your question\" [JSON=1 BURN=1 ANSWER_MODE=claim_lattice|claim_lattice_pointer|quote BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1]"; exit 2; \ fi - $(ABORIST) --shards-dir $(SHARDS_DIR) query --top-k $(QUERY_TOP_K) --dry-run $(if $(JSON),--json,) $(if $(BURN),--burn,) $(if $(ANSWER_MODE),--answer-mode $(ANSWER_MODE),) $(if $(BROAD),--apply-quantifier-caps,) $(if $(REJECT_BROAD),--reject-broad,) $(if $(ALLOW_BROAD),--allow-broad,) "$(Q)" + $(ARBORIST) --shards-dir $(SHARDS_DIR) query --top-k $(QUERY_TOP_K) --dry-run $(if $(JSON),--json,) $(if $(BURN),--burn,) $(if $(ANSWER_MODE),--answer-mode $(ANSWER_MODE),) $(if $(BROAD),--apply-quantifier-caps,) $(if $(REJECT_BROAD),--reject-broad,) $(if $(ALLOW_BROAD),--allow-broad,) "$(Q)" BENCH_QA_QUESTIONS ?= bench/qa_questions.txt BENCH_QA_OUT ?= bench/qa_results @@ -204,7 +204,7 @@ bench-qa-smoke: bootstrap ## quick 5-question smoke (all anchor classes; ~30s) --concurrency $(BENCH_QA_CONCURRENCY) test-live: bootstrap ## live QA quality tests against Hermes (gated; -n auto parallel) - ABORIST_LIVE_TESTS=1 ABORIST_LIVE_SHARDS_DIR=$(SHARDS_DIR) \ + ARBORIST_LIVE_TESTS=1 ARBORIST_LIVE_SHARDS_DIR=$(SHARDS_DIR) \ .venv/bin/pytest tests/test_qa_quality_live.py -v -n auto # Concept-layer backfill targets. Each runs an extractor across every @@ -224,7 +224,7 @@ backfill-concepts: bootstrap ## backfill all concept extractors in parallel acro # For pure smoke after a code change before the longer bench-qa-smoke. # Emergent stress-test: random word triangulation. Pick 3 words from # /usr/share/dict/words, ask Hermes @ temp=0.8 to weave them into a -# creative question, send to aborist, append the journey to +# creative question, send to arborist, append the journey to # bench/emergent_log.jsonl. Designed for blue-moon cadence — surfaces # combinatoric failure modes the curated bench-qa fixture set can't. # Teacher review (Opus) runs separately via `--print-pending`; fox @@ -243,10 +243,10 @@ bench-emergent-pending: bootstrap ## print log entries awaiting teacher review verify-shards: bootstrap ## cross-shard Merkle round-trip on a random sample - $(ABORIST) --shards-dir $(SHARDS_DIR) verify -n $(VERIFY_N) + $(ARBORIST) --shards-dir $(SHARDS_DIR) verify -n $(VERIFY_N) analyze-shards: bootstrap ## cross-shard compression spectrum + audit integrity - $(ABORIST) --shards-dir $(SHARDS_DIR) analyze + $(ARBORIST) --shards-dir $(SHARDS_DIR) analyze # Audit-chain integrity probe: counts dangling prev_event_hash references. # Faster than `analyze` and trivially scriptable. 0 = chain intact. @@ -269,7 +269,7 @@ chain-check-shards: ## audit-chain break count for every *.db in $(SHARDS_DIR) # because each shard is its own file. distill-shards-parallel: bootstrap ## one distill process per shard (parallel) @for shard in $(SHARDS_DIR)/*.db; do \ - $(ABORIST) --db $$shard distill --process first-sentence-v1 --kind surface & \ + $(ARBORIST) --db $$shard distill --process first-sentence-v1 --kind surface & \ done; wait # TF-IDF cores serve as enriched titles for retrieval — distinctive @@ -277,7 +277,7 @@ distill-shards-parallel: bootstrap ## one distill process per shard (parallel) # something queryable without a real title match. distill-shards-tfidf-parallel: bootstrap ## TF-IDF cores per shard, in parallel @for shard in $(SHARDS_DIR)/*.db; do \ - $(ABORIST) --db $$shard distill --process tfidf-keywords-v1 --kind surface & \ + $(ARBORIST) --db $$shard distill --process tfidf-keywords-v1 --kind surface & \ done; wait ingest: ingest-cur ## default ingest = cur (use ingest-old or *-parallel for full) @@ -293,11 +293,11 @@ GROK_EXPORT ?= $(HOME)/Downloads/ab8ef1f0-0d08-4f87-89c2-d4509e18115b GROK_SHARD := $(SHARDS_DIR)/grok.db ingest-grok-attached: bootstrap ## ingest Grok conversations into $(GROK_SHARD) @mkdir -p $(SHARDS_DIR) - $(ABORIST) --db $(GROK_SHARD) ingest --source grok_export --path $(GROK_EXPORT) --resume + $(ARBORIST) --db $(GROK_SHARD) ingest --source grok_export --path $(GROK_EXPORT) --resume ingest-grok-media-attached: bootstrap ## ingest Grok media prompts into $(GROK_SHARD) @mkdir -p $(SHARDS_DIR) - $(ABORIST) --db $(GROK_SHARD) ingest --source grok_media --path $(GROK_EXPORT) --resume + $(ARBORIST) --db $(GROK_SHARD) ingest --source grok_media --path $(GROK_EXPORT) --resume # ---------------------------------------------------------------------------- # Phase IV (2006+) Wikipedia XML dumps. Drop-in for any dated snapshot in @@ -333,10 +333,10 @@ fetch-xml: $(WP_XML) ## download Phase IV XML cur dump (default: enwiki 20101011 fetch-abstract: $(WP_ABSTRACT) ## download Phase IV abstract.xml (default: enwiki 20101011, ~3 GB) ingest-xml: bootstrap fetch-xml ## ingest INGEST_LIMIT pages from $(WP_XML) - $(ABORIST) --db $(DB) ingest --source wikipedia_xml --path $(WP_XML) --limit $(INGEST_LIMIT) + $(ARBORIST) --db $(DB) ingest --source wikipedia_xml --path $(WP_XML) --limit $(INGEST_LIMIT) ingest-xml-history: bootstrap ## ingest every revision (multi-revision mode); set WP_XML to a pages-meta-history file - $(ABORIST) --db $(DB) ingest --source wikipedia_xml_history --path $(WP_XML) --limit $(INGEST_LIMIT) + $(ARBORIST) --db $(DB) ingest --source wikipedia_xml_history --path $(WP_XML) --limit $(INGEST_LIMIT) # Sharded XML ingest into the attach-forever cluster — same pattern as # ingest-cur-attached. One process per shard, one SQLite file per shard, @@ -344,15 +344,15 @@ ingest-xml-history: bootstrap ## ingest every revision (multi-revision mode); se ingest-xml-attached: bootstrap fetch-xml ## sharded XML ingest, one process per shard @mkdir -p $(SHARDS_DIR) @for i in $$(seq 0 $$(($(SHARDS) - 1))); do \ - $(ABORIST) ingest --source wikipedia_xml --path $(WP_XML) \ + $(ARBORIST) ingest --source wikipedia_xml --path $(WP_XML) \ --shards-dir $(SHARDS_DIR) --shard $$i/$(SHARDS) & \ done; wait ingest-abstract: bootstrap fetch-abstract ## ingest INGEST_LIMIT abstract docs from $(WP_ABSTRACT) - $(ABORIST) --db $(DB) ingest --source wikipedia_abstract --path $(WP_ABSTRACT) --limit $(INGEST_LIMIT) + $(ARBORIST) --db $(DB) ingest --source wikipedia_abstract --path $(WP_ABSTRACT) --limit $(INGEST_LIMIT) # ---------------------------------------------------------------------------- -# Self-ingest: aborist consults its own source code as a queryable corpus. +# Self-ingest: arborist consults its own source code as a queryable corpus. # Re-running picks up new commits — same path + new content gets a fresh # document_root chained to the prior version via a `supersedes` edge, so the # audit chain grows as the repo grows. Lands in a dedicated shard file so it @@ -361,11 +361,11 @@ ingest-abstract: bootstrap fetch-abstract ## ingest INGEST_LIMIT abstract docs f # Override SELF_REPO to ingest a different repo's tree. # ---------------------------------------------------------------------------- SELF_REPO ?= $(CURDIR) -SELF_SHARD := $(SHARDS_DIR)/aborist-self.db +SELF_SHARD := $(SHARDS_DIR)/arborist-self.db ingest-self: bootstrap ## ingest this repo's HEAD into a dedicated shard @mkdir -p $(SHARDS_DIR) - $(ABORIST) --db $(SELF_SHARD) ingest --source git_repo --path $(SELF_REPO) + $(ARBORIST) --db $(SELF_SHARD) ingest --source git_repo --path $(SELF_REPO) # Self-reference: promote STRICT live providence records past the # kindergarten window into each shard's documents table. Each shard @@ -378,7 +378,7 @@ ingest-self-providence: bootstrap ## promote STRICT live providence records into @mkdir -p $(SHARDS_DIR) @for db in $(SHARDS_DIR)/*.db; do \ echo ">> promoting providence records: $$db"; \ - $(ABORIST) --db $$db ingest --source providence --kindergarten-seconds $(KG_SECONDS); \ + $(ARBORIST) --db $$db ingest --source providence --kindergarten-seconds $(KG_SECONDS); \ done # Generic git-repo ingest: aim it at any local clone via GIT_REPO=... @@ -386,7 +386,7 @@ GIT_REPO ?= $(CURDIR) GIT_SHARD := $(SHARDS_DIR)/$(notdir $(GIT_REPO))-git.db ingest-git: bootstrap ## ingest GIT_REPO= into its own shard @mkdir -p $(SHARDS_DIR) - $(ABORIST) --db $(GIT_SHARD) ingest --source git_repo --path $(GIT_REPO) + $(ARBORIST) --db $(GIT_SHARD) ingest --source git_repo --path $(GIT_REPO) # Generic hg-repo ingest. HG_REPO=. HG_REPO ?= @@ -394,16 +394,16 @@ HG_SHARD := $(SHARDS_DIR)/$(notdir $(HG_REPO))-hg.db ingest-hg: bootstrap ## ingest HG_REPO= (mercurial) into its own shard @if [ -z "$(HG_REPO)" ]; then echo "usage: make ingest-hg HG_REPO=/path/to/repo" >&2; exit 2; fi @mkdir -p $(SHARDS_DIR) - $(ABORIST) --db $(HG_SHARD) ingest --source hg_repo --path $(HG_REPO) + $(ARBORIST) --db $(HG_SHARD) ingest --source hg_repo --path $(HG_REPO) verify: bootstrap ## round-trip Merkle proofs for VERIFY_N random documents - $(ABORIST) --db $(DB) verify -n $(VERIFY_N) + $(ARBORIST) --db $(DB) verify -n $(VERIFY_N) search: bootstrap ## keyword search; override SEARCH_Q (or pass Q=...) - $(ABORIST) --db $(DB) search '$(if $(Q),$(Q),$(SEARCH_Q))' + $(ARBORIST) --db $(DB) search '$(if $(Q),$(Q),$(SEARCH_Q))' stats: bootstrap ## counts: documents, chunks, edges, audit chain - $(ABORIST) --db $(DB) stats + $(ARBORIST) --db $(DB) stats test: bootstrap ## run pytest suite (excludes opt-in crawler tests) $(VENV)/bin/pytest -q --ignore=tests/crawler -n auto @@ -435,7 +435,7 @@ crawl-ingest: bootstrap-crawler ## crawl URL=https://x.com [DEPTH=2 MAX=0 FAST=1 shard="$(SHARDS_DIR)/crawl_$${domain}.db"; \ fi; \ echo " shard: $$shard" >&2; \ - $(ABORIST) --db "$$shard" crawl --seed-url "$(URL)" --depth $(CRAWL_DEPTH) --max-pages $(CRAWL_MAX) $(if $(FAST),--fast,) --ingest + $(ARBORIST) --db "$$shard" crawl --seed-url "$(URL)" --depth $(CRAWL_DEPTH) --max-pages $(CRAWL_MAX) $(if $(FAST),--fast,) --ingest # Fast freshness probe: conditional HEAD per doc, classify fresh/stale/gone. # Send only If-None-Match + If-Modified-Since headers — server returns 304 @@ -445,12 +445,12 @@ RECRAWL_LIMIT ?= 100 # to scope to one. DOMAIN= filters to URLs containing the substring. recrawl-check: bootstrap-crawler ## conditional HEAD per ingested doc [DOMAIN=x.com LIMIT=100 CRAWL_SHARD=path] @if [ -n "$(CRAWL_SHARD)" ]; then \ - $(ABORIST) --db $(CRAWL_SHARD) crawler recrawl-check $(if $(DOMAIN),--domain $(DOMAIN),) --limit $(RECRAWL_LIMIT); \ + $(ARBORIST) --db $(CRAWL_SHARD) crawler recrawl-check $(if $(DOMAIN),--domain $(DOMAIN),) --limit $(RECRAWL_LIMIT); \ else \ for db in $(SHARDS_DIR)/*.db; do \ case "$$(basename $$db)" in qa.db|snapshots.db) continue;; esac; \ echo " shard: $$db" >&2; \ - $(ABORIST) --db "$$db" crawler recrawl-check $(if $(DOMAIN),--domain $(DOMAIN),) --limit $(RECRAWL_LIMIT); \ + $(ARBORIST) --db "$$db" crawler recrawl-check $(if $(DOMAIN),--domain $(DOMAIN),) --limit $(RECRAWL_LIMIT); \ done; \ fi @@ -476,15 +476,15 @@ docs-api-clean: ## remove Sphinx build artifacts # ETL throughput across configs and catches regressions on optimization # work. Override BENCH_DOCS=N (default 5000). BENCH_DOCS ?= 5000 -BENCH_DIR := /tmp/aborist-bench +BENCH_DIR := /tmp/arborist-bench bench: bootstrap fetch-cur ## benchmark serial vs parallel-shared vs attached at $(BENCH_DOCS) docs @bash bench/run.sh $(BENCH_DOCS) clean: ## remove venv + caches (keeps fetched data and db) - rm -rf $(VENV) .pytest_cache **/__pycache__ aborist.egg-info + rm -rf $(VENV) .pytest_cache **/__pycache__ arborist.egg-info find . -type d -name __pycache__ -prune -exec rm -rf {} + -clean-db: ## drop the aborist db (keeps fetched data and venv) +clean-db: ## drop the arborist db (keeps fetched data and venv) rm -f $(DB) $(DB)-journal $(DB)-wal $(DB)-shm clean-data: ## remove fetched dumps diff --git a/README.md b/README.md index e377509..162a071 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ -# aborist +# arborist An arborist for trees and forests of cross-linked information. -Aborist ingests documents into a content-addressed, Merkle-committed SQLite store, distills them into recursive cores, and answers questions over the resulting corpus via an OpenAI-compatible LLM. Every cached answer carries a verifiable Merkle proof tying it back to its source documents — Merkle-AGI v9.8 / Merkle Providence Reverse RAG, runnable end-to-end. +Arborist ingests documents into a content-addressed, Merkle-committed SQLite store, distills them into recursive cores, and answers questions over the resulting corpus via an OpenAI-compatible LLM. Every cached answer carries a verifiable Merkle proof tying it back to its source documents — Merkle-AGI v9.8 / Merkle Providence Reverse RAG, runnable end-to-end. ## Quickstart @@ -29,7 +29,7 @@ make crawl-ingest URL=https://russell.ballestrini.net DEPTH=2 # BFS + ingest make query Q="who is Russell Ballestrini?" # cross-shard query — picks up the new shard automatically ``` -The crawl shard is named after the seed hostname (`crawl_russell_ballestrini_net.db`) under `~/.aborist/shards/`. Add `FAST=1` for aggressive crawling against your own sites; `MAX=N` to cap discovery; `DEPTH=N` to bound BFS. Robots `Disallow` is always honored. After ingest, `make recrawl-check DOMAIN=russell.ballestrini.net` does a conditional-HEAD freshness probe per page. +The crawl shard is named after the seed hostname (`crawl_russell_ballestrini_net.db`) under `~/.arborist/shards/`. Add `FAST=1` for aggressive crawling against your own sites; `MAX=N` to cap discovery; `DEPTH=N` to bound BFS. Robots `Disallow` is always honored. After ingest, `make recrawl-check DOMAIN=russell.ballestrini.net` does a conditional-HEAD freshness probe per page. ### After the answer @@ -46,19 +46,19 @@ make burn KEY= REASON='…' # delete (kindergarten only — refuse ### Get the source ``` -git clone ssh://git@git.unturf.com:2222/engineering/unturf/aborist.git -cd aborist +git clone ssh://git@git.unturf.com:2222/engineering/unturf/arborist.git +cd arborist ``` HTTPS variant if SSH isn't set up: ``` -git clone https://git.unturf.com/engineering/unturf/aborist.git +git clone https://git.unturf.com/engineering/unturf/arborist.git ``` ### Install prerequisites -Aborist needs Python 3.10+, GNU make, `curl`, and `bzip2`. SQLite 3.35+ ships with CPython. +Arborist needs Python 3.10+, GNU make, `curl`, and `bzip2`. SQLite 3.35+ ships with CPython. **macOS** (Homebrew) @@ -96,7 +96,7 @@ Then inside the WSL Ubuntu shell, follow the Ubuntu instructions above. pkg_add git python-3.12 gmake curl ``` -OpenBSD's default `make` is BSD make. Aborist's Makefile uses GNU-make features (`?=`, conditional functions). Substitute `gmake` for `make` in every command, e.g. `gmake bootstrap`, `gmake query Q='…'`. +OpenBSD's default `make` is BSD make. Arborist's Makefile uses GNU-make features (`?=`, conditional functions). Substitute `gmake` for `make` in every command, e.g. `gmake bootstrap`, `gmake query Q='…'`. ### Bootstrap @@ -104,7 +104,7 @@ OpenBSD's default `make` is BSD make. Aborist's Makefile uses GNU-make features make bootstrap ``` -Creates `.venv/`, installs the package in editable mode with the `[dev,html]` extras, and exposes `aborist` at `.venv/bin/aborist`. No system-wide install. Re-running `make bootstrap` is a no-op if the venv is up to date. +Creates `.venv/`, installs the package in editable mode with the `[dev,html]` extras, and exposes `arborist` at `.venv/bin/arborist`. No system-wide install. Re-running `make bootstrap` is a no-op if the venv is up to date. After bootstrap, every workflow lives behind a `make` target. Run `make help` to list them. @@ -135,7 +135,7 @@ make ingest-cur-attached SHARDS=4 # cur snapshot, 4 parallel shards (~3 min) make ingest-old-attached SHARDS=4 # full history, ~30–40 min ``` -Shards land in `~/.aborist/shards/`. Override with `SHARDS_DIR=/path/to/somewhere`. +Shards land in `~/.arborist/shards/`. Override with `SHARDS_DIR=/path/to/somewhere`. ### Resumable @@ -146,7 +146,7 @@ Add `--resume` (or just re-run the make target — `--resume` is the default for For experiments under a few thousand docs, a single SQLite file is fine: ``` -make ingest-cur INGEST_LIMIT=1000 # one DB at $(DB), default ~/.aborist/aborist.db +make ingest-cur INGEST_LIMIT=1000 # one DB at $(DB), default ~/.arborist/arborist.db make ingest-cur-parallel SHARDS=4 # 4 processes, one shared DB (WAL serialized) ``` @@ -163,7 +163,7 @@ Run both — they generate independent cores per source. TF-IDF cores let neolog ## Data: Wikipedia 2010-11 (and other Phase IV snapshots) -In 2006 MediaWiki swapped its dumps from MySQL `INSERT INTO cur` syntax to XML. Aborist reads both — the SQL path above for 2003-2005 cur dumps, and a streaming XML path for any dated snapshot in . The largest single snapshot in that archive is enwiki 2010-11-08: +In 2006 MediaWiki swapped its dumps from MySQL `INSERT INTO cur` syntax to XML. Arborist reads both — the SQL path above for 2003-2005 cur dumps, and a streaming XML path for any dated snapshot in . The largest single snapshot in that archive is enwiki 2010-11-08: | file | size | what | |---|---|---| @@ -187,7 +187,7 @@ make ingest-abstract The XML source streams `.xml.bz2` directly via `iterparse` with bounded memory (each `` is processed and cleared). Same shard / resume / Merkle contract as the SQL source. Title-prefix namespace filtering kicks in for older export schemas that omit per-page ``. -To ingest historical revisions instead of just the current snapshot, point `WP_XML` at a `pages-meta-history.xml.bz2` file and use `make ingest-xml-history` — the source emits one Document per revision and aborist's prior-doc detection chains them with `supersedes` edges. +To ingest historical revisions instead of just the current snapshot, point `WP_XML` at a `pages-meta-history.xml.bz2` file and use `make ingest-xml-history` — the source emits one Document per revision and arborist's prior-doc detection chains them with `supersedes` edges. ## Data: personal Grok export @@ -205,10 +205,10 @@ Both walk the export tree, find `prod-grok-backend.json`, and yield one Document ## Data: git and Mercurial repos (self-play) -Aborist can consult itself. Point a source at any local clone and every text file at HEAD becomes a queryable Document; re-ingesting after new commits chains old → new via `supersedes` edges, so the audit trail grows alongside the repo: +Arborist can consult itself. Point a source at any local clone and every text file at HEAD becomes a queryable Document; re-ingesting after new commits chains old → new via `supersedes` edges, so the audit trail grows alongside the repo: ``` -make ingest-self # this aborist tree, into ~/.aborist/shards/aborist-self.db +make ingest-self # this arborist tree, into ~/.arborist/shards/arborist-self.db make ingest-git GIT_REPO=/path/to/repo # any other git clone make ingest-hg HG_REPO=/path/to/repo # mercurial flavor ``` @@ -217,7 +217,7 @@ URI shape: `git:///file/` (no commit hash — that's w ## Data: live websites (the crawler) -Aborist can BFS-discover and ingest a website starting from a seed URL, respecting robots.txt and crawl delays. The crawler is **off by default** — heavy deps (aiohttp, bs4, lxml, mwparserfromhell, etc.) ship as the `[crawler]` extras and aren't pulled into the default test suite. +Arborist can BFS-discover and ingest a website starting from a seed URL, respecting robots.txt and crawl delays. The crawler is **off by default** — heavy deps (aiohttp, bs4, lxml, mwparserfromhell, etc.) ship as the `[crawler]` extras and aren't pulled into the default test suite. ``` make bootstrap-crawler # one-time, install [crawler] extras @@ -254,7 +254,7 @@ make recrawl-check DOMAIN=russell.ballestrini.net Conditional `If-None-Match` / `If-Modified-Since` HEAD requests classify each ingested doc as fresh (304), stale (200), gone (404/410), or unreachable. One tiny round trip per URL with no body transfer when content's unchanged. -The crawler is a verbatim lift from `~/git/agents.ai.unturf.com/core/` (provenance documented in `aborist/sources/crawler/__init__.py`); aborist-side changes drop the chat-bot fetch triggers and skip `web_cache_manager.py` (aborist has its own content-addressed cache). Run `make test-crawler` for the lift's own tests. +The crawler is a verbatim lift from `~/git/agents.ai.unturf.com/core/` (provenance documented in `arborist/sources/crawler/__init__.py`); arborist-side changes drop the chat-bot fetch triggers and skip `web_cache_manager.py` (arborist has its own content-addressed cache). Run `make test-crawler` for the lift's own tests. ## Asking the corpus @@ -272,7 +272,7 @@ The query path: 1. **Search** — FTS5 (body) + SQL `LIKE` (title) + `JOIN` over derivations (TF-IDF core keywords) across every shard. Three accept paths to the relevance filter. 2. **Concept overlay** — per-shard `concept_relations` SQLite table (corpus-derived, not hand-curated). Synonyms widen retrieval; rivalries narrow it unless the query uses comparative phrasing ("compare X vs Y"). Built-in extractor `link_reciprocity_synonym` reads the existing `edges` table for reciprocal A↔B link pairs and emits synonym edges between their title-tokens. ~1.6% storage tax measured on 6 GB Wikipedia. -3. **Context assembly** — top-K sources concatenated up to a 60 KB budget. Wikitext is stripped to plain prose via `aborist.wikitext.to_base()` (the corpus stores raw `[[wikilinks]]` so the link graph is recoverable on demand; the LLM and verifier both see clean prose). +3. **Context assembly** — top-K sources concatenated up to a 60 KB budget. Wikitext is stripped to plain prose via `arborist.wikitext.to_base()` (the corpus stores raw `[[wikilinks]]` so the link graph is recoverable on demand; the LLM and verifier both see clean prose). 4. **LLM** — Hermes-3 with strict attribution rules in the system prompt + a user-turn grounding reminder. 5. **Verifier** — every claim runs through a layered lexical check; the result rolls up into the v9.8 trichotomy (`audit_mode` ∈ STRICT / HYBRID / UNGROUNDED) at the schema layer AND a four-rung display ladder at render time (POINTER-LINKED → ANCHOR-WARRANTED → EVIDENCE-WARRANTED → UNGROUNDED). See below. 6. **Cache** — the v9.8 8-dim cache_key (`source_root | question_hash | model_profile | conversation | governance_policy | schema | canonicalization | chunking`) keys the answer in `qa.db`. Cache hits replay in ~100 ms. Per-phase timings in every result. @@ -280,9 +280,9 @@ The query path: LLM endpoint defaults to `https://hermes.ai.unturf.com/v1` (Hermes-3 Llama-3.1-8B, 82K context, no auth). Override: ``` -export ABORIST_LLM_ENDPOINT="https://your-vllm.example/v1" -export ABORIST_LLM_MODEL="meta-llama/Llama-3.1-70B-Instruct" -export ABORIST_LLM_API_KEY="..." +export ARBORIST_LLM_ENDPOINT="https://your-vllm.example/v1" +export ARBORIST_LLM_MODEL="meta-llama/Llama-3.1-70B-Instruct" +export ARBORIST_LLM_API_KEY="..." ``` ## Verifying answers (audit modes & label ladder) @@ -322,7 +322,7 @@ The `claim_lattice` path runs **seven deterministic hard checks**: parser succee Trailing `(Source: https://...)` parentheticals the model appends to verbatim source sentences are stripped before substring testing, so verbatim-with-citation no longer flags HYBRID. -`unverified_quotes` on each record is the corpus-growth signal — model output that didn't ground anywhere. `aborist emergent --aggregate` ranks them by frequency (the worklist of "things to ingest more sources for"). `aborist reclassify` re-runs the verifier against existing live records after corpus growth without any LLM call; HYBRID promotes to STRICT, UNGROUNDED to HYBRID, and one `providence_reclassify` audit event per change. +`unverified_quotes` on each record is the corpus-growth signal — model output that didn't ground anywhere. `arborist emergent --aggregate` ranks them by frequency (the worklist of "things to ingest more sources for"). `arborist reclassify` re-runs the verifier against existing live records after corpus growth without any LLM call; HYBRID promotes to STRICT, UNGROUNDED to HYBRID, and one `providence_reclassify` audit event per change. To dig into a specific record's unverified spans: @@ -334,7 +334,7 @@ Read-only **sidecar** diagnostic: pulls source chunks, classifies each unverifie ## Mesh / federation (off by default) -Optional gossip layer for peer-to-peer corpus sync. Default off — no code path touches the network unless `mesh.enabled` is set. See [`docs/mesh.md`](docs/mesh.md) for protocol contract and the `aborist mesh` CLI subcommands. +Optional gossip layer for peer-to-peer corpus sync. Default off — no code path touches the network unless `mesh.enabled` is set. See [`docs/mesh.md`](docs/mesh.md) for protocol contract and the `arborist mesh` CLI subcommands. ## Inspecting @@ -350,7 +350,7 @@ make activity ACTIVITY_LIMIT=20 ## Architecture -![Aborist module graph](docs/diagrams/aborist-modules.svg) +![Arborist module graph](docs/diagrams/arborist-modules.svg) Generated API reference (every module, class, and function from docstrings): build with `make docs-api` (output at `docs/_source/_build/html/`) or browse diff --git a/aborist/search/__init__.py b/aborist/search/__init__.py deleted file mode 100644 index 383ee59..0000000 --- a/aborist/search/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Search backends.""" - -from aborist.search.base import AuditMode, Hit, SearchBackend -from aborist.search.fts5 import FTS5Backend - -__all__ = ["AuditMode", "Hit", "SearchBackend", "FTS5Backend"] diff --git a/aborist/__init__.py b/arborist/__init__.py similarity index 76% rename from aborist/__init__.py rename to arborist/__init__.py index 1fda08b..776684e 100644 --- a/aborist/__init__.py +++ b/arborist/__init__.py @@ -1,4 +1,4 @@ -"""aborist — an arborist for trees and forests of cross-linked information.""" +"""arborist — an arborist for trees and forests of cross-linked information.""" __version__ = "0.0.1" diff --git a/aborist/cli.py b/arborist/cli.py similarity index 96% rename from aborist/cli.py rename to arborist/cli.py index e3da553..428fd19 100644 --- a/aborist/cli.py +++ b/arborist/cli.py @@ -1,4 +1,4 @@ -"""Aborist CLI: ingest / search / verify / stats.""" +"""Arborist CLI: ingest / search / verify / stats.""" from __future__ import annotations @@ -8,12 +8,12 @@ import os import sys from pathlib import Path -from aborist import __version__ -from aborist.ingest import ingest_source, verify_random_sample -from aborist.progress import Progress -from aborist.search import FTS5Backend -from aborist.sources import WikipediaCurDump -from aborist.store import ( +from arborist import __version__ +from arborist.ingest import ingest_source, verify_random_sample +from arborist.progress import Progress +from arborist.search import FTS5Backend +from arborist.sources import WikipediaCurDump +from arborist.store import ( DEFAULT_DB_PATH, append_audit, connect, @@ -29,7 +29,7 @@ def _cmd_ingest(args: argparse.Namespace) -> int: if not args.path: print(f"--path is required for {args.source}", file=sys.stderr) return 2 - from aborist.sources import WikipediaSqlDump + from arborist.sources import WikipediaSqlDump table = "cur" if args.source == "wikipedia_cur" else "old" shard = None @@ -39,10 +39,10 @@ def _cmd_ingest(args: argparse.Namespace) -> int: src = WikipediaSqlDump(path=args.path, table=table, shard=shard) elif args.source == "html": # noqa: SIM114 — keep branch shape try: - from aborist.sources import HtmlPageSource + from arborist.sources import HtmlPageSource except ImportError: print( - "html source requires extras: pip install 'aborist[html]'", + "html source requires extras: pip install 'arborist[html]'", file=sys.stderr, ) return 2 @@ -61,7 +61,7 @@ def _cmd_ingest(args: argparse.Namespace) -> int: if not args.path: print(f"--path is required for {args.source}", file=sys.stderr) return 2 - from aborist.sources import GrokExportSource, GrokMediaPostsSource + from arborist.sources import GrokExportSource, GrokMediaPostsSource cls = GrokExportSource if args.source == "grok_export" else GrokMediaPostsSource src = cls(path=args.path) @@ -69,7 +69,7 @@ def _cmd_ingest(args: argparse.Namespace) -> int: if not args.path: print(f"--path is required for {args.source}", file=sys.stderr) return 2 - from aborist.sources import WikipediaAbstractDump, WikipediaXmlDump + from arborist.sources import WikipediaAbstractDump, WikipediaXmlDump if args.source == "wikipedia_abstract": src = WikipediaAbstractDump(path=args.path) @@ -87,7 +87,7 @@ def _cmd_ingest(args: argparse.Namespace) -> int: if not args.path: print(f"--path is required for {args.source}", file=sys.stderr) return 2 - from aborist.sources import GitRepoSource, MercurialRepoSource + from arborist.sources import GitRepoSource, MercurialRepoSource cls = GitRepoSource if args.source == "git_repo" else MercurialRepoSource src = cls(repo_path=args.path) @@ -95,11 +95,11 @@ def _cmd_ingest(args: argparse.Namespace) -> int: # Self-reference: promote STRICT live providence_cache records # past the kindergarten window into the document corpus. # See docs/self-reference-design.md. - from aborist.sources.providence import ( + from arborist.sources.providence import ( DEFAULT_KINDERGARTEN_SECONDS, ProvidenceSource, ) - from aborist.store import connect + from arborist.store import connect # The source reads from the SAME shard it's writing into — # promote each shard's own STRICT records to its own @@ -227,9 +227,9 @@ def _cmd_verify(args: argparse.Namespace) -> int: def _cmd_distill(args: argparse.Namespace) -> int: """Distill existing documents into cores (surface→core, or core→core+1).""" - from aborist.distill import get_distiller - from aborist.distill.runner import distill_existing - from aborist.store import discover_shards + from arborist.distill import get_distiller + from arborist.distill.runner import distill_existing + from arborist.store import discover_shards try: distiller = get_distiller(args.process) @@ -295,19 +295,19 @@ def _cmd_ask(args: argparse.Namespace) -> int: """Ask a question against one document; verifier classifies the answer.""" import os - from aborist.qa import ask - from aborist.qa.client import OpenAICompatibleClient, StubClient + from arborist.qa import ask + from arborist.qa.client import OpenAICompatibleClient, StubClient base_url = args.endpoint or os.environ.get( - "ABORIST_LLM_ENDPOINT", "https://hermes.ai.unturf.com/v1" + "ARBORIST_LLM_ENDPOINT", "https://hermes.ai.unturf.com/v1" ) model = args.model or os.environ.get( - "ABORIST_LLM_MODEL", + "ARBORIST_LLM_MODEL", "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic", ) - revision = os.environ.get("ABORIST_LLM_REVISION", "") - quantization = os.environ.get("ABORIST_LLM_QUANTIZATION", "fp8-dynamic") - api_key = os.environ.get("ABORIST_LLM_API_KEY") + revision = os.environ.get("ARBORIST_LLM_REVISION", "") + quantization = os.environ.get("ARBORIST_LLM_QUANTIZATION", "fp8-dynamic") + api_key = os.environ.get("ARBORIST_LLM_API_KEY") client: object if args.dry_run: @@ -324,7 +324,7 @@ def _cmd_ask(args: argparse.Namespace) -> int: ) # Per-call policy override for --answer-mode. Other knobs flow from # DEFAULT_POLICY. - from aborist.qa.runner import DEFAULT_POLICY as _DEFAULT_ASK_POLICY + from arborist.qa.runner import DEFAULT_POLICY as _DEFAULT_ASK_POLICY call_policy = dict(_DEFAULT_ASK_POLICY) if getattr(args, "answer_mode", None): call_policy["answer_mode"] = args.answer_mode @@ -349,19 +349,19 @@ def _cmd_query(args: argparse.Namespace) -> int: """Multi-source RAG: question -> top-K corpus docs -> Hermes -> cache.""" import os - from aborist.qa.client import OpenAICompatibleClient, StubClient - from aborist.qa.query import query + from arborist.qa.client import OpenAICompatibleClient, StubClient + from arborist.qa.query import query base_url = args.endpoint or os.environ.get( - "ABORIST_LLM_ENDPOINT", "https://hermes.ai.unturf.com/v1" + "ARBORIST_LLM_ENDPOINT", "https://hermes.ai.unturf.com/v1" ) model = args.model or os.environ.get( - "ABORIST_LLM_MODEL", + "ARBORIST_LLM_MODEL", "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic", ) - revision = os.environ.get("ABORIST_LLM_REVISION", "") - quantization = os.environ.get("ABORIST_LLM_QUANTIZATION", "fp8-dynamic") - api_key = os.environ.get("ABORIST_LLM_API_KEY") + revision = os.environ.get("ARBORIST_LLM_REVISION", "") + quantization = os.environ.get("ARBORIST_LLM_QUANTIZATION", "fp8-dynamic") + api_key = os.environ.get("ARBORIST_LLM_API_KEY") client: object if args.dry_run: @@ -376,7 +376,7 @@ def _cmd_query(args: argparse.Namespace) -> int: if args.global_shards_dir: qa_db = Path(args.global_shards_dir) / "qa.db" else: - qa_db = Path.home() / ".aborist" / "qa.db" + qa_db = Path.home() / ".arborist" / "qa.db" qa_db = Path(qa_db) shards_dir = ( @@ -387,7 +387,7 @@ def _cmd_query(args: argparse.Namespace) -> int: # Apply per-call policy overrides (question_dedup, repair, answer_mode) # on top of the default. fidelity is a function-level kwarg, not in # the policy. - from aborist.qa.query import DEFAULT_QUERY_POLICY + from arborist.qa.query import DEFAULT_QUERY_POLICY call_policy = dict(DEFAULT_QUERY_POLICY) if getattr(args, "question_dedup", None): call_policy["question_dedup"] = args.question_dedup @@ -415,7 +415,7 @@ def _cmd_query(args: argparse.Namespace) -> int: if getattr(args, "reject_broad", False): # Phase 4 reject-broad: the actual rejection happens inside # query() via the policy field; this CLI flag just sets the - # field. See aborist/qa/query.py for the early-return path. + # field. See arborist/qa/query.py for the early-return path. call_policy["quantifier_reject_broad"] = True if getattr(args, "apply_quantifier_caps", False): # Operator opts in to flipping the dry-run gate per-call. @@ -454,7 +454,7 @@ def _cmd_query(args: argparse.Namespace) -> int: ) # Emit unfirehose-compatible session journal. One JSONL file per - # `make query` invocation, written to ~/.aborist/unfirehose/{slug}/ + # `make query` invocation, written to ~/.arborist/unfirehose/{slug}/ # {session_uuid}.jsonl. Unfirehose's native-harness watcher picks # this up automatically (no registration). Failures here must NEVER # break the query path — wrap in a broad except & swallow. @@ -476,10 +476,10 @@ def _cmd_query(args: argparse.Namespace) -> int: def _emit_query_journal(question: str, result: dict, model: str) -> None: """Write one unfirehose/1.0 session for this query invocation.""" - from aborist.journal import SessionWriter + from arborist.journal import SessionWriter timings = result.get("timings") or {} answer = result.get("answer_text") or "" - aborist_meta = { + arborist_meta = { "audit_mode": result.get("audit_mode"), "verifier_method": result.get("verifier_method"), "n_quotes": result.get("n_quotes"), @@ -506,7 +506,7 @@ def _emit_query_journal(question: str, result: dict, model: str) -> None: provider="hermes", stop_reason="end_turn", duration_ms=int(timings.get("total_ms") or 0) or None, - aborist_meta=aborist_meta, + arborist_meta=arborist_meta, ) @@ -737,7 +737,7 @@ def _maybe_render_json_envelope_as_bullets(answer: str) -> str: if "claims" not in stripped: return answer try: - from aborist.qa.verify import _lenient_json_parse + from arborist.qa.verify import _lenient_json_parse parsed, _fixups = _lenient_json_parse(answer) except Exception: return answer @@ -1034,14 +1034,14 @@ def _cmd_inspect(args: argparse.Namespace) -> int: classifies each unverified span. Read-only; no audit events, no providence_cache mutations. """ - from aborist.qa.inspect import inspect_cache_key + from arborist.qa.inspect import inspect_cache_key qa_db = args.qa_db if qa_db is None: qa_db = ( Path(args.global_shards_dir) / "qa.db" if args.global_shards_dir - else Path.home() / ".aborist" / "qa.db" + else Path.home() / ".arborist" / "qa.db" ) shards_dir = ( Path(args.global_shards_dir) if args.global_shards_dir else None @@ -1167,7 +1167,7 @@ def _falsify_cache_key( """ import time as _time - from aborist.store import append_audit, discover_shards, transaction + from arborist.store import append_audit, discover_shards, transaction if state not in ("failed", "stale", "quarantined"): return {"status": "invalid_state", "value": state} @@ -1246,12 +1246,12 @@ def _burn_cache_key( referencing this cache_key. If any exist, refuse without ``--force``. Always writes a 'providence_burn' audit event so the chain records - that a leaf was removed and why. Use ``aborist providence --falsify`` + that a leaf was removed and why. Use ``arborist providence --falsify`` instead when downstream consumers may have built on this answer. """ import time as _time - from aborist.store import append_audit, discover_shards, transaction + from arborist.store import append_audit, discover_shards, transaction paths: list[Path] = ( discover_shards(shards_dir) if shards_dir else [Path(db_path)] @@ -1381,7 +1381,7 @@ def _burn_document_root( """ import time as _time - from aborist.store import append_audit, discover_shards, transaction + from arborist.store import append_audit, discover_shards, transaction paths: list[Path] = ( discover_shards(shards_dir) if shards_dir else [Path(db_path)] @@ -1501,7 +1501,7 @@ def _burn_core_root( """ import time as _time - from aborist.store import append_audit, discover_shards, transaction + from arborist.store import append_audit, discover_shards, transaction paths: list[Path] = ( discover_shards(shards_dir) if shards_dir else [Path(db_path)] @@ -1614,7 +1614,7 @@ def _cmd_burn_kindergarten(args: argparse.Namespace) -> int: chain integrity is verifiable via `make chain-check-shards` after. """ import time as _time - from aborist.store import discover_shards + from arborist.store import discover_shards now = int(_time.time()) # `kindergarten_seconds <= 0` means "no time gate — burn every live @@ -1901,7 +1901,7 @@ def _cmd_providence_show_preflight( # Verify the persisted payload hashes to the persisted leaf. # Mismatch would indicate post-write tampering or a serialization # drift; surface it explicitly so an auditor can detect. - from aborist.qa.dag import _canonical_json, _sha256_hex + from arborist.qa.dag import _canonical_json, _sha256_hex recomputed = _sha256_hex(_canonical_json(payload)) out["preflight_payload"] = payload out["payload_hash_check"] = ( @@ -1921,7 +1921,7 @@ def _cmd_providence_show_preflight( def _load_record_context(row, shards_dir, qa_db): """Reassemble context for a providence record. Returns text or None if any source doc has no hot chunks (cold).""" - from aborist.qa.query import _load_doc_text + from arborist.qa.query import _load_doc_text proof = json.loads(row["merkle_proof"]) sources = proof.get("sources", []) @@ -1955,7 +1955,7 @@ def _cmd_reclassify(args: argparse.Namespace) -> int: Cold-source records (where any source doc has no hot chunks) are skipped: we can't faithfully reclassify without the original context. - Run `aborist rehydrate` first if you want those covered too. + Run `arborist rehydrate` first if you want those covered too. `--compare` runs all four entity policies side-by-side without writing — use it to see what each policy would produce on real data @@ -1968,7 +1968,7 @@ def _cmd_reclassify(args: argparse.Namespace) -> int: import time from collections import defaultdict - from aborist.qa.verify import ( + from arborist.qa.verify import ( DEFAULT_ENTITY_POLICY, ENTITY_POLICIES, verify_quotes, @@ -1979,7 +1979,7 @@ def _cmd_reclassify(args: argparse.Namespace) -> int: if args.global_shards_dir: qa_db = Path(args.global_shards_dir) / "qa.db" else: - qa_db = Path.home() / ".aborist" / "qa.db" + qa_db = Path.home() / ".arborist" / "qa.db" qa_db = Path(qa_db) shards_dir = ( @@ -2199,7 +2199,7 @@ def _cmd_emergent(args: argparse.Namespace) -> int: def _cmd_evict(args: argparse.Namespace) -> int: """Demote surface chunks from hot to cold (NULL content). Cores never evict.""" - from aborist.evict import evict_to_cold + from arborist.evict import evict_to_cold conn = ( connect_query(args.db, shards_dir=args.global_shards_dir) @@ -2221,7 +2221,7 @@ def _cmd_evict(args: argparse.Namespace) -> int: def _cmd_rehydrate(args: argparse.Namespace) -> int: """Rehydrate cold chunks from source; non-zero exit if drift detected.""" - from aborist.evict import rehydrate + from arborist.evict import rehydrate conn = ( connect_query(args.db, shards_dir=args.global_shards_dir) @@ -2264,7 +2264,7 @@ def _cmd_activity(args: argparse.Namespace) -> int: """ import time as _time - from aborist.store import discover_shards + from arborist.store import discover_shards shard_paths: list[Path] = [] if args.global_shards_dir: @@ -2465,7 +2465,7 @@ def _check_audit_chain(conn: sqlite3.Connection) -> tuple[int, int]: def _cmd_analyze(args: argparse.Namespace) -> int: """Compression spectrum, depth distribution, audit chain integrity.""" - from aborist.store import discover_shards + from arborist.store import discover_shards # In sharded mode, audit chains live per-shard (each shard has its own # genesis -> latest). Check each independently and aggregate. @@ -2591,12 +2591,12 @@ def _cmd_snapshot_create(args: argparse.Namespace) -> int: Cross-shard mode (--shards-dir + --db): read against the in-memory UNION view to get the cluster-level Merkle root, then persist into args.db (a dedicated snapshots store, conventionally - `~/.aborist/shards/snapshots.db`). The writer's own documents table + `~/.arborist/shards/snapshots.db`). The writer's own documents table is irrelevant to the snapshot value — only the union scope counts. """ import time as _time - from aborist.snapshot import compute_snapshot_root, create_snapshot + from arborist.snapshot import compute_snapshot_root, create_snapshot if args.global_shards_dir is None: conn = connect(args.db) @@ -2677,7 +2677,7 @@ def _cmd_snapshot_create(args: argparse.Namespace) -> int: def _cmd_snapshot_list(args: argparse.Namespace) -> int: """List recent corpus snapshots (newest first, --limit N).""" - from aborist.snapshot import list_snapshots + from arborist.snapshot import list_snapshots conn = connect(args.db) try: @@ -2690,7 +2690,7 @@ def _cmd_snapshot_list(args: argparse.Namespace) -> int: def _cmd_snapshot_verify(args: argparse.Namespace) -> int: """Re-derive snapshot root from current corpus; non-zero exit on drift.""" - from aborist.snapshot import verify_snapshot + from arborist.snapshot import verify_snapshot conn = ( connect_query(args.db, shards_dir=args.global_shards_dir) @@ -2706,7 +2706,7 @@ def _cmd_snapshot_verify(args: argparse.Namespace) -> int: def _cmd_snapshot_diff(args: argparse.Namespace) -> int: - from aborist.snapshot import diff_against_current + from arborist.snapshot import diff_against_current conn = ( connect_query(args.db, shards_dir=args.global_shards_dir) @@ -2723,8 +2723,8 @@ def _cmd_snapshot_diff(args: argparse.Namespace) -> int: def _cmd_mesh_status(args: argparse.Namespace) -> int: """Show mesh state: enabled flag, identity, current epoch, roster.""" - from aborist.mesh import current_epoch, is_enabled, load_identity - from aborist.mesh.state import roster_at + from arborist.mesh import current_epoch, is_enabled, load_identity + from arborist.mesh.state import roster_at conn = connect(args.db) try: @@ -2762,7 +2762,7 @@ def _cmd_mesh_status(args: argparse.Namespace) -> int: def _cmd_mesh_init(args: argparse.Namespace) -> int: - from aborist.mesh import init_identity + from arborist.mesh import init_identity conn = connect(args.db) try: @@ -2790,7 +2790,7 @@ def _cmd_mesh_init(args: argparse.Namespace) -> int: def _cmd_mesh_enable(args: argparse.Namespace) -> int: - from aborist.mesh import set_enabled + from arborist.mesh import set_enabled conn = connect(args.db) try: @@ -2802,7 +2802,7 @@ def _cmd_mesh_enable(args: argparse.Namespace) -> int: def _cmd_mesh_disable(args: argparse.Namespace) -> int: - from aborist.mesh import set_enabled + from arborist.mesh import set_enabled conn = connect(args.db) try: @@ -2814,8 +2814,8 @@ def _cmd_mesh_disable(args: argparse.Namespace) -> int: def _cmd_mesh_members(args: argparse.Namespace) -> int: - from aborist.mesh import current_epoch - from aborist.mesh.state import roster_at + from arborist.mesh import current_epoch + from arborist.mesh.state import roster_at conn = connect(args.db) try: @@ -2847,7 +2847,7 @@ def _cmd_mesh_members(args: argparse.Namespace) -> int: def _cmd_mesh_add(args: argparse.Namespace) -> int: - from aborist.mesh.members import add_member + from arborist.mesh.members import add_member try: sign_pub = bytes.fromhex(args.sign_pub) @@ -2878,7 +2878,7 @@ def _cmd_mesh_add(args: argparse.Namespace) -> int: def _cmd_mesh_kick(args: argparse.Namespace) -> int: - from aborist.mesh.members import kick_member + from arborist.mesh.members import kick_member conn = connect(args.db) try: @@ -2898,7 +2898,7 @@ def _cmd_mesh_kick(args: argparse.Namespace) -> int: def _cmd_mesh_rotate(args: argparse.Namespace) -> int: - from aborist.mesh.members import scheduled_rotate + from arborist.mesh.members import scheduled_rotate conn = connect(args.db) try: @@ -2914,16 +2914,16 @@ def _cmd_mesh_rotate(args: argparse.Namespace) -> int: def _cmd_mesh_serve(args: argparse.Namespace) -> int: """Run the HTTP gossip server until SIGINT.""" - from aborist.mesh import is_enabled, load_identity - from aborist.mesh.wire import MeshWireServer + from arborist.mesh import is_enabled, load_identity + from arborist.mesh.wire import MeshWireServer conn = connect(args.db) try: if load_identity(conn) is None: - print("error: mesh not initialized; run 'aborist mesh init' first", file=sys.stderr) + print("error: mesh not initialized; run 'arborist mesh init' first", file=sys.stderr) return 2 if not is_enabled(conn): - print("error: mesh.enabled is off; run 'aborist mesh enable' first", file=sys.stderr) + print("error: mesh.enabled is off; run 'arborist mesh enable' first", file=sys.stderr) return 2 finally: conn.close() @@ -2971,8 +2971,8 @@ def _cmd_mesh_sync(args: argparse.Namespace) -> int: """ import time as _time - from aborist.mesh import is_enabled, load_identity - from aborist.mesh.wire import MeshWireClient + from arborist.mesh import is_enabled, load_identity + from arborist.mesh.wire import MeshWireClient now_ts = int(_time.time()) cutoff_ts = now_ts - max(0, args.kindergarten_seconds) @@ -3085,10 +3085,10 @@ def _cmd_mesh_pull(args: argparse.Namespace) -> int: canonicalization_version stay consistent — and rejects with rc=2 if the local re-ingest produces a different document_root than requested. """ - from aborist.document import Document - from aborist.ingest import ingest_source - from aborist.mesh import is_enabled, load_identity - from aborist.mesh.wire import MeshWireClient + from arborist.document import Document + from arborist.ingest import ingest_source + from arborist.mesh import is_enabled, load_identity + from arborist.mesh.wire import MeshWireClient conn = connect(args.db) try: @@ -3188,19 +3188,19 @@ def _cmd_crawl(args: argparse.Namespace) -> int: Two modes: - default: print discovered URLs to stdout (one per line). Compose - with `aborist ingest --source html` if you want to feed them + with `arborist ingest --source html` if you want to feed them through the standard ingest path manually. - ``--ingest``: run the discovery + ingest path in a single shot, capturing ETag + Last-Modified per page so a future ``crawler recrawl-check`` can do conditional HEADs. """ try: - from aborist.sources.crawler.bridge import crawl_seed, ingest_crawled + from arborist.sources.crawler.bridge import crawl_seed, ingest_crawled except ImportError as e: print(f"error: {e}", file=sys.stderr) return 2 - from aborist.progress import Progress + from arborist.progress import Progress cap = "no cap" if args.max_pages == 0 else f"max {args.max_pages}" speed = "fast" if args.fast else "polite" @@ -3262,7 +3262,7 @@ def _cmd_crawler_recrawl_check(args: argparse.Namespace) -> int: first. """ try: - from aborist.sources.crawler.bridge import recrawl_check + from arborist.sources.crawler.bridge import recrawl_check except ImportError as e: print(f"error: {e}", file=sys.stderr) return 2 @@ -3282,15 +3282,15 @@ def _cmd_crawler_recrawl_check(args: argparse.Namespace) -> int: def build_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser( - prog="aborist", + prog="arborist", description="An arborist for trees and forests of cross-linked information.", ) - p.add_argument("--version", action="version", version=f"aborist {__version__}") + p.add_argument("--version", action="version", version=f"arborist {__version__}") p.add_argument( "--db", type=Path, default=DEFAULT_DB_PATH, - help=f"path to aborist SQLite db (default: {DEFAULT_DB_PATH})", + help=f"path to arborist SQLite db (default: {DEFAULT_DB_PATH})", ) p.add_argument( "--shards-dir", @@ -3382,7 +3382,7 @@ def build_parser() -> argparse.ArgumentParser: help=( "directory for attach-forever sharding. With --shard rank/total, " "writes to shards-dir/.db instead of --db, removing the " - "WAL writer-lock contention entirely. Reads via aborist --shards-dir" + "WAL writer-lock contention entirely. Reads via arborist --shards-dir" ), ) ingest.add_argument( @@ -3480,12 +3480,12 @@ def build_parser() -> argparse.ArgumentParser: ask_cmd.add_argument( "--model", default=None, - help="model_id (default $ABORIST_LLM_MODEL or hermes-3)", + help="model_id (default $ARBORIST_LLM_MODEL or hermes-3)", ) ask_cmd.add_argument( "--endpoint", default=None, - help="OpenAI-compatible base URL (default $ABORIST_LLM_ENDPOINT)", + help="OpenAI-compatible base URL (default $ARBORIST_LLM_ENDPOINT)", ) ask_cmd.add_argument( "--dry-run", @@ -3550,16 +3550,16 @@ def build_parser() -> argparse.ArgumentParser: "--qa-db", dest="qa_db", default=None, help=( "providence_cache target DB. default: /qa.db, or " - "~/.aborist/qa.db when no shards-dir" + "~/.arborist/qa.db when no shards-dir" ), ) query_cmd.add_argument( "--model", default=None, - help="model_id (default $ABORIST_LLM_MODEL or hermes-3)", + help="model_id (default $ARBORIST_LLM_MODEL or hermes-3)", ) query_cmd.add_argument( "--endpoint", default=None, - help="OpenAI-compatible base URL (default $ABORIST_LLM_ENDPOINT)", + help="OpenAI-compatible base URL (default $ARBORIST_LLM_ENDPOINT)", ) query_cmd.add_argument( "--dry-run", dest="dry_run", action="store_true", @@ -3726,7 +3726,7 @@ def build_parser() -> argparse.ArgumentParser: ) inspect_cmd.add_argument( "--qa-db", dest="qa_db", default=None, - help="path to qa.db (default: /qa.db or ~/.aborist/qa.db)", + help="path to qa.db (default: /qa.db or ~/.arborist/qa.db)", ) inspect_cmd.add_argument( "--json", action="store_true", @@ -3873,7 +3873,7 @@ def build_parser() -> argparse.ArgumentParser: ) reclassify_cmd.add_argument( "--qa-db", dest="qa_db", default=None, - help="path to qa.db (default: /qa.db or ~/.aborist/qa.db)", + help="path to qa.db (default: /qa.db or ~/.arborist/qa.db)", ) reclassify_cmd.add_argument( "--limit", type=int, default=0, @@ -4123,7 +4123,7 @@ def build_parser() -> argparse.ArgumentParser: help=( "BFS-discover same-domain URLs from a seed; optionally ingest " "and store ETag/Last-Modified for cheap recrawl-checks " - "(requires aborist[crawler] extras)" + "(requires arborist[crawler] extras)" ), ) crawl_cmd.add_argument("--seed-url", dest="seed_url", required=True) diff --git a/aborist/compress.py b/arborist/compress.py similarity index 100% rename from aborist/compress.py rename to arborist/compress.py diff --git a/aborist/concepts/__init__.py b/arborist/concepts/__init__.py similarity index 86% rename from aborist/concepts/__init__.py rename to arborist/concepts/__init__.py index d87bd90..db38360 100644 --- a/aborist/concepts/__init__.py +++ b/arborist/concepts/__init__.py @@ -1,6 +1,6 @@ """Corpus-derived concept relations: synonyms, antonyms, rivalries, categories. -Replaces the hand-curated frozensets that lived in ``aborist.qa.concepts`` +Replaces the hand-curated frozensets that lived in ``arborist.qa.concepts`` through April 2026 (commit c6182ae). The frozensets were Phase 1; this is Phase 2. @@ -18,7 +18,7 @@ Architecture: - ``seed.py`` — One-time migration of the legacy frozensets to manual rows Public API for retrieval-time use (matches the legacy -``aborist.qa.concepts`` shape, so call sites in ``query.py`` keep working): +``arborist.qa.concepts`` shape, so call sites in ``query.py`` keep working): synonym_expand(tokens, *, shards_dir) -> set[str] rivalry_excluded(tokens, *, shards_dir, compare_phrasing=False) -> set[str] @@ -27,13 +27,13 @@ Public API for retrieval-time use (matches the legacy from __future__ import annotations -from aborist.concepts.query import ( +from arborist.concepts.query import ( has_compare_phrasing, invalidate_cache, rivalry_excluded, synonym_expand, ) -from aborist.concepts.store import ( +from arborist.concepts.store import ( add_concept_relation, concept_relations_for_token, purge_by_evidence_kind, diff --git a/aborist/concepts/extract.py b/arborist/concepts/extract.py similarity index 97% rename from aborist/concepts/extract.py rename to arborist/concepts/extract.py index 03e92fc..bea9927 100644 --- a/aborist/concepts/extract.py +++ b/arborist/concepts/extract.py @@ -5,7 +5,7 @@ extractors just read those rows & emit concept_relations. Each extractor has a stable ``evidence_kind`` string that lets an operator purge its output cleanly via -``aborist concepts purge --evidence-kind X``. +``arborist concepts purge --evidence-kind X``. Built-in extractors: @@ -21,7 +21,7 @@ Built-in extractors: Adding a new extractor: implement a callable ``(conn, *, derived_from) -> dict[str, int]`` that walks the shard & calls ``add_concept_relation`` for each finding. Register it under -a stable evidence_kind string. ``aborist concepts derive`` reads from +a stable evidence_kind string. ``arborist concepts derive`` reads from EXTRACTORS to dispatch. """ @@ -32,7 +32,7 @@ import sqlite3 import time from typing import Callable -from aborist.concepts.store import add_concept_relation +from arborist.concepts.store import add_concept_relation # Tokens too generic to use as anchor for a synonym edge. A reciprocal # link between two pages whose titles only share "the", "of", "and" @@ -237,7 +237,7 @@ def backfill_token_idf( # Registry: evidence_kind → extractor callable. # Adding a new extractor: pick a stable evidence_kind string, implement # the (conn, *, derived_from) -> dict signature, register it here. -# CLI command `aborist concepts derive --extractor X` reads this map. +# CLI command `arborist concepts derive --extractor X` reads this map. def backfill_documents_fts( conn: sqlite3.Connection, *, diff --git a/aborist/concepts/query.py b/arborist/concepts/query.py similarity index 97% rename from aborist/concepts/query.py rename to arborist/concepts/query.py index c278e75..d0a29e5 100644 --- a/aborist/concepts/query.py +++ b/arborist/concepts/query.py @@ -1,6 +1,6 @@ """Retrieval-time concept lookup. Cross-shard, read-only. -Public API matches the legacy ``aborist.qa.concepts`` shape so existing +Public API matches the legacy ``arborist.qa.concepts`` shape so existing call sites in ``query.py`` keep working unchanged. Behavior changes: - Backed by the ``concept_relations`` SQLite table instead of in-Python @@ -14,7 +14,7 @@ call sites in ``query.py`` keep working unchanged. Behavior changes: Cache: a per-process LRU keyed on ``shards_dir`` mtime. Lookups in a hot loop don't re-walk shards. Cache invalidates when any shard file's -mtime changes (e.g. after `aborist concepts derive` writes new rows). +mtime changes (e.g. after `arborist concepts derive` writes new rows). """ from __future__ import annotations @@ -23,7 +23,7 @@ import re import time from pathlib import Path -from aborist.store import connect_query +from arborist.store import connect_query # Tokens that mean "user wants both sides of any rivalry shown" — # kept here (not in DB) because compare-phrasing detection is a @@ -176,7 +176,7 @@ def invalidate_cache() -> None: # --------------------------------------------------------------------------- -# Public API — matches the legacy ``aborist.qa.concepts`` shape +# Public API — matches the legacy ``arborist.qa.concepts`` shape # --------------------------------------------------------------------------- diff --git a/aborist/concepts/seed.py b/arborist/concepts/seed.py similarity index 94% rename from aborist/concepts/seed.py rename to arborist/concepts/seed.py index 5644839..a4bfcbd 100644 --- a/aborist/concepts/seed.py +++ b/arborist/concepts/seed.py @@ -1,7 +1,7 @@ """One-time migration: legacy frozensets → concept_relations rows. Pre-2026-05-01 the synonym & rivalry data lived as hand-curated frozensets -in ``aborist.qa.concepts``. This module preserves those tuples & writes +in ``arborist.qa.concepts``. This module preserves those tuples & writes them as ``evidence_kind='manual_legacy'`` rows so the DB-backed lookup returns the same answers the frozenset lookup did. @@ -18,7 +18,7 @@ from __future__ import annotations import sqlite3 -from aborist.concepts.store import add_concept_relation +from arborist.concepts.store import add_concept_relation # Sentinel source_root for legacy seed data. Real concept relations # from corpus extractors use the document's actual document_root so @@ -99,7 +99,7 @@ def seed_legacy_concepts(conn: sqlite3.Connection) -> dict: token=a, target=b, evidence_kind=LEGACY_EVIDENCE_KIND, - derived_from="aborist.qa.concepts (legacy frozensets)", + derived_from="arborist.qa.concepts (legacy frozensets)", ) if inserted: syn_ins += 1 @@ -116,7 +116,7 @@ def seed_legacy_concepts(conn: sqlite3.Connection) -> dict: token=a, target=b, evidence_kind=LEGACY_EVIDENCE_KIND, - derived_from="aborist.qa.concepts (legacy frozensets)", + derived_from="arborist.qa.concepts (legacy frozensets)", ) if inserted: riv_ins += 1 diff --git a/aborist/concepts/store.py b/arborist/concepts/store.py similarity index 97% rename from aborist/concepts/store.py rename to arborist/concepts/store.py index 4af39fd..7559689 100644 --- a/aborist/concepts/store.py +++ b/arborist/concepts/store.py @@ -1,7 +1,7 @@ """DB read/write helpers for concept_relations. All operations are scoped to a single shard connection. Cross-shard -queries live in ``aborist.concepts.query``. +queries live in ``arborist.concepts.query``. Append-only by design: ``add_concept_relation`` uses INSERT OR IGNORE on the UNIQUE (source_root, relation_kind, token, target, evidence_kind) @@ -116,7 +116,7 @@ def purge_by_evidence_kind( def list_evidence_kinds(conn: sqlite3.Connection) -> list[tuple[str, int]]: """Return ``[(evidence_kind, row_count), ...]`` for the shard, ordered - by row_count descending. Useful for ``aborist concepts list --kinds``.""" + by row_count descending. Useful for ``arborist concepts list --kinds``.""" rows = conn.execute( "SELECT evidence_kind, COUNT(*) AS n " "FROM concept_relations GROUP BY evidence_kind ORDER BY n DESC" diff --git a/aborist/distill/__init__.py b/arborist/distill/__init__.py similarity index 70% rename from aborist/distill/__init__.py rename to arborist/distill/__init__.py index 2756c55..1ebf191 100644 --- a/aborist/distill/__init__.py +++ b/arborist/distill/__init__.py @@ -1,8 +1,8 @@ """Distillation: surface docs -> core docs, Merkle-signed back.""" -from aborist.distill.base import DistillationResult, Distiller -from aborist.distill.first_sentence import FirstSentenceDistiller -from aborist.distill.tfidf import TfidfKeywordDistiller +from arborist.distill.base import DistillationResult, Distiller +from arborist.distill.first_sentence import FirstSentenceDistiller +from arborist.distill.tfidf import TfidfKeywordDistiller __all__ = [ "DistillationResult", diff --git a/aborist/distill/base.py b/arborist/distill/base.py similarity index 96% rename from aborist/distill/base.py rename to arborist/distill/base.py index 80be4a8..02e2d24 100644 --- a/aborist/distill/base.py +++ b/arborist/distill/base.py @@ -10,7 +10,7 @@ from __future__ import annotations from abc import ABC, abstractmethod from dataclasses import dataclass -from aborist.document import Document +from arborist.document import Document @dataclass diff --git a/aborist/distill/first_sentence.py b/arborist/distill/first_sentence.py similarity index 95% rename from aborist/distill/first_sentence.py rename to arborist/distill/first_sentence.py index 15f0a76..ec95acc 100644 --- a/aborist/distill/first_sentence.py +++ b/arborist/distill/first_sentence.py @@ -9,8 +9,8 @@ from __future__ import annotations import re -from aborist.distill.base import DistillationResult, Distiller -from aborist.document import Document +from arborist.distill.base import DistillationResult, Distiller +from arborist.document import Document _SENTENCE_BOUNDARY = re.compile(r"(?<=[.!?])\s+(?=[A-Z0-9])") _MIN_SENTENCE_LEN = 10 diff --git a/aborist/distill/runner.py b/arborist/distill/runner.py similarity index 97% rename from aborist/distill/runner.py rename to arborist/distill/runner.py index a8983d7..a32b9ca 100644 --- a/aborist/distill/runner.py +++ b/arborist/distill/runner.py @@ -22,15 +22,15 @@ import sqlite3 import time from dataclasses import dataclass -from aborist import ( +from arborist import ( CANONICALIZATION_VERSION, SCHEMA_VERSION, ) -from aborist.compress import unpack_chunk -from aborist.distill.base import Distiller -from aborist.document import Document, canonicalize, get_chunker -from aborist.merkle import MerkleTree, hash_leaf, proof_to_dict -from aborist.store import append_audit, transaction +from arborist.compress import unpack_chunk +from arborist.distill.base import Distiller +from arborist.document import Document, canonicalize, get_chunker +from arborist.merkle import MerkleTree, hash_leaf, proof_to_dict +from arborist.store import append_audit, transaction DEFAULT_BATCH_SIZE = 200 diff --git a/aborist/distill/tfidf.py b/arborist/distill/tfidf.py similarity index 96% rename from aborist/distill/tfidf.py rename to arborist/distill/tfidf.py index 5d2d115..68cbd61 100644 --- a/aborist/distill/tfidf.py +++ b/arborist/distill/tfidf.py @@ -14,8 +14,8 @@ import math import re from collections import Counter -from aborist.distill.base import DistillationResult, Distiller -from aborist.document import Document +from arborist.distill.base import DistillationResult, Distiller +from arborist.document import Document _TOKEN_RE = re.compile(r"\b[a-zA-Z][a-zA-Z\-']{2,}\b") diff --git a/aborist/document.py b/arborist/document.py similarity index 100% rename from aborist/document.py rename to arborist/document.py diff --git a/aborist/evict.py b/arborist/evict.py similarity index 96% rename from aborist/evict.py rename to arborist/evict.py index 6ac3817..38ded75 100644 --- a/aborist/evict.py +++ b/arborist/evict.py @@ -18,10 +18,10 @@ import sqlite3 import time from typing import Callable, Iterable -from aborist.compress import pack_chunk -from aborist.document import canonicalize, get_chunker -from aborist.merkle import MerkleTree, hash_leaf -from aborist.store import append_audit, transaction +from arborist.compress import pack_chunk +from arborist.document import canonicalize, get_chunker +from arborist.merkle import MerkleTree, hash_leaf +from arborist.store import append_audit, transaction # Re-fetcher signature: takes a URI, returns parsed/canonicalized text or None. @@ -31,7 +31,7 @@ Fetcher = Callable[[str], str | None] def _default_html_fetcher(uri: str) -> str | None: """Reuse HtmlPageSource so rehydrate runs the exact same pipeline as ingest.""" try: - from aborist.sources.html_page import HtmlPageSource + from arborist.sources.html_page import HtmlPageSource except ImportError: return None src = HtmlPageSource([uri]) diff --git a/aborist/ingest.py b/arborist/ingest.py similarity index 97% rename from aborist/ingest.py rename to arborist/ingest.py index 8df831a..8efe674 100644 --- a/aborist/ingest.py +++ b/arborist/ingest.py @@ -17,16 +17,16 @@ import sqlite3 import time from dataclasses import dataclass -from aborist import ( +from arborist import ( CANONICALIZATION_VERSION, SCHEMA_VERSION, ) -from aborist.compress import pack_chunk, unpack_chunk -from aborist.document import Document, canonicalize, get_chunker -from aborist.merkle import MerkleTree, hash_leaf -from aborist.progress import Progress -from aborist.source import Source -from aborist.store import ( +from arborist.compress import pack_chunk, unpack_chunk +from arborist.document import Document, canonicalize, get_chunker +from arborist.merkle import MerkleTree, hash_leaf +from arborist.progress import Progress +from arborist.source import Source +from arborist.store import ( chain_audit_events, get_meta, latest_event_hash, @@ -72,7 +72,7 @@ def ingest_source( can rsync forward by re-running with --resume. `progress` (optional) gets a `tick(seen, inserted=...)` call after each - batch flush. Pass an `aborist.progress.Progress` for live stderr output. + batch flush. Pass an `arborist.progress.Progress` for live stderr output. """ chunker = get_chunker(chunker_name) stats = IngestStats() @@ -386,7 +386,7 @@ def _flush_edges( def verify_random_sample(conn: sqlite3.Connection, n: int = 10) -> dict: """Sample N documents, regenerate Merkle proof for chunk 0, verify.""" - from aborist.merkle import hash_leaf, verify_proof + from arborist.merkle import hash_leaf, verify_proof rows = conn.execute( "SELECT document_root FROM documents ORDER BY RANDOM() LIMIT ?", (n,) diff --git a/aborist/journal.py b/arborist/journal.py similarity index 85% rename from aborist/journal.py rename to arborist/journal.py index e094c5b..fd7badb 100644 --- a/aborist/journal.py +++ b/arborist/journal.py @@ -1,7 +1,7 @@ """Unfirehose-compatible session journal. Emits one JSONL file per query/bench-cycle to -``~/.aborist/unfirehose/{project-slug}/{session-uuid}.jsonl``. +``~/.arborist/unfirehose/{project-slug}/{session-uuid}.jsonl``. Unfirehose's native-harness auto-discovery picks up any ``~/.{name}/unfirehose/`` directory without registration; once a session lands, the unfirehose watcher debounces, ingests, and @@ -10,7 +10,7 @@ uncloseai sessions. Schema reference: ``~/git/unfirehose-nextjs-logger/docs/unfirehose-schema.md`` (``unfirehose/1.0``). Every record carries ``$schema: -"unfirehose/1.0"``; consumers ignore unknown fields, so aborist- +"unfirehose/1.0"``; consumers ignore unknown fields, so arborist- specific extras (audit_mode, cache_key, source list, verifier timings) ride along under namespaced keys. @@ -20,19 +20,19 @@ Per-query layout: user message role=user, content=[text] assistant message role=assistant, content=[text], usage=..., model=..., - aborist_meta={audit_mode, cache_key, + arborist_meta={audit_mode, cache_key, sources, timings} system session_end message subtype=session_end, durationMs Per-bench-cycle layout (bench_emergent): session header line (firstPrompt = generated question) - system init message subtype=init, aborist_meta={words} + system init message subtype=init, arborist_meta={words} user message the generated question - assistant message the student answer + aborist_meta + assistant message the student answer + arborist_meta system session_end message subtype=session_end -Everything aborist-specific lives under ``aborist_meta`` so the +Everything arborist-specific lives under ``arborist_meta`` so the canonical fields stay clean for off-the-shelf unfirehose consumers that expect the strict schema. """ @@ -48,23 +48,23 @@ from typing import Any # Default journal root. Override via env var or constructor. -DEFAULT_JOURNAL_ROOT = Path.home() / ".aborist" / "unfirehose" +DEFAULT_JOURNAL_ROOT = Path.home() / ".arborist" / "unfirehose" # Schema literal pinned to a single string — matches what unfirehose's # native-harness ingestion checks for. UNFIREHOSE_SCHEMA = "unfirehose/1.0" -# Harness identity. Any aborist process emitting JSONL claims this +# Harness identity. Any arborist process emitting JSONL claims this # `harness` name. Unfirehose surfaces it in its dashboard alongside # claude-code / fetch / uncloseai. -HARNESS_NAME = "aborist" +HARNESS_NAME = "arborist" -def _aborist_version() -> str: +def _arborist_version() -> str: """Best-effort version tag for the harnessVersion field.""" try: from importlib.metadata import version as _v - return _v("aborist") + return _v("arborist") except Exception: return "0.0.0+dev" @@ -77,7 +77,7 @@ def slugify_cwd(cwd: Path | str | None = None) -> str: - path separators (``/``) become ``-`` - dots (``.``) become ``-`` - ``/home/fox/git/aborist`` → ``-home-fox-git-aborist`` + ``/home/fox/git/arborist`` → ``-home-fox-git-arborist`` ``/home/fox/git/my.app`` → ``-home-fox-git-my-app`` """ p = Path(cwd) if cwd else Path.cwd() @@ -113,14 +113,14 @@ def _git_branch(cwd: Path | str | None = None) -> str | None: class SessionWriter: - """One open .jsonl file representing one aborist session. + """One open .jsonl file representing one arborist session. Use as a context manager so the close timestamp is set & the file flushed on exit: with SessionWriter(first_prompt=question, cwd=Path.cwd()) as s: s.user_message(question) - s.assistant_message(answer, model=..., usage=..., aborist_meta=...) + s.assistant_message(answer, model=..., usage=..., arborist_meta=...) The constructor writes the session header line; subsequent calls append message lines; ``__exit__`` writes the session_end line. @@ -137,11 +137,11 @@ class SessionWriter: ) -> None: self.session_id = session_id or new_session_id() self.cwd = Path(cwd or Path.cwd()) - self.root = Path(root or os.environ.get("ABORIST_JOURNAL_ROOT") or DEFAULT_JOURNAL_ROOT) + self.root = Path(root or os.environ.get("ARBORIST_JOURNAL_ROOT") or DEFAULT_JOURNAL_ROOT) self.project_slug = slugify_cwd(self.cwd) self.sidechain = sidechain self.git_branch = _git_branch(self.cwd) - self.harness_version = _aborist_version() + self.harness_version = _arborist_version() # File path: {root}/{slug}/{session_id}.jsonl self.session_dir = self.root / self.project_slug @@ -208,12 +208,12 @@ class SessionWriter: self, text: str, *, - aborist_meta: dict[str, Any] | None = None, + arborist_meta: dict[str, Any] | None = None, ) -> None: rec = self._base_message("user") rec["content"] = [{"type": "text", "text": text}] - if aborist_meta: - rec["aborist_meta"] = aborist_meta + if arborist_meta: + rec["arborist_meta"] = arborist_meta self._write(rec) def assistant_message( @@ -225,7 +225,7 @@ class SessionWriter: stop_reason: str | None = None, usage: dict[str, Any] | None = None, duration_ms: int | None = None, - aborist_meta: dict[str, Any] | None = None, + arborist_meta: dict[str, Any] | None = None, ) -> None: rec = self._base_message("assistant") rec["content"] = [{"type": "text", "text": text}] @@ -239,8 +239,8 @@ class SessionWriter: rec["usage"] = usage if duration_ms is not None: rec["durationMs"] = int(duration_ms) - if aborist_meta: - rec["aborist_meta"] = aborist_meta + if arborist_meta: + rec["arborist_meta"] = arborist_meta self._write(rec) def system_message( @@ -249,7 +249,7 @@ class SessionWriter: *, subtype: str | None = None, duration_ms: int | None = None, - aborist_meta: dict[str, Any] | None = None, + arborist_meta: dict[str, Any] | None = None, ) -> None: rec = self._base_message("system") if text: @@ -258,8 +258,8 @@ class SessionWriter: rec["subtype"] = subtype if duration_ms is not None: rec["durationMs"] = int(duration_ms) - if aborist_meta: - rec["aborist_meta"] = aborist_meta + if arborist_meta: + rec["arborist_meta"] = arborist_meta self._write(rec) # ──────────────────────────────────────────────────────────── lifecycle @@ -273,7 +273,7 @@ class SessionWriter: self.system_message( subtype="session_end", duration_ms=elapsed_ms, - aborist_meta={"status": status, "messageCount": self._message_count}, + arborist_meta={"status": status, "messageCount": self._message_count}, ) self._fp.close() self._closed = True diff --git a/aborist/merkle.py b/arborist/merkle.py similarity index 100% rename from aborist/merkle.py rename to arborist/merkle.py diff --git a/aborist/mesh/__init__.py b/arborist/mesh/__init__.py similarity index 82% rename from aborist/mesh/__init__.py rename to arborist/mesh/__init__.py index f54234c..c43db4b 100644 --- a/aborist/mesh/__init__.py +++ b/arborist/mesh/__init__.py @@ -1,8 +1,8 @@ -"""Mesh — gossip / membership layer for federated aborist trees. +"""Mesh — gossip / membership layer for federated arborist trees. Off by default. Loaded only when the user explicitly opts in via -`aborist mesh init` (creates this peer's identity) and -`aborist mesh enable` (flips the gating flag in the meta table). +`arborist mesh init` (creates this peer's identity) and +`arborist mesh enable` (flips the gating flag in the meta table). The cryptographic substrate: - Ed25519 (signing) — every gossip message and every membership @@ -22,11 +22,11 @@ the new secret, so any AEAD-protected gossip for epoch N+1 is opaque to them. This module deliberately avoids networking. The wire layer (HTTP/TLS -gossip server + sync client) lives in `aborist.mesh.wire` and is also +gossip server + sync client) lives in `arborist.mesh.wire` and is also opt-in. """ -from aborist.mesh.crypto import ( +from arborist.mesh.crypto import ( aead_decrypt, aead_encrypt, ecdh_shared_secret, @@ -35,7 +35,7 @@ from aborist.mesh.crypto import ( sign, verify, ) -from aborist.mesh.state import ( +from arborist.mesh.state import ( MESH_ENABLED_KEY, MeshIdentity, MeshRosterEntry, diff --git a/aborist/mesh/crypto.py b/arborist/mesh/crypto.py similarity index 99% rename from aborist/mesh/crypto.py rename to arborist/mesh/crypto.py index 086f890..cc04dbf 100644 --- a/aborist/mesh/crypto.py +++ b/arborist/mesh/crypto.py @@ -72,7 +72,7 @@ def ecdh_shared_secret(priv_bytes: bytes, peer_pub_bytes: bytes) -> bytes: return HKDF( algorithm=hashes.SHA256(), length=32, - salt=b"aborist.mesh.epoch.v1", + salt=b"arborist.mesh.epoch.v1", info=b"epoch-secret-wrap", ).derive(raw) diff --git a/aborist/mesh/members.py b/arborist/mesh/members.py similarity index 99% rename from aborist/mesh/members.py rename to arborist/mesh/members.py index 321a854..9534a33 100644 --- a/aborist/mesh/members.py +++ b/arborist/mesh/members.py @@ -10,7 +10,7 @@ from __future__ import annotations import sqlite3 -from aborist.mesh.state import ( +from arborist.mesh.state import ( MeshRosterEntry, current_epoch, load_identity, diff --git a/aborist/mesh/state.py b/arborist/mesh/state.py similarity index 98% rename from aborist/mesh/state.py rename to arborist/mesh/state.py index 54f181c..f992296 100644 --- a/aborist/mesh/state.py +++ b/arborist/mesh/state.py @@ -1,6 +1,6 @@ -"""Mesh state persisted in the standard aborist DB. +"""Mesh state persisted in the standard arborist DB. -Three tables (defined in `aborist.store.SCHEMA_SQL`): +Three tables (defined in `arborist.store.SCHEMA_SQL`): mesh_identity — this peer's keys + group name (singleton) mesh_roster — per-epoch (member_id, sign_pub, dh_pub, role) tuples mesh_epochs — epoch lifecycle: started_at, audit linkage, secret envelope @@ -20,14 +20,14 @@ import time import uuid from dataclasses import dataclass -from aborist.mesh.crypto import ( +from arborist.mesh.crypto import ( aead_decrypt, aead_encrypt, ecdh_shared_secret, generate_dh_keypair, generate_signing_keypair, ) -from aborist.store import ( +from arborist.store import ( append_audit, get_meta, set_meta, diff --git a/aborist/mesh/wire.py b/arborist/mesh/wire.py similarity index 98% rename from aborist/mesh/wire.py rename to arborist/mesh/wire.py index 8244481..55ac01a 100644 --- a/aborist/mesh/wire.py +++ b/arborist/mesh/wire.py @@ -72,14 +72,14 @@ from typing import Any import httpx -from aborist.mesh.crypto import aead_decrypt, aead_encrypt, sign, verify -from aborist.mesh.state import ( +from arborist.mesh.crypto import aead_decrypt, aead_encrypt, sign, verify +from arborist.mesh.state import ( current_epoch, load_identity, recover_epoch_secret, roster_at, ) -from aborist.store import append_audit, connect, latest_event_hash +from arborist.store import append_audit, connect, latest_event_hash # --------------------------------------------------------------------------- @@ -122,7 +122,7 @@ class WireEnvelope: """One signed gossip message. `body` is type-specific; canonicalization is via JSON sorted-keys with - no whitespace, identical to `aborist.store._canonical_json`. + no whitespace, identical to `arborist.store._canonical_json`. Confidentiality is opt-in. When `encrypted_body` is set, `body` is a placeholder (typically `{}`) and the real body lives AEAD-encrypted @@ -680,7 +680,7 @@ class MeshWireServer: "WHERE document_root=? AND tier='hot' ORDER BY idx", (root,), ).fetchall() - from aborist.compress import unpack_chunk + from arborist.compress import unpack_chunk leaves_hex = [r["leaf_hash"] for r in chunks] text = "\n\n".join(unpack_chunk(r["content"]) or "" for r in chunks) @@ -944,17 +944,17 @@ class MeshWireClient: def _canonical_json(obj: Any) -> str: """Stable JSON: sorted keys, no whitespace, ensure_ascii=False so unicode bytes survive a round-trip without \\uXXXX expansion. Same convention as - `aborist.store._canonical_json` — kept local to avoid a private import.""" + `arborist.store._canonical_json` — kept local to avoid a private import.""" return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False) def _merkle_root_matches(leaves_hex: list[str], expected_root_hex: str) -> bool: """Re-derive the Merkle root from `leaves_hex` (already-hashed chunk - leaves) and compare to `expected_root_hex`. Uses aborist.merkle's + leaves) and compare to `expected_root_hex`. Uses arborist.merkle's convention so single-leaf documents and odd-element trees behave identically to the local ingest path. """ - from aborist.merkle import MerkleTree + from arborist.merkle import MerkleTree if not leaves_hex: return False diff --git a/aborist/progress.py b/arborist/progress.py similarity index 100% rename from aborist/progress.py rename to arborist/progress.py diff --git a/aborist/qa/__init__.py b/arborist/qa/__init__.py similarity index 68% rename from aborist/qa/__init__.py rename to arborist/qa/__init__.py index 9e4d025..1a9413b 100644 --- a/aborist/qa/__init__.py +++ b/arborist/qa/__init__.py @@ -1,15 +1,15 @@ """Q&A layer: answer a question about a document, write a provable record.""" -from aborist.qa.client import ChatClient, OpenAICompatibleClient, StubClient -from aborist.qa.keys import ( +from arborist.qa.client import ChatClient, OpenAICompatibleClient, StubClient +from arborist.qa.keys import ( cache_key, conversation_hash, governance_policy_hash, model_profile_hash, question_hash, ) -from aborist.qa.query import DEFAULT_QUERY_POLICY, query -from aborist.qa.runner import DEFAULT_POLICY, ask +from arborist.qa.query import DEFAULT_QUERY_POLICY, query +from arborist.qa.runner import DEFAULT_POLICY, ask __all__ = [ "ChatClient", diff --git a/aborist/qa/client.py b/arborist/qa/client.py similarity index 100% rename from aborist/qa/client.py rename to arborist/qa/client.py diff --git a/aborist/qa/concepts.py b/arborist/qa/concepts.py similarity index 92% rename from aborist/qa/concepts.py rename to arborist/qa/concepts.py index cada4a2..aaec45f 100644 --- a/aborist/qa/concepts.py +++ b/arborist/qa/concepts.py @@ -1,10 +1,10 @@ -"""Backwards-compat shim — public API delegates to ``aborist.concepts``. +"""Backwards-compat shim — public API delegates to ``arborist.concepts``. The actual data lived as hand-curated frozensets in this module through April 2026 (commit c6182ae) and one entry was added in 2026-05-01 (commit bde1bd6 — mind/brain-tech group). 2026-05-01 the data layer moved to a per-shard ``concept_relations`` SQLite table -(see aborist/concepts/__init__.py for the rationale). +(see arborist/concepts/__init__.py for the rationale). This shim preserves the call-site signatures `query.py` already uses (``synonym_expand(qtokens)`` & ``rivalry_excluded(qtokens, compare_phrasing=...)``) @@ -21,7 +21,7 @@ from __future__ import annotations from pathlib import Path -from aborist.concepts.query import ( +from arborist.concepts.query import ( has_compare_phrasing, rivalry_excluded as _rivalry_excluded_impl, synonym_expand as _synonym_expand_impl, diff --git a/aborist/qa/dag.py b/arborist/qa/dag.py similarity index 98% rename from aborist/qa/dag.py rename to arborist/qa/dag.py index c55cd42..ff704db 100644 --- a/aborist/qa/dag.py +++ b/arborist/qa/dag.py @@ -7,7 +7,7 @@ Each query/ask call passes through several stages: Each stage emits a hash; the run's identity is the Merkle root over the ordered sequence of stage hashes. Stored on the providence record as ``run_dag_root`` (alongside ``cache_key``). The DAG is verifiable: given -the persisted node list & the same Merkle conventions aborist uses +the persisted node list & the same Merkle conventions arborist uses elsewhere (non-commutative HashCombine, prefix 0x03, leaf prefix 0x00, self-duplicate odd rule), an auditor can recompute the root from the nodes & confirm the run was constructed as recorded. @@ -41,7 +41,7 @@ from __future__ import annotations import hashlib import json -from aborist.merkle import MerkleTree +from arborist.merkle import MerkleTree def _sha256_hex(s: str) -> str: @@ -227,7 +227,7 @@ def build_run_dag( All inputs are already-computed hashes or text; no I/O. Idempotent & deterministic — same inputs always produce the same root, byte-for- byte across machines (as long as the Merkle conventions stay pinned; - they do, via ``aborist.merkle``). + they do, via ``arborist.merkle``). Two base DAG shapes; both gain an optional ``preflight`` stage when ``preflight_hash`` is supplied (Ticket #000009): @@ -365,7 +365,7 @@ def build_run_dag( out = {"root": root_hex, "nodes": nodes} # Ticket #000009 §7.2 — recoverable preflight payload. Storing # the canonical dict alongside the leaf hash means - # `aborist providence --show-preflight` can render the full + # `arborist providence --show-preflight` can render the full # 5-clause CTI contract (classifier / answer_contract / # prompt_contract / evidence_contract / policy_refs + # question_state) from `run_dag_blob` without needing a diff --git a/aborist/qa/evidence.py b/arborist/qa/evidence.py similarity index 99% rename from aborist/qa/evidence.py rename to arborist/qa/evidence.py index 6625770..632195e 100644 --- a/aborist/qa/evidence.py +++ b/arborist/qa/evidence.py @@ -47,12 +47,12 @@ import hashlib import re from dataclasses import asdict, dataclass -from aborist.merkle import MerkleTree +from arborist.merkle import MerkleTree def _sha256_hex(s: str) -> str: # ``errors='surrogatepass'`` for model-output text containing lone - # UTF-16 surrogates; same rationale as ``aborist.qa.dag._sha256_hex``. + # UTF-16 surrogates; same rationale as ``arborist.qa.dag._sha256_hex``. return hashlib.sha256(s.encode("utf-8", errors="surrogatepass")).hexdigest() diff --git a/aborist/qa/frame.py b/arborist/qa/frame.py similarity index 100% rename from aborist/qa/frame.py rename to arborist/qa/frame.py diff --git a/aborist/qa/inspect.py b/arborist/qa/inspect.py similarity index 98% rename from aborist/qa/inspect.py rename to arborist/qa/inspect.py index ebc3d70..8ae9727 100644 --- a/aborist/qa/inspect.py +++ b/arborist/qa/inspect.py @@ -21,17 +21,17 @@ import unicodedata from pathlib import Path from typing import Any -from aborist.compress import unpack_chunk -from aborist.store import connect +from arborist.compress import unpack_chunk +from arborist.store import connect try: - from aborist.wikitext import to_base as _wikitext_to_base + from arborist.wikitext import to_base as _wikitext_to_base except ImportError: # pragma: no cover _wikitext_to_base = None def _normalize(s: str) -> str: - """Same as aborist.qa.verify._normalize — kept local to avoid an + """Same as arborist.qa.verify._normalize — kept local to avoid an import cycle and to make the diagnostic self-contained.""" s = unicodedata.normalize("NFC", s) s = " ".join(s.split()) @@ -338,7 +338,7 @@ def _repair_for_trailing_artifact(span: str, diag: dict[str, Any]) -> dict[str, # Stopword set for question/answer token overlap. Mirrors the title-search -# stopwords in aborist.qa.query._TITLE_STOPWORDS but kept local so this +# stopwords in arborist.qa.query._TITLE_STOPWORDS but kept local so this # sidecar never reaches into the verifier's machinery for soft computations. _DEFLECTION_STOPWORDS = frozenset( """ @@ -354,7 +354,7 @@ _DEFLECTION_STOPWORDS = frozenset( def _stem_for_deflection(t: str) -> str: - """Mirror of aborist.qa.query._stem_token_for_match: strip trailing + """Mirror of arborist.qa.query._stem_token_for_match: strip trailing `'s` (possessive) then trailing `s` on tokens >4 chars (skip `ss`-enders). Keeps 4-char words like 'mars' intact while collapsing 'mars's' → 'mars' and 'rivers' → 'river'.""" @@ -529,7 +529,7 @@ _METAPHOR_PREPOSITION_CUES = frozenset({ # alongside American (`coloredly`, `realizingly`). # # Operators can supplement with corpus / domain-specific vocabulary -# via the ``ABORIST_METAPHOR_DICTS`` environment variable (colon- +# via the ``ARBORIST_METAPHOR_DICTS`` environment variable (colon- # separated list of paths, one word per line) or the # ``register_metaphor_dictionary(path)`` helper below. A forest with # its own jargon (aviation, medical, legal, dynastic) registers the @@ -554,7 +554,7 @@ def register_metaphor_dictionary(path: str | Path) -> None: vocabulary so the metaphor sidecar's morphological tests can pick up domain-specific stems. - Programmatic equivalent of the ``ABORIST_METAPHOR_DICTS`` env var. + Programmatic equivalent of the ``ARBORIST_METAPHOR_DICTS`` env var. Calling this invalidates the cache so subsequent lookups re-build the union. Idempotent — re-registering the same path is a no-op. """ @@ -583,7 +583,7 @@ def _english_wordlist() -> frozenset[str]: 1. ``/usr/share/dict/words`` (OS default) 2. ``/usr/share/dict/american-english`` (Debian split) 3. ``/usr/share/dict/british-english`` (Debian split) - 4. Paths in ``ABORIST_METAPHOR_DICTS`` (colon-separated env var) + 4. Paths in ``ARBORIST_METAPHOR_DICTS`` (colon-separated env var) 5. Paths registered via ``register_metaphor_dictionary()`` Cached on first call. Graceful degradation: missing paths skip @@ -596,7 +596,7 @@ def _english_wordlist() -> frozenset[str]: union: set[str] = set() for p in _DEFAULT_DICT_PATHS: union |= _load_dict(p) - env_paths = os.environ.get("ABORIST_METAPHOR_DICTS", "") + env_paths = os.environ.get("ARBORIST_METAPHOR_DICTS", "") if env_paths: for raw in env_paths.split(":"): raw = raw.strip() diff --git a/aborist/qa/keys.py b/arborist/qa/keys.py similarity index 98% rename from aborist/qa/keys.py rename to arborist/qa/keys.py index 99eccc1..514d73a 100644 --- a/aborist/qa/keys.py +++ b/arborist/qa/keys.py @@ -8,7 +8,7 @@ record is live (not failed/stale/quarantined): 3. model_profile_hash — model_id + revision + quantization 4. conversation_hash — full canonical OpenAI messages array 5. governance_policy_hash — sampling/policy parameters dict - 6. schema_version — aborist DB schema version + 6. schema_version — arborist DB schema version 7. canonicalization_version — text normalization rules 8. chunking_version — chunker name & parameters @@ -24,12 +24,12 @@ from __future__ import annotations import hashlib import json -from aborist.document import canonicalize +from arborist.document import canonicalize def _sha256(s: str) -> str: # ``errors='surrogatepass'`` survives lone UTF-16 surrogates from - # model output; same rationale as ``aborist.qa.dag._sha256_hex``. + # model output; same rationale as ``arborist.qa.dag._sha256_hex``. return hashlib.sha256(s.encode("utf-8", errors="surrogatepass")).hexdigest() diff --git a/aborist/qa/metacognition.py b/arborist/qa/metacognition.py similarity index 98% rename from aborist/qa/metacognition.py rename to arborist/qa/metacognition.py index 7bb2b23..247fa11 100644 --- a/aborist/qa/metacognition.py +++ b/arborist/qa/metacognition.py @@ -7,7 +7,7 @@ under-specified, broad-quantifier, time-sensitive, out-of-corpus, reference-frame ambiguous). Pure and deterministic. No I/O, no model call, no retrieval call. -Reuses ``aborist.qa.quantifier.classify_question_quantifier`` for +Reuses ``arborist.qa.quantifier.classify_question_quantifier`` for the broad-quantifier rung; adds four new lightweight detectors: - temporal sensitivity (current/latest/today/CEO/etc.) @@ -15,7 +15,7 @@ the broad-quantifier rung; adds four new lightweight detectors: - false-premise (lite) (presupposition patterns) - out-of-corpus (my-uploaded-X / my-file shapes) -Reference-frame detection lives in ``aborist.qa.query._detect_frame`` +Reference-frame detection lives in ``arborist.qa.query._detect_frame`` (ticket #000002) and is called from the surrounding runtime, not from this module — keeps detection pure-on-question (no corpus lookup needed here). @@ -396,13 +396,13 @@ def preflight_question( `reference_frames` is passed in by the caller because frame detection requires retrieved sources (lives in - `aborist.qa.query._detect_frame`). Empty tuple is the default + `arborist.qa.query._detect_frame`). Empty tuple is the default for "no frame routing happened". `policy` overrides for the per-detector enables. Defaults are permissive (all checks on) per ticket #000010 §7.3. """ - from aborist.qa.quantifier import classify_question_quantifier + from arborist.qa.quantifier import classify_question_quantifier policy = policy or {} enabled = bool(policy.get("metacognition_enabled", True)) diff --git a/aborist/qa/model_profiles.py b/arborist/qa/model_profiles.py similarity index 97% rename from aborist/qa/model_profiles.py rename to arborist/qa/model_profiles.py index e9ebbe3..6176973 100644 --- a/aborist/qa/model_profiles.py +++ b/arborist/qa/model_profiles.py @@ -7,8 +7,8 @@ model id, e.g. ``adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic``). Falls back to ``"default"`` profile when no entry matches. Folds into ``governance_policy_hash`` via two policy fields surfaced in -``aborist.qa.runner.DEFAULT_POLICY`` / -``aborist.qa.query.DEFAULT_QUERY_POLICY``: +``arborist.qa.runner.DEFAULT_POLICY`` / +``arborist.qa.query.DEFAULT_QUERY_POLICY``: quantifier_caps_by_intensity per-call cap dict (overrides the profile when set on the policy) @@ -16,7 +16,7 @@ Folds into ``governance_policy_hash`` via two policy fields surfaced in (default False through rollout) The profile table lives here; the *application* of the cap lives in the -runner (`aborist/qa/runner.py:ask`) and `aborist/qa/query.py:query`. +runner (`arborist/qa/runner.py:ask`) and `arborist/qa/query.py:query`. This file is pure data + a lookup helper. Per ticket §10.11.3 dry-run discipline: Phase 2 lands the cap-table and diff --git a/aborist/qa/parse_claims.py b/arborist/qa/parse_claims.py similarity index 100% rename from aborist/qa/parse_claims.py rename to arborist/qa/parse_claims.py diff --git a/aborist/qa/prompts.py b/arborist/qa/prompts.py similarity index 97% rename from aborist/qa/prompts.py rename to arborist/qa/prompts.py index bbc479b..09ccf5d 100644 --- a/aborist/qa/prompts.py +++ b/arborist/qa/prompts.py @@ -3,8 +3,8 @@ Single source of truth for the four prompt strings that travel across both QA paths: -- ``aborist.qa.runner.ask()`` — single-document path -- ``aborist.qa.query.query()`` — multi-source retrieval path +- ``arborist.qa.runner.ask()`` — single-document path +- ``arborist.qa.query.query()`` — multi-source retrieval path Both paths use the same lattice prompts (pointer + JSON variants, system + grounding-reminder each), so DRY them here. The @@ -109,7 +109,7 @@ CLAIM_LATTICE_GROUNDING_REMINDER = ( # JSON variant — pairs with grammar-constrained inference (vLLM # guided_json, Claude/GPT-4 native JSON, Qwen 3.6 reasoner). -# Lenient pre-parser in ``aborist.qa.verify.verify_claim_lattice_json`` +# Lenient pre-parser in ``arborist.qa.verify.verify_claim_lattice_json`` # keeps the path survivable on inference paths without grammar # guidance. CLAIM_LATTICE_JSON_SYSTEM_PROMPT = ( diff --git a/aborist/qa/quantifier.py b/arborist/qa/quantifier.py similarity index 99% rename from aborist/qa/quantifier.py rename to arborist/qa/quantifier.py index 2bd3a0f..1ccdd04 100644 --- a/aborist/qa/quantifier.py +++ b/arborist/qa/quantifier.py @@ -9,7 +9,7 @@ matters. Pure function. No I/O. No model call. No retrieval call. Folds into ``governance_policy_hash`` via ``classifier_version`` (added to -``aborist.qa.keys._VERIFIER_POLICY_FIELDS`` in Phase 2). +``arborist.qa.keys._VERIFIER_POLICY_FIELDS`` in Phase 2). Intensity rungs (highest wins for multi-quantifier questions): diff --git a/aborist/qa/quantifier_reminder.py b/arborist/qa/quantifier_reminder.py similarity index 100% rename from aborist/qa/quantifier_reminder.py rename to arborist/qa/quantifier_reminder.py diff --git a/aborist/qa/query.py b/arborist/qa/query.py similarity index 98% rename from aborist/qa/query.py rename to arborist/qa/query.py index 513b34a..7d308f8 100644 --- a/aborist/qa/query.py +++ b/arborist/qa/query.py @@ -39,26 +39,26 @@ import time from dataclasses import dataclass from pathlib import Path -from aborist import ( +from arborist import ( CANONICALIZATION_VERSION, CHUNKING_VERSION, SCHEMA_VERSION, ) -from aborist.compress import unpack_chunk -from aborist.merkle import MerkleTree -from aborist.qa.client import ChatClient -from aborist.qa.prompts import ( +from arborist.compress import unpack_chunk +from arborist.merkle import MerkleTree +from arborist.qa.client import ChatClient +from arborist.qa.prompts import ( CLAIM_LATTICE_GROUNDING_REMINDER, CLAIM_LATTICE_JSON_GROUNDING_REMINDER, CLAIM_LATTICE_JSON_SYSTEM_PROMPT, CLAIM_LATTICE_SYSTEM_PROMPT, ) -from aborist.qa.concepts import ( +from arborist.qa.concepts import ( has_compare_phrasing, rivalry_excluded, synonym_expand, ) -from aborist.qa.keys import ( +from arborist.qa.keys import ( DEFAULT_FIDELITY, DEFAULT_QUESTION_DEDUP, FIDELITY_MODES, @@ -71,17 +71,17 @@ from aborist.qa.keys import ( question_hash, verifier_policy_hash, ) -from aborist.qa.dag import build_run_dag -from aborist.qa.frame import FrameDetection, detect_frame as _detect_frame -from aborist.qa.retrieval_plan import RetrievalPlan, retrieval_plan_hash -from aborist.qa.evidence import ( +from arborist.qa.dag import build_run_dag +from arborist.qa.frame import FrameDetection, detect_frame as _detect_frame +from arborist.qa.retrieval_plan import RetrievalPlan, retrieval_plan_hash +from arborist.qa.evidence import ( build_evidence_map, evidence_map_root, render_evidence_map, render_evidence_map_for_json, ) -from aborist.qa.repair import mechanical_repair, reprompt_repair -from aborist.qa.verify import ( +from arborist.qa.repair import mechanical_repair, reprompt_repair +from arborist.qa.verify import ( ANSWER_MODES, CLAIM_LATTICE_JSON_SCHEMA, verify_claim_lattice_json, @@ -91,13 +91,13 @@ from aborist.qa.verify import ( ) try: - from aborist.wikitext import BASE_VERSION as _WIKITEXT_BASE_VERSION - from aborist.wikitext import to_base as _wikitext_to_base + from arborist.wikitext import BASE_VERSION as _WIKITEXT_BASE_VERSION + from arborist.wikitext import to_base as _wikitext_to_base except ImportError: # pragma: no cover _WIKITEXT_BASE_VERSION = None _wikitext_to_base = None -from aborist.search import FTS5Backend -from aborist.store import ( +from arborist.search import FTS5Backend +from arborist.store import ( append_audit, connect, discover_shards, @@ -115,7 +115,7 @@ _TITLE_TOKEN_RE = re.compile(r"[A-Za-z][A-Za-z0-9]*") _HYPHEN_RUN_RE = re.compile( r"[A-Za-z][A-Za-z0-9]*(?:-[A-Za-z][A-Za-z0-9]*)+" ) -# Kept in sync with FTS5 stopwords in aborist.search.fts5 — both filter +# Kept in sync with FTS5 stopwords in arborist.search.fts5 — both filter # the same set of question-shaping words. "tell" leaking into title-LIKE # search caused "tell me about permacomputer" to pull Tell_(poker), the # Tell-Tale_Heart movie, Tell_City Indiana, etc. @@ -365,7 +365,7 @@ DEFAULT_QUERY_POLICY = { "top_p": 1.0, "max_tokens": 768, # Entity-path policy for the faithfulness verifier. See - # aborist/qa/verify.py:ENTITY_POLICIES. Default 'proximity' promotes + # arborist/qa/verify.py:ENTITY_POLICIES. Default 'proximity' promotes # to STRICT only if N verified entities cluster within W chars of # each other in source — separating "source documents these entities # as a group" (cast list / infobox / roster) from "source mentions @@ -394,7 +394,7 @@ DEFAULT_QUERY_POLICY = { # asking the model to rewrite around the failed quotes. "repair_max_reprompts": 0, # G0 / CTI — claim-lattice-pointer answer mode. See - # aborist/qa/runner.py:DEFAULT_POLICY for full semantics. Default + # arborist/qa/runner.py:DEFAULT_POLICY for full semantics. Default # "quote" preserves existing behavior; "claim_lattice_pointer" # instructs the runtime to build an evidence map, show short # pointer ids (E1, E2, …) to the model, and accept pointer-line @@ -457,7 +457,7 @@ DEFAULT_QUERY_POLICY = { "claim_lattice_deflection_check_enabled": True, "claim_lattice_format_collapse_check_enabled": True, # Ticket #000008 Phase 2 — quantifier preflight guard. See - # aborist/qa/runner.py:DEFAULT_POLICY for the full rationale. + # arborist/qa/runner.py:DEFAULT_POLICY for the full rationale. # Phase 2 lands with apply_caps=False (dry-run); cap is # reported on result dict but not applied to the verifier. "quantifier_guard_enabled": True, @@ -585,8 +585,8 @@ def _classify_source_role( """Tag a source by its likely role for an N-token query. URI-scheme classification fires first: documents whose URI starts - with ``aborist://providence/`` are self-promoted providence - records (per ``aborist/sources/providence.py``) and classify as + with ``arborist://providence/`` are self-promoted providence + records (per ``arborist/sources/providence.py``) and classify as ``self_reference_source`` regardless of title shape — that role captures the trust model "STRICT-as-fact unless verifier falsifies." @@ -599,7 +599,7 @@ def _classify_source_role( question). Primary requires the strongest title coverage (N-1 of N stems present). """ - if document_uri and document_uri.startswith("aborist://providence/"): + if document_uri and document_uri.startswith("arborist://providence/"): return "self_reference_source" if not title: return "unclassified" @@ -1592,7 +1592,7 @@ def query( """Answer `question` using the corpus. Cache to qa_db. Returns a result dict. `fidelity` controls lookup tolerance — see ``FIDELITY_MODES`` in - ``aborist.qa.keys``. ``"strict"`` checks only the cache_key + ``arborist.qa.keys``. ``"strict"`` checks only the cache_key matching this call's ``policy["question_dedup"]``. ``"equivalence_class"`` (default) tries the primary cache_key first, then the alternate dedup-mode cache_key as a fallback so a fast-cache agent can reuse @@ -1659,8 +1659,8 @@ def query( # (quantifier_guard_enabled), per-mode (quantifier_guard_modes), # per-call (quantifier_caps_by_intensity overrides), per-test # (policy={"quantifier_guard_enabled": False}). - from aborist.qa.model_profiles import cap_for_intensity - from aborist.qa.quantifier import classify_question_quantifier + from arborist.qa.model_profiles import cap_for_intensity + from arborist.qa.quantifier import classify_question_quantifier quantifier_guard_on = bool(policy.get("quantifier_guard_enabled", True)) quantifier_guard_modes = policy.get( "quantifier_guard_modes", @@ -1730,7 +1730,7 @@ def query( # retrieval, and preflight here is pre-retrieval — frame info # lives on the result dict separately, not on QuestionState # in this pass). - from aborist.qa.metacognition import preflight_question + from arborist.qa.metacognition import preflight_question _t_preflight = time.monotonic() question_state = preflight_question( question, @@ -1744,7 +1744,7 @@ def query( # is True. Returns a stub hint (SOFT_DISABLED) when off so the # result-dict / run-DAG schema stays consistent. NEVER enters # the verifier proof path; advisory only. - from aborist.qa.soft_preflight import soft_preflight_question + from arborist.qa.soft_preflight import soft_preflight_question _t_soft_preflight = time.monotonic() soft_hint = soft_preflight_question( question, @@ -1777,7 +1777,7 @@ def query( # reject-broad DAG so the rejection is Merkle-auditable. # Without this, two rejections under different policy state # would be indistinguishable in audit replay. - from aborist.qa.dag import ( + from arborist.qa.dag import ( build_reject_run_dag, preflight_node_hash as _pre_hash, ) @@ -1813,7 +1813,7 @@ def query( # Same payload-then-hash pattern as the miss path so # `--show-preflight` can render the full clause set on # reject rows too. - from aborist.qa.dag import ( + from arborist.qa.dag import ( _canonical_json as _reject_canon, _sha256_hex as _reject_sha, build_preflight_node_payload as _reject_build_payload, @@ -2306,7 +2306,7 @@ def query( and quantifier.get("is_broad") and bool(policy.get("quantifier_reminder_enabled", False)) ): - from aborist.qa.quantifier_reminder import broad_quantifier_reminder + from arborist.qa.quantifier_reminder import broad_quantifier_reminder broad = broad_quantifier_reminder( intensity=quantifier["intensity"], cap=effective_max_claims, @@ -2843,7 +2843,7 @@ def query( # five nested clauses (classifier, answer_contract, # prompt_contract, evidence_contract, policy_refs) + the # metacognition QuestionState. - from aborist.qa.dag import preflight_node_hash + from arborist.qa.dag import preflight_node_hash # verifier_policy_hash + model_profile_hash already imported # at module top; reusing the existing names. Local re-imports # would shadow earlier free-variable uses. @@ -2875,12 +2875,12 @@ def query( ) # Build the canonical payload once; hash it AND persist it # alongside the DAG nodes so audit replay can render the - # full 5-clause CTI contract via `aborist providence + # full 5-clause CTI contract via `arborist providence # --show-preflight`. Hash is deterministic from payload, so # an auditor can re-verify: # _sha256_hex(_canonical_json(preflight_payload)) # == nodes[preflight_idx]["hash"] - from aborist.qa.dag import build_preflight_node_payload + from arborist.qa.dag import build_preflight_node_payload _preflight_payload = build_preflight_node_payload( question_state=question_state.to_dict(), quantifier=quantifier, @@ -2923,7 +2923,7 @@ def query( "answer_mode": answer_mode, }, ) - from aborist.qa.dag import _sha256_hex, _canonical_json + from arborist.qa.dag import _sha256_hex, _canonical_json preflight_hash = _sha256_hex(_canonical_json(_preflight_payload)) run_dag = build_run_dag( question_hash=qhash, @@ -3037,7 +3037,7 @@ def query( ) # The actual label is in the verify_payload, which we computed in # localize_failure earlier — recompute for the result dict. - from aborist.qa.dag import localize_failure as _localize + from arborist.qa.dag import localize_failure as _localize failure_stage = _localize( audit_mode=verdict["audit_mode"], n_sources=len(chosen), diff --git a/aborist/qa/repair.py b/arborist/qa/repair.py similarity index 99% rename from aborist/qa/repair.py rename to arborist/qa/repair.py index 8f978fb..576f592 100644 --- a/aborist/qa/repair.py +++ b/arborist/qa/repair.py @@ -1,7 +1,7 @@ """Mechanical answer repair — apply sidecar repair suggestions deterministically. When the verifier flags an answer's quoted span as unverified, the -sidecar (`aborist/qa/inspect.py:_classify_span`) emits a `repair` +sidecar (`arborist/qa/inspect.py:_classify_span`) emits a `repair` field naming a concrete fix: split a `[...]`-elided quote into two verbatim quotes, trim a trailing `(Source: ...)` artifact, restore a dropped parenthetical aside, drop a fully-invented claim. This module diff --git a/aborist/qa/retrieval_plan.py b/arborist/qa/retrieval_plan.py similarity index 96% rename from aborist/qa/retrieval_plan.py rename to arborist/qa/retrieval_plan.py index cf57306..c7bec10 100644 --- a/aborist/qa/retrieval_plan.py +++ b/arborist/qa/retrieval_plan.py @@ -2,7 +2,7 @@ Captures the *input* side of retrieval as a content-addressed hash so the audit chain reproduces both *what got retrieved* (sources_summary, -which `aborist.qa.dag` already binds) and *how retrieval got there* +which `arborist.qa.dag` already binds) and *how retrieval got there* (the operator-influenceable inputs: keywords, top_k, over_fetch, max_context_chars, shard set). @@ -80,7 +80,7 @@ def retrieval_plan_hash(plan: RetrievalPlan) -> str: Deterministic: same plan → same hash, byte-for-byte across machines. Folds into the run-DAG retrieval stage via - ``aborist.qa.dag.build_run_dag(retrieval_plan_hash=...)``. + ``arborist.qa.dag.build_run_dag(retrieval_plan_hash=...)``. """ canon = json.dumps(plan.canonical(), sort_keys=True, separators=(",", ":")) return hashlib.sha256(canon.encode("utf-8")).hexdigest() diff --git a/aborist/qa/runner.py b/arborist/qa/runner.py similarity index 96% rename from aborist/qa/runner.py rename to arborist/qa/runner.py index f824ca9..09fcdea 100644 --- a/aborist/qa/runner.py +++ b/arborist/qa/runner.py @@ -15,20 +15,20 @@ import json import sqlite3 import time -from aborist import ( +from arborist import ( CANONICALIZATION_VERSION, SCHEMA_VERSION, ) -from aborist.compress import unpack_chunk -from aborist.merkle import MerkleTree, proof_to_dict -from aborist.qa.client import ChatClient -from aborist.qa.prompts import ( +from arborist.compress import unpack_chunk +from arborist.merkle import MerkleTree, proof_to_dict +from arborist.qa.client import ChatClient +from arborist.qa.prompts import ( CLAIM_LATTICE_GROUNDING_REMINDER, CLAIM_LATTICE_JSON_GROUNDING_REMINDER, CLAIM_LATTICE_JSON_SYSTEM_PROMPT, CLAIM_LATTICE_SYSTEM_PROMPT, ) -from aborist.qa.keys import ( +from arborist.qa.keys import ( DEFAULT_FIDELITY, DEFAULT_QUESTION_DEDUP, FIDELITY_MODES, @@ -41,15 +41,15 @@ from aborist.qa.keys import ( question_hash, verifier_policy_hash, ) -from aborist.qa.dag import build_run_dag -from aborist.qa.evidence import ( +from arborist.qa.dag import build_run_dag +from arborist.qa.evidence import ( build_evidence_map, evidence_map_root, render_evidence_map, render_evidence_map_for_json, ) -from aborist.qa.repair import mechanical_repair, reprompt_repair -from aborist.qa.verify import ( +from arborist.qa.repair import mechanical_repair, reprompt_repair +from arborist.qa.verify import ( ANSWER_MODES, CLAIM_LATTICE_JSON_SCHEMA, DEFAULT_ANSWER_MODE, @@ -57,11 +57,11 @@ from aborist.qa.verify import ( verify_claim_lattice_json, verify_quotes, ) -from aborist.store import append_audit, transaction +from arborist.store import append_audit, transaction try: - from aborist.wikitext import BASE_VERSION as _WIKITEXT_BASE_VERSION - from aborist.wikitext import to_base as _wikitext_to_base + from arborist.wikitext import BASE_VERSION as _WIKITEXT_BASE_VERSION + from arborist.wikitext import to_base as _wikitext_to_base except ImportError: # pragma: no cover _WIKITEXT_BASE_VERSION = None _wikitext_to_base = None @@ -69,10 +69,10 @@ except ImportError: # pragma: no cover DEFAULT_POLICY = { # Ticket #000007 — query-layer hyphen-fold marker. See - # aborist/qa/query.py:DEFAULT_QUERY_POLICY for rationale. + # arborist/qa/query.py:DEFAULT_QUERY_POLICY for rationale. "hyphen_fold_v1": True, # Ticket #000006 amend 2026-05-02b (Rule 9). See - # aborist/qa/query.py:DEFAULT_QUERY_POLICY for full rationale. + # arborist/qa/query.py:DEFAULT_QUERY_POLICY for full rationale. "claim_lattice_subject_tokens_absent_threshold": 3, "system_prompt": ( "Answer the user's question based ONLY on the document below. " @@ -86,7 +86,7 @@ DEFAULT_POLICY = { "Stay inside the document at all times." ), # Restated rule fired as a user message right before the document + - # question arrive. See aborist/qa/query.py for the rationale (recent + # question arrive. See arborist/qa/query.py for the rationale (recent # user-turn instructions outweigh decayed system-turn rules in 8B # instruction-tuned models). "grounding_reminder": ( @@ -102,7 +102,7 @@ DEFAULT_POLICY = { "entity_proximity_n": 3, "entity_proximity_window": 300, # Mechanical answer repair after first verify. Off by default; see - # aborist/qa/query.py for semantics. + # arborist/qa/query.py for semantics. "repair_enabled": False, "repair_max_reprompts": 0, # Strip wikitext markup before the LLM ever sees the context. Lets @@ -128,7 +128,7 @@ DEFAULT_POLICY = { "claim_lattice_grounding_reminder": CLAIM_LATTICE_GROUNDING_REMINDER, # Allowed source roles for claim-lattice verification. Roles outside # this set get classified SOURCE_ROLE_BLOCKED and downgrade the - # verdict. Mirrors aborist.qa.verify.DEFAULT_ALLOWED_SOURCE_ROLES; + # verdict. Mirrors arborist.qa.verify.DEFAULT_ALLOWED_SOURCE_ROLES; # noisy_background_source / sequel_background_source are excluded by # default. Folds into governance_policy_hash on change. "claim_lattice_allowed_source_roles": [ @@ -136,7 +136,7 @@ DEFAULT_POLICY = { "secondary_context_source", "background_source", "unclassified", - # Self-promoted providence records (`aborist://providence/` + # Self-promoted providence records (`arborist://providence/` # URI scheme). Trusted-as-fact substrate per the # self-reference design — STRICT live records past the # kindergarten window. See @@ -187,7 +187,7 @@ DEFAULT_POLICY = { # phrase) from the claim. Catches the Homer-Simpson lazy-anchor # case where claim asserts "Mr. Burns" but cited span is the # voice-actor bio. WARRANT_MISSING violations cap audit_mode - # at HYBRID. See aborist/qa/warrant.py. + # at HYBRID. See arborist/qa/warrant.py. "claim_lattice_warrant_check_enabled": True, "claim_lattice_deflection_check_enabled": True, # Format-collapse check (pointer-mode only): when the model emits @@ -202,7 +202,7 @@ DEFAULT_POLICY = { "claim_lattice_format_collapse_check_enabled": True, # Quantifier preflight guard (Ticket #000008 Phase 2). Per-call # claim cap derived from the question's quantifier intensity and - # the configured model profile (aborist/qa/model_profiles.py). + # the configured model profile (arborist/qa/model_profiles.py). # Phase 2 lands the lookup wiring with apply_caps=False per # §10.11.3 dry-run discipline — claim_cap_applied is computed # and reported on the result dict, but the verifier still uses @@ -322,7 +322,7 @@ def ask( ) -> dict: """Look up cached answer or run inference. Returns a result dict. - See ``aborist.qa.query.query`` for `fidelity` semantics — it + See ``arborist.qa.query.query`` for `fidelity` semantics — it controls lookup tolerance: ``"strict"`` only checks the cache_key matching the call's ``policy["question_dedup"]``; the default ``"equivalence_class"`` falls back to the alternate dedup mode's @@ -337,10 +337,10 @@ def ask( f"fidelity must be one of {FIDELITY_MODES}, got {fidelity!r}" ) # Quantifier preflight (Ticket #000008 Phase 1+2). Same wiring - # as query() — see aborist/qa/query.py for the rationale and + # as query() — see arborist/qa/query.py for the rationale and # disable hierarchy. - from aborist.qa.model_profiles import cap_for_intensity - from aborist.qa.quantifier import classify_question_quantifier + from arborist.qa.model_profiles import cap_for_intensity + from arborist.qa.quantifier import classify_question_quantifier answer_mode_for_guard = policy.get("answer_mode", "quote") quantifier_guard_on = bool(policy.get("quantifier_guard_enabled", True)) quantifier_guard_modes = policy.get( @@ -387,7 +387,7 @@ def ask( else: effective_max_claims = _policy_max_claims # Ticket #000010 — meta-cognition preflight (mirror of query()). - from aborist.qa.metacognition import preflight_question + from arborist.qa.metacognition import preflight_question question_state = preflight_question( question, model_profile_id=model_id, @@ -516,7 +516,7 @@ def ask( and quantifier.get("is_broad") and bool(policy.get("quantifier_reminder_enabled", False)) ): - from aborist.qa.quantifier_reminder import broad_quantifier_reminder + from arborist.qa.quantifier_reminder import broad_quantifier_reminder broad = broad_quantifier_reminder( intensity=quantifier["intensity"], cap=effective_max_claims, @@ -528,7 +528,7 @@ def ask( mhash = model_profile_hash(model_id, revision, quantization) - # Dedup-mode-aware cache_key. See aborist/qa/query.py for rationale — + # Dedup-mode-aware cache_key. See arborist/qa/query.py for rationale — # policy_variant matches the alternate mode so governance_policy_hash # agrees with what an agent under that mode would have written, # enabling cross-silo fallback. @@ -837,7 +837,7 @@ def ask( } proof_blob = json.dumps(proof_obj, separators=(",", ":")) - # Per-run Merkle-DAG (see aborist/qa/dag.py). Single-doc shape: + # Per-run Merkle-DAG (see arborist/qa/dag.py). Single-doc shape: # the only "source" is document_root. Quote mode: 7 stages base # (8 with #000009 preflight). Pointer mode: 9 stages base (10 with # preflight); context drops out and answer splits into raw_answer @@ -861,7 +861,7 @@ def ask( ] # Ticket #000009 — preflight node binding (mirror of query(); # nested CTI clauses per ticket §8 / 2026-05-04 feedback). - from aborist.qa.dag import preflight_node_hash + from arborist.qa.dag import preflight_node_hash # verifier_policy_hash + model_profile_hash imported at module # top; do NOT re-import locally (free-variable shadowing). ghash_for_dag = verifier_policy_hash(policy) @@ -887,9 +887,9 @@ def ask( else "broad-quantifier-unbounded-v1" ) # Build payload + hash separately so we can persist both into - # run_dag_blob (Ticket #000009 §7.2 — `aborist providence + # run_dag_blob (Ticket #000009 §7.2 — `arborist providence # --show-preflight` renders the full clause set). - from aborist.qa.dag import ( + from arborist.qa.dag import ( _canonical_json as _runner_canon, _sha256_hex as _runner_sha, build_preflight_node_payload as _runner_build_payload, @@ -1036,7 +1036,7 @@ def ask( ), ) - from aborist.qa.dag import localize_failure as _localize + from arborist.qa.dag import localize_failure as _localize failure_stage = _localize( audit_mode=verdict["audit_mode"], n_sources=1, # ask() runs against one document @@ -1064,7 +1064,7 @@ def ask( "partially_verified_quotes": verdict.get("partially_verified_quotes") or [], # Quantifier preflight (Ticket #000008 Phase 1+2). See query.py # for full rationale; runner.ask carries the same schema for - # CLI-side `aborist ask` parity with `aborist query`. + # CLI-side `arborist ask` parity with `arborist query`. "quantifier_intensity": quantifier["intensity"], "quantifier_matched_token": quantifier["matched_token"], "scope_bound_hint": quantifier["scope_bound_hint"], diff --git a/aborist/qa/soft_preflight.py b/arborist/qa/soft_preflight.py similarity index 100% rename from aborist/qa/soft_preflight.py rename to arborist/qa/soft_preflight.py diff --git a/aborist/qa/verify.py b/arborist/qa/verify.py similarity index 98% rename from aborist/qa/verify.py rename to arborist/qa/verify.py index 0902216..9cb47bd 100644 --- a/aborist/qa/verify.py +++ b/arborist/qa/verify.py @@ -24,14 +24,14 @@ vocabulary; substrate calls UNGROUNDED "VISUAL"): `unverified_quotes` (kept under that name for schema continuity) collects spans the model produced that don't appear in any source — the -corpus-growth signal mined by `aborist emergent`. +corpus-growth signal mined by `arborist emergent`. Hard rule (CLAUDE.md "soft hash vs hard hash"): every check is a lexical substring test under norm-v1 + lowercase canonicalization. No embeddings, no semantic similarity, no fuzzy alignment. The contract is "this token sequence either is or isn't in the context." -Wikitext context is run through ``aborist.wikitext.to_base`` before the +Wikitext context is run through ``arborist.wikitext.to_base`` before the substring test. The corpus stores raw wikitext (so the link graph is recoverable from any page), but the LLM produces clean prose. Without the strip, every wikilink-carrying source paragraph compares as @@ -47,14 +47,14 @@ from __future__ import annotations import re import unicodedata -from aborist.qa.warrant import warrant_check +from arborist.qa.warrant import warrant_check -# Deferred import: aborist.qa.inspect imports aborist.compress & -# aborist.store at module load. The verifier doesn't need either +# Deferred import: arborist.qa.inspect imports arborist.compress & +# arborist.store at module load. The verifier doesn't need either # until deflection actually runs, so defer to call-site to keep # import order clean if anything else imports verify.py. try: - from aborist.wikitext import to_base as _wikitext_to_base + from arborist.wikitext import to_base as _wikitext_to_base except ImportError: # pragma: no cover _wikitext_to_base = None @@ -368,7 +368,7 @@ _TOKEN_PUNCT_STRIP = ".,;:!?\"()[]{}" # Common English stopwords of length >= 4 chars. Hand-curated rather -# than imported from NLTK to keep aborist dependency-light and the +# than imported from NLTK to keep arborist dependency-light and the # behavior pinned to a known set. Tokens are normalized form # (lowercase, NFC). Includes auxiliaries, prepositions, pronouns, # wh-words, conjunctions, and high-frequency adverbs/quantifiers that @@ -547,7 +547,7 @@ def verify_quotes( ) # Wikitext markup → plain prose. Identity if mwparserfromhell isn't - # installed (extras: pip install 'aborist[wikitext]'). + # installed (extras: pip install 'arborist[wikitext]'). if _wikitext_to_base is not None: context = _wikitext_to_base(context) @@ -893,7 +893,7 @@ def _lenient_json_parse(raw: str) -> tuple[object, list[str]]: # Default allowed source roles for claim_lattice_pointer mode. Roles # outside this set get classified as SOURCE_ROLE_BLOCKED. Mirrors the -# role classifications in aborist/qa/query.py:_classify_source_role; +# role classifications in arborist/qa/query.py:_classify_source_role; # "noisy_background_source" and "sequel_background_source" are # deliberately excluded by default. DEFAULT_ALLOWED_SOURCE_ROLES = ( @@ -969,7 +969,7 @@ def _claim_textually_overlaps_evidence( topical to check, and the verifier's other hard checks already own that case (claim_text_non_empty, no_manual_quotes, etc.). """ - from aborist.qa.evidence import _content_tokens + from arborist.qa.evidence import _content_tokens tokens = _content_tokens(claim_text) if not tokens: @@ -1012,7 +1012,7 @@ def _parroted_subject_tokens_absent( Empty cited_spans → return the full parroted set (defensive; no grounding at all is its own failure mode caught elsewhere). """ - from aborist.qa.evidence import _content_tokens + from arborist.qa.evidence import _content_tokens if not question_text or not claim_text: return set() @@ -1060,7 +1060,7 @@ def _claim_title_overlap(claim_text: str, source_title: str | None) -> bool: on degenerate inputs).""" if not source_title or not claim_text: return True - from aborist.qa.evidence import _content_tokens as _ct + from arborist.qa.evidence import _content_tokens as _ct claim_tokens = _ct(claim_text) title_tokens = _ct((source_title or "").replace("_", " ")) @@ -1167,11 +1167,11 @@ def verify_claim_lattice( form). Used to thread the parsed lattice into the run-DAG. """ - from aborist.qa.evidence import ( + from arborist.qa.evidence import ( evidence_map_by_pointer_id as _by_pointer, render_claim_lattice as _render, ) - from aborist.qa.parse_claims import parse_pointer_claims + from arborist.qa.parse_claims import parse_pointer_claims by_pointer = _by_pointer(evidence_map) violations: list[dict] = [] @@ -1263,7 +1263,7 @@ def verify_claim_lattice( # token-coverage bar so an off-topic chunk can no longer satisfy # the citation. Folds into governance_policy_hash via # ``claim_lattice_min_claim_content_tokens``. - from aborist.qa.evidence import _content_tokens as _ct + from arborist.qa.evidence import _content_tokens as _ct claim_content_tokens = _ct(claim_text) if len(claim_content_tokens) < min_claim_content_tokens: violations.append({ @@ -1458,7 +1458,7 @@ def verify_claim_lattice( # least one of the claim's named answer entities (proper-noun # phrases). Catches the Homer-Simpson lazy-anchor case fox # surfaced — claim asserts "Mr. Burns" but cited span is - # Castellaneta voice-actor prose. See aborist/qa/warrant.py + # Castellaneta voice-actor prose. See arborist/qa/warrant.py # for the lexical algorithm and rationale (deterministic, # not NLI). Fires only when the question shape suggests a # relation lookup AND the lookup is enabled by policy @@ -1610,9 +1610,9 @@ def verify_claim_lattice( # path. Render-layer ladder picks this up automatically. deflection_detected = False if deflection_check_enabled and question and rendered_text: - # Deferred import to avoid pulling aborist.compress + aborist.store + # Deferred import to avoid pulling arborist.compress + arborist.store # at verify.py module-load time when callers may not need them. - from aborist.qa.inspect import diagnose_deflection + from arborist.qa.inspect import diagnose_deflection signal = diagnose_deflection(question, rendered_text) if signal.get("kind") == "deflection": deflection_detected = True @@ -1782,7 +1782,7 @@ def verify_claim_lattice_json( peel (``"fence"`` / ``"prose_trim"`` / ``"curly_quotes"`` / ``"trailing_comma"``). Empty list = strict JSON parse on first try. """ - from aborist.qa.evidence import ( + from arborist.qa.evidence import ( evidence_map_by_pointer_id as _by_pointer, render_claim_lattice as _render, ) @@ -2112,9 +2112,9 @@ def verify_claim_lattice_json( # Deflection check (parallel to pointer-variant promotion). deflection_detected = False if deflection_check_enabled and question and rendered_text: - # Deferred import to avoid pulling aborist.compress + aborist.store + # Deferred import to avoid pulling arborist.compress + arborist.store # at verify.py module-load time when callers may not need them. - from aborist.qa.inspect import diagnose_deflection + from arborist.qa.inspect import diagnose_deflection signal = diagnose_deflection(question, rendered_text) if signal.get("kind") == "deflection": deflection_detected = True diff --git a/aborist/qa/warrant.py b/arborist/qa/warrant.py similarity index 100% rename from aborist/qa/warrant.py rename to arborist/qa/warrant.py diff --git a/arborist/search/__init__.py b/arborist/search/__init__.py new file mode 100644 index 0000000..56b5f0b --- /dev/null +++ b/arborist/search/__init__.py @@ -0,0 +1,6 @@ +"""Search backends.""" + +from arborist.search.base import AuditMode, Hit, SearchBackend +from arborist.search.fts5 import FTS5Backend + +__all__ = ["AuditMode", "Hit", "SearchBackend", "FTS5Backend"] diff --git a/aborist/search/base.py b/arborist/search/base.py similarity index 99% rename from aborist/search/base.py rename to arborist/search/base.py index 4382c2d..aaf7296 100644 --- a/aborist/search/base.py +++ b/arborist/search/base.py @@ -1,6 +1,6 @@ """Search backend ABC + Hit dataclass with explicit audit mode. -Every search hit carries an `audit_mode` so callers never overclaim. Aborist +Every search hit carries an `audit_mode` so callers never overclaim. Arborist adapts the Merkle-AGI v7 audit-mode trichotomy to the RAG layer: - STRICT — Merkle-verified evidence: every claim cited verbatim against the source-content tree. diff --git a/aborist/search/fts5.py b/arborist/search/fts5.py similarity index 98% rename from aborist/search/fts5.py rename to arborist/search/fts5.py index 0573efc..16423a9 100644 --- a/aborist/search/fts5.py +++ b/arborist/search/fts5.py @@ -11,8 +11,8 @@ from __future__ import annotations import re -from aborist.compress import unpack_chunk -from aborist.search.base import AuditMode, Hit, SearchBackend +from arborist.compress import unpack_chunk +from arborist.search.base import AuditMode, Hit, SearchBackend _FTS5_TOKEN_RE = re.compile(r"[A-Za-z][A-Za-z0-9]*") diff --git a/aborist/snapshot.py b/arborist/snapshot.py similarity index 97% rename from aborist/snapshot.py rename to arborist/snapshot.py index bd515b3..e0c6815 100644 --- a/aborist/snapshot.py +++ b/arborist/snapshot.py @@ -11,7 +11,7 @@ corpus state. Storage of the snapshots themselves is local: a `snapshots` row records the root, audit-chain pin, doc_count, optional parent (for chains), and human-readable reason. The root is reproducible from the documents -table at any time, so `aborist snapshot verify ` re-runs the +table at any time, so `arborist snapshot verify ` re-runs the computation and compares. In sharded mode (`--shards-dir`) the snapshot reads the UNION view @@ -26,8 +26,8 @@ import sqlite3 import time from typing import Iterable -from aborist.merkle import MerkleTree -from aborist.store import append_audit, transaction +from arborist.merkle import MerkleTree +from arborist.store import append_audit, transaction # Hash bytes of an empty corpus. Distinct from the genesis Merkle root over diff --git a/aborist/source.py b/arborist/source.py similarity index 81% rename from aborist/source.py rename to arborist/source.py index f862ee2..178094f 100644 --- a/aborist/source.py +++ b/arborist/source.py @@ -1,6 +1,6 @@ """Source ABC. -Adding a new corpus to aborist = one new Source subclass. The Source contract +Adding a new corpus to arborist = one new Source subclass. The Source contract is intentionally minimal: yield Document objects, one at a time. """ @@ -9,7 +9,7 @@ from __future__ import annotations from abc import ABC, abstractmethod from typing import Iterator -from aborist.document import Document +from arborist.document import Document class Source(ABC): diff --git a/aborist/sources/__init__.py b/arborist/sources/__init__.py similarity index 66% rename from aborist/sources/__init__.py rename to arborist/sources/__init__.py index 069e553..063b3bc 100644 --- a/aborist/sources/__init__.py +++ b/arborist/sources/__init__.py @@ -1,13 +1,13 @@ """Source implementations. Add a new corpus = add a new module here.""" -from aborist.sources.grok import GrokExportSource, GrokMediaPostsSource -from aborist.sources.vcs import GitRepoSource, MercurialRepoSource -from aborist.sources.wikipedia import ( +from arborist.sources.grok import GrokExportSource, GrokMediaPostsSource +from arborist.sources.vcs import GitRepoSource, MercurialRepoSource +from arborist.sources.wikipedia import ( WikipediaCurDump, WikipediaOldDump, WikipediaSqlDump, ) -from aborist.sources.wikipedia_xml import WikipediaAbstractDump, WikipediaXmlDump +from arborist.sources.wikipedia_xml import WikipediaAbstractDump, WikipediaXmlDump __all__ = [ "GitRepoSource", @@ -24,7 +24,7 @@ __all__ = [ # HtmlPageSource has optional deps (selectolax). Surface import errors only # when callers actually request it. try: - from aborist.sources.html_page import HtmlPageSource # noqa: F401 + from arborist.sources.html_page import HtmlPageSource # noqa: F401 __all__.append("HtmlPageSource") except ImportError: # pragma: no cover pass @@ -32,7 +32,7 @@ except ImportError: # pragma: no cover # HtmlPageSource has optional deps (httpx + selectolax). Surface import errors # only when callers actually request it. try: - from aborist.sources.html_page import HtmlPageSource # noqa: F401 + from arborist.sources.html_page import HtmlPageSource # noqa: F401 __all__.append("HtmlPageSource") except ImportError: # pragma: no cover pass diff --git a/aborist/sources/crawler/__init__.py b/arborist/sources/crawler/__init__.py similarity index 83% rename from aborist/sources/crawler/__init__.py rename to arborist/sources/crawler/__init__.py index fbcd665..bab7522 100644 --- a/aborist/sources/crawler/__init__.py +++ b/arborist/sources/crawler/__init__.py @@ -2,18 +2,18 @@ Verbatim lift from ``~/git/agents.ai.unturf.com/core/`` (rev as of 2026-04-28): - core/async_web_fetcher.py -> aborist/sources/crawler/async_web_fetcher.py - core/web_fetch.py -> aborist/sources/crawler/web_fetch.py + core/async_web_fetcher.py -> arborist/sources/crawler/async_web_fetcher.py + core/web_fetch.py -> arborist/sources/crawler/web_fetch.py Two source-side adaptations during the lift: 1. Chat-bot fetch triggers (``has_fresh_fetch_trigger`` / - ``has_web_fetch_trigger`` from ``core.keywords``) were dropped. Aborist + ``has_web_fetch_trigger`` from ``core.keywords``) were dropped. Arborist has no chat surface — fetch intent is detected at the application layer, not from message content. 2. ``web_cache_manager.py`` (SQLAlchemy-backed page cache) was NOT - lifted. Aborist has its own content-addressed store; cache through + lifted. Arborist has its own content-addressed store; cache through that path instead of carrying SQLAlchemy as a dep. **Off by default.** Heavy dependencies (aiohttp, beautifulsoup4, lxml, @@ -25,8 +25,8 @@ runs the lifted tests with the extras installed. Public surface — same as the agents repo: - from aborist.sources.crawler import AsyncWebFetcher, CrawlMode - from aborist.sources.crawler import ( + from arborist.sources.crawler import AsyncWebFetcher, CrawlMode + from arborist.sources.crawler import ( URIContentType, extract_all_uris_from_content, extract_url_from_content, @@ -42,7 +42,7 @@ when given a URL. # Suppress bs4's "you're parsing XML with the HTML parser" warning when the # crawler hits a sitemap.xml / RSS feed via the generic page handler. Benign # (parsing still works) but spams stderr during a deep BFS. Filtering here — -# at the aborist subpackage boundary — keeps the verbatim lift untouched. +# at the arborist subpackage boundary — keeps the verbatim lift untouched. import warnings as _warnings try: from bs4 import XMLParsedAsHTMLWarning as _XMLParsedAsHTMLWarning @@ -50,7 +50,7 @@ try: except ImportError: # pragma: no cover pass -from aborist.sources.crawler.async_web_fetcher import ( +from arborist.sources.crawler.async_web_fetcher import ( AsyncWebFetcher, CrawlMode, FeedItem, @@ -64,7 +64,7 @@ from aborist.sources.crawler.async_web_fetcher import ( parse_rss, parse_sitemap, ) -from aborist.sources.crawler.web_fetch import ( +from arborist.sources.crawler.web_fetch import ( URIContentType, build_sources_footer, detect_uri_content_type, diff --git a/aborist/sources/crawler/async_web_fetcher.py b/arborist/sources/crawler/async_web_fetcher.py similarity index 99% rename from aborist/sources/crawler/async_web_fetcher.py rename to arborist/sources/crawler/async_web_fetcher.py index fb6b30b..1ca32e4 100644 --- a/aborist/sources/crawler/async_web_fetcher.py +++ b/arborist/sources/crawler/async_web_fetcher.py @@ -26,8 +26,8 @@ from urllib.robotparser import RobotFileParser import time # Heavy crawler dependencies are optional. Install via: -# pip install 'aborist[crawler]' -# Off by default per CLAUDE.md — the core aborist test suite does not +# pip install 'arborist[crawler]' +# Off by default per CLAUDE.md — the core arborist test suite does not # touch the network and must not import this module unless asked. try: import aiohttp @@ -35,7 +35,7 @@ try: from miniuri import Uri except ImportError as e: # pragma: no cover raise ImportError( - "crawler module requires extras: pip install 'aborist[crawler]'" + "crawler module requires extras: pip install 'arborist[crawler]'" ) from e diff --git a/aborist/sources/crawler/bridge.py b/arborist/sources/crawler/bridge.py similarity index 96% rename from aborist/sources/crawler/bridge.py rename to arborist/sources/crawler/bridge.py index 68ffe73..7409a45 100644 --- a/aborist/sources/crawler/bridge.py +++ b/arborist/sources/crawler/bridge.py @@ -1,4 +1,4 @@ -"""Bridge: aborist sources/crawler ↔ aborist ingest pipeline. +"""Bridge: arborist sources/crawler ↔ arborist ingest pipeline. Two operations: @@ -13,7 +13,7 @@ Two operations: document into ``document_http_meta`` so a future recrawl-check can send conditional HEAD requests. -Off by default — both operations require ``aborist[crawler]`` extras. +Off by default — both operations require ``arborist[crawler]`` extras. """ from __future__ import annotations @@ -28,12 +28,12 @@ from typing import Iterable, Iterator import httpx -from aborist.document import Document -from aborist.ingest import ingest_source -from aborist.progress import Progress -from aborist.sources.crawler.async_web_fetcher import AsyncWebFetcher -from aborist.sources.html_page import USER_AGENT, parse_html -from aborist.store import transaction +from arborist.document import Document +from arborist.ingest import ingest_source +from arborist.progress import Progress +from arborist.sources.crawler.async_web_fetcher import AsyncWebFetcher +from arborist.sources.html_page import USER_AGENT, parse_html +from arborist.store import transaction def _same_domain(a: str, b: str) -> bool: @@ -117,10 +117,10 @@ async def _crawl_seed_async( """BFS from `seed_url`, staying on the same hostname. Discovery only — fetches each page enough to extract links, no - aborist-side ingestion happens here. Returns a deduplicated list + arborist-side ingestion happens here. Returns a deduplicated list of URLs in BFS order, capped at `max_pages`. - Pass ``progress`` (an `aborist.progress.Progress`) for stderr + Pass ``progress`` (an `arborist.progress.Progress`) for stderr heartbeats while the BFS runs. Lines are rate-limited by the Progress instance (default 2s) so a fast crawl prints once or twice; a slow one prints every interval. diff --git a/aborist/sources/crawler/web_fetch.py b/arborist/sources/crawler/web_fetch.py similarity index 98% rename from aborist/sources/crawler/web_fetch.py rename to arborist/sources/crawler/web_fetch.py index 49ffe3e..400a547 100644 --- a/aborist/sources/crawler/web_fetch.py +++ b/arborist/sources/crawler/web_fetch.py @@ -11,29 +11,29 @@ from enum import Enum from typing import Optional, Dict, List, Callable, Awaitable # Heavy crawler dependencies are optional. Install via: -# pip install 'aborist[crawler]' +# pip install 'arborist[crawler]' try: from miniuri import Uri except ImportError as e: # pragma: no cover raise ImportError( - "crawler module requires extras: pip install 'aborist[crawler]'" + "crawler module requires extras: pip install 'arborist[crawler]'" ) from e # `core.keywords` lived in agents.ai.unturf.com and detected fetch-intent -# from chat-bot messages. Aborist has no chat surface, so the verbatim +# from chat-bot messages. Arborist has no chat surface, so the verbatim # lift drops those triggers. Callers asking "is this content a fetch # request?" should adapt at the application layer instead. def has_fresh_fetch_trigger(content: str) -> bool: # pragma: no cover raise NotImplementedError( - "chat-bot fetch triggers were dropped during the aborist lift; " + "chat-bot fetch triggers were dropped during the arborist lift; " "callers should detect fetch intent at the application layer" ) def has_web_fetch_trigger(content: str) -> bool: # pragma: no cover raise NotImplementedError( - "chat-bot fetch triggers were dropped during the aborist lift; " + "chat-bot fetch triggers were dropped during the arborist lift; " "callers should detect fetch intent at the application layer" ) diff --git a/aborist/sources/grok.py b/arborist/sources/grok.py similarity index 99% rename from aborist/sources/grok.py rename to arborist/sources/grok.py index a8eab52..4e05063 100644 --- a/aborist/sources/grok.py +++ b/arborist/sources/grok.py @@ -21,8 +21,8 @@ from datetime import datetime, timezone from pathlib import Path from typing import Iterator -from aborist.document import Document, Edge -from aborist.source import Source +from arborist.document import Document, Edge +from arborist.source import Source # Strip Grok-internal render markers from message text. These appear as diff --git a/aborist/sources/html_page.py b/arborist/sources/html_page.py similarity index 94% rename from aborist/sources/html_page.py rename to arborist/sources/html_page.py index f5b15f8..2a8dcd9 100644 --- a/aborist/sources/html_page.py +++ b/arborist/sources/html_page.py @@ -3,7 +3,7 @@ Fetches URLs, honors robots.txt automatically, strips noise (script/style/nav/ footer/header), extracts main body text + outbound `` links as edges. -Optional dependency. Install with `pip install aborist[html]`. +Optional dependency. Install with `pip install arborist[html]`. """ from __future__ import annotations @@ -19,14 +19,14 @@ try: from selectolax.parser import HTMLParser except ImportError as e: # pragma: no cover raise ImportError( - "HTML source requires extras: pip install 'aborist[html]'" + "HTML source requires extras: pip install 'arborist[html]'" ) from e -from aborist.document import Document, Edge -from aborist.source import Source +from arborist.document import Document, Edge +from arborist.source import Source -USER_AGENT = "aborist/0.0.1 (+https://unturf.com)" +USER_AGENT = "arborist/0.0.1 (+https://unturf.com)" NOISE_SELECTORS = ("script", "style", "noscript", "nav", "header", "footer", "aside") diff --git a/aborist/sources/providence.py b/arborist/sources/providence.py similarity index 94% rename from aborist/sources/providence.py rename to arborist/sources/providence.py index 9c1e493..dfd5ba1 100644 --- a/aborist/sources/providence.py +++ b/arborist/sources/providence.py @@ -19,8 +19,8 @@ import sqlite3 import time from typing import Iterator -from aborist.document import Document -from aborist.source import Source +from arborist.document import Document +from arborist.source import Source # Mirrors the mesh-sync kindergarten convention: records younger @@ -32,7 +32,7 @@ DEFAULT_KINDERGARTEN_SECONDS = 3600 # URI scheme prefix for self-promoted providence documents. The # query-side `_classify_source_role` matches this prefix to assign # `self_reference_source`. -PROVIDENCE_URI_PREFIX = "aborist://providence/" +PROVIDENCE_URI_PREFIX = "arborist://providence/" class ProvidenceSource(Source): @@ -43,7 +43,7 @@ class ProvidenceSource(Source): 1. ``audit_mode == 'STRICT'`` — fully grounded only. HYBRID and UNGROUNDED records stay out of the substrate. 2. ``falsification_state == 'live'`` — falsified rows excluded. - The existing `aborist providence --falsify` machinery is the + The existing `arborist providence --falsify` machinery is the verifier-falsification mechanism: when a record's state flips, it stops being substrate on next ingest. 3. ``now - created_at >= kindergarten_seconds`` (default 1h) — @@ -56,7 +56,7 @@ class ProvidenceSource(Source): Each yielded Document: - - URI: ``aborist://providence/`` (content-addressed, + - URI: ``arborist://providence/`` (content-addressed, stable across runs, distinguishable from external URIs). - title: the question text, truncated to ~120 chars. - content: canonical layout ``Q: \\n\\nA: ``. diff --git a/aborist/sources/vcs.py b/arborist/sources/vcs.py similarity index 98% rename from aborist/sources/vcs.py rename to arborist/sources/vcs.py index 079ac0e..887a320 100644 --- a/aborist/sources/vcs.py +++ b/arborist/sources/vcs.py @@ -3,7 +3,7 @@ Each yields one Document per text file at HEAD/tip, with a stable URI that does NOT include the commit hash — so re-ingesting the same repo after new commits produces *new* documents for changed files which -aborist's prior-document detection auto-chains via `supersedes` edges. +arborist's prior-document detection auto-chains via `supersedes` edges. That gives "the Merkle tree grows over time" semantics for free: every new commit appends to the audit chain, every changed file gets a new content-addressed Document, and the supersedes edges connect them. @@ -25,8 +25,8 @@ import subprocess from pathlib import Path from typing import Iterator -from aborist.document import Document -from aborist.source import Source +from arborist.document import Document +from arborist.source import Source # Skip files >5 MB by default. Source code, prose, configs all fit diff --git a/aborist/sources/wikipedia.py b/arborist/sources/wikipedia.py similarity index 99% rename from aborist/sources/wikipedia.py rename to arborist/sources/wikipedia.py index 9721fa1..e691247 100644 --- a/aborist/sources/wikipedia.py +++ b/arborist/sources/wikipedia.py @@ -16,8 +16,8 @@ import re from pathlib import Path from typing import IO, Iterator -from aborist.document import Document, Edge -from aborist.source import Source +from arborist.document import Document, Edge +from arborist.source import Source # Match [[Target]], [[Target|display]], [[Target#anchor]] forms. diff --git a/aborist/sources/wikipedia_xml.py b/arborist/sources/wikipedia_xml.py similarity index 98% rename from aborist/sources/wikipedia_xml.py rename to arborist/sources/wikipedia_xml.py index 32d576b..001252f 100644 --- a/aborist/sources/wikipedia_xml.py +++ b/arborist/sources/wikipedia_xml.py @@ -28,9 +28,9 @@ from pathlib import Path from typing import IO, Iterator from xml.etree.ElementTree import Element, iterparse -from aborist.document import Document, Edge -from aborist.source import Source -from aborist.sources.wikipedia import _extract_wikilinks +from arborist.document import Document, Edge +from arborist.source import Source +from arborist.sources.wikipedia import _extract_wikilinks # MediaWiki export schema declares an xmlns; ElementTree returns tag names @@ -87,7 +87,7 @@ class WikipediaXmlDump(Source): page has many. By default we emit the LAST revision per page (matches cur semantics); pass `multi_revision=True` to emit every revision (matches old semantics, with `supersedes` chaining at the URI level - via aborist's existing prior-document detection). + via arborist's existing prior-document detection). """ source_type = "wikipedia_xml" diff --git a/aborist/store.py b/arborist/store.py similarity index 99% rename from aborist/store.py rename to arborist/store.py index d7852be..fad4117 100644 --- a/aborist/store.py +++ b/arborist/store.py @@ -23,7 +23,7 @@ from pathlib import Path from typing import Iterator -DEFAULT_DB_PATH = Path.home() / ".aborist" / "aborist.db" +DEFAULT_DB_PATH = Path.home() / ".arborist" / "arborist.db" SCHEMA_SQL = """ @@ -244,8 +244,8 @@ CREATE TABLE IF NOT EXISTS snapshots ( CREATE INDEX IF NOT EXISTS idx_snapshots_taken_at ON snapshots(taken_at); -- Mesh layer tables. Off by default — populated only when the user runs --- `aborist mesh init`. Never accessed by ingest / query / distill paths; --- mesh state is opt-in plumbing for federated peers (see aborist.mesh). +-- `arborist mesh init`. Never accessed by ingest / query / distill paths; +-- mesh state is opt-in plumbing for federated peers (see arborist.mesh). CREATE TABLE IF NOT EXISTS mesh_identity ( id INTEGER PRIMARY KEY CHECK (id = 1), -- singleton member_id TEXT NOT NULL UNIQUE, @@ -349,7 +349,7 @@ CREATE VIRTUAL TABLE IF NOT EXISTS documents_fts USING fts5( -- concept relations without invalidating any cached answers. -- -- Cross-shard lookup. Concept relations live in the shard whose document --- they were derived from; the lookup helpers in aborist.concepts walk all +-- they were derived from; the lookup helpers in arborist.concepts walk all -- shards (same pattern as cross-shard FTS5 search). Mesh sync moves shards -- between peers; concept relations come along for the ride automatically. -- @@ -363,7 +363,7 @@ CREATE VIRTUAL TABLE IF NOT EXISTS documents_fts USING fts5( -- 'category' - token belongs to category target (Wikipedia -- [[Category:X]] tail; HTML schema.org/ classification) -- --- evidence_kind: which extractor produced the row. Lets `aborist concepts +-- evidence_kind: which extractor produced the row. Lets `arborist concepts -- purge --evidence-kind X` revoke a single extractor's output cleanly -- without touching manual or other-extractor rows. New extractors register -- a stable evidence_kind string; legacy seeds are 'manual_legacy'. diff --git a/aborist/wikitext.py b/arborist/wikitext.py similarity index 95% rename from aborist/wikitext.py rename to arborist/wikitext.py index 5d33945..9176fa1 100644 --- a/aborist/wikitext.py +++ b/arborist/wikitext.py @@ -1,6 +1,6 @@ """Wikitext → base prose conversion. -Aborist stores raw MediaWiki wikitext in ``chunks.content`` so the link +Arborist stores raw MediaWiki wikitext in ``chunks.content`` so the link graph and original markup are recoverable from any page on demand. For LLM context and post-LLM faithfulness verification we need *prose* — a deterministic plain-text projection of the same chunk. @@ -13,7 +13,7 @@ Versioning protocol ------------------- Bump ``BASE_VERSION`` whenever the algorithm changes. Callers fold ``BASE_VERSION`` into ``governance_policy_hash`` (via ``policy["base_version"]`` -in ``aborist.qa.runner`` / ``aborist.qa.query``) so a bump invalidates every +in ``arborist.qa.runner`` / ``arborist.qa.query``) so a bump invalidates every prior providence-cache record's 8-dim cache_key on the next lookup. No schema migration; the next ``ask`` re-derives against fresh prose. @@ -33,7 +33,7 @@ Algorithm (wikitext-base-v1) 5. Whitespace pass: collapse runs of spaces/tabs, drop trailing space on lines, collapse 3+ newlines to 2. -Optional dependency. Install with ``pip install aborist[wikitext]``. +Optional dependency. Install with ``pip install arborist[wikitext]``. """ from __future__ import annotations @@ -45,7 +45,7 @@ try: except ImportError as e: # pragma: no cover raise ImportError( "wikitext base conversion requires extras: " - "pip install 'aborist[wikitext]'" + "pip install 'arborist[wikitext]'" ) from e diff --git a/pyproject.toml b/pyproject.toml index 938aa08..7d3284f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=68"] build-backend = "setuptools.build_meta" [project] -name = "aborist" +name = "arborist" version = "0.0.1" description = "An arborist for trees and forests of cross-linked information" readme = "README.md" @@ -35,7 +35,7 @@ mesh = [ crawler = [ # Verbatim lift from agents.ai.unturf.com/core. Off by default — the # default test suite never imports the crawler. Install with: - # pip install 'aborist[crawler]' + # pip install 'arborist[crawler]' # then run `make test-crawler`. "aiohttp>=3.8", "beautifulsoup4>=4.11", @@ -52,15 +52,15 @@ dev = [ "pytest>=8", "pytest-asyncio>=0.23", "pytest-xdist>=3.5", - "aborist[html]", - "aborist[wikitext]", - "aborist[mesh]", - "aborist[crawler]", + "arborist[html]", + "arborist[wikitext]", + "arborist[mesh]", + "arborist[crawler]", ] [project.scripts] -aborist = "aborist.cli:main" +arborist = "arborist.cli:main" [tool.setuptools.packages.find] where = ["."] -include = ["aborist*"] +include = ["arborist*"]