russell@unturf.com
3bc5ec1e6c
Pure-cloud consumer: client opens an arborist .db file IN PLACE on a bucket via HTTP RANGE reads, runs FTS5 + SQL locally, fetches chunk bodies from `blobs/<hash>` on the same bucket. No intermediate server in the data path. The bucket layout we already produce (Tier A clones plus --jit-blobs blobs/) is exactly what this consumer needs. Module `arborist/wallet/bucket.py`: - HttpRangeFile / HttpRangeVFS: apsw subclasses. xRead → HTTP Range GET; xFileSize → cached HEAD. xWrite/xTruncate raise (read-only). IOCAP_IMMUTABLE so SQLite skips locking/journaling. Empty tempfile backs the apsw VFSFile C-bookkeeping; never actually read. - _LRUByteCache: thread-safe (offset,length)-keyed LRU; soft byte budget (default 32 MB). SQLite's own page cache (~8 MB) handles most hot-path amortization, so our LRU is the second-level safety net for working sets that overflow SQLite's cache. - _HttpTransport: stdlib urllib (zero new runtime deps beyond apsw). - BucketClient: high-level — fts_search / chunks_for_doc / fetch_chunk_body / snapshot_root + page-cache stats. CLI (`arborist cloud <sub>`): - `cloud search Q --shard-url ...` - `cloud snapshot-root --shard-url ...` - `cloud fetch-chunk LEAF_HASH --blob-base ...` Makefile: - `make bootstrap-bucket` (installs apsw) - `make cloud-search Q="..." SHARD_URL=https://.../000.db` - `make cloud-snapshot-root SHARD_URL=...` - `make cloud-fetch-chunk LEAF_HASH=... BLOB_BASE=...` - `make cloud-demo` — end-to-end proof on a vanilla laptop: seeds a tiny bucket layout in tmp, serves it via a Range-aware static HTTP server, runs all three cloud commands from an isolated HOME that has no local arborist data. Asserts laptop HOME stays empty start-to-finish. Tests (tests/test_wallet_bucket.py, 4 passing): - bucket-direct FTS5 results == direct sqlite3 results - chunk fetch round-trip + hash verify - snapshot_root bucket-direct == snapshot_root local - second identical query adds 0 HTTP requests (SQLite-cached) pyproject: new `[bucket]` extra carries apsw>=3.45; folded into [dev].
176 lines
6.6 KiB
TOML
176 lines
6.6 KiB
TOML
[build-system]
|
|
requires = ["setuptools>=68"]
|
|
build-backend = "setuptools.build_meta"
|
|
|
|
[project]
|
|
name = "arborist"
|
|
version = "0.0.1"
|
|
description = "An arborist for trees and forests of cross-linked information"
|
|
readme = "README.md"
|
|
license = { text = "AGPL-3.0-only" }
|
|
requires-python = ">=3.10"
|
|
authors = [
|
|
{ name = "Russell Ballestrini", email = "russell@unturf.com" },
|
|
{ name = "foxhop" },
|
|
{ name = "TimeHexOn" },
|
|
]
|
|
dependencies = [
|
|
"httpx>=0.27",
|
|
"zstandard>=0.22",
|
|
"cryptography>=42",
|
|
]
|
|
|
|
[project.optional-dependencies]
|
|
html = [
|
|
"selectolax>=0.3",
|
|
]
|
|
wikitext = [
|
|
"mwparserfromhell>=0.6",
|
|
]
|
|
mesh = [
|
|
# httpx is already in core deps; mesh wire only depends on stdlib +
|
|
# cryptography (also core). This extras block exists as the documented
|
|
# opt-in surface even though no extra packages are required today.
|
|
]
|
|
math = [
|
|
# Symbolic algebra/calculus π* substrate (ticket #000030). SymPy is
|
|
# ~30 MB installed; pulling it into core deps would inflate every
|
|
# fresh checkout. Tests skip via pytest.importorskip when absent.
|
|
"sympy>=1.13",
|
|
]
|
|
hessian = [
|
|
# Phi_alignment_probe (ticket #000034 Phase 1a). Lanczos top-k +
|
|
# bottom-k eigendecomposition for measuring whether v7's frozen
|
|
# linear projection W aligns with the loss Hessian's low-eigenvalue
|
|
# subspace. Numpy + scipy together ~80 MB; gated separately from
|
|
# core to keep the default install lightweight. Tests skip via
|
|
# pytest.importorskip when absent. Install with:
|
|
# pip install 'arborist[hessian]'
|
|
"numpy>=1.26",
|
|
"scipy>=1.11",
|
|
]
|
|
crawler = [
|
|
# Verbatim lift from agents.ai.unturf.com/core. Off by default — the
|
|
# default test suite never imports the crawler. Install with:
|
|
# pip install 'arborist[crawler]'
|
|
# then run `make test-crawler`.
|
|
"aiohttp>=3.8",
|
|
"beautifulsoup4>=4.11",
|
|
"lxml>=4.9",
|
|
"html5lib>=1.1",
|
|
"html2text>=2024.2.26",
|
|
"miniuri>=1.1",
|
|
"feedparser>=6.0",
|
|
"Pillow>=10.0",
|
|
"cairosvg>=2.7",
|
|
"pypdf>=4.0",
|
|
]
|
|
vec = [
|
|
# Optional sqlite-vec semantic retrieval backend (ticket #000039).
|
|
# sqlite-vec ships only the loadable SQLite extension (~1 MB);
|
|
# fastembed pulls onnxruntime + tokenizers + huggingface-hub
|
|
# (~150 MB) and downloads the bge-small-en-v1.5 ONNX model
|
|
# (~130 MB) on first use. Gated separately so a fresh checkout
|
|
# stays python3.12 + venv + sqlite3. CLI surfaces `arborist embed`
|
|
# / `--backend vec` only when `sqlite_vec` imports. Install with:
|
|
# pip install 'arborist[vec]'
|
|
# (sentence-transformers is the heavier "official" embedder path
|
|
# the ticket §5 names; fastembed is the lightweight ONNX one.)
|
|
"sqlite-vec>=0.1.9",
|
|
"fastembed>=0.4",
|
|
]
|
|
nli = [
|
|
# Sentence-pair NLI for the #000049 Phase-2 *shadow* path
|
|
# (arborist/qa/nli/) — measures whether a clause-level contradiction
|
|
# veto would demote a weakly-grounded answer; never touches
|
|
# audit_mode. transformers + a CPU torch is ~600 MB installed, so it
|
|
# is gated hard out of core / dev — a fresh checkout stays
|
|
# python3.12 + venv + sqlite3, and the default test suite skips the
|
|
# NLI tests via pytest.importorskip when this extra is absent.
|
|
# Install with:
|
|
# pip install 'arborist[nli]'
|
|
# `optimum[onnxruntime]` gives the ONNX-export + int8-quantize path
|
|
# (`bench/scripts/export_nli_onnx.py`, `make export-nli-onnx`):
|
|
# `onnxruntime` on a quantized cross-encoder is ~2-4x faster on CPU
|
|
# than the torch forward path; `ShadowNLI._ensure_loaded` auto-prefers
|
|
# an export if it finds one. torch is still here because `optimum`'s
|
|
# exporter uses it, and it's the fallback when no export exists; a
|
|
# Phase-3 runtime could ship an `[nli-onnx]`-only extra (onnxruntime,
|
|
# no torch) once the export is committed/distributed (cf. [vec]).
|
|
"transformers>=4.40",
|
|
"torch>=2.2",
|
|
"sentencepiece>=0.2",
|
|
"protobuf>=4.0",
|
|
"optimum[onnxruntime]>=1.20",
|
|
]
|
|
bucket = [
|
|
# SPV-style bucket-direct queries (arborist/wallet/bucket.py): open a
|
|
# shard `.db` file in place on a bucket via SQLite HTTP-range VFS, no
|
|
# intermediate server, no local DB. Uses apsw because Python's stdlib
|
|
# sqlite3 doesn't expose the VFS API; apsw is a thin C wrapper that
|
|
# does. Gated separately so a fresh checkout stays python3.12 + venv
|
|
# + sqlite3; tests skip via pytest.importorskip when absent. Install
|
|
# with:
|
|
# pip install 'arborist[bucket]'
|
|
"apsw>=3.45",
|
|
]
|
|
object-store = [
|
|
# Cold-object eviction tier (ticket #000061). Pushes chunk bodies to an
|
|
# S3-compatible bucket keyed by leaf_hash so the corpus can grow past
|
|
# one machine while the Merkle tree stays intact. One backend covers
|
|
# AWS S3, DigitalOcean Spaces, Cloudflare R2, Backblaze B2, GCS
|
|
# (S3 interop), and MinIO — boto3 with a per-provider endpoint_url.
|
|
# Gated separately so a fresh checkout stays python3.12 + venv +
|
|
# sqlite3; tests skip via pytest.importorskip when absent. Install with:
|
|
# pip install 'arborist[object-store]'
|
|
# Credentials use boto3's standard discovery (env vars,
|
|
# ~/.aws/credentials, IAM role); never read into arborist code. The
|
|
# wire-level test gated additionally on `moto`.
|
|
"boto3>=1.34",
|
|
]
|
|
mt = [
|
|
# Local machine-translation engine for the #000056 "Operation
|
|
# Sandwich" cross-language grounding edges (arborist/qa/mt/):
|
|
# translate the query es->en (retrieval-side) and render the
|
|
# *verified English* answer en->es for display only. Helsinki-NLP
|
|
# opus-mt (MarianMT) is Apache-2.0, hash-pinnable by HF revision
|
|
# (reproducible — provenance-critical for a Merkle-replay system),
|
|
# and es<->en is its best-resourced pair. Hard out of core/dev like
|
|
# [nli]: a fresh checkout stays python3.12 + venv + sqlite3, and the
|
|
# default suite drives the sandwich via a deterministic
|
|
# StubTranslator so it never needs these weights. Install with:
|
|
# pip install 'arborist[mt]'
|
|
# Weights are NOT in the repo; they cache under
|
|
# ~/.arborist/models/mt/. Never Hermes-3-8B (8B unfit for
|
|
# human-language translation); never an external API (zero egress /
|
|
# Operation Voyeur; opaque API "versions" are provenance-hostile).
|
|
"transformers>=4.40",
|
|
"torch>=2.2",
|
|
"sentencepiece>=0.2",
|
|
"protobuf>=4.0",
|
|
"sacremoses>=0.1",
|
|
]
|
|
dev = [
|
|
"pytest>=8",
|
|
"pytest-asyncio>=0.23",
|
|
"pytest-xdist>=3.5",
|
|
"arborist[html]",
|
|
"arborist[wikitext]",
|
|
"arborist[mesh]",
|
|
"arborist[crawler]",
|
|
"arborist[math]",
|
|
"arborist[hessian]",
|
|
"arborist[vec]",
|
|
"arborist[object-store]",
|
|
"arborist[bucket]",
|
|
# moto is the wire-level boto3 test stub; only needed to run
|
|
# tests/test_cold_object_boto3.py (the default suite uses MemoryBackend).
|
|
"moto>=5.0",
|
|
]
|
|
|
|
[project.scripts]
|
|
arborist = "arborist.cli:main"
|
|
|
|
[tool.setuptools.packages.find]
|
|
where = ["."]
|
|
include = ["arborist*"]
|