russell@unturf.com
a5e709a470
Lifts the async web fetcher into aborist as an opt-in source. The implementation comes directly from ~/git/agents.ai.unturf.com/core (rev 2026-04-28); aborist's adaptations are minimal and documented in aborist/sources/crawler/__init__.py: core/async_web_fetcher.py -> aborist/sources/crawler/async_web_fetcher.py core/web_fetch.py -> aborist/sources/crawler/web_fetch.py Two source-side changes during the lift: 1. Heavy deps (aiohttp, bs4, miniuri) wrapped in try/except so a bare `import aborist.sources.crawler` raises ImportError with the install hint instead of leaking AttributeErrors deep in user code. 2. Chat-bot fetch triggers (`has_fresh_fetch_trigger`, `has_web_fetch_trigger` from agents.ai.unturf.com/core/keywords) replaced with NotImplementedError stubs. Aborist has no chat surface — fetch intent is detected at the application layer. The two test classes that exercised these triggers are `@pytest.mark.skip`'d with the same rationale. Not lifted: web_cache_manager.py — it backs page caching with SQLAlchemy. Aborist has its own content-addressed cache via providence_cache; no need to carry SQLAlchemy as a dep just for crawled-page memoization. Off by default: - `[crawler]` extras section in pyproject.toml carries the heavy deps. `[dev]` pulls them in so the crawler tests can run. - `make test` ignores tests/crawler/ entirely. - `make bootstrap-crawler` installs the extras into the venv. - `make test-crawler` runs only the lifted tests after extras land. Tests: 74 passed, 9 skipped (the chat-bot trigger tests deliberately dropped). Default `make test` stays at 273 passed, 1 skipped.
65 lines
1.5 KiB
TOML
65 lines
1.5 KiB
TOML
[build-system]
|
|
requires = ["setuptools>=68"]
|
|
build-backend = "setuptools.build_meta"
|
|
|
|
[project]
|
|
name = "aborist"
|
|
version = "0.0.1"
|
|
description = "An arborist for trees and forests of cross-linked information"
|
|
readme = "README.md"
|
|
license = { text = "AGPL-3.0-only" }
|
|
requires-python = ">=3.10"
|
|
authors = [
|
|
{ name = "Russell Ballestrini", email = "russell@unturf.com" },
|
|
{ name = "foxhop" },
|
|
{ name = "TimeHexOn" },
|
|
]
|
|
dependencies = [
|
|
"httpx>=0.27",
|
|
"zstandard>=0.22",
|
|
"cryptography>=42",
|
|
]
|
|
|
|
[project.optional-dependencies]
|
|
html = [
|
|
"selectolax>=0.3",
|
|
]
|
|
wikitext = [
|
|
"mwparserfromhell>=0.6",
|
|
]
|
|
mesh = [
|
|
# httpx is already in core deps; mesh wire only depends on stdlib +
|
|
# cryptography (also core). This extras block exists as the documented
|
|
# opt-in surface even though no extra packages are required today.
|
|
]
|
|
crawler = [
|
|
# Verbatim lift from agents.ai.unturf.com/core. Off by default — the
|
|
# default test suite never imports the crawler. Install with:
|
|
# pip install 'aborist[crawler]'
|
|
# then run `make test-crawler`.
|
|
"aiohttp>=3.8",
|
|
"beautifulsoup4>=4.11",
|
|
"lxml>=4.9",
|
|
"html5lib>=1.1",
|
|
"html2text>=2024.2.26",
|
|
"miniuri>=1.1",
|
|
"feedparser>=6.0",
|
|
"Pillow>=10.0",
|
|
"cairosvg>=2.7",
|
|
"pypdf>=4.0",
|
|
]
|
|
dev = [
|
|
"pytest>=8",
|
|
"pytest-asyncio>=0.23",
|
|
"aborist[html]",
|
|
"aborist[wikitext]",
|
|
"aborist[mesh]",
|
|
"aborist[crawler]",
|
|
]
|
|
|
|
[project.scripts]
|
|
aborist = "aborist.cli:main"
|
|
|
|
[tool.setuptools.packages.find]
|
|
where = ["."]
|
|
include = ["aborist*"]
|