russell@unturf.com
22a8071936
Adds aborist/wikitext.py with to_base() — deterministic wikitext → prose conversion via mwparserfromhell. Drops <ref>...</ref>, [[File:...]], [[Image:...]], [[Category:...]] entirely; resolves piped wikilinks to display text; collapses templates, formatting, and HTML markup. extract_wikilinks() preserves the link graph for the definition-cloud artifact (recoverable from any page on demand without re-parsing wikitext at query time). Wires to_base() into verify_quotes(). Without the strip, the verifier compared the model's clean prose against [[Cloud Strife]], [[Shinra Electric Power Company|Shinra]], etc. and falsely flagged real source quotes as VISUAL. Concrete case from a make-query run: 'Cloud Strife, an unsociable mercenary who claims to be a former 1st Class member of Shinra's SOLDIER unit;' is verbatim in the Final_Fantasy_VII article wikitext (modulo markup). With the strip that quote now verifies; the genuine model hallucinations in the same answer still flag honestly. Side-effect: 43% smaller context size on average so less LLM token waste. BASE_VERSION = 'wikitext-base-v1' is the algorithm pin. Bump when the strip rules change. Soft-imported in verify.py so environments without mwparserfromhell installed degrade gracefully (no strip, same behavior as before this commit). Build: - pyproject.toml: new [wikitext] extras (mwparserfromhell>=0.6), pulled in by [dev] - Makefile: chain-check / chain-check-shards targets — fast audit-chain integrity probe, counts dangling prev_event_hash references; 0 = chain intact Tests: - tests/test_wikitext.py: 31 tests (rules, idempotence, real-corpus fixture) - tests/test_verify.py: 2 regression tests pinning the FF7 flip (Cloud Strife quote: VISUAL → STRICT after strip; genuine hallucination: stays VISUAL) - tests/fixtures/ff7_characters_chunk0.wikitext: real chunk from a shard, used to validate the strip on actual Wikipedia content
41 lines
794 B
TOML
41 lines
794 B
TOML
[build-system]
|
|
requires = ["setuptools>=68"]
|
|
build-backend = "setuptools.build_meta"
|
|
|
|
[project]
|
|
name = "aborist"
|
|
version = "0.0.1"
|
|
description = "An arborist for trees and forests of cross-linked information"
|
|
readme = "README.md"
|
|
license = { text = "AGPL-3.0-only" }
|
|
requires-python = ">=3.10"
|
|
authors = [
|
|
{ name = "Russell Ballestrini", email = "russell@unturf.com" },
|
|
{ name = "foxhop" },
|
|
{ name = "TimeHexOn" },
|
|
]
|
|
dependencies = [
|
|
"httpx>=0.27",
|
|
"zstandard>=0.22",
|
|
"cryptography>=42",
|
|
]
|
|
|
|
[project.optional-dependencies]
|
|
html = [
|
|
"selectolax>=0.3",
|
|
]
|
|
wikitext = [
|
|
"mwparserfromhell>=0.6",
|
|
]
|
|
dev = [
|
|
"pytest>=8",
|
|
"aborist[html]",
|
|
"aborist[wikitext]",
|
|
]
|
|
|
|
[project.scripts]
|
|
aborist = "aborist.cli:main"
|
|
|
|
[tool.setuptools.packages.find]
|
|
where = ["."]
|
|
include = ["aborist*"]
|