russell@unturf.com
7f7eeefeb9
- make crawl-ingest writes to one central crawl db (CRAWL_DB, default ~/.arborist/crawl/web.db) instead of per-domain shards in the peer-shared main dir: keeps locally-crawled content out of peer sharing by default and a growing domain set under SQLite's 10-attach cap (Makefile, docs/crawler.md). - arborist query auto-includes the local crawl db (query() gains extra_shards; CLI --include-shard / --no-crawl-db, default-on when web.db exists). Fix latent --db single-file query AttributeError (cli.py). Persist used / used_pointer_ids + retrieval_purity into merkle_proof so read-only consumers can see which chunks fed the answer (qa/query.py). - arborist.read: read-only seam for dashboards / verifiers; on a multi-source context root surface the real primary source instead of the opaque corpus://multi-source sentinel (read.py). Backs the arborist-viz Merkle Command Center (#000069). - tests for extra_shards, the CLI crawl-db resolver, and the read seam.
422 lines
15 KiB
Python
422 lines
15 KiB
Python
"""Pretty-print + ensure_ascii=False on `arborist query` output.
|
|
|
|
The CLI default emits a human-readable render of the query result;
|
|
``--json`` switches to the raw record. Both paths must:
|
|
- render unicode literals (no \\uXXXX escape sequences)
|
|
- present the answer prominently; metadata is supporting info
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from arborist.cli import _render_query_human
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _render_query_human — direct unit tests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _result(**overrides) -> dict:
|
|
base = {
|
|
"status": "cache_miss_then_written",
|
|
"audit_mode": "HYBRID",
|
|
"cache_key": "35ab7d3355c723b759145d6d446cb1d9010abe0dce9992ca0e8b87837b596610",
|
|
"context_root": "ab" * 32,
|
|
"answer_text": "Pikachu is a species of Pokémon creatures.",
|
|
"sources": [
|
|
{
|
|
"document_root": "cd" * 32,
|
|
"document_uri": "https://en.wikipedia.org/wiki/Pikachu",
|
|
"title": "Pikachu",
|
|
"score": 36.6,
|
|
"chunk_idx": 6,
|
|
"shard": "002.db",
|
|
}
|
|
],
|
|
"n_quotes": 2,
|
|
"n_verified": 1,
|
|
"verifier_method": "quote",
|
|
"unverified_quotes": ["something the model said but didn't ground"],
|
|
"timings": {"total_ms": 9234.1},
|
|
}
|
|
base.update(overrides)
|
|
return base
|
|
|
|
|
|
def test_render_includes_question_at_top():
|
|
out = _render_query_human(_result(), "who is pikachu?")
|
|
assert out.splitlines()[0] == "who is pikachu?"
|
|
|
|
|
|
def test_render_summary_line_has_audit_n_verified_method_elapsed():
|
|
"""Summary line carries audit_label · n_verified/n_quotes · elapsed.
|
|
The label format updated 2026-05-01 to honesty-spell verifier
|
|
method (e.g. "EVIDENCE-LINKED · via claim_lattice" instead of
|
|
"STRICT") so user-facing output doesn't overclaim semantic truth.
|
|
Quote / span / entity / paraphrase modes keep the original
|
|
audit_mode token plus a "via {method}" tail."""
|
|
out = _render_query_human(_result(), "q")
|
|
second = out.splitlines()[1]
|
|
# Quote-mode keeps the audit_mode token (HYBRID) — it's the
|
|
# claim_lattice modes that get the EVIDENCE-LINKED relabel.
|
|
assert "HYBRID" in second
|
|
# Counts and method still in the line; "verified" word dropped
|
|
# to give the slimmer "1/2" form room for the longer label tail.
|
|
assert "1/2" in second
|
|
assert "via quote" in second
|
|
assert "9.2s" in second
|
|
|
|
|
|
def test_render_marks_cache_hit_distinctly():
|
|
out_hit = _render_query_human(_result(status="cache_hit"), "q")
|
|
out_fresh = _render_query_human(_result(status="cache_miss_then_written"), "q")
|
|
assert "(cached)" in out_hit
|
|
assert "(fresh)" in out_fresh
|
|
|
|
|
|
def test_render_emits_unicode_literals_not_escapes():
|
|
"""Pokémon, not Pok\\u00e9mon — the user's terminal should see real é."""
|
|
out = _render_query_human(_result(), "who is pikachu?")
|
|
assert "Pokémon" in out
|
|
assert "\\u00e9" not in out
|
|
|
|
|
|
def test_render_lists_sources_with_clean_host_path():
|
|
out = _render_query_human(_result(), "q")
|
|
# "[1] Pikachu — en.wikipedia.org/wiki/Pikachu (002.db)"
|
|
assert "[1] Pikachu" in out
|
|
assert "en.wikipedia.org/wiki/Pikachu" in out
|
|
# scheme stripped
|
|
assert "https://" not in out.split("sources (")[1] if "sources (" in out else True
|
|
assert "(002.db)" in out
|
|
|
|
|
|
def test_render_truncates_long_unverified_quotes():
|
|
long = "x" * 200
|
|
out = _render_query_human(_result(unverified_quotes=[long]), "q")
|
|
# Truncated form ends in ellipsis
|
|
assert "..." in out
|
|
# Original doesn't fully appear
|
|
assert long not in out
|
|
|
|
|
|
def test_render_omits_unverified_section_when_empty():
|
|
out = _render_query_human(_result(unverified_quotes=[]), "q")
|
|
assert "unverified" not in out.lower()
|
|
|
|
|
|
def test_render_omits_sources_section_when_empty():
|
|
out = _render_query_human(_result(sources=[]), "q")
|
|
assert "sources (" not in out
|
|
|
|
|
|
def test_render_falls_back_for_error_status():
|
|
"""no_sources / unknown_document / etc. produce a one-line status."""
|
|
out = _render_query_human(
|
|
{"status": "no_sources", "msg": "FTS5 returned no hits"},
|
|
"q",
|
|
)
|
|
assert "no_sources" in out
|
|
assert "FTS5" in out
|
|
|
|
|
|
def test_render_includes_short_cache_key_with_pointer_to_json():
|
|
out = _render_query_human(_result(), "q")
|
|
# Short prefix only — full key requires --json
|
|
assert "cache_key: 35ab7d33" in out
|
|
assert "--json" in out
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# json.dumps everywhere uses ensure_ascii=False (regression on the global pass)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_unicode_round_trips_through_print_path(capsys):
|
|
"""Pin: a result with a Pokémon-style answer prints é, not \\u00e9.
|
|
Uses argparse via build_parser to exercise the real CLI dispatch."""
|
|
from arborist.cli import build_parser
|
|
|
|
parser = build_parser()
|
|
# Build a fake result and call _cmd_query's render path indirectly.
|
|
out = _render_query_human(_result(), "who is pikachu?")
|
|
print(out)
|
|
captured = capsys.readouterr().out
|
|
assert "Pokémon" in captured
|
|
assert "\\u00e9" not in captured
|
|
|
|
|
|
def test_json_mode_also_uses_unicode():
|
|
"""The --json path should also emit unicode literals, not escapes."""
|
|
payload = {"answer": "Pokémon"}
|
|
s = json.dumps(payload, ensure_ascii=False)
|
|
assert "Pokémon" in s
|
|
assert "\\u00e9" not in s
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# capacity metrics — human render surfaces prompt_chars breakdown
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_render_shows_capacity_line_when_prompt_chars_present():
|
|
"""When the result carries `prompt_chars`, the human render
|
|
emits a one-liner summarizing prompt size + answer size so an
|
|
operator can tell at a glance whether STRICT came from a small
|
|
prompt or a context-stuffed one."""
|
|
r = _result(
|
|
prompt_chars={
|
|
"system_prompt": 1292,
|
|
"grounding_reminder": 342,
|
|
"user_question": 30,
|
|
"evidence_or_context": 20304,
|
|
"messages_total": 21996,
|
|
},
|
|
answer_chars=247,
|
|
)
|
|
out = _render_query_human(r, "what is the capital of france?")
|
|
assert "capacity:" in out
|
|
assert "21,996 chars" in out # messages_total
|
|
assert "sys 1,292" in out
|
|
assert "evidence 20,304" in out
|
|
assert "answer 247 chars" in out
|
|
|
|
|
|
def test_render_omits_capacity_line_on_legacy_results_without_prompt_chars():
|
|
"""Backwards-compat: results that pre-date capacity metrics (no
|
|
prompt_chars key) render without the capacity line — no KeyError,
|
|
no awkward 'capacity: 0 chars' noise."""
|
|
r = _result() # no prompt_chars
|
|
out = _render_query_human(r, "q")
|
|
assert "capacity:" not in out
|
|
|
|
|
|
def test_render_capacity_thousand_separators():
|
|
"""Large prompts must format with thousand-separator commas so
|
|
a 60K prompt reads as '60,000' not '60000' — operator legibility
|
|
on the daily render."""
|
|
r = _result(
|
|
prompt_chars={
|
|
"system_prompt": 1500,
|
|
"grounding_reminder": 0,
|
|
"user_question": 50,
|
|
"evidence_or_context": 60000,
|
|
"messages_total": 61550,
|
|
},
|
|
answer_chars=1234,
|
|
)
|
|
out = _render_query_human(r, "q")
|
|
assert "61,550 chars" in out
|
|
assert "60,000" in out
|
|
assert "1,234 chars" in out
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# label discipline — claim_lattice* modes get EVIDENCE-LINKED relabel
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_render_label_strict_lattice_no_violations_is_evidence_warranted():
|
|
"""claim-lattice STRICT with no soft demotes lands at the top
|
|
ladder rung (EVIDENCE-WARRANTED) — pointer verified, warrant
|
|
passed where it ran, no other demotes pulled the verdict back."""
|
|
r = _result(
|
|
audit_mode="STRICT",
|
|
verifier_method="claim_lattice",
|
|
n_quotes=1, n_verified=1,
|
|
violations=[],
|
|
)
|
|
out = _render_query_human(r, "q")
|
|
assert "EVIDENCE-WARRANTED" in out
|
|
assert "via claim_lattice" in out
|
|
# The bare STRICT token should NOT appear on the summary line.
|
|
summary_line = out.splitlines()[1]
|
|
assert "STRICT" not in summary_line
|
|
|
|
|
|
def test_render_label_strict_lattice_warrant_missing_is_pointer_linked():
|
|
"""claim-lattice STRICT (or HYBRID) with WARRANT_MISSING in
|
|
violations drops to POINTER-LINKED — the pointer resolved but
|
|
the cited evidence didn't anchor the claim."""
|
|
r = _result(
|
|
audit_mode="HYBRID",
|
|
verifier_method="claim_lattice_pointer",
|
|
n_quotes=2, n_verified=1,
|
|
violations=[{"kind": "WARRANT_MISSING", "claim_idx": 0}],
|
|
)
|
|
out = _render_query_human(r, "q")
|
|
assert "POINTER-LINKED-PARTIAL" in out
|
|
assert "via claim_lattice_pointer" in out
|
|
|
|
|
|
def test_render_label_strict_lattice_soft_demote_is_anchor_warranted():
|
|
"""claim-lattice STRICT/HYBRID with a soft-demote violation
|
|
(LAZY_ANCHOR_DEMOTED, POINTER_OVERFLOW_TRIMMED, TOO_MANY_CLAIMS)
|
|
but no WARRANT_MISSING lands at ANCHOR-WARRANTED — warrant
|
|
passed where it ran, but other demotes apply."""
|
|
r = _result(
|
|
audit_mode="STRICT",
|
|
verifier_method="claim_lattice",
|
|
n_quotes=3, n_verified=3,
|
|
violations=[{"kind": "LAZY_ANCHOR_DEMOTED"}],
|
|
)
|
|
out = _render_query_human(r, "q")
|
|
assert "ANCHOR-WARRANTED" in out
|
|
assert "via claim_lattice" in out
|
|
|
|
|
|
def test_render_label_hybrid_lattice_evidence_warranted_partial():
|
|
"""HYBRID + no violations + lattice mode → EVIDENCE-WARRANTED-PARTIAL.
|
|
The -PARTIAL suffix shows the audit didn't reach STRICT but the
|
|
claim that did verify reached the top rung."""
|
|
r = _result(
|
|
audit_mode="HYBRID",
|
|
verifier_method="claim_lattice",
|
|
n_quotes=3, n_verified=1,
|
|
violations=[],
|
|
)
|
|
out = _render_query_human(r, "q")
|
|
assert "EVIDENCE-WARRANTED-PARTIAL" in out
|
|
|
|
|
|
def test_render_label_ungrounded_in_claim_lattice_stays_ungrounded():
|
|
"""UNGROUNDED already names what it means — no rung-relabel."""
|
|
r = _result(
|
|
audit_mode="UNGROUNDED",
|
|
verifier_method="claim_lattice",
|
|
n_quotes=2, n_verified=0,
|
|
violations=[],
|
|
)
|
|
out = _render_query_human(r, "q")
|
|
assert "UNGROUNDED" in out
|
|
assert "via claim_lattice" in out
|
|
|
|
|
|
def test_render_shows_reference_frame_notes_when_detected():
|
|
"""When frame_detection.kind == 'reference', the renderer
|
|
surfaces the named reference work so an operator can see the
|
|
substrate routed to a fictional / reference source."""
|
|
r = _result(
|
|
audit_mode="STRICT",
|
|
verifier_method="claim_lattice",
|
|
frame_detection={
|
|
"kind": "reference",
|
|
"reference_title": "Nineteen Eighty-Four",
|
|
"reference_uri": "https://en.wikipedia.org/wiki/Nineteen_Eighty-Four",
|
|
"confidence": 0.8,
|
|
},
|
|
)
|
|
out = _render_query_human(r, "has oceania always been at war with east asia")
|
|
assert "reference frame: Nineteen Eighty-Four" in out
|
|
|
|
|
|
def test_render_omits_reference_frame_notes_for_literal_query():
|
|
"""Literal-frame queries (no allusion) don't render the frame
|
|
notes line — keeps the daily render clean for the common case."""
|
|
r = _result(
|
|
audit_mode="STRICT",
|
|
verifier_method="claim_lattice",
|
|
frame_detection={
|
|
"kind": "no_phrase_route",
|
|
"reference_title": None,
|
|
"reference_uri": None,
|
|
"confidence": 0.0,
|
|
},
|
|
)
|
|
out = _render_query_human(r, "what is the capital of france?")
|
|
assert "reference frame:" not in out
|
|
|
|
|
|
def test_render_omits_reference_frame_notes_for_ambiguous():
|
|
"""Ambiguous frame (phrase route fired but no clear reference
|
|
work) → no frame notes line. The signal isn't strong enough
|
|
to claim a named reference."""
|
|
r = _result(
|
|
audit_mode="STRICT",
|
|
verifier_method="claim_lattice",
|
|
frame_detection={
|
|
"kind": "ambiguous",
|
|
"reference_title": None,
|
|
"reference_uri": None,
|
|
"confidence": 0.4,
|
|
},
|
|
)
|
|
out = _render_query_human(r, "some allusion-shape query")
|
|
assert "reference frame:" not in out
|
|
|
|
|
|
def test_render_label_quote_mode_keeps_audit_mode_token():
|
|
"""Quote / span / entity / paraphrase modes verify against
|
|
pinned spans, not synthesis. STRICT in quote mode IS a strong
|
|
claim about evidence units — keep the label as-is."""
|
|
r = _result(
|
|
audit_mode="STRICT",
|
|
verifier_method="quote",
|
|
n_quotes=2, n_verified=2,
|
|
)
|
|
out = _render_query_human(r, "q")
|
|
assert "STRICT" in out
|
|
assert "via quote" in out
|
|
# Don't relabel quote-mode STRICT to either ladder rung.
|
|
assert "EVIDENCE-WARRANTED" not in out
|
|
assert "POINTER-LINKED" not in out
|
|
assert "ANCHOR-WARRANTED" not in out
|
|
|
|
|
|
def test_render_label_format_collapsed_surfaces_tail():
|
|
"""FORMAT_COLLAPSED soft-demotes STRICT → HYBRID and the renderer
|
|
surfaces a `· format collapsed` tail so an operator can tell from
|
|
the audit line whether UNGROUNDED meant 'tried & failed' vs
|
|
'abandoned the protocol entirely.' Pairs with WARRANT_MISSING /
|
|
TITLE_MISMATCH tails — multiple can coexist."""
|
|
r = _result(
|
|
audit_mode="UNGROUNDED",
|
|
verifier_method="claim_lattice_pointer",
|
|
n_quotes=2, n_verified=0,
|
|
violations=[{"kind": "FORMAT_COLLAPSED", "meaningful_lines": 50,
|
|
"bracket_count": 0}],
|
|
)
|
|
out = _render_query_human(r, "winners of all major sports?")
|
|
assert "format collapsed" in out
|
|
assert "UNGROUNDED" in out
|
|
|
|
|
|
# --- auto-include of the local crawl db -----------------------------
|
|
|
|
|
|
def test_resolve_extra_shards_auto_includes_crawl_db(tmp_path, monkeypatch):
|
|
"""`arborist query` auto-includes the local crawl db (when present),
|
|
honors --no-crawl-db, and appends --include-shard extras de-duped."""
|
|
import argparse
|
|
|
|
from arborist.cli import _default_crawl_db, _resolve_extra_shards
|
|
|
|
crawl_db = tmp_path / "web.db"
|
|
crawl_db.write_bytes(b"") # resolver only checks presence
|
|
monkeypatch.setenv("ARBORIST_CRAWL_DB", str(crawl_db))
|
|
assert _default_crawl_db() == crawl_db
|
|
|
|
# present → auto-included
|
|
ns = argparse.Namespace(no_crawl_db=False, include_shard=None)
|
|
assert _resolve_extra_shards(ns) == [crawl_db]
|
|
|
|
# --no-crawl-db opts out
|
|
ns_off = argparse.Namespace(no_crawl_db=True, include_shard=None)
|
|
assert _resolve_extra_shards(ns_off) == []
|
|
|
|
# explicit --include-shard appends extras; crawl db deduped
|
|
extra = tmp_path / "extra.db"
|
|
extra.write_bytes(b"")
|
|
ns_inc = argparse.Namespace(
|
|
no_crawl_db=False, include_shard=[str(extra), str(crawl_db)]
|
|
)
|
|
assert _resolve_extra_shards(ns_inc) == [crawl_db, extra]
|
|
|
|
# missing crawl db → silently dropped
|
|
monkeypatch.setenv("ARBORIST_CRAWL_DB", str(tmp_path / "nope.db"))
|
|
ns_missing = argparse.Namespace(no_crawl_db=False, include_shard=None)
|
|
assert _resolve_extra_shards(ns_missing) == []
|