Three small streams in one commit; each closes / expands a
recently-landed ticket without changing its hard contract.
#000026 Phase 3 wiring — authorship warrant ladder visible
============================================================
Phase 3 sidecar (arborist/qa/warrant_authorship.py landed in 60b5748)
exposed the classifier but didn't surface it. Two wirings:
- arborist/qa/inspect.py — diagnose_authorship_warrant runs against
the cached row's question + answer + per-source raw chunks +
URIs + titles; result lands as `authorship` field alongside the
other sidecars.
- arborist/cli.py _render_warrant_tail — appends ` · warrant:
<readable-tier>` when result['authorship'] is populated with a
non-quiet tier. AUTHOR_COPYRIGHT_FOOTER → "copyright-footer", etc.
NO_AUTHORSHIP_SIGNAL stays silent. Backward-compat: results
without an `authorship` key render unchanged.
Tests: 3 inspect-path tests (no-signal, copyright-footer,
repository-owner) + 4 render-tail tests (presence, no-signal
silence, missing-key silence, all-six-tiers readable mapping).
#000028 follow-ups — capital ledger + sample-rate
==================================================
Two policy fields layered on top of canonical_witness_enabled:
- canonical_witness_sample_rate (0.0..1.0; default 1.0). Operators
wanting passive calibration set 0.05 to fire witness on 5% of
canonical questions while paying 5% of LLM cost. 0.0 effectively
off; 1.0 = current always-on behavior. Gating uses random.random()
so distribution is uniform; clamped to [0, 1].
- Capital ledger row written for each FIRED witness (not skipped
ones). op_type='canonical_witness'; estimator inputs include
prompt_chars + answer_chars + llm_seconds + agreement_label +
pi_star_ref. Best-effort: ledger-write failure must never fail
the query (sidecar discipline).
Tests: 4 new — sample_rate=0.0 skips (no LLM call, no ledger row);
sample_rate=1.0 always fires; capital_ledger row written under
op_type='canonical_witness' with full input blob; sampled-out
witness records zero ledger rows.
Both fields fold into governance_policy_hash naturally via the
existing policy-hash machinery — flipping witness mode invalidates
prior records as expected.
#000025 Phase 1d — 5F fixture catalog 30 → 50
==============================================
Both synthetic and live sides of all 5 sub-batteries expanded
30 → 50 (+200 fixtures total: 5 × 20 synthetic, 5 × 20 live).
function — claim_count cycles 2..7 across new fixtures
falsification — 10-violation palette across new ids
feedback-loop — fact-N learning chains
finetuning — capability transitions across canonical π*
(math/logic/algebra/calculus pool)
formulate — multi-pointer claim shapes
500/500 pass through respective runners. test_session_integration
total bumped 562 → 662. Pinned test_5f_*_runs counts updated 30 →
50 (synthetic main + embedded + live).
Tests
=====
Full suite: 1467 passed, 36 skipped (was 1388; +79 across warrant
render + witness sample/ledger + 5F implicit coverage).
335 lines
11 KiB
Python
335 lines
11 KiB
Python
"""Tests for the authorship warrant ladder (#000026 Phase 3)."""
|
||
|
||
from __future__ import annotations
|
||
|
||
from arborist.qa.warrant_authorship import (
|
||
NO_AUTHORSHIP_SIGNAL,
|
||
TIER_RANK,
|
||
diagnose_authorship_warrant,
|
||
)
|
||
|
||
|
||
# ---------- Question gating ------------------------------------------------
|
||
|
||
|
||
def test_non_authorship_question_returns_no_signal():
|
||
"""Sidecar should stay quiet on irrelevant questions."""
|
||
out = diagnose_authorship_warrant(
|
||
question_text="what is the capital of france?",
|
||
answer_text="Paris.",
|
||
cited_evidence_spans=["© 2024 Wikipedia Foundation"],
|
||
)
|
||
assert out["tier"] == NO_AUTHORSHIP_SIGNAL
|
||
assert out["tier_rank"] == 99
|
||
|
||
|
||
def test_no_evidence_no_question_returns_no_signal():
|
||
out = diagnose_authorship_warrant()
|
||
assert out["tier"] == NO_AUTHORSHIP_SIGNAL
|
||
|
||
|
||
def test_no_evidence_with_authorship_question_returns_no_signal():
|
||
"""Authorship question but zero cited evidence — sidecar stays
|
||
quiet rather than reporting tier-6 secondary on no evidence."""
|
||
out = diagnose_authorship_warrant(
|
||
question_text="who wrote virt-back?",
|
||
answer_text="Russell Ballestrini.",
|
||
)
|
||
assert out["tier"] == NO_AUTHORSHIP_SIGNAL
|
||
|
||
|
||
# ---------- Tier 1 — package metadata --------------------------------------
|
||
|
||
|
||
def test_tier1_python_pyproject_author():
|
||
out = diagnose_authorship_warrant(
|
||
question_text="who wrote arborist?",
|
||
answer_text="Russell Ballestrini wrote arborist.",
|
||
cited_evidence_spans=[
|
||
'authors = [\n { name = "Russell Ballestrini" },\n]'
|
||
],
|
||
)
|
||
assert out["tier"] == "AUTHOR_PACKAGE_METADATA"
|
||
assert out["tier_rank"] == 1
|
||
assert "Russell Ballestrini" in out["candidate_names"]
|
||
|
||
|
||
def test_tier1_python_setup_py_author():
|
||
out = diagnose_authorship_warrant(
|
||
question_text="who is the author of virt-back?",
|
||
answer_text="Russell Ballestrini.",
|
||
cited_evidence_spans=['author="Russell Ballestrini"'],
|
||
)
|
||
assert out["tier"] == "AUTHOR_PACKAGE_METADATA"
|
||
|
||
|
||
def test_tier1_package_json_author():
|
||
out = diagnose_authorship_warrant(
|
||
question_text="who created this library?",
|
||
answer_text="Foo Bar.",
|
||
cited_evidence_spans=['{"name":"my-pkg","author":"Foo Bar"}'],
|
||
)
|
||
assert out["tier"] == "AUTHOR_PACKAGE_METADATA"
|
||
|
||
|
||
# ---------- Tier 2 — repository owner --------------------------------------
|
||
|
||
|
||
def test_tier2_github_uri_owner():
|
||
out = diagnose_authorship_warrant(
|
||
question_text="who maintains this?",
|
||
answer_text="The repository is owned by russellballestrini.",
|
||
cited_source_uris=["https://github.com/russellballestrini/virt-back"],
|
||
cited_evidence_spans=["A README about the project."],
|
||
)
|
||
assert out["tier"] == "AUTHOR_REPOSITORY_OWNER"
|
||
assert "russellballestrini" in out["candidate_names"]
|
||
|
||
|
||
def test_tier2_gitlab_uri_owner():
|
||
out = diagnose_authorship_warrant(
|
||
question_text="who owns this project?",
|
||
answer_text="The team-arborist group.",
|
||
cited_source_uris=[
|
||
"https://gitlab.com/team-arborist/some-project",
|
||
],
|
||
cited_evidence_spans=["x"],
|
||
)
|
||
assert out["tier"] == "AUTHOR_REPOSITORY_OWNER"
|
||
|
||
|
||
# ---------- Tier 3 — page byline -------------------------------------------
|
||
|
||
|
||
def test_tier3_byline_prose_by_pattern():
|
||
out = diagnose_authorship_warrant(
|
||
question_text="who wrote this article?",
|
||
answer_text="Russell Ballestrini.",
|
||
cited_evidence_spans=["By Russell Ballestrini, posted 2024-01-01."],
|
||
)
|
||
assert out["tier"] == "AUTHOR_PAGE_BYLINE"
|
||
|
||
|
||
def test_tier3_byline_meta_html():
|
||
out = diagnose_authorship_warrant(
|
||
question_text="who is the author of this page?",
|
||
answer_text="Russell Ballestrini.",
|
||
cited_evidence_spans=[
|
||
'<meta name="author" content="Russell Ballestrini">'
|
||
],
|
||
)
|
||
assert out["tier"] == "AUTHOR_PAGE_BYLINE"
|
||
|
||
|
||
# ---------- Tier 4 — primary page title ------------------------------------
|
||
|
||
|
||
def test_tier4_primary_page_title_match():
|
||
"""Site host token appears in the cited title AND the answer
|
||
references the same entity. The cited evidence is the entity's
|
||
own primary page."""
|
||
out = diagnose_authorship_warrant(
|
||
question_text="who wrote virt-back?",
|
||
answer_text="russellballestrini wrote virt-back.",
|
||
cited_source_uris=[
|
||
"https://russellballestrini.net/virt-back-restoring-from-backups/",
|
||
],
|
||
cited_source_titles=["virt-back: restoring from backups – Russell Ballestrini"],
|
||
cited_evidence_spans=["A blog post about virt-back."],
|
||
)
|
||
# Should fire at tier 4 absent stronger signals (no package
|
||
# metadata, no repo URL, no byline, no copyright).
|
||
assert out["tier"] == "AUTHOR_PRIMARY_PAGE_TITLE"
|
||
|
||
|
||
def test_tier4_third_party_host_does_not_fire():
|
||
"""Wikipedia is a third-party indexer; primary-page heuristic
|
||
must not fire even if tokens overlap."""
|
||
out = diagnose_authorship_warrant(
|
||
question_text="who wrote virt-back?",
|
||
answer_text="russell wrote virt-back.",
|
||
cited_source_uris=[
|
||
"https://en.wikipedia.org/wiki/virt-back-russell"
|
||
],
|
||
cited_source_titles=["virt-back russell"],
|
||
cited_evidence_spans=["x"],
|
||
)
|
||
# With NO other signals firing, falls through to secondary.
|
||
assert out["tier"] == "AUTHOR_SECONDARY_SOURCE"
|
||
|
||
|
||
# ---------- Tier 5 — copyright footer --------------------------------------
|
||
|
||
|
||
def test_tier5_copyright_footer_with_year():
|
||
out = diagnose_authorship_warrant(
|
||
question_text="who wrote virt-back?",
|
||
answer_text="Russell Ballestrini wrote virt-back.",
|
||
cited_evidence_spans=["© 2024 Russell Ballestrini"],
|
||
)
|
||
assert out["tier"] == "AUTHOR_COPYRIGHT_FOOTER"
|
||
|
||
|
||
def test_tier5_copyright_footer_no_year():
|
||
out = diagnose_authorship_warrant(
|
||
question_text="who wrote virt-back?",
|
||
answer_text="Russell Ballestrini.",
|
||
cited_evidence_spans=["© Russell Ballestrini"],
|
||
)
|
||
assert out["tier"] == "AUTHOR_COPYRIGHT_FOOTER"
|
||
|
||
|
||
def test_tier5_copyright_word_form():
|
||
out = diagnose_authorship_warrant(
|
||
question_text="who created this site?",
|
||
answer_text="Russell Ballestrini.",
|
||
cited_evidence_spans=["Copyright 2024 Russell Ballestrini. All rights reserved."],
|
||
)
|
||
assert out["tier"] == "AUTHOR_COPYRIGHT_FOOTER"
|
||
|
||
|
||
# ---------- Tier 6 — secondary source --------------------------------------
|
||
|
||
|
||
def test_tier6_third_party_evidence_no_authorship_markers():
|
||
"""Question is authorship; cited evidence has none of the direct
|
||
markers — falls through to secondary."""
|
||
out = diagnose_authorship_warrant(
|
||
question_text="who wrote virt-back?",
|
||
answer_text="Russell Ballestrini, according to the documentation.",
|
||
cited_evidence_spans=[
|
||
"virt-back is a tool for backing up libvirt VMs."
|
||
],
|
||
)
|
||
assert out["tier"] == "AUTHOR_SECONDARY_SOURCE"
|
||
assert out["tier_rank"] == TIER_RANK["AUTHOR_SECONDARY_SOURCE"]
|
||
|
||
|
||
# ---------- Tier ordering: strongest wins ----------------------------------
|
||
|
||
|
||
def test_strongest_tier_wins_when_multiple_fire():
|
||
"""Both copyright AND package-metadata fire — the package-metadata
|
||
signal is strictly stronger (tier 1 < tier 5)."""
|
||
out = diagnose_authorship_warrant(
|
||
question_text="who wrote virt-back?",
|
||
answer_text="Russell Ballestrini.",
|
||
cited_evidence_spans=[
|
||
"© Russell Ballestrini",
|
||
'author = "Russell Ballestrini"',
|
||
],
|
||
)
|
||
assert out["tier"] == "AUTHOR_PACKAGE_METADATA"
|
||
# But signals list captures BOTH for the operator.
|
||
tiers_seen = {s["tier"] for s in out["signals"]}
|
||
assert "AUTHOR_PACKAGE_METADATA" in tiers_seen
|
||
assert "AUTHOR_COPYRIGHT_FOOTER" in tiers_seen
|
||
|
||
|
||
# ---------- Sidecar contract -----------------------------------------------
|
||
|
||
|
||
def test_returns_dict_always():
|
||
"""Sidecar contract: never raises, always returns a dict."""
|
||
out = diagnose_authorship_warrant(
|
||
question_text=None,
|
||
answer_text="",
|
||
cited_evidence_spans=None,
|
||
cited_source_uris=None,
|
||
cited_source_titles=None,
|
||
)
|
||
assert isinstance(out, dict)
|
||
assert "tier" in out
|
||
assert "tier_rank" in out
|
||
|
||
|
||
def test_does_not_raise_on_garbage_input():
|
||
out = diagnose_authorship_warrant(
|
||
question_text="who wrote x?",
|
||
answer_text="\x00\x01" * 100,
|
||
cited_evidence_spans=["", None, "<<<>>>"], # type: ignore
|
||
)
|
||
assert out["tier"] in (
|
||
"NO_AUTHORSHIP_SIGNAL", "AUTHOR_SECONDARY_SOURCE",
|
||
"AUTHOR_PACKAGE_METADATA", "AUTHOR_REPOSITORY_OWNER",
|
||
"AUTHOR_PAGE_BYLINE", "AUTHOR_PRIMARY_PAGE_TITLE",
|
||
"AUTHOR_COPYRIGHT_FOOTER",
|
||
)
|
||
|
||
|
||
def test_noise_capitalized_words_not_classified_as_names():
|
||
"""`Author` / `Copyright` / etc. shouldn't slip through as names."""
|
||
out = diagnose_authorship_warrant(
|
||
question_text="who wrote x?",
|
||
answer_text="x",
|
||
cited_evidence_spans=[
|
||
"Copyright Reserved", # should NOT classify "Reserved" as name
|
||
"Author All Rights", # garbage shape
|
||
],
|
||
)
|
||
# Even if regex matches, _looks_like_noise filters; tier may fall
|
||
# to secondary or no-signal.
|
||
assert "Reserved" not in out.get("candidate_names", [])
|
||
assert "All" not in out.get("candidate_names", [])
|
||
|
||
|
||
# ---------- Render-tail integration (#000026 Phase 3 wiring) ---------------
|
||
|
||
|
||
def test_render_tail_emits_warrant_when_authorship_set():
|
||
"""`_render_warrant_tail` adds ` · warrant: <readable-tier>` when
|
||
result['authorship'] is populated with a non-quiet tier."""
|
||
from arborist.cli import _render_warrant_tail
|
||
result = {
|
||
"violations": [],
|
||
"authorship": {
|
||
"tier": "AUTHOR_COPYRIGHT_FOOTER",
|
||
"tier_rank": 5,
|
||
"signals": [],
|
||
"candidate_names": ["Russell Ballestrini"],
|
||
},
|
||
}
|
||
tail = _render_warrant_tail(result)
|
||
assert "warrant: copyright-footer" in tail
|
||
|
||
|
||
def test_render_tail_silent_for_no_authorship_signal():
|
||
"""Sidecar's quiet verdict (NO_AUTHORSHIP_SIGNAL) → no tail."""
|
||
from arborist.cli import _render_warrant_tail
|
||
result = {
|
||
"violations": [],
|
||
"authorship": {
|
||
"tier": "NO_AUTHORSHIP_SIGNAL",
|
||
"tier_rank": 99,
|
||
},
|
||
}
|
||
tail = _render_warrant_tail(result)
|
||
assert "warrant" not in tail
|
||
|
||
|
||
def test_render_tail_silent_when_no_authorship_field():
|
||
"""Backward-compat: results without an `authorship` key render
|
||
unchanged (the tail-builder must not raise / must not emit)."""
|
||
from arborist.cli import _render_warrant_tail
|
||
result = {"violations": []}
|
||
tail = _render_warrant_tail(result)
|
||
assert "warrant" not in tail
|
||
|
||
|
||
def test_render_tail_warrant_uses_readable_token_per_tier():
|
||
"""Each tier renders with hyphen-lowercase and AUTHOR_ stripped."""
|
||
from arborist.cli import _render_warrant_tail
|
||
expectations = {
|
||
"AUTHOR_PACKAGE_METADATA": "warrant: package-metadata",
|
||
"AUTHOR_REPOSITORY_OWNER": "warrant: repository-owner",
|
||
"AUTHOR_PAGE_BYLINE": "warrant: page-byline",
|
||
"AUTHOR_PRIMARY_PAGE_TITLE": "warrant: primary-page-title",
|
||
"AUTHOR_COPYRIGHT_FOOTER": "warrant: copyright-footer",
|
||
"AUTHOR_SECONDARY_SOURCE": "warrant: secondary-source",
|
||
}
|
||
for tier, expected in expectations.items():
|
||
tail = _render_warrant_tail({
|
||
"violations": [],
|
||
"authorship": {"tier": tier, "tier_rank": 1},
|
||
})
|
||
assert expected in tail, f"{tier} → tail={tail!r}"
|