russell@unturf.com
70ffc01ce4
Three small streams in one commit:
#000028 follow-up — witness divergence → 5F fixtures
=====================================================
Witness fan-out now writes a `providence_canonical_witness` audit
event when it fires (next to the capital-ledger record landed in
708aa45). Body carries pi_star_ref, question_text, agreement_label,
canonical_answer_text, llm_raw_text, llm_canonical_bytes,
cache_status. Best-effort write — chain failure never fails the
query.
New extractor `bench/scripts/witness_to_5f.py` reads those events
from a qa.db and writes them out as 5F-Falsification fixtures
matching the existing `falsification-live-v1` schema. Filtering
includes only divergence labels (LLM-DIVERGED / KERNEL-LLM-DIVERGED
/ CACHE-DRIFT); skips KERNEL-LLM-AGREE / STRICT-WITNESSED (no
calibration signal) and KERNEL-ONLY (LLM unparseable, not a
supervised-correction sample).
Idempotent: sorted by audit-event seq, so re-running against the
same qa.db produces byte-equal fixture files. The existing
fixture-digest discipline stays valid.
Makefile: `make bench-witness-divergence` (override default
qa.db / output path via WITNESS_QA_DB / WITNESS_OUT env-vars).
Closes the divergence → calibration data loop the witness ticket
imagined: every LLM hallucination on a canonical-shape question
becomes a supervised-correction fixture downstream prompt
improvements can grade against.
#000030 Phase 7 demo — function-sampled@v1 end-to-end
======================================================
`bench/scripts/demo_plot.py` — closes the loop on opencompletion's
activity24-math-plot.yaml. SymPy expression → quantized
integer-vector signature (canonical bytes) → optional matplotlib
PNG. Canonical bytes are the proof; PNG is just a downstream view
of the same evidence.
$ make demo-plot Q='sin(x)' PNG=/tmp/sin.png
Output JSON contains canonical_bytes_sha256 + canonical_bytes_preview
+ canonical_bytes_total_chars + grid metadata + the optional png_path.
matplotlib is gated — when absent, --png prints a warning to stderr
and skips the render; the canonical bytes still print. Tests skip
the PNG-presence assertion via `pytest.importorskip("matplotlib")`.
Public docs polish (#7)
========================
- docs/_source/bench.rst: updated fixture-count narrative (~660 →
662 default tasks + ~110 math π* fixtures); `make` quick-reference
now lists all per-π* 5S targets (tabular, calculus-limit/series,
linear-algebra, function-sampled) plus bench-real-shard,
bench-fork-baseline/score, bench-witness-divergence.
- docs/_source/v8-fork-score.rst: CLI section gained --out flag
documentation + a Make-harness sub-section covering
bench-fork-baseline / bench-fork-score / FORK_PARENT/CHILD/REPORT
env-vars.
Tests
=====
- tests/test_witness_to_5f.py — 8 new tests covering the audit-event
write (3) + extractor logic (5).
- tests/test_demo_plot.py — 6 new tests covering canonical-bytes
determinism + equivalence-class collapse + matplotlib gating.
Full suite: 1624 passed, 37 skipped (was 1568; +56).
97 lines
3.6 KiB
Python
97 lines
3.6 KiB
Python
"""Tests for the function-sampled@v1 demo-plot script.
|
|
|
|
The script's primary contract is the canonical-bytes path (always
|
|
prints SHA-256 + preview to stdout). The PNG render is an optional
|
|
matplotlib-gated downstream view; tests skip gracefully when
|
|
matplotlib isn't installed.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
|
|
sympy = pytest.importorskip("sympy")
|
|
|
|
|
|
def _run(*args: str) -> subprocess.CompletedProcess:
|
|
return subprocess.run(
|
|
[sys.executable, "-m", "bench.scripts.demo_plot", *args],
|
|
capture_output=True, text=True, check=False,
|
|
)
|
|
|
|
|
|
def test_demo_plot_emits_canonical_bytes_for_sin_x():
|
|
r = _run("--expr", "sin(x)", "--n-samples", "10", "--dv", "0.1")
|
|
assert r.returncode == 0, r.stderr
|
|
out = json.loads(r.stdout)
|
|
assert out["expr"] == "sin(x)"
|
|
assert out["pi_star_ref"] == "function-sampled@v1"
|
|
assert len(out["canonical_bytes_sha256"]) == 64
|
|
assert "canonical_bytes_preview" in out
|
|
assert out["grid"]["n_samples"] == 10
|
|
|
|
|
|
def test_demo_plot_canonical_bytes_deterministic():
|
|
"""Same expression + grid → same SHA-256."""
|
|
r1 = _run("--expr", "x**2", "--x-min", "0", "--x-max", "4",
|
|
"--n-samples", "5", "--dv", "1")
|
|
r2 = _run("--expr", "x**2", "--x-min", "0", "--x-max", "4",
|
|
"--n-samples", "5", "--dv", "1")
|
|
assert r1.returncode == 0 and r2.returncode == 0
|
|
out1 = json.loads(r1.stdout)
|
|
out2 = json.loads(r2.stdout)
|
|
assert out1["canonical_bytes_sha256"] == out2["canonical_bytes_sha256"]
|
|
|
|
|
|
def test_demo_plot_equivalent_expressions_collapse():
|
|
"""sin(x) and 2*sin(x)/2 must produce the same SHA-256 — they
|
|
sample to identical numeric values, so canonical bytes match."""
|
|
a = _run("--expr", "sin(x)", "--n-samples", "10", "--dv", "0.1")
|
|
b = _run("--expr", "2*sin(x)/2", "--n-samples", "10", "--dv", "0.1")
|
|
assert a.returncode == 0 and b.returncode == 0
|
|
assert (
|
|
json.loads(a.stdout)["canonical_bytes_sha256"]
|
|
== json.loads(b.stdout)["canonical_bytes_sha256"]
|
|
)
|
|
|
|
|
|
def test_demo_plot_invalid_expression_returns_1():
|
|
"""A genuinely unparseable expression should exit 1 with a
|
|
PiStarError surfaced on stderr."""
|
|
# Empty expression — sympify accepts garbage but lambdify or
|
|
# range checks fail; use an obviously-bad input.
|
|
r = _run("--expr", "x_min", "--x-min", "1", "--x-max", "1",
|
|
"--n-samples", "5", "--dv", "1")
|
|
# x_max <= x_min → PiStarError from function-sampled@v1
|
|
assert r.returncode == 1
|
|
assert "error" in r.stderr.lower()
|
|
|
|
|
|
def test_demo_plot_png_optional_no_crash_when_matplotlib_missing(tmp_path):
|
|
"""When matplotlib isn't installed, --png prints a warning to
|
|
stderr and skips the PNG; canonical bytes still print."""
|
|
png_path = tmp_path / "out.png"
|
|
r = _run("--expr", "x", "--n-samples", "5", "--dv", "1",
|
|
"--png", str(png_path))
|
|
assert r.returncode == 0, r.stderr
|
|
out = json.loads(r.stdout)
|
|
assert "canonical_bytes_sha256" in out
|
|
# matplotlib may or may not be installed in this env; the
|
|
# script's contract is "no crash either way."
|
|
|
|
|
|
def test_demo_plot_png_renders_when_matplotlib_present(tmp_path):
|
|
matplotlib = pytest.importorskip("matplotlib")
|
|
png_path = tmp_path / "out.png"
|
|
r = _run("--expr", "sin(x)", "--n-samples", "20", "--dv", "0.1",
|
|
"--png", str(png_path))
|
|
assert r.returncode == 0, r.stderr
|
|
assert png_path.is_file(), "matplotlib was importable; PNG must land"
|
|
# Sanity: the file isn't empty.
|
|
assert png_path.stat().st_size > 100
|