From 7cac942ea891dabe5fb927acc8d77ae84ce42127 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Sun, 10 May 2026 11:55:11 -0400 Subject: [PATCH] =?UTF-8?q?ticket=20#000036:=2051=20unit=20tests=20for=20t?= =?UTF-8?q?3=5Fbound=5Fcalculator=20+=20doc=20=C2=A711=20calibration?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds tests/test_t3_bound_calculator.py covering: - baseline (§11 worked example) bit-for-bit closed-form output - B1/B2/B3 isolation + monotonicity in each input - SNR_grad = g·‖∇L_max‖/σ_grad formula - B3 floor when N_b·σ_grad/‖∇L_max‖ ≤ 1 (adversary can't do worse than random shuffle) - constant-scaling (C_B1/B2/B3 in [0,1] fold linearly into B_i) - input-validation hard checks (gradient_fraction in (0,1], positive floats > 0, positive ints, constants in [0,1]) - recommendation text mode transitions (≤0 / <256 / ≥256 bit) - CLI subprocess invocation (argparse + JSON output, error path) - exact closed-form B1/B2 formulas across multiple configs - sum-of-three closure: I_window ≡ B1 + B2 + B3 (no missing term, no double-counting) 51 new tests; pure stdlib + subprocess invocation only. Full suite now 1720 passed / 45 skipped. Doc §11 calibration: The §11 worked-example output table quoted I_window ≈ 622.7, B1 ≈ 290.0, B3 ≈ 32.7. The closed-form actuals are 625.8716 / 292.4813 / 33.3904 — a ~3-bit total drift from rounding in the first-cut spec. Updated §11 to match the calculator's actual JSON output (calculator is the truth; doc was the approximation). SHA-256 single-window guarantee is broken at W=10000 either way; the calibration only sharpens the operator-guidance text. --- docs/soft-hash-channel-t3-bound.md | 19 +- tests/test_t3_bound_calculator.py | 351 +++++++++++++++++++++++++++++ 2 files changed, 362 insertions(+), 8 deletions(-) create mode 100644 tests/test_t3_bound_calculator.py diff --git a/docs/soft-hash-channel-t3-bound.md b/docs/soft-hash-channel-t3-bound.md index 1baebb2..6529b2d 100644 --- a/docs/soft-hash-channel-t3-bound.md +++ b/docs/soft-hash-channel-t3-bound.md @@ -399,16 +399,19 @@ $ python -m bench.scripts.t3_bound_calculator \ --batches-per-epoch 1024 \ --steps-per-epoch 1024 { - "I_window_bits_upper_bound": 622.7, - "B1_contribution": 290.0, + "I_window_bits_upper_bound": 625.8716, + "B1_contribution": 292.4813, "B2_contribution": 300.0, - "B3_contribution": 32.7, + "B3_contribution": 33.3904, + "snr_grad": 0.5, + "decisions_in_window": 100, + "epochs_in_window": 10, "constants": {"C_B1": 1.0, "C_B2": 1.0, "C_B3": 1.0}, - "recommendation": "I_window ≈ 622.7 bits/window. To steer - C(M) to a specific 256-bit target, adversary - needs ≥ 2^(256 - 622.7) ≈ {underflow} - windows (i.e., target trivially reachable - within one window)." + "recommendation": "I_window ≈ 625.9 bits/window EXCEEDS the + SHA-256 (256 bit) output size. M2's + single-window guarantee is broken at this W. + Reduce W (or reduce g / R / increase K) until + I_window < 256 bits/window." } ``` diff --git a/tests/test_t3_bound_calculator.py b/tests/test_t3_bound_calculator.py new file mode 100644 index 0000000..c0c0123 --- /dev/null +++ b/tests/test_t3_bound_calculator.py @@ -0,0 +1,351 @@ +"""Tests for the T3 per-window covert-channel bound calculator +(`bench/scripts/t3_bound_calculator.py`, ticket #000036 §11). + +Validates the closed-form bound from `docs/soft-hash-channel-t3- +bound.md` §6 against: + + - mathematical identities (B1 / B2 / B3 isolation, monotonicity in + each input) + - input-validation hard checks + - operator-guidance text mode transitions (≤0 / <256 / ≥256 bits) + - CLI surface (argparse + JSON output, error path) + - the §11 worked-example numbers in the ticket doc + +Pure stdlib — same dep profile as the calculator itself. +""" + +from __future__ import annotations + +import json +import math +import subprocess +import sys + +import pytest + +from bench.scripts.t3_bound_calculator import ( + CALCULATOR_VERSION, + t3_bound_bits, +) + + +# --- baseline (§11 worked example) ----------------------------------- + + +_BASELINE = dict( + gradient_fraction=0.05, + gradient_norm_max=1.0, + gradient_noise_stddev=0.1, + lr_decision_interval=100, + lr_grid_size=8, + window_length=10000, + batches_per_epoch=1024, + steps_per_epoch=1024, +) + + +def test_baseline_matches_section_11_doc(): + """The §11 worked example baseline. Calculator's actual numbers: + + I_window_bits_upper_bound: 625.8716 + B1_contribution: 292.4813 + B2_contribution: 300.0 + B3_contribution: 33.3904 + snr_grad: 0.5 + + Doc §11's quoted 622.7 / 290.0 / 32.7 are first-cut rounded + estimates; calculator's actual closed-form values differ by + ~3 bits total due to the doc's 290 vs actual 292.5 in B1 and + 32.7 vs actual 33.4 in B3. This test pins the closed-form + truth (the doc text needs a calibration pass; tracked in this + test's docstring as a follow-up). + """ + r = t3_bound_bits(**_BASELINE) + # Total: ⟨B1+B2+B3⟩ ≈ 292.48 + 300.0 + 33.39 = 625.87 + assert r["I_window_bits_upper_bound"] == pytest.approx(625.87, abs=0.05) + # B2 is exact: ⌈10000/100⌉ · log₂(8) = 100 · 3 = 300 + assert r["B2_contribution"] == pytest.approx(300.0, abs=1e-3) + # B3: epochs = ⌈10000/1024⌉ = 10; per-epoch factor = 1024·0.1/1.0 = 102.4; + # log₂(102.4)/2 ≈ 3.339; total ≈ 33.39 bits. + assert r["B3_contribution"] == pytest.approx(33.39, abs=0.05) + # B1: C_B1 · g · W · log₂(SNR+1) where SNR = 0.05·1.0/0.1 = 0.5 + # → log₂(1.5) ≈ 0.585; total ≈ 1.0 · 0.05 · 10000 · 0.585 = 292.48 + assert r["B1_contribution"] == pytest.approx(292.48, abs=0.05) + + +def test_returns_calculator_version_token(): + r = t3_bound_bits(**_BASELINE) + assert r["calculator_version"] == CALCULATOR_VERSION + # versioned-default discipline: matches the v1-bottou-refinement token + assert "v1" in CALCULATOR_VERSION + + +def test_b2_isolated_zero_when_decisions_zero(): + """If lr_decision_interval > window_length, decisions_in_window = 1 + (one initial decision); the bound floor is log₂ R, not zero. + Exercises the ceil semantics on small W / large K. + """ + r = t3_bound_bits(**{**_BASELINE, "window_length": 1, "lr_decision_interval": 100}) + # ceil(1/100) = 1, so B2 = log₂ 8 = 3.0 + assert r["B2_contribution"] == pytest.approx(3.0, abs=1e-6) + + +def test_snr_grad_formula(): + """SNR_grad = g · ‖∇L_max‖ / σ_grad. Check directly.""" + r = t3_bound_bits(**_BASELINE) + expected_snr = 0.05 * 1.0 / 0.1 # 0.5 + assert r["snr_grad"] == pytest.approx(expected_snr, abs=1e-6) + + +# --- monotonicity / bound shape -------------------------------------- + + +def test_monotone_in_window_length(): + """Doubling W roughly doubles each contribution (B1 linear in W, + B2 + B3 grow with ⌈W/K⌉ + ⌈W/E⌉ which scale linearly with W on + aligned periods).""" + base = t3_bound_bits(**_BASELINE) + long_window = t3_bound_bits(**{**_BASELINE, "window_length": 20000}) + # Total grows monotonically. + assert long_window["I_window_bits_upper_bound"] > base["I_window_bits_upper_bound"] + # B1 is exactly linear in W, so doubling W doubles B1. + assert long_window["B1_contribution"] == pytest.approx( + 2 * base["B1_contribution"], rel=1e-6 + ) + + +def test_monotone_in_gradient_fraction(): + """Increasing g raises the bound (more adversarial control).""" + base = t3_bound_bits(**_BASELINE) + high_g = t3_bound_bits(**{**_BASELINE, "gradient_fraction": 0.5}) + assert high_g["I_window_bits_upper_bound"] > base["I_window_bits_upper_bound"] + # B2 + B3 do not depend on g, so the increase is in B1 only. + assert high_g["B2_contribution"] == base["B2_contribution"] + assert high_g["B3_contribution"] == base["B3_contribution"] + + +def test_b3_floor_at_zero_when_factor_below_one(): + """When N_b · σ_grad / ‖∇L_max‖ ≤ 1, log₂ goes ≤ 0; the bound + floors at 0 — adversary can't do worse than random shuffle in + this regime. Verify the floor.""" + # σ_grad = 0.001, gradient_norm_max = 100, N_b = 4 → 0.00004 < 1 + args = {**_BASELINE, "gradient_noise_stddev": 0.001, + "gradient_norm_max": 100.0, "batches_per_epoch": 4, + "steps_per_epoch": 4} + r = t3_bound_bits(**args) + assert r["B3_contribution"] == pytest.approx(0.0, abs=1e-9) + + +def test_constant_C_B1_scales_b1(): + """C_B1 · B1 — halving C_B1 halves B1 (calculator rounds output + to 4 dp; tolerate rounding).""" + base = t3_bound_bits(**_BASELINE) + half_c = t3_bound_bits(**_BASELINE, c_b1=0.5) + assert half_c["B1_contribution"] == pytest.approx( + 0.5 * base["B1_contribution"], abs=1e-3 + ) + # B2 / B3 unchanged. + assert half_c["B2_contribution"] == base["B2_contribution"] + assert half_c["B3_contribution"] == base["B3_contribution"] + + +def test_constants_recorded(): + r = t3_bound_bits(**_BASELINE, c_b1=0.4, c_b2=0.3, c_b3=0.2) + assert r["constants"] == {"C_B1": 0.4, "C_B2": 0.3, "C_B3": 0.2} + + +# --- input validation ------------------------------------------------ + + +@pytest.mark.parametrize("g", [0.0, -0.1, 1.5, 2.0]) +def test_invalid_gradient_fraction_rejected(g): + args = {**_BASELINE, "gradient_fraction": g} + with pytest.raises(ValueError, match="gradient_fraction"): + t3_bound_bits(**args) + + +@pytest.mark.parametrize("name,value", [ + ("gradient_norm_max", 0.0), + ("gradient_norm_max", -1.0), + ("gradient_noise_stddev", 0.0), + ("gradient_noise_stddev", -0.1), +]) +def test_invalid_positive_floats_rejected(name, value): + args = {**_BASELINE, name: value} + with pytest.raises(ValueError, match=name): + t3_bound_bits(**args) + + +@pytest.mark.parametrize("name", [ + "lr_decision_interval", "lr_grid_size", "window_length", + "batches_per_epoch", "steps_per_epoch", +]) +@pytest.mark.parametrize("bad", [0, -1, 1.5, "a"]) +def test_invalid_positive_int_rejected(name, bad): + args = {**_BASELINE, name: bad} + with pytest.raises(ValueError, match=name): + t3_bound_bits(**args) + + +@pytest.mark.parametrize("c_name,c_val", [ + ("c_b1", -0.1), ("c_b1", 1.5), + ("c_b2", -0.5), ("c_b2", 2.0), + ("c_b3", -1.0), ("c_b3", 1.001), +]) +def test_constants_clamped_to_unit_interval(c_name, c_val): + args = {**_BASELINE, c_name: c_val} + with pytest.raises(ValueError, match=c_name): + t3_bound_bits(**args) + + +# --- recommendation text mode transitions ---------------------------- + + +def test_recommendation_total_le_zero(): + """When the bound floors at 0 across all three terms (extreme + benign config), recommendation reads as 'vacuously safe'.""" + args = { + **_BASELINE, + "gradient_fraction": 1e-9, # near-zero g → tiny B1 + "lr_decision_interval": 1_000_000, # huge K → ⌈W/K⌉ = 1, but log₂ R = log₂ 1 = 0 below + "lr_grid_size": 1, # log₂(1) = 0 → B2 = 0 + "gradient_noise_stddev": 0.001, + "gradient_norm_max": 100.0, + "batches_per_epoch": 4, + "steps_per_epoch": 4, # b3_factor = 4·0.001/100 = 4e-5 < 1 → B3 floor + } + r = t3_bound_bits(**args) + assert r["I_window_bits_upper_bound"] >= 0 + # B1 is tiny but not exactly 0; total floor regime depends on + # whether the sum reaches positive territory. Assert at least + # B2 + B3 are zero on this config. + assert r["B2_contribution"] == pytest.approx(0.0, abs=1e-9) + assert r["B3_contribution"] == pytest.approx(0.0, abs=1e-9) + + +def test_recommendation_below_sha256(): + """Baseline (~622 bits) actually exceeds 256 — wrong band for + this test. Use a config that lands in the (0, 256) range.""" + args = {**_BASELINE, "window_length": 1000} + r = t3_bound_bits(**args) + assert 0 < r["I_window_bits_upper_bound"] < 256, ( + f"window_length=1000 should land in (0, 256); got {r['I_window_bits_upper_bound']}" + ) + assert "windows" in r["recommendation"] + assert "256" in r["recommendation"] + + +def test_recommendation_exceeds_sha256(): + """Baseline (W=10000) → 622 bits > 256 — recommendation should + flag M2's single-window guarantee as broken.""" + r = t3_bound_bits(**_BASELINE) + assert r["I_window_bits_upper_bound"] > 256 + assert "EXCEEDS" in r["recommendation"] + assert "256" in r["recommendation"] + + +# --- CLI surface (subprocess invocation) ----------------------------- + + +def test_cli_baseline_runs_clean(tmp_path): + """Invoke the calculator CLI with §11 worked-example flags; + parse the JSON output; verify the I_window_bits_upper_bound + matches the function call.""" + cmd = [ + sys.executable, "-m", "bench.scripts.t3_bound_calculator", + "--gradient-fraction", "0.05", + "--gradient-norm-max", "1.0", + "--gradient-noise-stddev", "0.1", + "--lr-decision-interval", "100", + "--lr-grid-size", "8", + "--window-length", "10000", + "--batches-per-epoch", "1024", + "--steps-per-epoch", "1024", + ] + out = subprocess.run(cmd, capture_output=True, text=True, check=True, + cwd="/home/fox/git/arborist") + j = json.loads(out.stdout) + assert j["calculator_version"] == CALCULATOR_VERSION + # Closed-form actual; doc §11's 622.7 is approximate. + assert j["I_window_bits_upper_bound"] == pytest.approx(625.87, abs=0.05) + + +def test_cli_invalid_input_exits_2(): + """ValueError → exit code 2 + stderr error message.""" + cmd = [ + sys.executable, "-m", "bench.scripts.t3_bound_calculator", + "--gradient-fraction", "1.5", # invalid + "--gradient-norm-max", "1.0", + "--gradient-noise-stddev", "0.1", + "--lr-decision-interval", "100", + "--lr-grid-size", "8", + "--window-length", "100", + "--batches-per-epoch", "1024", + "--steps-per-epoch", "1024", + ] + out = subprocess.run(cmd, capture_output=True, text=True, + cwd="/home/fox/git/arborist") + assert out.returncode == 2 + assert "gradient_fraction" in out.stderr + + +# --- closed-form sanity --------------------------------------------- + + +def test_b2_exact_formula(): + """B2 = C_B2 · ⌈W/K⌉ · log₂(R). Direct closed-form check + against several configurations.""" + cases = [ + # (W, K, R) → expected ⌈W/K⌉ · log₂ R + (1000, 100, 8, 10 * 3.0), + (1000, 200, 4, 5 * 2.0), + (999, 100, 16, 10 * 4.0), # ceil(999/100)=10 + (1, 100, 2, 1 * 1.0), + ] + for W, K, R, expected in cases: + r = t3_bound_bits( + **{**_BASELINE, "window_length": W, + "lr_decision_interval": K, "lr_grid_size": R}, + ) + assert r["B2_contribution"] == pytest.approx(expected, abs=1e-9), ( + f"B2 mismatch on (W={W}, K={K}, R={R}): " + f"{r['B2_contribution']} vs expected {expected}" + ) + + +def test_b1_exact_formula(): + """B1 = C_B1 · g · W · log₂(SNR + 1) where SNR = g·‖∇L_max‖/σ_grad. + Hand-compute for one config and check exact agreement.""" + g, gnm, sigma, W = 0.1, 2.0, 0.5, 5000 + snr = g * gnm / sigma # 0.4 + expected = 1.0 * g * W * math.log2(snr + 1) + r = t3_bound_bits(**{ + **_BASELINE, + "gradient_fraction": g, + "gradient_norm_max": gnm, + "gradient_noise_stddev": sigma, + "window_length": W, + }) + # Output rounded to 4 dp inside the calculator. + assert r["B1_contribution"] == pytest.approx(expected, abs=1e-3) + assert r["snr_grad"] == pytest.approx(snr, abs=1e-3) + + +# --- integration with §6 framework ----------------------------------- + + +def test_total_equals_sum_of_three_contributions(): + """Closure check: I_window ≡ B1 + B2 + B3 (no missing term, no + double-counting). Multiple inputs to widen the cone.""" + for window_length, lr_decision_interval in [ + (100, 10), + (5000, 250), + (10000, 100), + (50000, 1000), + ]: + args = {**_BASELINE, "window_length": window_length, + "lr_decision_interval": lr_decision_interval} + r = t3_bound_bits(**args) + total = r["B1_contribution"] + r["B2_contribution"] + r["B3_contribution"] + # Allow small rounding (round to 4 dp on each contribution). + assert r["I_window_bits_upper_bound"] == pytest.approx(total, abs=1e-3)