russell@unturf.com
1665893321
29 new files publish factory infra (V2 autoscaler with live VRAM sampling + EWMA peak tracking, HUGE solo-dispatch, two-tier DLQ/rDLQ classifier + retry), general quantum circuit primitives (Cuccaro ripple-carry adder, Clifford gate library, Clifford tableau simulator, mod-arith family, dialog GCD reversible inverse, Karatsuba multiplier, Solinas fast reduction), and a TCRAUDT reducer harness. Originally developed in ~/git/www.foxhop.net/ecdsa/ for secp256k1 attack-surface research; published upstream as obligated by AGPLv3. Parametrization contract at factory/CONTRACT.md. Consumers export LUMBDA_REPO_DIR + LUMBDA_QUEUE_DIR + LUMBDA_BACKEND_CMD + LUMBDA_EMITTER_CMD then exec factory scripts. No fork-and-modify; single source of truth upstream. Integration tests gate 7 V2 defect classes that wedged a live factory on 2026-06-12 (skewed-demand starve, zero-floor reservation, multi-tier greedy, +-25%% damping, cold-start ramp, DLQ surge halve, post-damp CPU ceiling) + 28 DLQ classifier cases (auto-retry vs escalate partition) + bash -n syntax lint across every script. GPU backend stays in consumer trees; rationale in factory/GPU-BACKEND-NOTE.md. Bend wire protocol + gpu-worker.lsp already upstream at examples/cuda-fanout/. make factory-lint bash -n on every factory/*.sh make test-integration V2 reducer + DLQ classifier + syntax gate make sweep-doctrine TCRAUDT reducer gate (serial) make sweep-doctrine-parallel xargs -P fan-out Verified on neoblanka: factory-lint 12 scripts PASS; test-integration 14 V2 cases + 28 DLQ classifier cases + 12 syntax cases all PASS.
178 lines
6.5 KiB
Bash
Executable file
178 lines
6.5 KiB
Bash
Executable file
#!/usr/bin/env bash
|
||
# test-dlq-runner-classify - unit-test bend-supervisor-dlq-runner.sh's
|
||
# classify_reason() function. Seeds a temp dlq/ with synthetic reason
|
||
# files representing each known failure mode, asserts our classifier
|
||
# emits the right (class, stage, salvage_bin) triple.
|
||
#
|
||
# Independent of live LUMBDA_QUEUE_DIR; runs in mktemp -d. Safe on any
|
||
# host.
|
||
#
|
||
# Env contract per ~/git/lumbda/factory/CONTRACT.md:
|
||
# LUMBDA_FACTORY_DIR where bend-supervisor-dlq-runner.sh lives
|
||
# (default $HOME/git/lumbda/factory).
|
||
#
|
||
# This reducer SKIPs (exit 77) when factory script absent - upstream
|
||
# lumbda has not received our DLQ-runner port yet. After our port lands,
|
||
# this gate fires.
|
||
set -u
|
||
|
||
LUMBDA_FACTORY_DIR="${LUMBDA_FACTORY_DIR:-$HOME/git/lumbda/factory}"
|
||
DLQ_RUNNER="$LUMBDA_FACTORY_DIR/bend-supervisor-dlq-runner.sh"
|
||
|
||
if [ ! -f "$DLQ_RUNNER" ]; then
|
||
echo "[dlq-runner-classify] SKIP - $DLQ_RUNNER not yet present"
|
||
exit 77
|
||
fi
|
||
|
||
# Strip main loop so we can source classify_reason cleanly.
|
||
STUB=$(mktemp /tmp/dlq-runner-stub-XXXXXX.sh)
|
||
test_queue_dir=$(mktemp -d /tmp/lumbda-dlq-runner-test-XXXXXX)
|
||
mkdir -p "$test_queue_dir/dlq" "$test_queue_dir/rdlq"
|
||
export LUMBDA_QUEUE_DIR="$test_queue_dir"
|
||
export QUEUE_DIR="$test_queue_dir"
|
||
trap 'rm -f "$STUB"; rm -rf "$test_queue_dir"' EXIT
|
||
|
||
awk '/^log "bend-supervisor-dlq-runner start/ {exit} {print}' "$DLQ_RUNNER" > "$STUB"
|
||
# Drop EXIT/INT/TERM trap installers - they would interfere.
|
||
sed -i '/^trap on_exit EXIT$/d; /^trap .EXIT_REASON=signal-int/d; /^trap .EXIT_REASON=signal-term/d' "$STUB"
|
||
|
||
# shellcheck source=/dev/null
|
||
. "$STUB" 2>/dev/null
|
||
|
||
fails=0
|
||
pass() { echo "PASS $*"; }
|
||
fail() { echo "FAIL $*"; fails=$((fails + 1)); }
|
||
|
||
# assert_classify <label> <reason-text> <expected-class> <expected-stage> <expected-salvage>
|
||
assert_classify() {
|
||
local label="$1" content="$2" exp_class="$3" exp_stage="$4" exp_salv="$5"
|
||
local f="$test_queue_dir/dlq/__test_$$_$RANDOM.reason"
|
||
printf '%s' "$content" > "$f"
|
||
local got
|
||
got=$(classify_reason "$f")
|
||
local got_class got_stage got_salv
|
||
got_class=$(echo "$got" | awk '{print $1}')
|
||
got_stage=$(echo "$got" | awk '{print $2}')
|
||
got_salv=$(echo "$got" | awk '{print $3}')
|
||
if [ "$got_class" = "$exp_class" ] && [ "$got_stage" = "$exp_stage" ] && [ "$got_salv" = "$exp_salv" ]; then
|
||
pass "$label -> $got_class/$got_stage/$got_salv"
|
||
else
|
||
fail "$label expected=$exp_class/$exp_stage/$exp_salv got=$got_class/$got_stage/$got_salv"
|
||
fi
|
||
rm -f "$f"
|
||
}
|
||
|
||
echo "── classifier tests ──"
|
||
|
||
assert_classify "cuda-oom (typed Phase 7 portal)" \
|
||
'(cuda-sim-error (code cuda-oom) (reason "out of memory at sim_gpu_packed.cu:179"))' \
|
||
"cuda-oom" "sim" "yes"
|
||
|
||
assert_classify "cuda-oom (legacy stderr leak)" \
|
||
'# dispatch foo.bin (0.5 GB) × 141 batches
|
||
wall_s=42
|
||
raw_resp=(error (no-portal "/tmp/bend-sim-ops-X.portal"))
|
||
--- bend worker log context ---
|
||
CUDA error cudaMalloc(&c->d_bits, ...) at sim_gpu_packed.cu:179: out of memory' \
|
||
"cuda-oom" "sim" "yes"
|
||
|
||
assert_classify "cuda-illegal-addr (typed)" \
|
||
'(cuda-sim-error (code cuda-illegal-addr) (reason "..."))' \
|
||
"cuda-illegal-addr" "sim" "yes"
|
||
|
||
assert_classify "memory-cap-refused" \
|
||
'(cuda-sim-error (code memory-cap-refused) (reason "peak host+device memory ~22.4 GB exceeds 16 GB cap"))' \
|
||
"memory-cap-refused" "sim" "yes"
|
||
|
||
assert_classify "vram-budget-refused (Phase 8 fast-fail)" \
|
||
'(cuda-sim-error (code vram-budget-refused) (reason "predicted 5.4 GB > free 1.2 GB at sim_gpu_packed_alloc; reduce n_batches"))' \
|
||
"vram-budget-refused" "sim" "yes"
|
||
|
||
assert_classify "bin-load-fail" \
|
||
'(cuda-sim-error (code bin-load-fail) (reason "load_ops_bin returned non-zero"))' \
|
||
"bin-load-fail" "emit" "no"
|
||
|
||
assert_classify "no-portal (pre-Phase-7)" \
|
||
'raw_resp=(error (no-portal "/tmp/bend-sim-ops-X.portal"))' \
|
||
"no-portal" "sim" "yes"
|
||
|
||
# Note: queue path inside reason body deliberately reads /tmp/lumbda-queue
|
||
# (CONTRACT default) - classifier matches on `missing:` prefix not on path
|
||
# substring, so this reads cleanly under upstream's generic env.
|
||
assert_classify "missing-bin" \
|
||
'missing: /tmp/lumbda-queue/foo-cell.bin' \
|
||
"missing-bin" "dispatch" "yes"
|
||
|
||
assert_classify "tier=zero-magic (emit interrupted)" \
|
||
'tier=zero-magic bin_bytes=146800656
|
||
reason: emit-interrupted' \
|
||
"emit-broken" "emit" "no"
|
||
|
||
assert_classify "tier=zero-n_ops (emit-finalize never fired)" \
|
||
'tier=zero-n_ops bin_bytes=146800656
|
||
reason: emit-finalize-never-fired' \
|
||
"emit-broken" "emit" "no"
|
||
|
||
assert_classify "tier=above-max" \
|
||
'tier=above-max bin_bytes=8144699544
|
||
reason: bin-too-large (over tier-huge max ...)' \
|
||
"tier-classify" "emit" "no"
|
||
|
||
assert_classify "BISECT-DIAG (pool finalize race)" \
|
||
'BISECT-DIAG: ipmul step6 swap target<->tmp
|
||
BISECT-DIAG: ipmul step7 free tmp
|
||
BISECT-DIAG: exit dgcd-raw-pa! dispatch arm' \
|
||
"bisect-pool-race" "emit" "no"
|
||
|
||
assert_classify "transient cuda-error (catchall typed)" \
|
||
'(cuda-sim-error (code cuda-error) (reason "transient driver"))' \
|
||
"cuda-error-transient" "sim" "yes"
|
||
|
||
assert_classify "unknown (catchall)" \
|
||
'completely unrelated text' \
|
||
"unknown" "unknown" "no"
|
||
|
||
# 2026-06-13: empty / whitespace-only .reason files used to land in
|
||
# "unknown" class + rDLQ. Now classified as transient-empty-reason
|
||
# (auto-retry up to AUTO_RETRY_MAX before escalating). Catches our
|
||
# dispatcher socket-disconnect / minimal-output failure mode. Detection
|
||
# stays whitespace-aware so short-but-meaningful reasons (like
|
||
# "missing: ...foo.bin") never get swept into transient class.
|
||
assert_classify "empty .reason (0 bytes)" \
|
||
'' \
|
||
"transient-empty-reason" "sim" "yes"
|
||
|
||
assert_classify "whitespace-only .reason" \
|
||
'
|
||
|
||
' \
|
||
"transient-empty-reason" "sim" "yes"
|
||
|
||
# Class-set checks - partitioning by is_auto_class / is_escalate_class
|
||
# must cover every named class exactly once.
|
||
echo ""
|
||
echo "── class-partition tests ──"
|
||
|
||
for c in missing-bin bisect-pool-race cuda-error-transient no-portal transient-empty-reason; do
|
||
if is_auto_class "$c" && ! is_escalate_class "$c"; then
|
||
pass "$c is auto (not escalate)"
|
||
else
|
||
fail "$c partitioning wrong"
|
||
fi
|
||
done
|
||
|
||
for c in cuda-oom cuda-illegal-addr memory-cap-refused vram-budget-refused bin-load-fail tier-classify emit-broken unknown; do
|
||
if is_escalate_class "$c" && ! is_auto_class "$c"; then
|
||
pass "$c is escalate (not auto)"
|
||
else
|
||
fail "$c partitioning wrong"
|
||
fi
|
||
done
|
||
|
||
echo ""
|
||
if [ "$fails" -gt 0 ]; then
|
||
echo "[dlq-runner-classify] FAIL - $fails assertion(s) failed"
|
||
exit 1
|
||
fi
|
||
echo "[dlq-runner-classify] PASS - all assertions OK"
|
||
exit 0
|