russell@unturf.com
823e8da1ff
Two follow-up defects from 2026-06-14 factory triage:
(1) DLQ runner classifier did not recognize "vram-oversized" reason text
introduced by foxhop dispatcher pre-flight (commit 8d45e0d on the foxhop
side). 65 of 87 rDLQ cells got escalated as class=unknown instead of a
properly named bucket. Adds pattern + escalate-class entry + reducer
test case mapped to (vram-oversized sim no — bin is dead weight on this
card, salvage skipped).
(2) sweep-doctrine reducers had no per-test timeout. K=5 doctrine tests
(test-k5-apply-forward-ipmul + 4 siblings) ran lumbda at 99% CPU for
2h43m on a remote node without ever emitting their DOCTRINE verdict
line — accumulating 30+ runaway lumbda procs under two stuck `make
sweep-doctrine` invocations. run.sh + run-parallel.sh now wrap our
lumbda invocation in `timeout ${SWEEP_DOCTRINE_TEST_TIMEOUT_S:-300}`;
hit exits 124, our existing "no DOCTRINE line" branch logs HARNESS-FAIL.
K=5 substrate has a documented non-terminating compute defect AND a
load-time buffer overflow (commit 6d18c59 on foxhop). Bisect deferred
per ticket 0007 in foxhop tree; needs qemu apparatus we currently lack.
183 lines
6.9 KiB
Bash
Executable file
183 lines
6.9 KiB
Bash
Executable file
#!/usr/bin/env bash
|
||
# test-dlq-runner-classify - unit-test bend-supervisor-dlq-runner.sh's
|
||
# classify_reason() function. Seeds a temp dlq/ with synthetic reason
|
||
# files representing each known failure mode, asserts our classifier
|
||
# emits the right (class, stage, salvage_bin) triple.
|
||
#
|
||
# Independent of live LUMBDA_QUEUE_DIR; runs in mktemp -d. Safe on any
|
||
# host.
|
||
#
|
||
# Env contract per ~/git/lumbda/factory/CONTRACT.md:
|
||
# LUMBDA_FACTORY_DIR where bend-supervisor-dlq-runner.sh lives
|
||
# (default $HOME/git/lumbda/factory).
|
||
#
|
||
# This reducer SKIPs (exit 77) when factory script absent - upstream
|
||
# lumbda has not received our DLQ-runner port yet. After our port lands,
|
||
# this gate fires.
|
||
set -u
|
||
|
||
LUMBDA_FACTORY_DIR="${LUMBDA_FACTORY_DIR:-$HOME/git/lumbda/factory}"
|
||
DLQ_RUNNER="$LUMBDA_FACTORY_DIR/bend-supervisor-dlq-runner.sh"
|
||
|
||
if [ ! -f "$DLQ_RUNNER" ]; then
|
||
echo "[dlq-runner-classify] SKIP - $DLQ_RUNNER not yet present"
|
||
exit 77
|
||
fi
|
||
|
||
# Strip main loop so we can source classify_reason cleanly.
|
||
STUB=$(mktemp /tmp/dlq-runner-stub-XXXXXX.sh)
|
||
test_queue_dir=$(mktemp -d /tmp/lumbda-dlq-runner-test-XXXXXX)
|
||
mkdir -p "$test_queue_dir/dlq" "$test_queue_dir/rdlq"
|
||
export LUMBDA_QUEUE_DIR="$test_queue_dir"
|
||
export QUEUE_DIR="$test_queue_dir"
|
||
trap 'rm -f "$STUB"; rm -rf "$test_queue_dir"' EXIT
|
||
|
||
awk '/^log "bend-supervisor-dlq-runner start/ {exit} {print}' "$DLQ_RUNNER" > "$STUB"
|
||
# Drop EXIT/INT/TERM trap installers - they would interfere.
|
||
sed -i '/^trap on_exit EXIT$/d; /^trap .EXIT_REASON=signal-int/d; /^trap .EXIT_REASON=signal-term/d' "$STUB"
|
||
|
||
# shellcheck source=/dev/null
|
||
. "$STUB" 2>/dev/null
|
||
|
||
fails=0
|
||
pass() { echo "PASS $*"; }
|
||
fail() { echo "FAIL $*"; fails=$((fails + 1)); }
|
||
|
||
# assert_classify <label> <reason-text> <expected-class> <expected-stage> <expected-salvage>
|
||
assert_classify() {
|
||
local label="$1" content="$2" exp_class="$3" exp_stage="$4" exp_salv="$5"
|
||
local f="$test_queue_dir/dlq/__test_$$_$RANDOM.reason"
|
||
printf '%s' "$content" > "$f"
|
||
local got
|
||
got=$(classify_reason "$f")
|
||
local got_class got_stage got_salv
|
||
got_class=$(echo "$got" | awk '{print $1}')
|
||
got_stage=$(echo "$got" | awk '{print $2}')
|
||
got_salv=$(echo "$got" | awk '{print $3}')
|
||
if [ "$got_class" = "$exp_class" ] && [ "$got_stage" = "$exp_stage" ] && [ "$got_salv" = "$exp_salv" ]; then
|
||
pass "$label -> $got_class/$got_stage/$got_salv"
|
||
else
|
||
fail "$label expected=$exp_class/$exp_stage/$exp_salv got=$got_class/$got_stage/$got_salv"
|
||
fi
|
||
rm -f "$f"
|
||
}
|
||
|
||
echo "── classifier tests ──"
|
||
|
||
assert_classify "cuda-oom (typed Phase 7 portal)" \
|
||
'(cuda-sim-error (code cuda-oom) (reason "out of memory at sim_gpu_packed.cu:179"))' \
|
||
"cuda-oom" "sim" "yes"
|
||
|
||
assert_classify "cuda-oom (legacy stderr leak)" \
|
||
'# dispatch foo.bin (0.5 GB) × 141 batches
|
||
wall_s=42
|
||
raw_resp=(error (no-portal "/tmp/bend-sim-ops-X.portal"))
|
||
--- bend worker log context ---
|
||
CUDA error cudaMalloc(&c->d_bits, ...) at sim_gpu_packed.cu:179: out of memory' \
|
||
"cuda-oom" "sim" "yes"
|
||
|
||
assert_classify "cuda-illegal-addr (typed)" \
|
||
'(cuda-sim-error (code cuda-illegal-addr) (reason "..."))' \
|
||
"cuda-illegal-addr" "sim" "yes"
|
||
|
||
assert_classify "memory-cap-refused" \
|
||
'(cuda-sim-error (code memory-cap-refused) (reason "peak host+device memory ~22.4 GB exceeds 16 GB cap"))' \
|
||
"memory-cap-refused" "sim" "yes"
|
||
|
||
assert_classify "vram-budget-refused (Phase 8 fast-fail)" \
|
||
'(cuda-sim-error (code vram-budget-refused) (reason "predicted 5.4 GB > free 1.2 GB at sim_gpu_packed_alloc; reduce n_batches"))' \
|
||
"vram-budget-refused" "sim" "yes"
|
||
|
||
assert_classify "vram-oversized (dispatcher pre-flight reject 2026-06-14)" \
|
||
'tier=vram-oversized bin_bytes=4002302992 bin_mib=3816 vram_need_mib=46304 vram_total_mib=24576
|
||
reason: vram-oversized — predicted 46304MiB exceeds card capacity 24576MiB. Cell cannot fit.' \
|
||
"vram-oversized" "sim" "no"
|
||
|
||
assert_classify "bin-load-fail" \
|
||
'(cuda-sim-error (code bin-load-fail) (reason "load_ops_bin returned non-zero"))' \
|
||
"bin-load-fail" "emit" "no"
|
||
|
||
assert_classify "no-portal (pre-Phase-7)" \
|
||
'raw_resp=(error (no-portal "/tmp/bend-sim-ops-X.portal"))' \
|
||
"no-portal" "sim" "yes"
|
||
|
||
# Note: queue path inside reason body deliberately reads /tmp/lumbda-queue
|
||
# (CONTRACT default) - classifier matches on `missing:` prefix not on path
|
||
# substring, so this reads cleanly under upstream's generic env.
|
||
assert_classify "missing-bin" \
|
||
'missing: /tmp/lumbda-queue/foo-cell.bin' \
|
||
"missing-bin" "dispatch" "yes"
|
||
|
||
assert_classify "tier=zero-magic (emit interrupted)" \
|
||
'tier=zero-magic bin_bytes=146800656
|
||
reason: emit-interrupted' \
|
||
"emit-broken" "emit" "no"
|
||
|
||
assert_classify "tier=zero-n_ops (emit-finalize never fired)" \
|
||
'tier=zero-n_ops bin_bytes=146800656
|
||
reason: emit-finalize-never-fired' \
|
||
"emit-broken" "emit" "no"
|
||
|
||
assert_classify "tier=above-max" \
|
||
'tier=above-max bin_bytes=8144699544
|
||
reason: bin-too-large (over tier-huge max ...)' \
|
||
"tier-classify" "emit" "no"
|
||
|
||
assert_classify "BISECT-DIAG (pool finalize race)" \
|
||
'BISECT-DIAG: ipmul step6 swap target<->tmp
|
||
BISECT-DIAG: ipmul step7 free tmp
|
||
BISECT-DIAG: exit dgcd-raw-pa! dispatch arm' \
|
||
"bisect-pool-race" "emit" "no"
|
||
|
||
assert_classify "transient cuda-error (catchall typed)" \
|
||
'(cuda-sim-error (code cuda-error) (reason "transient driver"))' \
|
||
"cuda-error-transient" "sim" "yes"
|
||
|
||
assert_classify "unknown (catchall)" \
|
||
'completely unrelated text' \
|
||
"unknown" "unknown" "no"
|
||
|
||
# 2026-06-13: empty / whitespace-only .reason files used to land in
|
||
# "unknown" class + rDLQ. Now classified as transient-empty-reason
|
||
# (auto-retry up to AUTO_RETRY_MAX before escalating). Catches our
|
||
# dispatcher socket-disconnect / minimal-output failure mode. Detection
|
||
# stays whitespace-aware so short-but-meaningful reasons (like
|
||
# "missing: ...foo.bin") never get swept into transient class.
|
||
assert_classify "empty .reason (0 bytes)" \
|
||
'' \
|
||
"transient-empty-reason" "sim" "yes"
|
||
|
||
assert_classify "whitespace-only .reason" \
|
||
'
|
||
|
||
' \
|
||
"transient-empty-reason" "sim" "yes"
|
||
|
||
# Class-set checks - partitioning by is_auto_class / is_escalate_class
|
||
# must cover every named class exactly once.
|
||
echo ""
|
||
echo "── class-partition tests ──"
|
||
|
||
for c in missing-bin bisect-pool-race cuda-error-transient no-portal transient-empty-reason; do
|
||
if is_auto_class "$c" && ! is_escalate_class "$c"; then
|
||
pass "$c is auto (not escalate)"
|
||
else
|
||
fail "$c partitioning wrong"
|
||
fi
|
||
done
|
||
|
||
for c in cuda-oom cuda-illegal-addr memory-cap-refused vram-budget-refused vram-oversized bin-load-fail tier-classify emit-broken unknown; do
|
||
if is_escalate_class "$c" && ! is_auto_class "$c"; then
|
||
pass "$c is escalate (not auto)"
|
||
else
|
||
fail "$c partitioning wrong"
|
||
fi
|
||
done
|
||
|
||
echo ""
|
||
if [ "$fails" -gt 0 ]; then
|
||
echo "[dlq-runner-classify] FAIL - $fails assertion(s) failed"
|
||
exit 1
|
||
fi
|
||
echo "[dlq-runner-classify] PASS - all assertions OK"
|
||
exit 0
|