factory + sweep-doctrine: vram-oversized class + per-test timeout
Two follow-up defects from 2026-06-14 factory triage:
(1) DLQ runner classifier did not recognize "vram-oversized" reason text
introduced by foxhop dispatcher pre-flight (commit 8d45e0d on the foxhop
side). 65 of 87 rDLQ cells got escalated as class=unknown instead of a
properly named bucket. Adds pattern + escalate-class entry + reducer
test case mapped to (vram-oversized sim no — bin is dead weight on this
card, salvage skipped).
(2) sweep-doctrine reducers had no per-test timeout. K=5 doctrine tests
(test-k5-apply-forward-ipmul + 4 siblings) ran lumbda at 99% CPU for
2h43m on a remote node without ever emitting their DOCTRINE verdict
line — accumulating 30+ runaway lumbda procs under two stuck `make
sweep-doctrine` invocations. run.sh + run-parallel.sh now wrap our
lumbda invocation in `timeout ${SWEEP_DOCTRINE_TEST_TIMEOUT_S:-300}`;
hit exits 124, our existing "no DOCTRINE line" branch logs HARNESS-FAIL.
K=5 substrate has a documented non-terminating compute defect AND a
load-time buffer overflow (commit 6d18c59 on foxhop). Bisect deferred
per ticket 0007 in foxhop tree; needs qemu apparatus we currently lack.
This commit is contained in:
parent
e547b54750
commit
823e8da1ff
4 changed files with 24 additions and 4 deletions
|
|
@ -94,6 +94,12 @@ classify_reason() {
|
|||
if grep -q "vram-budget-refused" "$r" 2>/dev/null; then
|
||||
echo "vram-budget-refused sim yes"; return
|
||||
fi
|
||||
# Structural reject (dispatcher pre-flight 2026-06-14): predicted VRAM
|
||||
# exceeds card capacity. Cell cannot ever fit. Bin is dead weight;
|
||||
# salvage=no since retry on same card is futile.
|
||||
if grep -q "vram-oversized" "$r" 2>/dev/null; then
|
||||
echo "vram-oversized sim no"; return
|
||||
fi
|
||||
if grep -q "bin-load-fail" "$r" 2>/dev/null; then
|
||||
echo "bin-load-fail emit no"; return
|
||||
fi
|
||||
|
|
@ -138,7 +144,7 @@ is_auto_class() {
|
|||
# Escalate-on-first-fail classes — no point in auto-retrying.
|
||||
is_escalate_class() {
|
||||
case "$1" in
|
||||
cuda-oom|cuda-illegal-addr|memory-cap-refused|vram-budget-refused|bin-load-fail|tier-classify|emit-broken|unknown) return 0 ;;
|
||||
cuda-oom|cuda-illegal-addr|memory-cap-refused|vram-budget-refused|vram-oversized|bin-load-fail|tier-classify|emit-broken|unknown) return 0 ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
|
|
|||
|
|
@ -88,6 +88,11 @@ assert_classify "vram-budget-refused (Phase 8 fast-fail)" \
|
|||
'(cuda-sim-error (code vram-budget-refused) (reason "predicted 5.4 GB > free 1.2 GB at sim_gpu_packed_alloc; reduce n_batches"))' \
|
||||
"vram-budget-refused" "sim" "yes"
|
||||
|
||||
assert_classify "vram-oversized (dispatcher pre-flight reject 2026-06-14)" \
|
||||
'tier=vram-oversized bin_bytes=4002302992 bin_mib=3816 vram_need_mib=46304 vram_total_mib=24576
|
||||
reason: vram-oversized — predicted 46304MiB exceeds card capacity 24576MiB. Cell cannot fit.' \
|
||||
"vram-oversized" "sim" "no"
|
||||
|
||||
assert_classify "bin-load-fail" \
|
||||
'(cuda-sim-error (code bin-load-fail) (reason "load_ops_bin returned non-zero"))' \
|
||||
"bin-load-fail" "emit" "no"
|
||||
|
|
@ -161,7 +166,7 @@ for c in missing-bin bisect-pool-race cuda-error-transient no-portal transient-e
|
|||
fi
|
||||
done
|
||||
|
||||
for c in cuda-oom cuda-illegal-addr memory-cap-refused vram-budget-refused bin-load-fail tier-classify emit-broken unknown; do
|
||||
for c in cuda-oom cuda-illegal-addr memory-cap-refused vram-budget-refused vram-oversized bin-load-fail tier-classify emit-broken unknown; do
|
||||
if is_escalate_class "$c" && ! is_auto_class "$c"; then
|
||||
pass "$c is escalate (not auto)"
|
||||
else
|
||||
|
|
|
|||
|
|
@ -99,7 +99,10 @@ run_one() {
|
|||
local name rel out cmd
|
||||
name="$(basename "$t" .lsp)"
|
||||
rel="tests/sweep-doctrine/${name}.lsp"
|
||||
cmd="cd $repo_root; $LUMBDA_BIN --fast $rel 2>&1"
|
||||
# Per-test timeout (2026-06-14): see run.sh for rationale. Non-terminating
|
||||
# reducers fail-fast (timeout exit 124, no DOCTRINE line, runner logs
|
||||
# HARNESS-FAIL). SWEEP_DOCTRINE_TEST_TIMEOUT_S overrides default 300s.
|
||||
cmd="cd $repo_root; timeout ${SWEEP_DOCTRINE_TEST_TIMEOUT_S:-300} $LUMBDA_BIN --fast $rel 2>&1"
|
||||
if [ -n "$REMOTE" ]; then
|
||||
out=$(ssh "$REMOTE" "$cmd" 2>&1 | grep -E "^DOCTRINE ")
|
||||
else
|
||||
|
|
|
|||
|
|
@ -88,7 +88,13 @@ run_one_local_or_remote() {
|
|||
local name rel out cmd
|
||||
name="$(basename "$t" .lsp)"
|
||||
rel="tests/sweep-doctrine/${name}.lsp"
|
||||
cmd="cd $repo_root; $LUMBDA_BIN --fast $rel 2>&1"
|
||||
# Per-test timeout (2026-06-14 defect fix): a doctrine reducer that does
|
||||
# not terminate should fail-fast, not leak compute hours. K=5 doctrine
|
||||
# tests on raw-pa substrate exhausted CPU + accumulated lumbda procs on
|
||||
# remote until manual SIGTERM. SWEEP_DOCTRINE_TEST_TIMEOUT_S sets per-
|
||||
# test wall (default 300s); timeout exits 124 on hit and run.sh logs
|
||||
# HARNESS-FAIL via the "no DOCTRINE line" path.
|
||||
cmd="cd $repo_root; timeout ${SWEEP_DOCTRINE_TEST_TIMEOUT_S:-300} $LUMBDA_BIN --fast $rel 2>&1"
|
||||
if [ -n "$REMOTE" ]; then
|
||||
out=$(ssh "$REMOTE" "$cmd" 2>&1 | grep -E "^DOCTRINE ")
|
||||
else
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue