From 823e8da1ffa6c7dfe7b75c8018e8658c1257d9e9 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Sun, 14 Jun 2026 13:59:48 -0400 Subject: [PATCH] factory + sweep-doctrine: vram-oversized class + per-test timeout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two follow-up defects from 2026-06-14 factory triage: (1) DLQ runner classifier did not recognize "vram-oversized" reason text introduced by foxhop dispatcher pre-flight (commit 8d45e0d on the foxhop side). 65 of 87 rDLQ cells got escalated as class=unknown instead of a properly named bucket. Adds pattern + escalate-class entry + reducer test case mapped to (vram-oversized sim no — bin is dead weight on this card, salvage skipped). (2) sweep-doctrine reducers had no per-test timeout. K=5 doctrine tests (test-k5-apply-forward-ipmul + 4 siblings) ran lumbda at 99% CPU for 2h43m on a remote node without ever emitting their DOCTRINE verdict line — accumulating 30+ runaway lumbda procs under two stuck `make sweep-doctrine` invocations. run.sh + run-parallel.sh now wrap our lumbda invocation in `timeout ${SWEEP_DOCTRINE_TEST_TIMEOUT_S:-300}`; hit exits 124, our existing "no DOCTRINE line" branch logs HARNESS-FAIL. K=5 substrate has a documented non-terminating compute defect AND a load-time buffer overflow (commit 6d18c59 on foxhop). Bisect deferred per ticket 0007 in foxhop tree; needs qemu apparatus we currently lack. --- factory/bend-supervisor-dlq-runner.sh | 8 +++++++- tests/integration/test-dlq-runner-classify.sh | 7 ++++++- tests/sweep-doctrine/run-parallel.sh | 5 ++++- tests/sweep-doctrine/run.sh | 8 +++++++- 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/factory/bend-supervisor-dlq-runner.sh b/factory/bend-supervisor-dlq-runner.sh index aebae5a..9532da1 100755 --- a/factory/bend-supervisor-dlq-runner.sh +++ b/factory/bend-supervisor-dlq-runner.sh @@ -94,6 +94,12 @@ classify_reason() { if grep -q "vram-budget-refused" "$r" 2>/dev/null; then echo "vram-budget-refused sim yes"; return fi + # Structural reject (dispatcher pre-flight 2026-06-14): predicted VRAM + # exceeds card capacity. Cell cannot ever fit. Bin is dead weight; + # salvage=no since retry on same card is futile. + if grep -q "vram-oversized" "$r" 2>/dev/null; then + echo "vram-oversized sim no"; return + fi if grep -q "bin-load-fail" "$r" 2>/dev/null; then echo "bin-load-fail emit no"; return fi @@ -138,7 +144,7 @@ is_auto_class() { # Escalate-on-first-fail classes — no point in auto-retrying. is_escalate_class() { case "$1" in - cuda-oom|cuda-illegal-addr|memory-cap-refused|vram-budget-refused|bin-load-fail|tier-classify|emit-broken|unknown) return 0 ;; + cuda-oom|cuda-illegal-addr|memory-cap-refused|vram-budget-refused|vram-oversized|bin-load-fail|tier-classify|emit-broken|unknown) return 0 ;; *) return 1 ;; esac } diff --git a/tests/integration/test-dlq-runner-classify.sh b/tests/integration/test-dlq-runner-classify.sh index 5641c06..0585403 100755 --- a/tests/integration/test-dlq-runner-classify.sh +++ b/tests/integration/test-dlq-runner-classify.sh @@ -88,6 +88,11 @@ assert_classify "vram-budget-refused (Phase 8 fast-fail)" \ '(cuda-sim-error (code vram-budget-refused) (reason "predicted 5.4 GB > free 1.2 GB at sim_gpu_packed_alloc; reduce n_batches"))' \ "vram-budget-refused" "sim" "yes" +assert_classify "vram-oversized (dispatcher pre-flight reject 2026-06-14)" \ + 'tier=vram-oversized bin_bytes=4002302992 bin_mib=3816 vram_need_mib=46304 vram_total_mib=24576 +reason: vram-oversized — predicted 46304MiB exceeds card capacity 24576MiB. Cell cannot fit.' \ + "vram-oversized" "sim" "no" + assert_classify "bin-load-fail" \ '(cuda-sim-error (code bin-load-fail) (reason "load_ops_bin returned non-zero"))' \ "bin-load-fail" "emit" "no" @@ -161,7 +166,7 @@ for c in missing-bin bisect-pool-race cuda-error-transient no-portal transient-e fi done -for c in cuda-oom cuda-illegal-addr memory-cap-refused vram-budget-refused bin-load-fail tier-classify emit-broken unknown; do +for c in cuda-oom cuda-illegal-addr memory-cap-refused vram-budget-refused vram-oversized bin-load-fail tier-classify emit-broken unknown; do if is_escalate_class "$c" && ! is_auto_class "$c"; then pass "$c is escalate (not auto)" else diff --git a/tests/sweep-doctrine/run-parallel.sh b/tests/sweep-doctrine/run-parallel.sh index c2f0244..00466e5 100755 --- a/tests/sweep-doctrine/run-parallel.sh +++ b/tests/sweep-doctrine/run-parallel.sh @@ -99,7 +99,10 @@ run_one() { local name rel out cmd name="$(basename "$t" .lsp)" rel="tests/sweep-doctrine/${name}.lsp" - cmd="cd $repo_root; $LUMBDA_BIN --fast $rel 2>&1" + # Per-test timeout (2026-06-14): see run.sh for rationale. Non-terminating + # reducers fail-fast (timeout exit 124, no DOCTRINE line, runner logs + # HARNESS-FAIL). SWEEP_DOCTRINE_TEST_TIMEOUT_S overrides default 300s. + cmd="cd $repo_root; timeout ${SWEEP_DOCTRINE_TEST_TIMEOUT_S:-300} $LUMBDA_BIN --fast $rel 2>&1" if [ -n "$REMOTE" ]; then out=$(ssh "$REMOTE" "$cmd" 2>&1 | grep -E "^DOCTRINE ") else diff --git a/tests/sweep-doctrine/run.sh b/tests/sweep-doctrine/run.sh index 175fd90..90a189a 100755 --- a/tests/sweep-doctrine/run.sh +++ b/tests/sweep-doctrine/run.sh @@ -88,7 +88,13 @@ run_one_local_or_remote() { local name rel out cmd name="$(basename "$t" .lsp)" rel="tests/sweep-doctrine/${name}.lsp" - cmd="cd $repo_root; $LUMBDA_BIN --fast $rel 2>&1" + # Per-test timeout (2026-06-14 defect fix): a doctrine reducer that does + # not terminate should fail-fast, not leak compute hours. K=5 doctrine + # tests on raw-pa substrate exhausted CPU + accumulated lumbda procs on + # remote until manual SIGTERM. SWEEP_DOCTRINE_TEST_TIMEOUT_S sets per- + # test wall (default 300s); timeout exits 124 on hit and run.sh logs + # HARNESS-FAIL via the "no DOCTRINE line" path. + cmd="cd $repo_root; timeout ${SWEEP_DOCTRINE_TEST_TIMEOUT_S:-300} $LUMBDA_BIN --fast $rel 2>&1" if [ -n "$REMOTE" ]; then out=$(ssh "$REMOTE" "$cmd" 2>&1 | grep -E "^DOCTRINE ") else