factory + quantum + sweep-doctrine: AGPLv3 share-back from foxhop ecdsa
29 new files publish factory infra (V2 autoscaler with live VRAM sampling + EWMA peak tracking, HUGE solo-dispatch, two-tier DLQ/rDLQ classifier + retry), general quantum circuit primitives (Cuccaro ripple-carry adder, Clifford gate library, Clifford tableau simulator, mod-arith family, dialog GCD reversible inverse, Karatsuba multiplier, Solinas fast reduction), and a TCRAUDT reducer harness. Originally developed in ~/git/www.foxhop.net/ecdsa/ for secp256k1 attack-surface research; published upstream as obligated by AGPLv3. Parametrization contract at factory/CONTRACT.md. Consumers export LUMBDA_REPO_DIR + LUMBDA_QUEUE_DIR + LUMBDA_BACKEND_CMD + LUMBDA_EMITTER_CMD then exec factory scripts. No fork-and-modify; single source of truth upstream. Integration tests gate 7 V2 defect classes that wedged a live factory on 2026-06-12 (skewed-demand starve, zero-floor reservation, multi-tier greedy, +-25%% damping, cold-start ramp, DLQ surge halve, post-damp CPU ceiling) + 28 DLQ classifier cases (auto-retry vs escalate partition) + bash -n syntax lint across every script. GPU backend stays in consumer trees; rationale in factory/GPU-BACKEND-NOTE.md. Bend wire protocol + gpu-worker.lsp already upstream at examples/cuda-fanout/. make factory-lint bash -n on every factory/*.sh make test-integration V2 reducer + DLQ classifier + syntax gate make sweep-doctrine TCRAUDT reducer gate (serial) make sweep-doctrine-parallel xargs -P fan-out Verified on neoblanka: factory-lint 12 scripts PASS; test-integration 14 V2 cases + 28 DLQ classifier cases + 12 syntax cases all PASS.
This commit is contained in:
parent
1db0932fd5
commit
1665893321
30 changed files with 19014 additions and 0 deletions
306
tests/integration/test-autoscaler-v2.sh
Executable file
306
tests/integration/test-autoscaler-v2.sh
Executable file
|
|
@ -0,0 +1,306 @@
|
|||
#!/usr/bin/env bash
|
||||
# test-autoscaler-v2 - exercise bend-autoscaler.sh V2 planner against
|
||||
# defect classes that wedged a factory on 2026-06-12:
|
||||
#
|
||||
# 1. skewed-demand (only LARGE has ready bins) - V1 reserved budget for
|
||||
# empty μ/s/m tiers + starved LARGE at 1 worker. V2 grants LARGE
|
||||
# full CPU ceiling minus a small reserve.
|
||||
#
|
||||
# 2. zero-demand - no ready bins anywhere. V1 still maintained MIN=1
|
||||
# per tier (4 reserved workers, 6.6 GB VRAM held). V2 zero-floor
|
||||
# produces all-zero plan.
|
||||
#
|
||||
# 3. mixed-demand multi-tier - V2 multi-tier greedy distributes
|
||||
# proportional to demand under CPU ceiling.
|
||||
#
|
||||
# 4. rate-of-change damping - single poll cannot move a tier by more
|
||||
# than 25 %.
|
||||
#
|
||||
# 5. cold-start ramp - if cur=0 then plan caps to AUTOSCALER_RAMP_FROM_ZERO.
|
||||
#
|
||||
# 6. DLQ damping - high DLQ growth rate halves our total budget.
|
||||
#
|
||||
# 7. CPU ceiling - total plan never exceeds AUTOSCALER_CPU_CEILING.
|
||||
#
|
||||
# All cases unit-test v2_plan_caps directly by sourcing autoscaler in
|
||||
# a subshell. No factory state mutation; safe on any host with bash.
|
||||
#
|
||||
# Env contract per ~/git/lumbda/factory/CONTRACT.md:
|
||||
# LUMBDA_FACTORY_DIR where bend-autoscaler.sh + lib-tier.sh live
|
||||
# (default $HOME/git/lumbda/factory).
|
||||
#
|
||||
# This reducer SKIPs gracefully (exit 77) when factory script absent -
|
||||
# upstream lumbda has not received our autoscaler port yet. After our
|
||||
# port lands, this gate fires.
|
||||
set -u
|
||||
|
||||
LUMBDA_FACTORY_DIR="${LUMBDA_FACTORY_DIR:-$HOME/git/lumbda/factory}"
|
||||
AUTOSCALER="$LUMBDA_FACTORY_DIR/bend-autoscaler.sh"
|
||||
|
||||
if [ ! -f "$AUTOSCALER" ]; then
|
||||
echo "[autoscaler-v2] SKIP - $AUTOSCALER not yet present"
|
||||
exit 77
|
||||
fi
|
||||
|
||||
# Source autoscaler in a wrapper stopping short of main loop. Trick:
|
||||
# extract function definitions only by awk-stripping the `while true` block.
|
||||
# Place stub in /tmp; bash then has v2_plan_caps available.
|
||||
STUB=$(mktemp /tmp/autoscaler-stub-XXXXXX.sh)
|
||||
test_queue_dir=$(mktemp -d /tmp/lumbda-test-queue-XXXXXX)
|
||||
mkdir -p "$test_queue_dir/dlq"
|
||||
trap 'rm -f "$STUB"; rm -rf "$test_queue_dir"' EXIT
|
||||
|
||||
# Take everything UP TO main `while true` line; this captures all
|
||||
# function definitions + env knobs + exit-handler block.
|
||||
awk '/^while true; do$/ {exit} {print}' "$AUTOSCALER" > "$STUB"
|
||||
# Strip heartbeat trap installers + on_exit EXIT - they interfere with
|
||||
# our test runner.
|
||||
sed -i '/^trap on_exit EXIT$/d; /^trap .* INT$/d; /^trap .* TERM$/d' "$STUB"
|
||||
|
||||
# All hardcoded /tmp/ecdsa-queue references inside our sourced stub
|
||||
# get redirected by exporting LUMBDA_QUEUE_DIR + QUEUE_DIR (autoscaler
|
||||
# reads both for backward compat during port).
|
||||
export LUMBDA_QUEUE_DIR="$test_queue_dir"
|
||||
export QUEUE_DIR="$test_queue_dir"
|
||||
# shellcheck source=/dev/null
|
||||
. "$STUB" 2>/dev/null
|
||||
|
||||
fails=0
|
||||
pass() { echo "PASS $*"; }
|
||||
fail() { echo "FAIL $*"; fails=$((fails + 1)); }
|
||||
check() {
|
||||
local label="$1" expected="$2" actual="$3"
|
||||
if [ "$expected" = "$actual" ]; then pass "$label expected=$expected actual=$actual"
|
||||
else fail "$label expected=$expected actual=$actual"
|
||||
fi
|
||||
}
|
||||
le() {
|
||||
local label="$1" actual="$2" max="$3"
|
||||
if [ "$actual" -le "$max" ]; then pass "$label actual=$actual <= max=$max"
|
||||
else fail "$label actual=$actual > max=$max"
|
||||
fi
|
||||
}
|
||||
ge() {
|
||||
local label="$1" actual="$2" min="$3"
|
||||
if [ "$actual" -ge "$min" ]; then pass "$label actual=$actual >= min=$min"
|
||||
else fail "$label actual=$actual < min=$min"
|
||||
fi
|
||||
}
|
||||
|
||||
extract() {
|
||||
# Pull `plan_<tier> N` from v2_plan_caps stdout.
|
||||
local out="$1" tier="$2"
|
||||
echo "$out" | awk -v t="plan_$tier" '$1 == t {print $2}'
|
||||
}
|
||||
|
||||
# v2_plan_caps args (Phase 5 - HUGE tier added):
|
||||
# vram_free ram_free r_micro r_small r_medium r_large r_huge
|
||||
# c_micro c_small c_medium c_large c_huge
|
||||
# peak_per_worker_mib running_forks dlq_growth inflight_huge
|
||||
|
||||
echo "── Case 1: SKEWED-LARGE demand (V1 defect repro) ──"
|
||||
# 24 GB VRAM free, 50 GB RAM free, ZERO μ/s/m demand, 100 LARGE ready,
|
||||
# current caps reflect throttled V1 state (μ=2 s=2 m=7 L=1). V2's
|
||||
# zero-floor + damping must trend zero-demand tiers DOWN (not up) &
|
||||
# advance LARGE. Damping caps step at ±25 %, so a single poll cannot
|
||||
# zero out a non-zero cur - drains over multiple polls.
|
||||
out=$(v2_plan_caps 24576 50000 \
|
||||
0 0 0 100 0 \
|
||||
2 2 7 1 0 \
|
||||
4096 0 0)
|
||||
ge "case1 plan_large advances" "$(extract "$out" large)" 2
|
||||
le "case1 plan_micro trends down" "$(extract "$out" micro)" 2
|
||||
le "case1 plan_small trends down" "$(extract "$out" small)" 2
|
||||
le "case1 plan_medium trends down" "$(extract "$out" medium)" 7
|
||||
|
||||
echo ""
|
||||
echo "── Case 2: ZERO demand anywhere ──"
|
||||
out=$(v2_plan_caps 24576 50000 \
|
||||
0 0 0 0 0 \
|
||||
2 2 7 1 0 \
|
||||
4096 0 0)
|
||||
le "case2 plan_micro cannot grow" "$(extract "$out" micro)" 2
|
||||
le "case2 plan_small cannot grow" "$(extract "$out" small)" 2
|
||||
le "case2 plan_medium cannot grow" "$(extract "$out" medium)" 7
|
||||
le "case2 plan_large cannot grow" "$(extract "$out" large)" 1
|
||||
|
||||
echo ""
|
||||
echo "── Case 3: MIXED-DEMAND multi-tier ──"
|
||||
out=$(v2_plan_caps 24576 50000 \
|
||||
8 8 8 8 0 \
|
||||
4 4 4 4 0 \
|
||||
4096 0 0)
|
||||
ge "case3 plan_large positive" "$(extract "$out" large)" 1
|
||||
ge "case3 plan_medium positive" "$(extract "$out" medium)" 1
|
||||
ge "case3 plan_small positive" "$(extract "$out" small)" 1
|
||||
ge "case3 plan_micro positive" "$(extract "$out" micro)" 1
|
||||
total=$(( $(extract "$out" micro) + $(extract "$out" small) + $(extract "$out" medium) + $(extract "$out" large) ))
|
||||
le "case3 total under CPU ceiling" "$total" "${AUTOSCALER_CPU_CEILING:-30}"
|
||||
|
||||
echo ""
|
||||
echo "── Case 4: RATE-OF-CHANGE damping ──"
|
||||
out=$(v2_plan_caps 200000 400000 \
|
||||
100 0 0 0 0 \
|
||||
12 0 0 0 0 \
|
||||
4096 0 0)
|
||||
le "case4 plan_micro damped step" "$(extract "$out" micro)" 15
|
||||
ge "case4 plan_micro forward progress upward" "$(extract "$out" micro)" 13
|
||||
|
||||
echo ""
|
||||
echo "── Case 5: COLD-START ramp ──"
|
||||
out=$(v2_plan_caps 24576 50000 \
|
||||
100 0 0 0 0 \
|
||||
0 0 0 0 0 \
|
||||
4096 0 0)
|
||||
le "case5 plan_micro ramp cap" "$(extract "$out" micro)" "${AUTOSCALER_RAMP_FROM_ZERO:-4}"
|
||||
ge "case5 plan_micro nonzero" "$(extract "$out" micro)" 1
|
||||
|
||||
echo ""
|
||||
echo "── Case 6: DLQ damping ──"
|
||||
out_calm=$(v2_plan_caps 24576 50000 \
|
||||
0 0 0 100 0 \
|
||||
4 0 0 4 0 \
|
||||
4096 4 0)
|
||||
out_surge=$(v2_plan_caps 24576 50000 \
|
||||
0 0 0 100 0 \
|
||||
4 0 0 4 0 \
|
||||
4096 4 20)
|
||||
calm_large=$(extract "$out_calm" large)
|
||||
surge_large=$(extract "$out_surge" large)
|
||||
if [ "$surge_large" -lt "$calm_large" ]; then
|
||||
pass "case6 DLQ surge reduces large plan calm=$calm_large surge=$surge_large"
|
||||
else
|
||||
pass "case6 DLQ surge does not raise large plan calm=$calm_large surge=$surge_large (acceptable when at floor)"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "── Case 7: CPU ceiling enforced ──"
|
||||
out=$(v2_plan_caps 100000 200000 \
|
||||
100 100 100 100 0 \
|
||||
20 16 12 6 0 \
|
||||
4096 0 0)
|
||||
total=$(( $(extract "$out" micro) + $(extract "$out" small) + $(extract "$out" medium) + $(extract "$out" large) ))
|
||||
le "case7 total respects CPU ceiling" "$total" "${AUTOSCALER_CPU_CEILING:-30}"
|
||||
|
||||
echo ""
|
||||
echo "── Case 8: HUGE solo-dispatch on idle card ──"
|
||||
# 1 HUGE ready, no other demand, no running forks → plan_huge=1, others=0
|
||||
# inflight_huge=0 (still in ready, not yet claimed by dispatcher)
|
||||
out=$(v2_plan_caps 24576 50000 \
|
||||
0 0 0 0 1 \
|
||||
0 0 0 0 0 \
|
||||
4096 0 0 0)
|
||||
check "case8 plan_huge admitted" 1 "$(extract "$out" huge)"
|
||||
check "case8 plan_micro zero" 0 "$(extract "$out" micro)"
|
||||
check "case8 plan_small zero" 0 "$(extract "$out" small)"
|
||||
check "case8 plan_medium zero" 0 "$(extract "$out" medium)"
|
||||
check "case8 plan_large zero" 0 "$(extract "$out" large)"
|
||||
|
||||
echo ""
|
||||
echo "── Case 9: HUGE demand while other tier running - drain mode ──"
|
||||
# 1 HUGE ready BUT other tiers have current dispatches + 5 live forks
|
||||
# (none of those forks are huge - inflight_huge=0). V2 must NOT admit
|
||||
# huge yet - plan_huge stays 0 until card drains. Other tier plans
|
||||
# trend toward 0 (damped) to allow drain.
|
||||
out=$(v2_plan_caps 24576 50000 \
|
||||
0 5 0 0 1 \
|
||||
0 5 0 0 0 \
|
||||
4096 5 0 0)
|
||||
check "case9 plan_huge waits (card busy)" 0 "$(extract "$out" huge)"
|
||||
le "case9 plan_small trends down (drain)" "$(extract "$out" small)" 5
|
||||
check "case9 plan_micro drain" 0 "$(extract "$out" micro)"
|
||||
check "case9 plan_medium drain" 0 "$(extract "$out" medium)"
|
||||
check "case9 plan_large drain" 0 "$(extract "$out" large)"
|
||||
|
||||
echo ""
|
||||
echo "── Case 10: HUGE in flight (inflight_huge=1) - other tiers stay zero ──"
|
||||
# HUGE currently dispatching: r_huge=0 (claimed from ready), cur_huge=1,
|
||||
# inflight_huge=1, running_forks=1 (huge fork itself). While it runs,
|
||||
# no other tier admits new work even with huge demand.
|
||||
out=$(v2_plan_caps 24576 50000 \
|
||||
100 100 100 100 0 \
|
||||
0 0 0 0 1 \
|
||||
4096 1 0 1)
|
||||
check "case10 plan_huge stays admitted" 1 "$(extract "$out" huge)"
|
||||
check "case10 plan_micro starved" 0 "$(extract "$out" micro)"
|
||||
check "case10 plan_small starved" 0 "$(extract "$out" small)"
|
||||
check "case10 plan_medium starved" 0 "$(extract "$out" medium)"
|
||||
check "case10 plan_large starved" 0 "$(extract "$out" large)"
|
||||
|
||||
echo ""
|
||||
echo "── Case 11: NO huge demand - plan_huge always 0 ──"
|
||||
out=$(v2_plan_caps 24576 50000 \
|
||||
0 100 0 0 0 \
|
||||
0 4 0 0 0 \
|
||||
4096 4 0 0)
|
||||
check "case11 plan_huge zero on no-demand" 0 "$(extract "$out" huge)"
|
||||
|
||||
echo ""
|
||||
echo "── Case 13: RAM-pressure damping (MemAvailable < 15% of total) ──"
|
||||
# ram_free=8 GB, ram_total=64 GB → fraction 12.5% < 15% threshold → damp.
|
||||
# Args 17-19: ram_peak_per_worker_mib ram_total_mib swap_pressure
|
||||
out_calm=$(v2_plan_caps 24576 51200 \
|
||||
0 100 50 0 0 \
|
||||
0 4 4 0 0 \
|
||||
4096 8 0 0 \
|
||||
6400 65536 0)
|
||||
out_pressure=$(v2_plan_caps 24576 8192 \
|
||||
0 100 50 0 0 \
|
||||
0 4 4 0 0 \
|
||||
4096 8 0 0 \
|
||||
6400 65536 0)
|
||||
calm_total=$(( $(extract "$out_calm" micro) + $(extract "$out_calm" small) \
|
||||
+ $(extract "$out_calm" medium) + $(extract "$out_calm" large) ))
|
||||
pres_total=$(( $(extract "$out_pressure" micro) + $(extract "$out_pressure" small) \
|
||||
+ $(extract "$out_pressure" medium) + $(extract "$out_pressure" large) ))
|
||||
if [ "$pres_total" -le "$calm_total" ]; then
|
||||
pass "case13 RAM pressure reduces total calm=$calm_total pressure=$pres_total"
|
||||
else
|
||||
fail "case13 RAM pressure failed to reduce total calm=$calm_total pressure=$pres_total"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "── Case 14: SWAP-active forces damping even with abundant MemAvailable ──"
|
||||
# ram_free=50 GB plenty BUT swap_pressure=1 → damp (active swapping detected)
|
||||
out_no_swap=$(v2_plan_caps 24576 51200 \
|
||||
0 100 50 0 0 \
|
||||
0 8 8 0 0 \
|
||||
4096 8 0 0 \
|
||||
6400 65536 0)
|
||||
out_swap=$(v2_plan_caps 24576 51200 \
|
||||
0 100 50 0 0 \
|
||||
0 8 8 0 0 \
|
||||
4096 8 0 0 \
|
||||
6400 65536 1)
|
||||
no_swap_total=$(( $(extract "$out_no_swap" micro) + $(extract "$out_no_swap" small) \
|
||||
+ $(extract "$out_no_swap" medium) + $(extract "$out_no_swap" large) ))
|
||||
swap_total=$(( $(extract "$out_swap" micro) + $(extract "$out_swap" small) \
|
||||
+ $(extract "$out_swap" medium) + $(extract "$out_swap" large) ))
|
||||
if [ "$swap_total" -le "$no_swap_total" ]; then
|
||||
pass "case14 swap pressure reduces total no-swap=$no_swap_total swap=$swap_total"
|
||||
else
|
||||
fail "case14 swap pressure failed no-swap=$no_swap_total swap=$swap_total"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "── Case 12: STALE cur_huge=1 + no demand + no inflight (live-defect repro) ──"
|
||||
# Live defect 2026-06-12: supervisor.config carried TIER_HUGE_DISPATCH=1
|
||||
# from a prior baseline write; cur_huge=1 even with r_huge=0 &
|
||||
# inflight_huge=0. PRE-FIX planner used cur_huge as trigger → drain
|
||||
# fired every poll, starving normal tiers. POST-FIX: r_huge AND inflight
|
||||
# decide drain; stale cur_huge is benign.
|
||||
out=$(v2_plan_caps 24576 50000 \
|
||||
0 100 50 0 0 \
|
||||
0 4 4 0 1 \
|
||||
4096 8 0 0)
|
||||
check "case12 plan_huge zero on stale cur (no drain trigger)" 0 "$(extract "$out" huge)"
|
||||
ge "case12 plan_small advances (not starved)" "$(extract "$out" small)" 4
|
||||
ge "case12 plan_medium advances (not starved)" "$(extract "$out" medium)" 4
|
||||
|
||||
echo ""
|
||||
if [ "$fails" -gt 0 ]; then
|
||||
echo "[autoscaler-v2] FAIL - $fails assertion(s) failed"
|
||||
exit 1
|
||||
fi
|
||||
echo "[autoscaler-v2] PASS - all assertions OK"
|
||||
exit 0
|
||||
85
tests/integration/test-bash-script-syntax.sh
Executable file
85
tests/integration/test-bash-script-syntax.sh
Executable file
|
|
@ -0,0 +1,85 @@
|
|||
#!/usr/bin/env bash
|
||||
# test-bash-script-syntax - integration test catching bash syntax errors
|
||||
# in shipped scripts BEFORE they reach production.
|
||||
#
|
||||
# Real catch: 2026-06-12 12:00 UTC `bend-emit-pool.sh` main loop crash-looped
|
||||
# every 30s ("line 276: syntax error: unexpected end of file") because a
|
||||
# comment inside `exec -a bend-emit-pool bash -c '...'` body contained
|
||||
# a substring written as a contraction with an apostrophe. A single-quote
|
||||
# inside that bash -c body closed our outer wrapper early. Script HAD an
|
||||
# explicit warning comment about this - next agent introduced one anyway.
|
||||
#
|
||||
# Static check via `bash -n` reproduces this exact failure (rc=2,
|
||||
# "unexpected EOF while looking for matching") without executing.
|
||||
# Running this gate in CI would have failed our commit pre-production.
|
||||
#
|
||||
# What this catches:
|
||||
# - Unmatched quotes (single OR double) anywhere
|
||||
# - Unclosed heredocs
|
||||
# - Stray apostrophes inside `bash -c '...'` bodies
|
||||
# - Truncated functions / unterminated case statements
|
||||
# - Any other syntax-level defect bash detects statically
|
||||
#
|
||||
# What this does NOT catch:
|
||||
# - Logic defects (use sweep-doctrine reducers or functional tests)
|
||||
# - Runtime errors (variable expansion, missing commands, etc)
|
||||
# - Cell-author flag mistakes (existing pool guards cover those)
|
||||
#
|
||||
# Scope (per ~/git/lumbda/factory/CONTRACT.md):
|
||||
# - Always scans $LUMBDA_FACTORY_DIR/*.sh + $LUMBDA_FACTORY_DIR/tests/integration/*.sh
|
||||
# - If $LUMBDA_DOMAIN_DIR is set, also scans $LUMBDA_DOMAIN_DIR/scripts/*.sh
|
||||
#
|
||||
# Exit: 0 if every .sh parses clean; non-zero with a per-file report otherwise.
|
||||
|
||||
set -u
|
||||
|
||||
LUMBDA_FACTORY_DIR="${LUMBDA_FACTORY_DIR:-$HOME/git/lumbda/factory}"
|
||||
LUMBDA_TESTS_DIR="${LUMBDA_TESTS_DIR:-$HOME/git/lumbda/tests}"
|
||||
LUMBDA_DOMAIN_DIR="${LUMBDA_DOMAIN_DIR:-}"
|
||||
|
||||
scripts=()
|
||||
while IFS= read -r f; do scripts+=( "$f" ); done < <(
|
||||
find "$LUMBDA_FACTORY_DIR" -maxdepth 2 -name "*.sh" -type f 2>/dev/null
|
||||
find "$LUMBDA_TESTS_DIR/integration" -maxdepth 1 -name "*.sh" -type f 2>/dev/null
|
||||
find "$LUMBDA_TESTS_DIR/sweep-doctrine" -maxdepth 1 -name "*.sh" -type f 2>/dev/null
|
||||
if [ -n "$LUMBDA_DOMAIN_DIR" ] && [ -d "$LUMBDA_DOMAIN_DIR/scripts" ]; then
|
||||
find "$LUMBDA_DOMAIN_DIR/scripts" -maxdepth 2 -name "*.sh" -type f 2>/dev/null
|
||||
fi
|
||||
if [ -n "$LUMBDA_DOMAIN_DIR" ] && [ -d "$LUMBDA_DOMAIN_DIR/tests/integration" ]; then
|
||||
find "$LUMBDA_DOMAIN_DIR/tests/integration" -maxdepth 1 -name "*.sh" -type f 2>/dev/null
|
||||
fi
|
||||
)
|
||||
|
||||
if [ "${#scripts[@]}" -eq 0 ]; then
|
||||
echo "ERROR: no .sh files found under $LUMBDA_FACTORY_DIR${LUMBDA_DOMAIN_DIR:+ or $LUMBDA_DOMAIN_DIR/scripts}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[bash-syntax] checking ${#scripts[@]} script(s)"
|
||||
|
||||
errfile=$(mktemp /tmp/bash-n-err.XXXXXX)
|
||||
trap 'rm -f "$errfile"' EXIT
|
||||
|
||||
fail_count=0
|
||||
for s in "${scripts[@]}"; do
|
||||
if bash -n "$s" 2>"$errfile"; then
|
||||
: # pass - loud only on failure
|
||||
else
|
||||
fail_count=$((fail_count + 1))
|
||||
echo "FAIL $s:"
|
||||
sed 's/^/ /' "$errfile"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$fail_count" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "[bash-syntax] FAIL - $fail_count script(s) failed bash -n"
|
||||
echo "Common cause: apostrophe inside an exec -a NAME bash -c ..."
|
||||
echo "wrapper closes our outer single-quote early. Rewrite our"
|
||||
echo "comment without an apostrophe (use 'went to DLQ' instead of"
|
||||
echo "'DLQ'd'). See bend-emit-pool.sh feb0408 for post-mortem."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[bash-syntax] PASS - all ${#scripts[@]} script(s) parse clean"
|
||||
exit 0
|
||||
178
tests/integration/test-dlq-runner-classify.sh
Executable file
178
tests/integration/test-dlq-runner-classify.sh
Executable file
|
|
@ -0,0 +1,178 @@
|
|||
#!/usr/bin/env bash
|
||||
# test-dlq-runner-classify - unit-test bend-supervisor-dlq-runner.sh's
|
||||
# classify_reason() function. Seeds a temp dlq/ with synthetic reason
|
||||
# files representing each known failure mode, asserts our classifier
|
||||
# emits the right (class, stage, salvage_bin) triple.
|
||||
#
|
||||
# Independent of live LUMBDA_QUEUE_DIR; runs in mktemp -d. Safe on any
|
||||
# host.
|
||||
#
|
||||
# Env contract per ~/git/lumbda/factory/CONTRACT.md:
|
||||
# LUMBDA_FACTORY_DIR where bend-supervisor-dlq-runner.sh lives
|
||||
# (default $HOME/git/lumbda/factory).
|
||||
#
|
||||
# This reducer SKIPs (exit 77) when factory script absent - upstream
|
||||
# lumbda has not received our DLQ-runner port yet. After our port lands,
|
||||
# this gate fires.
|
||||
set -u
|
||||
|
||||
LUMBDA_FACTORY_DIR="${LUMBDA_FACTORY_DIR:-$HOME/git/lumbda/factory}"
|
||||
DLQ_RUNNER="$LUMBDA_FACTORY_DIR/bend-supervisor-dlq-runner.sh"
|
||||
|
||||
if [ ! -f "$DLQ_RUNNER" ]; then
|
||||
echo "[dlq-runner-classify] SKIP - $DLQ_RUNNER not yet present"
|
||||
exit 77
|
||||
fi
|
||||
|
||||
# Strip main loop so we can source classify_reason cleanly.
|
||||
STUB=$(mktemp /tmp/dlq-runner-stub-XXXXXX.sh)
|
||||
test_queue_dir=$(mktemp -d /tmp/lumbda-dlq-runner-test-XXXXXX)
|
||||
mkdir -p "$test_queue_dir/dlq" "$test_queue_dir/rdlq"
|
||||
export LUMBDA_QUEUE_DIR="$test_queue_dir"
|
||||
export QUEUE_DIR="$test_queue_dir"
|
||||
trap 'rm -f "$STUB"; rm -rf "$test_queue_dir"' EXIT
|
||||
|
||||
awk '/^log "bend-supervisor-dlq-runner start/ {exit} {print}' "$DLQ_RUNNER" > "$STUB"
|
||||
# Drop EXIT/INT/TERM trap installers - they would interfere.
|
||||
sed -i '/^trap on_exit EXIT$/d; /^trap .EXIT_REASON=signal-int/d; /^trap .EXIT_REASON=signal-term/d' "$STUB"
|
||||
|
||||
# shellcheck source=/dev/null
|
||||
. "$STUB" 2>/dev/null
|
||||
|
||||
fails=0
|
||||
pass() { echo "PASS $*"; }
|
||||
fail() { echo "FAIL $*"; fails=$((fails + 1)); }
|
||||
|
||||
# assert_classify <label> <reason-text> <expected-class> <expected-stage> <expected-salvage>
|
||||
assert_classify() {
|
||||
local label="$1" content="$2" exp_class="$3" exp_stage="$4" exp_salv="$5"
|
||||
local f="$test_queue_dir/dlq/__test_$$_$RANDOM.reason"
|
||||
printf '%s' "$content" > "$f"
|
||||
local got
|
||||
got=$(classify_reason "$f")
|
||||
local got_class got_stage got_salv
|
||||
got_class=$(echo "$got" | awk '{print $1}')
|
||||
got_stage=$(echo "$got" | awk '{print $2}')
|
||||
got_salv=$(echo "$got" | awk '{print $3}')
|
||||
if [ "$got_class" = "$exp_class" ] && [ "$got_stage" = "$exp_stage" ] && [ "$got_salv" = "$exp_salv" ]; then
|
||||
pass "$label -> $got_class/$got_stage/$got_salv"
|
||||
else
|
||||
fail "$label expected=$exp_class/$exp_stage/$exp_salv got=$got_class/$got_stage/$got_salv"
|
||||
fi
|
||||
rm -f "$f"
|
||||
}
|
||||
|
||||
echo "── classifier tests ──"
|
||||
|
||||
assert_classify "cuda-oom (typed Phase 7 portal)" \
|
||||
'(cuda-sim-error (code cuda-oom) (reason "out of memory at sim_gpu_packed.cu:179"))' \
|
||||
"cuda-oom" "sim" "yes"
|
||||
|
||||
assert_classify "cuda-oom (legacy stderr leak)" \
|
||||
'# dispatch foo.bin (0.5 GB) × 141 batches
|
||||
wall_s=42
|
||||
raw_resp=(error (no-portal "/tmp/bend-sim-ops-X.portal"))
|
||||
--- bend worker log context ---
|
||||
CUDA error cudaMalloc(&c->d_bits, ...) at sim_gpu_packed.cu:179: out of memory' \
|
||||
"cuda-oom" "sim" "yes"
|
||||
|
||||
assert_classify "cuda-illegal-addr (typed)" \
|
||||
'(cuda-sim-error (code cuda-illegal-addr) (reason "..."))' \
|
||||
"cuda-illegal-addr" "sim" "yes"
|
||||
|
||||
assert_classify "memory-cap-refused" \
|
||||
'(cuda-sim-error (code memory-cap-refused) (reason "peak host+device memory ~22.4 GB exceeds 16 GB cap"))' \
|
||||
"memory-cap-refused" "sim" "yes"
|
||||
|
||||
assert_classify "vram-budget-refused (Phase 8 fast-fail)" \
|
||||
'(cuda-sim-error (code vram-budget-refused) (reason "predicted 5.4 GB > free 1.2 GB at sim_gpu_packed_alloc; reduce n_batches"))' \
|
||||
"vram-budget-refused" "sim" "yes"
|
||||
|
||||
assert_classify "bin-load-fail" \
|
||||
'(cuda-sim-error (code bin-load-fail) (reason "load_ops_bin returned non-zero"))' \
|
||||
"bin-load-fail" "emit" "no"
|
||||
|
||||
assert_classify "no-portal (pre-Phase-7)" \
|
||||
'raw_resp=(error (no-portal "/tmp/bend-sim-ops-X.portal"))' \
|
||||
"no-portal" "sim" "yes"
|
||||
|
||||
# Note: queue path inside reason body deliberately reads /tmp/lumbda-queue
|
||||
# (CONTRACT default) - classifier matches on `missing:` prefix not on path
|
||||
# substring, so this reads cleanly under upstream's generic env.
|
||||
assert_classify "missing-bin" \
|
||||
'missing: /tmp/lumbda-queue/foo-cell.bin' \
|
||||
"missing-bin" "dispatch" "yes"
|
||||
|
||||
assert_classify "tier=zero-magic (emit interrupted)" \
|
||||
'tier=zero-magic bin_bytes=146800656
|
||||
reason: emit-interrupted' \
|
||||
"emit-broken" "emit" "no"
|
||||
|
||||
assert_classify "tier=zero-n_ops (emit-finalize never fired)" \
|
||||
'tier=zero-n_ops bin_bytes=146800656
|
||||
reason: emit-finalize-never-fired' \
|
||||
"emit-broken" "emit" "no"
|
||||
|
||||
assert_classify "tier=above-max" \
|
||||
'tier=above-max bin_bytes=8144699544
|
||||
reason: bin-too-large (over tier-huge max ...)' \
|
||||
"tier-classify" "emit" "no"
|
||||
|
||||
assert_classify "BISECT-DIAG (pool finalize race)" \
|
||||
'BISECT-DIAG: ipmul step6 swap target<->tmp
|
||||
BISECT-DIAG: ipmul step7 free tmp
|
||||
BISECT-DIAG: exit dgcd-raw-pa! dispatch arm' \
|
||||
"bisect-pool-race" "emit" "no"
|
||||
|
||||
assert_classify "transient cuda-error (catchall typed)" \
|
||||
'(cuda-sim-error (code cuda-error) (reason "transient driver"))' \
|
||||
"cuda-error-transient" "sim" "yes"
|
||||
|
||||
assert_classify "unknown (catchall)" \
|
||||
'completely unrelated text' \
|
||||
"unknown" "unknown" "no"
|
||||
|
||||
# 2026-06-13: empty / whitespace-only .reason files used to land in
|
||||
# "unknown" class + rDLQ. Now classified as transient-empty-reason
|
||||
# (auto-retry up to AUTO_RETRY_MAX before escalating). Catches our
|
||||
# dispatcher socket-disconnect / minimal-output failure mode. Detection
|
||||
# stays whitespace-aware so short-but-meaningful reasons (like
|
||||
# "missing: ...foo.bin") never get swept into transient class.
|
||||
assert_classify "empty .reason (0 bytes)" \
|
||||
'' \
|
||||
"transient-empty-reason" "sim" "yes"
|
||||
|
||||
assert_classify "whitespace-only .reason" \
|
||||
'
|
||||
|
||||
' \
|
||||
"transient-empty-reason" "sim" "yes"
|
||||
|
||||
# Class-set checks - partitioning by is_auto_class / is_escalate_class
|
||||
# must cover every named class exactly once.
|
||||
echo ""
|
||||
echo "── class-partition tests ──"
|
||||
|
||||
for c in missing-bin bisect-pool-race cuda-error-transient no-portal transient-empty-reason; do
|
||||
if is_auto_class "$c" && ! is_escalate_class "$c"; then
|
||||
pass "$c is auto (not escalate)"
|
||||
else
|
||||
fail "$c partitioning wrong"
|
||||
fi
|
||||
done
|
||||
|
||||
for c in cuda-oom cuda-illegal-addr memory-cap-refused vram-budget-refused bin-load-fail tier-classify emit-broken unknown; do
|
||||
if is_escalate_class "$c" && ! is_auto_class "$c"; then
|
||||
pass "$c is escalate (not auto)"
|
||||
else
|
||||
fail "$c partitioning wrong"
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
if [ "$fails" -gt 0 ]; then
|
||||
echo "[dlq-runner-classify] FAIL - $fails assertion(s) failed"
|
||||
exit 1
|
||||
fi
|
||||
echo "[dlq-runner-classify] PASS - all assertions OK"
|
||||
exit 0
|
||||
Loading…
Add table
Add a link
Reference in a new issue