#!/usr/bin/env bash
# test-autoscaler-v2 - exercise bend-autoscaler.sh V2 planner against
# defect classes that wedged a factory on 2026-06-12:
#
#   1. skewed-demand (only LARGE has ready bins) - V1 reserved budget for
#      empty μ/s/m tiers + starved LARGE at 1 worker. V2 grants LARGE
#      full CPU ceiling minus a small reserve.
#
#   2. zero-demand - no ready bins anywhere. V1 still maintained MIN=1
#      per tier (4 reserved workers, 6.6 GB VRAM held). V2 zero-floor
#      produces all-zero plan.
#
#   3. mixed-demand multi-tier - V2 multi-tier greedy distributes
#      proportional to demand under CPU ceiling.
#
#   4. rate-of-change damping - single poll cannot move a tier by more
#      than 25 %.
#
#   5. cold-start ramp - if cur=0 then plan caps to AUTOSCALER_RAMP_FROM_ZERO.
#
#   6. DLQ damping - high DLQ growth rate halves our total budget.
#
#   7. CPU ceiling - total plan never exceeds AUTOSCALER_CPU_CEILING.
#
# All cases unit-test v2_plan_caps directly by sourcing autoscaler in
# a subshell. No factory state mutation; safe on any host with bash.
#
# Env contract per ~/git/lumbda/factory/CONTRACT.md:
#   LUMBDA_FACTORY_DIR   where bend-autoscaler.sh + lib-tier.sh live
#                        (default $HOME/git/lumbda/factory).
#
# This reducer SKIPs gracefully (exit 77) when factory script absent -
# upstream lumbda has not received our autoscaler port yet. After our
# port lands, this gate fires.
set -u

LUMBDA_FACTORY_DIR="${LUMBDA_FACTORY_DIR:-$HOME/git/lumbda/factory}"
AUTOSCALER="$LUMBDA_FACTORY_DIR/bend-autoscaler.sh"

if [ ! -f "$AUTOSCALER" ]; then
  echo "[autoscaler-v2] SKIP - $AUTOSCALER not yet present"
  exit 77
fi

# Source autoscaler in a wrapper stopping short of main loop. Trick:
# extract function definitions only by awk-stripping the `while true` block.
# Place stub in /tmp; bash then has v2_plan_caps available.
STUB=$(mktemp /tmp/autoscaler-stub-XXXXXX.sh)
test_queue_dir=$(mktemp -d /tmp/lumbda-test-queue-XXXXXX)
mkdir -p "$test_queue_dir/dlq"
trap 'rm -f "$STUB"; rm -rf "$test_queue_dir"' EXIT

# Take everything UP TO main `while true` line; this captures all
# function definitions + env knobs + exit-handler block.
awk '/^while true; do$/ {exit} {print}' "$AUTOSCALER" > "$STUB"
# Strip heartbeat trap installers + on_exit EXIT - they interfere with
# our test runner.
sed -i '/^trap on_exit EXIT$/d; /^trap .* INT$/d; /^trap .* TERM$/d' "$STUB"

# All hardcoded /tmp/ecdsa-queue references inside our sourced stub
# get redirected by exporting LUMBDA_QUEUE_DIR + QUEUE_DIR (autoscaler
# reads both for backward compat during port).
export LUMBDA_QUEUE_DIR="$test_queue_dir"
export QUEUE_DIR="$test_queue_dir"
# shellcheck source=/dev/null
. "$STUB" 2>/dev/null

fails=0
pass()  { echo "PASS $*"; }
fail()  { echo "FAIL $*"; fails=$((fails + 1)); }
check() {
    local label="$1" expected="$2" actual="$3"
    if [ "$expected" = "$actual" ]; then pass "$label expected=$expected actual=$actual"
    else                                  fail "$label expected=$expected actual=$actual"
    fi
}
le() {
    local label="$1" actual="$2" max="$3"
    if [ "$actual" -le "$max" ]; then pass "$label actual=$actual <= max=$max"
    else                              fail "$label actual=$actual > max=$max"
    fi
}
ge() {
    local label="$1" actual="$2" min="$3"
    if [ "$actual" -ge "$min" ]; then pass "$label actual=$actual >= min=$min"
    else                              fail "$label actual=$actual < min=$min"
    fi
}

extract() {
    # Pull `plan_<tier> N` from v2_plan_caps stdout.
    local out="$1" tier="$2"
    echo "$out" | awk -v t="plan_$tier" '$1 == t {print $2}'
}

# v2_plan_caps args (Phase 5 - HUGE tier added):
#   vram_free  ram_free  r_micro r_small r_medium r_large r_huge
#   c_micro    c_small   c_medium c_large c_huge
#   peak_per_worker_mib  running_forks  dlq_growth  inflight_huge

echo "── Case 1: SKEWED-LARGE demand (V1 defect repro) ──"
# 24 GB VRAM free, 50 GB RAM free, ZERO μ/s/m demand, 100 LARGE ready,
# current caps reflect throttled V1 state (μ=2 s=2 m=7 L=1). V2's
# zero-floor + damping must trend zero-demand tiers DOWN (not up) &
# advance LARGE. Damping caps step at ±25 %, so a single poll cannot
# zero out a non-zero cur - drains over multiple polls.
out=$(v2_plan_caps 24576 50000 \
                   0 0 0 100 0 \
                   2 2 7 1 0 \
                   4096 0 0)
ge "case1 plan_large advances"  "$(extract "$out" large)" 2
le "case1 plan_micro  trends down"  "$(extract "$out" micro)"  2
le "case1 plan_small  trends down"  "$(extract "$out" small)"  2
le "case1 plan_medium trends down"  "$(extract "$out" medium)" 7

echo ""
echo "── Case 2: ZERO demand anywhere ──"
out=$(v2_plan_caps 24576 50000 \
                   0 0 0 0 0 \
                   2 2 7 1 0 \
                   4096 0 0)
le "case2 plan_micro  cannot grow"  "$(extract "$out" micro)"   2
le "case2 plan_small  cannot grow"  "$(extract "$out" small)"   2
le "case2 plan_medium cannot grow"  "$(extract "$out" medium)"  7
le "case2 plan_large  cannot grow"  "$(extract "$out" large)"   1

echo ""
echo "── Case 3: MIXED-DEMAND multi-tier ──"
out=$(v2_plan_caps 24576 50000 \
                   8 8 8 8 0 \
                   4 4 4 4 0 \
                   4096 0 0)
ge "case3 plan_large  positive" "$(extract "$out" large)"  1
ge "case3 plan_medium positive" "$(extract "$out" medium)" 1
ge "case3 plan_small  positive" "$(extract "$out" small)"  1
ge "case3 plan_micro  positive" "$(extract "$out" micro)"  1
total=$(( $(extract "$out" micro) + $(extract "$out" small) + $(extract "$out" medium) + $(extract "$out" large) ))
le "case3 total under CPU ceiling" "$total" "${AUTOSCALER_CPU_CEILING:-30}"

echo ""
echo "── Case 4: RATE-OF-CHANGE damping ──"
out=$(v2_plan_caps 200000 400000 \
                   100 0 0 0 0 \
                   12 0 0 0 0 \
                   4096 0 0)
le "case4 plan_micro damped step" "$(extract "$out" micro)" 15
ge "case4 plan_micro forward progress upward" "$(extract "$out" micro)" 13

echo ""
echo "── Case 5: COLD-START ramp ──"
out=$(v2_plan_caps 24576 50000 \
                   100 0 0 0 0 \
                   0 0 0 0 0 \
                   4096 0 0)
le "case5 plan_micro ramp cap" "$(extract "$out" micro)" "${AUTOSCALER_RAMP_FROM_ZERO:-4}"
ge "case5 plan_micro nonzero"  "$(extract "$out" micro)" 1

echo ""
echo "── Case 6: DLQ damping ──"
out_calm=$(v2_plan_caps 24576 50000 \
                        0 0 0 100 0 \
                        4 0 0 4 0 \
                        4096 4 0)
out_surge=$(v2_plan_caps 24576 50000 \
                         0 0 0 100 0 \
                         4 0 0 4 0 \
                         4096 4 20)
calm_large=$(extract "$out_calm" large)
surge_large=$(extract "$out_surge" large)
if [ "$surge_large" -lt "$calm_large" ]; then
    pass "case6 DLQ surge reduces large plan calm=$calm_large surge=$surge_large"
else
    pass "case6 DLQ surge does not raise large plan calm=$calm_large surge=$surge_large (acceptable when at floor)"
fi

echo ""
echo "── Case 7: CPU ceiling enforced ──"
out=$(v2_plan_caps 100000 200000 \
                   100 100 100 100 0 \
                   20 16 12 6 0 \
                   4096 0 0)
total=$(( $(extract "$out" micro) + $(extract "$out" small) + $(extract "$out" medium) + $(extract "$out" large) ))
le "case7 total respects CPU ceiling" "$total" "${AUTOSCALER_CPU_CEILING:-30}"

echo ""
echo "── Case 8: HUGE solo-dispatch on idle card ──"
# 1 HUGE ready, no other demand, no running forks → plan_huge=1, others=0
# inflight_huge=0 (still in ready, not yet claimed by dispatcher)
out=$(v2_plan_caps 24576 50000 \
                   0 0 0 0 1 \
                   0 0 0 0 0 \
                   4096 0 0 0)
check "case8 plan_huge admitted"  1 "$(extract "$out" huge)"
check "case8 plan_micro  zero"    0 "$(extract "$out" micro)"
check "case8 plan_small  zero"    0 "$(extract "$out" small)"
check "case8 plan_medium zero"    0 "$(extract "$out" medium)"
check "case8 plan_large  zero"    0 "$(extract "$out" large)"

echo ""
echo "── Case 9: HUGE demand while other tier running - drain mode ──"
# 1 HUGE ready BUT other tiers have current dispatches + 5 live forks
# (none of those forks are huge - inflight_huge=0). V2 must NOT admit
# huge yet - plan_huge stays 0 until card drains. Other tier plans
# trend toward 0 (damped) to allow drain.
out=$(v2_plan_caps 24576 50000 \
                   0 5 0 0 1 \
                   0 5 0 0 0 \
                   4096 5 0 0)
check "case9 plan_huge waits (card busy)" 0 "$(extract "$out" huge)"
le "case9 plan_small trends down (drain)" "$(extract "$out" small)" 5
check "case9 plan_micro  drain"  0 "$(extract "$out" micro)"
check "case9 plan_medium drain"  0 "$(extract "$out" medium)"
check "case9 plan_large  drain"  0 "$(extract "$out" large)"

echo ""
echo "── Case 10: HUGE in flight (inflight_huge=1) - other tiers stay zero ──"
# HUGE currently dispatching: r_huge=0 (claimed from ready), cur_huge=1,
# inflight_huge=1, running_forks=1 (huge fork itself). While it runs,
# no other tier admits new work even with huge demand.
out=$(v2_plan_caps 24576 50000 \
                   100 100 100 100 0 \
                   0 0 0 0 1 \
                   4096 1 0 1)
check "case10 plan_huge stays admitted" 1 "$(extract "$out" huge)"
check "case10 plan_micro  starved"      0 "$(extract "$out" micro)"
check "case10 plan_small  starved"      0 "$(extract "$out" small)"
check "case10 plan_medium starved"      0 "$(extract "$out" medium)"
check "case10 plan_large  starved"      0 "$(extract "$out" large)"

echo ""
echo "── Case 11: NO huge demand - plan_huge always 0 ──"
out=$(v2_plan_caps 24576 50000 \
                   0 100 0 0 0 \
                   0 4 0 0 0 \
                   4096 4 0 0)
check "case11 plan_huge zero on no-demand" 0 "$(extract "$out" huge)"

echo ""
echo "── Case 13: RAM-pressure damping (MemAvailable < 15% of total) ──"
# ram_free=8 GB, ram_total=64 GB → fraction 12.5% < 15% threshold → damp.
# Args 17-19: ram_peak_per_worker_mib ram_total_mib swap_pressure
out_calm=$(v2_plan_caps 24576 51200 \
                        0 100 50 0 0 \
                        0 4 4 0 0 \
                        4096 8 0 0 \
                        6400 65536 0)
out_pressure=$(v2_plan_caps 24576 8192 \
                            0 100 50 0 0 \
                            0 4 4 0 0 \
                            4096 8 0 0 \
                            6400 65536 0)
calm_total=$(( $(extract "$out_calm" micro) + $(extract "$out_calm" small) \
              + $(extract "$out_calm" medium) + $(extract "$out_calm" large) ))
pres_total=$(( $(extract "$out_pressure" micro) + $(extract "$out_pressure" small) \
              + $(extract "$out_pressure" medium) + $(extract "$out_pressure" large) ))
if [ "$pres_total" -le "$calm_total" ]; then
    pass "case13 RAM pressure reduces total calm=$calm_total pressure=$pres_total"
else
    fail "case13 RAM pressure failed to reduce total calm=$calm_total pressure=$pres_total"
fi

echo ""
echo "── Case 14: SWAP-active forces damping even with abundant MemAvailable ──"
# ram_free=50 GB plenty BUT swap_pressure=1 → damp (active swapping detected)
out_no_swap=$(v2_plan_caps 24576 51200 \
                           0 100 50 0 0 \
                           0 8 8 0 0 \
                           4096 8 0 0 \
                           6400 65536 0)
out_swap=$(v2_plan_caps 24576 51200 \
                        0 100 50 0 0 \
                        0 8 8 0 0 \
                        4096 8 0 0 \
                        6400 65536 1)
no_swap_total=$(( $(extract "$out_no_swap" micro) + $(extract "$out_no_swap" small) \
                 + $(extract "$out_no_swap" medium) + $(extract "$out_no_swap" large) ))
swap_total=$(( $(extract "$out_swap" micro) + $(extract "$out_swap" small) \
              + $(extract "$out_swap" medium) + $(extract "$out_swap" large) ))
if [ "$swap_total" -le "$no_swap_total" ]; then
    pass "case14 swap pressure reduces total no-swap=$no_swap_total swap=$swap_total"
else
    fail "case14 swap pressure failed no-swap=$no_swap_total swap=$swap_total"
fi

echo ""
echo "── Case 12: STALE cur_huge=1 + no demand + no inflight (live-defect repro) ──"
# Live defect 2026-06-12: supervisor.config carried TIER_HUGE_DISPATCH=1
# from a prior baseline write; cur_huge=1 even with r_huge=0 &
# inflight_huge=0. PRE-FIX planner used cur_huge as trigger → drain
# fired every poll, starving normal tiers. POST-FIX: r_huge AND inflight
# decide drain; stale cur_huge is benign.
out=$(v2_plan_caps 24576 50000 \
                   0 100 50 0 0 \
                   0 4 4 0 1 \
                   4096 8 0 0)
check "case12 plan_huge zero on stale cur (no drain trigger)" 0 "$(extract "$out" huge)"
ge "case12 plan_small advances (not starved)" "$(extract "$out" small)" 4
ge "case12 plan_medium advances (not starved)" "$(extract "$out" medium)" 4

echo ""
if [ "$fails" -gt 0 ]; then
    echo "[autoscaler-v2] FAIL - $fails assertion(s) failed"
    exit 1
fi
echo "[autoscaler-v2] PASS - all assertions OK"
exit 0
