russell@unturf.com
3e47814cf3
shopt -s nullglob (set so outer for-loop tolerates empty *.bin) made an unmatched .inflight-* glob expand to nothing. Bare "ls >/dev/null" then succeeded by listing CWD, and our if-branch incorrectly skipped every orphan whose .inflight-* did not match. Net effect: medium-sized orphan bins (100 MiB+, valid QECCOPS1 magic, no markers) accumulated in our queue indefinitely across pool restarts. Replace with compgen -G which returns success only when our pattern matches, immune to nullglob. Adds tests/integration/test-heal-orphan-bins.sh as TCRAUDT reducer covering our contract: medium-size orphan promotes to .ready, sub-threshold truncated to .done, bad-magic DLQs to dlq/. Incident 2026-06-14: 8 vecC-tri-*-w8c.bin orphans (493 MiB each, Jun 12 emit) survived multiple foxhop pool restarts. Live factory queue visible to operator as a persistent ~8-bin floor that never drained.
121 lines
4 KiB
Bash
Executable file
121 lines
4 KiB
Bash
Executable file
#!/bin/bash
|
|
# test-heal-orphan-bins.sh — pin heal_orphan_bins() contract
|
|
#
|
|
# Defect observed 2026-06-14: 8 medium-size .bin files (493 MiB each,
|
|
# valid QECCOPS1 magic, no markers, dated Jun 12) sat as orphans through
|
|
# multiple pool restarts. heal_orphan_bins() should have promoted them
|
|
# to .ready on next iter. Root cause not yet isolated — likely stale
|
|
# running-process baked-in code, or a subtle skip condition not in file
|
|
# source. Reducer pins the function contract so any regression to file
|
|
# source is caught.
|
|
#
|
|
# Three positive cases + one negative:
|
|
# 1. medium bin (500 MiB, valid magic, no markers) -> .ready
|
|
# 2. large bin (4 GiB, valid magic, no markers) -> .ready
|
|
# 3. tiny bin (1 MiB, valid magic, no markers) -> .done (truncated)
|
|
# 4. bad-magic bin (any size, no markers) -> .done.fail + .done + dlq/.bin
|
|
#
|
|
# Sources heal_orphan_bins from $LUMBDA_FACTORY_DIR/bend-emit-pool.sh,
|
|
# defaulting to upstream factory dir. Consumers can override via
|
|
# LUMBDA_FACTORY_DIR env to test their own copy.
|
|
|
|
set -e
|
|
HERE="$(cd "$(dirname "$0")" && pwd)"
|
|
FACTORY_DIR="${LUMBDA_FACTORY_DIR:-$HERE/../../factory}"
|
|
POOL_SCRIPT="$FACTORY_DIR/bend-emit-pool.sh"
|
|
|
|
if [ ! -f "$POOL_SCRIPT" ]; then
|
|
echo "SKIP: $POOL_SCRIPT missing (factory not deployed)"
|
|
exit 77
|
|
fi
|
|
|
|
TMP="$(mktemp -d /tmp/heal-orphan-bins-test-XXXXXX)"
|
|
trap 'rm -rf "$TMP"' EXIT
|
|
|
|
# Extract heal_orphan_bins function definition from pool script.
|
|
# Function starts at "^heal_orphan_bins()" + ends at first standalone "^}".
|
|
awk '/^heal_orphan_bins\(\)/,/^}$/' "$POOL_SCRIPT" > "$TMP/heal-fn.sh"
|
|
|
|
# Sanity check extraction.
|
|
if ! grep -q "^heal_orphan_bins()" "$TMP/heal-fn.sh"; then
|
|
echo "FAIL: could not extract heal_orphan_bins() from $POOL_SCRIPT"
|
|
exit 1
|
|
fi
|
|
|
|
# Stub date for deterministic log timestamps (function calls date inline).
|
|
# Source the function definition into the current shell.
|
|
# shellcheck source=/dev/null
|
|
. "$TMP/heal-fn.sh"
|
|
|
|
QUEUE_DIR="$TMP/queue"
|
|
MIN_BIN_SIZE=67108864 # 64 MiB — same default as production pool
|
|
mkdir -p "$QUEUE_DIR"
|
|
export QUEUE_DIR MIN_BIN_SIZE
|
|
|
|
# Helper to build a fake .bin with given magic + size in MiB.
|
|
# Uses bs=1M (fast) + dd conv=notrunc to overlay magic header on
|
|
# the first 8 bytes. Avoids bs=1 byte-by-byte fill which makes
|
|
# 4 GiB fixtures take an hour.
|
|
make_bin() {
|
|
local tag="$1" magic="$2" size_mb="$3"
|
|
local bin="$QUEUE_DIR/$tag.bin"
|
|
dd if=/dev/zero of="$bin" bs=1M count="$size_mb" 2>/dev/null
|
|
printf '%s' "$magic" | dd of="$bin" bs=1 count=8 conv=notrunc 2>/dev/null
|
|
}
|
|
|
|
echo "── building test fixtures ──"
|
|
make_bin "medium-orphan" "QECCOPS1" 100 # > 64 MiB MIN, valid magic
|
|
make_bin "tiny-orphan" "QECCOPS1" 1 # < 64 MiB MIN, valid magic
|
|
make_bin "badmagic-orphan" "XXXXXXXX" 80 # bad magic
|
|
|
|
ls -la "$QUEUE_DIR" | head -8
|
|
|
|
echo "── invoking heal_orphan_bins() ──"
|
|
heal_orphan_bins
|
|
|
|
echo "── assertions ──"
|
|
ok=0; fail=0
|
|
assert_ready() {
|
|
local tag="$1"
|
|
if [ -f "$QUEUE_DIR/$tag.ready" ]; then
|
|
echo " PASS $tag -> .ready"
|
|
ok=$((ok+1))
|
|
else
|
|
echo " FAIL $tag -> .ready missing"
|
|
fail=$((fail+1))
|
|
fi
|
|
}
|
|
assert_done_no_bin() {
|
|
local tag="$1" reason="$2"
|
|
if [ -f "$QUEUE_DIR/$tag.done" ] && [ ! -f "$QUEUE_DIR/$tag.bin" ]; then
|
|
echo " PASS $tag -> .done + bin removed ($reason)"
|
|
ok=$((ok+1))
|
|
else
|
|
echo " FAIL $tag -> expected .done + bin removed ($reason)"
|
|
fail=$((fail+1))
|
|
fi
|
|
}
|
|
assert_dlq() {
|
|
local tag="$1"
|
|
if [ -f "$QUEUE_DIR/dlq/$tag.reason" ]; then
|
|
echo " PASS $tag -> dlq/$tag.reason written"
|
|
ok=$((ok+1))
|
|
else
|
|
echo " FAIL $tag -> dlq/$tag.reason missing"
|
|
fail=$((fail+1))
|
|
fi
|
|
}
|
|
|
|
assert_ready medium-orphan
|
|
assert_done_no_bin tiny-orphan "truncated < MIN_BIN_SIZE"
|
|
assert_done_no_bin badmagic-orphan "zero-magic DLQ"
|
|
assert_dlq badmagic-orphan
|
|
|
|
echo
|
|
if [ "$fail" -eq 0 ]; then
|
|
echo "[heal-orphan-bins] PASS — all $ok assertions OK"
|
|
exit 0
|
|
else
|
|
echo "[heal-orphan-bins] FAIL — $fail of $((ok+fail)) assertions failed"
|
|
exit 1
|
|
fi
|