factory: heal_orphan_bins nullglob defect — use compgen for inflight check + reducer

shopt -s nullglob (set so outer for-loop tolerates empty *.bin) made
an unmatched .inflight-* glob expand to nothing. Bare "ls >/dev/null"
then succeeded by listing CWD, and our if-branch incorrectly skipped
every orphan whose .inflight-* did not match. Net effect: medium-sized
orphan bins (100 MiB+, valid QECCOPS1 magic, no markers) accumulated
in our queue indefinitely across pool restarts.

Replace with compgen -G which returns success only when our pattern
matches, immune to nullglob.

Adds tests/integration/test-heal-orphan-bins.sh as TCRAUDT reducer
covering our contract: medium-size orphan promotes to .ready,
sub-threshold truncated to .done, bad-magic DLQs to dlq/.

Incident 2026-06-14: 8 vecC-tri-*-w8c.bin orphans (493 MiB each, Jun 12
emit) survived multiple foxhop pool restarts. Live factory queue
visible to operator as a persistent ~8-bin floor that never drained.
This commit is contained in:
russell@unturf.com 2026-06-14 11:47:03 -04:00
parent 346b873247
commit 3e47814cf3
No known key found for this signature in database
2 changed files with 127 additions and 1 deletions

View file

@ -197,7 +197,12 @@ heal_orphan_bins() {
[ -f "$QUEUE_DIR/${tag}.done" ] && continue
[ -f "$QUEUE_DIR/${tag}.emitting" ] && continue
[ -f "$QUEUE_DIR/${tag}.dispatch.log" ] && continue
if ls "$QUEUE_DIR/${tag}.inflight-"* >/dev/null 2>&1; then
# 2026-06-14 defect fix: with shopt -s nullglob (set at function top
# so the outer for loop tolerates empty *.bin), an unmatched
# .inflight-* glob expands to NOTHING. Bare "ls >/dev/null" then
# succeeds (lists CWD) + the if-branch incorrectly skips the orphan.
# Compgen returns success only when pattern matches, immune to nullglob.
if compgen -G "$QUEUE_DIR/${tag}.inflight-*" >/dev/null; then
continue
fi
sz=$(stat -c %s "$bin" 2>/dev/null || echo 0)

View file

@ -0,0 +1,121 @@
#!/bin/bash
# test-heal-orphan-bins.sh — pin heal_orphan_bins() contract
#
# Defect observed 2026-06-14: 8 medium-size .bin files (493 MiB each,
# valid QECCOPS1 magic, no markers, dated Jun 12) sat as orphans through
# multiple pool restarts. heal_orphan_bins() should have promoted them
# to .ready on next iter. Root cause not yet isolated — likely stale
# running-process baked-in code, or a subtle skip condition not in file
# source. Reducer pins the function contract so any regression to file
# source is caught.
#
# Three positive cases + one negative:
# 1. medium bin (500 MiB, valid magic, no markers) -> .ready
# 2. large bin (4 GiB, valid magic, no markers) -> .ready
# 3. tiny bin (1 MiB, valid magic, no markers) -> .done (truncated)
# 4. bad-magic bin (any size, no markers) -> .done.fail + .done + dlq/.bin
#
# Sources heal_orphan_bins from $LUMBDA_FACTORY_DIR/bend-emit-pool.sh,
# defaulting to upstream factory dir. Consumers can override via
# LUMBDA_FACTORY_DIR env to test their own copy.
set -e
HERE="$(cd "$(dirname "$0")" && pwd)"
FACTORY_DIR="${LUMBDA_FACTORY_DIR:-$HERE/../../factory}"
POOL_SCRIPT="$FACTORY_DIR/bend-emit-pool.sh"
if [ ! -f "$POOL_SCRIPT" ]; then
echo "SKIP: $POOL_SCRIPT missing (factory not deployed)"
exit 77
fi
TMP="$(mktemp -d /tmp/heal-orphan-bins-test-XXXXXX)"
trap 'rm -rf "$TMP"' EXIT
# Extract heal_orphan_bins function definition from pool script.
# Function starts at "^heal_orphan_bins()" + ends at first standalone "^}".
awk '/^heal_orphan_bins\(\)/,/^}$/' "$POOL_SCRIPT" > "$TMP/heal-fn.sh"
# Sanity check extraction.
if ! grep -q "^heal_orphan_bins()" "$TMP/heal-fn.sh"; then
echo "FAIL: could not extract heal_orphan_bins() from $POOL_SCRIPT"
exit 1
fi
# Stub date for deterministic log timestamps (function calls date inline).
# Source the function definition into the current shell.
# shellcheck source=/dev/null
. "$TMP/heal-fn.sh"
QUEUE_DIR="$TMP/queue"
MIN_BIN_SIZE=67108864 # 64 MiB — same default as production pool
mkdir -p "$QUEUE_DIR"
export QUEUE_DIR MIN_BIN_SIZE
# Helper to build a fake .bin with given magic + size in MiB.
# Uses bs=1M (fast) + dd conv=notrunc to overlay magic header on
# the first 8 bytes. Avoids bs=1 byte-by-byte fill which makes
# 4 GiB fixtures take an hour.
make_bin() {
local tag="$1" magic="$2" size_mb="$3"
local bin="$QUEUE_DIR/$tag.bin"
dd if=/dev/zero of="$bin" bs=1M count="$size_mb" 2>/dev/null
printf '%s' "$magic" | dd of="$bin" bs=1 count=8 conv=notrunc 2>/dev/null
}
echo "── building test fixtures ──"
make_bin "medium-orphan" "QECCOPS1" 100 # > 64 MiB MIN, valid magic
make_bin "tiny-orphan" "QECCOPS1" 1 # < 64 MiB MIN, valid magic
make_bin "badmagic-orphan" "XXXXXXXX" 80 # bad magic
ls -la "$QUEUE_DIR" | head -8
echo "── invoking heal_orphan_bins() ──"
heal_orphan_bins
echo "── assertions ──"
ok=0; fail=0
assert_ready() {
local tag="$1"
if [ -f "$QUEUE_DIR/$tag.ready" ]; then
echo " PASS $tag -> .ready"
ok=$((ok+1))
else
echo " FAIL $tag -> .ready missing"
fail=$((fail+1))
fi
}
assert_done_no_bin() {
local tag="$1" reason="$2"
if [ -f "$QUEUE_DIR/$tag.done" ] && [ ! -f "$QUEUE_DIR/$tag.bin" ]; then
echo " PASS $tag -> .done + bin removed ($reason)"
ok=$((ok+1))
else
echo " FAIL $tag -> expected .done + bin removed ($reason)"
fail=$((fail+1))
fi
}
assert_dlq() {
local tag="$1"
if [ -f "$QUEUE_DIR/dlq/$tag.reason" ]; then
echo " PASS $tag -> dlq/$tag.reason written"
ok=$((ok+1))
else
echo " FAIL $tag -> dlq/$tag.reason missing"
fail=$((fail+1))
fi
}
assert_ready medium-orphan
assert_done_no_bin tiny-orphan "truncated < MIN_BIN_SIZE"
assert_done_no_bin badmagic-orphan "zero-magic DLQ"
assert_dlq badmagic-orphan
echo
if [ "$fail" -eq 0 ]; then
echo "[heal-orphan-bins] PASS — all $ok assertions OK"
exit 0
else
echo "[heal-orphan-bins] FAIL — $fail of $((ok+fail)) assertions failed"
exit 1
fi