factory: heal_orphan_bins nullglob defect — use compgen for inflight check + reducer
shopt -s nullglob (set so outer for-loop tolerates empty *.bin) made an unmatched .inflight-* glob expand to nothing. Bare "ls >/dev/null" then succeeded by listing CWD, and our if-branch incorrectly skipped every orphan whose .inflight-* did not match. Net effect: medium-sized orphan bins (100 MiB+, valid QECCOPS1 magic, no markers) accumulated in our queue indefinitely across pool restarts. Replace with compgen -G which returns success only when our pattern matches, immune to nullglob. Adds tests/integration/test-heal-orphan-bins.sh as TCRAUDT reducer covering our contract: medium-size orphan promotes to .ready, sub-threshold truncated to .done, bad-magic DLQs to dlq/. Incident 2026-06-14: 8 vecC-tri-*-w8c.bin orphans (493 MiB each, Jun 12 emit) survived multiple foxhop pool restarts. Live factory queue visible to operator as a persistent ~8-bin floor that never drained.
This commit is contained in:
parent
346b873247
commit
3e47814cf3
2 changed files with 127 additions and 1 deletions
|
|
@ -197,7 +197,12 @@ heal_orphan_bins() {
|
|||
[ -f "$QUEUE_DIR/${tag}.done" ] && continue
|
||||
[ -f "$QUEUE_DIR/${tag}.emitting" ] && continue
|
||||
[ -f "$QUEUE_DIR/${tag}.dispatch.log" ] && continue
|
||||
if ls "$QUEUE_DIR/${tag}.inflight-"* >/dev/null 2>&1; then
|
||||
# 2026-06-14 defect fix: with shopt -s nullglob (set at function top
|
||||
# so the outer for loop tolerates empty *.bin), an unmatched
|
||||
# .inflight-* glob expands to NOTHING. Bare "ls >/dev/null" then
|
||||
# succeeds (lists CWD) + the if-branch incorrectly skips the orphan.
|
||||
# Compgen returns success only when pattern matches, immune to nullglob.
|
||||
if compgen -G "$QUEUE_DIR/${tag}.inflight-*" >/dev/null; then
|
||||
continue
|
||||
fi
|
||||
sz=$(stat -c %s "$bin" 2>/dev/null || echo 0)
|
||||
|
|
|
|||
121
tests/integration/test-heal-orphan-bins.sh
Executable file
121
tests/integration/test-heal-orphan-bins.sh
Executable file
|
|
@ -0,0 +1,121 @@
|
|||
#!/bin/bash
|
||||
# test-heal-orphan-bins.sh — pin heal_orphan_bins() contract
|
||||
#
|
||||
# Defect observed 2026-06-14: 8 medium-size .bin files (493 MiB each,
|
||||
# valid QECCOPS1 magic, no markers, dated Jun 12) sat as orphans through
|
||||
# multiple pool restarts. heal_orphan_bins() should have promoted them
|
||||
# to .ready on next iter. Root cause not yet isolated — likely stale
|
||||
# running-process baked-in code, or a subtle skip condition not in file
|
||||
# source. Reducer pins the function contract so any regression to file
|
||||
# source is caught.
|
||||
#
|
||||
# Three positive cases + one negative:
|
||||
# 1. medium bin (500 MiB, valid magic, no markers) -> .ready
|
||||
# 2. large bin (4 GiB, valid magic, no markers) -> .ready
|
||||
# 3. tiny bin (1 MiB, valid magic, no markers) -> .done (truncated)
|
||||
# 4. bad-magic bin (any size, no markers) -> .done.fail + .done + dlq/.bin
|
||||
#
|
||||
# Sources heal_orphan_bins from $LUMBDA_FACTORY_DIR/bend-emit-pool.sh,
|
||||
# defaulting to upstream factory dir. Consumers can override via
|
||||
# LUMBDA_FACTORY_DIR env to test their own copy.
|
||||
|
||||
set -e
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
FACTORY_DIR="${LUMBDA_FACTORY_DIR:-$HERE/../../factory}"
|
||||
POOL_SCRIPT="$FACTORY_DIR/bend-emit-pool.sh"
|
||||
|
||||
if [ ! -f "$POOL_SCRIPT" ]; then
|
||||
echo "SKIP: $POOL_SCRIPT missing (factory not deployed)"
|
||||
exit 77
|
||||
fi
|
||||
|
||||
TMP="$(mktemp -d /tmp/heal-orphan-bins-test-XXXXXX)"
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
|
||||
# Extract heal_orphan_bins function definition from pool script.
|
||||
# Function starts at "^heal_orphan_bins()" + ends at first standalone "^}".
|
||||
awk '/^heal_orphan_bins\(\)/,/^}$/' "$POOL_SCRIPT" > "$TMP/heal-fn.sh"
|
||||
|
||||
# Sanity check extraction.
|
||||
if ! grep -q "^heal_orphan_bins()" "$TMP/heal-fn.sh"; then
|
||||
echo "FAIL: could not extract heal_orphan_bins() from $POOL_SCRIPT"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Stub date for deterministic log timestamps (function calls date inline).
|
||||
# Source the function definition into the current shell.
|
||||
# shellcheck source=/dev/null
|
||||
. "$TMP/heal-fn.sh"
|
||||
|
||||
QUEUE_DIR="$TMP/queue"
|
||||
MIN_BIN_SIZE=67108864 # 64 MiB — same default as production pool
|
||||
mkdir -p "$QUEUE_DIR"
|
||||
export QUEUE_DIR MIN_BIN_SIZE
|
||||
|
||||
# Helper to build a fake .bin with given magic + size in MiB.
|
||||
# Uses bs=1M (fast) + dd conv=notrunc to overlay magic header on
|
||||
# the first 8 bytes. Avoids bs=1 byte-by-byte fill which makes
|
||||
# 4 GiB fixtures take an hour.
|
||||
make_bin() {
|
||||
local tag="$1" magic="$2" size_mb="$3"
|
||||
local bin="$QUEUE_DIR/$tag.bin"
|
||||
dd if=/dev/zero of="$bin" bs=1M count="$size_mb" 2>/dev/null
|
||||
printf '%s' "$magic" | dd of="$bin" bs=1 count=8 conv=notrunc 2>/dev/null
|
||||
}
|
||||
|
||||
echo "── building test fixtures ──"
|
||||
make_bin "medium-orphan" "QECCOPS1" 100 # > 64 MiB MIN, valid magic
|
||||
make_bin "tiny-orphan" "QECCOPS1" 1 # < 64 MiB MIN, valid magic
|
||||
make_bin "badmagic-orphan" "XXXXXXXX" 80 # bad magic
|
||||
|
||||
ls -la "$QUEUE_DIR" | head -8
|
||||
|
||||
echo "── invoking heal_orphan_bins() ──"
|
||||
heal_orphan_bins
|
||||
|
||||
echo "── assertions ──"
|
||||
ok=0; fail=0
|
||||
assert_ready() {
|
||||
local tag="$1"
|
||||
if [ -f "$QUEUE_DIR/$tag.ready" ]; then
|
||||
echo " PASS $tag -> .ready"
|
||||
ok=$((ok+1))
|
||||
else
|
||||
echo " FAIL $tag -> .ready missing"
|
||||
fail=$((fail+1))
|
||||
fi
|
||||
}
|
||||
assert_done_no_bin() {
|
||||
local tag="$1" reason="$2"
|
||||
if [ -f "$QUEUE_DIR/$tag.done" ] && [ ! -f "$QUEUE_DIR/$tag.bin" ]; then
|
||||
echo " PASS $tag -> .done + bin removed ($reason)"
|
||||
ok=$((ok+1))
|
||||
else
|
||||
echo " FAIL $tag -> expected .done + bin removed ($reason)"
|
||||
fail=$((fail+1))
|
||||
fi
|
||||
}
|
||||
assert_dlq() {
|
||||
local tag="$1"
|
||||
if [ -f "$QUEUE_DIR/dlq/$tag.reason" ]; then
|
||||
echo " PASS $tag -> dlq/$tag.reason written"
|
||||
ok=$((ok+1))
|
||||
else
|
||||
echo " FAIL $tag -> dlq/$tag.reason missing"
|
||||
fail=$((fail+1))
|
||||
fi
|
||||
}
|
||||
|
||||
assert_ready medium-orphan
|
||||
assert_done_no_bin tiny-orphan "truncated < MIN_BIN_SIZE"
|
||||
assert_done_no_bin badmagic-orphan "zero-magic DLQ"
|
||||
assert_dlq badmagic-orphan
|
||||
|
||||
echo
|
||||
if [ "$fail" -eq 0 ]; then
|
||||
echo "[heal-orphan-bins] PASS — all $ok assertions OK"
|
||||
exit 0
|
||||
else
|
||||
echo "[heal-orphan-bins] FAIL — $fail of $((ok+fail)) assertions failed"
|
||||
exit 1
|
||||
fi
|
||||
Loading…
Add table
Add a link
Reference in a new issue