From 3e47814cf38acc41f77316d00af4e859cd9117e7 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Sun, 14 Jun 2026 11:47:03 -0400 Subject: [PATCH] =?UTF-8?q?factory:=20heal=5Forphan=5Fbins=20nullglob=20de?= =?UTF-8?q?fect=20=E2=80=94=20use=20compgen=20for=20inflight=20check=20+?= =?UTF-8?q?=20reducer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit shopt -s nullglob (set so outer for-loop tolerates empty *.bin) made an unmatched .inflight-* glob expand to nothing. Bare "ls >/dev/null" then succeeded by listing CWD, and our if-branch incorrectly skipped every orphan whose .inflight-* did not match. Net effect: medium-sized orphan bins (100 MiB+, valid QECCOPS1 magic, no markers) accumulated in our queue indefinitely across pool restarts. Replace with compgen -G which returns success only when our pattern matches, immune to nullglob. Adds tests/integration/test-heal-orphan-bins.sh as TCRAUDT reducer covering our contract: medium-size orphan promotes to .ready, sub-threshold truncated to .done, bad-magic DLQs to dlq/. Incident 2026-06-14: 8 vecC-tri-*-w8c.bin orphans (493 MiB each, Jun 12 emit) survived multiple foxhop pool restarts. Live factory queue visible to operator as a persistent ~8-bin floor that never drained. --- factory/bend-emit-pool.sh | 7 +- tests/integration/test-heal-orphan-bins.sh | 121 +++++++++++++++++++++ 2 files changed, 127 insertions(+), 1 deletion(-) create mode 100755 tests/integration/test-heal-orphan-bins.sh diff --git a/factory/bend-emit-pool.sh b/factory/bend-emit-pool.sh index c212c4d..afa7b54 100755 --- a/factory/bend-emit-pool.sh +++ b/factory/bend-emit-pool.sh @@ -197,7 +197,12 @@ heal_orphan_bins() { [ -f "$QUEUE_DIR/${tag}.done" ] && continue [ -f "$QUEUE_DIR/${tag}.emitting" ] && continue [ -f "$QUEUE_DIR/${tag}.dispatch.log" ] && continue - if ls "$QUEUE_DIR/${tag}.inflight-"* >/dev/null 2>&1; then + # 2026-06-14 defect fix: with shopt -s nullglob (set at function top + # so the outer for loop tolerates empty *.bin), an unmatched + # .inflight-* glob expands to NOTHING. Bare "ls >/dev/null" then + # succeeds (lists CWD) + the if-branch incorrectly skips the orphan. + # Compgen returns success only when pattern matches, immune to nullglob. + if compgen -G "$QUEUE_DIR/${tag}.inflight-*" >/dev/null; then continue fi sz=$(stat -c %s "$bin" 2>/dev/null || echo 0) diff --git a/tests/integration/test-heal-orphan-bins.sh b/tests/integration/test-heal-orphan-bins.sh new file mode 100755 index 0000000..1d1eafe --- /dev/null +++ b/tests/integration/test-heal-orphan-bins.sh @@ -0,0 +1,121 @@ +#!/bin/bash +# test-heal-orphan-bins.sh — pin heal_orphan_bins() contract +# +# Defect observed 2026-06-14: 8 medium-size .bin files (493 MiB each, +# valid QECCOPS1 magic, no markers, dated Jun 12) sat as orphans through +# multiple pool restarts. heal_orphan_bins() should have promoted them +# to .ready on next iter. Root cause not yet isolated — likely stale +# running-process baked-in code, or a subtle skip condition not in file +# source. Reducer pins the function contract so any regression to file +# source is caught. +# +# Three positive cases + one negative: +# 1. medium bin (500 MiB, valid magic, no markers) -> .ready +# 2. large bin (4 GiB, valid magic, no markers) -> .ready +# 3. tiny bin (1 MiB, valid magic, no markers) -> .done (truncated) +# 4. bad-magic bin (any size, no markers) -> .done.fail + .done + dlq/.bin +# +# Sources heal_orphan_bins from $LUMBDA_FACTORY_DIR/bend-emit-pool.sh, +# defaulting to upstream factory dir. Consumers can override via +# LUMBDA_FACTORY_DIR env to test their own copy. + +set -e +HERE="$(cd "$(dirname "$0")" && pwd)" +FACTORY_DIR="${LUMBDA_FACTORY_DIR:-$HERE/../../factory}" +POOL_SCRIPT="$FACTORY_DIR/bend-emit-pool.sh" + +if [ ! -f "$POOL_SCRIPT" ]; then + echo "SKIP: $POOL_SCRIPT missing (factory not deployed)" + exit 77 +fi + +TMP="$(mktemp -d /tmp/heal-orphan-bins-test-XXXXXX)" +trap 'rm -rf "$TMP"' EXIT + +# Extract heal_orphan_bins function definition from pool script. +# Function starts at "^heal_orphan_bins()" + ends at first standalone "^}". +awk '/^heal_orphan_bins\(\)/,/^}$/' "$POOL_SCRIPT" > "$TMP/heal-fn.sh" + +# Sanity check extraction. +if ! grep -q "^heal_orphan_bins()" "$TMP/heal-fn.sh"; then + echo "FAIL: could not extract heal_orphan_bins() from $POOL_SCRIPT" + exit 1 +fi + +# Stub date for deterministic log timestamps (function calls date inline). +# Source the function definition into the current shell. +# shellcheck source=/dev/null +. "$TMP/heal-fn.sh" + +QUEUE_DIR="$TMP/queue" +MIN_BIN_SIZE=67108864 # 64 MiB — same default as production pool +mkdir -p "$QUEUE_DIR" +export QUEUE_DIR MIN_BIN_SIZE + +# Helper to build a fake .bin with given magic + size in MiB. +# Uses bs=1M (fast) + dd conv=notrunc to overlay magic header on +# the first 8 bytes. Avoids bs=1 byte-by-byte fill which makes +# 4 GiB fixtures take an hour. +make_bin() { + local tag="$1" magic="$2" size_mb="$3" + local bin="$QUEUE_DIR/$tag.bin" + dd if=/dev/zero of="$bin" bs=1M count="$size_mb" 2>/dev/null + printf '%s' "$magic" | dd of="$bin" bs=1 count=8 conv=notrunc 2>/dev/null +} + +echo "── building test fixtures ──" +make_bin "medium-orphan" "QECCOPS1" 100 # > 64 MiB MIN, valid magic +make_bin "tiny-orphan" "QECCOPS1" 1 # < 64 MiB MIN, valid magic +make_bin "badmagic-orphan" "XXXXXXXX" 80 # bad magic + +ls -la "$QUEUE_DIR" | head -8 + +echo "── invoking heal_orphan_bins() ──" +heal_orphan_bins + +echo "── assertions ──" +ok=0; fail=0 +assert_ready() { + local tag="$1" + if [ -f "$QUEUE_DIR/$tag.ready" ]; then + echo " PASS $tag -> .ready" + ok=$((ok+1)) + else + echo " FAIL $tag -> .ready missing" + fail=$((fail+1)) + fi +} +assert_done_no_bin() { + local tag="$1" reason="$2" + if [ -f "$QUEUE_DIR/$tag.done" ] && [ ! -f "$QUEUE_DIR/$tag.bin" ]; then + echo " PASS $tag -> .done + bin removed ($reason)" + ok=$((ok+1)) + else + echo " FAIL $tag -> expected .done + bin removed ($reason)" + fail=$((fail+1)) + fi +} +assert_dlq() { + local tag="$1" + if [ -f "$QUEUE_DIR/dlq/$tag.reason" ]; then + echo " PASS $tag -> dlq/$tag.reason written" + ok=$((ok+1)) + else + echo " FAIL $tag -> dlq/$tag.reason missing" + fail=$((fail+1)) + fi +} + +assert_ready medium-orphan +assert_done_no_bin tiny-orphan "truncated < MIN_BIN_SIZE" +assert_done_no_bin badmagic-orphan "zero-magic DLQ" +assert_dlq badmagic-orphan + +echo +if [ "$fail" -eq 0 ]; then + echo "[heal-orphan-bins] PASS — all $ok assertions OK" + exit 0 +else + echo "[heal-orphan-bins] FAIL — $fail of $((ok+fail)) assertions failed" + exit 1 +fi