russell@unturf.com
823e8da1ff
Two follow-up defects from 2026-06-14 factory triage:
(1) DLQ runner classifier did not recognize "vram-oversized" reason text
introduced by foxhop dispatcher pre-flight (commit 8d45e0d on the foxhop
side). 65 of 87 rDLQ cells got escalated as class=unknown instead of a
properly named bucket. Adds pattern + escalate-class entry + reducer
test case mapped to (vram-oversized sim no — bin is dead weight on this
card, salvage skipped).
(2) sweep-doctrine reducers had no per-test timeout. K=5 doctrine tests
(test-k5-apply-forward-ipmul + 4 siblings) ran lumbda at 99% CPU for
2h43m on a remote node without ever emitting their DOCTRINE verdict
line — accumulating 30+ runaway lumbda procs under two stuck `make
sweep-doctrine` invocations. run.sh + run-parallel.sh now wrap our
lumbda invocation in `timeout ${SWEEP_DOCTRINE_TEST_TIMEOUT_S:-300}`;
hit exits 124, our existing "no DOCTRINE line" branch logs HARNESS-FAIL.
K=5 substrate has a documented non-terminating compute defect AND a
load-time buffer overflow (commit 6d18c59 on foxhop). Bisect deferred
per ticket 0007 in foxhop tree; needs qemu apparatus we currently lack.
162 lines
6.7 KiB
Bash
Executable file
162 lines
6.7 KiB
Bash
Executable file
#!/usr/bin/env bash
|
||
# sweep-doctrine/run-parallel.sh - same contract as run.sh but xargs -P
|
||
# fan-out over independent reducers.
|
||
#
|
||
# Each reducer is independent (no shared state across processes) so
|
||
# parallelism is safe. Wall-clock improvement on a champion-class
|
||
# verify: ~89 reducers × ~30s serial → ~30s + slowest reducer wall
|
||
# at JOBS=16.
|
||
#
|
||
# Usage:
|
||
# ./run-parallel.sh # all reducers, JOBS=16
|
||
# ./run-parallel.sh TEST=... # one reducer (no fan-out needed)
|
||
# JOBS=8 ./run-parallel.sh # cap parallelism (lower = lighter REMOTE load)
|
||
# SLOW=1 ./run-parallel.sh # include reducers tagged ";;; SLOW-TIER"
|
||
#
|
||
# Env contract per ~/git/lumbda/factory/CONTRACT.md:
|
||
# LUMBDA_REMOTE / REMOTE SSH target. Unset = run locally.
|
||
# LUMBDA_BIN lumbda binary (default $HOME/git/lumbda/c/lumbda).
|
||
# LUMBDA_REPO_DIR repo root for cd (default $HOME/git/lumbda).
|
||
# LUMBDA_DOMAIN_DIR optional consumer repo (no default).
|
||
set -u
|
||
|
||
REMOTE="${LUMBDA_REMOTE:-${REMOTE:-}}"
|
||
LUMBDA_BIN="${LUMBDA_BIN:-$HOME/git/lumbda/c/lumbda}"
|
||
LUMBDA_REPO_DIR="${LUMBDA_REPO_DIR:-$HOME/git/lumbda}"
|
||
LUMBDA_DOMAIN_DIR="${LUMBDA_DOMAIN_DIR:-}"
|
||
JOBS="${JOBS:-16}"
|
||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||
|
||
# Resolve reducer set. SLOW=1 includes reducers whose header carries
|
||
# ";;; SLOW-TIER" marker; default skips them so fast-path stays under
|
||
# ~5 min wall-clock.
|
||
upstream_tests=()
|
||
domain_tests=()
|
||
if [ -n "${TEST:-}" ]; then
|
||
if [ -f "$HERE/${TEST}.lsp" ]; then
|
||
upstream_tests=( "$HERE/${TEST}.lsp" )
|
||
elif [ -n "$LUMBDA_DOMAIN_DIR" ] && [ -f "$LUMBDA_DOMAIN_DIR/tests/sweep-doctrine/${TEST}.lsp" ]; then
|
||
domain_tests=( "$LUMBDA_DOMAIN_DIR/tests/sweep-doctrine/${TEST}.lsp" )
|
||
else
|
||
echo "ERROR: TEST=$TEST not found upstream or under \$LUMBDA_DOMAIN_DIR"
|
||
exit 1
|
||
fi
|
||
else
|
||
if [ "${SLOW:-0}" = "1" ]; then
|
||
while IFS= read -r f; do upstream_tests+=( "$f" ); done < <(ls "$HERE"/test-*.lsp 2>/dev/null)
|
||
if [ -n "$LUMBDA_DOMAIN_DIR" ] && [ -d "$LUMBDA_DOMAIN_DIR/tests/sweep-doctrine" ]; then
|
||
while IFS= read -r f; do domain_tests+=( "$f" ); done < <(ls "$LUMBDA_DOMAIN_DIR/tests/sweep-doctrine"/test-*.lsp 2>/dev/null)
|
||
fi
|
||
else
|
||
while IFS= read -r f; do upstream_tests+=( "$f" ); done < <(grep -L "^;;; SLOW-TIER" "$HERE"/test-*.lsp 2>/dev/null)
|
||
if [ -n "$LUMBDA_DOMAIN_DIR" ] && [ -d "$LUMBDA_DOMAIN_DIR/tests/sweep-doctrine" ]; then
|
||
while IFS= read -r f; do domain_tests+=( "$f" ); done < <(grep -L "^;;; SLOW-TIER" "$LUMBDA_DOMAIN_DIR/tests/sweep-doctrine"/test-*.lsp 2>/dev/null)
|
||
fi
|
||
fi
|
||
fi
|
||
|
||
total_count=$(( ${#upstream_tests[@]} + ${#domain_tests[@]} ))
|
||
[ "$total_count" -gt 0 ] || { echo "ERROR: no test-*.lsp found in $HERE${LUMBDA_DOMAIN_DIR:+ or $LUMBDA_DOMAIN_DIR/tests/sweep-doctrine}"; exit 1; }
|
||
|
||
# Single-reducer path: defer to serial runner - fan-out has no payoff.
|
||
if [ "$total_count" -eq 1 ]; then
|
||
exec env TEST="${TEST:-}" REMOTE="$REMOTE" LUMBDA_REPO_DIR="$LUMBDA_REPO_DIR" \
|
||
LUMBDA_DOMAIN_DIR="$LUMBDA_DOMAIN_DIR" LUMBDA_BIN="$LUMBDA_BIN" \
|
||
"$HERE/run.sh"
|
||
fi
|
||
|
||
# Remote auto-pull (same discipline as run.sh).
|
||
if [ -n "$REMOTE" ]; then
|
||
pull_out=$(ssh "$REMOTE" "cd $LUMBDA_REPO_DIR; git pull --ff-only origin master 2>&1" 2>&1)
|
||
echo "$pull_out" | sed 's/^/[remote-pull-upstream] /'
|
||
if echo "$pull_out" | grep -qE "^(error|Aborting|fatal:)"; then
|
||
echo "[remote-pull-upstream] HARD ABORT - pull failed; reducers not run."
|
||
exit 1
|
||
fi
|
||
if [ -n "$LUMBDA_DOMAIN_DIR" ] && [ "${#domain_tests[@]}" -gt 0 ]; then
|
||
domain_repo="$(cd "$LUMBDA_DOMAIN_DIR/.." 2>/dev/null && pwd || echo "$LUMBDA_DOMAIN_DIR")"
|
||
pull_out=$(ssh "$REMOTE" "cd $domain_repo; git pull --ff-only origin master 2>&1" 2>&1)
|
||
echo "$pull_out" | sed 's/^/[remote-pull-domain] /'
|
||
if echo "$pull_out" | grep -qE "^(error|Aborting|fatal:)"; then
|
||
echo "[remote-pull-domain] HARD ABORT - pull failed; reducers not run."
|
||
exit 1
|
||
fi
|
||
fi
|
||
fi
|
||
|
||
WORKDIR=$(mktemp -d -t sweep-doctrine-parallel.XXXXXX)
|
||
trap 'rm -rf "$WORKDIR"' EXIT
|
||
|
||
echo "[run-parallel] $(date -u +%H:%M:%S) launching $total_count reducer(s) with JOBS=$JOBS"
|
||
|
||
# Worker: run one reducer, grep DOCTRINE lines, write per-test file.
|
||
# Exit codes:
|
||
# 0 = at least one DOCTRINE line, no unexpected verdicts
|
||
# 1 = unexpected FAIL / UNEXPECTED-PASS / HARNESS-FAIL
|
||
# 2 = no DOCTRINE line (harness fail)
|
||
run_one() {
|
||
local t="$1" repo_root="$2" workdir="$3"
|
||
local name rel out cmd
|
||
name="$(basename "$t" .lsp)"
|
||
rel="tests/sweep-doctrine/${name}.lsp"
|
||
# Per-test timeout (2026-06-14): see run.sh for rationale. Non-terminating
|
||
# reducers fail-fast (timeout exit 124, no DOCTRINE line, runner logs
|
||
# HARNESS-FAIL). SWEEP_DOCTRINE_TEST_TIMEOUT_S overrides default 300s.
|
||
cmd="cd $repo_root; timeout ${SWEEP_DOCTRINE_TEST_TIMEOUT_S:-300} $LUMBDA_BIN --fast $rel 2>&1"
|
||
if [ -n "$REMOTE" ]; then
|
||
out=$(ssh "$REMOTE" "$cmd" 2>&1 | grep -E "^DOCTRINE ")
|
||
else
|
||
out=$(bash -c "$cmd" 2>&1 | grep -E "^DOCTRINE ")
|
||
fi
|
||
if [ -z "$out" ]; then
|
||
echo "DOCTRINE $name HARNESS-FAIL (no DOCTRINE line emitted)" > "$workdir/$name.out"
|
||
return 2
|
||
fi
|
||
echo "$out" > "$workdir/$name.out"
|
||
if echo "$out" | grep -qE "DOCTRINE [^ ]+ (FAIL|UNEXPECTED-PASS|HARNESS-FAIL)( |$)"; then
|
||
return 1
|
||
fi
|
||
return 0
|
||
}
|
||
export -f run_one
|
||
export REMOTE LUMBDA_REPO_DIR LUMBDA_BIN LUMBDA_DOMAIN_DIR
|
||
|
||
# Build per-test (path, repo_root) pairs. xargs -0 -n2 reads two
|
||
# null-delimited tokens per worker invocation.
|
||
{
|
||
for t in "${upstream_tests[@]}"; do printf "%s\0%s\0" "$t" "$LUMBDA_REPO_DIR"; done
|
||
for t in "${domain_tests[@]}"; do printf "%s\0%s\0" "$t" "$LUMBDA_DOMAIN_DIR"; done
|
||
} | xargs -0 -P "$JOBS" -n2 bash -c 'run_one "$1" "$2" "'"$WORKDIR"'"' _
|
||
|
||
# Collect outputs in lexicographic order (deterministic).
|
||
overall_rc=0
|
||
failed_tests=()
|
||
collect_one() {
|
||
local t="$1" name
|
||
name="$(basename "$t" .lsp)"
|
||
if [ -f "$WORKDIR/$name.out" ]; then
|
||
cat "$WORKDIR/$name.out"
|
||
# Same field-anchored + summary-aware verdict logic as run.sh.
|
||
if awk '
|
||
$1=="DOCTRINE" && $2~/-summary$/ && $3=="EXPECTED-FAIL" {expected_fail=1}
|
||
$1=="DOCTRINE" && $3~/^(FAIL|UNEXPECTED-PASS|HARNESS-FAIL)$/ {found=1}
|
||
END{ exit !(found && !expected_fail) }
|
||
' "$WORKDIR/$name.out"; then
|
||
overall_rc=1
|
||
failed_tests+=("$name")
|
||
fi
|
||
else
|
||
echo "DOCTRINE $name HARNESS-FAIL (no output file - worker crashed)"
|
||
overall_rc=1
|
||
failed_tests+=("$name (missing output)")
|
||
fi
|
||
}
|
||
for t in "${upstream_tests[@]}"; do collect_one "$t"; done
|
||
for t in "${domain_tests[@]}"; do collect_one "$t"; done
|
||
|
||
echo "[run-parallel] $(date -u +%H:%M:%S) done; exit=$overall_rc"
|
||
if [ "$overall_rc" -ne 0 ]; then
|
||
echo "[run-parallel] failed reducers (${#failed_tests[@]}):"
|
||
printf " - %s\n" "${failed_tests[@]}"
|
||
fi
|
||
exit "$overall_rc"
|