russell@unturf.com
823e8da1ff
Two follow-up defects from 2026-06-14 factory triage:
(1) DLQ runner classifier did not recognize "vram-oversized" reason text
introduced by foxhop dispatcher pre-flight (commit 8d45e0d on the foxhop
side). 65 of 87 rDLQ cells got escalated as class=unknown instead of a
properly named bucket. Adds pattern + escalate-class entry + reducer
test case mapped to (vram-oversized sim no — bin is dead weight on this
card, salvage skipped).
(2) sweep-doctrine reducers had no per-test timeout. K=5 doctrine tests
(test-k5-apply-forward-ipmul + 4 siblings) ran lumbda at 99% CPU for
2h43m on a remote node without ever emitting their DOCTRINE verdict
line — accumulating 30+ runaway lumbda procs under two stuck `make
sweep-doctrine` invocations. run.sh + run-parallel.sh now wrap our
lumbda invocation in `timeout ${SWEEP_DOCTRINE_TEST_TIMEOUT_S:-300}`;
hit exits 124, our existing "no DOCTRINE line" branch logs HARNESS-FAIL.
K=5 substrate has a documented non-terminating compute defect AND a
load-time buffer overflow (commit 6d18c59 on foxhop). Bisect deferred
per ticket 0007 in foxhop tree; needs qemu apparatus we currently lack.
132 lines
5.8 KiB
Bash
Executable file
132 lines
5.8 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# sweep-doctrine/run.sh - runs every doctrine reducer serially.
|
|
#
|
|
# Discovers `tests/sweep-doctrine/test-*.lsp` in this repo. If
|
|
# $LUMBDA_DOMAIN_DIR points at a consumer repo (e.g. foxhop ecdsa) that
|
|
# also ships reducers under `$LUMBDA_DOMAIN_DIR/tests/sweep-doctrine/`,
|
|
# discovers + runs those too.
|
|
#
|
|
# Greps DOCTRINE lines, exits non-zero on any unexpected FAIL /
|
|
# UNEXPECTED-PASS / HARNESS-FAIL.
|
|
#
|
|
# Usage:
|
|
# ./run.sh # all reducers
|
|
# ./run.sh TEST=test-some-lesson # one reducer by name
|
|
# REMOTE=user@host ./run.sh # via SSH
|
|
# LUMBDA_BIN=/path/to/lumbda ./run.sh # override tier binary
|
|
# LUMBDA_DOMAIN_DIR=$HOME/git/foo ./run.sh # also run consumer reducers
|
|
#
|
|
# Env contract per ~/git/lumbda/factory/CONTRACT.md:
|
|
# LUMBDA_REMOTE / REMOTE SSH target. Unset = run locally.
|
|
# LUMBDA_BIN lumbda binary (default $HOME/git/lumbda/c/lumbda).
|
|
# LUMBDA_REPO_DIR repo root for remote cd (default $HOME/git/lumbda).
|
|
# LUMBDA_DOMAIN_DIR optional consumer repo (no default).
|
|
set -u
|
|
|
|
REMOTE="${LUMBDA_REMOTE:-${REMOTE:-}}"
|
|
LUMBDA_BIN="${LUMBDA_BIN:-$HOME/git/lumbda/c/lumbda}"
|
|
LUMBDA_REPO_DIR="${LUMBDA_REPO_DIR:-$HOME/git/lumbda}"
|
|
LUMBDA_DOMAIN_DIR="${LUMBDA_DOMAIN_DIR:-}"
|
|
HERE="$(cd "$(dirname "$0")" && pwd)"
|
|
|
|
# Resolve which reducers to run. TEST=<name> picks one; default = every
|
|
# test-*.lsp in HERE plus (optionally) every test-*.lsp under
|
|
# $LUMBDA_DOMAIN_DIR/tests/sweep-doctrine/.
|
|
upstream_tests=()
|
|
domain_tests=()
|
|
if [ -n "${TEST:-}" ]; then
|
|
# Single-test mode: prefer upstream, fall back to domain.
|
|
if [ -f "$HERE/${TEST}.lsp" ]; then
|
|
upstream_tests=( "$HERE/${TEST}.lsp" )
|
|
elif [ -n "$LUMBDA_DOMAIN_DIR" ] && [ -f "$LUMBDA_DOMAIN_DIR/tests/sweep-doctrine/${TEST}.lsp" ]; then
|
|
domain_tests=( "$LUMBDA_DOMAIN_DIR/tests/sweep-doctrine/${TEST}.lsp" )
|
|
else
|
|
echo "ERROR: TEST=$TEST not found upstream or under \$LUMBDA_DOMAIN_DIR"
|
|
exit 1
|
|
fi
|
|
else
|
|
while IFS= read -r f; do upstream_tests+=( "$f" ); done < <(ls "$HERE"/test-*.lsp 2>/dev/null)
|
|
if [ -n "$LUMBDA_DOMAIN_DIR" ] && [ -d "$LUMBDA_DOMAIN_DIR/tests/sweep-doctrine" ]; then
|
|
while IFS= read -r f; do domain_tests+=( "$f" ); done < <(ls "$LUMBDA_DOMAIN_DIR/tests/sweep-doctrine"/test-*.lsp 2>/dev/null)
|
|
fi
|
|
fi
|
|
|
|
total_count=$(( ${#upstream_tests[@]} + ${#domain_tests[@]} ))
|
|
[ "$total_count" -gt 0 ] || { echo "ERROR: no test-*.lsp found in $HERE${LUMBDA_DOMAIN_DIR:+ or $LUMBDA_DOMAIN_DIR/tests/sweep-doctrine}"; exit 1; }
|
|
|
|
# Remote auto-pull mirrors bend-emit-pool's merge-to-master discipline:
|
|
# every reducer that runs here must already live in master. Pre-merge
|
|
# scp staging is removed (was a recurring defect source - scp'd tests
|
|
# landed as untracked files on REMOTE then every subsequent pull
|
|
# aborted on "would overwrite untracked working tree files"). Fail
|
|
# HARD on pull error so stale-substrate wrong-output surfaces instead
|
|
# of silently masking.
|
|
if [ -n "$REMOTE" ]; then
|
|
pull_out=$(ssh "$REMOTE" "cd $LUMBDA_REPO_DIR; git pull --ff-only origin master 2>&1" 2>&1)
|
|
echo "$pull_out" | sed 's/^/[remote-pull-upstream] /'
|
|
if echo "$pull_out" | grep -qE "^(error|Aborting|fatal:)"; then
|
|
echo "[remote-pull-upstream] HARD ABORT - pull failed; reducers not run."
|
|
exit 1
|
|
fi
|
|
# Domain repo pull (if domain reducers will fire).
|
|
if [ -n "$LUMBDA_DOMAIN_DIR" ] && [ "${#domain_tests[@]}" -gt 0 ]; then
|
|
domain_repo="$(cd "$LUMBDA_DOMAIN_DIR/.." 2>/dev/null && pwd || echo "$LUMBDA_DOMAIN_DIR")"
|
|
pull_out=$(ssh "$REMOTE" "cd $domain_repo; git pull --ff-only origin master 2>&1" 2>&1)
|
|
echo "$pull_out" | sed 's/^/[remote-pull-domain] /'
|
|
if echo "$pull_out" | grep -qE "^(error|Aborting|fatal:)"; then
|
|
echo "[remote-pull-domain] HARD ABORT - pull failed; reducers not run."
|
|
exit 1
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# Run one reducer. Resolves cwd to repo root so (load "lumbda/...") &
|
|
# (load "tests/sweep-doctrine/lib.lsp") both succeed. Upstream reducers
|
|
# run from $LUMBDA_REPO_DIR; domain reducers run from $LUMBDA_DOMAIN_DIR.
|
|
run_one_local_or_remote() {
|
|
local t="$1" repo_root="$2"
|
|
local name rel out cmd
|
|
name="$(basename "$t" .lsp)"
|
|
rel="tests/sweep-doctrine/${name}.lsp"
|
|
# Per-test timeout (2026-06-14 defect fix): a doctrine reducer that does
|
|
# not terminate should fail-fast, not leak compute hours. K=5 doctrine
|
|
# tests on raw-pa substrate exhausted CPU + accumulated lumbda procs on
|
|
# remote until manual SIGTERM. SWEEP_DOCTRINE_TEST_TIMEOUT_S sets per-
|
|
# test wall (default 300s); timeout exits 124 on hit and run.sh logs
|
|
# HARNESS-FAIL via the "no DOCTRINE line" path.
|
|
cmd="cd $repo_root; timeout ${SWEEP_DOCTRINE_TEST_TIMEOUT_S:-300} $LUMBDA_BIN --fast $rel 2>&1"
|
|
if [ -n "$REMOTE" ]; then
|
|
out=$(ssh "$REMOTE" "$cmd" 2>&1 | grep -E "^DOCTRINE ")
|
|
else
|
|
out=$(bash -c "$cmd" 2>&1 | grep -E "^DOCTRINE ")
|
|
fi
|
|
if [ -z "$out" ]; then
|
|
echo "DOCTRINE $name HARNESS-FAIL (no DOCTRINE line emitted)"
|
|
return 2
|
|
fi
|
|
echo "$out"
|
|
# Field-anchored verdict scan. Only bare FAIL / UNEXPECTED-PASS /
|
|
# HARNESS-FAIL in third token position gates a run. Data-line tokens
|
|
# like EXPECTED-FAIL or ARITH-FAIL never gate.
|
|
#
|
|
# Summary-aware override: if any DOCTRINE line whose second token
|
|
# ends in "-summary" emits EXPECTED-FAIL, per-row FAILs are
|
|
# documented-defect evidence (not regression) & do not gate.
|
|
if echo "$out" | awk '
|
|
$1=="DOCTRINE" && $2~/-summary$/ && $3=="EXPECTED-FAIL" {expected_fail=1}
|
|
$1=="DOCTRINE" && $3~/^(FAIL|UNEXPECTED-PASS|HARNESS-FAIL)$/ {found=1}
|
|
END{ exit !(found && !expected_fail) }
|
|
'; then
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
overall_rc=0
|
|
for t in "${upstream_tests[@]}"; do
|
|
run_one_local_or_remote "$t" "$LUMBDA_REPO_DIR" || overall_rc=1
|
|
done
|
|
for t in "${domain_tests[@]}"; do
|
|
run_one_local_or_remote "$t" "$LUMBDA_DOMAIN_DIR" || overall_rc=1
|
|
done
|
|
exit "$overall_rc"
|