russell@unturf.com
1665893321
29 new files publish factory infra (V2 autoscaler with live VRAM sampling + EWMA peak tracking, HUGE solo-dispatch, two-tier DLQ/rDLQ classifier + retry), general quantum circuit primitives (Cuccaro ripple-carry adder, Clifford gate library, Clifford tableau simulator, mod-arith family, dialog GCD reversible inverse, Karatsuba multiplier, Solinas fast reduction), and a TCRAUDT reducer harness. Originally developed in ~/git/www.foxhop.net/ecdsa/ for secp256k1 attack-surface research; published upstream as obligated by AGPLv3. Parametrization contract at factory/CONTRACT.md. Consumers export LUMBDA_REPO_DIR + LUMBDA_QUEUE_DIR + LUMBDA_BACKEND_CMD + LUMBDA_EMITTER_CMD then exec factory scripts. No fork-and-modify; single source of truth upstream. Integration tests gate 7 V2 defect classes that wedged a live factory on 2026-06-12 (skewed-demand starve, zero-floor reservation, multi-tier greedy, +-25%% damping, cold-start ramp, DLQ surge halve, post-damp CPU ceiling) + 28 DLQ classifier cases (auto-retry vs escalate partition) + bash -n syntax lint across every script. GPU backend stays in consumer trees; rationale in factory/GPU-BACKEND-NOTE.md. Bend wire protocol + gpu-worker.lsp already upstream at examples/cuda-fanout/. make factory-lint bash -n on every factory/*.sh make test-integration V2 reducer + DLQ classifier + syntax gate make sweep-doctrine TCRAUDT reducer gate (serial) make sweep-doctrine-parallel xargs -P fan-out Verified on neoblanka: factory-lint 12 scripts PASS; test-integration 14 V2 cases + 28 DLQ classifier cases + 12 syntax cases all PASS.
108 lines
4.2 KiB
Bash
Executable file
108 lines
4.2 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# lib-heartbeat.sh — shared library for factory liveness signals.
|
|
#
|
|
# Sourced by bend-emit-pool.sh, bend-dispatcher.sh, bend-supervisor.sh,
|
|
# bend-autoscaler.sh, bend-supervisor-dlq-runner.sh. Pure POSIX-ish bash;
|
|
# no external daemons; portable to busybox-bash + flock + setsid + nohup.
|
|
#
|
|
# Env vars consumed: none. See $LUMBDA_FACTORY_DIR/CONTRACT.md for full
|
|
# factory-wide knob list.
|
|
#
|
|
# Contract:
|
|
# - heartbeat_touch <state_dir> <service_name>
|
|
# Writes "<pid> <unix_ts>" to <state_dir>/<service_name>.heartbeat.
|
|
# Called at top of every loop iteration; mtime + content authoritative.
|
|
# Returns 0 on success, 1 on write failure.
|
|
#
|
|
# - heartbeat_is_alive <state_dir> <service_name> <stale_threshold_sec>
|
|
# Returns 0 if BOTH:
|
|
# (a) heartbeat file mtime within <stale_threshold_sec> of now
|
|
# (b) pid recorded in heartbeat file responds to kill -0
|
|
# Returns 1 otherwise (missing file, stale mtime, dead pid).
|
|
# (a) gates daemon liveness; (b) gates against post-mortem stale files.
|
|
#
|
|
# - heartbeat_exit_cause <state_dir> <service_name> <reason>
|
|
# Appends "<utc_iso8601> <pid> <reason>" to
|
|
# <state_dir>/<service_name>.exit-cause, removes <service_name>.heartbeat.
|
|
# Caller invokes from every exit path (STOP marker, signal trap, drain
|
|
# timeout, error abort). Operators read .exit-cause to triage post-mortem.
|
|
#
|
|
# File formats:
|
|
# <service>.heartbeat — single line: "<pid> <unix_ts>" (whitespace-sep)
|
|
# <service>.exit-cause — append-only log, one line per exit:
|
|
# "<utc_iso8601> <pid> <reason>"
|
|
#
|
|
# Backwards compatibility: legacy per-worker dispatcher.heartbeat-<wid>
|
|
# files used freeform text content + only mtime for liveness. heartbeat_is_alive
|
|
# falls back to mtime-only when a heartbeat file does not parse as
|
|
# "<pid> <unix_ts>".
|
|
#
|
|
# Caller responsibilities:
|
|
# - mkdir -p "$state_dir" BEFORE calling any function (library does NOT
|
|
# auto-create; refusing to silently swallow path typos).
|
|
# - Wrap heartbeat_touch in an EXIT trap that calls heartbeat_exit_cause
|
|
# so an unexpected crash still leaves a forensic trail.
|
|
|
|
# heartbeat_touch <state_dir> <service_name>
|
|
heartbeat_touch() {
|
|
local state_dir="$1"
|
|
local service_name="$2"
|
|
local hb_file="${state_dir}/${service_name}.heartbeat"
|
|
printf '%s %s\n' "$$" "$(date +%s)" > "$hb_file" 2>/dev/null
|
|
}
|
|
|
|
# heartbeat_is_alive <state_dir> <service_name> <stale_threshold_sec>
|
|
# Returns 0 alive, 1 dead/stale.
|
|
heartbeat_is_alive() {
|
|
local state_dir="$1"
|
|
local service_name="$2"
|
|
local threshold="$3"
|
|
local hb_file="${state_dir}/${service_name}.heartbeat"
|
|
|
|
[ -f "$hb_file" ] || return 1
|
|
|
|
# mtime gate
|
|
local now mtime age
|
|
now=$(date +%s)
|
|
mtime=$(stat -c %Y "$hb_file" 2>/dev/null) || return 1
|
|
age=$(( now - mtime ))
|
|
[ "$age" -le "$threshold" ] || return 1
|
|
|
|
# pid gate: parse first token. Tolerate freeform legacy content by
|
|
# treating non-numeric first token as "mtime-only mode" — return 0 if
|
|
# mtime gate already passed.
|
|
local first_token
|
|
first_token=$(awk 'NR==1 {print $1}' "$hb_file" 2>/dev/null)
|
|
if [ -z "$first_token" ]; then
|
|
return 0
|
|
fi
|
|
case "$first_token" in
|
|
''|*[!0-9]*)
|
|
# non-numeric (e.g., ISO timestamp from legacy writer) → mtime-only
|
|
return 0
|
|
;;
|
|
esac
|
|
|
|
# Numeric pid: must respond to kill -0. Cross-host caveat: this library
|
|
# runs local to whichever process reads our heartbeat. heartbeat_is_alive
|
|
# called against a remote host's state dir over SSH must run on that
|
|
# host (kill -0 lives in local pid space only). Supervisor + status
|
|
# callers SSH per-host before calling, so this holds.
|
|
if kill -0 "$first_token" 2>/dev/null; then
|
|
return 0
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
# heartbeat_exit_cause <state_dir> <service_name> <reason>
|
|
heartbeat_exit_cause() {
|
|
local state_dir="$1"
|
|
local service_name="$2"
|
|
local reason="$3"
|
|
local ec_file="${state_dir}/${service_name}.exit-cause"
|
|
local hb_file="${state_dir}/${service_name}.heartbeat"
|
|
local ts
|
|
ts=$(date -u +%FT%TZ)
|
|
printf '%s %s %s\n' "$ts" "$$" "$reason" >> "$ec_file" 2>/dev/null
|
|
rm -f "$hb_file" 2>/dev/null
|
|
}
|