russell@unturf.com
1665893321
29 new files publish factory infra (V2 autoscaler with live VRAM sampling + EWMA peak tracking, HUGE solo-dispatch, two-tier DLQ/rDLQ classifier + retry), general quantum circuit primitives (Cuccaro ripple-carry adder, Clifford gate library, Clifford tableau simulator, mod-arith family, dialog GCD reversible inverse, Karatsuba multiplier, Solinas fast reduction), and a TCRAUDT reducer harness. Originally developed in ~/git/www.foxhop.net/ecdsa/ for secp256k1 attack-surface research; published upstream as obligated by AGPLv3. Parametrization contract at factory/CONTRACT.md. Consumers export LUMBDA_REPO_DIR + LUMBDA_QUEUE_DIR + LUMBDA_BACKEND_CMD + LUMBDA_EMITTER_CMD then exec factory scripts. No fork-and-modify; single source of truth upstream. Integration tests gate 7 V2 defect classes that wedged a live factory on 2026-06-12 (skewed-demand starve, zero-floor reservation, multi-tier greedy, +-25%% damping, cold-start ramp, DLQ surge halve, post-damp CPU ceiling) + 28 DLQ classifier cases (auto-retry vs escalate partition) + bash -n syntax lint across every script. GPU backend stays in consumer trees; rationale in factory/GPU-BACKEND-NOTE.md. Bend wire protocol + gpu-worker.lsp already upstream at examples/cuda-fanout/. make factory-lint bash -n on every factory/*.sh make test-integration V2 reducer + DLQ classifier + syntax gate make sweep-doctrine TCRAUDT reducer gate (serial) make sweep-doctrine-parallel xargs -P fan-out Verified on neoblanka: factory-lint 12 scripts PASS; test-integration 14 V2 cases + 28 DLQ classifier cases + 12 syntax cases all PASS.
203 lines
9.2 KiB
Bash
Executable file
203 lines
9.2 KiB
Bash
Executable file
#!/usr/bin/env bash
|
||
# lib-tier.sh — bin-size tier classifier + policy table.
|
||
#
|
||
# Bins sized below MIN or above MAX go to DLQ unconditionally. In-range
|
||
# bins get classified into a tier name. Each tier carries a default
|
||
# concurrency cap pair (POOL_MAX, DISPATCH) that supervisor / pool /
|
||
# dispatcher consult — overridable via $LUMBDA_QUEUE_DIR/supervisor.config
|
||
# `TIER_<NAME>_*` keys for hot-reload tuning.
|
||
#
|
||
# Why tiers: a backend's CPU+VRAM residency scales ~linearly with bin
|
||
# size. A 24-way fleet that fits 250-MB bins crushes a box on 2.4-GB
|
||
# bins. Per-tier admission lets a factory autopilot through a mixed
|
||
# workload without dead-zoning either extreme.
|
||
#
|
||
# Pure bash — no jq / awk / python deps, sourceable by every factory
|
||
# component.
|
||
#
|
||
# Env vars consumed: see $LUMBDA_FACTORY_DIR/CONTRACT.md for our canonical
|
||
# table. TIER_* knobs (boundaries + per-tier caps + per-tier MIB estimates)
|
||
# all readable via env override; defaults below match a 24-core / 62 GB RAM
|
||
# / 24 GB VRAM workstation profile.
|
||
|
||
# ── tier boundaries (bytes) ──────────────────────────────────────
|
||
# A bin's tier = lowest band whose MAX it does NOT exceed.
|
||
# A bin below TIER_MICRO_MIN gets DLQ'd as "too-small" (broken emit).
|
||
# A bin above TIER_LARGE_MAX gets DLQ'd as "too-large" (would OOM a fleet).
|
||
TIER_MICRO_MIN="${TIER_MICRO_MIN:-33554432}" # 32 MB — emit-broken floor
|
||
TIER_MICRO_MAX="${TIER_MICRO_MAX:-268435456}" # 256 MB
|
||
TIER_SMALL_MAX="${TIER_SMALL_MAX:-536870912}" # 512 MB
|
||
TIER_MEDIUM_MAX="${TIER_MEDIUM_MAX:-2147483648}" # 2 GB
|
||
TIER_LARGE_MAX="${TIER_LARGE_MAX:-4294967296}" # 4 GB — co-resident ceiling
|
||
# huge tier: bins above LARGE that still fit a fleet but only when
|
||
# a card has no co-tenants. Largest candidate bins land here (6-8 GB
|
||
# observed). HUGE_MAX is "single bin needs ALL of VRAM" — our autoscaler
|
||
# must drain other tiers before dispatching huge.
|
||
# At 8 GB bin × 1.55 ratio = ~12.4 GB RSS, well under 24 GB VRAM / 62 GB
|
||
# RAM ceilings of a typical workstation when run alone.
|
||
TIER_HUGE_MAX="${TIER_HUGE_MAX:-12884901888}" # 12 GB — solo-dispatch ceiling
|
||
|
||
# ── default concurrency caps per tier ────────────────────────────
|
||
# Numbers tuned for a 24-core, 62 GB RAM, 24 GB VRAM workstation.
|
||
# Override via supervisor.config: TIER_<NAME>_POOL_MAX / TIER_<NAME>_DISPATCH.
|
||
# Autoscaler may rewrite these based on live RAM headroom.
|
||
TIER_MICRO_POOL_MAX="${TIER_MICRO_POOL_MAX:-24}"
|
||
TIER_MICRO_DISPATCH="${TIER_MICRO_DISPATCH:-24}"
|
||
TIER_SMALL_POOL_MAX="${TIER_SMALL_POOL_MAX:-16}"
|
||
TIER_SMALL_DISPATCH="${TIER_SMALL_DISPATCH:-16}"
|
||
TIER_MEDIUM_POOL_MAX="${TIER_MEDIUM_POOL_MAX:-12}"
|
||
TIER_MEDIUM_DISPATCH="${TIER_MEDIUM_DISPATCH:-12}"
|
||
TIER_LARGE_POOL_MAX="${TIER_LARGE_POOL_MAX:-6}"
|
||
TIER_LARGE_DISPATCH="${TIER_LARGE_DISPATCH:-6}"
|
||
# HUGE: solo-dispatch tier. Both caps = 1; one huge bin at a time.
|
||
# Autoscaler must enforce "no other tier dispatches in flight before
|
||
# admitting a huge dispatch" — cap=1 alone does not guarantee that
|
||
# (small + huge could overlap if scheduled naively).
|
||
TIER_HUGE_POOL_MAX="${TIER_HUGE_POOL_MAX:-1}"
|
||
TIER_HUGE_DISPATCH="${TIER_HUGE_DISPATCH:-1}"
|
||
|
||
# ── per-tier per-dispatch memory estimates (MiB) ──────────────────
|
||
# Used by our autoscaler to compute "total estimated memory at current
|
||
# caps" for tier-mix admission. Calibrated from /usr/bin/time -v on a
|
||
# backend running against representative bins, scaled to each tier ceiling.
|
||
#
|
||
# Sample calibration data (peak RSS):
|
||
# 683 MB bin → 1.05 GB RSS (ratio 1.54×) → MEDIUM tier sample
|
||
# 2.4 GB bin → 3.55 GB RSS (ratio 1.48×) → LARGE tier sample
|
||
#
|
||
# CPU_MIB = tier-ceiling × 1.55× × small safety pad, so a tier-full bin
|
||
# never exceeds an admitted budget.
|
||
# MICRO ceil 256 MB × 1.55 → ~397 → round 512
|
||
# SMALL ceil 512 MB × 1.55 → ~794 → round 1024
|
||
# MEDIUM ceil 2 GB × 1.55 → ~3174 → round 3328
|
||
# LARGE ceil 4 GB × 1.55 → ~6349 → round 6400
|
||
TIER_MICRO_CPU_MIB="${TIER_MICRO_CPU_MIB:-512}"
|
||
TIER_MICRO_VRAM_MIB="${TIER_MICRO_VRAM_MIB:-512}"
|
||
TIER_SMALL_CPU_MIB="${TIER_SMALL_CPU_MIB:-1024}"
|
||
TIER_SMALL_VRAM_MIB="${TIER_SMALL_VRAM_MIB:-1024}"
|
||
TIER_MEDIUM_CPU_MIB="${TIER_MEDIUM_CPU_MIB:-3328}"
|
||
TIER_MEDIUM_VRAM_MIB="${TIER_MEDIUM_VRAM_MIB:-2048}"
|
||
TIER_LARGE_CPU_MIB="${TIER_LARGE_CPU_MIB:-6400}"
|
||
TIER_LARGE_VRAM_MIB="${TIER_LARGE_VRAM_MIB:-3072}"
|
||
# HUGE: 12 GB bin × 1.55 ratio ≈ 19000 MiB RSS at peak. Numbers below
|
||
# act as upper-bound "needs whole card" — autoscaler treats them as
|
||
# a wholesale claim, not adds-to-budget like other tiers.
|
||
TIER_HUGE_CPU_MIB="${TIER_HUGE_CPU_MIB:-19000}"
|
||
TIER_HUGE_VRAM_MIB="${TIER_HUGE_VRAM_MIB:-20480}"
|
||
|
||
# tier_of_size BYTES → echoes one of:
|
||
# below-min bin too small; DLQ as broken emit
|
||
# micro in-band [TIER_MICRO_MIN, TIER_MICRO_MAX]
|
||
# small (TIER_MICRO_MAX, TIER_SMALL_MAX]
|
||
# medium (TIER_SMALL_MAX, TIER_MEDIUM_MAX]
|
||
# large (TIER_MEDIUM_MAX, TIER_LARGE_MAX]
|
||
# huge (TIER_LARGE_MAX, TIER_HUGE_MAX] — solo-dispatch
|
||
# above-max bin too large; DLQ as fleet overflow
|
||
tier_of_size() {
|
||
local sz="$1"
|
||
if [ -z "$sz" ] || [ "$sz" -lt "$TIER_MICRO_MIN" ] 2>/dev/null; then
|
||
echo "below-min"; return
|
||
fi
|
||
if [ "$sz" -le "$TIER_MICRO_MAX" ]; then echo "micro"; return; fi
|
||
if [ "$sz" -le "$TIER_SMALL_MAX" ]; then echo "small"; return; fi
|
||
if [ "$sz" -le "$TIER_MEDIUM_MAX" ]; then echo "medium"; return; fi
|
||
if [ "$sz" -le "$TIER_LARGE_MAX" ]; then echo "large"; return; fi
|
||
if [ "$sz" -le "$TIER_HUGE_MAX" ]; then echo "huge"; return; fi
|
||
echo "above-max"
|
||
}
|
||
|
||
# Re-read tier caps from supervisor.config each call so autoscaler tweaks
|
||
# land in long-running dispatcher / pool workers WITHOUT a process restart.
|
||
# Old behavior baked TIER_* env at fork time → autoscaler had to kill the
|
||
# dispatcher to update caps → 30 s downtime per autoscale event. With
|
||
# live-reload, autoscaler edits supervisor.config + next tier_dispatch /
|
||
# tier_pool_max call sees a new value with no restart.
|
||
#
|
||
# Cost: sourcing a small KEY=VALUE file via grep+eval per call costs a
|
||
# few ms. tier_dispatch fires once per cell claim ≈ every 10-30 s per
|
||
# worker, so overhead stays invisible.
|
||
#
|
||
# Gated by TIER_CONFIG_LIVE_RELOAD=1 (default ON); set to 0 to fall back
|
||
# to env-only behavior (legacy / debugging).
|
||
TIER_CONFIG_FILE="${TIER_CONFIG_FILE:-${LUMBDA_QUEUE_DIR:-/tmp/lumbda-queue}/supervisor.config}"
|
||
TIER_CONFIG_LIVE_RELOAD="${TIER_CONFIG_LIVE_RELOAD:-1}"
|
||
|
||
tier_reload_caps_from_file() {
|
||
[ "$TIER_CONFIG_LIVE_RELOAD" = "1" ] || return 0
|
||
[ -f "$TIER_CONFIG_FILE" ] || return 0
|
||
# Eval only TIER_* lines from a config file — never inherit
|
||
# unrelated vars (POOL_MAX etc. stay supervisor-only, change those
|
||
# via supervisor restart).
|
||
eval "$(grep -E '^TIER_[A-Z_]+=' "$TIER_CONFIG_FILE" 2>/dev/null)"
|
||
}
|
||
|
||
# tier_pool_max NAME → echoes integer cap (0 for DLQ tiers)
|
||
tier_pool_max() {
|
||
tier_reload_caps_from_file
|
||
case "$1" in
|
||
micro) echo "$TIER_MICRO_POOL_MAX" ;;
|
||
small) echo "$TIER_SMALL_POOL_MAX" ;;
|
||
medium) echo "$TIER_MEDIUM_POOL_MAX" ;;
|
||
large) echo "$TIER_LARGE_POOL_MAX" ;;
|
||
huge) echo "$TIER_HUGE_POOL_MAX" ;;
|
||
*) echo 0 ;;
|
||
esac
|
||
}
|
||
|
||
# tier_dispatch NAME → echoes integer cap (0 for DLQ tiers)
|
||
tier_dispatch() {
|
||
tier_reload_caps_from_file
|
||
case "$1" in
|
||
micro) echo "$TIER_MICRO_DISPATCH" ;;
|
||
small) echo "$TIER_SMALL_DISPATCH" ;;
|
||
medium) echo "$TIER_MEDIUM_DISPATCH" ;;
|
||
large) echo "$TIER_LARGE_DISPATCH" ;;
|
||
huge) echo "$TIER_HUGE_DISPATCH" ;;
|
||
*) echo 0 ;;
|
||
esac
|
||
}
|
||
|
||
# tier_cpu_mib NAME → estimated CPU RSS per dispatch in MiB
|
||
tier_cpu_mib() {
|
||
case "$1" in
|
||
micro) echo "$TIER_MICRO_CPU_MIB" ;;
|
||
small) echo "$TIER_SMALL_CPU_MIB" ;;
|
||
medium) echo "$TIER_MEDIUM_CPU_MIB" ;;
|
||
large) echo "$TIER_LARGE_CPU_MIB" ;;
|
||
huge) echo "$TIER_HUGE_CPU_MIB" ;;
|
||
*) echo 0 ;;
|
||
esac
|
||
}
|
||
|
||
# tier_vram_mib NAME → estimated VRAM per dispatch in MiB
|
||
tier_vram_mib() {
|
||
case "$1" in
|
||
micro) echo "$TIER_MICRO_VRAM_MIB" ;;
|
||
small) echo "$TIER_SMALL_VRAM_MIB" ;;
|
||
medium) echo "$TIER_MEDIUM_VRAM_MIB" ;;
|
||
large) echo "$TIER_LARGE_VRAM_MIB" ;;
|
||
huge) echo "$TIER_HUGE_VRAM_MIB" ;;
|
||
*) echo 0 ;;
|
||
esac
|
||
}
|
||
|
||
# tier_solo NAME → 1 when a tier needs an empty card (no co-tenant
|
||
# dispatches); 0 when it shares a fleet with other tiers. Used by our
|
||
# autoscaler / dispatcher: before admitting a `huge` dispatch, poll all
|
||
# other tiers + verify zero inflight. lib-tier owns CLASSIFICATION;
|
||
# a controller owns ENFORCEMENT.
|
||
tier_solo() {
|
||
case "$1" in
|
||
huge) echo 1 ;;
|
||
*) echo 0 ;;
|
||
esac
|
||
}
|
||
|
||
# tier_label_dlq NAME → reason string for DLQ (or empty if not DLQ)
|
||
tier_label_dlq() {
|
||
case "$1" in
|
||
below-min) echo "bin-too-small (under tier-micro min ${TIER_MICRO_MIN} bytes) — emit produced no usable circuit" ;;
|
||
above-max) echo "bin-too-large (over tier-huge max ${TIER_HUGE_MAX} bytes) — would exceed fleet RAM/VRAM ceiling even with solo dispatch" ;;
|
||
*) echo "" ;;
|
||
esac
|
||
}
|