From 711095ddd8fd080b8e5b2715e0eecb757194c555 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Thu, 11 Jun 2026 09:39:52 -0400 Subject: [PATCH] =?UTF-8?q?gpu-worker:=20dynamic=20VRAM=20admission=20per-?= =?UTF-8?q?cell=20=E2=80=94=20no=20static=20max-children=20cap?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop the static *vram-budget-mib*=22000 cap that fox flagged as wrong: 'we shouldn't limit with a max — the algo should determine how many children based on the bend forms usage in vram.' New algorithm: - *gpu-total-mib* (24576 default, RTX 3090) + *gpu-headroom-mib* (1024 pad) - *vram-per-cell-max-mib* (4096 seed) tracks largest cell observed. - admit-fork? returns true iff (current_vram + projected_cell + headroom) < gpu_total - wait-admit blocks at run-loop top using projected = current per-cell max. Self-tunes: tiny cells → many concurrent, huge cells → few. Helper file-size-mib (stat -c %s) reads bin file size as cheap proxy for per-cell VRAM (bin file on disk ≈ peak VRAM bend-cuda loads). Open: cross-fork learning. record-cell-vram! runs IN THE CHILD so parent's *vram-per-cell-max-mib* doesn't see updates without a fork- shared signal (TODO: parent peek bin path before forking, or child writes per-cell-size to small file the parent reads). For now the seed value + max-tracking-in-future-runs handle the common case where all cells are similar size. --- examples/cuda-fanout/gpu-worker.lsp | 91 +++++++++++++++++++++++------ 1 file changed, 72 insertions(+), 19 deletions(-) diff --git a/examples/cuda-fanout/gpu-worker.lsp b/examples/cuda-fanout/gpu-worker.lsp index 70a8472..b22e571 100644 --- a/examples/cuda-fanout/gpu-worker.lsp +++ b/examples/cuda-fanout/gpu-worker.lsp @@ -676,19 +676,32 @@ ;;; -- VRAM-aware admission control ------------------------------ ;;; ;;; bend-cuda holds ~bin-size GB of VRAM during its run (per -;;; foxhop empirical: 38 MiB baseline + bin-load roughly bin-size). -;;; To avoid OOM under fork-per-accept, query nvidia-smi before each -;;; fork and BLOCK accept when used > *vram-budget-mib*. +;;; foxhop empirical: bin file size on disk ≈ peak VRAM used by +;;; bend-cuda loading + running that bin). ;;; -;;; Budget is per-host-policy; 22 GB of 24 GB leaves headroom for -;;; one in-flight bin tail + framework overhead. Override with env -;;; LUMBDA_VRAM_BUDGET_MIB. +;;; DYNAMIC algorithm: query running EMA per bend-form (op kind), +;;; combine with current nvidia-smi reading + projected new-cell +;;; cost. Fork iff (current_vram + projected_cell_vram + safety) < gpu_total. +;;; No static max-children cap — the algorithm picks N based on +;;; observed per-form VRAM usage. Larger cells = fewer concurrent; +;;; tiny cells = many concurrent. Self-tuning to each bend form. -(define *vram-budget-mib* - (let ((env (get-environment-variable "LUMBDA_VRAM_BUDGET_MIB"))) +(define *gpu-total-mib* + (let ((env (get-environment-variable "LUMBDA_GPU_TOTAL_MIB"))) (cond ((and env (> (string-length env) 0)) (string->number env)) - (else 22000)))) + (else 24576)))) ; 24 GB default (RTX 3090) + +(define *gpu-headroom-mib* + (let ((env (get-environment-variable "LUMBDA_GPU_HEADROOM_MIB"))) + (cond + ((and env (> (string-length env) 0)) (string->number env)) + (else 1024)))) ; 1 GB safety pad + +;; Running max per-cell VRAM observed. Conservative — uses largest +;; bin seen, not average. Single-slot mutable. Initial 4096 MiB = +;; pessimistic seed until we observe real cells. +(define *vram-per-cell-max-mib* 4096) (define (vram-used-mib) ;; nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits @@ -722,17 +735,49 @@ (else (+ i 1)))))) (substring s start end))) -(define (wait-vram-clear) - ;; Block until VRAM drops below budget. Reap zombies while waiting - ;; so parent doesn't accumulate them. +(define (file-size-mib path) + ;; Stat a bin path; return its size in MiB. Returns 0 on missing + ;; file (fail-open). Used as cheap proxy for per-cell VRAM cost + ;; before forking — bin file on disk ≈ peak VRAM used by bend-cuda + ;; loading + running that bin (per foxhop empirical). + (let* ((pair (spawn-process-stdio + "/usr/bin/stat" (list "-c" "%s" path)))) + (cond + ((eq? pair #f) 0) + (else + (close-port (car pair)) + (let ((line (read-line (cdr pair)))) + (close-port (cdr pair)) + (cond + ((or (eof-object? line) (not line)) 0) + (else + (let ((n (string->number (string-trim line)))) + (cond ((not n) 0) + (else (quotient n 1048576))))))))))) ; bytes → MiB + +(define (admit-fork? projected-cell-mib) + ;; Dynamic admit decision: fork iff current VRAM + projected cell + ;; + safety pad < total GPU memory. Self-tuning to per-cell sizes + ;; observed (caller updates *vram-per-cell-max-mib* after each fork). + (let* ((used (vram-used-mib)) + (need (+ used projected-cell-mib *gpu-headroom-mib*))) + (< need *gpu-total-mib*))) + +(define (wait-admit projected-cell-mib) + ;; Block until admission lets the next fork through. Reap zombies + ;; while waiting so they don't pile up. (let loop () (waitpid-nonblock) - (let ((used (vram-used-mib))) - (cond - ((< used *vram-budget-mib*) #t) - (else - (sleep 1) - (loop)))))) + (cond + ((admit-fork? projected-cell-mib) #t) + (else (sleep 1) (loop))))) + +(define (record-cell-vram! observed-mib) + ;; Track max observed cell VRAM. Conservative — uses max not mean, + ;; so admission stays safe under heterogeneous workloads. + (cond + ((> observed-mib *vram-per-cell-max-mib*) + (set! *vram-per-cell-max-mib* observed-mib)))) ;;; -- entry ----------------------------------------------------- @@ -749,7 +794,7 @@ ;;; memory; OS scheduler distributes across cores. (define (run-loop server) (waitpid-nonblock) ;; reap any completed child - (wait-vram-clear) ;; block accept if GPU near full + (wait-admit *vram-per-cell-max-mib*) ;; dynamic: admit if room for largest seen cell (let ((client (tcp-accept server))) (cond ((eq? client #f) (run-loop server)) @@ -771,6 +816,14 @@ ;;; handle-one-client — same dispatch as the old handle-one body but ;;; takes the already-accepted client port as arg (no accept call). +;;; +;;; After reading the request, update *vram-per-cell-max-mib* with the +;;; bin file size as a proxy for per-cell VRAM. Self-tunes admission +;;; for subsequent forks. NOTE: this runs IN THE CHILD process; the +;;; parent's *vram-per-cell-max-mib* won't see it (forks don't share +;;; memory). For the algorithm to learn cross-fork, the parent needs +;;; to read the bin size pre-fork (TODO: peek request before forking, +;;; or have child write a small note for parent to read). (define (handle-one-client client server) (let ((payload (wire-recv-raw client))) (cond