gpu-worker: pool + queue health in (health) RPC response

bend co-lives with ecdsa-emit-pool on each foxhop production host.
When pool dies but bend stays up, .lsp cells pile un-emitted; bend
sits idle waiting for .ready bins that never arrive. Today's incident
took 30+ min to surface because feeder couldn't tell from bend health
alone — needed a separate SSH+pgrep per host.

Add pool/queue counts to (health) so one RPC returns the full picture:
  (ok (load-avg L) (vram-free-mb V) (uptime-ms U)
      (pool-procs P) (queue-ready R) (queue-emitting E) (queue-done D))

Helpers:
  health-pool-procs        pgrep -cf ecdsa-emit-pool
  health-queue-count EXT   ls $BEND_QUEUE_DIR/*.EXT | wc -l

BEND_QUEUE_DIR env var — set when bend is launched on a host with an
associated pool. Absent → queue counts return -1 (caller treats as
'unknown / not applicable').

Caller now has single-RPC view of bend + pool + queue health; feeder
can drop its separate SSH pool-watchdog probe in favor of the bend
(health) RPC field.
This commit is contained in:
russell@unturf.com 2026-06-09 14:26:26 -04:00
parent 29fcdcf367
commit e294decd74
No known key found for this signature in database

View file

@ -350,11 +350,98 @@
((eof-object? line) 0)
(else (parse-leading-number line))))))))))
;;; -- pool / queue health -------------------------------------------
;;;
;;; bend co-lives with an ecdsa-emit-pool on each foxhop production host.
;;; When the pool dies but bend stays up, .lsp cells pile up un-emitted
;;; and bend sits idle waiting for .ready bins that never arrive. The
;;; feeder daemon polls bend's (health) to know whether to push more
;;; work — adding pool/queue counts here means a single RPC tells the
;;; feeder everything: bend health, pool health, queue depth.
;;;
;;; BEND_QUEUE_DIR env var — set when bend is launched on a host with
;;; an associated pool. Absent on hosts that only run bend without a
;;; pool. When absent, queue counts return -1 (caller treats as
;;; "unknown / not applicable").
(define (run-and-count cmd-args)
;; Spawn cmd-args, read all stdout lines, return integer line count.
;; Returns 0 on spawn failure. Used to count ls / pgrep output.
(cond
((not (file-exists? (car cmd-args))) 0)
(else
(let ((pair (spawn-process-stdio (car cmd-args) (cdr cmd-args))))
(cond
((eq? pair #f) 0)
(else
(let loop ((count 0))
(let ((line (read-line (cdr pair))))
(cond
((or (eq? line #f) (eof-object? line))
(close-port (car pair))
(close-port (cdr pair))
count)
(else (loop (+ count 1))))))))))))
(define (health-pool-procs)
;; Count of ecdsa-emit-pool processes on this host. 0 = pool dead.
(let* ((pair (spawn-process-stdio "/usr/bin/pgrep"
'("-cf" "ecdsa-emit-pool"))))
(cond
((eq? pair #f) 0)
(else
(let ((line (read-line (cdr pair))))
(close-port (car pair))
(close-port (cdr pair))
(cond
((or (eq? line #f) (eof-object? line)) 0)
(else (or (string->number (parse-trim line)) 0))))))))
(define (parse-trim s)
;; Strip leading/trailing whitespace from a single-line string.
(let* ((n (string-length s))
(lo (let ll ((i 0))
(cond
((>= i n) i)
((char-whitespace? (string-ref s i)) (ll (+ i 1)))
(else i))))
(hi (let lh ((i (- n 1)))
(cond
((< i lo) lo)
((char-whitespace? (string-ref s i)) (lh (- i 1)))
(else (+ i 1))))))
(substring s lo hi)))
(define (health-queue-count ext)
;; Count of files matching $BEND_QUEUE_DIR/*.<ext>. Returns -1 when
;; BEND_QUEUE_DIR env is unset (caller treats as unknown).
(let ((qdir (get-environment-variable "BEND_QUEUE_DIR")))
(cond
((or (eq? qdir #f) (string=? qdir "")) -1)
(else
(let* ((pattern (string-append qdir "/*." ext))
(pair (spawn-process-stdio "/bin/sh"
(list "-c"
(string-append "ls " pattern " 2>/dev/null | wc -l")))))
(cond
((eq? pair #f) 0)
(else
(let ((line (read-line (cdr pair))))
(close-port (car pair))
(close-port (cdr pair))
(cond
((or (eq? line #f) (eof-object? line)) 0)
(else (or (string->number (parse-trim line)) 0)))))))))))
(define (handle-health)
(list 'ok
(list 'load-avg (health-load-avg))
(list 'vram-free-mb (health-vram-free-mb))
(list 'uptime-ms (current-time-ms))))
(list 'uptime-ms (current-time-ms))
(list 'pool-procs (health-pool-procs))
(list 'queue-ready (health-queue-count "ready"))
(list 'queue-emitting (health-queue-count "emitting"))
(list 'queue-done (health-queue-count "done"))))
;;; -- dispatch --------------------------------------------------