gpu-worker: dispatcher-procs in (health) RPC response
Mirror of pool-procs added earlier — bend health now reports both pool-procs and dispatcher-procs counts so a single RPC tells the feeder/factory consumer whether either supervisor is dead while queue has work. Failure mode this fixes: 4090's bend-dispatcher hit MAX_IDLE_LOOPS drain-exit; balance moved 5 cells into its queue but no dispatcher to consume them. Bend health response previously didn't surface the gap; feeder's separate SSH probe (now added in www.foxhop.net commit) caught it but a single RPC is cheaper than per-host SSH.
This commit is contained in:
parent
f53894df11
commit
6af72c706c
1 changed files with 17 additions and 0 deletions
|
|
@ -397,6 +397,22 @@
|
|||
((or (eq? line #f) (eof-object? line)) 0)
|
||||
(else (or (string->number (parse-trim line)) 0))))))))
|
||||
|
||||
(define (health-dispatcher-procs)
|
||||
;; Count of bend-dispatcher processes on this host. <=1 = dispatcher
|
||||
;; functionally dead (only stale wrapper bash matched). Live
|
||||
;; multi-dispatcher = parent + 2 worker forks = ≥3 procs.
|
||||
(let* ((pair (spawn-process-stdio "/usr/bin/pgrep"
|
||||
'("-cf" "bend-dispatcher"))))
|
||||
(cond
|
||||
((eq? pair #f) 0)
|
||||
(else
|
||||
(let ((line (read-line (cdr pair))))
|
||||
(close-port (car pair))
|
||||
(close-port (cdr pair))
|
||||
(cond
|
||||
((or (eq? line #f) (eof-object? line)) 0)
|
||||
(else (or (string->number (parse-trim line)) 0))))))))
|
||||
|
||||
(define (parse-trim s)
|
||||
;; Strip leading/trailing whitespace from a single-line string.
|
||||
(let* ((n (string-length s))
|
||||
|
|
@ -439,6 +455,7 @@
|
|||
(list 'vram-free-mb (health-vram-free-mb))
|
||||
(list 'uptime-ms (current-time-ms))
|
||||
(list 'pool-procs (health-pool-procs))
|
||||
(list 'dispatcher-procs (health-dispatcher-procs))
|
||||
(list 'queue-ready (health-queue-count "ready"))
|
||||
(list 'queue-emitting (health-queue-count "emitting"))
|
||||
(list 'queue-done (health-queue-count "done"))))
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue