gpu-worker: guard portal-timing/find-number against #f form
Root cause for repeated bend gpu-worker crashes mid-pipeline: ;;; bend RECV cuda-sim-ops-bin ops=monitor-r00007-c006.bin error: not a pair: #f When demo_ops crashes mid-write (GPU OOM, segfault, etc) the portal file may exist but be empty or malformed. read-from-string returns #f. Both portal-timing + portal-find-number then call (cdr #f) = runtime crash, taking the port-8320 listener down. Fix: gate both walkers on (pair? form) BEFORE descending. When form is not a pair (= #f from empty/malformed portal), return #f cleanly. Caller's (or cpu-ms 'NA) display already handles #f. Watchdog still exists for true OOM / kill -9, but routine empty-portal events no longer take bend down. Reproduces by sending a malformed S-expr to portal-find-number or portal-timing.
This commit is contained in:
parent
1731304ed8
commit
3599c23f23
1 changed files with 25 additions and 13 deletions
|
|
@ -215,22 +215,34 @@
|
|||
;; Pull a numeric value out of a parsed portal expression for telemetry.
|
||||
;; Walks (a b c ...) looking for (name <number>); returns #f if missing.
|
||||
(define (portal-find-number form name)
|
||||
(let loop ((rest (cdr form)))
|
||||
(cond
|
||||
((null? rest) #f)
|
||||
((and (pair? (car rest)) (eq? (car (car rest)) name)
|
||||
(pair? (cdr (car rest))) (number? (car (cdr (car rest)))))
|
||||
(car (cdr (car rest))))
|
||||
(else (loop (cdr rest))))))
|
||||
;; Guard: form may arrive as #f when read-from-string sees an empty
|
||||
;; or malformed portal file (demo_ops crashed mid-write, OOM, etc).
|
||||
;; A bare (cdr #f) here historically crashed the gpu-worker with
|
||||
;; "error: not a pair: #f", taking the port-8320 listener down.
|
||||
(cond
|
||||
((not (pair? form)) #f)
|
||||
(else
|
||||
(let loop ((rest (cdr form)))
|
||||
(cond
|
||||
((null? rest) #f)
|
||||
((not (pair? rest)) #f)
|
||||
((and (pair? (car rest)) (eq? (car (car rest)) name)
|
||||
(pair? (cdr (car rest))) (number? (car (cdr (car rest)))))
|
||||
(car (cdr (car rest))))
|
||||
(else (loop (cdr rest))))))))
|
||||
|
||||
;; Walk through (timing-ms (cpu-total X) (gpu-kernel Y)) shape.
|
||||
(define (portal-timing form which)
|
||||
(let loop ((rest (cdr form)))
|
||||
(cond
|
||||
((null? rest) #f)
|
||||
((and (pair? (car rest)) (eq? (car (car rest)) 'timing-ms))
|
||||
(portal-find-number (car rest) which))
|
||||
(else (loop (cdr rest))))))
|
||||
(cond
|
||||
((not (pair? form)) #f)
|
||||
(else
|
||||
(let loop ((rest (cdr form)))
|
||||
(cond
|
||||
((null? rest) #f)
|
||||
((not (pair? rest)) #f)
|
||||
((and (pair? (car rest)) (eq? (car (car rest)) 'timing-ms))
|
||||
(portal-find-number (car rest) which))
|
||||
(else (loop (cdr rest))))))))
|
||||
|
||||
(define (handle-cuda-sim-ops-bin args)
|
||||
(let* ((ops-path (car args))
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue