55 lines
2.1 KiB
Common Lisp
55 lines
2.1 KiB
Common Lisp
#!/usr/bin/env sbcl --script
|
|
|
|
;;; Hermes AI Streaming Example
|
|
;;; Uses Dexador with :want-stream for SSE streaming
|
|
|
|
(load "~/quicklisp/setup.lisp")
|
|
(ql:quickload '(:dexador :jonathan) :silent t)
|
|
|
|
(defparameter *base-url* "https://hermes.ai.unturf.com/v1/chat/completions")
|
|
(defparameter *model* "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic")
|
|
|
|
(defun process-sse-line (line)
|
|
"Process a single Server-Sent Event line"
|
|
(when (and line (> (length line) 6) (string= (subseq line 0 6) "data: "))
|
|
(let ((data (subseq line 6)))
|
|
(unless (string= data "[DONE]")
|
|
(handler-case
|
|
(let* ((parsed (jonathan:parse data))
|
|
(choices (getf parsed :|choices|))
|
|
(delta (when (> (length choices) 0)
|
|
(getf (aref choices 0) :|delta|)))
|
|
(content (when delta (getf delta :|content|))))
|
|
(when content
|
|
(format t "~A" content)
|
|
(force-output)))
|
|
(error (e) nil))))))
|
|
|
|
(defun make-streaming-request (prompt)
|
|
"Make a streaming chat completion request"
|
|
(let ((payload (jonathan:to-json
|
|
(list :|model| *model*
|
|
:|messages| (vector (list :|role| "user"
|
|
:|content| prompt))
|
|
:|temperature| 0.5
|
|
:|max_tokens| 150
|
|
:|stream| t))))
|
|
(dex:request *base-url*
|
|
:method :post
|
|
:headers '(("Content-Type" . "application/json"))
|
|
:content payload
|
|
:want-stream t
|
|
:stream-callback
|
|
(lambda (stream)
|
|
(loop for line = (read-line stream nil nil)
|
|
while line
|
|
do (process-sse-line line))))))
|
|
|
|
;; Main execution
|
|
(handler-case
|
|
(progn
|
|
(format t "Streaming from Hermes AI...~%~%")
|
|
(make-streaming-request "Give a Python Fizzbuzz solution in one line of code?")
|
|
(format t "~%~%Done!~%"))
|
|
(error (e)
|
|
(format t "~%Error: ~A~%" e)))
|