fix elixir httpoison SSE parsing (String.replace_prefix), document openai_ex streaming blocks for full response
This commit is contained in:
parent
b5bead65cb
commit
fc4b021b7f
4 changed files with 44 additions and 8 deletions
|
|
@ -33,5 +33,9 @@ COPY --from=build /app/lib /app/lib
|
|||
COPY --from=build /app/run.exs /app/run.exs
|
||||
COPY --from=build /app/mix.exs /app/mix.exs
|
||||
|
||||
# Set mix environment to skip Mix.install
|
||||
ENV MIX_ENV=prod
|
||||
ENV ERL_FLAGS="-pa _build/prod/lib/*/ebin -pa deps/*/ebin"
|
||||
|
||||
# Run the application
|
||||
CMD ["elixir", "run.exs"]
|
||||
CMD ["elixir", "-pa", "_build/prod/lib/*/ebin", "-pa", "deps/*/ebin", "run.exs"]
|
||||
|
|
|
|||
|
|
@ -174,7 +174,7 @@ defmodule UncloseAI do
|
|||
contents =
|
||||
lines
|
||||
|> Enum.filter(&String.starts_with?(&1, "data: "))
|
||||
|> Enum.map(&String.slice(&1, 6..-1))
|
||||
|> Enum.map(&String.replace_prefix(&1, "data: ", ""))
|
||||
|> Enum.reject(&(&1 == "[DONE]"))
|
||||
|> Enum.map(&extract_content/1)
|
||||
|> Enum.reject(&is_nil/1)
|
||||
|
|
|
|||
|
|
@ -1,10 +1,12 @@
|
|||
#!/usr/bin/env elixir
|
||||
|
||||
# Load dependencies
|
||||
Mix.install([
|
||||
{:httpoison, "~> 2.2"},
|
||||
{:jason, "~> 1.4"}
|
||||
])
|
||||
# Load dependencies (skip Mix.install when running via Docker with pre-compiled deps)
|
||||
unless System.get_env("MIX_ENV") == "prod" do
|
||||
Mix.install([
|
||||
{:httpoison, "~> 2.2"},
|
||||
{:jason, "~> 1.4"}
|
||||
])
|
||||
end
|
||||
|
||||
# Load the module
|
||||
Code.require_file("lib/uncloseai.ex", __DIR__)
|
||||
|
|
|
|||
|
|
@ -54,7 +54,37 @@ end
|
|||
|
||||
# Streaming chat with Model 1
|
||||
IO.puts("=== Streaming Chat (Model 1) ===")
|
||||
IO.puts("[SKIPPED] Streaming not compatible with vLLM endpoints in openai_ex 0.9.18\n")
|
||||
|
||||
stream_req1 = OpenaiEx.Chat.Completions.new(
|
||||
model: model_1_id,
|
||||
messages: [
|
||||
%{role: "user", content: "Count from 1 to 20, one number per line."}
|
||||
],
|
||||
temperature: 0.5,
|
||||
max_tokens: 150,
|
||||
stream: true
|
||||
)
|
||||
|
||||
IO.write("Response (with timing): ")
|
||||
start_time = :os.system_time(:millisecond)
|
||||
first_chunk_time = nil
|
||||
|
||||
case OpenaiEx.Chat.Completions.create(client1, stream_req1) do
|
||||
{:ok, stream} ->
|
||||
stream
|
||||
|> Stream.with_index()
|
||||
|> Stream.each(fn {chunk, index} ->
|
||||
current_time = :os.system_time(:millisecond)
|
||||
if index == 0 do
|
||||
IO.puts("\n[First chunk at #{current_time - start_time}ms]")
|
||||
end
|
||||
IO.puts("[Chunk #{index} at #{current_time - start_time}ms]: #{inspect(chunk)}")
|
||||
end)
|
||||
|> Stream.run()
|
||||
IO.puts("\n")
|
||||
{:error, error} ->
|
||||
IO.puts("Error: #{inspect(error)}\n")
|
||||
end
|
||||
|
||||
# Non-streaming chat with Model 2
|
||||
IO.puts("=== Non-Streaming Chat (Model 2) ===")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue