From 6a42dacf65c22982076b4f9b5b7a05caa58b597e Mon Sep 17 00:00:00 2001 From: Russell Ballestrini Date: Sun, 7 Jan 2024 14:57:40 -0500 Subject: [PATCH] localhost openchat is working. lol I replaced gpt-3.5-turbo workloads with openchat a local GPU powered inference server The openchat inference server supports using the latest and official openai python client. This means you can replace both standard and streaming workloads with an "offline" LLM. modified: README.rst modified: app.py --- README.rst | 2 +- app.py | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 944de0a..cc1258f 100644 --- a/README.rst +++ b/README.rst @@ -50,7 +50,7 @@ To set up the project, follow these steps: 2. Create a virtual environment and activate it:: - python3 -m venv ven + python3 -m venv env source env/bin/activate # On Windows use `env\Scripts\activate` 3. Install the required dependencies:: diff --git a/app.py b/app.py index 82e86a9..b36095a 100644 --- a/app.py +++ b/app.py @@ -250,6 +250,7 @@ def handle_message(data): or "gpt-4" in data["message"] or "mistral-" in data["message"] or "together/" in data["message"] + or "localhost/" in data["message"] ): # Emit a temporary message indicating that llm is processing emit( @@ -336,6 +337,15 @@ def handle_message(data): model_name="upstage/SOLAR-10.7B-Instruct-v1.0", stop=["###", ""], ) + if "localhost/openchat" in data["message"]: + eventlet.spawn( + chat_gpt, + data["username"], + room.name, + data["message"], + model_name="openchat_3.5", + ) + @socketio.on("delete_message") @@ -511,7 +521,10 @@ def chat_claude(username, room_name, message, model_name="anthropic.claude-v1"): def chat_gpt(username, room_name, message, model_name="gpt-3.5-turbo"): - openai_client = OpenAI() + if model_name == "openchat_3.5": + openai_client = OpenAI(base_url="http://localhost:18888/v1", api_key="not-needed") + else: + openai_client = OpenAI() limit = 15 if model_name == "gpt-4-1106-preview": limit = 1000