diff --git a/README.rst b/README.rst index 24d6b97..e1bc73b 100644 --- a/README.rst +++ b/README.rst @@ -37,6 +37,7 @@ Requirements - openai (for interacting with OpenAI's language models) - mistralai (for interacting with MistralAI's language models) - together (for interacting with together.ai language models) +- groq (for interacting with very fast groq language models) Installation ------------ @@ -75,6 +76,7 @@ Set up optional environment variables for your AWS, OpenAI, MistralAI, or togeth export OPENAI_API_KEY="your_openai_api_key" export MISTRAL_API_KEY="your_mistralai_api_key" export TOGETHER_API_KEY="your_togetherai_api_key" + export GROQ_API_KEY="your_groq_api_key" export VLLM_API_KEY="not-needed" export VLLM_ENDPOINT="http://localhost:18888/v1" @@ -103,6 +105,8 @@ To interact with the various language models, you can use the following commands - For Together Mistral, send a message with ``together/mistral`` and include your prompt. - For Together Mixtral, send a message with ``together/mixtral`` and include your prompt. - For Together Solar, send a message with ``together/solar`` and include your prompt. +- For Groq Mixtral, send a message with ``groq/mixtral`` and include your prompt. +- For Groq Llama-2, send a message with ``groq/llama2`` and include your prompt. - For vLLM OpenChat, send a message with ``vllm/openchat`` and include your prompt. - For vLLM OpenHermes, send a message with ``vllm/openhermes`` and include your prompt. - For Dall-e-3, send a message with ``dall-e-3`` and include your prompt. diff --git a/app.py b/app.py index ad7ae16..1101c3a 100644 --- a/app.py +++ b/app.py @@ -8,6 +8,8 @@ from mistralai.models.chat_completion import ChatMessage from openai import OpenAI +from groq import Groq + import tiktoken import os @@ -44,6 +46,8 @@ system_users = [ "mistral-medium", "mistralai/Mixtral-8x7B-v0.1", "mistralai/Mistral-7B-Instruct-v0.1", + "mixtral-8x7b-32768", + "llama2-70b-4096", "openchat/openchat-3.5-1210", "openchat/openchat-3.5-0106", "upstage/SOLAR-10.7B-Instruct-v1.0", @@ -258,6 +262,7 @@ def handle_message(data): or "together/" in data["message"] or "localhost/" in data["message"] or "vllm/" in data["message"] + or "groq/" in data["message"] ): # Emit a temporary message indicating that llm is processing emit( @@ -344,6 +349,22 @@ def handle_message(data): model_name="upstage/SOLAR-10.7B-Instruct-v1.0", stop=["###", ""], ) + if "groq/mixtral" in data["message"]: + eventlet.spawn( + chat_groq, + data["username"], + room.name, + data["message"], + model_name="mixtral-8x7b-32768", + ) + if "groq/llama2" in data["message"]: + eventlet.spawn( + chat_groq, + data["username"], + room.name, + data["message"], + model_name="llama2-70b-4096", + ) if "vllm/openchat" in data["message"]: eventlet.spawn( chat_gpt, @@ -582,7 +603,8 @@ def chat_gpt(username, room_name, message, model_name="gpt-3.5-turbo"): chat_history = [ { "role": "system" if msg.username in system_users else "user", - "content": f"{msg.username}: {msg.content}", + #"content": f"{msg.username}: {msg.content}", + "content": msg.content, } for msg in reversed(last_messages) if not msg.is_base64_image() @@ -899,6 +921,112 @@ def chat_together( socketio.emit("delete_processing_message", msg_id, room=room.name) +def chat_groq(username, room_name, message, model_name="mixtral-8x7b-32768"): + + _limit = 15 + if "mixtral" in model_name: + _limit = 50 + + with app.app_context(): + room = get_room(room_name) + last_messages = ( + Message.query.filter_by(room_id=room.id) + .order_by(Message.id.desc()) + .limit(_limit) + .all() + ) + + chat_history = [ + { + "role": "system" if msg.username in system_users else "user", + "content": msg.content, + } + for msg in reversed(last_messages) + if not msg.is_base64_image() + ] + + # Initialize the Groq client + client = Groq() + + buffer = "" # Content buffer for accumulating the chunks + + # Save an empty message to get an ID for the chunks + with app.app_context(): + new_message = Message(username=model_name, content=buffer, room_id=room.id) + db.session.add(new_message) + db.session.commit() + msg_id = new_message.id + + first_chunk = True + + try: + # Use the Groq client to stream the chat completion + stream = client.chat.completions.create( + messages=chat_history, + model=model_name, + stream=True, + ) + + for chunk in stream: + content_chunk = chunk.choices[0].delta.content + + if content_chunk: + buffer += content_chunk # Accumulate content + + if first_chunk: + socketio.emit( + "message_chunk", + { + "id": msg_id, + "content": f"**{username} ({model_name}):**\n\n{content_chunk}", + }, + room=room.name, + ) + first_chunk = False + else: + socketio.emit( + "message_chunk", + {"id": msg_id, "content": content_chunk}, + room=room.name, + ) + socketio.sleep(0) # Force immediate handling + + except Exception as e: + with app.app_context(): + message_content = f"Groq Error: {e}" + new_message = ( + db.session.query(Message).filter(Message.id == msg_id).one_or_none() + ) + if new_message: + new_message.content = message_content + new_message.count_tokens() + db.session.add(new_message) + db.session.commit() + socketio.emit( + "message", + { + "id": msg_id, + "username": model_name, + "content": message_content, + }, + room=room.name, + ) + return None + + # Save the entire completion to the database + with app.app_context(): + new_message = ( + db.session.query(Message).filter(Message.id == msg_id).one_or_none() + ) + if new_message: + new_message.content = buffer + new_message.count_tokens() + db.session.add(new_message) + db.session.commit() + + socketio.emit("delete_processing_message", msg_id, room=room.name) + + def chat_llama( username, room_name, message, model_name="mistral-7b-instruct-v0.2.Q3_K_L.gguf" ): diff --git a/requirements.txt b/requirements.txt index 8452ac6..a9078f7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ eventlet mistralai together openai +groq tiktoken #llama-cpp-python[server]