Added groq platform support for ultra fast LLM inference

modified:   README.rst
	modified:   app.py
	modified:   requirements.txt

README.rst

@@ -37,6 +37,7 @@ Requirements
 - openai (for interacting with OpenAI's language models)
 - mistralai (for interacting with MistralAI's language models)
 - together (for interacting with together.ai language models)
+- groq (for interacting with very fast groq language models)
 
 Installation
 ------------
@@ -75,6 +76,7 @@ Set up optional environment variables for your AWS, OpenAI, MistralAI, or togeth
     export OPENAI_API_KEY="your_openai_api_key"
     export MISTRAL_API_KEY="your_mistralai_api_key"
     export TOGETHER_API_KEY="your_togetherai_api_key"
+    export GROQ_API_KEY="your_groq_api_key"
     export VLLM_API_KEY="not-needed"
     export VLLM_ENDPOINT="http://localhost:18888/v1"
 
@@ -103,6 +105,8 @@ To interact with the various language models, you can use the following commands
 - For Together Mistral, send a message with ``together/mistral`` and include your prompt.
 - For Together Mixtral, send a message with ``together/mixtral`` and include your prompt.
 - For Together Solar, send a message with ``together/solar`` and include your prompt.
+- For Groq Mixtral, send a message with ``groq/mixtral`` and include your prompt.
+- For Groq Llama-2, send a message with ``groq/llama2`` and include your prompt.
 - For vLLM OpenChat, send a message with ``vllm/openchat`` and include your prompt.
 - For vLLM OpenHermes, send a message with ``vllm/openhermes`` and include your prompt.
 - For Dall-e-3, send a message with ``dall-e-3`` and include your prompt.

app.py

@@ -8,6 +8,8 @@ from mistralai.models.chat_completion import ChatMessage
 
 from openai import OpenAI
 
+from groq import Groq
+
 import tiktoken
 
 import os
@@ -44,6 +46,8 @@ system_users = [
     "mistral-medium",
     "mistralai/Mixtral-8x7B-v0.1",
     "mistralai/Mistral-7B-Instruct-v0.1",
+    "mixtral-8x7b-32768",
+    "llama2-70b-4096",
     "openchat/openchat-3.5-1210",
     "openchat/openchat-3.5-0106",
     "upstage/SOLAR-10.7B-Instruct-v1.0",
@@ -258,6 +262,7 @@ def handle_message(data):
         or "together/" in data["message"]
         or "localhost/" in data["message"]
         or "vllm/" in data["message"]
+        or "groq/" in data["message"]
     ):
         # Emit a temporary message indicating that llm is processing
         emit(
@@ -344,6 +349,22 @@ def handle_message(data):
                 model_name="upstage/SOLAR-10.7B-Instruct-v1.0",
                 stop=["###", "</s>"],
             )
+        if "groq/mixtral" in data["message"]:
+            eventlet.spawn(
+                chat_groq,
+                data["username"],
+                room.name,
+                data["message"],
+                model_name="mixtral-8x7b-32768",
+            )
+        if "groq/llama2" in data["message"]:
+            eventlet.spawn(
+                chat_groq,
+                data["username"],
+                room.name,
+                data["message"],
+                model_name="llama2-70b-4096",
+            )
         if "vllm/openchat" in data["message"]:
             eventlet.spawn(
                 chat_gpt,
@@ -582,7 +603,8 @@ def chat_gpt(username, room_name, message, model_name="gpt-3.5-turbo"):
         chat_history = [
             {
                 "role": "system" if msg.username in system_users else "user",
-                "content": f"{msg.username}: {msg.content}",
+                #"content": f"{msg.username}: {msg.content}",
+                "content": msg.content,
             }
             for msg in reversed(last_messages)
             if not msg.is_base64_image()
@@ -899,6 +921,112 @@ def chat_together(
     socketio.emit("delete_processing_message", msg_id, room=room.name)
 
 
+def chat_groq(username, room_name, message, model_name="mixtral-8x7b-32768"):
+
+    _limit = 15
+    if "mixtral" in model_name:
+        _limit = 50
+
+    with app.app_context():
+        room = get_room(room_name)
+        last_messages = (
+            Message.query.filter_by(room_id=room.id)
+            .order_by(Message.id.desc())
+            .limit(_limit)
+            .all()
+        )
+
+        chat_history = [
+            {
+                "role": "system" if msg.username in system_users else "user",
+                "content": msg.content,
+            }
+            for msg in reversed(last_messages)
+            if not msg.is_base64_image()
+        ]
+
+    # Initialize the Groq client
+    client = Groq()
+
+    buffer = ""  # Content buffer for accumulating the chunks
+
+    # Save an empty message to get an ID for the chunks
+    with app.app_context():
+        new_message = Message(username=model_name, content=buffer, room_id=room.id)
+        db.session.add(new_message)
+        db.session.commit()
+        msg_id = new_message.id
+
+    first_chunk = True
+
+    try:
+        # Use the Groq client to stream the chat completion
+        stream = client.chat.completions.create(
+            messages=chat_history,
+            model=model_name,
+            stream=True,
+        )
+
+        for chunk in stream:
+            content_chunk = chunk.choices[0].delta.content
+
+            if content_chunk:
+                buffer += content_chunk  # Accumulate content
+
+                if first_chunk:
+                    socketio.emit(
+                        "message_chunk",
+                        {
+                            "id": msg_id,
+                            "content": f"**{username} ({model_name}):**\n\n{content_chunk}",
+                        },
+                        room=room.name,
+                    )
+                    first_chunk = False
+                else:
+                    socketio.emit(
+                        "message_chunk",
+                        {"id": msg_id, "content": content_chunk},
+                        room=room.name,
+                    )
+                socketio.sleep(0)  # Force immediate handling
+
+    except Exception as e:
+        with app.app_context():
+            message_content = f"Groq Error: {e}"
+            new_message = (
+                db.session.query(Message).filter(Message.id == msg_id).one_or_none()
+            )
+            if new_message:
+                new_message.content = message_content
+                new_message.count_tokens()
+                db.session.add(new_message)
+                db.session.commit()
+        socketio.emit(
+            "message",
+            {
+                "id": msg_id,
+                "username": model_name,
+                "content": message_content,
+            },
+            room=room.name,
+        )
+        return None
+
+    # Save the entire completion to the database
+    with app.app_context():
+        new_message = (
+            db.session.query(Message).filter(Message.id == msg_id).one_or_none()
+        )
+        if new_message:
+            new_message.content = buffer
+            new_message.count_tokens()
+            db.session.add(new_message)
+            db.session.commit()
+
+    socketio.emit("delete_processing_message", msg_id, room=room.name)
+
+
 def chat_llama(
     username, room_name, message, model_name="mistral-7b-instruct-v0.2.Q3_K_L.gguf"
 ):

requirements.txt

@@ -4,6 +4,7 @@ eventlet
 mistralai
 together
 openai
+groq
 tiktoken
 
 #llama-cpp-python[server]