Fixes4Carlton.txt

@@ -0,0 +1,42 @@
+# I'm hardmoding linux so I've only done a hard-reset once
+# (when I switched from an AMD card to nvidia) 
+
+# I'm pretty sure my everything is borked so-
+# Running these commands when I get a complaint. 
+# don't question it, I'm not a neckbeard, I'm a linux enjoyer.
+
+# Didn't use the install scripts cause, yeah.
+ 
+### THE COMMANDS BELOW
+
+#Openai error - You don't need a valid API key to run, it just complains
+
+export OPENAI_API_KEY=balls
+
+
+pip uninstall -y httpcore
+pip install httpx==0.24.1
+
+#additionally, I have multiple python packages so I have to specify - 
+#python3.10 app.py
+
+
+# if you do not have s3, use --local-activities like :
+# python3.10 'app.py' --local-activities
+#
+#
+#
+#
+# To add new ollama models, open app.py and duplicate the following :
+ 
+        if "ollama/thewindmom/hermes-3-llama-3.1-8b" in data["message"]:
+            gevent.spawn(
+                chat_ollama,
+                data["username"],
+                room.name,
+                model_name="thewindmom/hermes-3-llama-3.1-8b",
+            )
+# remember it's python so don't f up the tabs.
+# change the model name after ollama/modelnamehere and the variable
+# model_name="changethistoo"
+

README.rst

@@ -1,6 +1,16 @@
 flask-socketio-llm-completions
 ========================================
 
+Ollama compatible version of flask-socketio-llm-completions. Read Fixes4Carlton.txt on how to add additional models. 
+
+
+[![Open Completion Welcome video](https://img.youtube.com/vi/4YQhgwQfBv0/0.jpg)](https://www.youtube.com/watch?v=4YQhgwQfBv0
+
+
+#######################################
+
+Original Description below.
+
 This project is a chatroom application that allows users to join different chat rooms, send messages, and interact with multiple language models in real-time. The backend is built with Flask and Flask-SocketIO for real-time web communication, while the frontend uses HTML, CSS, and JavaScript to provide an interactive user interface.
 
 To view a short video of the chat in action click this screenshot:

app.py

@@ -34,6 +34,8 @@ from groq import Groq
 from mistralai import Mistral
 from openai import OpenAI
 
+import ollama
+
 app = Flask(__name__)
 
 app.config["SECRET_KEY"] = "your_secret_key"
@@ -96,7 +98,7 @@ HELP_MESSAGE = """
 - `/s3 load [s3_file_path]`: Load a file from S3.
 - `/s3 save [s3_key_path]`: Save the most recent code block from the chatroom to S3.
 - `/title new`: Generates a new title which reflects conversation content for the current chatroom using gpt-4.
-- `/cancel`: Cancel the most recent chat completion from streaming into the chatroom.
+- `/cancel`: Cancel the most recent chat completion from stllmsreaming into the chatroom.
 - `/help`: Display this help message.
 
 **Available Models:**
@@ -457,7 +459,7 @@ def on_join(data):
 
     message_count = len(previous_messages)
     if room.title is None and message_count >= 6:
-        room.title = gpt_generate_room_title(previous_messages)
+        room.title = ollama_generate_room_title(previous_messages)
         db.session.add(room)
         db.session.commit()
         socketio.emit("update_room_title", {"title": room.title}, room=room.name)
@@ -586,6 +588,7 @@ def handle_message(data):
         or "localhost/" in data["message"]
         or "vllm/" in data["message"]
         or "groq/" in data["message"]
+        or "ollama/" in data["message"]
     ):
         # Emit a temporary message indicating that the llm is processing
         emit(
@@ -770,6 +773,22 @@ def handle_message(data):
                 room.name,
                 model_name="openhermes-2.5-mistral-7b.Q6_K.gguf",
             )
+        if "localhost/thewindmom/hermes-3-llama-3.1-8b" in data["message"]:
+            gevent.spawn(
+                chat_llama,
+                data["username"],
+                room.name,
+                model_name="thewindmom/hermes-3-llama-3.1-8b",
+            )
+
+        if "ollama/thewindmom/hermes-3-llama-3.1-8b" in data["message"]:
+            gevent.spawn(
+                chat_ollama,
+                data["username"],
+                room.name,
+                model_name="thewindmom/hermes-3-llama-3.1-8b",
+            )
+
 
 
 @socketio.on("delete_message")
@@ -975,7 +994,7 @@ def chat_claude(
     socketio.emit("delete_processing_message", msg_id, room=room.name)
 
 
-def get_openai_client_and_model(model_name="NousResearch/Hermes-3-Llama-3.1-8B"):
+def get_openai_client_and_model(model_name="thewindmom/hermes-3-llama-3.1-8b"):
     vllm_endpoint = os.environ.get("VLLM_ENDPOINT")
     vllm_api_key = os.environ.get("VLLM_API_KEY", "not-needed")
 
@@ -989,6 +1008,7 @@ def get_openai_client_and_model(model_name="NousResearch/Hermes-3-Llama-3.1-8B")
     return openai_client, model_name
 
 
+
 def chat_gpt(username, room_name, model_name="gpt-4o-mini"):
     openai_client, model_name = get_openai_client_and_model(model_name)
 
@@ -1547,6 +1567,117 @@ def chat_llama(username, room_name, model_name="mistral-7b-instruct-v0.2.Q3_K_L.
     socketio.emit("delete_processing_message", msg_id, room=room.name)
 
 
+
+def chat_ollama(username, room_name, model_name="thewindmom/hermes-3-llama-3.1-8b"):
+    import llama_cpp
+
+    # https://llama-cpp-python.readthedocs.io/en/latest/api-reference/
+    #model = llama_cpp.Llama(model_name, n_gpu_layers=-1, n_ctx=32000)
+
+    limit = 15
+    with app.app_context():
+        room = get_room(room_name)
+        last_messages = (
+            Message.query.filter_by(room_id=room.id)
+            .order_by(Message.id.desc())
+            .limit(limit)
+            .all()
+        )
+
+        chat_history = [
+            {
+                "role": "system" if msg.username in system_users else "user",
+                "content": f"{msg.username}: {msg.content}",
+            }
+            for msg in reversed(last_messages)
+            if not msg.is_base64_image()
+        ]
+
+    buffer = ""  # Content buffer for accumulating the chunks
+
+    # save empty message, we need the ID when we chunk the response.
+    with app.app_context():
+        new_message = Message(username=model_name, content=buffer, room_id=room.id)
+        db.session.add(new_message)
+        db.session.commit()
+        msg_id = new_message.id
+
+    first_chunk = True
+    
+    
+    try:
+        chunks = ollama.chat(model=model_name,
+            messages=chat_history,
+            stream=True,
+        )
+    except Exception as e:
+        with app.app_context():
+            message_content = f"LLama Error: {e}"
+            new_message = (
+                db.session.query(Message).filter(Message.id == msg_id).one_or_none()
+            )
+            if new_message:
+                new_message.content = message_content
+                new_message.count_tokens()
+                db.session.add(new_message)
+                db.session.commit()
+        socketio.emit(
+            "chat_message",
+            {
+                "id": msg_id,
+                "username": model_name,
+                "content": message_content,
+            },
+            room=room_name,
+        )
+        socketio.emit("delete_processing_message", msg_id, room=room.name)
+        # exit early to avoid clobbering the error message.
+        return None
+
+    for chunk in chunks:
+        # Check if there has been a cancellation request, break if there is.
+        if cancellation_requests.get(msg_id):
+            del cancellation_requests[msg_id]
+            break
+            
+        print(chunk)
+        content = chunk["message"]["content"]
+        
+        if content:
+            buffer += content  # Accumulate content
+
+            if first_chunk:
+                socketio.emit(
+                    "message_chunk",
+                    {
+                        "id": msg_id,
+                        "content": f"**{username} ({model_name}):**\n\n{content}",
+                    },
+                    room=room.name,
+                )
+                first_chunk = False
+            else:
+                socketio.emit(
+                    "message_chunk",
+                    {"id": msg_id, "content": content},
+                    room=room.name,
+                )
+            socketio.sleep(0)  # Force immediate handling
+
+    # Save the entire completion to the database
+    with app.app_context():
+        new_message = (
+            db.session.query(Message).filter(Message.id == msg_id).one_or_none()
+        )
+        if new_message:
+            new_message.content = buffer
+            new_message.count_tokens()
+            db.session.add(new_message)
+            db.session.commit()
+
+    socketio.emit("delete_processing_message", msg_id, room=room.name)
+
+
 def gpt_generate_room_title(messages):
     """
     Generate a title for the room based on a list of messages.
@@ -1581,6 +1712,46 @@ def gpt_generate_room_title(messages):
     return title.replace('"', "")
     
     
+############################################################ C WUZ HERE
+def ollama_generate_room_title(messages):
+    """
+    Generate a title for the room based on a list of messages.
+    """
+    #openai_client, model_name = get_openai_client_and_model()
+    model_name = 'thewindmom/hermes-3-llama-3.1-8b'
+    chat_history = [
+        {
+            "role": "system" if msg.username in system_users else "user",
+            "content": f"{msg.username}: {msg.content}",
+        }
+        for msg in reversed(messages)
+        if not msg.is_base64_image()
+    ]
+
+    chat_history.append(
+        {
+            "role": "system",
+            "content": "return a short title for the title bar of this conversation. Reply with the title only",
+        }
+    )
+
+    # Interaction with LLM to generate summary
+    # For example, using OpenAI's GPT model
+    response = ollama.chat(
+        messages=chat_history,
+        model=model_name,  # or any appropriate model
+        #max_tokens=20,
+    )
+    
+    print(response)
+    content = response["message"]["content"]
+    title = content.strip().replace('"', "")
+
+    return title
+
+#############################################################################
+
+
 def generate_new_title(room_name, username):
     with app.app_context():
         room = get_room(room_name)
@@ -2928,6 +3099,7 @@ if __name__ == "__main__":
         "--local-activities",
         action="store_true",
         help="Use local activity files instead of S3",
+     
     )
     args = parser.parse_args()