engineering/opencompletion.com
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

modified: app.py

Browse source
This commit is contained in:
Russell Ballestrini 2024-01-19 08:59:11 -05:00
parent f256d65391
commit 0efbe93910
1 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
app.py
View file

@ -8,8 +8,6 @@ from mistralai.models.chat_completion import ChatMessage
from openai import OpenAI
import llama_cpp
import tiktoken
import os
@ -869,6 +867,8 @@ def chat_together(
def chat_llama(username, room_name, message, model_name="mistral-7b-instruct-v0.2.Q3_K_L.gguf"):
import llama_cpp
# https://llama-cpp-python.readthedocs.io/en/latest/api-reference/
model = llama_cpp.Llama(model_name, n_gpu_layers=-1, n_ctx=32000)