engineering/opencompletion.com
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

support both vllm and openai at the same time.

Browse source 
modified:   app.py
This commit is contained in:
Russell Ballestrini 2024-08-04 11:35:12 -04:00
parent 7a433bfccb
commit c4df19132f
1 changed files with 1 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
app.py
View file

@ -762,7 +762,7 @@ def get_openai_client_and_model(model_name="gpt-4o-mini"):
vllm_endpoint = os.environ.get("VLLM_ENDPOINT")
vllm_api_key = os.environ.get("VLLM_API_KEY", "not-needed")
if vllm_endpoint:
if "gpt" not in model_name and vllm_endpoint:
openai_client = OpenAI(base_url=vllm_endpoint, api_key=vllm_api_key)
model_name = "NousResearch/Hermes-2-Pro-Llama-3-8B"
else: