diff --git a/index.html b/index.html index c7f10b1..da89857 100644 --- a/index.html +++ b/index.html @@ -106,7 +106,8 @@ from openai import OpenAI client = OpenAI(base_url="https://hermes.ai.unturf.com/v1", api_key="choose-any-value") -MODEL = "NousResearch/Hermes-3-Llama-3.1-8B" +#MODEL = "NousResearch/Hermes-3-Llama-3.1-8B" +MODEL = "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic" messages = [{"role": "user", "content": "Give a Python Fizzbuzz solution in one line of code?"}] @@ -126,7 +127,7 @@ from openai import OpenAI client = OpenAI(base_url="https://hermes.ai.unturf.com/v1", api_key="choose-any-value") -MODEL = "NousResearch/Hermes-3-Llama-3.1-8B" +MODEL = "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic" messages = [ {"role": "user", "content": "Give a Python Fizzbuzz solution in one line of code?"} @@ -154,7 +155,7 @@ const client = new OpenAI({ apiKey: "dummy-api-key", }); -const MODEL = "NousResearch/Hermes-3-Llama-3.1-8B"; +const MODEL = "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic"; const messages = [{"role": "user", "content": "Give a Python Fizzbuzz solution in one line of code?"}]; @@ -185,7 +186,7 @@ const client = new OpenAI({ apiKey: "dummy-api-key", }); -const MODEL = "NousResearch/Hermes-3-Llama-3.1-8B"; +const MODEL = "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic"; const messages = [{"role": "user", "content": "Give a Python Fizzbuzz solution in one line of code?"}]; @@ -269,7 +270,7 @@ cd ~ python3 -m venv env source env/bin/activate pip install vllm -python -m vllm.entrypoints.openai.api_server --model NousResearch/Hermes-3-Llama-3.1-8B --host 0.0.0.0 --port 18888 --max-model-len 16000 +python -m vllm.entrypoints.openai.api_server --model adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic --host 0.0.0.0 --port 18888 --max-model-len 82000
The Speech endpoint or TTS uses openedai-speech running via Docker. diff --git a/uncloseai.js b/uncloseai.js index 6fea554..7fe5768 100644 --- a/uncloseai.js +++ b/uncloseai.js @@ -23,7 +23,7 @@ const API_URL = "https://hermes.ai.unturf.com/v1/chat/completions"; const TTS_API_URL = "https://speech.ai.unturf.com/v1/audio/speech"; const MEGAPARCE_API_URL = "https://megaparce.ai.unturf.com/v1/file"; const API_KEY = "dummy-api-key"; -const MODEL = "NousResearch/Hermes-3-Llama-3.1-8B"; +const MODEL = "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic"; // Initialize chat history let chatHistory = [ @@ -152,7 +152,7 @@ Ready? Breath and then return a stream of tokens to be used in a TTS system. } ], temperature: 0, - max_tokens: 13000 + max_tokens: 80000 }; console.log("Sending payload to Hermes for spoken tokens:", JSON.stringify(payload, null, 2)); @@ -196,7 +196,7 @@ async function extractSpokenTokens(content) { } ], temperature: 0, - max_tokens: 13000 + max_tokens: 80000 }; console.log("Sending payload to Hermes for spoken tokens:", JSON.stringify(payload, null, 2)); @@ -349,7 +349,7 @@ async function* sendMessage(message) { model: MODEL, messages: chatHistory, temperature: 0.5, - max_tokens: 5600, + max_tokens: 80000, stream: true }) });