engineering/uncloseai.com
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

much bigger context window switching to fp8!

Browse source 
modified:   index.html
	modified:   uncloseai.js
This commit is contained in:
Russell Ballestrini 2025-02-01 14:29:39 -05:00
parent 472d2e1100
commit 7abe4975de
2 changed files with 10 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

11
index.html
View file

@ -106,7 +106,8 @@ from openai import OpenAI
client = OpenAI(base_url="https://hermes.ai.unturf.com/v1", api_key="choose-any-value")
MODEL = "NousResearch/Hermes-3-Llama-3.1-8B"
#MODEL = "NousResearch/Hermes-3-Llama-3.1-8B"
MODEL = "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic"
messages = [{"role": "user", "content": "Give a Python Fizzbuzz solution in one line of code?"}]
@ -126,7 +127,7 @@ from openai import OpenAI
client = OpenAI(base_url="https://hermes.ai.unturf.com/v1", api_key="choose-any-value")
MODEL = "NousResearch/Hermes-3-Llama-3.1-8B"
MODEL = "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic"
messages = [
{"role": "user", "content": "Give a Python Fizzbuzz solution in one line of code?"}
@ -154,7 +155,7 @@ const client = new OpenAI({
apiKey: "dummy-api-key",
});
const MODEL = "NousResearch/Hermes-3-Llama-3.1-8B";
const MODEL = "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic";
const messages = [{"role": "user", "content": "Give a Python Fizzbuzz solution in one line of code?"}];
@ -185,7 +186,7 @@ const client = new OpenAI({
apiKey: "dummy-api-key",
});
const MODEL = "NousResearch/Hermes-3-Llama-3.1-8B";
const MODEL = "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic";
const messages = [{"role": "user", "content": "Give a Python Fizzbuzz solution in one line of code?"}];
@ -269,7 +270,7 @@ cd ~
python3 -m venv env
source env/bin/activate
pip install vllm
python -m vllm.entrypoints.openai.api_server --model NousResearch/Hermes-3-Llama-3.1-8B --host 0.0.0.0 --port 18888 --max-model-len 16000
python -m vllm.entrypoints.openai.api_server --model adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic --host 0.0.0.0 --port 18888 --max-model-len 82000
</code></pre>
<p>The Speech endpoint or TTS uses <a href="https://github.com/matatonic/openedai-speech?tab=readme-ov-file#nvidia-gpu-cuda">openedai-speech</a> running via Docker.

8
uncloseai.js
View file

@ -23,7 +23,7 @@ const API_URL = "https://hermes.ai.unturf.com/v1/chat/completions";
const TTS_API_URL = "https://speech.ai.unturf.com/v1/audio/speech";
const MEGAPARCE_API_URL = "https://megaparce.ai.unturf.com/v1/file";
const API_KEY = "dummy-api-key";
const MODEL = "NousResearch/Hermes-3-Llama-3.1-8B";
const MODEL = "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic";
// Initialize chat history
let chatHistory = [
@ -152,7 +152,7 @@ Ready? Breath and then return a stream of tokens to be used in a TTS system.
}
],
temperature: 0,
max_tokens: 13000
max_tokens: 80000
};
console.log("Sending payload to Hermes for spoken tokens:", JSON.stringify(payload, null, 2));
@ -196,7 +196,7 @@ async function extractSpokenTokens(content) {
}
],
temperature: 0,
max_tokens: 13000
max_tokens: 80000
};
console.log("Sending payload to Hermes for spoken tokens:", JSON.stringify(payload, null, 2));
@ -349,7 +349,7 @@ async function* sendMessage(message) {
model: MODEL,
messages: chatHistory,
temperature: 0.5,
max_tokens: 5600,
max_tokens: 80000,
stream: true
})
});