261 lines
11 KiB
HTML
261 lines
11 KiB
HTML
<!DOCTYPE html>
|
|
<html lang="en">
|
|
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<meta name="theme-color" content="#43a047">
|
|
<meta name="color-scheme" content="light dark">
|
|
<title>How We Run Inference | uncloseai.com</title>
|
|
|
|
<!-- PicoCSS -->
|
|
<link rel="stylesheet" href="/css/pico.classless.min.css">
|
|
<!-- ChunkFive Font -->
|
|
<link rel="stylesheet" href="/css/chunkfive/stylesheet.css" type="text/css" charset="utf-8" />
|
|
<!-- Sidebar Theme -->
|
|
<link rel="stylesheet" href="/css/sidebar-theme.css">
|
|
|
|
<!-- Highlight.js for syntax highlighting -->
|
|
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.10.0/styles/a11y-dark.min.css" />
|
|
<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.6.0/highlight.min.js"></script>
|
|
|
|
<!-- Theme Switcher Script -->
|
|
<script>
|
|
function switchTheme(theme) {
|
|
if (theme === "auto") {
|
|
document.documentElement.removeAttribute('data-theme');
|
|
} else {
|
|
document.documentElement.setAttribute('data-theme', theme);
|
|
}
|
|
}
|
|
|
|
// Disable custom styling for uncloseai.js - use PicoCSS instead
|
|
window.UNCLOSEAI_CUSTOM_STYLING = false;
|
|
</script>
|
|
|
|
<script src="https://uncloseai.com/uncloseai.js" type="module"></script>
|
|
</head>
|
|
|
|
<body>
|
|
<!-- Mobile menu toggle -->
|
|
<button class="sidebar-toggle" onclick="document.querySelector('.sidebar').classList.toggle('open')">
|
|
☰
|
|
</button>
|
|
|
|
<!-- Left sidebar - Main Navigation -->
|
|
<aside class="sidebar">
|
|
<div class="table-of-contents">
|
|
|
|
<nav>
|
|
<ul>
|
|
<li><a href="/">Home</a></li>
|
|
<li><a href="/c-examples.html">C Examples</a></li>
|
|
<li><a href="/csharp-examples.html">C# Examples</a></li>
|
|
<li><a href="/dart-examples.html">Dart Examples</a></li>
|
|
<li><a href="/elixir-examples.html">Elixir Examples</a></li>
|
|
<li><a href="/go-examples.html">Go Examples</a></li>
|
|
<li><a href="/java-examples.html">Java Examples</a></li>
|
|
<li><a href="/kotlin-examples.html">Kotlin Examples</a></li>
|
|
<li><a href="/nodejs-examples.html">Node.js Examples</a></li>
|
|
<li><a href="/php-examples.html">PHP Examples</a></li>
|
|
<li><a href="/python-examples.html">Python Examples</a></li>
|
|
<li><a href="/ruby-examples.html">Ruby Examples</a></li>
|
|
<li><a href="/rust-examples.html">Rust Examples</a></li>
|
|
<li><a href="/swift-examples.html">Swift Examples</a></li>
|
|
<li><a href="/uncloseai-js.html">uncloseai.js Docs</a></li>
|
|
<li><a href="/cli.html">uncloseai-cli</a></li>
|
|
<li><a href="/inference.html" class="active">Inference Setup</a></li>
|
|
<li><a href="/text-to-speech.html">Text-to-Speech</a></li>
|
|
<li><a href="/crawler.html">Our Crawler</a></li>
|
|
<li><a href="/languages" target="_blank">🔗 All Languages</a></li>
|
|
<li><a href="https://shop.unturf.com/p/8486f492-a93e-11f0-b477-02dfe05770ee/uncloseai-machine-learning-reference-guide-to-inference-clients" target="_blank">📚 Book</a></li>
|
|
</ul>
|
|
</nav>
|
|
</div>
|
|
</aside>
|
|
|
|
<!-- Main content -->
|
|
<main>
|
|
<header>
|
|
<hgroup>
|
|
<a href="https://uncloseai.com"><h1 class="unturf" style="font-family: 'ChunkFiveRegular';">uncloseai.</h1></a>
|
|
<p>How We Run Inference</p>
|
|
</hgroup>
|
|
<nav>
|
|
<ul>
|
|
<li><a href="#" onclick="switchTheme('auto')">Auto</a></li>
|
|
<li><a href="#" onclick="switchTheme('light')">Light</a></li>
|
|
<li><a href="#" onclick="switchTheme('dark')">Dark</a></li>
|
|
</ul>
|
|
</nav>
|
|
</header>
|
|
|
|
<h2 id="inference-intro">How we run inference</h2>
|
|
<p>This section is optional. This is only if you wanted to try to contribute idle GPU time to the project or if you wanted to reproduce everything in your own cluster.</p>
|
|
|
|
<h3 id="vllm-setup">vLLM Setup</h3>
|
|
<p>We use vLLM to run models, generally with full f16 safetensors. We make sure to use a virtualenv to hold the dependencies.</p>
|
|
<p><strong>Note:</strong> For Hermes, we use an FP8 quant by adamo1139 (adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic), which is optimized for 4090 and 3090 GPUs.</p>
|
|
<p><strong>Tool Calling:</strong> The <code>--enable-auto-tool-choice</code> and <code>--tool-call-parser hermes</code> flags are required for Hermes 3 to support function calling via the OpenAI-compatible API. Without these flags, tool calling requests will fail.</p>
|
|
<p>We are considering supporting ollama for better quant support.</p>
|
|
<p>Stand up a replica cluster on a new domain.</p>
|
|
<pre><code>sudo apt-get install gcc python3.12-dev python3.12-venv
|
|
cd ~
|
|
python3 -m venv env
|
|
source env/bin/activate
|
|
pip install vllm
|
|
python -m vllm.entrypoints.openai.api_server --model adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic --host 0.0.0.0 --port 18888 --max-model-len 82000 --enable-auto-tool-choice --tool-call-parser hermes
|
|
</code></pre>
|
|
|
|
<h3 id="proxy-setup">Proxy Setup</h3>
|
|
<p>If you want to see how we setup the proxy, check out <a href="https://git.unturf.com/-/snippets/3" target="_blank">/etc/caddy/Caddyfile</a></p>
|
|
|
|
<pre><code>ai.unturf.com {
|
|
root * /opt/www
|
|
file_server
|
|
log {
|
|
output file /var/log/caddy/ai.unturf.com.log {
|
|
roll_size 50mb
|
|
roll_keep 5
|
|
}
|
|
}
|
|
tls {
|
|
on_demand
|
|
}
|
|
}
|
|
|
|
hermes.ai.unturf.com {
|
|
reverse_proxy <removed>:18888
|
|
log {
|
|
output file /var/log/caddy/hermes.ai.unturf.com.log {
|
|
roll_size 50mb
|
|
roll_keep 5
|
|
}
|
|
}
|
|
tls {
|
|
on_demand
|
|
}
|
|
}
|
|
|
|
speech.ai.unturf.com {
|
|
reverse_proxy <removed>:8000
|
|
log {
|
|
output file /var/log/caddy/speech.ai.unturf.com.log {
|
|
roll_size 50mb
|
|
roll_keep 5
|
|
}
|
|
}
|
|
tls {
|
|
on_demand
|
|
}
|
|
}
|
|
</code></pre>
|
|
|
|
<h3 id="model-discovery">Model Discovery</h3>
|
|
<p>vLLM provides an OpenAI-compatible API with built-in documentation. You can discover available models and explore the full API using these endpoints:</p>
|
|
|
|
<h4>Swagger Documentation</h4>
|
|
<p>Access the interactive API docs at the <code>/docs</code> endpoint:</p>
|
|
<p><a href="https://hermes.ai.unturf.com/docs" target="_blank">hermes.ai.unturf.com/docs</a> - Hermes endpoint Swagger docs</p>
|
|
<p>The Swagger UI lets you explore all available endpoints, see request/response schemas, and test API calls directly in your browser.</p>
|
|
|
|
<h4>Model Discovery</h4>
|
|
<p>To get the current model ID being hosted, query the <code>/v1/models</code> endpoint:</p>
|
|
<ul>
|
|
<li><a href="https://hermes.ai.unturf.com/v1/models" target="_blank">hermes.ai.unturf.com/v1/models</a> - Hermes models</li>
|
|
<li><a href="https://qwen.ai.unturf.com/v1/models" target="_blank">qwen.ai.unturf.com/v1/models</a> - Qwen models</li>
|
|
</ul>
|
|
<p>Or via curl:</p>
|
|
<pre><code class="language-bash">curl https://hermes.ai.unturf.com/v1/models
|
|
curl https://qwen.ai.unturf.com/v1/models
|
|
</code></pre>
|
|
|
|
<p>Example response:</p>
|
|
<pre><code class="language-json">{
|
|
"object": "list",
|
|
"data": [
|
|
{
|
|
"id": "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic",
|
|
"object": "model",
|
|
"created": 1735689600,
|
|
"owned_by": "vllm",
|
|
"root": "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic",
|
|
"max_model_len": 82000
|
|
}
|
|
]
|
|
}
|
|
</code></pre>
|
|
|
|
<p>The <code>id</code> field contains the model name you should use in your API calls. The <code>max_model_len</code> field tells you the maximum context length supported.</p>
|
|
|
|
<p><strong>Tip:</strong> Always query <code>/v1/models</code> programmatically rather than hardcoding model names. This ensures your code works even when models are updated or swapped.</p>
|
|
|
|
<h3 id="rate-limiting">Rate Limiting</h3>
|
|
<p>Rate limiting is configured based on client IP address: 3 requests per second per IP per endpoint.</p>
|
|
|
|
<h3 id="next-steps">Next Steps</h3>
|
|
<p>Ready to add text-to-speech to your application?</p>
|
|
<p><a href="/text-to-speech.html"><strong>📖 Read the Text-to-Speech documentation →</strong></a></p>
|
|
|
|
<script>hljs.highlightAll();</script>
|
|
|
|
<footer>
|
|
<small>© uncloseai. 2025</small>
|
|
<br>
|
|
<small>Stylesheets by <a href="https://picocss.com" target="_blank">PicoCSS</a></small>
|
|
<small>& <a href="https://highlightjs.org/" target="_blank">highlight.js</a></small>
|
|
<br>
|
|
<small><a href="/privacy-policy.html">Privacy Policy</a> | <a href="/terms-of-use.html">Terms of Use</a></small>
|
|
</footer>
|
|
|
|
<script>
|
|
// Simple active link highlighting
|
|
document.querySelectorAll('.table-of-contents a').forEach(link => {
|
|
link.addEventListener('click', function() {
|
|
document.querySelectorAll('.table-of-contents a').forEach(a => a.classList.remove('active'));
|
|
this.classList.add('active');
|
|
});
|
|
});
|
|
|
|
// Highlight current section on scroll
|
|
const sections = document.querySelectorAll('h2[id], h3[id]');
|
|
const navLinks = document.querySelectorAll('.table-of-contents a');
|
|
|
|
window.addEventListener('scroll', () => {
|
|
let current = '';
|
|
sections.forEach(section => {
|
|
const sectionTop = section.offsetTop;
|
|
const sectionHeight = section.clientHeight;
|
|
if (window.scrollY >= sectionTop - 100) {
|
|
current = section.getAttribute('id');
|
|
}
|
|
});
|
|
|
|
navLinks.forEach(link => {
|
|
link.classList.remove('active');
|
|
if (link.getAttribute('href') === '#' + current) {
|
|
link.classList.add('active');
|
|
}
|
|
});
|
|
});
|
|
</script>
|
|
</main>
|
|
|
|
<!-- Right sidebar - Page TOC -->
|
|
<aside class="sidebar-right">
|
|
<div class="table-of-contents">
|
|
|
|
<nav>
|
|
<ul>
|
|
<li><a href="#inference-intro" class="active">Overview</a></li>
|
|
<li><a href="#vllm-setup">vLLM Setup</a></li>
|
|
<li><a href="#proxy-setup">Proxy Setup</a></li>
|
|
<li><a href="#model-discovery">Model Discovery</a></li>
|
|
<li><a href="#rate-limiting">Rate Limiting</a></li>
|
|
</ul>
|
|
</nav>
|
|
</div>
|
|
</aside>
|
|
</body>
|
|
|
|
</html>
|