Russell Ballestrini
caeeb85c63
- Added text-to-speech link in navigation after Inference Setup link - Updated inference.html TTS section to link to text-to-speech page - Removed redundant openedai-speech references from inference page - Applied changes to 18 public HTML pages for consistent navigation
216 lines
8.3 KiB
HTML
216 lines
8.3 KiB
HTML
<!DOCTYPE html>
|
|
<html lang="en">
|
|
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<meta name="theme-color" content="#43a047">
|
|
<meta name="color-scheme" content="light dark">
|
|
<title>How We Run Inference | uncloseai.com</title>
|
|
|
|
<!-- PicoCSS -->
|
|
<link rel="stylesheet" href="/css/pico.classless.min.css">
|
|
<!-- ChunkFive Font -->
|
|
<link rel="stylesheet" href="/css/chunkfive/stylesheet.css" type="text/css" charset="utf-8" />
|
|
<!-- Sidebar Theme -->
|
|
<link rel="stylesheet" href="/css/sidebar-theme.css">
|
|
|
|
<!-- Highlight.js for syntax highlighting -->
|
|
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.10.0/styles/a11y-dark.min.css" />
|
|
<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.6.0/highlight.min.js"></script>
|
|
|
|
<!-- Theme Switcher Script -->
|
|
<script>
|
|
function switchTheme(theme) {
|
|
if (theme === "auto") {
|
|
document.documentElement.removeAttribute('data-theme');
|
|
} else {
|
|
document.documentElement.setAttribute('data-theme', theme);
|
|
}
|
|
}
|
|
|
|
// Disable custom styling for uncloseai.js - use PicoCSS instead
|
|
window.UNCLOSEAI_CUSTOM_STYLING = false;
|
|
</script>
|
|
|
|
<script src="https://uncloseai.com/uncloseai.js" type="module"></script>
|
|
</head>
|
|
|
|
<body>
|
|
<!-- Mobile menu toggle -->
|
|
<button class="sidebar-toggle" onclick="document.querySelector('.sidebar').classList.toggle('open')">
|
|
☰
|
|
</button>
|
|
|
|
<!-- Left sidebar - Main Navigation -->
|
|
<aside class="sidebar">
|
|
<div class="table-of-contents">
|
|
|
|
<nav>
|
|
<ul>
|
|
<li><a href="/">Home</a></li>
|
|
<li><a href="/csharp-examples.html">C# Examples</a></li>
|
|
<li><a href="/dart-examples.html">Dart Examples</a></li>
|
|
<li><a href="/elixir-examples.html">Elixir Examples</a></li>
|
|
<li><a href="/go-examples.html">Go Examples</a></li>
|
|
<li><a href="/java-examples.html">Java Examples</a></li>
|
|
<li><a href="/kotlin-examples.html">Kotlin Examples</a></li>
|
|
<li><a href="/nodejs-examples.html">Node.js Examples</a></li>
|
|
<li><a href="/php-examples.html">PHP Examples</a></li>
|
|
<li><a href="/python-examples.html">Python Examples</a></li>
|
|
<li><a href="/ruby-examples.html">Ruby Examples</a></li>
|
|
<li><a href="/rust-examples.html">Rust Examples</a></li>
|
|
<li><a href="/swift-examples.html">Swift Examples</a></li>
|
|
<li><a href="/uncloseai-js.html">uncloseai.js Docs</a></li>
|
|
<li><a href="/inference.html" class="active">Inference Setup</a></li>
|
|
<li><a href="/languages" target="_blank">🔗 All Languages</a></li>
|
|
<li><a href="https://shop.unturf.com/p/8486f492-a93e-11f0-b477-02dfe05770ee/uncloseai-machine-learning-reference-guide-to-inference-clients" target="_blank">📚 Book</a></li>
|
|
</ul>
|
|
</nav>
|
|
</div>
|
|
</aside>
|
|
|
|
<!-- Main content -->
|
|
<main>
|
|
<header>
|
|
<hgroup>
|
|
<a href="https://uncloseai.com"><h1 class="unturf" style="font-family: 'ChunkFiveRegular';">uncloseai.</h1></a>
|
|
<p>How We Run Inference</p>
|
|
</hgroup>
|
|
<nav>
|
|
<ul>
|
|
<li><a href="#" onclick="switchTheme('auto')">Auto</a></li>
|
|
<li><a href="#" onclick="switchTheme('light')">Light</a></li>
|
|
<li><a href="#" onclick="switchTheme('dark')">Dark</a></li>
|
|
</ul>
|
|
</nav>
|
|
</header>
|
|
|
|
<h2 id="inference-intro">How we run inference</h2>
|
|
<p>This section is optional. This is only if you wanted to try to contribute idle GPU time to the project or if you wanted to reproduce everything in your own cluster.</p>
|
|
|
|
<h3 id="vllm-setup">vLLM Setup</h3>
|
|
<p>We use vLLM to run models, generally with full f16 safetensors. We make sure to use a virtualenv to hold the dependencies.</p>
|
|
<p><strong>Note:</strong> For Hermes, we use an FP8 quant by adamo1139 (adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic), which is optimized for 4090 and 3090 GPUs.</p>
|
|
<p>We are considering supporting ollama for better quant support.</p>
|
|
<p>Stand up a replica cluster on a new domain.</p>
|
|
<pre><code>sudo apt-get install gcc python3.12-dev python3.12-venv
|
|
cd ~
|
|
python3 -m venv env
|
|
source env/bin/activate
|
|
pip install vllm
|
|
python -m vllm.entrypoints.openai.api_server --model adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic --host 0.0.0.0 --port 18888 --max-model-len 82000
|
|
</code></pre>
|
|
|
|
<h3 id="tts-setup">TTS Setup</h3>
|
|
<p>For usage examples and documentation, see the <a href="/text-to-speech.html">Text-to-Speech page</a>.</p>
|
|
|
|
<h3 id="proxy-setup">Proxy Setup</h3>
|
|
<p>If you want to see how we setup the proxy, check out <a href="https://git.unturf.com/-/snippets/3" target="_blank">/etc/caddy/Caddyfile</a></p>
|
|
|
|
<pre><code>ai.unturf.com {
|
|
root * /opt/www
|
|
file_server
|
|
log {
|
|
output file /var/log/caddy/ai.unturf.com.log {
|
|
roll_size 50mb
|
|
roll_keep 5
|
|
}
|
|
}
|
|
tls {
|
|
on_demand
|
|
}
|
|
}
|
|
|
|
hermes.ai.unturf.com {
|
|
reverse_proxy <removed>:18888
|
|
log {
|
|
output file /var/log/caddy/hermes.ai.unturf.com.log {
|
|
roll_size 50mb
|
|
roll_keep 5
|
|
}
|
|
}
|
|
tls {
|
|
on_demand
|
|
}
|
|
}
|
|
|
|
speech.ai.unturf.com {
|
|
reverse_proxy <removed>:8000
|
|
log {
|
|
output file /var/log/caddy/speech.ai.unturf.com.log {
|
|
roll_size 50mb
|
|
roll_keep 5
|
|
}
|
|
}
|
|
tls {
|
|
on_demand
|
|
}
|
|
}
|
|
</code></pre>
|
|
|
|
<h3 id="rate-limiting">Rate Limiting</h3>
|
|
<p>Rate limiting is configured based on client IP address: 3 requests per second per IP per endpoint.</p>
|
|
|
|
<script>hljs.highlightAll();</script>
|
|
|
|
<footer>
|
|
<small>© uncloseai. 2025</small>
|
|
<br>
|
|
<small>Stylesheets by <a href="https://picocss.com" target="_blank">PicoCSS</a></small>
|
|
<small>& <a href="https://highlightjs.org/" target="_blank">highlight.js</a></small>
|
|
<br>
|
|
<small><a href="/privacy-policy.html">Privacy Policy</a> | <a href="/terms-of-use.html">Terms of Use</a></small>
|
|
</footer>
|
|
|
|
<script>
|
|
// Simple active link highlighting
|
|
document.querySelectorAll('.table-of-contents a').forEach(link => {
|
|
link.addEventListener('click', function() {
|
|
document.querySelectorAll('.table-of-contents a').forEach(a => a.classList.remove('active'));
|
|
this.classList.add('active');
|
|
});
|
|
});
|
|
|
|
// Highlight current section on scroll
|
|
const sections = document.querySelectorAll('h2[id], h3[id]');
|
|
const navLinks = document.querySelectorAll('.table-of-contents a');
|
|
|
|
window.addEventListener('scroll', () => {
|
|
let current = '';
|
|
sections.forEach(section => {
|
|
const sectionTop = section.offsetTop;
|
|
const sectionHeight = section.clientHeight;
|
|
if (window.scrollY >= sectionTop - 100) {
|
|
current = section.getAttribute('id');
|
|
}
|
|
});
|
|
|
|
navLinks.forEach(link => {
|
|
link.classList.remove('active');
|
|
if (link.getAttribute('href') === '#' + current) {
|
|
link.classList.add('active');
|
|
}
|
|
});
|
|
});
|
|
</script>
|
|
</main>
|
|
|
|
<!-- Right sidebar - Page TOC -->
|
|
<aside class="sidebar-right">
|
|
<div class="table-of-contents">
|
|
|
|
<nav>
|
|
<ul>
|
|
<li><a href="#inference-intro" class="active">Overview</a></li>
|
|
<li><a href="#vllm-setup">vLLM Setup</a></li>
|
|
<li><a href="#tts-setup">TTS Setup</a></li>
|
|
<li><a href="#proxy-setup">Proxy Setup</a></li>
|
|
<li><a href="#rate-limiting">Rate Limiting</a></li>
|
|
</ul>
|
|
</nav>
|
|
</div>
|
|
</aside>
|
|
</body>
|
|
|
|
</html>
|