Russell Ballestrini
bfee917049
Created /crawler.html page explaining: - Our user agent (uncloseai.com/1.42) - What we use the crawler for (research, Q&A, content aggregation) - Ethical practices (robots.txt, crawl delays, caching, smart depth) - How to block the crawler - Link to Discord bot Added crawler link to sidebar navigation in inference.html This provides transparency for webmasters and explains our ethical crawling practices in human-friendly terms.
218 lines
8.7 KiB
HTML
218 lines
8.7 KiB
HTML
<!DOCTYPE html>
|
|
<html lang="en">
|
|
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<meta name="theme-color" content="#43a047">
|
|
<meta name="color-scheme" content="light dark">
|
|
<title>How We Run Inference | uncloseai.com</title>
|
|
|
|
<!-- PicoCSS -->
|
|
<link rel="stylesheet" href="/css/pico.classless.min.css">
|
|
<!-- ChunkFive Font -->
|
|
<link rel="stylesheet" href="/css/chunkfive/stylesheet.css" type="text/css" charset="utf-8" />
|
|
<!-- Sidebar Theme -->
|
|
<link rel="stylesheet" href="/css/sidebar-theme.css">
|
|
|
|
<!-- Highlight.js for syntax highlighting -->
|
|
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.10.0/styles/a11y-dark.min.css" />
|
|
<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.6.0/highlight.min.js"></script>
|
|
|
|
<!-- Theme Switcher Script -->
|
|
<script>
|
|
function switchTheme(theme) {
|
|
if (theme === "auto") {
|
|
document.documentElement.removeAttribute('data-theme');
|
|
} else {
|
|
document.documentElement.setAttribute('data-theme', theme);
|
|
}
|
|
}
|
|
|
|
// Disable custom styling for uncloseai.js - use PicoCSS instead
|
|
window.UNCLOSEAI_CUSTOM_STYLING = false;
|
|
</script>
|
|
|
|
<script src="https://uncloseai.com/uncloseai.js" type="module"></script>
|
|
</head>
|
|
|
|
<body>
|
|
<!-- Mobile menu toggle -->
|
|
<button class="sidebar-toggle" onclick="document.querySelector('.sidebar').classList.toggle('open')">
|
|
☰
|
|
</button>
|
|
|
|
<!-- Left sidebar - Main Navigation -->
|
|
<aside class="sidebar">
|
|
<div class="table-of-contents">
|
|
|
|
<nav>
|
|
<ul>
|
|
<li><a href="/">Home</a></li>
|
|
<li><a href="/csharp-examples.html">C# Examples</a></li>
|
|
<li><a href="/dart-examples.html">Dart Examples</a></li>
|
|
<li><a href="/elixir-examples.html">Elixir Examples</a></li>
|
|
<li><a href="/go-examples.html">Go Examples</a></li>
|
|
<li><a href="/java-examples.html">Java Examples</a></li>
|
|
<li><a href="/kotlin-examples.html">Kotlin Examples</a></li>
|
|
<li><a href="/nodejs-examples.html">Node.js Examples</a></li>
|
|
<li><a href="/php-examples.html">PHP Examples</a></li>
|
|
<li><a href="/python-examples.html">Python Examples</a></li>
|
|
<li><a href="/ruby-examples.html">Ruby Examples</a></li>
|
|
<li><a href="/rust-examples.html">Rust Examples</a></li>
|
|
<li><a href="/swift-examples.html">Swift Examples</a></li>
|
|
<li><a href="/uncloseai-js.html">uncloseai.js Docs</a></li>
|
|
<li><a href="/inference.html" class="active">Inference Setup</a></li>
|
|
<li><a href="/crawler.html">Our Crawler</a></li>
|
|
<li><a href="/languages" target="_blank">🔗 All Languages</a></li>
|
|
<li><a href="https://shop.unturf.com/p/8486f492-a93e-11f0-b477-02dfe05770ee/uncloseai-machine-learning-reference-guide-to-inference-clients" target="_blank">📚 Book</a></li>
|
|
</ul>
|
|
</nav>
|
|
</div>
|
|
</aside>
|
|
|
|
<!-- Main content -->
|
|
<main>
|
|
<header>
|
|
<hgroup>
|
|
<a href="https://uncloseai.com"><h1 class="unturf" style="font-family: 'ChunkFiveRegular';">uncloseai.</h1></a>
|
|
<p>How We Run Inference</p>
|
|
</hgroup>
|
|
<nav>
|
|
<ul>
|
|
<li><a href="#" onclick="switchTheme('auto')">Auto</a></li>
|
|
<li><a href="#" onclick="switchTheme('light')">Light</a></li>
|
|
<li><a href="#" onclick="switchTheme('dark')">Dark</a></li>
|
|
</ul>
|
|
</nav>
|
|
</header>
|
|
|
|
<h2 id="inference-intro">How we run inference</h2>
|
|
<p>This section is optional. This is only if you wanted to try to contribute idle GPU time to the project or if you wanted to reproduce everything in your own cluster.</p>
|
|
|
|
<h3 id="vllm-setup">vLLM Setup</h3>
|
|
<p>We use vLLM to run models, generally with full f16 safetensors. We make sure to use a virtualenv to hold the dependencies.</p>
|
|
<p><strong>Note:</strong> For Hermes, we use an FP8 quant by adamo1139 (adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic), which is optimized for 4090 and 3090 GPUs.</p>
|
|
<p><strong>Tool Calling:</strong> The <code>--enable-auto-tool-choice</code> and <code>--tool-call-parser hermes</code> flags are required for Hermes 3 to support function calling via the OpenAI-compatible API. Without these flags, tool calling requests will fail.</p>
|
|
<p>We are considering supporting ollama for better quant support.</p>
|
|
<p>Stand up a replica cluster on a new domain.</p>
|
|
<pre><code>sudo apt-get install gcc python3.12-dev python3.12-venv
|
|
cd ~
|
|
python3 -m venv env
|
|
source env/bin/activate
|
|
pip install vllm
|
|
python -m vllm.entrypoints.openai.api_server --model adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic --host 0.0.0.0 --port 18888 --max-model-len 82000 --enable-auto-tool-choice --tool-call-parser hermes
|
|
</code></pre>
|
|
|
|
<h3 id="tts-setup">TTS Setup</h3>
|
|
<p>For usage examples and documentation, see the <a href="/text-to-speech.html">Text-to-Speech page</a>.</p>
|
|
|
|
<h3 id="proxy-setup">Proxy Setup</h3>
|
|
<p>If you want to see how we setup the proxy, check out <a href="https://git.unturf.com/-/snippets/3" target="_blank">/etc/caddy/Caddyfile</a></p>
|
|
|
|
<pre><code>ai.unturf.com {
|
|
root * /opt/www
|
|
file_server
|
|
log {
|
|
output file /var/log/caddy/ai.unturf.com.log {
|
|
roll_size 50mb
|
|
roll_keep 5
|
|
}
|
|
}
|
|
tls {
|
|
on_demand
|
|
}
|
|
}
|
|
|
|
hermes.ai.unturf.com {
|
|
reverse_proxy <removed>:18888
|
|
log {
|
|
output file /var/log/caddy/hermes.ai.unturf.com.log {
|
|
roll_size 50mb
|
|
roll_keep 5
|
|
}
|
|
}
|
|
tls {
|
|
on_demand
|
|
}
|
|
}
|
|
|
|
speech.ai.unturf.com {
|
|
reverse_proxy <removed>:8000
|
|
log {
|
|
output file /var/log/caddy/speech.ai.unturf.com.log {
|
|
roll_size 50mb
|
|
roll_keep 5
|
|
}
|
|
}
|
|
tls {
|
|
on_demand
|
|
}
|
|
}
|
|
</code></pre>
|
|
|
|
<h3 id="rate-limiting">Rate Limiting</h3>
|
|
<p>Rate limiting is configured based on client IP address: 3 requests per second per IP per endpoint.</p>
|
|
|
|
<script>hljs.highlightAll();</script>
|
|
|
|
<footer>
|
|
<small>© uncloseai. 2025</small>
|
|
<br>
|
|
<small>Stylesheets by <a href="https://picocss.com" target="_blank">PicoCSS</a></small>
|
|
<small>& <a href="https://highlightjs.org/" target="_blank">highlight.js</a></small>
|
|
<br>
|
|
<small><a href="/privacy-policy.html">Privacy Policy</a> | <a href="/terms-of-use.html">Terms of Use</a></small>
|
|
</footer>
|
|
|
|
<script>
|
|
// Simple active link highlighting
|
|
document.querySelectorAll('.table-of-contents a').forEach(link => {
|
|
link.addEventListener('click', function() {
|
|
document.querySelectorAll('.table-of-contents a').forEach(a => a.classList.remove('active'));
|
|
this.classList.add('active');
|
|
});
|
|
});
|
|
|
|
// Highlight current section on scroll
|
|
const sections = document.querySelectorAll('h2[id], h3[id]');
|
|
const navLinks = document.querySelectorAll('.table-of-contents a');
|
|
|
|
window.addEventListener('scroll', () => {
|
|
let current = '';
|
|
sections.forEach(section => {
|
|
const sectionTop = section.offsetTop;
|
|
const sectionHeight = section.clientHeight;
|
|
if (window.scrollY >= sectionTop - 100) {
|
|
current = section.getAttribute('id');
|
|
}
|
|
});
|
|
|
|
navLinks.forEach(link => {
|
|
link.classList.remove('active');
|
|
if (link.getAttribute('href') === '#' + current) {
|
|
link.classList.add('active');
|
|
}
|
|
});
|
|
});
|
|
</script>
|
|
</main>
|
|
|
|
<!-- Right sidebar - Page TOC -->
|
|
<aside class="sidebar-right">
|
|
<div class="table-of-contents">
|
|
|
|
<nav>
|
|
<ul>
|
|
<li><a href="#inference-intro" class="active">Overview</a></li>
|
|
<li><a href="#vllm-setup">vLLM Setup</a></li>
|
|
<li><a href="#tts-setup">TTS Setup</a></li>
|
|
<li><a href="#proxy-setup">Proxy Setup</a></li>
|
|
<li><a href="#rate-limiting">Rate Limiting</a></li>
|
|
</ul>
|
|
</nav>
|
|
</div>
|
|
</aside>
|
|
</body>
|
|
|
|
</html>
|