diff --git a/public/crawler.html b/public/crawler.html
new file mode 100644
index 0000000..b27d27e
--- /dev/null
+++ b/public/crawler.html
@@ -0,0 +1,246 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <meta name="theme-color" content="#43a047">
+    <meta name="color-scheme" content="light dark">
+    <title>Our Ethical Web Crawler | uncloseai.com</title>
+
+    <!-- PicoCSS -->
+    <link rel="stylesheet" href="/css/pico.classless.min.css">
+    <!-- ChunkFive Font -->
+    <link rel="stylesheet" href="/css/chunkfive/stylesheet.css" type="text/css" charset="utf-8" />
+    <!-- Sidebar Theme -->
+    <link rel="stylesheet" href="/css/sidebar-theme.css">
+
+    <!-- Highlight.js for syntax highlighting -->
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.10.0/styles/a11y-dark.min.css" />
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.6.0/highlight.min.js"></script>
+
+    <!-- Theme Switcher Script -->
+    <script>
+        function switchTheme(theme) {
+            if (theme === "auto") {
+                document.documentElement.removeAttribute('data-theme');
+            } else {
+                document.documentElement.setAttribute('data-theme', theme);
+            }
+        }
+
+        // Disable custom styling for uncloseai.js - use PicoCSS instead
+        window.UNCLOSEAI_CUSTOM_STYLING = false;
+    </script>
+
+    <script src="https://uncloseai.com/uncloseai.js" type="module"></script>
+</head>
+
+<body>
+    <!-- Mobile menu toggle -->
+    <button class="sidebar-toggle" onclick="document.querySelector('.sidebar').classList.toggle('open')">
+        ☰
+    </button>
+
+    <!-- Left sidebar - Main Navigation -->
+    <aside class="sidebar">
+        <div class="table-of-contents">
+
+            <nav>
+                <ul>
+                    <li><a href="/">Home</a></li>
+                    <li><a href="/csharp-examples.html">C# Examples</a></li>
+                    <li><a href="/dart-examples.html">Dart Examples</a></li>
+                    <li><a href="/elixir-examples.html">Elixir Examples</a></li>
+                    <li><a href="/go-examples.html">Go Examples</a></li>
+                    <li><a href="/java-examples.html">Java Examples</a></li>
+                    <li><a href="/kotlin-examples.html">Kotlin Examples</a></li>
+                    <li><a href="/nodejs-examples.html">Node.js Examples</a></li>
+                    <li><a href="/php-examples.html">PHP Examples</a></li>
+                    <li><a href="/python-examples.html">Python Examples</a></li>
+                    <li><a href="/ruby-examples.html">Ruby Examples</a></li>
+                    <li><a href="/rust-examples.html">Rust Examples</a></li>
+                    <li><a href="/swift-examples.html">Swift Examples</a></li>
+                    <li><a href="/uncloseai-js.html">uncloseai.js Docs</a></li>
+                    <li><a href="/inference.html">Inference Setup</a></li>
+                    <li><a href="/crawler.html" class="active">Our Crawler</a></li>
+                    <li><a href="/languages" target="_blank">🔗 All Languages</a></li>
+                    <li><a href="https://shop.unturf.com/p/8486f492-a93e-11f0-b477-02dfe05770ee/uncloseai-machine-learning-reference-guide-to-inference-clients" target="_blank">📚 Book</a></li>
+                </ul>
+            </nav>
+        </div>
+    </aside>
+
+    <!-- Main content -->
+    <main>
+        <header>
+        <hgroup>
+            <a href="https://uncloseai.com"><h1 class="unturf" style="font-family: 'ChunkFiveRegular';">uncloseai.</h1></a>
+            <p>Our Ethical Web Crawler</p>
+        </hgroup>
+        <nav>
+            <ul>
+                <li><a href="#" onclick="switchTheme('auto')">Auto</a></li>
+                <li><a href="#" onclick="switchTheme('light')">Light</a></li>
+                <li><a href="#" onclick="switchTheme('dark')">Dark</a></li>
+            </ul>
+        </nav>
+        </header>
+
+        <h2 id="intro">About Our Web Crawler</h2>
+        <p>Our team uses an ethical web crawler for research and development on the open web. We believe in responsible data collection that respects website owners and follows industry best practices.</p>
+
+        <h3 id="user-agent">User Agent</h3>
+        <p>Our crawler identifies itself with the following user agent:</p>
+        <pre><code>uncloseai.com/1.42 (ethical web crawler; +https://uncloseai.com)</code></pre>
+        <p>This clearly identifies who we are and provides a link back to this page for more information.</p>
+
+        <h3 id="what-we-do">What We Do</h3>
+        <p>Our crawler is used by our AI systems to:</p>
+        <ul>
+            <li><strong>Answer questions</strong> - When users ask about websites or specific information, our bot can fetch and analyze web content to provide accurate answers</li>
+            <li><strong>Research & development</strong> - We study how information is structured on the web to improve our AI models and tools</li>
+            <li><strong>Content aggregation</strong> - We help users discover and understand information across multiple web pages</li>
+        </ul>
+
+        <h3 id="ethical-practices">Our Ethical Practices</h3>
+        <p>We follow strict ethical guidelines to ensure we're good web citizens:</p>
+
+        <h4>1. Robots.txt Compliance</h4>
+        <p>We <strong>always respect robots.txt</strong> files. If your site's robots.txt disallows our crawler, we won't access those pages.</p>
+        <pre><code># Example: Block our crawler from specific paths
+User-agent: uncloseai.com
+Disallow: /private/
+Disallow: /admin/</code></pre>
+
+        <h4>2. Crawl Delays</h4>
+        <p>We respect crawl delays specified in robots.txt. Our default is <strong>2 seconds between requests</strong> to the same domain, but we'll honor any delay you specify:</p>
+        <pre><code># Example: Set custom crawl delay
+User-agent: uncloseai.com
+Crawl-delay: 5</code></pre>
+
+        <h4>3. Intelligent Caching</h4>
+        <p>We cache fetched content for <strong>7 days</strong> by default. This means:</p>
+        <ul>
+            <li>We won't repeatedly hit your server for the same content</li>
+            <li>Reduces load on your infrastructure</li>
+            <li>Faster responses for our users</li>
+        </ul>
+
+        <h4>4. Smart Depth Crawling</h4>
+        <p>We use intelligent depth-based crawling:</p>
+        <ul>
+            <li><strong>Depth 0</strong> - Just the homepage</li>
+            <li><strong>Depth 1</strong> - Homepage plus directly linked pages</li>
+            <li><strong>Depth 2</strong> - Goes two levels deep when needed</li>
+        </ul>
+        <p>Our crawler automatically decides how deep to go based on whether it found relevant information, so we don't waste resources fetching unnecessary pages.</p>
+
+        <h4>5. Query-Aware Relevance</h4>
+        <p>We score pages based on relevance to the user's question, prioritizing:</p>
+        <ul>
+            <li>Pages that contain keywords from the user's query</li>
+            <li>Clean, readable URLs over dynamic query strings</li>
+            <li>Pages with substantial, unique content</li>
+            <li>Descriptive titles that match the topic</li>
+        </ul>
+
+        <h3 id="transparency">Transparency</h3>
+        <p>We believe in being open about our crawling practices:</p>
+        <ul>
+            <li><strong>Clear identification</strong> - Our user agent clearly identifies us</li>
+            <li><strong>Contact information</strong> - This page provides details on how to reach us</li>
+            <li><strong>Open source approach</strong> - Our crawler follows public best practices</li>
+            <li><strong>No stealth crawling</strong> - We never hide our identity or bypass restrictions</li>
+        </ul>
+
+        <h3 id="blocking">How to Block Our Crawler</h3>
+        <p>If you don't want our crawler accessing your site, you can block it using robots.txt:</p>
+        <pre><code># Block uncloseai.com crawler entirely
+User-agent: uncloseai.com
+Disallow: /</code></pre>
+        <p>Or use your server's firewall/WAF to block our user agent string.</p>
+
+        <h3 id="contact">Questions or Concerns?</h3>
+        <p>If you have questions about our crawler or want to discuss our access to your site, please reach out:</p>
+        <ul>
+            <li><strong>Website</strong>: <a href="https://uncloseai.com">uncloseai.com</a></li>
+            <li><strong>Discord</strong>: Join our community at <a href="https://discord.agents.ai.unturf.com">discord.agents.ai.unturf.com</a></li>
+            <li><strong>Email</strong>: Contact us through our website</li>
+        </ul>
+
+        <h3 id="discord-bot">Try Our Discord Bot</h3>
+        <p>Our ethical web crawler powers the research capabilities in our Discord bot. The bot can:</p>
+        <ul>
+            <li>Answer questions by fetching and analyzing web content</li>
+            <li>Execute code in 42+ programming languages</li>
+            <li>Provide intelligent summaries of web pages</li>
+            <li>Respect user privacy and website policies</li>
+        </ul>
+        <p><strong>Want to try it?</strong> Visit our Discord bot page at <a href="https://discord.agents.ai.unturf.com" target="_blank">discord.agents.ai.unturf.com</a> to add the bot to your server and see our ethical crawler in action.</p>
+
+        <script>hljs.highlightAll();</script>
+
+        <footer>
+        <small>© uncloseai. 2025</small>
+        <br>
+        <small>Stylesheets by <a href="https://picocss.com" target="_blank">PicoCSS</a></small>
+        <small>& <a href="https://highlightjs.org/" target="_blank">highlight.js</a></small>
+        <br>
+        <small><a href="/privacy-policy.html">Privacy Policy</a> | <a href="/terms-of-use.html">Terms of Use</a></small>
+        </footer>
+
+        <script>
+            // Simple active link highlighting
+            document.querySelectorAll('.table-of-contents a').forEach(link => {
+                link.addEventListener('click', function() {
+                    document.querySelectorAll('.table-of-contents a').forEach(a => a.classList.remove('active'));
+                    this.classList.add('active');
+                });
+            });
+
+            // Highlight current section on scroll
+            const sections = document.querySelectorAll('h2[id], h3[id], h4[id]');
+            const navLinks = document.querySelectorAll('.table-of-contents a');
+
+            window.addEventListener('scroll', () => {
+                let current = '';
+                sections.forEach(section => {
+                    const sectionTop = section.offsetTop;
+                    const sectionHeight = section.clientHeight;
+                    if (window.scrollY >= sectionTop - 100) {
+                        current = section.getAttribute('id');
+                    }
+                });
+
+                navLinks.forEach(link => {
+                    link.classList.remove('active');
+                    if (link.getAttribute('href') === '#' + current) {
+                        link.classList.add('active');
+                    }
+                });
+            });
+        </script>
+    </main>
+
+    <!-- Right sidebar - Page TOC -->
+    <aside class="sidebar-right">
+        <div class="table-of-contents">
+
+            <nav>
+                <ul>
+                    <li><a href="#intro" class="active">About</a></li>
+                    <li><a href="#user-agent">User Agent</a></li>
+                    <li><a href="#what-we-do">What We Do</a></li>
+                    <li><a href="#ethical-practices">Ethical Practices</a></li>
+                    <li><a href="#transparency">Transparency</a></li>
+                    <li><a href="#blocking">How to Block</a></li>
+                    <li><a href="#contact">Contact</a></li>
+                    <li><a href="#discord-bot">Discord Bot</a></li>
+                </ul>
+            </nav>
+        </div>
+    </aside>
+</body>
+
+</html>
diff --git a/public/inference.html b/public/inference.html
index 42fa278..9a31ca7 100644
--- a/public/inference.html
+++ b/public/inference.html
@@ -63,6 +63,7 @@
                     <li><a href="/swift-examples.html">Swift Examples</a></li>
                     <li><a href="/uncloseai-js.html">uncloseai.js Docs</a></li>
                     <li><a href="/inference.html" class="active">Inference Setup</a></li>
+                    <li><a href="/crawler.html">Our Crawler</a></li>
                     <li><a href="/languages" target="_blank">🔗 All Languages</a></li>
                     <li><a href="https://shop.unturf.com/p/8486f492-a93e-11f0-b477-02dfe05770ee/uncloseai-machine-learning-reference-guide-to-inference-clients" target="_blank">📚 Book</a></li>
                 </ul>
