268 lines
13 KiB
HTML
268 lines
13 KiB
HTML
<!--
|
|
This is free software for the public good of a permacomputer hosted at
|
|
permacomputer.com, an always-on computer by the people, for the people.
|
|
One which is durable, easy to repair, & distributed like tap water
|
|
for machine learning intelligence.
|
|
|
|
The permacomputer is community-owned infrastructure optimized around
|
|
four values:
|
|
|
|
TRUTH First principles, math & science, open source code freely distributed
|
|
FREEDOM Voluntary partnerships, freedom from tyranny & corporate control
|
|
HARMONY Minimal waste, self-renewing systems with diverse thriving connections
|
|
LOVE Be yourself without hurting others, cooperation through natural law
|
|
|
|
This software contributes to that vision by making machine learning
|
|
accessible to everyone through a free, open, embeddable chat interface.
|
|
Code is seeds to sprout on any abandoned technology.
|
|
-->
|
|
<!DOCTYPE html>
|
|
<html lang="en">
|
|
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<meta name="theme-color" content="#43a047">
|
|
<meta name="color-scheme" content="light dark">
|
|
<title>Our Ethical Web Crawler | uncloseai.com</title>
|
|
|
|
<!-- PicoCSS -->
|
|
<link rel="stylesheet" href="/css/pico.classless.min.css">
|
|
<!-- ChunkFive Font -->
|
|
<link rel="stylesheet" href="/css/chunkfive/stylesheet.css" type="text/css" charset="utf-8" />
|
|
<!-- Sidebar Theme -->
|
|
<link rel="stylesheet" href="/css/sidebar-theme.css">
|
|
|
|
<!-- Highlight.js for syntax highlighting -->
|
|
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.10.0/styles/a11y-dark.min.css" />
|
|
<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.6.0/highlight.min.js"></script>
|
|
|
|
<!-- Theme Switcher Script -->
|
|
<script>
|
|
function switchTheme(theme) {
|
|
if (theme === "auto") {
|
|
document.documentElement.removeAttribute('data-theme');
|
|
} else {
|
|
document.documentElement.setAttribute('data-theme', theme);
|
|
}
|
|
}
|
|
|
|
// Disable custom styling for uncloseai.js - use PicoCSS instead
|
|
window.UNCLOSEAI_CUSTOM_STYLING = false;
|
|
</script>
|
|
|
|
<script src="https://uncloseai.com/uncloseai.js" type="module"></script>
|
|
</head>
|
|
|
|
<body>
|
|
<!-- Mobile menu toggle -->
|
|
<button class="sidebar-toggle" onclick="document.querySelector('.sidebar').classList.toggle('open')">
|
|
☰
|
|
</button>
|
|
|
|
<!-- Left sidebar - Main Navigation -->
|
|
<aside class="sidebar">
|
|
<div class="table-of-contents">
|
|
|
|
<nav>
|
|
<ul>
|
|
<li><a href="/">Home</a></li>
|
|
<li><a href="/c-examples.html">C Examples</a></li>
|
|
<li><a href="/csharp-examples.html">C# Examples</a></li>
|
|
<li><a href="/dart-examples.html">Dart Examples</a></li>
|
|
<li><a href="/elixir-examples.html">Elixir Examples</a></li>
|
|
<li><a href="/go-examples.html">Go Examples</a></li>
|
|
<li><a href="/java-examples.html">Java Examples</a></li>
|
|
<li><a href="/kotlin-examples.html">Kotlin Examples</a></li>
|
|
<li><a href="/nodejs-examples.html">Node.js Examples</a></li>
|
|
<li><a href="/php-examples.html">PHP Examples</a></li>
|
|
<li><a href="/python-examples.html">Python Examples</a></li>
|
|
<li><a href="/ruby-examples.html">Ruby Examples</a></li>
|
|
<li><a href="/rust-examples.html">Rust Examples</a></li>
|
|
<li><a href="/swift-examples.html">Swift Examples</a></li>
|
|
<li><a href="/uncloseai-js.html">uncloseai.js Docs</a></li>
|
|
<li><a href="/uncloseai-js-styleguide.html">Styleguide</a></li>
|
|
<li><a href="/cli.html">uncloseai-cli</a></li>
|
|
<li><a href="/browser-toys.html">Browser Toys</a></li>
|
|
<li><a href="/inference.html">Inference Setup</a></li>
|
|
<li><a href="/crawler.html" class="active">Our Crawler</a></li>
|
|
<li><a href="/languages" target="_blank">🔗 All Languages</a></li>
|
|
<li><a href="https://shop.unturf.com/p/8486f492-a93e-11f0-b477-02dfe05770ee/uncloseai-machine-learning-reference-guide-to-inference-clients" target="_blank">📚 Book</a></li>
|
|
</ul>
|
|
</nav>
|
|
</div>
|
|
</aside>
|
|
|
|
<!-- Main content -->
|
|
<main>
|
|
<header>
|
|
<hgroup>
|
|
<a href="https://uncloseai.com"><h1 class="unturf" style="font-family: 'ChunkFiveRegular';">uncloseai.</h1></a>
|
|
<p>Our Ethical Web Crawler</p>
|
|
</hgroup>
|
|
<nav>
|
|
<ul>
|
|
<li><a href="#" onclick="switchTheme('auto')">Auto</a></li>
|
|
<li><a href="#" onclick="switchTheme('light')">Light</a></li>
|
|
<li><a href="#" onclick="switchTheme('dark')">Dark</a></li>
|
|
</ul>
|
|
</nav>
|
|
</header>
|
|
|
|
<h2 id="intro">About Our Web Crawler</h2>
|
|
<p>Our team uses an ethical web crawler for research and development on the open web. We believe in responsible data collection that respects website owners and follows industry best practices.</p>
|
|
|
|
<h3 id="user-agent">User Agent</h3>
|
|
<p>Our crawler identifies itself with the following user agent:</p>
|
|
<pre><code>uncloseai.com/1.42 (ethical web crawler; +https://uncloseai.com)</code></pre>
|
|
<p>This clearly identifies who we are and provides a link back to this page for more information.</p>
|
|
|
|
<h3 id="what-we-do">What We Do</h3>
|
|
<p>Our crawler is used by our AI systems to:</p>
|
|
<ul>
|
|
<li><strong>Answer questions</strong> - When users ask about websites or specific information, our bot can fetch and analyze web content to provide accurate answers</li>
|
|
<li><strong>Research & development</strong> - We study how information is structured on the web to improve our AI models and tools</li>
|
|
<li><strong>Content aggregation</strong> - We help users discover and understand information across multiple web pages</li>
|
|
</ul>
|
|
|
|
<h3 id="ethical-practices">Our Ethical Practices</h3>
|
|
<p>We follow strict ethical guidelines to ensure we're good web citizens:</p>
|
|
|
|
<h4>1. Robots.txt Compliance</h4>
|
|
<p>We <strong>always respect robots.txt</strong> files. If your site's robots.txt disallows our crawler, we won't access those pages.</p>
|
|
<pre><code># Example: Block our crawler from specific paths
|
|
User-agent: uncloseai.com
|
|
Disallow: /private/
|
|
Disallow: /admin/</code></pre>
|
|
|
|
<h4>2. Crawl Delays</h4>
|
|
<p>We respect crawl delays specified in robots.txt. Our default is <strong>2 seconds between requests</strong> to the same domain, but we'll honor any delay you specify:</p>
|
|
<pre><code># Example: Set custom crawl delay
|
|
User-agent: uncloseai.com
|
|
Crawl-delay: 5</code></pre>
|
|
|
|
<h4>3. Intelligent Caching</h4>
|
|
<p>We cache fetched content for <strong>7 days</strong> by default. This means:</p>
|
|
<ul>
|
|
<li>We won't repeatedly hit your server for the same content</li>
|
|
<li>Reduces load on your infrastructure</li>
|
|
<li>Faster responses for our users</li>
|
|
</ul>
|
|
|
|
<h4>4. Smart Depth Crawling</h4>
|
|
<p>We use intelligent depth-based crawling:</p>
|
|
<ul>
|
|
<li><strong>Depth 0</strong> - Just the homepage</li>
|
|
<li><strong>Depth 1</strong> - Homepage plus directly linked pages</li>
|
|
<li><strong>Depth 2</strong> - Goes two levels deep when needed</li>
|
|
</ul>
|
|
<p>Our crawler automatically decides how deep to go based on whether it found relevant information, so we don't waste resources fetching unnecessary pages.</p>
|
|
|
|
<h4>5. Query-Aware Relevance</h4>
|
|
<p>We score pages based on relevance to the user's question, prioritizing:</p>
|
|
<ul>
|
|
<li>Pages that contain keywords from the user's query</li>
|
|
<li>Clean, readable URLs over dynamic query strings</li>
|
|
<li>Pages with substantial, unique content</li>
|
|
<li>Descriptive titles that match the topic</li>
|
|
</ul>
|
|
|
|
<h3 id="transparency">Transparency</h3>
|
|
<p>We believe in being open about our crawling practices:</p>
|
|
<ul>
|
|
<li><strong>Clear identification</strong> - Our user agent clearly identifies us</li>
|
|
<li><strong>Contact information</strong> - This page provides details on how to reach us</li>
|
|
<li><strong>Open source approach</strong> - Our crawler follows public best practices</li>
|
|
<li><strong>No stealth crawling</strong> - We never hide our identity or bypass restrictions</li>
|
|
</ul>
|
|
|
|
<h3 id="blocking">How to Block Our Crawler</h3>
|
|
<p>If you don't want our crawler accessing your site, you can block it using robots.txt:</p>
|
|
<pre><code># Block uncloseai.com crawler entirely
|
|
User-agent: uncloseai.com
|
|
Disallow: /</code></pre>
|
|
<p>Or use your server's firewall/WAF to block our user agent string.</p>
|
|
|
|
<h3 id="contact">Questions or Concerns?</h3>
|
|
<p>If you have questions about our crawler or want to discuss our access to your site, please reach out:</p>
|
|
<ul>
|
|
<li><strong>Website</strong>: <a href="https://uncloseai.com">uncloseai.com</a></li>
|
|
<li><strong>Discord</strong>: Join our community at <a href="https://discord.com/invite/TZPmQdPj8a" target="_blank">discord.com/invite/TZPmQdPj8a</a></li>
|
|
<li><strong>Email</strong>: Contact us through our website</li>
|
|
</ul>
|
|
|
|
<h3 id="discord-bot">Try Our Discord Bot</h3>
|
|
<p>Our ethical web crawler powers the research capabilities in our Discord bot. The bot can:</p>
|
|
<ul>
|
|
<li>Answer questions by fetching and analyzing web content</li>
|
|
<li>Execute code in 42+ programming languages</li>
|
|
<li>Provide intelligent summaries of web pages</li>
|
|
<li>Respect user privacy and website policies</li>
|
|
</ul>
|
|
<p><strong>Want to try it?</strong> Check out our <a href="https://git.unturf.com/engineering/unturf/discord.agents.ai.unturf.com" target="_blank">Discord bot source code</a> to see how our ethical crawler powers the bot's research capabilities.</p>
|
|
|
|
<script>hljs.highlightAll();</script>
|
|
|
|
<footer>
|
|
<small>© uncloseai. 2025</small>
|
|
<br>
|
|
<small>Stylesheets by <a href="https://picocss.com" target="_blank">PicoCSS</a></small>
|
|
<small>& <a href="https://highlightjs.org/" target="_blank">highlight.js</a></small>
|
|
<br>
|
|
<small><a href="/privacy-policy.html">Privacy Policy</a> | <a href="/terms-of-use.html">Terms of Use</a></small>
|
|
</footer>
|
|
|
|
<script>
|
|
// Simple active link highlighting
|
|
document.querySelectorAll('.table-of-contents a').forEach(link => {
|
|
link.addEventListener('click', function() {
|
|
document.querySelectorAll('.table-of-contents a').forEach(a => a.classList.remove('active'));
|
|
this.classList.add('active');
|
|
});
|
|
});
|
|
|
|
// Highlight current section on scroll
|
|
const sections = document.querySelectorAll('h2[id], h3[id], h4[id]');
|
|
const navLinks = document.querySelectorAll('.table-of-contents a');
|
|
|
|
window.addEventListener('scroll', () => {
|
|
let current = '';
|
|
sections.forEach(section => {
|
|
const sectionTop = section.offsetTop;
|
|
const sectionHeight = section.clientHeight;
|
|
if (window.scrollY >= sectionTop - 100) {
|
|
current = section.getAttribute('id');
|
|
}
|
|
});
|
|
|
|
navLinks.forEach(link => {
|
|
link.classList.remove('active');
|
|
if (link.getAttribute('href') === '#' + current) {
|
|
link.classList.add('active');
|
|
}
|
|
});
|
|
});
|
|
</script>
|
|
</main>
|
|
|
|
<!-- Right sidebar - Page TOC -->
|
|
<aside class="sidebar-right">
|
|
<div class="table-of-contents">
|
|
|
|
<nav>
|
|
<ul>
|
|
<li><a href="#intro" class="active">About</a></li>
|
|
<li><a href="#user-agent">User Agent</a></li>
|
|
<li><a href="#what-we-do">What We Do</a></li>
|
|
<li><a href="#ethical-practices">Ethical Practices</a></li>
|
|
<li><a href="#transparency">Transparency</a></li>
|
|
<li><a href="#blocking">How to Block</a></li>
|
|
<li><a href="#contact">Contact</a></li>
|
|
<li><a href="#discord-bot">Discord Bot</a></li>
|
|
</ul>
|
|
</nav>
|
|
</div>
|
|
</aside>
|
|
</body>
|
|
|
|
</html>
|