russell@unturf.com
49607a5997
Implements Read the Docs infrastructure to generate API documentation directly from code docstrings. Replaces static modules.md (1200+ lines). New structure: - docs/_source/conf.py — Sphinx configuration (furo theme) - docs/_source/index.rst — Main TOC - docs/_source/api/*.rst — Module groups (substrate, storage, retrieval, qa, distill, mesh, cli) - docs/_source/Makefile — Local build targets - docs/_source/README.md — Documentation on building and extending Makefile integration: - make docs-api — generate HTML (output: docs/_source/_build/html/) - make docs-api-clean — remove build artifacts Build output (40 HTML files): - API module reference with docstrings - Source code links (:viewcode: extension) - Full-text search index - Module index (genindex, py-modindex) Sphinx installed in venv as dev dependency. HTML is browseable at docs/_source/_build/html/index.html (open in browser after build). This justifies the deletion of modules.md: code docstrings + Sphinx autodoc = automatically-generated, always-current API reference.
882 lines
No EOL
83 KiB
HTML
882 lines
No EOL
83 KiB
HTML
<!doctype html>
|
||
<html class="no-js" lang="en" data-content_root="../">
|
||
<head><meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||
<meta name="color-scheme" content="light dark"><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||
<link rel="index" title="Index" href="../genindex.html"><link rel="search" title="Search" href="../search.html"><link rel="next" title="Q&A Pipeline: question → answer → verify → cache" href="qa.html"><link rel="prev" title="Storage: SQLite schema and audit" href="storage.html">
|
||
|
||
<!-- Generated with Sphinx 9.1.0 and Furo 2025.12.19 -->
|
||
<title>Retrieval: FTS5 search and concepts - Aborist API Reference</title>
|
||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=d111a655" />
|
||
<link rel="stylesheet" type="text/css" href="../_static/styles/furo.css?v=7bdb33bb" />
|
||
<link rel="stylesheet" type="text/css" href="../_static/styles/furo-extensions.css?v=8dab3a3b" />
|
||
|
||
|
||
|
||
|
||
<style>
|
||
body {
|
||
--color-code-background: #f2f2f2;
|
||
--color-code-foreground: #1e1e1e;
|
||
|
||
}
|
||
@media not print {
|
||
body[data-theme="dark"] {
|
||
--color-code-background: #202020;
|
||
--color-code-foreground: #d0d0d0;
|
||
|
||
}
|
||
@media (prefers-color-scheme: dark) {
|
||
body:not([data-theme="light"]) {
|
||
--color-code-background: #202020;
|
||
--color-code-foreground: #d0d0d0;
|
||
|
||
}
|
||
}
|
||
}
|
||
</style></head>
|
||
<body>
|
||
|
||
<script>
|
||
document.body.dataset.theme = localStorage.getItem("theme") || "auto";
|
||
</script>
|
||
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" style="display: none;">
|
||
<symbol id="svg-toc" viewBox="0 0 24 24">
|
||
<title>Contents</title>
|
||
<svg stroke="currentColor" fill="currentColor" stroke-width="0" viewBox="0 0 1024 1024">
|
||
<path d="M408 442h480c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8H408c-4.4 0-8 3.6-8 8v56c0 4.4 3.6 8 8 8zm-8 204c0 4.4 3.6 8 8 8h480c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8H408c-4.4 0-8 3.6-8 8v56zm504-486H120c-4.4 0-8 3.6-8 8v56c0 4.4 3.6 8 8 8h784c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8zm0 632H120c-4.4 0-8 3.6-8 8v56c0 4.4 3.6 8 8 8h784c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8zM115.4 518.9L271.7 642c5.8 4.6 14.4.5 14.4-6.9V388.9c0-7.4-8.5-11.5-14.4-6.9L115.4 505.1a8.74 8.74 0 0 0 0 13.8z"/>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-menu" viewBox="0 0 24 24">
|
||
<title>Menu</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
|
||
stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather-menu">
|
||
<line x1="3" y1="12" x2="21" y2="12"></line>
|
||
<line x1="3" y1="6" x2="21" y2="6"></line>
|
||
<line x1="3" y1="18" x2="21" y2="18"></line>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-arrow-right" viewBox="0 0 24 24">
|
||
<title>Expand</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
|
||
stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather-chevron-right">
|
||
<polyline points="9 18 15 12 9 6"></polyline>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-sun" viewBox="0 0 24 24">
|
||
<title>Light mode</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
|
||
stroke-width="1" stroke-linecap="round" stroke-linejoin="round" class="feather-sun">
|
||
<circle cx="12" cy="12" r="5"></circle>
|
||
<line x1="12" y1="1" x2="12" y2="3"></line>
|
||
<line x1="12" y1="21" x2="12" y2="23"></line>
|
||
<line x1="4.22" y1="4.22" x2="5.64" y2="5.64"></line>
|
||
<line x1="18.36" y1="18.36" x2="19.78" y2="19.78"></line>
|
||
<line x1="1" y1="12" x2="3" y2="12"></line>
|
||
<line x1="21" y1="12" x2="23" y2="12"></line>
|
||
<line x1="4.22" y1="19.78" x2="5.64" y2="18.36"></line>
|
||
<line x1="18.36" y1="5.64" x2="19.78" y2="4.22"></line>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-moon" viewBox="0 0 24 24">
|
||
<title>Dark mode</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
|
||
stroke-width="1" stroke-linecap="round" stroke-linejoin="round" class="icon-tabler-moon">
|
||
<path stroke="none" d="M0 0h24v24H0z" fill="none" />
|
||
<path d="M12 3c.132 0 .263 0 .393 0a7.5 7.5 0 0 0 7.92 12.446a9 9 0 1 1 -8.313 -12.454z" />
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-sun-with-moon" viewBox="0 0 24 24">
|
||
<title>Auto light/dark, in light mode</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
|
||
stroke-width="1" stroke-linecap="round" stroke-linejoin="round"
|
||
class="icon-custom-derived-from-feather-sun-and-tabler-moon">
|
||
<path style="opacity: 50%" d="M 5.411 14.504 C 5.471 14.504 5.532 14.504 5.591 14.504 C 3.639 16.319 4.383 19.569 6.931 20.352 C 7.693 20.586 8.512 20.551 9.25 20.252 C 8.023 23.207 4.056 23.725 2.11 21.184 C 0.166 18.642 1.702 14.949 4.874 14.536 C 5.051 14.512 5.231 14.5 5.411 14.5 L 5.411 14.504 Z"/>
|
||
<line x1="14.5" y1="3.25" x2="14.5" y2="1.25"/>
|
||
<line x1="14.5" y1="15.85" x2="14.5" y2="17.85"/>
|
||
<line x1="10.044" y1="5.094" x2="8.63" y2="3.68"/>
|
||
<line x1="19" y1="14.05" x2="20.414" y2="15.464"/>
|
||
<line x1="8.2" y1="9.55" x2="6.2" y2="9.55"/>
|
||
<line x1="20.8" y1="9.55" x2="22.8" y2="9.55"/>
|
||
<line x1="10.044" y1="14.006" x2="8.63" y2="15.42"/>
|
||
<line x1="19" y1="5.05" x2="20.414" y2="3.636"/>
|
||
<circle cx="14.5" cy="9.55" r="3.6"/>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-moon-with-sun" viewBox="0 0 24 24">
|
||
<title>Auto light/dark, in dark mode</title>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
|
||
stroke-width="1" stroke-linecap="round" stroke-linejoin="round"
|
||
class="icon-custom-derived-from-feather-sun-and-tabler-moon">
|
||
<path d="M 8.282 7.007 C 8.385 7.007 8.494 7.007 8.595 7.007 C 5.18 10.184 6.481 15.869 10.942 17.24 C 12.275 17.648 13.706 17.589 15 17.066 C 12.851 22.236 5.91 23.143 2.505 18.696 C -0.897 14.249 1.791 7.786 7.342 7.063 C 7.652 7.021 7.965 7 8.282 7 L 8.282 7.007 Z"/>
|
||
<line style="opacity: 50%" x1="18" y1="3.705" x2="18" y2="2.5"/>
|
||
<line style="opacity: 50%" x1="18" y1="11.295" x2="18" y2="12.5"/>
|
||
<line style="opacity: 50%" x1="15.316" y1="4.816" x2="14.464" y2="3.964"/>
|
||
<line style="opacity: 50%" x1="20.711" y1="10.212" x2="21.563" y2="11.063"/>
|
||
<line style="opacity: 50%" x1="14.205" y1="7.5" x2="13.001" y2="7.5"/>
|
||
<line style="opacity: 50%" x1="21.795" y1="7.5" x2="23" y2="7.5"/>
|
||
<line style="opacity: 50%" x1="15.316" y1="10.184" x2="14.464" y2="11.036"/>
|
||
<line style="opacity: 50%" x1="20.711" y1="4.789" x2="21.563" y2="3.937"/>
|
||
<circle style="opacity: 50%" cx="18" cy="7.5" r="2.169"/>
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-pencil" viewBox="0 0 24 24">
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
|
||
stroke-width="1" stroke-linecap="round" stroke-linejoin="round" class="icon-tabler-pencil-code">
|
||
<path d="M4 20h4l10.5 -10.5a2.828 2.828 0 1 0 -4 -4l-10.5 10.5v4" />
|
||
<path d="M13.5 6.5l4 4" />
|
||
<path d="M20 21l2 -2l-2 -2" />
|
||
<path d="M17 17l-2 2l2 2" />
|
||
</svg>
|
||
</symbol>
|
||
<symbol id="svg-eye" viewBox="0 0 24 24">
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
|
||
stroke-width="1" stroke-linecap="round" stroke-linejoin="round" class="icon-tabler-eye-code">
|
||
<path stroke="none" d="M0 0h24v24H0z" fill="none" />
|
||
<path d="M10 12a2 2 0 1 0 4 0a2 2 0 0 0 -4 0" />
|
||
<path
|
||
d="M11.11 17.958c-3.209 -.307 -5.91 -2.293 -8.11 -5.958c2.4 -4 5.4 -6 9 -6c3.6 0 6.6 2 9 6c-.21 .352 -.427 .688 -.647 1.008" />
|
||
<path d="M20 21l2 -2l-2 -2" />
|
||
<path d="M17 17l-2 2l2 2" />
|
||
</svg>
|
||
</symbol>
|
||
</svg>
|
||
|
||
<input type="checkbox" class="sidebar-toggle" name="__navigation" id="__navigation" aria-label="Toggle site navigation sidebar">
|
||
<input type="checkbox" class="sidebar-toggle" name="__toc" id="__toc" aria-label="Toggle table of contents sidebar">
|
||
<label class="overlay sidebar-overlay" for="__navigation"></label>
|
||
<label class="overlay toc-overlay" for="__toc"></label>
|
||
|
||
<a class="skip-to-content muted-link" href="#furo-main-content">Skip to content</a>
|
||
|
||
|
||
|
||
<div class="page">
|
||
<header class="mobile-header">
|
||
<div class="header-left">
|
||
<label class="nav-overlay-icon" for="__navigation">
|
||
<span class="icon"><svg><use href="#svg-menu"></use></svg></span>
|
||
</label>
|
||
</div>
|
||
<div class="header-center">
|
||
<a href="../index.html"><div class="brand">Aborist API Reference</div></a>
|
||
</div>
|
||
<div class="header-right">
|
||
<div class="theme-toggle-container theme-toggle-header">
|
||
<button class="theme-toggle" aria-label="Toggle Light / Dark / Auto color theme">
|
||
<svg class="theme-icon-when-auto-light"><use href="#svg-sun-with-moon"></use></svg>
|
||
<svg class="theme-icon-when-auto-dark"><use href="#svg-moon-with-sun"></use></svg>
|
||
<svg class="theme-icon-when-dark"><use href="#svg-moon"></use></svg>
|
||
<svg class="theme-icon-when-light"><use href="#svg-sun"></use></svg>
|
||
</button>
|
||
</div>
|
||
<label class="toc-overlay-icon toc-header-icon" for="__toc">
|
||
<span class="icon"><svg><use href="#svg-toc"></use></svg></span>
|
||
</label>
|
||
</div>
|
||
</header>
|
||
<aside class="sidebar-drawer">
|
||
<div class="sidebar-container">
|
||
|
||
<div class="sidebar-sticky"><div class="sidebar-tree">
|
||
<p class="caption" role="heading"><span class="caption-text">API Modules</span></p>
|
||
<ul class="current">
|
||
<li class="toctree-l1"><a class="reference internal" href="substrate.html">Substrate: Core data structures</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="storage.html">Storage: SQLite schema and audit</a></li>
|
||
<li class="toctree-l1 current current-page"><a class="current reference internal" href="#">Retrieval: FTS5 search and concepts</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="qa.html">Q&A Pipeline: question → answer → verify → cache</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="distill.html">Distillation: surface → core compression</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="mesh.html">Federation: multiplayer aborist</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="cli.html">CLI: Command-line interface</a></li>
|
||
</ul>
|
||
|
||
</div>
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</aside>
|
||
<div class="main">
|
||
<div class="content">
|
||
<div class="article-container">
|
||
<a href="#" class="back-to-top muted-link">
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24">
|
||
<path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8v12z"></path>
|
||
</svg>
|
||
<span>Back to top</span>
|
||
</a>
|
||
<div class="content-icon-container">
|
||
<div class="view-this-page">
|
||
<a class="muted-link" href="../_sources/api/retrieval.rst.txt" title="View this page">
|
||
<svg><use href="#svg-eye"></use></svg>
|
||
<span class="visually-hidden">View this page</span>
|
||
</a>
|
||
</div>
|
||
<div class="theme-toggle-container theme-toggle-content">
|
||
<button class="theme-toggle" aria-label="Toggle Light / Dark / Auto color theme">
|
||
<svg class="theme-icon-when-auto-light"><use href="#svg-sun-with-moon"></use></svg>
|
||
<svg class="theme-icon-when-auto-dark"><use href="#svg-moon-with-sun"></use></svg>
|
||
<svg class="theme-icon-when-dark"><use href="#svg-moon"></use></svg>
|
||
<svg class="theme-icon-when-light"><use href="#svg-sun"></use></svg>
|
||
</button>
|
||
</div>
|
||
<label class="toc-overlay-icon toc-content-icon" for="__toc">
|
||
<span class="icon"><svg><use href="#svg-toc"></use></svg></span>
|
||
</label>
|
||
</div>
|
||
<article role="main" id="furo-main-content">
|
||
<section id="retrieval-fts5-search-and-concepts">
|
||
<h1>Retrieval: FTS5 search and concepts<a class="headerlink" href="#retrieval-fts5-search-and-concepts" title="Link to this heading">¶</a></h1>
|
||
<p>Full-text search, concept relations (synonym/rivalry overlay).</p>
|
||
<section id="module-aborist.search">
|
||
<span id="search"></span><h2>search<a class="headerlink" href="#module-aborist.search" title="Link to this heading">¶</a></h2>
|
||
<p>Search backends.</p>
|
||
<dl class="py class">
|
||
<dt class="sig sig-object py" id="aborist.search.AuditMode">
|
||
<span class="property"><span class="k"><span class="pre">class</span></span><span class="w"> </span></span><span class="sig-prename descclassname"><span class="pre">aborist.search.</span></span><span class="sig-name descname"><span class="pre">AuditMode</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="o"><span class="pre">*</span></span><span class="n"><span class="pre">values</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/aborist/search/base.html#AuditMode"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#aborist.search.AuditMode" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Bases: <a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><code class="xref py py-class docutils literal notranslate"><span class="pre">str</span></code></a>, <a class="reference external" href="https://docs.python.org/3/library/enum.html#enum.Enum" title="(in Python v3.14)"><code class="xref py py-class docutils literal notranslate"><span class="pre">Enum</span></code></a></p>
|
||
<dl class="py attribute">
|
||
<dt class="sig sig-object py" id="aborist.search.AuditMode.STRICT">
|
||
<span class="sig-name descname"><span class="pre">STRICT</span></span><span class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">'STRICT'</span></span><a class="headerlink" href="#aborist.search.AuditMode.STRICT" title="Link to this definition">¶</a></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="py attribute">
|
||
<dt class="sig sig-object py" id="aborist.search.AuditMode.HYBRID">
|
||
<span class="sig-name descname"><span class="pre">HYBRID</span></span><span class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">'HYBRID'</span></span><a class="headerlink" href="#aborist.search.AuditMode.HYBRID" title="Link to this definition">¶</a></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="py attribute">
|
||
<dt class="sig sig-object py" id="aborist.search.AuditMode.UNGROUNDED">
|
||
<span class="sig-name descname"><span class="pre">UNGROUNDED</span></span><span class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">'UNGROUNDED'</span></span><a class="headerlink" href="#aborist.search.AuditMode.UNGROUNDED" title="Link to this definition">¶</a></dt>
|
||
<dd></dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
<dl class="py class">
|
||
<dt class="sig sig-object py" id="aborist.search.Hit">
|
||
<span class="property"><span class="k"><span class="pre">class</span></span><span class="w"> </span></span><span class="sig-prename descclassname"><span class="pre">aborist.search.</span></span><span class="sig-name descname"><span class="pre">Hit</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">document_root</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="s"><span class="pre">'str'</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">document_uri</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="s"><span class="pre">'str'</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">chunk_idx</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="s"><span class="pre">'int'</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">snippet</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="s"><span class="pre">'str'</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">score</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="s"><span class="pre">'float'</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">audit_mode</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="s"><span class="pre">'AuditMode'</span></span></span></em>, <em class="sig-param"><span class="n"><span class="pre">title</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="s"><span class="pre">'str</span> <span class="pre">|</span> <span class="pre">None'</span></span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/aborist/search/base.html#Hit"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#aborist.search.Hit" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Bases: <a class="reference external" href="https://docs.python.org/3/library/functions.html#object" title="(in Python v3.14)"><code class="xref py py-class docutils literal notranslate"><span class="pre">object</span></code></a></p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><ul class="simple">
|
||
<li><p><strong>document_root</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><em>str</em></a>)</p></li>
|
||
<li><p><strong>document_uri</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><em>str</em></a>)</p></li>
|
||
<li><p><strong>chunk_idx</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.14)"><em>int</em></a>)</p></li>
|
||
<li><p><strong>snippet</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><em>str</em></a>)</p></li>
|
||
<li><p><strong>score</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.14)"><em>float</em></a>)</p></li>
|
||
<li><p><strong>audit_mode</strong> (<a class="reference internal" href="#aborist.search.AuditMode" title="aborist.search.base.AuditMode"><em>AuditMode</em></a>)</p></li>
|
||
<li><p><strong>title</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><em>str</em></a><em> | </em><em>None</em>)</p></li>
|
||
</ul>
|
||
</dd>
|
||
</dl>
|
||
<dl class="py attribute">
|
||
<dt class="sig sig-object py" id="aborist.search.Hit.document_root">
|
||
<span class="sig-name descname"><span class="pre">document_root</span></span><span class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><span class="pre">str</span></a></span><a class="headerlink" href="#aborist.search.Hit.document_root" title="Link to this definition">¶</a></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="py attribute">
|
||
<dt class="sig sig-object py" id="aborist.search.Hit.document_uri">
|
||
<span class="sig-name descname"><span class="pre">document_uri</span></span><span class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><span class="pre">str</span></a></span><a class="headerlink" href="#aborist.search.Hit.document_uri" title="Link to this definition">¶</a></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="py attribute">
|
||
<dt class="sig sig-object py" id="aborist.search.Hit.chunk_idx">
|
||
<span class="sig-name descname"><span class="pre">chunk_idx</span></span><span class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.14)"><span class="pre">int</span></a></span><a class="headerlink" href="#aborist.search.Hit.chunk_idx" title="Link to this definition">¶</a></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="py attribute">
|
||
<dt class="sig sig-object py" id="aborist.search.Hit.snippet">
|
||
<span class="sig-name descname"><span class="pre">snippet</span></span><span class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><span class="pre">str</span></a></span><a class="headerlink" href="#aborist.search.Hit.snippet" title="Link to this definition">¶</a></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="py attribute">
|
||
<dt class="sig sig-object py" id="aborist.search.Hit.score">
|
||
<span class="sig-name descname"><span class="pre">score</span></span><span class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.14)"><span class="pre">float</span></a></span><a class="headerlink" href="#aborist.search.Hit.score" title="Link to this definition">¶</a></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="py attribute">
|
||
<dt class="sig sig-object py" id="aborist.search.Hit.audit_mode">
|
||
<span class="sig-name descname"><span class="pre">audit_mode</span></span><span class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><a class="reference internal" href="#aborist.search.AuditMode" title="aborist.search.base.AuditMode"><span class="pre">AuditMode</span></a></span><a class="headerlink" href="#aborist.search.Hit.audit_mode" title="Link to this definition">¶</a></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="py attribute">
|
||
<dt class="sig sig-object py" id="aborist.search.Hit.title">
|
||
<span class="sig-name descname"><span class="pre">title</span></span><span class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><span class="pre">str</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><a class="reference external" href="https://docs.python.org/3/library/constants.html#None" title="(in Python v3.14)"><span class="pre">None</span></a></span><span class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">None</span></span><a class="headerlink" href="#aborist.search.Hit.title" title="Link to this definition">¶</a></dt>
|
||
<dd></dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
<dl class="py class">
|
||
<dt class="sig sig-object py" id="aborist.search.SearchBackend">
|
||
<span class="property"><span class="k"><span class="pre">class</span></span><span class="w"> </span></span><span class="sig-prename descclassname"><span class="pre">aborist.search.</span></span><span class="sig-name descname"><span class="pre">SearchBackend</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">conn</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/aborist/search/base.html#SearchBackend"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#aborist.search.SearchBackend" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Bases: <a class="reference external" href="https://docs.python.org/3/library/abc.html#abc.ABC" title="(in Python v3.14)"><code class="xref py py-class docutils literal notranslate"><span class="pre">ABC</span></code></a></p>
|
||
<p>A search hook over the chunk store.</p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><p><strong>conn</strong> (<a class="reference external" href="https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection" title="(in Python v3.14)"><em>sqlite3.Connection</em></a>)</p>
|
||
</dd>
|
||
</dl>
|
||
<dl class="py attribute">
|
||
<dt class="sig sig-object py" id="aborist.search.SearchBackend.name">
|
||
<span class="sig-name descname"><span class="pre">name</span></span><span class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><span class="pre">str</span></a></span><a class="headerlink" href="#aborist.search.SearchBackend.name" title="Link to this definition">¶</a></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="py attribute">
|
||
<dt class="sig sig-object py" id="aborist.search.SearchBackend.audit_mode">
|
||
<span class="sig-name descname"><span class="pre">audit_mode</span></span><span class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><a class="reference internal" href="#aborist.search.AuditMode" title="aborist.search.base.AuditMode"><span class="pre">AuditMode</span></a></span><a class="headerlink" href="#aborist.search.SearchBackend.audit_mode" title="Link to this definition">¶</a></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="py method">
|
||
<dt class="sig sig-object py" id="aborist.search.SearchBackend.search">
|
||
<span class="property"><span class="k"><span class="pre">abstractmethod</span></span><span class="w"> </span></span><span class="sig-name descname"><span class="pre">search</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">query</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">limit</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">20</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/aborist/search/base.html#SearchBackend.search"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#aborist.search.SearchBackend.search" title="Link to this definition">¶</a></dt>
|
||
<dd><dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><ul class="simple">
|
||
<li><p><strong>query</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><em>str</em></a>)</p></li>
|
||
<li><p><strong>limit</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.14)"><em>int</em></a>)</p></li>
|
||
</ul>
|
||
</dd>
|
||
<dt class="field-even">Return type<span class="colon">:</span></dt>
|
||
<dd class="field-even"><p><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#list" title="(in Python v3.14)">list</a>[<a class="reference internal" href="#aborist.search.Hit" title="aborist.search.base.Hit"><em>Hit</em></a>]</p>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
<dl class="py class">
|
||
<dt class="sig sig-object py" id="aborist.search.FTS5Backend">
|
||
<span class="property"><span class="k"><span class="pre">class</span></span><span class="w"> </span></span><span class="sig-prename descclassname"><span class="pre">aborist.search.</span></span><span class="sig-name descname"><span class="pre">FTS5Backend</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">conn</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/aborist/search/fts5.html#FTS5Backend"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#aborist.search.FTS5Backend" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Bases: <a class="reference internal" href="#aborist.search.SearchBackend" title="aborist.search.base.SearchBackend"><code class="xref py py-class docutils literal notranslate"><span class="pre">SearchBackend</span></code></a></p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><p><strong>conn</strong> (<a class="reference external" href="https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection" title="(in Python v3.14)"><em>sqlite3.Connection</em></a>)</p>
|
||
</dd>
|
||
</dl>
|
||
<dl class="py attribute">
|
||
<dt class="sig sig-object py" id="aborist.search.FTS5Backend.name">
|
||
<span class="sig-name descname"><span class="pre">name</span></span><span class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><span class="pre">str</span></a></span><span class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">'fts5'</span></span><a class="headerlink" href="#aborist.search.FTS5Backend.name" title="Link to this definition">¶</a></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="py attribute">
|
||
<dt class="sig sig-object py" id="aborist.search.FTS5Backend.audit_mode">
|
||
<span class="sig-name descname"><span class="pre">audit_mode</span></span><span class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><a class="reference internal" href="#aborist.search.AuditMode" title="aborist.search.base.AuditMode"><span class="pre">AuditMode</span></a></span><span class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">'UNGROUNDED'</span></span><a class="headerlink" href="#aborist.search.FTS5Backend.audit_mode" title="Link to this definition">¶</a></dt>
|
||
<dd></dd></dl>
|
||
|
||
<dl class="py method">
|
||
<dt class="sig sig-object py" id="aborist.search.FTS5Backend.search">
|
||
<span class="sig-name descname"><span class="pre">search</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">query</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">limit</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">20</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">extra_or_tokens</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/aborist/search/fts5.html#FTS5Backend.search"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#aborist.search.FTS5Backend.search" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Run FTS5 BM25 over chunk content.</p>
|
||
<p><code class="docutils literal notranslate"><span class="pre">extra_or_tokens</span></code> (synonym-expanded set) is passed through to
|
||
the OR-mode fallback only. AND mode stays on original query
|
||
tokens (adding synonyms there would relax the AND constraint
|
||
and pull in noise). The intended caller is the retrieval
|
||
pipeline that has already computed <code class="docutils literal notranslate"><span class="pre">synonym_expand(qtokens)</span></code>
|
||
— passing it here saves the OR-mode pool from missing topical
|
||
synonym terms.</p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><ul class="simple">
|
||
<li><p><strong>query</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><em>str</em></a>)</p></li>
|
||
<li><p><strong>limit</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.14)"><em>int</em></a>)</p></li>
|
||
<li><p><strong>extra_or_tokens</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#set" title="(in Python v3.14)"><em>set</em></a><em>[</em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><em>str</em></a><em>] </em><em>| </em><em>None</em>)</p></li>
|
||
</ul>
|
||
</dd>
|
||
<dt class="field-even">Return type<span class="colon">:</span></dt>
|
||
<dd class="field-even"><p><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#list" title="(in Python v3.14)">list</a>[<a class="reference internal" href="#aborist.search.Hit" title="aborist.search.base.Hit"><em>Hit</em></a>]</p>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
</section>
|
||
<section id="module-aborist.source">
|
||
<span id="sources"></span><h2>sources<a class="headerlink" href="#module-aborist.source" title="Link to this heading">¶</a></h2>
|
||
<p>Source ABC.</p>
|
||
<p>Adding a new corpus to aborist = one new Source subclass. The Source contract
|
||
is intentionally minimal: yield Document objects, one at a time.</p>
|
||
<dl class="py class">
|
||
<dt class="sig sig-object py" id="aborist.source.Source">
|
||
<span class="property"><span class="k"><span class="pre">class</span></span><span class="w"> </span></span><span class="sig-prename descclassname"><span class="pre">aborist.source.</span></span><span class="sig-name descname"><span class="pre">Source</span></span><a class="reference internal" href="../_modules/aborist/source.html#Source"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#aborist.source.Source" title="Link to this definition">¶</a></dt>
|
||
<dd><p>A corpus that yields documents into the ingest pipeline.</p>
|
||
<dl class="py attribute">
|
||
<dt class="sig sig-object py" id="aborist.source.Source.source_type">
|
||
<span class="sig-name descname"><span class="pre">source_type</span></span><span class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><span class="pre">str</span></a></span><a class="headerlink" href="#aborist.source.Source.source_type" title="Link to this definition">¶</a></dt>
|
||
<dd><p>source_type tag stored on every Document this source produces.</p>
|
||
</dd></dl>
|
||
|
||
<dl class="py method">
|
||
<dt class="sig sig-object py" id="aborist.source.Source.iter_documents">
|
||
<span class="property"><span class="k"><span class="pre">abstractmethod</span></span><span class="w"> </span></span><span class="sig-name descname"><span class="pre">iter_documents</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="../_modules/aborist/source.html#Source.iter_documents"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#aborist.source.Source.iter_documents" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Yield Document objects. Must be deterministic & idempotent.</p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Return type<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><p><a class="reference external" href="https://docs.python.org/3/library/typing.html#typing.Iterator" title="(in Python v3.14)"><em>Iterator</em></a>[<a class="reference internal" href="substrate.html#aborist.document.Document" title="aborist.document.Document"><em>Document</em></a>]</p>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
<p id="module-aborist.sources.wikipedia">MediaWiki ‘cur’ table SQL dump source.</p>
|
||
<p>Handles 2003-era SQL dumps in bz2 format (e.g. 20030516_cur_tablesql.bz2).
|
||
Yields one Document per non-redirect main-namespace article, with
|
||
[[wikilinks]] extracted as outbound edges.</p>
|
||
<p>Implements a stream parser for MySQL extended INSERT syntax. The cur table
|
||
schema for that era starts: cur_id, cur_namespace, cur_title, cur_text, …
|
||
We rely on positional access for the first four columns.</p>
|
||
<dl class="py class">
|
||
<dt class="sig sig-object py" id="aborist.sources.wikipedia.WikipediaSqlDump">
|
||
<span class="property"><span class="k"><span class="pre">class</span></span><span class="w"> </span></span><span class="sig-prename descclassname"><span class="pre">aborist.sources.wikipedia.</span></span><span class="sig-name descname"><span class="pre">WikipediaSqlDump</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path</span></span></em>, <em class="sig-param"><span class="keyword-only-separator o"><abbr title="Keyword-only parameters separator (PEP 3102)"><span class="pre">*</span></abbr></span></em>, <em class="sig-param"><span class="n"><span class="pre">table</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'cur'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">namespace</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">0</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">base_uri</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'https://en.wikipedia.org/wiki/'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">shard</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">start_id</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">0</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">encoding</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'latin-1'</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/aborist/sources/wikipedia.html#WikipediaSqlDump"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#aborist.sources.wikipedia.WikipediaSqlDump" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Iterates a MediaWiki SQL table dump (cur or old), bz2 or plain.</p>
|
||
<p>Both <cite>cur</cite> (current snapshot) and <cite>old</cite> (revision history) tables share
|
||
the first four column positions: id, namespace, title, text. The <cite>cur</cite>
|
||
table has <cite>cur_is_redirect</cite> at position 10 (we skip redirects); <cite>old</cite>
|
||
has no redirect flag (every revision is real).</p>
|
||
<p>Shard support: pass <cite>shard=(rank, total)</cite> and the source yields only
|
||
docs whose 0-based index satisfies <cite>index % total == rank</cite>. Useful for
|
||
spawning N parallel ingest processes against the same dump file —
|
||
parser CPU runs in parallel, writes serialize at the WAL writer lock.</p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><ul class="simple">
|
||
<li><p><strong>path</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><em>str</em></a><em> | </em><em>Path</em>)</p></li>
|
||
<li><p><strong>table</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><em>str</em></a>)</p></li>
|
||
<li><p><strong>namespace</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.14)"><em>int</em></a>)</p></li>
|
||
<li><p><strong>base_uri</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><em>str</em></a>)</p></li>
|
||
<li><p><strong>shard</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#tuple" title="(in Python v3.14)"><em>tuple</em></a><em>[</em><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.14)"><em>int</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.14)"><em>int</em></a><em>] </em><em>| </em><em>None</em>)</p></li>
|
||
<li><p><strong>start_id</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.14)"><em>int</em></a>)</p></li>
|
||
<li><p><strong>encoding</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><em>str</em></a>)</p></li>
|
||
</ul>
|
||
</dd>
|
||
</dl>
|
||
<dl class="py method">
|
||
<dt class="sig sig-object py" id="aborist.sources.wikipedia.WikipediaSqlDump.iter_documents">
|
||
<span class="sig-name descname"><span class="pre">iter_documents</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="../_modules/aborist/sources/wikipedia.html#WikipediaSqlDump.iter_documents"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#aborist.sources.wikipedia.WikipediaSqlDump.iter_documents" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Yield Document objects. Must be deterministic & idempotent.</p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Return type<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><p><a class="reference external" href="https://docs.python.org/3/library/typing.html#typing.Iterator" title="(in Python v3.14)"><em>Iterator</em></a>[<a class="reference internal" href="substrate.html#aborist.document.Document" title="aborist.document.Document"><em>Document</em></a>]</p>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
<dl class="py class">
|
||
<dt class="sig sig-object py" id="aborist.sources.wikipedia.WikipediaCurDump">
|
||
<span class="property"><span class="k"><span class="pre">class</span></span><span class="w"> </span></span><span class="sig-prename descclassname"><span class="pre">aborist.sources.wikipedia.</span></span><span class="sig-name descname"><span class="pre">WikipediaCurDump</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">namespace</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">0</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">base_uri</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'https://en.wikipedia.org/wiki/'</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/aborist/sources/wikipedia.html#WikipediaCurDump"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#aborist.sources.wikipedia.WikipediaCurDump" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Iterates a MediaWiki ‘cur’ table SQL dump.</p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><ul class="simple">
|
||
<li><p><strong>path</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><em>str</em></a><em> | </em><em>Path</em>)</p></li>
|
||
<li><p><strong>namespace</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.14)"><em>int</em></a>)</p></li>
|
||
<li><p><strong>base_uri</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><em>str</em></a>)</p></li>
|
||
</ul>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="py class">
|
||
<dt class="sig sig-object py" id="aborist.sources.wikipedia.WikipediaOldDump">
|
||
<span class="property"><span class="k"><span class="pre">class</span></span><span class="w"> </span></span><span class="sig-prename descclassname"><span class="pre">aborist.sources.wikipedia.</span></span><span class="sig-name descname"><span class="pre">WikipediaOldDump</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">namespace</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">0</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">base_uri</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'https://en.wikipedia.org/wiki/'</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/aborist/sources/wikipedia.html#WikipediaOldDump"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#aborist.sources.wikipedia.WikipediaOldDump" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Iterates a MediaWiki ‘old’ (revision history) table SQL dump.</p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><ul class="simple">
|
||
<li><p><strong>path</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><em>str</em></a><em> | </em><em>Path</em>)</p></li>
|
||
<li><p><strong>namespace</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.14)"><em>int</em></a>)</p></li>
|
||
<li><p><strong>base_uri</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><em>str</em></a>)</p></li>
|
||
</ul>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<p id="module-aborist.sources.html_page">HTML page source.</p>
|
||
<p>Fetches URLs, honors robots.txt automatically, strips noise (script/style/nav/
|
||
footer/header), extracts main body text + outbound <cite><a href></cite> links as edges.</p>
|
||
<p>Optional dependency. Install with <cite>pip install aborist[html]</cite>.</p>
|
||
<dl class="py function">
|
||
<dt class="sig sig-object py" id="aborist.sources.html_page.parse_html">
|
||
<span class="sig-prename descclassname"><span class="pre">aborist.sources.html_page.</span></span><span class="sig-name descname"><span class="pre">parse_html</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">url</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">html</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">source_type</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">'html'</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/aborist/sources/html_page.html#parse_html"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#aborist.sources.html_page.parse_html" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Pure parse function. Separated so tests can run without network.</p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><ul class="simple">
|
||
<li><p><strong>url</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><em>str</em></a>)</p></li>
|
||
<li><p><strong>html</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><em>str</em></a>)</p></li>
|
||
<li><p><strong>source_type</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><em>str</em></a>)</p></li>
|
||
</ul>
|
||
</dd>
|
||
<dt class="field-even">Return type<span class="colon">:</span></dt>
|
||
<dd class="field-even"><p><a class="reference internal" href="substrate.html#aborist.document.Document" title="aborist.document.Document"><em>Document</em></a> | None</p>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="py class">
|
||
<dt class="sig sig-object py" id="aborist.sources.html_page.HtmlPageSource">
|
||
<span class="property"><span class="k"><span class="pre">class</span></span><span class="w"> </span></span><span class="sig-prename descclassname"><span class="pre">aborist.sources.html_page.</span></span><span class="sig-name descname"><span class="pre">HtmlPageSource</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">urls</span></span></em>, <em class="sig-param"><span class="keyword-only-separator o"><abbr title="Keyword-only parameters separator (PEP 3102)"><span class="pre">*</span></abbr></span></em>, <em class="sig-param"><span class="n"><span class="pre">respect_robots</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">True</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">timeout</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">30.0</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/aborist/sources/html_page.html#HtmlPageSource"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#aborist.sources.html_page.HtmlPageSource" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Iterates a list of URLs, fetching and parsing each as HTML.</p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><ul class="simple">
|
||
<li><p><strong>urls</strong> (<em>Iterable</em><em>[</em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><em>str</em></a><em>]</em>)</p></li>
|
||
<li><p><strong>respect_robots</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.14)"><em>bool</em></a>)</p></li>
|
||
<li><p><strong>timeout</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.14)"><em>float</em></a>)</p></li>
|
||
</ul>
|
||
</dd>
|
||
</dl>
|
||
<dl class="py attribute">
|
||
<dt class="sig sig-object py" id="aborist.sources.html_page.HtmlPageSource.source_type">
|
||
<span class="sig-name descname"><span class="pre">source_type</span></span><span class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><span class="pre">str</span></a></span><span class="property"><span class="w"> </span><span class="p"><span class="pre">=</span></span><span class="w"> </span><span class="pre">'html'</span></span><a class="headerlink" href="#aborist.sources.html_page.HtmlPageSource.source_type" title="Link to this definition">¶</a></dt>
|
||
<dd><p>source_type tag stored on every Document this source produces.</p>
|
||
</dd></dl>
|
||
|
||
<dl class="py method">
|
||
<dt class="sig sig-object py" id="aborist.sources.html_page.HtmlPageSource.from_file">
|
||
<span class="property"><span class="k"><span class="pre">classmethod</span></span><span class="w"> </span></span><span class="sig-name descname"><span class="pre">from_file</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">path</span></span></em>, <em class="sig-param"><span class="o"><span class="pre">**</span></span><span class="n"><span class="pre">kwargs</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/aborist/sources/html_page.html#HtmlPageSource.from_file"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#aborist.sources.html_page.HtmlPageSource.from_file" title="Link to this definition">¶</a></dt>
|
||
<dd><dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><p><strong>path</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><em>str</em></a><em> | </em><a class="reference external" href="https://docs.python.org/3/library/pathlib.html#pathlib.Path" title="(in Python v3.14)"><em>Path</em></a>)</p>
|
||
</dd>
|
||
<dt class="field-even">Return type<span class="colon">:</span></dt>
|
||
<dd class="field-even"><p><a class="reference internal" href="#aborist.sources.html_page.HtmlPageSource" title="aborist.sources.html_page.HtmlPageSource"><em>HtmlPageSource</em></a></p>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="py method">
|
||
<dt class="sig sig-object py" id="aborist.sources.html_page.HtmlPageSource.iter_documents">
|
||
<span class="sig-name descname"><span class="pre">iter_documents</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="../_modules/aborist/sources/html_page.html#HtmlPageSource.iter_documents"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#aborist.sources.html_page.HtmlPageSource.iter_documents" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Yield Document objects. Must be deterministic & idempotent.</p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Return type<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><p><a class="reference external" href="https://docs.python.org/3/library/typing.html#typing.Iterator" title="(in Python v3.14)"><em>Iterator</em></a>[<a class="reference internal" href="substrate.html#aborist.document.Document" title="aborist.document.Document"><em>Document</em></a>]</p>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
</dd></dl>
|
||
|
||
</section>
|
||
<section id="module-aborist.concepts">
|
||
<span id="concepts"></span><h2>concepts<a class="headerlink" href="#module-aborist.concepts" title="Link to this heading">¶</a></h2>
|
||
<p>Corpus-derived concept relations: synonyms, antonyms, rivalries, categories.</p>
|
||
<p>Replaces the hand-curated frozensets that lived in <code class="docutils literal notranslate"><span class="pre">aborist.qa.concepts</span></code>
|
||
through April 2026 (commit c6182ae). The frozensets were Phase 1; this is
|
||
Phase 2.</p>
|
||
<p>The concept-relations layer is a <strong>secondary index</strong> over the existing
|
||
Merkle-committed corpus. Writes to <code class="docutils literal notranslate"><span class="pre">concept_relations</span></code> NEVER affect
|
||
<code class="docutils literal notranslate"><span class="pre">document_root</span></code>, <code class="docutils literal notranslate"><span class="pre">chunk_root</span></code>, or <code class="docutils literal notranslate"><span class="pre">cache_key</span></code> — backfilling
|
||
relations is safe across the entire corpus without invalidating cached
|
||
answers or breaking audit chains.</p>
|
||
<p>Architecture:</p>
|
||
<ul class="simple">
|
||
<li><p><code class="docutils literal notranslate"><span class="pre">store.py</span></code> — DB read/write helpers, append-only with UNIQUE-key idempotency</p></li>
|
||
<li><p><code class="docutils literal notranslate"><span class="pre">extract.py</span></code> — Extractor ABC + registry; per-source extractors plug in here</p></li>
|
||
<li><p><code class="docutils literal notranslate"><span class="pre">query.py</span></code> — Cross-shard <code class="docutils literal notranslate"><span class="pre">synonyms_for(token)</span></code> & <code class="docutils literal notranslate"><span class="pre">rivalries_for(token)</span></code></p></li>
|
||
<li><p><code class="docutils literal notranslate"><span class="pre">seed.py</span></code> — One-time migration of the legacy frozensets to manual rows</p></li>
|
||
</ul>
|
||
<p>Public API for retrieval-time use (matches the legacy
|
||
<code class="docutils literal notranslate"><span class="pre">aborist.qa.concepts</span></code> shape, so call sites in <code class="docutils literal notranslate"><span class="pre">query.py</span></code> keep working):</p>
|
||
<blockquote>
|
||
<div><p>synonym_expand(tokens, <a href="#id1"><span class="problematic" id="id2">*</span></a>, shards_dir) -> set[str]
|
||
rivalry_excluded(tokens, <a href="#id3"><span class="problematic" id="id4">*</span></a>, shards_dir, compare_phrasing=False) -> set[str]
|
||
has_compare_phrasing(question) -> bool</p>
|
||
</div></blockquote>
|
||
<dl class="py function">
|
||
<dt class="sig sig-object py" id="aborist.concepts.add_concept_relation">
|
||
<span class="sig-prename descclassname"><span class="pre">aborist.concepts.</span></span><span class="sig-name descname"><span class="pre">add_concept_relation</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">conn</span></span></em>, <em class="sig-param"><span class="keyword-only-separator o"><abbr title="Keyword-only parameters separator (PEP 3102)"><span class="pre">*</span></abbr></span></em>, <em class="sig-param"><span class="n"><span class="pre">source_root</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">relation_kind</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">token</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">target</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">evidence_kind</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">confidence</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">1.0</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">derived_from</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">derived_at</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/aborist/concepts/store.html#add_concept_relation"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#aborist.concepts.add_concept_relation" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Append a concept relation. Returns True if a row was inserted,
|
||
False if the (source_root, relation_kind, token, target, evidence_kind)
|
||
tuple already existed (idempotent re-derivation).</p>
|
||
<p>Tokens are stored exactly as given — case preservation lets the
|
||
query layer decide normalization. Substring lookup at retrieval
|
||
time is case-insensitive via SQLite’s NOCASE comparator.</p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><ul class="simple">
|
||
<li><p><strong>conn</strong> (<a class="reference external" href="https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection" title="(in Python v3.14)"><em>Connection</em></a>)</p></li>
|
||
<li><p><strong>source_root</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><em>str</em></a>)</p></li>
|
||
<li><p><strong>relation_kind</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><em>str</em></a>)</p></li>
|
||
<li><p><strong>token</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><em>str</em></a>)</p></li>
|
||
<li><p><strong>target</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><em>str</em></a>)</p></li>
|
||
<li><p><strong>evidence_kind</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><em>str</em></a>)</p></li>
|
||
<li><p><strong>confidence</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.14)"><em>float</em></a>)</p></li>
|
||
<li><p><strong>derived_from</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><em>str</em></a><em> | </em><em>None</em>)</p></li>
|
||
<li><p><strong>derived_at</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.14)"><em>int</em></a><em> | </em><em>None</em>)</p></li>
|
||
</ul>
|
||
</dd>
|
||
<dt class="field-even">Return type<span class="colon">:</span></dt>
|
||
<dd class="field-even"><p><a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.14)">bool</a></p>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="py function">
|
||
<dt class="sig sig-object py" id="aborist.concepts.concept_relations_for_token">
|
||
<span class="sig-prename descclassname"><span class="pre">aborist.concepts.</span></span><span class="sig-name descname"><span class="pre">concept_relations_for_token</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">conn</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">token</span></span></em>, <em class="sig-param"><span class="keyword-only-separator o"><abbr title="Keyword-only parameters separator (PEP 3102)"><span class="pre">*</span></abbr></span></em>, <em class="sig-param"><span class="n"><span class="pre">relation_kind</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/aborist/concepts/store.html#concept_relations_for_token"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#aborist.concepts.concept_relations_for_token" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Return all relations whose <code class="docutils literal notranslate"><span class="pre">token</span></code> matches (case-insensitive).
|
||
If <code class="docutils literal notranslate"><span class="pre">relation_kind</span></code> is given, filter to that kind.</p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><ul class="simple">
|
||
<li><p><strong>conn</strong> (<a class="reference external" href="https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection" title="(in Python v3.14)"><em>Connection</em></a>)</p></li>
|
||
<li><p><strong>token</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><em>str</em></a>)</p></li>
|
||
<li><p><strong>relation_kind</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><em>str</em></a><em> | </em><em>None</em>)</p></li>
|
||
</ul>
|
||
</dd>
|
||
<dt class="field-even">Return type<span class="colon">:</span></dt>
|
||
<dd class="field-even"><p><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#list" title="(in Python v3.14)">list</a>[<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#dict" title="(in Python v3.14)">dict</a>]</p>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="py function">
|
||
<dt class="sig sig-object py" id="aborist.concepts.has_compare_phrasing">
|
||
<span class="sig-prename descclassname"><span class="pre">aborist.concepts.</span></span><span class="sig-name descname"><span class="pre">has_compare_phrasing</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">question</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/aborist/concepts/query.html#has_compare_phrasing"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#aborist.concepts.has_compare_phrasing" title="Link to this definition">¶</a></dt>
|
||
<dd><p>True if the question contains comparison language.</p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><p><strong>question</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><em>str</em></a>)</p>
|
||
</dd>
|
||
<dt class="field-even">Return type<span class="colon">:</span></dt>
|
||
<dd class="field-even"><p><a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.14)">bool</a></p>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="py function">
|
||
<dt class="sig sig-object py" id="aborist.concepts.invalidate_cache">
|
||
<span class="sig-prename descclassname"><span class="pre">aborist.concepts.</span></span><span class="sig-name descname"><span class="pre">invalidate_cache</span></span><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="../_modules/aborist/concepts/query.html#invalidate_cache"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#aborist.concepts.invalidate_cache" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Drop all cached indices. Call after a writer commits new rows
|
||
(the mtime check would catch this on next read, but invalidating
|
||
explicitly is faster on the same-process write+read pattern).</p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Return type<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><p>None</p>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="py function">
|
||
<dt class="sig sig-object py" id="aborist.concepts.purge_by_evidence_kind">
|
||
<span class="sig-prename descclassname"><span class="pre">aborist.concepts.</span></span><span class="sig-name descname"><span class="pre">purge_by_evidence_kind</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">conn</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">evidence_kind</span></span></em>, <em class="sig-param"><span class="keyword-only-separator o"><abbr title="Keyword-only parameters separator (PEP 3102)"><span class="pre">*</span></abbr></span></em>, <em class="sig-param"><span class="n"><span class="pre">derived_from</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/aborist/concepts/store.html#purge_by_evidence_kind"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#aborist.concepts.purge_by_evidence_kind" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Delete every row with the given <code class="docutils literal notranslate"><span class="pre">evidence_kind</span></code> (and optional
|
||
<code class="docutils literal notranslate"><span class="pre">derived_from</span></code>). Returns the number of rows removed.</p>
|
||
<p>The intended use: revoke a buggy extractor’s output cleanly. Manual
|
||
rows live under <code class="docutils literal notranslate"><span class="pre">evidence_kind='manual'</span></code> and are NOT touched by
|
||
a purge of any other kind.</p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><ul class="simple">
|
||
<li><p><strong>conn</strong> (<a class="reference external" href="https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection" title="(in Python v3.14)"><em>Connection</em></a>)</p></li>
|
||
<li><p><strong>evidence_kind</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><em>str</em></a>)</p></li>
|
||
<li><p><strong>derived_from</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><em>str</em></a><em> | </em><em>None</em>)</p></li>
|
||
</ul>
|
||
</dd>
|
||
<dt class="field-even">Return type<span class="colon">:</span></dt>
|
||
<dd class="field-even"><p><a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.14)">int</a></p>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="py function">
|
||
<dt class="sig sig-object py" id="aborist.concepts.rivalry_excluded">
|
||
<span class="sig-prename descclassname"><span class="pre">aborist.concepts.</span></span><span class="sig-name descname"><span class="pre">rivalry_excluded</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">tokens</span></span></em>, <em class="sig-param"><span class="keyword-only-separator o"><abbr title="Keyword-only parameters separator (PEP 3102)"><span class="pre">*</span></abbr></span></em>, <em class="sig-param"><span class="n"><span class="pre">shards_dir</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">compare_phrasing</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">False</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/aborist/concepts/query.html#rivalry_excluded"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#aborist.concepts.rivalry_excluded" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Tokens whose presence in a doc title means EXCLUDE that doc.</p>
|
||
<p>For each rivalry pair (A, B): if exactly ONE side appears in the
|
||
query AND no comparison language was used, exclude the OTHER side’s
|
||
tokens. If both sides appear, or if the user asked for a comparison,
|
||
no exclusion (they wanted both).</p>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><ul class="simple">
|
||
<li><p><strong>tokens</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#set" title="(in Python v3.14)"><em>set</em></a><em>[</em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><em>str</em></a><em>]</em>)</p></li>
|
||
<li><p><strong>shards_dir</strong> (<a class="reference external" href="https://docs.python.org/3/library/pathlib.html#pathlib.Path" title="(in Python v3.14)"><em>Path</em></a><em> | </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><em>str</em></a><em> | </em><em>None</em>)</p></li>
|
||
<li><p><strong>compare_phrasing</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.14)"><em>bool</em></a>)</p></li>
|
||
</ul>
|
||
</dd>
|
||
<dt class="field-even">Return type<span class="colon">:</span></dt>
|
||
<dd class="field-even"><p><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#set" title="(in Python v3.14)">set</a>[<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)">str</a>]</p>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
<dl class="py function">
|
||
<dt class="sig sig-object py" id="aborist.concepts.synonym_expand">
|
||
<span class="sig-prename descclassname"><span class="pre">aborist.concepts.</span></span><span class="sig-name descname"><span class="pre">synonym_expand</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">tokens</span></span></em>, <em class="sig-param"><span class="keyword-only-separator o"><abbr title="Keyword-only parameters separator (PEP 3102)"><span class="pre">*</span></abbr></span></em>, <em class="sig-param"><span class="n"><span class="pre">shards_dir</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">max_neighbors_per_token</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">8</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">max_total</span></span><span class="o"><span class="pre">=</span></span><span class="default_value"><span class="pre">50</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/aborist/concepts/query.html#synonym_expand"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#aborist.concepts.synonym_expand" title="Link to this definition">¶</a></dt>
|
||
<dd><p>Add direct synonym neighbors for any input token whose degree
|
||
is bounded enough that its neighbors are likely topical, not
|
||
topic-adjacency noise.</p>
|
||
<p>Two caps protect retrieval performance & quality:</p>
|
||
<ol class="arabic simple">
|
||
<li><p><code class="docutils literal notranslate"><span class="pre">max_neighbors_per_token</span></code> — tokens with more direct neighbors
|
||
than this contribute NO expansion. Generic tokens (“person”,
|
||
“thoughts”) have huge degree in the Wikipedia reciprocal-link
|
||
graph; expanding them dumps random topical-cluster noise.
|
||
Specific named entities (“athlon”, “telepathy”) have small
|
||
focused neighborhoods that pass the cap.</p></li>
|
||
<li><p><code class="docutils literal notranslate"><span class="pre">max_total</span></code> — overall cap on expanded set size. Bounds the
|
||
SQL clause count downstream. Original query tokens are always
|
||
preserved; if total > cap, neighbors are sorted alphabetically
|
||
& truncated.</p></li>
|
||
</ol>
|
||
<dl class="field-list simple">
|
||
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
|
||
<dd class="field-odd"><ul class="simple">
|
||
<li><p><strong>tokens</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#set" title="(in Python v3.14)"><em>set</em></a><em>[</em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><em>str</em></a><em>]</em>)</p></li>
|
||
<li><p><strong>shards_dir</strong> (<a class="reference external" href="https://docs.python.org/3/library/pathlib.html#pathlib.Path" title="(in Python v3.14)"><em>Path</em></a><em> | </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)"><em>str</em></a><em> | </em><em>None</em>)</p></li>
|
||
<li><p><strong>max_neighbors_per_token</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.14)"><em>int</em></a>)</p></li>
|
||
<li><p><strong>max_total</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.14)"><em>int</em></a>)</p></li>
|
||
</ul>
|
||
</dd>
|
||
<dt class="field-even">Return type<span class="colon">:</span></dt>
|
||
<dd class="field-even"><p><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#set" title="(in Python v3.14)">set</a>[<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.14)">str</a>]</p>
|
||
</dd>
|
||
</dl>
|
||
</dd></dl>
|
||
|
||
</section>
|
||
</section>
|
||
|
||
</article>
|
||
</div>
|
||
<footer>
|
||
|
||
<div class="related-pages">
|
||
<a class="next-page" href="qa.html">
|
||
<div class="page-info">
|
||
<div class="context">
|
||
<span>Next</span>
|
||
</div>
|
||
<div class="title">Q&A Pipeline: question → answer → verify → cache</div>
|
||
</div>
|
||
<svg class="furo-related-icon"><use href="#svg-arrow-right"></use></svg>
|
||
</a>
|
||
<a class="prev-page" href="storage.html">
|
||
<svg class="furo-related-icon"><use href="#svg-arrow-right"></use></svg>
|
||
<div class="page-info">
|
||
<div class="context">
|
||
<span>Previous</span>
|
||
</div>
|
||
|
||
<div class="title">Storage: SQLite schema and audit</div>
|
||
|
||
</div>
|
||
</a>
|
||
</div>
|
||
<div class="bottom-of-page">
|
||
<div class="left-details">
|
||
<div class="copyright">
|
||
Copyright © 2026, agent blackops
|
||
</div>
|
||
Made with <a href="https://www.sphinx-doc.org/">Sphinx</a> and <a class="muted-link" href="https://pradyunsg.me">@pradyunsg</a>'s
|
||
|
||
<a href="https://github.com/pradyunsg/furo">Furo</a>
|
||
|
||
</div>
|
||
<div class="right-details">
|
||
|
||
</div>
|
||
</div>
|
||
|
||
</footer>
|
||
</div>
|
||
<aside class="toc-drawer">
|
||
|
||
|
||
<div class="toc-sticky toc-scroll">
|
||
<div class="toc-title-container">
|
||
<span class="toc-title">
|
||
On this page
|
||
</span>
|
||
</div>
|
||
<div class="toc-tree-container">
|
||
<div class="toc-tree">
|
||
<ul>
|
||
<li><a class="reference internal" href="#">Retrieval: FTS5 search and concepts</a><ul>
|
||
<li><a class="reference internal" href="#module-aborist.search">search</a><ul>
|
||
<li><a class="reference internal" href="#aborist.search.AuditMode"><code class="docutils literal notranslate"><span class="pre">AuditMode</span></code></a><ul>
|
||
<li><a class="reference internal" href="#aborist.search.AuditMode.STRICT"><code class="docutils literal notranslate"><span class="pre">AuditMode.STRICT</span></code></a></li>
|
||
<li><a class="reference internal" href="#aborist.search.AuditMode.HYBRID"><code class="docutils literal notranslate"><span class="pre">AuditMode.HYBRID</span></code></a></li>
|
||
<li><a class="reference internal" href="#aborist.search.AuditMode.UNGROUNDED"><code class="docutils literal notranslate"><span class="pre">AuditMode.UNGROUNDED</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#aborist.search.Hit"><code class="docutils literal notranslate"><span class="pre">Hit</span></code></a><ul>
|
||
<li><a class="reference internal" href="#aborist.search.Hit.document_root"><code class="docutils literal notranslate"><span class="pre">Hit.document_root</span></code></a></li>
|
||
<li><a class="reference internal" href="#aborist.search.Hit.document_uri"><code class="docutils literal notranslate"><span class="pre">Hit.document_uri</span></code></a></li>
|
||
<li><a class="reference internal" href="#aborist.search.Hit.chunk_idx"><code class="docutils literal notranslate"><span class="pre">Hit.chunk_idx</span></code></a></li>
|
||
<li><a class="reference internal" href="#aborist.search.Hit.snippet"><code class="docutils literal notranslate"><span class="pre">Hit.snippet</span></code></a></li>
|
||
<li><a class="reference internal" href="#aborist.search.Hit.score"><code class="docutils literal notranslate"><span class="pre">Hit.score</span></code></a></li>
|
||
<li><a class="reference internal" href="#aborist.search.Hit.audit_mode"><code class="docutils literal notranslate"><span class="pre">Hit.audit_mode</span></code></a></li>
|
||
<li><a class="reference internal" href="#aborist.search.Hit.title"><code class="docutils literal notranslate"><span class="pre">Hit.title</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#aborist.search.SearchBackend"><code class="docutils literal notranslate"><span class="pre">SearchBackend</span></code></a><ul>
|
||
<li><a class="reference internal" href="#aborist.search.SearchBackend.name"><code class="docutils literal notranslate"><span class="pre">SearchBackend.name</span></code></a></li>
|
||
<li><a class="reference internal" href="#aborist.search.SearchBackend.audit_mode"><code class="docutils literal notranslate"><span class="pre">SearchBackend.audit_mode</span></code></a></li>
|
||
<li><a class="reference internal" href="#aborist.search.SearchBackend.search"><code class="docutils literal notranslate"><span class="pre">SearchBackend.search()</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#aborist.search.FTS5Backend"><code class="docutils literal notranslate"><span class="pre">FTS5Backend</span></code></a><ul>
|
||
<li><a class="reference internal" href="#aborist.search.FTS5Backend.name"><code class="docutils literal notranslate"><span class="pre">FTS5Backend.name</span></code></a></li>
|
||
<li><a class="reference internal" href="#aborist.search.FTS5Backend.audit_mode"><code class="docutils literal notranslate"><span class="pre">FTS5Backend.audit_mode</span></code></a></li>
|
||
<li><a class="reference internal" href="#aborist.search.FTS5Backend.search"><code class="docutils literal notranslate"><span class="pre">FTS5Backend.search()</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#module-aborist.source">sources</a><ul>
|
||
<li><a class="reference internal" href="#aborist.source.Source"><code class="docutils literal notranslate"><span class="pre">Source</span></code></a><ul>
|
||
<li><a class="reference internal" href="#aborist.source.Source.source_type"><code class="docutils literal notranslate"><span class="pre">Source.source_type</span></code></a></li>
|
||
<li><a class="reference internal" href="#aborist.source.Source.iter_documents"><code class="docutils literal notranslate"><span class="pre">Source.iter_documents()</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#aborist.sources.wikipedia.WikipediaSqlDump"><code class="docutils literal notranslate"><span class="pre">WikipediaSqlDump</span></code></a><ul>
|
||
<li><a class="reference internal" href="#aborist.sources.wikipedia.WikipediaSqlDump.iter_documents"><code class="docutils literal notranslate"><span class="pre">WikipediaSqlDump.iter_documents()</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#aborist.sources.wikipedia.WikipediaCurDump"><code class="docutils literal notranslate"><span class="pre">WikipediaCurDump</span></code></a></li>
|
||
<li><a class="reference internal" href="#aborist.sources.wikipedia.WikipediaOldDump"><code class="docutils literal notranslate"><span class="pre">WikipediaOldDump</span></code></a></li>
|
||
<li><a class="reference internal" href="#aborist.sources.html_page.parse_html"><code class="docutils literal notranslate"><span class="pre">parse_html()</span></code></a></li>
|
||
<li><a class="reference internal" href="#aborist.sources.html_page.HtmlPageSource"><code class="docutils literal notranslate"><span class="pre">HtmlPageSource</span></code></a><ul>
|
||
<li><a class="reference internal" href="#aborist.sources.html_page.HtmlPageSource.source_type"><code class="docutils literal notranslate"><span class="pre">HtmlPageSource.source_type</span></code></a></li>
|
||
<li><a class="reference internal" href="#aborist.sources.html_page.HtmlPageSource.from_file"><code class="docutils literal notranslate"><span class="pre">HtmlPageSource.from_file()</span></code></a></li>
|
||
<li><a class="reference internal" href="#aborist.sources.html_page.HtmlPageSource.iter_documents"><code class="docutils literal notranslate"><span class="pre">HtmlPageSource.iter_documents()</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
<li><a class="reference internal" href="#module-aborist.concepts">concepts</a><ul>
|
||
<li><a class="reference internal" href="#aborist.concepts.add_concept_relation"><code class="docutils literal notranslate"><span class="pre">add_concept_relation()</span></code></a></li>
|
||
<li><a class="reference internal" href="#aborist.concepts.concept_relations_for_token"><code class="docutils literal notranslate"><span class="pre">concept_relations_for_token()</span></code></a></li>
|
||
<li><a class="reference internal" href="#aborist.concepts.has_compare_phrasing"><code class="docutils literal notranslate"><span class="pre">has_compare_phrasing()</span></code></a></li>
|
||
<li><a class="reference internal" href="#aborist.concepts.invalidate_cache"><code class="docutils literal notranslate"><span class="pre">invalidate_cache()</span></code></a></li>
|
||
<li><a class="reference internal" href="#aborist.concepts.purge_by_evidence_kind"><code class="docutils literal notranslate"><span class="pre">purge_by_evidence_kind()</span></code></a></li>
|
||
<li><a class="reference internal" href="#aborist.concepts.rivalry_excluded"><code class="docutils literal notranslate"><span class="pre">rivalry_excluded()</span></code></a></li>
|
||
<li><a class="reference internal" href="#aborist.concepts.synonym_expand"><code class="docutils literal notranslate"><span class="pre">synonym_expand()</span></code></a></li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
</aside>
|
||
</div>
|
||
</div><script src="../_static/documentation_options.js?v=335b8a46"></script>
|
||
<script src="../_static/doctools.js?v=fd6eb6e6"></script>
|
||
<script src="../_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||
<script src="../_static/scripts/furo.js?v=46bd48cc"></script>
|
||
</body>
|
||
</html> |