arborist/docs/diagrams/aborist-modules.dot

// Aborist module graph — top-level packages & their dependencies.
//
// Arrows point in the import direction (A → B means A imports from B).
// Cluster boxes group modules with shared concerns (storage, retrieval,
// federation, etc.).
//
// Render: dot -Tsvg aborist-modules.dot -o aborist-modules.svg

digraph aborist_modules {
    rankdir=LR
    node [shape=box, style="rounded,filled", fontname="Helvetica", fontsize=10]
    edge [fontname="Helvetica", fontsize=9]
    bgcolor="white"

    // ---- Substrate (pure crypto / serialization, no SQL) ----
    subgraph cluster_substrate {
        label="substrate"
        style="rounded,dashed"
        color="#999999"
        merkle    [label="merkle.py\n(Python port of Go merkle.go)", fillcolor="#fff7e6"]
        document  [label="document.py\nDocument · Edge · Chunker", fillcolor="#fff7e6"]
        wikitext  [label="wikitext.py\nto_base() prose-strip", fillcolor="#fff7e6"]
    }

    // ---- Storage layer ----
    subgraph cluster_storage {
        label="storage"
        style="rounded,dashed"
        color="#999999"
        store     [label="store.py\nv9.8 SQLite schema +\naudit chain helpers", fillcolor="#e6f0ff"]
        ingest    [label="ingest.py\nnormalize → chunk → merkle → upsert", fillcolor="#e6f0ff"]
        evict     [label="evict.py\nhot ↔ cold tier", fillcolor="#e6f0ff"]
    }

    // ---- Sources (corpus producers) ----
    subgraph cluster_sources {
        label="sources/"
        style="rounded,dashed"
        color="#999999"
        wikipedia [label="wikipedia.py\ncur + old SQL dumps", fillcolor="#e7ffe7"]
        wiki_xml  [label="wikipedia_xml.py\nphase IV iterparse", fillcolor="#e7ffe7"]
        html_page [label="html_page.py\nselectolax + httpx", fillcolor="#e7ffe7"]
        crawler   [label="crawler/\nasync HTML BFS", fillcolor="#e7ffe7"]
        grok      [label="grok.py\nxAI export", fillcolor="#e7ffe7"]
        vcs       [label="vcs.py\ngit + Mercurial", fillcolor="#e7ffe7"]
    }

    // ---- Retrieval & verifier ----
    subgraph cluster_retrieval {
        label="retrieval & verifier"
        style="rounded,dashed"
        color="#999999"
        search    [label="search/\nFTS5 backend + ABC", fillcolor="#ffe6f0"]
        concepts  [label="concepts/\nsynonym & rivalry overlay\n(per-shard concept_relations)", fillcolor="#ffe6f0"]
        qa_query  [label="qa/query.py\nmulti-source RAG", fillcolor="#ffe6f0"]
        qa_verify [label="qa/verify.py\nquote/span/entity/paraphrase\n+ claim_lattice (7 hard checks)", fillcolor="#ffe6f0"]
        qa_warrant[label="qa/warrant.py\n5 anchor classes\n(proper-noun · date · count\n · entity-list · cause)", fillcolor="#ffe6f0"]
        qa_evidence[label="qa/evidence.py\nEvidenceObject + spotlight", fillcolor="#ffe6f0"]
        qa_runner [label="qa/runner.py\nask(): single-doc Q&A", fillcolor="#ffe6f0"]
        qa_dag    [label="qa/dag.py\nper-run Merkle DAG\n(7 / 9 stages)", fillcolor="#ffe6f0"]
        qa_keys   [label="qa/keys.py\n8-dim cache_key + question_hash", fillcolor="#ffe6f0"]
        qa_inspect[label="qa/inspect.py\nsidecar diagnostic\n(read-only)", fillcolor="#ffe6f0"]
    }

    // ---- Distill (surface → core) ----
    distill   [label="distill/\nDistiller ABC + tfidf + first_sentence", fillcolor="#fff0e0"]

    // ---- Federation (off by default) ----
    mesh      [label="mesh/\nfederation: identity, roster,\nAEAD epoch secret, gossip wire", fillcolor="#f0e6ff"]

    // ---- Entry point ----
    cli       [label="cli.py\nargparse entrypoint\n(make targets call into here)", fillcolor="#dddddd", shape=note]

    // Substrate dependencies
    document   -> merkle
    store      -> merkle
    store      -> document
    ingest     -> store
    ingest     -> document

    // Sources -> ingest path
    wikipedia  -> ingest
    wiki_xml   -> ingest
    html_page  -> ingest  [label="parse_html →\nedges rows"]
    crawler    -> html_page
    grok       -> ingest
    vcs        -> ingest

    // Retrieval pipeline
    qa_query   -> search
    qa_query   -> concepts  [label="synonym_expand /\nrivalry_excluded"]
    qa_query   -> qa_keys
    qa_query   -> qa_verify
    qa_query   -> qa_evidence
    qa_query   -> qa_dag
    qa_query   -> wikitext  [label="to_base()"]
    qa_verify  -> qa_warrant
    qa_verify  -> qa_evidence
    qa_runner  -> qa_query
    qa_inspect -> qa_query  [style=dashed, label="read-only sidecar"]

    // Concepts reads existing edges
    concepts   -> store     [label="reads edges,\nwrites concept_relations"]

    // Search reads chunks
    search     -> store

    // Distill consumes store, writes derivations
    distill    -> store
    distill    -> document

    // Eviction
    evict      -> store

    // Mesh
    mesh       -> store

    // CLI dispatches everywhere
    cli        -> ingest
    cli        -> qa_runner
    cli        -> qa_query
    cli        -> qa_inspect
    cli        -> distill
    cli        -> evict
    cli        -> mesh
    cli        -> concepts
}