// Ingest pipeline: how a source document becomes Merkle-committed
// content-addressed storage in a v9.8 shard.
//
// Render: dot -Tsvg ingest-pipeline.dot -o ingest-pipeline.svg

digraph ingest_pipeline {
    rankdir=TB
    node [shape=box, style="rounded,filled", fontname="Helvetica", fontsize=10]
    edge [fontname="Helvetica", fontsize=9]
    bgcolor="white"

    source [label="SOURCE\nWikipedia SQL dump · XML page · HTML URL ·\nGrok export · git/hg repo · crawled site", fillcolor="#fff7e6", shape=note]

    iter_documents [label="Source.iter_documents()\nABC method per source kind\nyields Document(uri, raw_content, kind, ...)", fillcolor="#e7ffe7"]

    parse [label="source-specific parse\n• wikipedia.py: SQL row → Document\n• html_page.py: selectolax → text + edges\n• vcs.py: HEAD walk + supersedes chain\n• grok.py: conversation → Document(s)", fillcolor="#e7ffe7"]

    canonicalize [label="canonicalize(text)\nNFC + ws-collapse + strip ends\n(pinned by canonicalization_version)", fillcolor="#fff0e0"]

    chunker [label="Chunker.chunk(text)\ntok-512-v1 default\n(pinned by chunking_version)", fillcolor="#fff0e0"]

    leaf_hash [label="per-chunk leaf hash\nsha256(0x00 || canonical_chunk_bytes)\nleaves are content-addressed", fillcolor="#fff0e0"]

    merkle_tree [label="MerkleTree.build(leaves)\nnon-commutative HashCombine 0x03\nodd-element rule = self-duplicate\n→ document_root (32 bytes)", fillcolor="#fff0e0"]

    upsert [label="store.upsert_document(doc)\n• documents row keyed on document_root\n• chunks rows with leaf_hash + tier\n• merkle_nodes for proof reconstruction\n• edges per outbound link\n• audit_event chained", fillcolor="#e6f0ff"]

    chunks_fts [label="chunks_fts INSERT\nFTS5 contentless index\nrowid = chunks.chunk_id", fillcolor="#e6f0ff"]

    supersedes [label="supersedes edge\nif same uri but different content,\nlink new → old (lossless history)", fillcolor="#e6f0ff"]

    audit [label="audit_events INSERT\nevent_hash = sha256(prev_hash || canonical(body))\nlinear chain — 0 breaks invariant", fillcolor="#e6f0ff"]

    // Optional: distill cores AFTER ingest
    distill [label="distill/runner.py\n(optional, separate phase)\n• first_sentence\n• tfidf keyword core\n• per-contrib-chunk Merkle proof", fillcolor="#fff0e0", style="rounded,filled,dashed"]

    derivations [label="derivations row\ncore_root ← src_root\nproof_blob = inclusion proofs\nfor every contributing chunk", fillcolor="#fff0e0", style="rounded,filled,dashed"]

    end [label="SHARD READY\ndocuments + chunks + edges + merkle_nodes\n+ audit_events + (concept_relations\nback-derived later)", fillcolor="#d4edda", shape=note]

    source -> iter_documents
    iter_documents -> parse
    parse -> canonicalize
    canonicalize -> chunker
    chunker -> leaf_hash
    leaf_hash -> merkle_tree
    merkle_tree -> upsert
    upsert -> chunks_fts
    upsert -> supersedes
    upsert -> audit
    upsert -> end
    upsert -> distill [style=dashed]
    distill -> derivations [style=dashed]
    derivations -> end [style=dashed]
}
