From 508fec69753d92025c596d31dc3c88400d6235c6 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Mon, 1 Jun 2026 16:58:08 -0400 Subject: [PATCH] session: Merkle-rooted multi-turn Q&A REPL with Bates ledger MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `arborist session` is an interactive multi-turn Q&A REPL where every turn (or fork) mints one node in a per-session SQLite-backed tree. Each node carries a stable Bates id (`-<6-digit>`) and folds into a Merkle subtree-hash chain; the root node's subtree_hash is the session_root. Tree shape lets: - **Forks** happen implicitly: `/cd ` to a prior node, ask again → sibling under that parent. Branch points (≥2 children) surfaced by `/branches`. - **Page-refresh caching** stay cheap: a client tracking (bates → subtree_hash, body) only refetches subtrees whose hash changed. Sibling subtrees that didn't change are byte-identical → cache-equivalent. Same property git pack-protocol and IPFS MFS use. - **Audit-chain verification** be per-session and independent: each session db has its own session_audit_events with event_hash = sha256(prev_hash || canonical_body). `make session-chain-check` walks all sessions; 0 breaks each = intact. Wire: - arborist/qa/session.py — Session class, Bates minting, Merkle recompute on O(depth) insert, audit chain, helpers (list, render, resolve ). - arborist/cli.py — `session` subcommand: REPL + --list / --tree / --chain-check / --gc / --json flags. Ancestor-titles → retrieval keywords (parsed from cited-pointer lines in answer_text) flow down the branch via policy["retrieval_keywords"]. - arborist/qa/providence_query.py — honor policy["retrieval_keywords"]: augment FTS5 retrieval query without touching cache_key (mirrors legacy --retrieval-keywords discipline, #000001). - Makefile — `make session [SID=...]`, `make session-list`, `make session-tree SID=...`, `make session-chain-check`, `make session-gc SESSION_KEEP=N`. - docs/sessions.md — schema, Merkle conventions (portability for non-Python consumers), REPL command reference. - tests/test_session.py — 15 tests: create, resume, add_node, fork via cd, branches, root determinism, audit chain (intact + tampered), resolve, list, render, sibling-invariance of subtree_hash. Storage: ~/.arborist/sessions/.db (self-contained — no FK into main store). Answers live in providence_cache keyed by cache_key; session only carries conversation shape. Cache hits stay live across sessions. Bounded growth via --gc. Phase 1 scope: tree + Merkle + Bates + retrieval-keyword flow. NOT in Phase 1: LLM-side conversation_history (threading prior Q&A into the LLM prompt + conversation_hash). A bare-pronoun follow-up ("who created him?") gets the right retrieval today but the LLM may still UNGROUNDED because it sees only the new question as user message. Folding conversation_history into the prompt + cache_key's conversation_hash dimension is the natural Phase 2. 197 tests pass. --- Makefile | 24 ++ arborist/cli.py | 432 +++++++++++++++++++++++ arborist/qa/providence_query.py | 70 +++- arborist/qa/session.py | 608 ++++++++++++++++++++++++++++++++ docs/sessions.md | 205 +++++++++++ tests/test_session.py | 211 +++++++++++ 6 files changed, 1541 insertions(+), 9 deletions(-) create mode 100644 arborist/qa/session.py create mode 100644 docs/sessions.md create mode 100644 tests/test_session.py diff --git a/Makefile b/Makefile index df6809b..47e4168 100644 --- a/Makefile +++ b/Makefile @@ -207,6 +207,30 @@ query: bootstrap ## ask the corpus a question [JSON=1 BURN=1 LLM=qwen|hermes REP $(if $(LLM_ENDPOINT),@echo "# llm: $(LLM_ENDPOINT) / $(LLM_MODEL)" >&2,) $(ARBORIST) --shards-dir $(SHARDS_DIR) query --top-k $(QUERY_TOP_K) $(if $(LLM_ENDPOINT),--endpoint $(LLM_ENDPOINT) --model $(LLM_MODEL),) $(if $(JSON),--json,) $(if $(BURN),--burn,) $(if $(REPAIR),--repair,) $(if $(REPROMPTS),--repair-reprompts $(REPROMPTS),) $(if $(ANSWER_MODE),--answer-mode $(ANSWER_MODE),) --user-payload-layout $(LAYOUT) $(if $(K),--retrieval-keywords "$(K)",) $(if $(BROAD),--apply-quantifier-caps,) $(if $(REJECT_BROAD),--reject-broad,) $(if $(ALLOW_BROAD),--allow-broad,) $(if $(WITNESS),--witness,) $(if $(XLANG),--crosslang-guard,) $(if $(XLANG_MT),--crosslang-translate,) "$(Q)" +# Merkle-rooted multi-turn Q&A REPL. Each turn (or fork) mints one +# tree node with a Bates id + a session-wide subtree_hash. Forking is +# implicit: /cd to a parent node and ask again → sibling under that +# parent. See arborist/qa/session.py + docs/sessions.md. +session: bootstrap ## interactive multi-turn Q&A REPL [SID=... LLM=qwen|hermes ANSWER_MODE=... BURN=1] + $(if $(LLM_ENDPOINT),@echo "# llm: $(LLM_ENDPOINT) / $(LLM_MODEL)" >&2,) + $(ARBORIST) --shards-dir $(SHARDS_DIR) session $(if $(SID),--sid $(SID),) --top-k $(QUERY_TOP_K) $(if $(LLM_ENDPOINT),--endpoint $(LLM_ENDPOINT) --model $(LLM_MODEL),) $(if $(ANSWER_MODE),--answer-mode $(ANSWER_MODE),) $(if $(BURN),--burn,) + +session-list: bootstrap ## list all session dbs in ~/.arborist/sessions/ + $(ARBORIST) session --list $(if $(JSON),--json,) + +session-tree: bootstrap ## print one session's tree [SID=...] + @if [ -z "$$SID" ] && [ -z "$(SID)" ]; then \ + echo "usage: make session-tree SID= [JSON=1]"; exit 2; \ + fi + $(ARBORIST) session --sid $(SID) --tree $(if $(JSON),--json,) + +session-chain-check: bootstrap ## verify session audit-chain integrity [SID=... (default: all)] + $(ARBORIST) session --chain-check $(if $(SID),--sid $(SID),) + +SESSION_KEEP ?= 100 +session-gc: bootstrap ## delete old session dbs, keep $(SESSION_KEEP) most recent [SESSION_KEEP=N] + $(ARBORIST) session --gc $(SESSION_KEEP) + query-dry: bootstrap ## like 'make query' but skip the LLM call (dry-run) [JSON=1 BURN=1 ANSWER_MODE=... BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1 XLANG=1 XLANG_MT=1] @if [ -z "$$Q" ] && [ -z "$(Q)" ]; then \ echo "usage: make query-dry Q=\"your question\" [JSON=1 BURN=1 ANSWER_MODE=claim_lattice|claim_lattice_pointer|quote BROAD=1 REJECT_BROAD=1 ALLOW_BROAD=1 XLANG=1 XLANG_MT=1]"; exit 2; \ diff --git a/arborist/cli.py b/arborist/cli.py index 448a924..afe5860 100644 --- a/arborist/cli.py +++ b/arborist/cli.py @@ -7447,9 +7447,441 @@ def build_parser() -> argparse.ArgumentParser: ) sidecar_build_fts.set_defaults(func=_cmd_sidecar_build_fts) + # `arborist session` — Merkle-rooted multi-turn Q&A REPL with + # forks. See arborist/qa/session.py for the tree + audit-chain + # discipline; see docs/sessions.md for the wire format consumers + # outside arborist can read (the on-disk schema is self-contained). + session_cmd = sub.add_parser( + "session", + help="Merkle-rooted multi-turn Q&A REPL (forks via /cd)", + ) + session_cmd.add_argument( + "--sid", default=None, + help="resume an existing session id (default: new session)", + ) + session_cmd.add_argument( + "--sessions-dir", default=None, + help="directory containing session dbs (default ~/.arborist/sessions/)", + ) + session_cmd.add_argument( + "--list", action="store_true", + help="list all sessions and exit", + ) + session_cmd.add_argument( + "--tree", action="store_true", + help="print the session tree and exit (use with --sid)", + ) + session_cmd.add_argument( + "--chain-check", action="store_true", + help="verify session audit chain integrity (all sessions if no --sid)", + ) + session_cmd.add_argument( + "--gc", type=int, default=None, metavar="N", + help="keep the N most-recent sessions, delete older db files", + ) + session_cmd.add_argument( + "--json", action="store_true", + help="emit structured JSON (for --list and --tree)", + ) + # Query-side options for the REPL turns. Same shape as `arborist + # query` so $LLM=qwen / $K=… style env-driven flags work. + session_cmd.add_argument("--top-k", type=int, default=8) + session_cmd.add_argument( + "--answer-mode", + choices=["quote", "claim_lattice_pointer", "claim_lattice"], + default="claim_lattice", + ) + session_cmd.add_argument("--endpoint", default=None) + session_cmd.add_argument("--model", default=None) + session_cmd.add_argument( + "--burn", action="store_true", + help="burn an existing cache entry for each turn (force re-LLM)", + ) + session_cmd.set_defaults(func=_cmd_session) + return p +def _cmd_session(args: argparse.Namespace) -> int: + """Merkle-rooted multi-turn Q&A REPL with forks. + + Subcommands flagged on the parser: + --list list sessions + --tree print tree (requires --sid) + --chain-check verify audit chain + --gc N keep N most recent sessions + + Default (no flag): start an interactive REPL on a new session + (or --sid to resume). REPL commands: + /tree print tree + /branches list branch-point Bates (≥2 children) + /cd set current to + /back cd to parent of current + /label label the current node + /show show stored cache_key + audit for that node + /sid print session id + /root print session_root + /help print this help + /quit exit (or Ctrl-D) + new question, child of current node + """ + from pathlib import Path as _Path + + from arborist.qa.session import ( + Session, default_sessions_dir, list_sessions, render_tree, + session_path, + ) + + sessions_dir = ( + _Path(args.sessions_dir) if args.sessions_dir + else default_sessions_dir() + ) + + # --list + if getattr(args, "list", False): + listing = list_sessions(sessions_dir) + if args.json: + print(json.dumps(listing, indent=2)) + else: + if not listing: + print(f"no sessions in {sessions_dir}") + return 0 + print(f"{len(listing)} session(s) in {sessions_dir}:") + for s in listing: + print( + f" {s['sid']:32s} nodes={s['n_nodes']:3d} " + f"root={s['session_root'][:12]}… " + f"updated={s['updated_at']}" + ) + return 0 + + # --gc + if args.gc is not None: + listing = list_sessions(sessions_dir) + listing.sort(key=lambda s: s["updated_at"], reverse=True) + keep, drop = listing[: args.gc], listing[args.gc:] + for s in drop: + try: + _Path(s["path"]).unlink() + except Exception: + pass + print( + f"kept {len(keep)}, deleted {len(drop)} " + f"session db(s) in {sessions_dir}" + ) + return 0 + + # --chain-check + if args.chain_check: + if args.sid: + paths = [session_path(args.sid, sessions_dir)] + else: + paths = sorted(sessions_dir.glob("*.db")) + any_break = False + for p in paths: + if not p.is_file(): + print(f"missing: {p}") + any_break = True + continue + sess = Session.open(p) + intact, breaks = sess.chain_check() + sess.close() + print(f"{p.name} intact={intact} breaks={breaks}") + if breaks: + any_break = True + return 1 if any_break else 0 + + # --tree + if args.tree: + if not args.sid: + print("--tree requires --sid", file=sys.stderr) + return 2 + p = session_path(args.sid, sessions_dir) + if not p.is_file(): + print(f"no such session: {args.sid}", file=sys.stderr) + return 2 + sess = Session.open(p) + if args.json: + nodes = [ + { + "bates": n.bates, "seq": n.seq, + "parent_bates": n.parent_bates, + "question": n.question, "cache_key": n.cache_key, + "audit_mode": n.audit_mode, + "created_at": n.created_at, "label": n.label, + "node_hash": n.node_hash, + "subtree_hash": n.subtree_hash, + } + for n in sess.all_nodes() + ] + print(json.dumps({ + "sid": sess.sid, + "session_root": sess.session_root, + "current_bates": sess.current_bates, + "branches": sess.branches(), + "nodes": nodes, + }, indent=2)) + else: + print(render_tree(sess)) + sess.close() + return 0 + + # Default: interactive REPL. + if args.sid: + p = session_path(args.sid, sessions_dir) + sess = Session.open(p, sid=args.sid) + else: + # Mint a fresh sid via Session.open's default generator. + from arborist.qa.session import _gen_sid + sid = _gen_sid() + p = session_path(sid, sessions_dir) + sess = Session.open(p, sid=sid) + print(f"# session {sess.sid}", file=sys.stderr) + print( + f"# db {p} · root {sess.session_root[:12]}…", + file=sys.stderr, + ) + print( + "# /help for commands · Ctrl-D to exit · " + "any other line asks a question", + file=sys.stderr, + ) + + rc = _session_repl(args, sess) + sess.close() + return rc + + +def _session_repl(args: argparse.Namespace, sess) -> int: + """REPL loop. Each non-/ line becomes a new node under current.""" + from arborist.qa.session import render_tree + from arborist.qa.providence_query import providence_query + from arborist.qa.corpus import ( + MultiShardSqliteCorpus, SqliteShardCorpus, + ) + from arborist.store import connect as _connect + + shards_dir = getattr(args, "global_shards_dir", None) or getattr( + args, "shards_dir", None + ) + single_db = getattr(args, "db", None) + _all_shard_paths: list[Path] = [] + if shards_dir: + candidates = sorted(Path(shards_dir).glob("*.db")) + skip = {"qa.db", "snapshots.db", "selfmodel-chain.db"} + _all_shard_paths = [ + pp for pp in candidates if pp.name not in skip + ] or candidates + if _all_shard_paths: + corpus_obj = MultiShardSqliteCorpus(_all_shard_paths) + _closer = corpus_obj.close + elif single_db: + _conn = _connect(single_db) + corpus_obj = SqliteShardCorpus(_conn) + _closer = _conn.close + else: + print( + "session needs --shards-dir or --db (set on the top-level " + "parser).", + file=sys.stderr, + ) + return 2 + + # LLM client. Same env-resolution shape as `arborist query`. + endpoint = args.endpoint or os.environ.get( + "ARBORIST_LLM_ENDPOINT", + "https://hermes.ai.unturf.com/v1", + ) + model = args.model or os.environ.get( + "ARBORIST_LLM_MODEL", + "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic", + ) + from arborist.qa.client import OpenAICompatibleClient + client = OpenAICompatibleClient(base_url=endpoint) + + qa_db = Path.home() / ".arborist" / "qa.db" + qa_db.parent.mkdir(parents=True, exist_ok=True) + + policy = {"answer_mode": args.answer_mode} + + help_text = ( + "/tree show the tree\n" + "/branches list branch points\n" + "/cd move current pointer\n" + "/back cd to parent of current\n" + "/label label current node\n" + "/show show stored info for a node\n" + "/sid print session id\n" + "/root print session_root\n" + "/help this help\n" + "/quit exit" + ) + + try: + while True: + try: + line = input(f"[{sess.current_bates}] > ") + except (EOFError, KeyboardInterrupt): + print(file=sys.stderr) + break + line = line.strip() + if not line: + continue + if line in ("/quit", "/exit"): + break + if line == "/help": + print(help_text) + continue + if line == "/sid": + print(sess.sid) + continue + if line == "/root": + print(sess.session_root) + continue + if line == "/tree": + print(render_tree(sess)) + continue + if line == "/branches": + bs = sess.branches() + if not bs: + print("(no branch points yet)") + else: + for b in bs: + n = sess.get_node(b) + print(f" {b} {(n.question or '(root)')[:60]}") + continue + if line == "/back": + cur = sess.get_node(sess.current_bates) + if cur and cur.parent_bates: + sess.cd(cur.parent_bates) + print(f"cd → {sess.current_bates}") + else: + print("already at root") + continue + if line.startswith("/cd "): + ref = line[4:].strip() + tgt = sess.resolve(ref) + if tgt is None: + print(f"unknown ref: {ref}") + else: + sess.cd(tgt) + print(f"cd → {tgt}") + continue + if line.startswith("/label "): + name = line[7:].strip() + sess.label(sess.current_bates, name) + print(f"labeled {sess.current_bates} as {name!r}") + continue + if line.startswith("/show"): + ref = line[5:].strip() or sess.current_bates + tgt = sess.resolve(ref) or ref + n = sess.get_node(tgt) + if not n: + print(f"unknown ref: {ref}") + continue + print(json.dumps({ + "bates": n.bates, "parent_bates": n.parent_bates, + "question": n.question, "cache_key": n.cache_key, + "audit_mode": n.audit_mode, + "label": n.label, + "node_hash": n.node_hash, + "subtree_hash": n.subtree_hash, + "created_at": n.created_at, + }, indent=2)) + continue + if line.startswith("/"): + print(f"unknown command: {line} (try /help)") + continue + + # Non-command line → new question. Pull ancestor cited + # titles into retrieval_keywords so context flows down the + # branch (verifier-blind; same discipline as the existing + # --retrieval-keywords flag, see #000001). + ancestor_keywords = _session_ancestor_keywords(sess, qa_db) + policy_local = dict(policy) + if ancestor_keywords: + policy_local["retrieval_keywords"] = ancestor_keywords + + try: + result = providence_query( + corpus_obj, line, client, + qa_db=qa_db, + policy=policy_local, + model_id=model, + burn_existing=bool(args.burn), + top_k=args.top_k, + ) + except Exception as e: + print(f"query failed: {e}") + continue + + audit_mode = result.get("audit_mode", "UNGROUNDED") + ckey = result.get("cache_key", "") + n = sess.add_node(line, ckey, audit_mode) + ans = (result.get("answer_text") or "").strip() + print(f"\n[{n.bates}] {audit_mode}") + print(ans) + print( + f"\n# root → {sess.session_root[:16]}… " + f"cache_key={ckey[:12]}…" + ) + finally: + try: + _closer() + except Exception: + pass + try: + client.close() + except Exception: + pass + return 0 + + +def _session_ancestor_keywords(sess, qa_db: Path) -> str: + """Gather cited source titles along the path-to-root → flat + keyword string suitable for retrieval_keywords. Verifier-blind: + these terms widen retrieval, never change governance_policy_hash. + + Reads the ancestor nodes' stored answer_text from providence_cache + and parses ``[E# | | <chunk_prefix>: …`` pointer-lines. + Only titles the model actually cited count — exactly what the + user just saw on the prior turn — so retrieval flows along the + visible thread of the conversation, not the wider retrieval pool. + """ + import re + pointer_re = re.compile(r"\[E\d+\s*\|\s*([^|]+?)\s*\|") + try: + import sqlite3 as _sql + conn = _sql.connect(f"file:{qa_db}?mode=ro", uri=True) + conn.row_factory = _sql.Row + except Exception: + return "" + seen_titles: list[str] = [] + try: + cur = sess.current_bates + ancestors = sess.path_to_root(cur)[1:] # skip current + for n in ancestors: + if not n.cache_key: + continue + r = conn.execute( + "SELECT answer_text FROM providence_cache " + "WHERE cache_key=? AND falsification_state='live' LIMIT 1", + (n.cache_key,), + ).fetchone() + if not r: + continue + for m in pointer_re.finditer(r["answer_text"] or ""): + t = m.group(1).strip() + if t and t not in seen_titles: + seen_titles.append(t) + finally: + try: + conn.close() + except Exception: + pass + return " ".join(seen_titles[:8]) # cap to avoid keyword sprawl + + def _cmd_serve(args: argparse.Namespace) -> int: """Start the HTTP wallet server. Blocks until SIGINT/SIGTERM.""" from arborist.wallet.server import WalletServer, serve diff --git a/arborist/qa/providence_query.py b/arborist/qa/providence_query.py index 9916148..4def8ef 100644 --- a/arborist/qa/providence_query.py +++ b/arborist/qa/providence_query.py @@ -166,6 +166,7 @@ def _persist_miss( question: str, run_result: dict, sources: list, + proof_blob: str, source_root: str, qhash: str, mhash: str, @@ -179,11 +180,10 @@ def _persist_miss( """Write one providence_cache row + one audit event for a fresh (miss) result. Returns (audit_event_hash, run_dag_dict). - Minimal-viable persist: merkle_proof is an empty JSON array - placeholder. Real per-chunk Merkle proofs are a follow-up — the - schema requires the column NOT NULL, so empty array satisfies - that without claiming proof structure the caller didn't actually - build. + ``proof_blob`` is a JSON-serialized {context_root, sources, + retrieval_purity} object — same shape legacy query() writes, and + what arborist-viz reads to synthesize the leaf set under a + multi-source context root. Audit chain append happens BEFORE the providence_cache INSERT so event_hash is available to set audit_event_hash on the row. @@ -252,7 +252,7 @@ def _persist_miss( "ON CONFLICT(cache_key) DO NOTHING", ( ckey, source_root, "corpus://multi-source", qhash, - question, answer_text, "[]", + question, answer_text, proof_blob, mhash, chash, ghash, schema_version, canonicalization_version, chunking_version, chain, event_hash, now, @@ -320,12 +320,24 @@ def providence_query( progress.emit("search.start", top_k=top_k) t_ret = _time.time() + # Retrieval-side keyword augmentation: when ``policy[ + # "retrieval_keywords"]`` is set (free-text), append to the question + # passed into FTS5 routes. Verifier-blind, never folded into + # cache_key (mirrors the legacy ``--retrieval-keywords`` flag, + # see #000001). Used by ``arborist session`` to flow ancestor + # cited-titles down a branch so pronoun follow-ups ("who created + # him?") still surface the prior turn's primary source. + retrieval_kw = (policy.get("retrieval_keywords") or "").strip() + retrieval_question = ( + f"{question} {retrieval_kw}" if retrieval_kw else question + ) + def _safe(method: str): try: fn = getattr(corpus, method, None) if fn is None: return [] - return fn(question, limit=top_k * 4) + return fn(retrieval_question, limit=top_k * 4) except NotSupportedError: return [] @@ -349,7 +361,7 @@ def providence_query( hits = sorted(best.values(), key=lambda h: h.score) hits = apply_title_boost( - hits, question, + hits, retrieval_question, higher_is_better=getattr(corpus, "higher_is_better", False), )[:top_k] retrieval_s = _time.time() - t_ret @@ -502,12 +514,52 @@ def providence_query( # Use the LLM-run sources (with used/used_pointer_ids) for # the persist path; fall back to provisional if absent. sources = run_result.get("sources") or sources + # Enrich each source with hit-level retrieval metadata so the + # persisted merkle_proof matches legacy query()'s shape. + # Arborist-viz reads `merkle_proof.sources` to synthesize the + # leaf set under the context_root — an empty list (the prior + # "[]" placeholder) showed 0 leaves in the lattice. + hit_by_root = {h.document_root: h for h in hits} + for s in sources: + h = hit_by_root.get(s.get("document_root")) + if h is None: + continue + s.setdefault("score", float(h.score)) + s.setdefault("shard", Path(h.shard_id).name if h.shard_id else "") + # Retrieval-purity sidecar (legacy parity — proof_obj has it). + # Sidecar signal, not commitment: lets bench scripts spot + # creeping retrieval-quality regressions without touching the + # audit chain (the audit body excludes merkle_proof). + noisy_roles = {"noisy_background_source", "sequel_background_source"} + primary_rank = next( + (i for i, s in enumerate(sources, start=1) + if s.get("source_role") == "primary_answer_source"), + 0, + ) + noise_sources = [s for s in sources if s.get("source_role") in noisy_roles] + retrieval_purity = { + "primary_rank": primary_rank, + "primary_used": ( + primary_rank > 0 + and sources[primary_rank - 1].get("used") is True + ), + "noise_sources_count": len(noise_sources), + "noise_sources_used": sum(1 for s in noise_sources if s.get("used")), + "total_sources": len(sources), + "used_sources": sum(1 for s in sources if s.get("used")), + } + proof_obj = { + "context_root": source_root, + "sources": sources, + "retrieval_purity": retrieval_purity, + } + proof_blob = _json.dumps(proof_obj, separators=(",", ":")) progress.emit("persist.start") t_persist = _time.time() audit_event_hash, run_dag = _persist_miss( qa_conn, ckey=ckey, question=question, run_result=run_result, - sources=sources, source_root=source_root, + sources=sources, proof_blob=proof_blob, source_root=source_root, qhash=qhash, mhash=mhash, chash=chash, ghash=ghash, schema_version=schema_version, canonicalization_version=canonicalization_version, diff --git a/arborist/qa/session.py b/arborist/qa/session.py new file mode 100644 index 0000000..94bbf21 --- /dev/null +++ b/arborist/qa/session.py @@ -0,0 +1,608 @@ +"""Session: Merkle-rooted conversation tree over providence_cache. + +A session is a tree of Q&A nodes. Each turn (or fork) mints a new node +under a chosen parent. Every node carries a stable Bates identifier +(``<sid>-<6-digit>``) AND a content-addressed ``node_hash``. The tree +maintains a Merkle ``subtree_hash`` per node; the root node's +``subtree_hash`` is the ``session_root``. + +When a new node is inserted under parent P: + 1. Compute ``node_hash = sha256(canonical(parent_node_hash || bates || + question || cache_key || audit_mode || created_at || label))`` + 2. Set ``subtree_hash = leaf_hash(node_hash)`` (new node is a leaf). + 3. Walk N → P → P.parent → … → root, recomputing each ancestor's + ``subtree_hash`` to fold in the new child. + 4. Update ``session_meta.session_root`` to the root's new ``subtree_hash``. + 5. Append one ``session_audit_events`` row whose ``event_hash`` chains + off the previous event. + +This is O(depth) per insert. Caching for page-refresh works the way it +does in any Merkle tree: a UI client stores (bates → subtree_hash, body) +and on refresh fetches only nodes whose subtree_hash changed. Sibling +subtrees that didn't change keep their hash → no re-fetch. + +The session db is self-contained at ``~/.arborist/sessions/<sid>.db`` — +no foreign keys into the main store. Cross-session verification: +``arborist session --chain-check SID``. + +Bates discipline: never reassigned, never re-used. ``seq=0`` is a +synthetic root with empty question; all real questions are descendants. +This matches arborist's broader "every fact has a stable name" identity +philosophy. +""" +from __future__ import annotations + +import hashlib +import json +import sqlite3 +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +from arborist.merkle import hash_combine, hash_leaf + + +def _canonical(body: dict) -> bytes: + """Stable canonical JSON: sorted keys, no whitespace, UTF-8.""" + return json.dumps( + body, sort_keys=True, separators=(",", ":"), ensure_ascii=False + ).encode("utf-8", errors="surrogatepass") + + +def _sha256_hex(*parts: bytes) -> str: + h = hashlib.sha256() + for p in parts: + h.update(p) + return h.hexdigest() + + +def _now_iso() -> str: + return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + + +def _gen_sid() -> str: + """Generate a session id: YYYYMMDD-HHMMSS-<4-hex>.""" + import secrets + return time.strftime("%Y%m%d-%H%M%S", time.gmtime()) + "-" + secrets.token_hex(2) + + +SCHEMA = """ +CREATE TABLE IF NOT EXISTS session_meta ( + session_id TEXT PRIMARY KEY, + root_bates TEXT NOT NULL, + session_root TEXT NOT NULL, + current_bates TEXT NOT NULL, + next_seq INTEGER NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS nodes ( + bates TEXT PRIMARY KEY, + seq INTEGER NOT NULL, + parent_bates TEXT REFERENCES nodes(bates), + question TEXT NOT NULL, + cache_key TEXT NOT NULL, + audit_mode TEXT NOT NULL, + created_at TEXT NOT NULL, + label TEXT, + node_hash TEXT NOT NULL, + subtree_hash TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_nodes_parent ON nodes(parent_bates); +CREATE INDEX IF NOT EXISTS idx_nodes_label ON nodes(label); + +CREATE TABLE IF NOT EXISTS session_audit_events ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + event_type TEXT NOT NULL, + bates TEXT, + prev_root TEXT NOT NULL, + session_root TEXT NOT NULL, + body TEXT NOT NULL, + event_hash TEXT NOT NULL, + created_at TEXT NOT NULL +); +""" + + +@dataclass(frozen=True) +class Node: + bates: str + seq: int + parent_bates: Optional[str] + question: str + cache_key: str + audit_mode: str + created_at: str + label: Optional[str] + node_hash: str + subtree_hash: str + + +def _bates(sid: str, seq: int) -> str: + return f"{sid}-{seq:06d}" + + +def _compute_node_hash( + parent_node_hash: str, + bates: str, + question: str, + cache_key: str, + audit_mode: str, + created_at: str, + label: Optional[str], +) -> str: + body = { + "parent_node_hash": parent_node_hash, + "bates": bates, + "question": question, + "cache_key": cache_key, + "audit_mode": audit_mode, + "created_at": created_at, + "label": label or "", + } + return _sha256_hex(_canonical(body)) + + +def _compute_subtree_hash(node_hash: str, children_subtree_hashes: list[str]) -> str: + """Merkle hash over node + sorted children. + + Children are sorted by their own subtree_hash (deterministic). Two + leaves combine via ``HashCombine`` to keep the prefix-discipline + consistent with the rest of the repo. A node with N children folds + them pairwise as a balanced tree (left-leaning self-duplicate on + odd counts, same as ``arborist.merkle.MerkleTree``). + """ + leaf = hash_leaf(bytes.fromhex(node_hash)) + if not children_subtree_hashes: + return leaf.hex() + # Sort children for determinism — Bates is monotonic but inserts can + # arrive out of order under future concurrent writers; subtree_hash + # is content-addressed so sorting on it is the natural canonical form. + levels: list[bytes] = sorted(bytes.fromhex(h) for h in children_subtree_hashes) + while len(levels) > 1: + nxt: list[bytes] = [] + for i in range(0, len(levels), 2): + if i + 1 < len(levels): + nxt.append(hash_combine(levels[i], levels[i + 1])) + else: + # Odd-count rule: self-duplicate. + nxt.append(hash_combine(levels[i], levels[i])) + levels = nxt + children_root = levels[0] + return hash_combine(leaf, children_root).hex() + + +class Session: + """Open Merkle-rooted conversation-tree store on disk.""" + + def __init__(self, conn: sqlite3.Connection): + self._conn = conn + self._conn.row_factory = sqlite3.Row + + @property + def conn(self) -> sqlite3.Connection: + return self._conn + + # ---- factory ---- + + @classmethod + def open(cls, db_path: Path, sid: Optional[str] = None) -> "Session": + """Open or create a session at db_path. ``sid`` is required only + when creating a NEW session db; for existing dbs we read the + single session_meta row to discover the sid. + """ + db_path = Path(db_path) + db_path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(str(db_path)) + conn.executescript(SCHEMA) + sess = cls(conn) + existing = conn.execute( + "SELECT session_id FROM session_meta LIMIT 1" + ).fetchone() + if existing is None: + sess._init_new(sid or _gen_sid()) + return sess + + def _init_new(self, sid: str) -> None: + """Create the synthetic root node (seq=0, empty question).""" + now = _now_iso() + root_bates = _bates(sid, 0) + # Synthetic root has no parent → parent_node_hash="" (empty). + nh = _compute_node_hash( + parent_node_hash="", + bates=root_bates, + question="", + cache_key="", + audit_mode="", + created_at=now, + label=None, + ) + sh = _compute_subtree_hash(nh, []) + with self._conn: + self._conn.execute( + "INSERT INTO nodes (bates, seq, parent_bates, question, " + "cache_key, audit_mode, created_at, label, node_hash, " + "subtree_hash) VALUES (?, ?, NULL, '', '', '', ?, NULL, ?, ?)", + (root_bates, 0, now, nh, sh), + ) + self._conn.execute( + "INSERT INTO session_meta (session_id, root_bates, " + "session_root, current_bates, next_seq, created_at, " + "updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)", + (sid, root_bates, sh, root_bates, 1, now, now), + ) + self._append_event("session_init", root_bates, prev_root="", + session_root=sh, + body={"sid": sid, "root_bates": root_bates}) + + # ---- properties ---- + + @property + def sid(self) -> str: + r = self._conn.execute( + "SELECT session_id FROM session_meta" + ).fetchone() + return r["session_id"] if r else "" + + @property + def root_bates(self) -> str: + r = self._conn.execute("SELECT root_bates FROM session_meta").fetchone() + return r["root_bates"] if r else "" + + @property + def session_root(self) -> str: + r = self._conn.execute("SELECT session_root FROM session_meta").fetchone() + return r["session_root"] if r else "" + + @property + def current_bates(self) -> str: + r = self._conn.execute("SELECT current_bates FROM session_meta").fetchone() + return r["current_bates"] if r else "" + + # ---- queries ---- + + def get_node(self, bates: str) -> Optional[Node]: + r = self._conn.execute( + "SELECT * FROM nodes WHERE bates=?", (bates,) + ).fetchone() + if not r: + return None + return Node( + bates=r["bates"], seq=r["seq"], + parent_bates=r["parent_bates"], + question=r["question"], cache_key=r["cache_key"], + audit_mode=r["audit_mode"], created_at=r["created_at"], + label=r["label"], node_hash=r["node_hash"], + subtree_hash=r["subtree_hash"], + ) + + def resolve(self, ref: str) -> Optional[str]: + """Resolve a user-facing reference to a Bates. + + Accepts: full Bates (``<sid>-<6digit>``), short seq (``23``), + or a label. Returns the canonical Bates or None. + """ + sid = self.sid + # Full Bates direct match. + if "-" in ref and self.get_node(ref) is not None: + return ref + # Bare numeric → prefix with sid. + try: + n = int(ref) + cand = _bates(sid, n) + if self.get_node(cand): + return cand + except ValueError: + pass + # Label lookup. + r = self._conn.execute( + "SELECT bates FROM nodes WHERE label=?", (ref,) + ).fetchone() + if r: + return r["bates"] + return None + + def children_of(self, bates: str) -> list[Node]: + return [ + self.get_node(r["bates"]) # type: ignore[misc] + for r in self._conn.execute( + "SELECT bates FROM nodes WHERE parent_bates=? ORDER BY seq", + (bates,), + ) + ] + + def path_to_root(self, bates: str) -> list[Node]: + """Return [bates, parent, ..., root]. Root last.""" + out: list[Node] = [] + cur: Optional[str] = bates + while cur is not None: + n = self.get_node(cur) + if n is None: + break + out.append(n) + cur = n.parent_bates + return out + + def branches(self) -> list[str]: + """Bates with ≥2 children — explicit branch points.""" + return [ + r["parent_bates"] + for r in self._conn.execute( + "SELECT parent_bates FROM nodes WHERE parent_bates IS NOT NULL " + "GROUP BY parent_bates HAVING COUNT(*) >= 2 " + "ORDER BY MIN(seq)" + ) + ] + + def all_nodes(self) -> list[Node]: + return [ + Node( + bates=r["bates"], seq=r["seq"], + parent_bates=r["parent_bates"], + question=r["question"], cache_key=r["cache_key"], + audit_mode=r["audit_mode"], created_at=r["created_at"], + label=r["label"], node_hash=r["node_hash"], + subtree_hash=r["subtree_hash"], + ) + for r in self._conn.execute( + "SELECT * FROM nodes ORDER BY seq" + ) + ] + + # ---- mutations ---- + + def add_node( + self, + question: str, + cache_key: str, + audit_mode: str, + parent_bates: Optional[str] = None, + ) -> Node: + """Mint a new node under parent (or current_bates). Recompute + subtree_hashes from new node up to root, append audit event.""" + parent_bates = parent_bates or self.current_bates + parent = self.get_node(parent_bates) + if parent is None: + raise ValueError(f"parent node not found: {parent_bates}") + meta = self._conn.execute( + "SELECT session_id, next_seq, session_root FROM session_meta" + ).fetchone() + sid = meta["session_id"] + seq = meta["next_seq"] + prev_root = meta["session_root"] + bates = _bates(sid, seq) + created_at = _now_iso() + node_hash = _compute_node_hash( + parent_node_hash=parent.node_hash, + bates=bates, question=question, cache_key=cache_key, + audit_mode=audit_mode, created_at=created_at, label=None, + ) + subtree_hash = _compute_subtree_hash(node_hash, []) + + with self._conn: + self._conn.execute( + "INSERT INTO nodes (bates, seq, parent_bates, question, " + "cache_key, audit_mode, created_at, label, node_hash, " + "subtree_hash) VALUES (?, ?, ?, ?, ?, ?, ?, NULL, ?, ?)", + (bates, seq, parent_bates, question, cache_key, + audit_mode, created_at, node_hash, subtree_hash), + ) + # Walk parent → root, recompute each ancestor's subtree_hash. + new_root = self._recompute_path(parent_bates) + self._conn.execute( + "UPDATE session_meta SET session_root=?, current_bates=?, " + "next_seq=?, updated_at=?", + (new_root, bates, seq + 1, created_at), + ) + self._append_event( + "node_added", bates, prev_root=prev_root, + session_root=new_root, + body={ + "bates": bates, "parent_bates": parent_bates, + "question": question, "cache_key": cache_key, + "audit_mode": audit_mode, "node_hash": node_hash, + }, + ) + return self.get_node(bates) # type: ignore[return-value] + + def _recompute_path(self, start_bates: str) -> str: + """Recompute subtree_hash from start_bates up to root. Returns + the (possibly new) root subtree_hash.""" + cur: Optional[str] = start_bates + last_subtree = "" + while cur is not None: + n = self.get_node(cur) + if n is None: + break + child_hashes = [ + c.subtree_hash for c in self.children_of(cur) + ] + sh = _compute_subtree_hash(n.node_hash, child_hashes) + self._conn.execute( + "UPDATE nodes SET subtree_hash=? WHERE bates=?", + (sh, cur), + ) + last_subtree = sh + cur = n.parent_bates + return last_subtree + + def cd(self, bates: str) -> None: + if self.get_node(bates) is None: + raise ValueError(f"node not found: {bates}") + with self._conn: + self._conn.execute( + "UPDATE session_meta SET current_bates=?, updated_at=?", + (bates, _now_iso()), + ) + + def label(self, bates: str, name: str) -> None: + if self.get_node(bates) is None: + raise ValueError(f"node not found: {bates}") + # Labels participate in node_hash on the next recompute — + # however changing a label retroactively would invalidate + # downstream subtree_hashes. Keep labels OUT of node_hash for + # now (they're a UI affordance, not a forensic field). Stored + # in nodes.label but not folded into the hash chain. + with self._conn: + self._conn.execute( + "UPDATE nodes SET label=? WHERE bates=?", (name, bates), + ) + self._conn.execute( + "UPDATE session_meta SET updated_at=?", (_now_iso(),) + ) + + # ---- audit chain ---- + + def _append_event( + self, event_type: str, bates: Optional[str], *, + prev_root: str, session_root: str, body: dict, + ) -> str: + # Chain off the previous event's event_hash, NOT the + # session_root. session_root tells you "what does the tree + # look like now"; event_hash tells you "every state change in + # order is intact." Both are independent integrity signals. + prev_event = self._conn.execute( + "SELECT event_hash FROM session_audit_events " + "ORDER BY seq DESC LIMIT 1" + ).fetchone() + prev_hash_hex = prev_event["event_hash"] if prev_event else "" + full_body = { + "event_type": event_type, + "bates": bates, + "prev_root": prev_root, + "session_root": session_root, + **body, + } + body_canonical = _canonical(full_body) + h = hashlib.sha256() + if prev_hash_hex: + h.update(bytes.fromhex(prev_hash_hex)) + h.update(body_canonical) + event_hash = h.hexdigest() + self._conn.execute( + "INSERT INTO session_audit_events (event_type, bates, " + "prev_root, session_root, body, event_hash, created_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?)", + (event_type, bates, prev_root, session_root, + body_canonical.decode("utf-8"), event_hash, _now_iso()), + ) + return event_hash + + def chain_check(self) -> tuple[int, int]: + """Walk audit events in order, recompute each event_hash, + compare to stored. Returns (intact_count, break_count).""" + intact = 0 + breaks = 0 + prev_hex = "" + for r in self._conn.execute( + "SELECT body, event_hash FROM session_audit_events ORDER BY seq" + ): + body_bytes = r["body"].encode("utf-8", errors="surrogatepass") + h = hashlib.sha256() + if prev_hex: + h.update(bytes.fromhex(prev_hex)) + h.update(body_bytes) + expected = h.hexdigest() + if expected == r["event_hash"]: + intact += 1 + else: + breaks += 1 + prev_hex = r["event_hash"] + return intact, breaks + + def close(self) -> None: + try: + self._conn.close() + except Exception: + pass + + +# ---- top-level helpers (dir-level operations) ---- + +def default_sessions_dir() -> Path: + return Path.home() / ".arborist" / "sessions" + + +def list_sessions(sessions_dir: Optional[Path] = None) -> list[dict]: + """List all session dbs in the directory. Returns one dict per + session with sid, root, n_nodes, updated_at, current_bates.""" + sessions_dir = sessions_dir or default_sessions_dir() + if not sessions_dir.is_dir(): + return [] + out: list[dict] = [] + for p in sorted(sessions_dir.glob("*.db")): + try: + conn = sqlite3.connect(str(p)) + conn.row_factory = sqlite3.Row + m = conn.execute( + "SELECT * FROM session_meta LIMIT 1" + ).fetchone() + if not m: + conn.close() + continue + n_nodes = conn.execute( + "SELECT COUNT(*) AS c FROM nodes" + ).fetchone()["c"] + out.append({ + "sid": m["session_id"], + "path": str(p), + "session_root": m["session_root"], + "current_bates": m["current_bates"], + "n_nodes": n_nodes, + "updated_at": m["updated_at"], + "created_at": m["created_at"], + }) + conn.close() + except Exception: + continue + return out + + +def session_path(sid: str, sessions_dir: Optional[Path] = None) -> Path: + sessions_dir = sessions_dir or default_sessions_dir() + return sessions_dir / f"{sid}.db" + + +def render_tree(sess: Session) -> str: + """Render the session tree as a text outline. + + Synthetic root is collapsed to a header. Current node is marked with + a trailing ← current. + """ + cur = sess.current_bates + sid = sess.sid + n_nodes = len(sess.all_nodes()) - 1 # exclude synthetic root + bps = sess.branches() + lines: list[str] = [ + f"session {sid} ({n_nodes} nodes · {len(bps)} branch points · " + f"root={sess.session_root[:12]}… · current → {cur})", + ] + root = sess.root_bates + + def _walk(bates: str, prefix: str, is_last: bool, is_root: bool) -> None: + n = sess.get_node(bates) + if n is None: + return + if is_root: + # Don't print the synthetic root; jump straight to children. + kids = sess.children_of(bates) + for i, c in enumerate(kids): + _walk(c.bates, "", i == len(kids) - 1, False) + return + marker = "└── " if is_last else "├── " + cur_tag = " ← current" if n.bates == cur else "" + label_tag = f" {{{n.label}}}" if n.label else "" + q = (n.question or "")[:60] + lines.append( + f"{prefix}{marker}[{n.bates}] {q}{label_tag} " + f"[{n.audit_mode}]{cur_tag}" + ) + nxt_prefix = prefix + (" " if is_last else "│ ") + kids = sess.children_of(bates) + for i, c in enumerate(kids): + _walk(c.bates, nxt_prefix, i == len(kids) - 1, False) + + _walk(root, "", True, True) + return "\n".join(lines) diff --git a/docs/sessions.md b/docs/sessions.md new file mode 100644 index 0000000..5a33d4d --- /dev/null +++ b/docs/sessions.md @@ -0,0 +1,205 @@ +# Sessions — Merkle-rooted conversation trees + +Multi-turn Q&A with branching, where the tree itself is Merkle-rooted. +Each turn (or fork) mints a node with a stable Bates id and folds into +a per-session subtree-hash chain; the root node's `subtree_hash` is the +`session_root`, a single hash that pins the entire conversation state. + +## On-disk shape + +One SQLite file per session at `~/.arborist/sessions/<sid>.db`: + +```sql +CREATE TABLE session_meta ( + session_id TEXT PRIMARY KEY, + root_bates TEXT NOT NULL, -- always seq=0 + session_root TEXT NOT NULL, -- subtree_hash of root node + current_bates TEXT NOT NULL, -- REPL "you are here" pointer + next_seq INTEGER NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); + +CREATE TABLE nodes ( + bates TEXT PRIMARY KEY, -- "<sid>-<6-digit>" + seq INTEGER NOT NULL, -- monotonic per session + parent_bates TEXT REFERENCES nodes(bates), + question TEXT NOT NULL, + cache_key TEXT NOT NULL, -- → providence_cache + audit_mode TEXT NOT NULL, + created_at TEXT NOT NULL, + label TEXT, -- optional human name + node_hash TEXT NOT NULL, -- sha256 of canonical body + subtree_hash TEXT NOT NULL -- Merkle hash over self + children +); + +CREATE TABLE session_audit_events ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + event_type TEXT NOT NULL, -- session_init | node_added + bates TEXT, + prev_root TEXT NOT NULL, + session_root TEXT NOT NULL, + body TEXT NOT NULL, -- canonical JSON + event_hash TEXT NOT NULL, -- sha256(prev_hash || body) + created_at TEXT NOT NULL +); +``` + +The session db is **self-contained**: no foreign keys into the main +store. Answers live in `~/.arborist/qa.db:providence_cache` keyed by +`cache_key`; the session only carries the **shape** of the conversation +(parent links, Bates, hashes). Cache hits across sessions stay live. + +## Bates discipline + +`<sid>-<6-digit-seq>`. `sid` is `YYYYMMDD-HHMMSS-<4-hex>`; `seq` is +monotonic per session, never reassigned, never re-used. `seq=0` is a +synthetic root with empty question — collapsed in tree rendering, kept +so all real questions descend from a single node. + +## Merkle hashes + +Two derived hashes per node: + +``` +node_hash = sha256(canonical_json({ + parent_node_hash, bates, question, cache_key, + audit_mode, created_at, label, +})) + +subtree_hash = HashCombine( + leaf_hash(node_hash), + children_root, +) +``` + +Where `children_root` is built bottom-up by `HashCombine` over the +children's `subtree_hash` values, sorted by hash (deterministic), +with the standard arborist odd-element rule (self-duplicate). + +Sorting children by `subtree_hash` (content-addressed) keeps the +combine order independent of insertion timing — two sessions that +land the same nodes in different order produce the same root. + +## Insert path is O(depth) + +When a new node `N` is inserted under parent `P`: + +1. Compute `N.node_hash`. +2. `N.subtree_hash = leaf_hash(N.node_hash)` (new node is a leaf). +3. Walk `N → P → P.parent → … → root` recomputing each ancestor's + `subtree_hash`. +4. Update `session_meta.session_root` to root's new `subtree_hash`. +5. Append one `session_audit_events` row whose `event_hash` chains + off the previous event. + +A sibling subtree that didn't change keeps its `subtree_hash`. This +is the property that makes page-refresh caching cheap. + +## Caching for page refresh + +A UI client tracks `(bates → subtree_hash, body)` locally. On refresh: + +1. Client sends current `session_root`. +2. Server returns either "still current" or the new root + a list of + `subtree_hash` deltas along the changed paths. +3. Client walks top-down, fetching only nodes whose `subtree_hash` + differs from cache. + +Same shape as git's pack-protocol or IPFS MFS: content-addressed +sibling subtrees are byte-identical → cache-equivalent. + +## Audit chain + +`session_audit_events` is a hash-chain mirror of `arborist.audit_events`, +scoped per session. Every state change emits one event whose `event_hash` +is `sha256(prev_event_hash || canonical_body)`. Verify with +`arborist session --chain-check [--sid SID]` (or +`make session-chain-check`). + +`event_hash` and `session_root` are independent integrity signals: +`session_root` answers "what does the tree look like now?"; +`event_hash` answers "is every state change in order intact?". Both +moves on every insert, but for different reasons. + +## Forking + +There is no `/fork` verb. Forking is implicit: `/cd <bates>` moves the +"current" pointer to any node in the tree; the next question becomes a +child of that node. Branch points are nodes with ≥2 children — listed +by `/branches` or in the tree header. + +## Retrieval-side context flow + +When a turn is asked under parent P, the REPL gathers the source titles +the LLM **cited** in P and P's ancestors (parsed from each ancestor's +stored `answer_text` for `[E# | <title> | …]` pointers) and passes them +into `providence_query` via `policy["retrieval_keywords"]`. The keyword +string augments FTS5 retrieval ONLY — it never enters `cache_key`, +`question_hash`, or `governance_policy_hash` (mirrors the discipline of +the legacy `--retrieval-keywords` flag, see ticket #000001). + +This makes pronoun follow-ups ("who created him?") still surface the +prior turn's primary source. **It does not solve the LLM-side antecedent +problem** — the model receives only the new question as the user +message, so a bare pronoun question may still fail to ground. Threading +prior Q&A pairs as `conversation_history` into the LLM prompt (+ folding +that history into `conversation_hash`) is a future enhancement. + +## CLI + +``` +arborist session # new session, REPL +arborist session --sid <id> # resume +arborist session --list [--json] # list sessions +arborist session --tree --sid <id> # print tree +arborist session --chain-check # verify audit chain (all) +arborist session --gc N # keep N most recent +``` + +REPL commands: + +| command | effect | +|---|---| +| `/tree` | print the tree | +| `/branches` | list branch points (Bates with ≥2 children) | +| `/cd <bates\|seq\|label>` | move current pointer | +| `/back` | cd to parent of current | +| `/label <name>` | name current node | +| `/show <ref>` | dump stored info for a node | +| `/sid` | print session id | +| `/root` | print session_root | +| `/help` | help | +| `/quit` | exit (or Ctrl-D) | +| anything else | new question, child of current | + +Make targets: + +``` +make session [SID=...] # interactive REPL +make session-list # list all +make session-tree SID=... # print one tree +make session-chain-check # verify audit chain +make session-gc # keep SESSION_KEEP=100 (default) most recent +``` + +## Bounded growth + +`make session-gc` (or `arborist session --gc N`) keeps the N most-recent +sessions, deletes the rest. Each session is a single SQLite file → +trivially deletable, shareable, archivable. + +## Portability + +The on-disk schema has no FK dependency on the main arborist store. +A non-Python consumer that wants to read a session db needs only: + +1. SQLite3 with FTS5 (not required — sessions don't use FTS). +2. The canonical-JSON convention: sorted keys, no whitespace, UTF-8. +3. Three hash primitives: + - `sha256(bytes) → hex` + - `leaf_hash(bytes) = sha256(0x00 || bytes)` + - `hash_combine(L, R) = sha256(0x03 || L || R)` + +Those are the same conventions as the proxy.unturf.com Go merkle and +the rest of arborist (see `arborist/merkle.py`). diff --git a/tests/test_session.py b/tests/test_session.py new file mode 100644 index 0000000..1f04d91 --- /dev/null +++ b/tests/test_session.py @@ -0,0 +1,211 @@ +"""Tests for arborist.qa.session — Merkle-rooted conversation tree.""" +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +import pytest + +from arborist.qa.session import ( + Session, + list_sessions, + render_tree, + session_path, +) + + +@pytest.fixture +def session_db(tmp_path: Path) -> Path: + return tmp_path / "test-sid.db" + + +def test_open_creates_synthetic_root(session_db: Path): + sess = Session.open(session_db, sid="sid001") + assert sess.sid == "sid001" + assert sess.root_bates == "sid001-000000" + root = sess.get_node("sid001-000000") + assert root is not None + assert root.question == "" + assert root.parent_bates is None + assert root.subtree_hash == sess.session_root + assert sess.current_bates == sess.root_bates + sess.close() + + +def test_open_resumes_existing_session(session_db: Path): + s1 = Session.open(session_db, sid="sid002") + s1.add_node("first question", "cache-key-1", "STRICT") + root_after_1 = s1.session_root + s1.close() + s2 = Session.open(session_db) # no sid → resume + assert s2.sid == "sid002" + assert s2.session_root == root_after_1 + assert s2.current_bates == "sid002-000001" + s2.close() + + +def test_add_node_mints_bates_and_recomputes_root(session_db: Path): + sess = Session.open(session_db, sid="s") + root_root = sess.session_root + n1 = sess.add_node("q1", "ck1", "STRICT") + assert n1.bates == "s-000001" + assert sess.current_bates == "s-000001" + assert sess.session_root != root_root # tree changed → root changed + n2 = sess.add_node("q2", "ck2", "HYBRID") + assert n2.bates == "s-000002" + assert n2.parent_bates == "s-000001" + sess.close() + + +def test_fork_via_cd_creates_sibling(session_db: Path): + sess = Session.open(session_db, sid="s") + n1 = sess.add_node("q1", "ck1", "STRICT") + n2 = sess.add_node("q2", "ck2", "STRICT") + # Fork: cd back to n1, ask q3 — q3 is a sibling of q2 under n1. + sess.cd(n1.bates) + n3 = sess.add_node("q3", "ck3", "STRICT") + assert n3.parent_bates == n1.bates + assert sess.current_bates == n3.bates + kids = sess.children_of(n1.bates) + bates_set = {k.bates for k in kids} + assert n2.bates in bates_set + assert n3.bates in bates_set + sess.close() + + +def test_branches_lists_branch_points(session_db: Path): + sess = Session.open(session_db, sid="s") + n1 = sess.add_node("q1", "ck1", "STRICT") + sess.add_node("q2", "ck2", "STRICT") # child of n1 + sess.cd(n1.bates) + sess.add_node("q3", "ck3", "STRICT") # second child of n1 → branch + bps = sess.branches() + assert n1.bates in bps + assert sess.root_bates not in bps # root only has 1 child so far + sess.close() + + +def test_session_root_changes_per_insert(session_db: Path): + sess = Session.open(session_db, sid="s") + roots = [sess.session_root] + sess.add_node("q1", "ck1", "STRICT") + roots.append(sess.session_root) + sess.add_node("q2", "ck2", "STRICT") + roots.append(sess.session_root) + assert len(set(roots)) == 3 # all distinct + sess.close() + + +def test_session_root_deterministic(tmp_path: Path): + """Two sessions with the same insert sequence produce the same root. + Note: timestamps differ in real wall-clock — we fix them by + passing the same created_at via mocking. Here we just confirm the + HASH formula is deterministic when inputs match. + """ + from arborist.qa.session import _compute_node_hash, _compute_subtree_hash + nh = _compute_node_hash("", "s-000001", "q", "ck", "STRICT", + "2026-06-01T00:00:00Z", None) + sh = _compute_subtree_hash(nh, []) + # Same inputs → same hash. + nh2 = _compute_node_hash("", "s-000001", "q", "ck", "STRICT", + "2026-06-01T00:00:00Z", None) + sh2 = _compute_subtree_hash(nh2, []) + assert nh == nh2 + assert sh == sh2 + + +def test_path_to_root_walks_ancestors(session_db: Path): + sess = Session.open(session_db, sid="s") + n1 = sess.add_node("q1", "ck1", "STRICT") + n2 = sess.add_node("q2", "ck2", "STRICT") + path = sess.path_to_root(n2.bates) + assert [n.bates for n in path] == [n2.bates, n1.bates, sess.root_bates] + sess.close() + + +def test_audit_chain_intact_after_inserts(session_db: Path): + sess = Session.open(session_db, sid="s") + sess.add_node("q1", "ck1", "STRICT") + sess.add_node("q2", "ck2", "STRICT") + sess.add_node("q3", "ck3", "HYBRID") + intact, breaks = sess.chain_check() + assert breaks == 0 + assert intact >= 4 # session_init + 3 node_added + sess.close() + + +def test_audit_chain_detects_tampered_event(session_db: Path): + sess = Session.open(session_db, sid="s") + sess.add_node("q1", "ck1", "STRICT") + sess.add_node("q2", "ck2", "STRICT") + # Tamper with one event's body. + sess.conn.execute( + "UPDATE session_audit_events SET body=? " + "WHERE event_type='node_added' LIMIT 1", + ('{"tampered": true}',), + ) + sess.conn.commit() + intact, breaks = sess.chain_check() + assert breaks >= 1 + sess.close() + + +def test_resolve_accepts_bates_seq_and_label(session_db: Path): + sess = Session.open(session_db, sid="sid") + n1 = sess.add_node("q1", "ck1", "STRICT") + sess.label(n1.bates, "alpha") + assert sess.resolve(n1.bates) == n1.bates + assert sess.resolve("1") == n1.bates + assert sess.resolve("alpha") == n1.bates + assert sess.resolve("nope") is None + sess.close() + + +def test_list_sessions_finds_db_in_dir(tmp_path: Path): + p1 = tmp_path / "s1.db" + p2 = tmp_path / "s2.db" + Session.open(p1, sid="s1").close() + Session.open(p2, sid="s2").close() + listing = list_sessions(tmp_path) + sids = {s["sid"] for s in listing} + assert sids == {"s1", "s2"} + + +def test_render_tree_includes_current_marker(session_db: Path): + sess = Session.open(session_db, sid="s") + sess.add_node("first", "ck1", "STRICT") + out = render_tree(sess) + assert "first" in out + assert "← current" in out + sess.close() + + +def test_subtree_hash_propagates_up(session_db: Path): + """Inserting a deep node changes the root's subtree_hash.""" + sess = Session.open(session_db, sid="s") + n1 = sess.add_node("q1", "ck1", "STRICT") + root_after_1 = sess.session_root + # Add a deep child of n1. + sess.cd(n1.bates) + sess.add_node("q1.child", "ck2", "STRICT") + root_after_2 = sess.session_root + assert root_after_1 != root_after_2 + sess.close() + + +def test_sibling_insert_changes_only_changed_paths(session_db: Path): + """When a node is added under parent P, P's subtree_hash AND root + change, but unrelated siblings' subtree_hash stays the same.""" + sess = Session.open(session_db, sid="s") + a = sess.add_node("a", "ck-a", "STRICT") + sess.cd(sess.root_bates) + b = sess.add_node("b", "ck-b", "STRICT") + a_hash_before = sess.get_node(a.bates).subtree_hash + b_hash_before = sess.get_node(b.bates).subtree_hash + # Add child under a. + sess.cd(a.bates) + sess.add_node("a.child", "ck-ac", "STRICT") + # a's subtree_hash changed; b's didn't. + assert sess.get_node(a.bates).subtree_hash != a_hash_before + assert sess.get_node(b.bates).subtree_hash == b_hash_before + sess.close()