diff --git a/Makefile b/Makefile index a5baead..37db34c 100644 --- a/Makefile +++ b/Makefile @@ -227,6 +227,25 @@ backfill-concepts: bootstrap ## backfill all concept extractors in parallel acro # Quick bench mode — 1 sample, smoke fixture, all 3 modes. ~10s. # For pure smoke after a code change before the longer bench-qa-smoke. +# Emergent stress-test: random word triangulation. Pick 3 words from +# /usr/share/dict/words, ask Hermes @ temp=0.8 to weave them into a +# creative question, send to aborist, append the journey to +# bench/emergent_log.jsonl. Designed for blue-moon cadence — surfaces +# combinatoric failure modes the curated bench-qa fixture set can't. +# Teacher review (Opus) runs separately via `--print-pending`; fox +# brings entries here & gets back guidance to append to the log. +EMERGENT_N ?= 10 +EMERGENT_SEED ?= +bench-emergent: bootstrap ## blue-moon emergent stress test (3-word triangulation; N=10) + PYTHONUNBUFFERED=1 $(PY) scripts/bench_emergent.py \ + --n $(EMERGENT_N) \ + $(if $(EMERGENT_SEED),--seed $(EMERGENT_SEED),) \ + --shards-dir $(SHARDS_DIR) \ + --qa-db $(SHARDS_DIR)/qa.db + +bench-emergent-pending: bootstrap ## print log entries awaiting teacher review + $(PY) scripts/bench_emergent.py --print-pending + bench-qa-quick: bootstrap ## fastest bench (1 sample × 5 questions × 3 modes; ~10s) PYTHONUNBUFFERED=1 $(PY) bench/qa_sweep.py \ --questions bench/qa_questions_smoke.txt \ diff --git a/bench/emergent_log.jsonl b/bench/emergent_log.jsonl new file mode 100644 index 0000000..bcd9fc3 --- /dev/null +++ b/bench/emergent_log.jsonl @@ -0,0 +1,2 @@ +{"ts": 1777740048, "iso_ts": "2026-05-02T16:40:48Z", "words": ["brainier", "Goldman", "jealously"], "question": "Could the remarkable intellect of a brainier individual, possibly even surpassing that of the renowned investment banker Goldman, inadvertently lead to feelings of jealously among those who have not yet achieved such academic and professional heights?", "question_gen_seconds": 1.01, "answer": "{\"claims\":[{\"text\":\"A brainier individual surpassing the intellect of a renowned investment banker like Goldman could potentially lead to feelings of jealousy among those who have not yet achieved similar academic and professional heights.\",\"evidence_ids\":[\"E1\",\"E2\"]}]}", "audit_mode": "UNGROUNDED", "verifier_method": "claim_lattice", "n_quotes": 2, "n_verified": 0, "violation_kinds": ["CITATION_MISMATCH"], "sources": [{"title": "Virtue jurisprudence", "uri": "https://en.wikipedia.org/wiki/Virtue_jurisprudence", "used": false}, {"title": "Dirty War", "uri": "https://en.wikipedia.org/wiki/Dirty_War", "used": false}, {"title": "Salvation", "uri": "https://en.wikipedia.org/wiki/Salvation", "used": false}, {"title": "The Lord of the Rings film trilogy", "uri": "https://en.wikipedia.org/wiki/The_Lord_of_the_Rings_film_trilogy", "used": false}, {"title": "Central Philippine University", "uri": "https://en.wikipedia.org/wiki/Central_Philippine_University", "used": false}, {"title": "Koreans in Japan", "uri": "https://en.wikipedia.org/wiki/Koreans_in_Japan", "used": false}, {"title": "Korean Buddhist sculpture", "uri": "https://en.wikipedia.org/wiki/Korean_Buddhist_sculpture", "used": false}, {"title": "Jihad", "uri": "https://en.wikipedia.org/wiki/Jihad", "used": false}], "cache_key": "31860906e6424914f2f98dc515b189c42d4dce8e140245e691c281e73f09b707", "answer_mode": "claim_lattice", "answer_seconds": 25.13, "total_seconds": 26.14, "student_error": null, "teacher": null} +{"ts": 1777740074, "iso_ts": "2026-05-02T16:41:14Z", "words": ["hastes", "freelanced", "coincident"], "question": "How do these three seemingly unrelated concepts - hastes, freelanced, and coincident - unexpectedly intertwine in the context of a time-crunched artist rushing to complete a project, resulting in a unique and unanticipated outcome?", "question_gen_seconds": 0.71, "answer": "{\"claims\":[{\"text\":\"In a time-crunched artist rushing to complete a project, hastes can lead to freelanced, coincident elements unexpectedly intertwining, resulting in a unique and unanticipated outcome.\",\"evidence_ids\":[\"E1\",\"E2\",\"E3\"]}]}", "audit_mode": "UNGROUNDED", "verifier_method": "claim_lattice", "n_quotes": 3, "n_verified": 0, "violation_kinds": ["CITATION_MISMATCH", "TOO_MANY_EVIDENCE_IDS"], "sources": [{"title": "Time reversibility", "uri": "https://en.wikipedia.org/wiki/Time_reversibility", "used": false}, {"title": "Wall clock time", "uri": "https://en.wikipedia.org/wiki/Wall_clock_time", "used": false}, {"title": "Ordinary least squares", "uri": "https://en.wikipedia.org/wiki/Ordinary_least_squares", "used": false}, {"title": "Bernheim Arboretum and Research Forest", "uri": "https://en.wikipedia.org/wiki/Bernheim_Arboretum_and_Research_Forest", "used": false}, {"title": "Dirac large numbers hypothesis", "uri": "https://en.wikipedia.org/wiki/Dirac_large_numbers_hypothesis", "used": false}, {"title": "1000000000 (number)", "uri": "https://en.wikipedia.org/wiki/1000000000_(number)", "used": false}, {"title": "José Antonio Conde", "uri": "https://en.wikipedia.org/wiki/José_Antonio_Conde", "used": false}, {"title": "L'étoile (opera)", "uri": "https://en.wikipedia.org/wiki/L'étoile_(opera)", "used": false}], "cache_key": "3aca45f06eea324602f28f120e6ca353c49107082e28c4392e85904c8ef735b8", "answer_mode": "claim_lattice", "answer_seconds": 13.51, "total_seconds": 14.23, "student_error": null, "teacher": null} diff --git a/docs/bench-emergent-design.md b/docs/bench-emergent-design.md new file mode 100644 index 0000000..02c5328 --- /dev/null +++ b/docs/bench-emergent-design.md @@ -0,0 +1,205 @@ +# Bench-emergent: random-word triangulation stress test + +**Status:** landed — `scripts/bench_emergent.py`, 2026-05-02 +**Cadence:** blue-moon — runs on demand, NOT every commit +**Audience:** anyone hunting for failure modes the curated +`bench/qa_questions.txt` doesn't surface + +## Why + +`bench/qa_questions.txt` is 71 hand-curated questions that pin +known-good answers + known-failure shapes. It tells us whether the +substrate handles the failure shapes we already named. It can't +tell us what we *haven't* named. + +The curated bench is a scoreboard. This is a sonar — random pings +into the combinatoric space of "what happens when somebody asks +something we never thought to ask." The 2010-11 Wikipedia corpus +is fixed; the question space is infinite; emergent surfacing is +how we find what we don't know. + +## Loop shape + +``` + /usr/share/dict/words + │ random.sample(3) + ▼ + ╔════════════════════╗ + ║ generator: Hermes ║ temp=0.8 + ║ "weave 3 words → ║ creative paragraph + ║ question paragraph║ with question framing + ╚════════════════════╝ + │ + ▼ + ╔════════════════════╗ + ║ student: aborist ║ full retrieval pipeline, + ║ query() against ║ claim_lattice mode, + ║ live shards ║ burn=True (always fresh) + ╚════════════════════╝ + │ + ▼ + ┌────────────────────┐ + │ append to │ one JSONL line per cycle + │ bench/emergent_ │ fields: words, question, + │ log.jsonl │ answer, audit_mode, sources, + │ │ timings, teacher: null + └────────────────────┘ + │ + │ (teacher review — separate, manual) + ▼ + ╔════════════════════╗ + ║ teacher: Opus ║ read pending entries, + ║ (Claude 4.7, this ║ judge match / novelty / + ║ conversation) ║ bench-max signal, + ║ ║ append `teacher: {...}` + ╚════════════════════╝ +``` + +## Why "blue moon" + +- Wall-clock cost: ~20s per cycle (Hermes generator @ 5-10s + + aborist student @ 5-15s). N=10 ≈ 4 min, N=50 ≈ 17 min. +- Most cycles are UNGROUNDED-by-corpus-design (random word triplets + rarely overlap with 2010 Wikipedia coverage). The interesting + cases are the rare HYBRID/STRICT verdicts on triplets that + surprise us, plus the verifier-disagreement cases the teacher + catches. +- Curated bench (`make bench-qa{,-quick,-smoke}`) is the every- + iteration signal. This is the once-a-month sonar. + +## Word filter + +`scripts/bench_emergent.py` uses `^[a-z]{5,12}$` after lowercasing +to admit a word. Skips: + +- length ≤ 4 (short words like "the", "and" produce trivially + vague questions) +- length > 12 (rare scientific terms / loanwords; Hermes struggles + to weave them) +- non-alpha (apostrophes, hyphens — the unix words file mixes + these in) +- ALLCAPS (filtered after the lowercasing step admits "Goldman" → + "goldman" as a generic noun, which is fine; Hermes treats it + as a name regardless) + +Override the filter or word source via: + +```bash +python scripts/bench_emergent.py --words-path /custom/words.txt +``` + +## Teacher review protocol + +The bench script does NOT auto-invoke a teacher. Two reasons: + +1. **Separation of concerns**: generation is automated, judgment + is contextual. The teacher (Opus, currently) needs the loop's + raw output PLUS access to the corpus-knowledge frame ("is this + a 2010 Wikipedia gap or a substrate failure?"). That frame + lives in this repo's docs, not in a per-call API spec. + +2. **Future flexibility**: today the teacher is Claude Opus 4.7 + in this conversation. Tomorrow it might be GPT-5, Claude 5, + or a Mixture-of-Experts review committee. Keeping teacher + review out of the bench script means swapping teachers is a + workflow change, not a code change. + +### How fox brings entries to the teacher + +```bash +make bench-emergent-pending # print every entry with teacher==null +# … pipe into a Claude session, paste, ask for judgment +``` + +The teacher's output is a JSON dict to append to the same line: + +```json +{ + "teacher": { + "match": true|false, + "audit_agreement": "agree"|"disagree"|"unsure", + "novelty_class": "known_truth_grounding" + | "emergent_synthesis" + | "novel_claim" + | "no_signal", + "score_0_5": 0..5, + "reasoning": "", + "bench_max_signal": "", + "reviewed_by": "claude-opus-4-7[1m]", + "reviewed_ts": + } +} +``` + +The `bench_max_signal` field is the actionable bit: which subsystem +should be tuned to address this kind of failure? `retrieval` / +`warrant` / `prompt` / `nil` (no action — corpus genuinely lacks +the answer). + +### Fields the teacher considers + +- **match**: did the answer address the question? Not "is the + answer correct" — the corpus may legitimately not have the + answer. The check is "is this answer about the same topic as + the question?" +- **audit_agreement**: does the substrate's audit_mode (STRICT/ + HYBRID/UNGROUNDED) or display rung (POINTER-LINKED → + ANCHOR-WARRANTED → EVIDENCE-WARRANTED) match what the teacher + thinks the answer's grounding deserves? +- **novelty_class**: how does the answer relate to 2010 Wikipedia + knowledge? + - `known_truth_grounding`: cites well-known facts present + verbatim in the corpus. The expected case for narrow + factoids ("capital of France", "founder of Microsoft"). + - `emergent_synthesis`: the answer connects facts in a way no + single source contains; the substrate did real work + composing across sources. The interesting case. + - `novel_claim`: the answer goes beyond what the 2010 corpus + can ground (post-2010 science, personal opinion, made-up + detail). The dangerous case if STRICT/EVIDENCE-WARRANTED. + - `no_signal`: UNGROUNDED, or the question was incoherent. + Most random-word triplets land here — that's fine. + +## Future: multiple upstreams + +When the substrate supports multiple inference endpoints, the +generator and student can use different upstreams to surface +upstream-specific failure modes. The CLI flags for this aren't +implemented yet but the shape is clear: + +```bash +python scripts/bench_emergent.py \ + --generator-endpoint https://hermes.ai.unturf.com/v1 \ + --student-endpoint https://other-llm.example/v1 +``` + +Today both default to Hermes. The student is always aborist's +`query()` against the configured shard set; only the LLM behind +that pipeline is plugged. + +## How to run + +```bash +make bench-emergent # 10 cycles, default seed (random) +make bench-emergent EMERGENT_N=50 EMERGENT_SEED=42 # bigger sample, reproducible +make bench-emergent-pending # print pending teacher review + +# Direct: +python scripts/bench_emergent.py --n 50 --seed 42 +python scripts/bench_emergent.py --print-pending +``` + +## Append-only log + +`bench/emergent_log.jsonl` accumulates every cycle ever run, +across branches, across days. Never rewritten. Each line is one +self-describing JSON record (timestamp, words, question, answer, +audit, sources, timings, teacher). The log is the substrate's +own version of `/var/log/syslog` — every interaction with the +emergent harness leaves a trace, and a future you can grep for +"every entry where the answer mentioned X" or "every UNGROUNDED +result on triplets containing Y." + +A line whose `teacher` field is `null` is awaiting review. A line +with a populated `teacher` field is closed. There is no DELETE +path; corrections add new lines that supersede old ones. diff --git a/scripts/bench_emergent.py b/scripts/bench_emergent.py new file mode 100644 index 0000000..a434b94 --- /dev/null +++ b/scripts/bench_emergent.py @@ -0,0 +1,310 @@ +"""Emergent stress test — random word triangulation against the substrate. + +Pick three random words from /usr/share/dict/words, ask Hermes at +temp 0.8 to weave them into a creative question paragraph, send that +paragraph to aborist, log the whole journey (words → question → +answer → audit_mode → sources → timings) as a single JSONL line. + +Designed for **blue moon cadence**, NOT every-commit benching. The +combinatoric word space surfaces failure modes the curated bench +(`bench/qa_questions.txt`) doesn't reach: question shapes the +authors didn't anticipate, vocabulary the corpus barely covers, +adversarial premises that emerge by accident. + +The teacher review step is intentionally **not** automated here. +Fox brings interesting log entries to a teacher model (Claude +Opus 4.7 in the current setup) and asks for guidance: + + - did the answer match the question? (mismatch → bench-max signal) + - novelty class: known_truth_grounding / emergent_synthesis / + novel_claim / no_signal (accounting for the 2010-11 Wikipedia + corpus legitimately lacking post-2010 advances in science / + math / engineering) + - which hyperparam to tune for the next iteration + +Future: `--generator-endpoint` & `--student-endpoint` to swap +upstreams (different model per role), and a separate review-side +script that prompts a teacher model via API. + +Usage: + make bench-emergent # 10 cycles, default + make bench-emergent N=50 SEED=42 # bigger sample, reproducible + python scripts/bench_emergent.py --n 10 --seed 42 + python scripts/bench_emergent.py --print-pending # show un-reviewed entries +""" + +from __future__ import annotations + +import argparse +import json +import os +import random +import re +import sys +import time +from pathlib import Path + +# All paths default to the repo's bench/ tree so re-running across +# branches accumulates a continuous log of every emergent cycle. +DEFAULT_WORDS_PATH = Path("/usr/share/dict/words") +DEFAULT_LOG_PATH = Path("bench/emergent_log.jsonl") + +# Word filter: skip too-short / proper-noun / abbreviation / weird +# punctuation. The goal is "common English content tokens" that +# Hermes can actually weave into a coherent paragraph. +_WORD_RE = re.compile(r"^[a-z]{5,12}$") + + +def pick_words( + n: int = 3, + *, + words_path: Path = DEFAULT_WORDS_PATH, + rng: random.Random | None = None, +) -> list[str]: + """Sample N random words satisfying the content-token filter.""" + rng = rng or random.Random() + pool = [ + w for w in words_path.read_text().splitlines() + if _WORD_RE.fullmatch(w.strip().lower()) + ] + if len(pool) < n: + raise RuntimeError( + f"word pool too small ({len(pool)} usable words at " + f"{words_path}); expected ≥{n}" + ) + return rng.sample(pool, n) + + +GENERATOR_PROMPT = """\ +Write a single short paragraph (2-3 sentences) that uses ALL THREE \ +of these words: + + - {w1} + - {w2} + - {w3} + +Then phrase the paragraph as a question that explores the connections \ +between them. The question should be the kind a curious reader would \ +genuinely ask about how these three concepts relate. + +Output ONLY the question paragraph. No prefix, no commentary, no \ +quotes around it, no numbering. Just the question.\ +""" + + +def generate_question( + client, + model_id: str, + words: list[str], + *, + temperature: float = 0.8, + max_tokens: int = 256, +) -> str: + """Hermes at creative temperature weaves the 3 words into a question.""" + prompt = GENERATOR_PROMPT.format(w1=words[0], w2=words[1], w3=words[2]) + response = client.chat_completion( + messages=[{"role": "user", "content": prompt}], + model=model_id, + temperature=temperature, + max_tokens=max_tokens, + ) + text = response.strip() if isinstance(response, str) else str(response).strip() + # Strip wrapping quotes / leading "Question:" labels in case the + # model ignored the no-prefix instruction. + text = text.removeprefix("Question:").strip() + if text.startswith('"') and text.endswith('"'): + text = text[1:-1] + return text + + +def run_one_cycle( + *, + words: list[str], + client, + model_id: str, + qa_db: Path, + shards_dir: Path, + answer_mode: str, + top_k: int, +) -> dict: + """One emergent cycle: words → question → answer → log entry dict.""" + from aborist.qa.query import DEFAULT_QUERY_POLICY, query + + t_pick = time.time() + question = generate_question(client, model_id, words) + t_question = time.time() + + policy = dict(DEFAULT_QUERY_POLICY) + policy["answer_mode"] = answer_mode + + try: + result = query( + question=question, + qa_db=qa_db, + chat_client=client, + model_id=model_id, + shards_dir=shards_dir, + top_k=top_k, + policy=policy, + burn_existing=True, # always fresh — this is an emergent test + ) + student_error = None + except Exception as e: + result = {} + student_error = f"{type(e).__name__}: {e}" + + t_answer = time.time() + + return { + "ts": int(t_pick), + "iso_ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(t_pick)), + "words": words, + "question": question, + "question_gen_seconds": round(t_question - t_pick, 2), + "answer": result.get("answer_text"), + "audit_mode": result.get("audit_mode"), + "verifier_method": result.get("verifier_method"), + "n_quotes": result.get("n_quotes"), + "n_verified": result.get("n_verified"), + "violation_kinds": sorted( + {v.get("kind") for v in (result.get("violations") or []) if v.get("kind")} + ), + "sources": [ + {"title": s.get("title"), "uri": s.get("document_uri"), "used": s.get("used")} + for s in (result.get("sources") or []) + ], + "cache_key": result.get("cache_key"), + "answer_mode": answer_mode, + "answer_seconds": round(t_answer - t_question, 2), + "total_seconds": round(t_answer - t_pick, 2), + "student_error": student_error, + # Teacher review fields — left null on generation, populated + # by a separate review pass (fox brings entries to Opus, gets + # back an entry to append to the same log). + "teacher": None, + } + + +def append_log(entry: dict, log_path: Path) -> None: + log_path.parent.mkdir(parents=True, exist_ok=True) + with log_path.open("a") as f: + f.write(json.dumps(entry, ensure_ascii=False) + "\n") + + +def print_pending(log_path: Path) -> int: + """Print every log entry with `teacher: None` — what fox should + bring to a teacher model for review.""" + if not log_path.exists(): + print(f"no log at {log_path}") + return 0 + pending = 0 + for line in log_path.read_text().splitlines(): + if not line.strip(): + continue + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + if entry.get("teacher") is None: + pending += 1 + print(json.dumps(entry, indent=2)) + print() # blank line between entries + print(f"\n>> {pending} entry/entries awaiting teacher review", file=sys.stderr) + return pending + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.strip().splitlines()[0]) + parser.add_argument("--n", type=int, default=10, help="cycles to run") + parser.add_argument("--seed", type=int, default=None, help="RNG seed for reproducibility") + parser.add_argument( + "--words-path", + type=Path, + default=DEFAULT_WORDS_PATH, + help="path to a unix-style words file", + ) + parser.add_argument("--log-path", type=Path, default=DEFAULT_LOG_PATH) + parser.add_argument( + "--shards-dir", + type=Path, + default=Path.home() / ".aborist" / "shards", + ) + parser.add_argument( + "--qa-db", + type=Path, + default=Path.home() / ".aborist" / "shards" / "qa.db", + ) + parser.add_argument("--answer-mode", default="claim_lattice") + parser.add_argument("--top-k", type=int, default=8) + parser.add_argument( + "--endpoint", + default=os.environ.get("ABORIST_LLM_ENDPOINT", "https://hermes.ai.unturf.com/v1"), + ) + parser.add_argument( + "--model", + default=os.environ.get( + "ABORIST_LLM_MODEL", + "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic", + ), + ) + parser.add_argument( + "--print-pending", + action="store_true", + help="print log entries with teacher==null & exit", + ) + ns = parser.parse_args(argv) + + if ns.print_pending: + return 0 if print_pending(ns.log_path) >= 0 else 1 + + if not ns.words_path.exists(): + parser.error(f"words file not found: {ns.words_path}") + if not ns.shards_dir.exists(): + parser.error(f"shards dir not found: {ns.shards_dir}") + + rng = random.Random(ns.seed) + from aborist.qa.client import OpenAICompatibleClient + + api_key = os.environ.get("ABORIST_LLM_API_KEY") + client = OpenAICompatibleClient(base_url=ns.endpoint, api_key=api_key) + + print(f">> emergent bench: {ns.n} cycles · seed={ns.seed} · log={ns.log_path}", flush=True) + for i in range(ns.n): + words = pick_words(3, words_path=ns.words_path, rng=rng) + print(f"[{i+1}/{ns.n}] words={words}", flush=True) + entry = run_one_cycle( + words=words, + client=client, + model_id=ns.model, + qa_db=ns.qa_db, + shards_dir=ns.shards_dir, + answer_mode=ns.answer_mode, + top_k=ns.top_k, + ) + append_log(entry, ns.log_path) + # Compact stdout summary so a long sweep is observable. + audit = entry.get("audit_mode") or "?" + ratio = ( + f"{entry['n_verified']}/{entry['n_quotes']}" + if entry.get("n_quotes") is not None else "?/?" + ) + err = entry.get("student_error") + if err: + print(f" → ERROR: {err}", flush=True) + else: + print( + f" → {audit} {ratio} {entry['total_seconds']}s " + f"q={entry['question'][:60]!r}", + flush=True, + ) + + print(f"\n>> done. log: {ns.log_path}") + print( + f">> review: bring un-reviewed entries to Opus via\n" + f" python scripts/bench_emergent.py --print-pending" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main())