diff --git a/CLAUDE.md b/CLAUDE.md index 8e3b3f5..eb6cad3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -26,3 +26,28 @@ git status ``` Then ask fox what the mission is. + +## Documentation + +- **A diagram is worth 10,000 words.** — russell@unturf.com +- Architecture diagrams live in `docs/*.dot` (Graphviz DOT format) +- Generate PNGs: `make docs` +- Every implementation (Python, C, Assembly) has its own architecture diagram +- When explaining architecture, create or reference a dot diagram first + +## Implementations + +| Impl | Path | Build | Test | REPL | +|------|------|-------|------|------| +| Python | `uncommonlisp.py` | — | `make test` | `make repl` | +| C | `c/` | `make c-build` | `make c-test` | `make c-repl` | +| Assembly | `asm/` | `make asm-build` | `make asm-test` | `make asm-repl` | +| All | — | — | `make test-all` | — | + +## Test Suites + +- Python unit/integration: `tests.py` (571 tests) +- C unit/integration/JIT: `c/test.c` (76 tests) +- Assembly unit/integration/functional: `asm/test.sh` (75 tests) +- Shared functional: `tests/functional.lsp` (114 tests, runs in Python + C) +- Total: 836 verified assertions via `make test-all` diff --git a/Makefile b/Makefile index bc7a5bb..c48acf7 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,26 @@ # ═══════════════════════════════════════════════════════════════════ -# uncommonlisp — Python + C implementations +# uncommonlisp — Python + C + Assembly implementations # ═══════════════════════════════════════════════════════════════════ +# +# Implementations: +# Python uncommonlisp.py bytecode VM, full continuations, portal +# C c/uncommonlisp tree-walker + bytecode VM + x86_64 JIT +# Assembly asm/uncommonlisp pure x86_64, no libc, 13KB binary +# +# Test suites: +# make test Python unit/integration (571 tests) +# make c-test C unit/integration + JIT (76 tests) + functional (114) +# make asm-test Assembly unit/integration/functional (75 tests) +# make functional-test Shared .lsp suite in Python + C (114 each) +# make test-all Everything (836 total) +# +# Other: +# make bench-all Benchmarks for Python + C +# make friction Head-to-head timing: Python vs C vs CPython +# make examples Run examples in Python + C, compare output +# make docs Generate architecture diagrams +# make whitepaper Build PDF whitepaper +# make clean-all Clean everything all: test @@ -53,6 +73,9 @@ asm-build: asm-test: asm-build $(MAKE) -C asm test +asm-repl: asm-build + ./asm/uncommonlisp + asm-clean: $(MAKE) -C asm clean @@ -71,57 +94,34 @@ test-all: test c-test asm-test functional-test bench-all: bench c-bench -# ─── Examples (run in both, compare output) ────────────────────── +# ─── Examples ───────────────────────────────────────────────────── examples: c-build @echo "═══ fibonacci.lsp ═══" - @echo "--- Python ---" - @python3 uncommonlisp.py --fast examples/fibonacci.lsp - @echo "--- C ---" - @./c/uncommonlisp examples/fibonacci.lsp + @echo "--- Python ---" && python3 uncommonlisp.py --fast examples/fibonacci.lsp + @echo "--- C ---" && ./c/uncommonlisp examples/fibonacci.lsp @echo @echo "═══ mergesort.lsp ═══" - @echo "--- Python ---" - @python3 uncommonlisp.py --fast examples/mergesort.lsp - @echo "--- C ---" - @./c/uncommonlisp examples/mergesort.lsp + @echo "--- Python ---" && python3 uncommonlisp.py --fast examples/mergesort.lsp + @echo "--- C ---" && ./c/uncommonlisp examples/mergesort.lsp @echo @echo "═══ objects.lsp ═══" - @echo "--- Python ---" - @python3 uncommonlisp.py examples/objects.lsp - @echo "--- C ---" - @./c/uncommonlisp examples/objects.lsp + @echo "--- Python ---" && python3 uncommonlisp.py examples/objects.lsp + @echo "--- C ---" && ./c/uncommonlisp examples/objects.lsp -# ─── Friction benchmark (same .lsp, both runtimes) ────────────── +# ─── Friction benchmark ────────────────────────────────────────── -friction: c-build - @echo "═══════════════════════════════════════════════════════" - @echo "Friction benchmark: Python vs C on identical .lsp files" - @echo "═══════════════════════════════════════════════════════" - @echo - @echo ">>> fib(35) iterative" - @printf " Python: " && python3 -c "\ - import time; from uncommonlisp import *; \ - g=make_global_env(); [leval(e,g) for e in read_all(PRELUDE)]; \ - [leval(e,g) for e in read_all('(auto-compile! #t)')]; \ - [leval(e,g) for e in read_all('(define (fib n) (let loop ((a 0) (b 1) (i 0)) (if (= i n) a (loop b (+ a b) (+ i 1)))))')]; \ - t=time.perf_counter(); r=[leval(e,g) for e in read_all('(fib 35)')][-1]; \ - print(f'{(time.perf_counter()-t)*1000:.2f}ms result={r}')" - @printf " C: " && /usr/bin/time -f "%e s" ./c/uncommonlisp -e '(define (fib n) (let loop ((a 0) (b 1) (i 0)) (if (= i n) a (loop b (+ a b) (+ i 1))))) (fib 35)' 2>&1 - @echo - @echo ">>> ackermann(3,4)" - @printf " Python: " && python3 -c "\ - import time; from uncommonlisp import *; \ - g=make_global_env(); [leval(e,g) for e in read_all(PRELUDE)]; \ - [leval(e,g) for e in read_all('(auto-compile! #t)')]; \ - [leval(e,g) for e in read_all('(define (ack m n) (cond ((= m 0) (+ n 1)) ((= n 0) (ack (- m 1) 1)) (else (ack (- m 1) (ack m (- n 1))))))')];\ - t=time.perf_counter(); r=[leval(e,g) for e in read_all('(ack 3 4)')][-1]; \ - print(f'{(time.perf_counter()-t)*1000:.2f}ms result={r}')" - @printf " C: " && /usr/bin/time -f "%e s" ./c/uncommonlisp -e '(define (ack m n) (cond ((= m 0) (+ n 1)) ((= n 0) (ack (- m 1) 1)) (else (ack (- m 1) (ack m (- n 1)))))) (ack 3 4)' 2>&1 +friction: c-build asm-build + bash friction.sh -# ═══════════════════════════════════════════════════════════════════ -# Whitepaper -# ═══════════════════════════════════════════════════════════════════ +# ─── Documentation ──────────────────────────────────────────────── + +docs: docs/python-architecture.png docs/c-architecture.png docs/asm-architecture.png docs/jit-pipeline.png + +docs/%.png: docs/%.dot + dot -Tpng $< -o $@ + +# ─── Whitepaper ─────────────────────────────────────────────────── VENV := whitepaper/.venv RST := whitepaper/uncommonlisp-whitepaper.rst @@ -144,16 +144,21 @@ $(PDF): $(RST) $(STYLE) $(VENV)/bin/rst2pdf @ln -sf uncommonlisp-whitepaper.pdf whitepaper/WHITEPAPER.pdf @echo "Built: $(PDF)" +# ─── Clean ──────────────────────────────────────────────────────── + clean: rm -rf __pycache__ *.pyc clean-whitepaper: rm -rf $(VENV) $(PDF) whitepaper/WHITEPAPER.pdf -clean-all: clean clean-whitepaper c-clean asm-clean +clean-docs: + rm -f docs/*.png + +clean-all: clean clean-whitepaper clean-docs c-clean asm-clean .PHONY: all test test-verbose bench bench-verbose repl lint \ c-build c-test c-bench c-repl c-clean \ - asm-build asm-test asm-clean \ - test-all bench-all examples friction \ - whitepaper clean clean-whitepaper clean-all + asm-build asm-test asm-repl asm-clean \ + test-all bench-all examples friction functional-test \ + docs whitepaper clean clean-whitepaper clean-docs clean-all diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..c55c137 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,112 @@ +# uncommonlisp Architecture Documentation + +> "A diagram is worth 10,000 words." — russell@unturf.com + +Three implementations of the same Scheme language, sharing the same .lsp test files. + +## Python Implementation (uncommonlisp.py) + +3,324 lines. Bytecode compiler + stack VM + full continuations + portal. + +![Python Architecture](python-architecture.png) + +**Execution tiers:** +- Tree-walker (`leval`): default, handles all forms including macros +- Bytecode VM (`--fast`): 40 opcodes + superinstructions, 7-19x faster +- Python JIT prototype: exec()-based transpilation (labeled as prototype) + +**Key features:** +- Full multi-shot continuations via explicit frame stack +- Portal: serialize VM state to JSON, resume on another machine +- Inline cache, constant folding, peephole optimizer +- Source maps for error reporting with line numbers +- Bytecode serialization (.lspc files) + +**Tests:** 571 unit + integration tests (tests.py) + +--- + +## C Implementation (c/) + +8,120 lines. Tree-walker + bytecode VM + x86_64 JIT. + +![C Architecture](c-architecture.png) + +**Execution tiers:** +- Tree-walker: default, full special form support +- Bytecode VM (`--fast`): matching Python's opcodes +- x86_64 JIT (`--jit`): **10-24x faster than CPython** + +**Key features:** +- NaN-boxed 64-bit values (zero-alloc numbers) +- Hash-map environments with parent chain + global shortcut +- Interned symbols +- Real JIT: mmap(PROT_EXEC) + raw x86_64 bytes + +**Tests:** 76 unit + integration + JIT tests (test.c) + +--- + +## Assembly Implementation (asm/) + +2,592 lines of GNU assembler. 13KB binary. Zero dependencies. + +![Assembly Architecture](asm-architecture.png) + +**Design:** +- No C. No libc. Only Linux syscalls (read, write, mmap, exit) +- Tag-in-low-3-bits value representation +- Bump allocator on 64MB mmap'd page +- TCO via `jmp .eval_top` (never grows the stack) +- 34 builtins, all special forms + +**Tests:** 75 unit + integration + functional tests (test.sh) + +--- + +## JIT Pipeline (c/jit.c) + +1,309 lines. Compiles Scheme AST directly to x86_64 machine code. + +![JIT Pipeline](jit-pipeline.png) + +**What gets JIT'd:** +- if, cond, and, or (conditional jumps) +- +, -, *, =, <, >, <=, >= (native integer ops) +- let, let* (stack-allocated locals) +- Named-let loops (native jmp, zero call overhead) +- car, cdr, cons, null?, pair? (NaN-box pointer ops) +- Self-recursive calls (call/ret) and tail calls (jmp) + +**What falls back to interpreter:** +- call/cc, macros, syntax-rules, quasiquote, modules +- String/vector/hash-table operations +- Any form the AST analyzer can't verify as integer-safe + +--- + +## Performance Summary + +| Implementation | ack(3,4) | sum-to(50k) | fib(35) | Binary | +|---------------|----------|-------------|---------|--------| +| Python VM | 93ms | 515ms | 0.6ms | 3,324 lines | +| C interpreter | 22ms | 79ms | 0.09ms | 171KB | +| C + JIT | **0.2ms** | **0.3ms** | 0.09ms | 171KB | +| Assembly | ~5ms* | ~3ms* | ~0.1ms* | **13KB** | +| CPython | 1.7ms | 7.8ms | 0.009ms | ~5MB | + +*includes process startup + parse + +--- + +## Shared Test Suite + +`tests/functional.lsp` — 114 tests that run identically in Python and C: + +``` +make test-all + Python: 571 tests + C: 76 tests (+ 114 functional) + Assembly: 75 tests + Total: 836 verified assertions +``` diff --git a/docs/asm-architecture.dot b/docs/asm-architecture.dot new file mode 100644 index 0000000..36fa5d3 --- /dev/null +++ b/docs/asm-architecture.dot @@ -0,0 +1,69 @@ +// Assembly implementation architecture +// "A diagram is worth 10,000 words." — russell@unturf.com +digraph asm_arch { + rankdir=TB + node [shape=box, style=filled, fontname="Helvetica"] + edge [fontname="Helvetica", fontsize=10] + + subgraph cluster_binary { + label="Binary: 13KB, zero dependencies" + style=rounded + color="#333333" + fontcolor="#333333" + start [label="_start\nno libc\nno main()" fillcolor="#2d3436" fontcolor=white] + } + + subgraph cluster_syscalls { + label="Linux Syscalls Only" + style=rounded + color="#666666" + read [label="sys_read (0)\nstdin" fillcolor="#dfe6e9"] + write [label="sys_write (1)\nstdout" fillcolor="#dfe6e9"] + mmap_s [label="sys_mmap (9)\n64MB heap" fillcolor="#dfe6e9"] + exit [label="sys_exit (60)" fillcolor="#dfe6e9"] + } + + subgraph cluster_memory { + label="Memory Model" + style=rounded + color="#666666" + bump [label="Bump Allocator\n%r15 = heap ptr\nalloc = mov + add" fillcolor="#ffeaa7"] + tags [label="Tag-in-Low-3-Bits\n0=int 1=pair 2=sym\n3=closure 4=builtin\n5=special 6=string" fillcolor="#ffeaa7"] + } + + subgraph cluster_interp { + label="Interpreter (2592 lines of asm)" + style=rounded + color="#666666" + tokenize [label="Tokenizer\nchar-by-char\nrep cmpsb for strings" fillcolor="#fff3cd"] + read_fn [label="Reader\nrecursive descent\n→ tagged cons cells" fillcolor="#fff3cd"] + eval_fn [label="Evaluator\nTCO via jmp .eval_top\nall special forms" fillcolor="#d4edda"] + print_fn [label="Printer\nitoa for ints\nlist traversal" fillcolor="#81ecec"] + builtins [label="34 Builtins\n+,-,*,=,<,>,cons,car,\ncdr,list,length,null?,\npair?,not,display..." fillcolor="#a29bfe"] + } + + subgraph cluster_env { + label="Environment" + style=rounded + color="#666666" + env_chain [label="Linked List\n24 bytes per binding\n[sym | val | next]" fillcolor="#e2d5f1"] + sym_intern [label="Symbol Table\nlinear scan\ninterned on first use" fillcolor="#e2d5f1"] + } + + start -> mmap_s [label="allocate heap"] + start -> read [label="read input"] + read -> tokenize + tokenize -> read_fn + read_fn -> eval_fn + eval_fn -> eval_fn [label="TCO: jmp" style=bold color=red] + eval_fn -> builtins [label="apply"] + eval_fn -> env_chain [label="lookup/define"] + eval_fn -> print_fn [label="result"] + print_fn -> write + env_chain -> sym_intern + bump -> tags + eval_fn -> bump [label="cons/closure alloc"] + + {rank=same; read; write; mmap_s; exit} + {rank=same; tokenize; read_fn} +} diff --git a/docs/asm-architecture.png b/docs/asm-architecture.png new file mode 100644 index 0000000..db0c340 Binary files /dev/null and b/docs/asm-architecture.png differ diff --git a/docs/c-architecture.dot b/docs/c-architecture.dot new file mode 100644 index 0000000..5331408 --- /dev/null +++ b/docs/c-architecture.dot @@ -0,0 +1,71 @@ +// C implementation architecture +// "A diagram is worth 10,000 words." — russell@unturf.com +digraph c_arch { + rankdir=TB + node [shape=box, style=filled, fontname="Helvetica"] + edge [fontname="Helvetica", fontsize=10] + + subgraph cluster_input { + label="Input" + style=dashed + source [label=".lsp source" fillcolor="#e8f4fd"] + cli [label="CLI flags\n-e --fast --jit" fillcolor="#e8f4fd"] + } + + subgraph cluster_frontend { + label="Frontend (reader.c)" + style=rounded + color="#666666" + tokenizer [label="Tokenizer\nregex-free\nchar-by-char" fillcolor="#fff3cd"] + parser [label="Parser\nrecursive descent\n→ Pair/Value AST" fillcolor="#fff3cd"] + } + + subgraph cluster_types { + label="Types (types.c, uncommonlisp.h)" + style=rounded + color="#666666" + nanbox [label="NaN-Boxing\n64-bit doubles\ntype tags in NaN payload\nzero-alloc numbers" fillcolor="#e2d5f1"] + intern [label="Symbol Interning\nhash table" fillcolor="#e2d5f1"] + pairs [label="Cons Cells\n16 bytes (car+cdr)" fillcolor="#e2d5f1"] + envs [label="Environment\nhash-map bindings\nparent chain + global shortcut" fillcolor="#e2d5f1"] + } + + subgraph cluster_eval { + label="Three Execution Tiers" + style=rounded + color="#666666" + + interp [label="Tree-Walker\nleval()\nTCO via while loop\nall special forms" fillcolor="#d4edda"] + bytecode [label="Bytecode VM\ncompile → execute\nsuperinstructions\nSELF_TAIL_CALL" fillcolor="#cce5ff"] + jit [label="x86_64 JIT\nmmap(PROT_EXEC)\nnative machine code\nif/cond/let/named-let\ncar/cdr/cons\n10-24x faster than CPython" fillcolor="#ff6b6b" fontcolor=white] + } + + subgraph cluster_jit_detail { + label="JIT Pipeline (jit.c)" + style=rounded + color="#cc0000" + analyze [label="AST Analysis\ncan_jit_proc()" fillcolor="#ffcccc"] + emit [label="x86_64 Emission\nmov/add/sub/cmp\ncall/ret/jmp" fillcolor="#ffcccc"] + mmap [label="mmap()\nPROT_READ|WRITE|EXEC\nfunction pointer" fillcolor="#ffcccc"] + } + + source -> tokenizer + cli -> tokenizer + tokenizer -> parser + parser -> interp [label="default"] + parser -> bytecode [label="--fast"] + parser -> analyze [label="--jit"] + analyze -> emit [label="jittable"] + analyze -> interp [label="fallback" style=dashed] + emit -> mmap + mmap -> jit [label="JitFunc"] + interp -> envs + bytecode -> envs + jit -> nanbox [label="unbox/rebox"] + envs -> nanbox + nanbox -> intern + nanbox -> pairs + + {rank=same; interp; bytecode; jit} + {rank=same; analyze; emit; mmap} +} diff --git a/docs/c-architecture.png b/docs/c-architecture.png new file mode 100644 index 0000000..1800291 Binary files /dev/null and b/docs/c-architecture.png differ diff --git a/docs/jit-pipeline.dot b/docs/jit-pipeline.dot new file mode 100644 index 0000000..e0a6836 --- /dev/null +++ b/docs/jit-pipeline.dot @@ -0,0 +1,35 @@ +// JIT compilation pipeline +// "A diagram is worth 10,000 words." — russell@unturf.com +digraph jit_pipeline { + rankdir=LR + node [shape=box, style=filled, fontname="Helvetica"] + edge [fontname="Helvetica", fontsize=10] + + scheme [label="Scheme Source\n(define (ack m n)\n (cond ...))" fillcolor="#e8f4fd" shape=note] + ast [label="AST\n(Pair tree)" fillcolor="#fff3cd"] + analysis [label="can_jit_proc()\ncheck: only ints,\narith, cond, if,\nself-recursion" fillcolor="#ffeaa7"] + + subgraph cluster_codegen { + label="x86_64 Code Generation" + style=rounded + color="#cc0000" + prologue [label="Prologue\npush rbp\nmov rbp,rsp\npush r12-r15" fillcolor="#ffcccc"] + body [label="Body Emission\ncmp → je/jne\nadd/sub/imul\ncall self\njmp (TCO)" fillcolor="#ffcccc"] + epilogue [label="Epilogue\npop r15-r12\npop rbp\nret" fillcolor="#ffcccc"] + } + + mmap [label="mmap()\nPROT_READ|\nPROT_WRITE|\nPROT_EXEC" fillcolor="#ff6b6b" fontcolor=white] + native [label="Native Function\nJitFunc ptr\n0.12ms ack(3,4)" fillcolor="#00b894" fontcolor=white shape=doubleoctagon] + + fallback [label="Interpreter\nFallback\n(call/cc, macros)" fillcolor="#dfe6e9" style="filled,dashed"] + + scheme -> ast [label="parse"] + ast -> analysis [label="walk"] + analysis -> prologue [label="jittable"] + analysis -> fallback [label="complex" style=dashed] + prologue -> body -> epilogue + epilogue -> mmap [label="raw bytes"] + mmap -> native [label="cast to\nfunction ptr"] + + {rank=same; analysis; fallback} +} diff --git a/docs/jit-pipeline.png b/docs/jit-pipeline.png new file mode 100644 index 0000000..89bee6f Binary files /dev/null and b/docs/jit-pipeline.png differ diff --git a/docs/python-architecture.dot b/docs/python-architecture.dot new file mode 100644 index 0000000..e8ed1fb --- /dev/null +++ b/docs/python-architecture.dot @@ -0,0 +1,71 @@ +// Python implementation architecture +// "A diagram is worth 10,000 words." — russell@unturf.com +digraph python_arch { + rankdir=TB + node [shape=box, style=filled, fontname="Helvetica"] + edge [fontname="Helvetica", fontsize=10] + + subgraph cluster_input { + label="Input" + style=dashed + source [label=".lsp source\nstdlib.lsp" fillcolor="#e8f4fd"] + repl [label="REPL\ninteractive" fillcolor="#e8f4fd"] + } + + subgraph cluster_frontend { + label="Frontend" + style=rounded + color="#666666" + tokenizer [label="Tokenizer\n_tokenize_lines()" fillcolor="#fff3cd"] + parser [label="Parser\n_read() → Pair AST" fillcolor="#fff3cd"] + macros [label="Macro Expander\nsyntax-rules\ndefine-macro" fillcolor="#fff3cd"] + } + + subgraph cluster_eval { + label="Evaluators (two paths)" + style=rounded + color="#666666" + + leval [label="Tree-Walker\nleval()\nTCO via while loop" fillcolor="#d4edda"] + compiler [label="Bytecode Compiler\n_bc() → CodeObj\n40 opcodes + superinstrs" fillcolor="#cce5ff"] + vm [label="Stack VM\n_vm_loop()\nexplicit frame stack" fillcolor="#cce5ff"] + jit_py [label="Python JIT\n(prototype)\nexec() transpile" fillcolor="#f8d7da" style="filled,dashed"] + } + + subgraph cluster_runtime { + label="Runtime" + style=rounded + color="#666666" + env [label="Environment\nEnv chain + global shortcut\ninline cache" fillcolor="#e2d5f1"] + types [label="Types\nSymbol, Pair, Proc\nCompiledProc, FullCont\nMutableString, Fraction" fillcolor="#e2d5f1"] + gc [label="Memory\nPython GC\n(automatic)" fillcolor="#e2d5f1"] + } + + subgraph cluster_features { + label="Features" + style=rounded + color="#666666" + callcc [label="Full Continuations\ncall/cc\nmulti-shot" fillcolor="#ffeaa7"] + portal [label="Portal\nserialize VM state\nresume on another machine" fillcolor="#ffeaa7"] + serial [label="Bytecode Serialization\n.lspc files\nJSON format" fillcolor="#ffeaa7"] + } + + source -> tokenizer + repl -> tokenizer + tokenizer -> parser + parser -> macros + macros -> leval [label="interpreted"] + macros -> compiler [label="--fast"] + compiler -> vm + vm -> jit_py [label="hot funcs" style=dashed] + leval -> env + vm -> env + env -> types + types -> gc + vm -> callcc + callcc -> portal + compiler -> serial + + {rank=same; leval; compiler} + {rank=same; callcc; portal; serial} +} diff --git a/docs/python-architecture.png b/docs/python-architecture.png new file mode 100644 index 0000000..8da4095 Binary files /dev/null and b/docs/python-architecture.png differ