russell@unturf.com
67e4fef85c
Every benchmark in the whitepaper now has a Makefile target and
each in-paper result is tagged with its reproduce command.
New / refactored Make targets:
make bench Python tree-walker vs bytecode (§6.1-6.3)
make bench-3way 3-way Python/C/asm head-to-head (§6.4)
make bench-portal portal save+load timings (§7.5)
make bench-portal-cross 3x3 cross-impl portal matrix (§7.2)
make bench-web HTTP vs busybox / python http.server (§11.3)
make bench-rpc-chain Python → C relay → asm chain (§11.4)
make bench-all runs every bench above
bench-3way is a new script (tests/bench-3way.sh) that drives each
impl in its recommended high-performance mode and prints a clean
best-of-two comparison table matching §6.4.
Every script uses the six-layer safety envelope from CLAUDE.md
(ulimit -v + trap + timeout + explicit kill + pgrep verify).
Documented in the whitepaper's §6 Methodology block.
Whitepaper additions:
- §6 Methodology paragraph adds a "Reproducibility" block listing
every Makefile target alongside the section it backs.
- §12 MOAD Audit now cites the canonical MOAD taxonomy:
https://undefect.com/moad-cheat-sheet/
(MOAD-0001 through MOAD-0005) so readers can look up the defect
classes the paper references.
- §6.4, §7.2, §7.5, §11.3, §11.4 each end with a "Reproduce: make
bench-<name>" pointer tying the number to the script that
produces it.
Ran bench-3way on the i5-8350U:
Python --fast: sum-to(100k)=555ms, sum-to(1M)=5038ms, ack(3,8)=18740ms
C --fast: sum-to(100k)= 27ms, sum-to(1M)= 255ms, ack(3,8)= 1465ms
asm: sum-to(100k)= 67ms, sum-to(1M)= 692ms, ack(3,8)= 2300ms
Matches the table in the paper (best-of-two).
77 lines
3.1 KiB
Bash
Executable file
77 lines
3.1 KiB
Bash
Executable file
#!/bin/bash
|
|
# bench-3way.sh — head-to-head: Python --fast vs C --fast vs asm
|
|
#
|
|
# Runs sum-to(100k), sum-to(1M), ackermann(3,8) under each impl's
|
|
# high-performance mode and prints a comparison table. Drives the
|
|
# numbers in §6.4 of the whitepaper.
|
|
#
|
|
# Usage: bash tests/bench-3way.sh
|
|
# Safety: no backgrounded servers, no sockets, no stray processes.
|
|
# Each run is a single short-lived foreground process.
|
|
|
|
set -e
|
|
cd "$(dirname "$0")/.."
|
|
ulimit -v 2097152 # 2 GB virt cap (bytecode VM can use more than asm)
|
|
|
|
cat > /tmp/bench-3way.lsp <<'EOF'
|
|
(define (sum-to n)
|
|
(let loop ((i 0) (acc 0))
|
|
(if (= i n) acc (loop (+ i 1) (+ acc i)))))
|
|
(define (ack m n)
|
|
(cond ((= m 0) (+ n 1))
|
|
((= n 0) (ack (- m 1) 1))
|
|
(else (ack (- m 1) (ack m (- n 1))))))
|
|
(define t0 (current-time-ms)) (sum-to 100000) (define t1 (current-time-ms))
|
|
(define t2 (current-time-ms)) (sum-to 1000000) (define t3 (current-time-ms))
|
|
(define t4 (current-time-ms)) (ack 3 8) (define t5 (current-time-ms))
|
|
(display "sum-to(100k): ") (display (- t1 t0)) (newline)
|
|
(display "sum-to(1M): ") (display (- t3 t2)) (newline)
|
|
(display "ack(3,8): ") (display (- t5 t4)) (newline)
|
|
EOF
|
|
|
|
echo "═══════════════════════════════════════════════════════════════════"
|
|
echo "Three implementations head-to-head (best of 2 runs, ms)"
|
|
echo " Python --fast (bytecode VM) | C --fast (bytecode VM) | asm (tree-walker)"
|
|
echo "═══════════════════════════════════════════════════════════════════"
|
|
echo
|
|
|
|
best_of_two() {
|
|
# Run twice, take the smaller time per metric. Each run prints
|
|
# sum-to(100k): N
|
|
# sum-to(1M): N
|
|
# ack(3,8): N
|
|
# (asm additionally prints the results themselves first; we grep
|
|
# only the metric lines.)
|
|
local cmd="$1"
|
|
local r1 r2
|
|
r1=$(mktemp); r2=$(mktemp)
|
|
eval "timeout 60 $cmd" 2>&1 | grep -E "sum-to|ack" > "$r1"
|
|
eval "timeout 60 $cmd" 2>&1 | grep -E "sum-to|ack" > "$r2"
|
|
paste "$r1" "$r2" | awk -F'\t' '{
|
|
# Each side is "label: N". Parse each label/number.
|
|
n1 = $1; n2 = $2
|
|
sub(/.*: */, "", n1); n1 += 0
|
|
sub(/.*: */, "", n2); n2 += 0
|
|
label = $1; sub(/:.*/, ":", label)
|
|
min = (n1 < n2) ? n1 : n2
|
|
printf " %-16s %6d ms\n", label, min
|
|
}'
|
|
rm -f "$r1" "$r2"
|
|
}
|
|
|
|
echo "── Python (--fast) ──"
|
|
best_of_two "python3 uncommonlisp.py --fast /tmp/bench-3way.lsp"
|
|
echo
|
|
|
|
echo "── C (--fast) ──"
|
|
best_of_two "c/uncommonlisp --fast /tmp/bench-3way.lsp"
|
|
echo
|
|
|
|
echo "── asm ──"
|
|
best_of_two "asm/uncommonlisp < /tmp/bench-3way.lsp"
|
|
echo
|
|
|
|
rm -f /tmp/bench-3way.lsp
|
|
echo "═══════════════════════════════════════════════════════════════════"
|
|
echo "Hardware: $(grep -m1 'model name' /proc/cpuinfo | cut -d: -f2 | xargs)"
|
|
echo "Kernel: $(uname -r)"
|