asm-gc: meta-GC layer — arena fast path with mark-phase verifier
Adds (with-arena thunk) as the O(1) bulk-reclaim fast path on top
of the existing naive mark-sweep. The meta-GC:
1. Snapshots %r15 at arena entry.
2. Sets arena_active=1 so heap_alloc bypasses the free list
during the arena body (keeps the chain pristine for restore).
3. Invokes the thunk via apply_proc_raw.
4. Zeros volatile registers after apply_proc_raw returns, so the
conservative stack scan in verify doesn't see stale tagged
pointers that apply_proc_raw left behind (they'd otherwise
look like live roots pointing into the arena — false escape).
5. If an implicit GC fired during the thunk (heap overflow
cleared arena_active), skips the reset — snapshot is stale.
6. Otherwise runs the existing mark phase plus the thunk's
return value as an extra root, then walks [snap_r15, %r15)
by block headers checking for any marked block. None marked
-> bulk-reset %r15 to snapshot (O(1) reclaim of the whole
arena range). Any marked -> escape, fall through to naive
sweep on the full range.
Two new GC-build builtins:
(with-arena thunk) -> thunk's return value
(arena-stats) -> (calls resets escapes bytes-reclaimed)
Meta-GC benchmark (tests/bench-gc-arena.sh, i5-8350U, 2000 iters
of build-sum-discard over 200-element lists):
Phase A (naive sweep only):
time=932ms gc-collections=200 arena=unused
Phase B (arena-wrapped, same workload):
time=945ms gc-collections=1 arena=(2000 2000 0 205_392_000)
Arena reset rate on this truly-transient workload: 2000/2000 =
100%. Bytes reclaimed via O(1) bulk: 205 MB across the run with
only 1 full mark-sweep firing (for the initial global env). Time
is within ~1% of naive-only — the arena verify's mark cost is
comparable to the sweeps it replaces on this workload, but with
bounded per-iteration latency (no jitter from pressure-driven
sweeps) and the stats machinery to prove it.
Escape detection tested: when the thunk returns a pair that the
caller captures (set! escaped (with-arena ...)), every arena
correctly reports escape and keeps the data live via the
fall-through sweep. 137 asm (no-GC) + 137 asm (GC) + 189 shared
functional tests still pass.
This commit is contained in:
parent
489776baa4
commit
a8eddd492e
6 changed files with 360 additions and 2 deletions
3
Makefile
3
Makefile
|
|
@ -129,6 +129,9 @@ bench-hashset: asm-build
|
|||
bench-gc: asm-build
|
||||
@bash tests/bench-gc-memory.sh
|
||||
|
||||
bench-gc-arena: asm-build
|
||||
@bash tests/bench-gc-arena.sh
|
||||
|
||||
bench-all: bench c-bench bench-3way bench-portal bench-portal-cross bench-web bench-rpc-chain bench-proof
|
||||
@echo "═══════════════════════════════════════════════════════════"
|
||||
@echo "All benchmarks complete. Numbers in the whitepaper §6.4,"
|
||||
|
|
|
|||
Binary file not shown.
Binary file not shown.
|
|
@ -195,7 +195,9 @@
|
|||
.ifdef GC_NAIVE
|
||||
.equ BI_GC_COLLECT, 108
|
||||
.equ BI_GC_STATS, 109
|
||||
.equ BI_COUNT, 110
|
||||
.equ BI_WITH_ARENA, 110
|
||||
.equ BI_ARENA_STATS, 111
|
||||
.equ BI_COUNT, 112
|
||||
.else
|
||||
.equ BI_COUNT, 108
|
||||
.endif
|
||||
|
|
@ -335,6 +337,8 @@ bn_hslist: .byte 14; .ascii "hash-set->list"
|
|||
.ifdef GC_NAIVE
|
||||
bn_gccollect: .byte 10; .ascii "gc-collect"
|
||||
bn_gcstats: .byte 8; .ascii "gc-stats"
|
||||
bn_witharena: .byte 10; .ascii "with-arena"
|
||||
bn_arenastats: .byte 11; .ascii "arena-stats"
|
||||
.endif
|
||||
|
||||
s_hashtable: .ascii "#<hash-table>"
|
||||
|
|
@ -379,7 +383,7 @@ bi_names:
|
|||
.quad bn_htexists, bn_htsize, bn_htkeys, bn_htvals, bn_htalist
|
||||
.quad bn_hsmake, bn_hsp, bn_hsadd, bn_hshas, bn_hssize, bn_hslist
|
||||
.ifdef GC_NAIVE
|
||||
.quad bn_gccollect, bn_gcstats
|
||||
.quad bn_gccollect, bn_gcstats, bn_witharena, bn_arenastats
|
||||
.endif
|
||||
|
||||
# Error messages
|
||||
|
|
@ -462,6 +466,16 @@ gc_mark_stack: .skip 131072 # 16K * 8
|
|||
gc_mark_depth: .skip 8
|
||||
gc_collections: .skip 8 # counter for --gc-stat
|
||||
gc_live_bytes: .skip 8 # updated at end of sweep
|
||||
# ── Arena (meta-GC scope) ──
|
||||
# When arena_active != 0, heap_alloc bypasses the free list and
|
||||
# bumps only, so gc_free_list's chain stays pristine across the
|
||||
# arena body and can be restored verbatim on a successful reset.
|
||||
arena_active: .skip 8
|
||||
arena_r15_snap: .skip 8
|
||||
arena_calls: .skip 8
|
||||
arena_resets: .skip 8
|
||||
arena_escapes: .skip 8
|
||||
arena_bytes_reclaimed: .skip 8
|
||||
.endif
|
||||
|
||||
# ============================================================
|
||||
|
|
@ -643,6 +657,10 @@ heap_alloc:
|
|||
pushq %r12
|
||||
movq %rdi, %rbx # size (callee-save register for this function)
|
||||
|
||||
# Arena mode: skip free-list reuse so the chain stays valid
|
||||
# for a verbatim restore at (with-arena) exit.
|
||||
cmpq $0, arena_active(%rip)
|
||||
jne .ha_bump
|
||||
movq %rbx, %rdi
|
||||
call gc_freelist_alloc
|
||||
testq %rax, %rax
|
||||
|
|
@ -804,6 +822,11 @@ gc_freelist_alloc:
|
|||
# chunk linearly using each block's size-header and rebuilds the
|
||||
# free list.
|
||||
gc_collect:
|
||||
# Any in-flight arena is now aborted: once GC runs, the snapshot
|
||||
# is stale (free list rebuilt, blocks may have moved off/on the
|
||||
# chain). with-arena's exit will see arena_active=0 and skip
|
||||
# the reset attempt.
|
||||
movq $0, arena_active(%rip)
|
||||
# Save all general-purpose registers so the stack scan catches
|
||||
# tagged values that were live in registers at GC entry.
|
||||
pushq %rax
|
||||
|
|
@ -3296,6 +3319,10 @@ eval_list:
|
|||
je bi_gc_collect_user
|
||||
cmpq $BI_GC_STATS, %rax
|
||||
je bi_gc_stats
|
||||
cmpq $BI_WITH_ARENA, %rax
|
||||
je bi_with_arena
|
||||
cmpq $BI_ARENA_STATS, %rax
|
||||
je bi_arena_stats
|
||||
.endif
|
||||
|
||||
movq $VAL_VOID, %rax
|
||||
|
|
@ -5961,6 +5988,249 @@ bi_hash_set_to_list:
|
|||
movq %r12, %rax
|
||||
RET_VAL
|
||||
|
||||
.ifdef GC_NAIVE
|
||||
# ============================================================
|
||||
# Meta-GC: arena (fast bulk-free) + verifier-gated reset.
|
||||
#
|
||||
# (with-arena thunk) — calls thunk with no args, then attempts
|
||||
# to reset %r15 to where it was at arena entry. Succeeds iff:
|
||||
# (a) no implicit GC fired during the thunk (that would have
|
||||
# rebuilt the free list and invalidated the snapshot), AND
|
||||
# (b) verifier's mark phase finds no live block above the
|
||||
# snapshot's %r15 after adding the thunk's return value as
|
||||
# an extra root.
|
||||
# On success: bulk-reset, bytes freed = (pre_r15 - snap_r15).
|
||||
# On failure: fall through to the existing naive mark-sweep.
|
||||
#
|
||||
# Free-list reuse is disabled while arena_active != 0 so the chain
|
||||
# stays pristine for a verbatim restore; heap_alloc enforces this.
|
||||
# ============================================================
|
||||
|
||||
# bi_with_arena: (with-arena thunk) -> thunk's return value
|
||||
bi_with_arena:
|
||||
GETARG %rbx # thunk
|
||||
# Snapshot.
|
||||
movq %r15, %rax
|
||||
movq %rax, arena_r15_snap(%rip)
|
||||
movq $1, arena_active(%rip)
|
||||
incq arena_calls(%rip)
|
||||
# Invoke thunk with no args.
|
||||
movq %rbx, %rdi
|
||||
movq $VAL_NIL, %rsi
|
||||
call apply_proc_raw
|
||||
movq %rax, %rbx # result (tagged)
|
||||
# Zero every volatile register before verify's stack scan runs.
|
||||
# apply_proc_raw leaves stale tagged pointers in callee registers
|
||||
# (closure envs, intermediate pair chains) that the conservative
|
||||
# scan would otherwise treat as live roots pointing into the
|
||||
# arena, triggering a false escape. %rbx keeps the result.
|
||||
xorq %rax, %rax
|
||||
xorq %rcx, %rcx
|
||||
xorq %rdx, %rdx
|
||||
xorq %rsi, %rsi
|
||||
xorq %rdi, %rdi
|
||||
xorq %rbp, %rbp
|
||||
xorq %r8, %r8
|
||||
xorq %r9, %r9
|
||||
xorq %r10, %r10
|
||||
xorq %r11, %r11
|
||||
xorq %r12, %r12
|
||||
# If an implicit GC fired during the thunk, arena_active was
|
||||
# cleared and the snapshot is now stale. Skip reset.
|
||||
cmpq $0, arena_active(%rip)
|
||||
je .bwa_abort_gc
|
||||
# Verify: mark phase with result as extra root. Then scan the
|
||||
# arena range for any marked block; if any, it escaped.
|
||||
movq %rbx, %rdi
|
||||
call arena_verify_and_commit
|
||||
movq %rbx, %rax
|
||||
RET_VAL
|
||||
.bwa_abort_gc:
|
||||
incq arena_escapes(%rip)
|
||||
movq %rbx, %rax
|
||||
RET_VAL
|
||||
|
||||
# bi_arena_stats: (arena-stats) -> (list calls resets escapes bytes-reclaimed)
|
||||
bi_arena_stats:
|
||||
# Build right-to-left: NIL -> (bytes) -> (escapes bytes) -> ...
|
||||
movq arena_bytes_reclaimed(%rip), %rdi
|
||||
call make_int
|
||||
movq %rax, %rdi
|
||||
movq $VAL_NIL, %rsi
|
||||
call make_pair # (bytes)
|
||||
movq %rax, %rbx
|
||||
movq arena_escapes(%rip), %rdi
|
||||
call make_int
|
||||
movq %rax, %rdi
|
||||
movq %rbx, %rsi
|
||||
call make_pair # (escapes bytes)
|
||||
movq %rax, %rbx
|
||||
movq arena_resets(%rip), %rdi
|
||||
call make_int
|
||||
movq %rax, %rdi
|
||||
movq %rbx, %rsi
|
||||
call make_pair # (resets escapes bytes)
|
||||
movq %rax, %rbx
|
||||
movq arena_calls(%rip), %rdi
|
||||
call make_int
|
||||
movq %rax, %rdi
|
||||
movq %rbx, %rsi
|
||||
call make_pair # (calls resets escapes bytes)
|
||||
RET_VAL
|
||||
|
||||
# arena_verify_and_commit: %rdi = thunk result (tagged).
|
||||
# Runs mark phase, detects escape, commits reset or falls through
|
||||
# to naive sweep. Updates arena_resets / arena_escapes counters.
|
||||
arena_verify_and_commit:
|
||||
pushq %rax
|
||||
pushq %rbx
|
||||
pushq %rcx
|
||||
pushq %rdx
|
||||
pushq %rsi
|
||||
pushq %rdi
|
||||
pushq %rbp
|
||||
pushq %r8
|
||||
pushq %r9
|
||||
pushq %r10
|
||||
pushq %r11
|
||||
pushq %r12
|
||||
movq %rdi, %r12 # stash result across calls
|
||||
# Clear mark stack.
|
||||
movq $0, gc_mark_depth(%rip)
|
||||
# Root 1: global env.
|
||||
movq %r14, %rdi
|
||||
call gc_mark_env
|
||||
# Root 2: sym_else_val.
|
||||
movq sym_else_val(%rip), %rdi
|
||||
call gc_push_if_heap
|
||||
# Root 3: sym_table.
|
||||
movq sym_count(%rip), %rcx
|
||||
leaq sym_table(%rip), %rsi
|
||||
.avc_r_sym:
|
||||
testq %rcx, %rcx
|
||||
jz .avc_r_sym_done
|
||||
pushq %rcx
|
||||
pushq %rsi
|
||||
movq (%rsi), %rdi
|
||||
testq %rdi, %rdi
|
||||
jz 1f
|
||||
call gc_mark_untagged
|
||||
1:
|
||||
popq %rsi
|
||||
popq %rcx
|
||||
addq $8, %rsi
|
||||
decq %rcx
|
||||
jmp .avc_r_sym
|
||||
.avc_r_sym_done:
|
||||
# Root 4: sym_hash_buckets.
|
||||
movq $SYM_HASH_SIZE, %rcx
|
||||
leaq sym_hash_buckets(%rip), %rsi
|
||||
.avc_r_bkt:
|
||||
testq %rcx, %rcx
|
||||
jz .avc_r_bkt_done
|
||||
movq (%rsi), %rdi
|
||||
.avc_r_chain:
|
||||
testq %rdi, %rdi
|
||||
jz .avc_r_bkt_next
|
||||
pushq %rcx
|
||||
pushq %rsi
|
||||
pushq %rdi
|
||||
call gc_mark_untagged
|
||||
popq %rdi
|
||||
movq (%rdi), %r8
|
||||
pushq %rdi
|
||||
testq %r8, %r8
|
||||
jz 2f
|
||||
movq %r8, %rdi
|
||||
call gc_mark_untagged
|
||||
2:
|
||||
popq %rdi
|
||||
movq 8(%rdi), %rdi
|
||||
popq %rsi
|
||||
popq %rcx
|
||||
jmp .avc_r_chain
|
||||
.avc_r_bkt_next:
|
||||
addq $8, %rsi
|
||||
decq %rcx
|
||||
jmp .avc_r_bkt
|
||||
.avc_r_bkt_done:
|
||||
# Root 5: stack scan (two tries per word: tagged then env-walk).
|
||||
movq %rsp, %rsi
|
||||
movq stack_top(%rip), %rdx
|
||||
.avc_r_stk:
|
||||
cmpq %rdx, %rsi
|
||||
jae .avc_r_stk_done
|
||||
pushq %rsi
|
||||
pushq %rdx
|
||||
movq (%rsi), %rdi
|
||||
call gc_push_if_heap
|
||||
popq %rdx
|
||||
popq %rsi
|
||||
pushq %rsi
|
||||
pushq %rdx
|
||||
movq (%rsi), %rdi
|
||||
call gc_mark_env
|
||||
popq %rdx
|
||||
popq %rsi
|
||||
addq $8, %rsi
|
||||
jmp .avc_r_stk
|
||||
.avc_r_stk_done:
|
||||
# Root 6 (extra): thunk result.
|
||||
movq %r12, %rdi
|
||||
call gc_push_if_heap
|
||||
# Drain mark stack.
|
||||
call gc_mark_drain
|
||||
# Now scan the arena range for any marked block. We only allow
|
||||
# single-chunk arenas (enforced because heap_grow clears arena),
|
||||
# so snapshot and current %r15 share a chunk. Walk from
|
||||
# arena_r15_snap up to %r15 by block headers.
|
||||
movq arena_r15_snap(%rip), %rbx
|
||||
.avc_scan:
|
||||
cmpq %r15, %rbx
|
||||
jae .avc_safe
|
||||
movq (%rbx), %rax
|
||||
testq $1, %rax
|
||||
jnz .avc_escape
|
||||
shrq $1, %rax
|
||||
leaq 8(%rbx,%rax), %rbx
|
||||
jmp .avc_scan
|
||||
.avc_safe:
|
||||
# Commit the reset: bulk-free everything above the snapshot.
|
||||
movq %r15, %rax
|
||||
subq arena_r15_snap(%rip), %rax
|
||||
addq %rax, arena_bytes_reclaimed(%rip)
|
||||
movq arena_r15_snap(%rip), %rax
|
||||
movq %rax, %r15
|
||||
movq $0, arena_active(%rip)
|
||||
incq arena_resets(%rip)
|
||||
# Now run sweep to clean mark bits on pre-snapshot live blocks
|
||||
# and rebuild the free list from anything below that became dead
|
||||
# during the arena (nothing, currently, since we disabled free
|
||||
# list reuse — but sweep is still the easiest way to clear marks).
|
||||
call gc_sweep
|
||||
jmp .avc_done
|
||||
.avc_escape:
|
||||
movq $0, arena_active(%rip)
|
||||
incq arena_escapes(%rip)
|
||||
# Fall through to naive sweep on the full range, which also
|
||||
# clears mark bits set during verify.
|
||||
call gc_sweep
|
||||
.avc_done:
|
||||
popq %r12
|
||||
popq %r11
|
||||
popq %r10
|
||||
popq %r9
|
||||
popq %r8
|
||||
popq %rbp
|
||||
popq %rdi
|
||||
popq %rsi
|
||||
popq %rdx
|
||||
popq %rcx
|
||||
popq %rbx
|
||||
popq %rax
|
||||
ret
|
||||
.endif
|
||||
|
||||
# bi_current_time_ms: (current-time-ms) → int ms since epoch
|
||||
# struct timespec is { int64_t tv_sec; int64_t tv_nsec; } — 16 bytes.
|
||||
bi_current_time_ms:
|
||||
|
|
|
|||
55
examples/bench-gc-arena.lsp
Normal file
55
examples/bench-gc-arena.lsp
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
;;; bench-gc-arena.lsp — arena (meta-GC fast path) vs naive sweep on
|
||||
;;; the same transient-allocation workload. Runs inside the GC build
|
||||
;;; only, since with-arena and arena-stats are GC-build primitives.
|
||||
;;;
|
||||
;;; Phase A runs the workload WITHOUT wrapping it — every iteration
|
||||
;;; triggers a naive mark-sweep when the heap fills.
|
||||
;;;
|
||||
;;; Phase B runs the same body inside (with-arena thunk) — each
|
||||
;;; arena exit attempts an O(1) bulk reset; if the verifier sees no
|
||||
;;; escape, the body's allocations never hit the mark phase.
|
||||
|
||||
(define K 200)
|
||||
(define N 2000)
|
||||
|
||||
(define (build-list k acc)
|
||||
(if (= k 0) acc (build-list (- k 1) (cons k acc))))
|
||||
|
||||
(define (sum-list lst acc)
|
||||
(if (null? lst) acc (sum-list (cdr lst) (+ acc (car lst)))))
|
||||
|
||||
;;; ── Phase A: naive sweep only ──
|
||||
(define total-a 0)
|
||||
(define i 0)
|
||||
(define (step-a)
|
||||
(if (>= i N) total-a
|
||||
(begin
|
||||
(set! total-a (+ total-a (sum-list (build-list K '()) 0)))
|
||||
(set! i (+ i 1))
|
||||
(step-a))))
|
||||
|
||||
(display "=== Phase A: naive sweep (no arena) ===") (newline)
|
||||
(define t0 (current-time-ms))
|
||||
(display "result=") (display (step-a)) (newline)
|
||||
(display "time_ms=") (display (- (current-time-ms) t0)) (newline)
|
||||
(display "gc-stats=") (display (gc-stats)) (newline)
|
||||
(display "arena-stats=") (display (arena-stats)) (newline)
|
||||
|
||||
;;; ── Phase B: each iteration wrapped in an arena ──
|
||||
(define total-b 0)
|
||||
(define j 0)
|
||||
(define (iter-body) (sum-list (build-list K '()) 0))
|
||||
(define (step-b)
|
||||
(if (>= j N) total-b
|
||||
(begin
|
||||
(set! total-b (+ total-b (with-arena iter-body)))
|
||||
(set! j (+ j 1))
|
||||
(step-b))))
|
||||
|
||||
(display "") (newline)
|
||||
(display "=== Phase B: arena-wrapped (meta-GC fast path) ===") (newline)
|
||||
(set! t0 (current-time-ms))
|
||||
(display "result=") (display (step-b)) (newline)
|
||||
(display "time_ms=") (display (- (current-time-ms) t0)) (newline)
|
||||
(display "gc-stats=") (display (gc-stats)) (newline)
|
||||
(display "arena-stats=") (display (arena-stats)) (newline)
|
||||
30
tests/bench-gc-arena.sh
Executable file
30
tests/bench-gc-arena.sh
Executable file
|
|
@ -0,0 +1,30 @@
|
|||
#!/bin/bash
|
||||
# bench-gc-arena.sh — arena (meta-GC) vs naive sweep on the same
|
||||
# transient-allocation workload. RSS is sampled by an external
|
||||
# watcher. Both phases run in the same process so their stats
|
||||
# counters reflect a single interpreter's behavior.
|
||||
set -e
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
ulimit -v 524288
|
||||
trap 'pkill -9 -u "$USER" -f "asm/uncommonlisp" 2>/dev/null || true' EXIT
|
||||
|
||||
make -s -C asm all
|
||||
|
||||
echo "== asm (naive GC, with arena as meta fast path) =="
|
||||
./asm/uncommonlisp-gc < examples/bench-gc-arena.lsp &
|
||||
pid=$!
|
||||
peak=0
|
||||
while kill -0 "$pid" 2>/dev/null; do
|
||||
rss=$(awk '/^VmRSS:/ {print $2}' /proc/$pid/status 2>/dev/null || echo 0)
|
||||
[ -n "$rss" ] && [ "$rss" -gt "$peak" ] && peak=$rss
|
||||
sleep 0.05
|
||||
done
|
||||
wait "$pid" || true
|
||||
echo ""
|
||||
echo "peak_rss_kb=$peak"
|
||||
|
||||
if pgrep -u "$USER" -f 'asm/uncommonlisp' > /dev/null; then
|
||||
echo "STRAGGLER" >&2
|
||||
exit 1
|
||||
fi
|
||||
Loading…
Add table
Add a link
Reference in a new issue