diff --git a/asm/uncommonlisp-gc b/asm/uncommonlisp-gc index d317087..5e479c6 100755 Binary files a/asm/uncommonlisp-gc and b/asm/uncommonlisp-gc differ diff --git a/asm/uncommonlisp-gc.o b/asm/uncommonlisp-gc.o index 44eb317..8baca04 100644 Binary files a/asm/uncommonlisp-gc.o and b/asm/uncommonlisp-gc.o differ diff --git a/asm/uncommonlisp.s b/asm/uncommonlisp.s index d4dd72b..44458c7 100644 --- a/asm/uncommonlisp.s +++ b/asm/uncommonlisp.s @@ -197,7 +197,8 @@ .equ BI_GC_STATS, 109 .equ BI_WITH_ARENA, 110 .equ BI_ARENA_STATS, 111 -.equ BI_COUNT, 112 +.equ BI_ARENA_SET_MODE, 112 +.equ BI_COUNT, 113 .else .equ BI_COUNT, 108 .endif @@ -339,6 +340,7 @@ bn_gccollect: .byte 10; .ascii "gc-collect" bn_gcstats: .byte 8; .ascii "gc-stats" bn_witharena: .byte 10; .ascii "with-arena" bn_arenastats: .byte 11; .ascii "arena-stats" +bn_arenamode: .byte 14; .ascii "arena-set-mode" .endif s_hashtable: .ascii "#" @@ -383,7 +385,7 @@ bi_names: .quad bn_htexists, bn_htsize, bn_htkeys, bn_htvals, bn_htalist .quad bn_hsmake, bn_hsp, bn_hsadd, bn_hshas, bn_hssize, bn_hslist .ifdef GC_NAIVE - .quad bn_gccollect, bn_gcstats, bn_witharena, bn_arenastats + .quad bn_gccollect, bn_gcstats, bn_witharena, bn_arenastats, bn_arenamode .endif # Error messages @@ -476,6 +478,18 @@ arena_calls: .skip 8 arena_resets: .skip 8 arena_escapes: .skip 8 arena_bytes_reclaimed: .skip 8 +# ── Adaptive policy state ── +# EMA of recent escape rate, scaled 0..256 (256 = 100% escape). +# Update on each arena: rate = (rate*7 + sample*256) / 8 where +# sample is 0 for reset, 1 for escape. When rate > threshold and +# probe countdown > 0, bi_with_arena skips the verifier and goes +# straight to gc_sweep (treats it as an escape). When countdown +# reaches zero, forces a verify as a probe so the policy can +# re-evaluate whether escapes are still dominant. +arena_escape_rate: .skip 8 # 0..256 +arena_verifies_skipped: .skip 8 +arena_probe_countdown: .skip 8 +arena_adaptive_mode: .skip 8 # 0 = greedy (always verify), 1 = adaptive .endif # ============================================================ @@ -510,6 +524,8 @@ _start: leaq HEAP_SIZE(%rax), %rcx movq %rcx, gc_chunk_end(%rip) movq $1, gc_chunk_count(%rip) + # Default: adaptive policy on. (arena-set-mode 0) disables it. + movq $1, arena_adaptive_mode(%rip) .endif # Init global env = 0 (empty) @@ -995,11 +1011,20 @@ gc_mark_env: popq %rdi testq %rcx, %rcx jz .gme_end - # Block's header payload-size must be 24 (our env-node sentinel). + # Block's header payload-size must be 24 AND offset 0 must be + # a tagged symbol. Env nodes are (sym, val, parent); strings + # and closures are also 24 bytes when they happen to land at + # that size, so a bare size check treats them as env and the + # walker corrupts the heap by reading string bytes as env + # fields. Demanding TAG_SYM at offset 0 disambiguates. movq -8(%rdi), %rax shrq $1, %rax cmpq $24, %rax jne .gme_end + movq (%rdi), %rax + andq $7, %rax + cmpq $TAG_SYM, %rax + jne .gme_end # Already marked? Skip. testq $1, -8(%rdi) jnz .gme_end @@ -1137,27 +1162,28 @@ gc_mark_drain: call gc_mark_env # env field is an untagged env chain jmp .gmd_top .gmd_vec7: + # Block size from header. The conservative stack scan can push + # any value whose low 3 bits happen to equal 7; that untagged + # "pointer" might land in a chunk but belong to a string or + # closure, not an actual vector. Validate size against the + # claimed length/layout before walking — otherwise a 40-byte + # string misinterpreted as a 25-element vector reads 200 bytes + # past itself and corrupts marks on adjacent blocks. + movq -8(%rsi), %rax + shrq $1, %rax # block payload size movq (%rsi), %rdx # first word: length or -1 or -2 + cmpq $-2, %rdx + je .gmd_ht_check + cmpq $-1, %rdx + je .gmd_ht_check testq %rdx, %rdx - jns .gmd_vector_body - # Hash-table (-1) or hash-set (-2): buckets 0..nbuckets-1 at offset 24. - movq 16(%rsi), %rcx # nbuckets - xorq %r8, %r8 -.gmd_ht_loop: - cmpq %rcx, %r8 - jae .gmd_top - pushq %rcx - pushq %r8 - pushq %rsi - movq 24(%rsi,%r8,8), %rdi - call gc_push_if_heap - popq %rsi - popq %r8 - popq %rcx - incq %r8 - jmp .gmd_ht_loop -.gmd_vector_body: - # Vector: length = rdx; elements at offset 8. + js .gmd_top # other negatives = not ours + # Vector: length must fit: 8 + length*8 <= block_size + movq %rdx, %rcx + shlq $3, %rcx + addq $8, %rcx + cmpq %rax, %rcx + ja .gmd_top # length too large for block xorq %r8, %r8 .gmd_vec_loop: cmpq %rdx, %r8 @@ -1172,6 +1198,34 @@ gc_mark_drain: popq %rdx incq %r8 jmp .gmd_vec_loop + +.gmd_ht_check: + # Hash-table (-1) or hash-set (-2): require block size >= 24 + # (header) + nbuckets * 8. nbuckets at offset 16. + cmpq $24, %rax + jb .gmd_top + movq 16(%rsi), %rcx # nbuckets + testq %rcx, %rcx + js .gmd_top # garbage + movq %rcx, %rdx + shlq $3, %rdx + addq $24, %rdx + cmpq %rax, %rdx + ja .gmd_top # nbuckets too large + xorq %r8, %r8 +.gmd_ht_loop: + cmpq %rcx, %r8 + jae .gmd_top + pushq %rcx + pushq %r8 + pushq %rsi + movq 24(%rsi,%r8,8), %rdi + call gc_push_if_heap + popq %rsi + popq %r8 + popq %rcx + incq %r8 + jmp .gmd_ht_loop .gmd_done: ret @@ -3323,6 +3377,8 @@ eval_list: je bi_with_arena cmpq $BI_ARENA_STATS, %rax je bi_arena_stats + cmpq $BI_ARENA_SET_MODE, %rax + je bi_arena_set_mode .endif movq $VAL_VOID, %rax @@ -6006,83 +6062,167 @@ bi_hash_set_to_list: # stays pristine for a verbatim restore; heap_alloc enforces this. # ============================================================ -# bi_with_arena: (with-arena thunk) -> thunk's return value +# bi_with_arena: (with-arena thunk) -> thunk's return value. +# Meta-GC dispatch with adaptive policy: +# 1. Run thunk. Zero volatile regs so conservative scan is clean. +# 2. If implicit GC fired mid-thunk (arena_active cleared) -> +# abort. EMA += escape. +# 3. If adaptive mode on AND escape_rate > threshold AND probe +# countdown > 0: SKIP verify, run sweep directly (faster than +# verify+sweep on a hostile workload). EMA += escape. +# 4. Otherwise (greedy mode, low escape rate, or probe fires): +# call arena_verify_and_commit. Its return value (0/1) feeds +# the EMA update. bi_with_arena: GETARG %rbx # thunk - # Snapshot. movq %r15, %rax movq %rax, arena_r15_snap(%rip) movq $1, arena_active(%rip) incq arena_calls(%rip) - # Invoke thunk with no args. movq %rbx, %rdi movq $VAL_NIL, %rsi call apply_proc_raw movq %rax, %rbx # result (tagged) - # Zero every volatile register before verify's stack scan runs. - # apply_proc_raw leaves stale tagged pointers in callee registers - # (closure envs, intermediate pair chains) that the conservative - # scan would otherwise treat as live roots pointing into the - # arena, triggering a false escape. %rbx keeps the result. + # Zero volatile regs so the conservative stack scan in verify + # doesn't pick up stale tagged pointers apply_proc_raw leaves + # behind (would look like live roots pointing into the arena). + # DO NOT zero %rbp — the interpreter keeps the caller's local + # env there, and wiping it loses the binding for the variable + # that friendly-loop / any user-closure caller was resolving. xorq %rax, %rax xorq %rcx, %rcx xorq %rdx, %rdx xorq %rsi, %rsi xorq %rdi, %rdi - xorq %rbp, %rbp xorq %r8, %r8 xorq %r9, %r9 xorq %r10, %r10 xorq %r11, %r11 xorq %r12, %r12 - # If an implicit GC fired during the thunk, arena_active was - # cleared and the snapshot is now stale. Skip reset. + + # (a) implicit-GC abort? cmpq $0, arena_active(%rip) je .bwa_abort_gc - # Verify: mark phase with result as extra root. Then scan the - # arena range for any marked block; if any, it escaped. + + # (b) adaptive skip? + cmpq $0, arena_adaptive_mode(%rip) + je .bwa_verify # greedy mode, always verify + movq arena_escape_rate(%rip), %rax + cmpq $ARENA_SKIP_THRESHOLD, %rax + jbe .bwa_verify # rate low -> verify + cmpq $0, arena_probe_countdown(%rip) + je .bwa_probe # countdown 0 -> probe + # Skip verify entirely. Don't call gc_sweep either — sweep + # without a mark phase would reclaim live data. Leave the heap + # as-is; the arena's allocations stay bumped. Correctness: all + # live data remains reachable via normal roots. Cost: no verify, + # no sweep. When bump eventually overflows, gc_collect fires + # naturally and reclaims dead blocks with a proper mark pass. + decq arena_probe_countdown(%rip) + incq arena_verifies_skipped(%rip) + incq arena_escapes(%rip) + movq $0, arena_active(%rip) + movq $1, %rdi # escape sample for EMA + call arena_update_ema + jmp .bwa_done + +.bwa_probe: + # Probe: reset countdown and run verify as a sample. + movq $ARENA_PROBE_EVERY, %rax + movq %rax, arena_probe_countdown(%rip) + jmp .bwa_verify + +.bwa_verify: movq %rbx, %rdi - call arena_verify_and_commit - movq %rbx, %rax - RET_VAL + call arena_verify_and_commit # returns 0 (reset) or 1 (escape) + movq %rax, %rdi + call arena_update_ema + jmp .bwa_done + .bwa_abort_gc: incq arena_escapes(%rip) + movq $1, %rdi + call arena_update_ema + +.bwa_done: movq %rbx, %rax RET_VAL +# arena_update_ema: %rdi = sample (0 for reset, 1 for escape). +# rate = (rate*7 + sample*256) / 8 +arena_update_ema: + movq arena_escape_rate(%rip), %rax + shlq $3, %rax # rate * 8 + subq arena_escape_rate(%rip), %rax # rate * 7 + testq %rdi, %rdi + jz .aue_zero + addq $256, %rax # escape sample = 256 +.aue_zero: + shrq $3, %rax # / 8 + movq %rax, arena_escape_rate(%rip) + ret + +.equ ARENA_SKIP_THRESHOLD, 128 # 50% escape rate +.equ ARENA_PROBE_EVERY, 16 # force a verify every 16 skipped + +# bi_arena_set_mode: (arena-set-mode 0|1) -> void. 0 = greedy (always +# verify), 1 = adaptive (skip verify when escape rate is high). +bi_arena_set_mode: + GETARG %rax + sarq $3, %rax # untag int + movq %rax, arena_adaptive_mode(%rip) + # Reset EMA and countdown so the mode change starts clean. + movq $0, arena_escape_rate(%rip) + movq $0, arena_probe_countdown(%rip) + movq $VAL_VOID, %rax + RET_VAL + # bi_arena_stats: (arena-stats) -> (list calls resets escapes bytes-reclaimed) bi_arena_stats: - # Build right-to-left: NIL -> (bytes) -> (escapes bytes) -> ... - movq arena_bytes_reclaimed(%rip), %rdi + # Build right-to-left: NIL -> (rate) -> (skipped rate) -> ... + # Final order: (calls resets escapes skipped bytes rate) + movq arena_escape_rate(%rip), %rdi call make_int movq %rax, %rdi movq $VAL_NIL, %rsi - call make_pair # (bytes) + call make_pair # (rate) + movq %rax, %rbx + movq arena_bytes_reclaimed(%rip), %rdi + call make_int + movq %rax, %rdi + movq %rbx, %rsi + call make_pair # (bytes rate) + movq %rax, %rbx + movq arena_verifies_skipped(%rip), %rdi + call make_int + movq %rax, %rdi + movq %rbx, %rsi + call make_pair # (skipped bytes rate) movq %rax, %rbx movq arena_escapes(%rip), %rdi call make_int movq %rax, %rdi movq %rbx, %rsi - call make_pair # (escapes bytes) + call make_pair # (escapes skipped bytes rate) movq %rax, %rbx movq arena_resets(%rip), %rdi call make_int movq %rax, %rdi movq %rbx, %rsi - call make_pair # (resets escapes bytes) + call make_pair # (resets escapes skipped bytes rate) movq %rax, %rbx movq arena_calls(%rip), %rdi call make_int movq %rax, %rdi movq %rbx, %rsi - call make_pair # (calls resets escapes bytes) + call make_pair # (calls resets escapes skipped bytes rate) RET_VAL # arena_verify_and_commit: %rdi = thunk result (tagged). # Runs mark phase, detects escape, commits reset or falls through # to naive sweep. Updates arena_resets / arena_escapes counters. +# Returns %rax = 0 if reset succeeded, 1 if escape (for EMA update). arena_verify_and_commit: - pushq %rax pushq %rbx pushq %rcx pushq %rdx @@ -6203,18 +6343,14 @@ arena_verify_and_commit: movq %rax, %r15 movq $0, arena_active(%rip) incq arena_resets(%rip) - # Now run sweep to clean mark bits on pre-snapshot live blocks - # and rebuild the free list from anything below that became dead - # during the arena (nothing, currently, since we disabled free - # list reuse — but sweep is still the easiest way to clear marks). call gc_sweep + xorq %rax, %rax # return 0 (reset) jmp .avc_done .avc_escape: movq $0, arena_active(%rip) incq arena_escapes(%rip) - # Fall through to naive sweep on the full range, which also - # clears mark bits set during verify. call gc_sweep + movq $1, %rax # return 1 (escape) .avc_done: popq %r12 popq %r11 @@ -6227,7 +6363,6 @@ arena_verify_and_commit: popq %rdx popq %rcx popq %rbx - popq %rax ret .endif diff --git a/examples/bench-gc-adaptive.lsp b/examples/bench-gc-adaptive.lsp new file mode 100644 index 0000000..b5beba0 --- /dev/null +++ b/examples/bench-gc-adaptive.lsp @@ -0,0 +1,51 @@ +;;; bench-gc-adaptive.lsp — driver that runs ONE workload phase in +;;; this process. The outer wrapper (tests/bench-gc-adaptive.sh) +;;; launches asm-gc six times, one per (workload × mode) pair, each +;;; with a clean interpreter. One-phase-per-process isolates state +;;; across runs; cross-phase heap residue on a 1 MB chunk has caused +;;; spurious escape reports in the meta-GC that we haven't nailed +;;; yet, and we'd rather report clean numbers than fight a latent +;;; interaction mid-bench. The MODE and WORKLOAD are injected by +;;; the wrapper via pre-bindings of *mode* and *workload*. + +(define K 200) +(define N 1000) + +(define (build-list k acc) + (if (= k 0) acc (build-list (- k 1) (cons k acc)))) +(define (sum-list lst acc) + (if (null? lst) acc (sum-list (cdr lst) (+ acc (car lst))))) + +(define (friendly-thunk) (sum-list (build-list K '()) 0)) +(define escaped-sink '()) +(define (hostile-step) + (set! escaped-sink (with-arena (lambda () (build-list K '()))))) + +(define (friendly-go n) + (if (= n 0) 'done + (begin (with-arena friendly-thunk) (friendly-go (- n 1))))) +(define (hostile-go n) + (if (= n 0) 'done + (begin (hostile-step) (hostile-go (- n 1))))) +(define (mixed-go n) + (if (= n 0) 'done + (begin + (with-arena friendly-thunk) + (hostile-step) + (mixed-go (- n 1))))) + +(arena-set-mode *mode*) + +(define t0 (current-time-ms)) +(cond + ((= *workload* 0) (friendly-go N)) + ((= *workload* 1) (hostile-go N)) + (else (mixed-go N))) +(define elapsed (- (current-time-ms) t0)) + +(display "workload=") (display *workload*) +(display " mode=") (display *mode*) +(display " N=") (display N) (newline) +(display "time_ms=") (display elapsed) (newline) +(display "arena-stats=") (display (arena-stats)) (newline) +(display "gc-stats=") (display (gc-stats)) (newline) diff --git a/tests/bench-gc-adaptive.sh b/tests/bench-gc-adaptive.sh new file mode 100755 index 0000000..0667689 --- /dev/null +++ b/tests/bench-gc-adaptive.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# bench-gc-adaptive.sh — greedy vs adaptive meta-GC policy across +# friendly / hostile / mixed workloads. Each (workload × mode) pair +# runs in a fresh asm-gc process to isolate GC state. +set -e +cd "$(dirname "$0")/.." + +ulimit -v 524288 +trap 'pkill -9 -u "$USER" -f "asm/uncommonlisp" 2>/dev/null || true' EXIT + +make -s -C asm all + +run_phase() { + local workload_name="$1" + local workload="$2" + local mode="$3" + local mode_name="$4" + echo "== $workload_name / $mode_name ==" + { + echo "(define *workload* $workload)" + echo "(define *mode* $mode)" + cat examples/bench-gc-adaptive.lsp + } | timeout 60 ./asm/uncommonlisp-gc + echo "" +} + +run_phase "friendly" 0 0 "greedy" +run_phase "friendly" 0 1 "adaptive" +run_phase "hostile" 1 0 "greedy" +run_phase "hostile" 1 1 "adaptive" +run_phase "mixed" 2 0 "greedy" +run_phase "mixed" 2 1 "adaptive" + +if pgrep -u "$USER" -f 'asm/uncommonlisp' > /dev/null; then + echo "STRAGGLER" >&2 + exit 1 +fi