From 78fbd906f048d564d1351d21517a893294f84c19 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Sun, 14 Jun 2026 14:58:30 -0400 Subject: [PATCH] python + c bytecode VM: OP_SELF_TAIL_CALL frame-unwind fix Both bytecode VMs had a latent O(n^2) defect on self-recursive tail calls invoked from inside let/let*/letrec/letrec*/do bodies. The self-tail-call op assumed reusing "current env" was safe, but current env was the innermost let* frame, not the lambda body env. Each iter pushed a fresh let* frame on top (PUSH_ENV at compile site), the self-tail-call rebound params into that frame & jumped to ip=0 without unwinding. Env chain grew linearly with iters; every var lookup walked O(n) chain; effective O(n^2) behaviour. Symptom observed 2026-06-14: 156k circ-ops walk hung > 5min instead of 1.4s. K=5 doctrine reducers ran 30+ runaway lumbda procs at 99% CPU across multiple `make sweep-doctrine` invocations before we tracked it back to language layer (initially misdiagnosed as K=5 substrate). Fix: track scope depth at compile time on CodeObj (scope_depth bumped on PUSH_ENV emit, decremented on POP_ENV emit). Record self_base at lambda body entry (0 unless internal defines pushed a frame). At self-tail-call emit, encode pops_needed = scope_depth - self_base in the op arg. Runtime handler unwinds that many env frames before rebinding params + jumping to ip=0. Tree-walker (c/lumbda without --fast) already worked - it walks the ast & lets recursion clean up frames naturally. Asm tier also fine - no self-tail-call op, uses different lambda-call convention. Verification: python tier: 571 tests PASS, our 100k let* repro 1.04s wall (was infinite) c tier: 205 tests PASS, same repro 0.05s wall (was infinite) asm tier: 158 tests PASS (no fix needed, never had the bug) Portal-resume backwards-compat: pre-fix portals stored OP_SELF_TAIL_CALL arg as 2-tuple. Deserializer fills pops=0 when 'pops' key is absent, so an old portal resumes at correct behaviour at the cost of slow walk on its very next self-tail-call body (no worse than pre-fix). Memory note saved at reference_lumbda_let_star_in_tail_loop in our foxhop blackops memory for future agents. --- c/lumbda.h | 7 +++++++ c/vm.c | 26 ++++++++++++++++++++++---- lumbda.py | 47 ++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 69 insertions(+), 11 deletions(-) diff --git a/c/lumbda.h b/c/lumbda.h index ad300fc..9579525 100644 --- a/c/lumbda.h +++ b/c/lumbda.h @@ -538,6 +538,13 @@ typedef struct CodeObj { const char *self_name; Value *self_params; int self_nparams; + /* 2026-06-14 self-tail-call frame-unwind: tracks env-frame depth at + compile time so OP_SELF_TAIL_CALL pops accumulated let/let-star/letrec/do + frames before reusing our lambda body env. Without this, each iter + let-star frame stayed on the env chain; lookup walked O(n) chain; + effective O(n^2). */ + int scope_depth; + int self_base; } CodeObj; #define IS_CODE(v) (IS_PTR(v) && obj_type(v) == OBJ_CODE) diff --git a/c/vm.c b/c/vm.c index 1bd24e6..4227a3e 100644 --- a/c/vm.c +++ b/c/vm.c @@ -28,6 +28,10 @@ int code_emit(CodeObj *c, Opcode op, Value arg) { c->instrs = (Instruction *)ul_realloc_values(c->instrs, sizeof(Instruction) * c->cap); c->source_map = (int *)ul_realloc(c->source_map, sizeof(int) * c->cap); } + /* 2026-06-14 self-tail-call frame-unwind: track env-frame depth so + OP_SELF_TAIL_CALL pops accumulated let/let-star/letrec/do frames. */ + if (op == OP_PUSH_ENV) c->scope_depth++; + else if (op == OP_POP_ENV) c->scope_depth--; int idx = c->count; c->instrs[idx].op = op; c->instrs[idx].arg = arg; @@ -134,6 +138,9 @@ CodeObj *bc_lambda(Value *body, int nbody, Value *params, int nparams, code_emit(inner, OP_BIND, def_names[j]); } } + /* 2026-06-14: record baseline depth after internal-defines frame. + Self-tail-call unwinds back to here, not all the way to 0. */ + inner->self_base = inner->scope_depth; bc_body(expanded, n, inner, env, true); code_emit(inner, OP_RETURN, VAL_NIL); @@ -614,7 +621,11 @@ void bc_compile(Value expr, CodeObj *code, Env *env, bool tail) { if (tail && IS_SYM(head) && code->self_name && strcmp(sym_name(head), code->self_name) == 0) { Value *call_args; int nca = value_to_list(args, &call_args); for (int i = 0; i < nca; i++) bc_compile(call_args[i], code, env, false); - code_emit2(code, OP_SELF_TAIL_CALL, VAL_INT(nca), code->self_nparams); + /* 2026-06-14 frame-unwind: pops accumulated let/let-star/letrec/do + frames before we reuse our lambda body env. arg2 now carries + pops_needed (was self_nparams, redundant since cur_code knows it). */ + int pops_needed = code->scope_depth - code->self_base; + code_emit2(code, OP_SELF_TAIL_CALL, VAL_INT(nca), pops_needed); ul_free(call_args); return; } @@ -963,11 +974,18 @@ Value vm_exec(CodeObj *code, Env *env) { case OP_LOOK_SUB1: vs_push(&stack, num_sub(env_lookup(env, arg), VAL_INT(1))); break; case OP_SELF_TAIL_CALL: { int nargs = (int)as_int(arg); - int nparams_expected = instr->arg2; + int pops = instr->arg2; Value *args_arr = stack.data + stack.len - nargs; - /* Rebind in current env */ + /* 2026-06-14 frame-unwind: pop let/let-star/letrec/do frames + accumulated since lambda body entry. Otherwise env grows + per iter & every var lookup walks an O(n) chain. */ + while (pops > 0 && env->parent) { + env = env->parent; + pops--; + } + /* Rebind in lambda body env (post-unwind) */ if (cur_code->self_params) { - for (int i = 0; i < nargs && i < nparams_expected; i++) { + for (int i = 0; i < nargs && i < cur_code->self_nparams; i++) { env_set(env, cur_code->self_params[i], args_arr[i]); } } diff --git a/lumbda.py b/lumbda.py index 3863d12..8ed1fa0 100644 --- a/lumbda.py +++ b/lumbda.py @@ -1203,13 +1203,28 @@ _BC_FOLDABLE = { class CodeObj: """Compiled bytecode chunk.""" - __slots__ = ('instrs', 'name', 'source_map', 'ic', '_cur_line', '_self_name', '_self_params') + __slots__ = ('instrs', 'name', 'source_map', 'ic', '_cur_line', + '_self_name', '_self_params', '_scope_depth', '_self_base') def __init__(self, name=None): self.instrs = []; self.name = name self.source_map = [] # parallel to instrs: line number or None self.ic = None # inline cache (populated at runtime) self._cur_line = None # current source line during compilation + # 2026-06-14 self-tail-call frame-unwind: tracks env-frame depth at + # compile time so OP_SELF_TAIL_CALL can pop accumulated let/let*/ + # letrec/do frames before reusing our lambda body env. Without this + # a (let* (...) (loop ...)) inside (let loop ...) bloated env per + # iter — 156k-element walk hung > 5 min instead of completing in + # 1.4s. _self_base records depth at lambda body entry; _scope_depth + # is current depth; pops_needed = depth - base at tail call site. + self._scope_depth = 0 + self._self_base = 0 def emit(self, op, arg=None): + # 2026-06-14 self-tail-call frame-unwind: track env-frame depth so + # OP_SELF_TAIL_CALL knows how many let/let*/letrec/do frames sit + # between us & our lambda body env. + if op == OP_PUSH_ENV: self._scope_depth += 1 + elif op == OP_POP_ENV: self._scope_depth -= 1 idx = len(self.instrs); self.instrs.append((op, arg)) self.source_map.append(self._cur_line) return idx @@ -1529,7 +1544,12 @@ def _bc(expr, code, env, tail=False): if tail and isinstance(head, Symbol) and hasattr(code, '_self_name') and str(head) == code._self_name: params = code._self_params for arg in call_args: _bc(arg, code, env) - code.emit(OP_SELF_TAIL_CALL, (len(call_args), tuple(params))); return + # 2026-06-14 frame-unwind: pop accumulated let/let*/letrec/do frames + # before we reuse our lambda body env. Without this each iter's + # let* frame stays on the env chain — env grows linearly with iters + # & every var lookup walks an O(n) chain → effective O(n^2). + pops_needed = code._scope_depth - code._self_base + code.emit(OP_SELF_TAIL_CALL, (len(call_args), tuple(params), pops_needed)); return # --- Function call --- _bc(head, code, env) @@ -1569,6 +1589,9 @@ def _bc_lambda(body, params, rest, env, name=None, self_name=None, self_params=N if def_names: inner.emit(OP_PUSH_ENV) for nm in def_names: inner.emit(OP_VOID); inner.emit(OP_BIND, nm) + # 2026-06-14: record baseline depth after any internal-defines frame. + # Our self-tail-call unwind pops back to here, not all the way to 0. + inner._self_base = inner._scope_depth _bc_body(body_list, inner, env, tail=True) inner.emit(OP_RETURN) _peephole(inner) @@ -1832,9 +1855,14 @@ def _vm_loop(instrs, ip, stack, env, frames, vm_id): c, addr = arg if _po() != c: ip = addr elif op == OP_SELF_TAIL_CALL: - n, params = arg + # 2026-06-14: arg now (n_args, params, pops_needed). pops_needed + # unwinds accumulated let/let*/letrec/do frames before we reuse + # our lambda body env — otherwise self-tail-call from inside a + # let* bloats env per iter & every var lookup walks O(n) chain. + n, params, pops = arg if n: args_ = stack[-n:]; del stack[-n:] else: args_ = [] + for _ in range(pops): env = env.p b = env.b for p, a in zip(params, args_): b[p] = a ip = 0; stack.clear(); continue @@ -1888,9 +1916,12 @@ def _serialize_operand(val): return {'t': 'closure', 'code': _serialize_code(code), 'params': [str(p) for p in params], 'rest': str(rest) if rest else None} - # OP_SELF_TAIL_CALL: (n_args, params_tuple) + # OP_SELF_TAIL_CALL: (n_args, params_tuple, pops_needed) + if len(val) == 3 and isinstance(val[0], int) and isinstance(val[1], tuple) and isinstance(val[2], int): + return {'t': 'stc', 'n': val[0], 'p': [str(p) for p in val[1]], 'pops': val[2]} + # Backwards-compat: pre-2026-06-14 portals have 2-tuple form. if len(val) == 2 and isinstance(val[0], int) and isinstance(val[1], tuple): - return {'t': 'stc', 'n': val[0], 'p': [str(p) for p in val[1]]} + return {'t': 'stc', 'n': val[0], 'p': [str(p) for p in val[1]], 'pops': 0} return {'t': 'repr', 'v': repr(val)} def _deserialize_operand(data): @@ -1921,7 +1952,8 @@ def _deserialize_operand(data): rest = S(data['rest']) if data['rest'] else None return (code, params, rest) if t == 'stc': - return (data['n'], tuple(S(p) for p in data['p'])) + # 2026-06-14: stc now carries pops field; older portals (no pops) → 0. + return (data['n'], tuple(S(p) for p in data['p']), data.get('pops', 0)) return data def _serialize_code(code): @@ -2632,7 +2664,8 @@ def _jit_transpile(instrs, params, name, has_self_tc): val = stack.pop() if stack else 'VOID' stmts.append(('return', val)); ip+=1 elif op == OP_SELF_TAIL_CALL: - tc_n, tc_params = arg + # arg is (n_args, params, pops_needed) since 2026-06-14 + tc_n = arg[0]; tc_params = arg[1] pnames = [str(p) for p in tc_params] args = []; for _ in range(tc_n): args.insert(0, stack.pop())