engineering/lumbda
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

python + c bytecode VM: OP_SELF_TAIL_CALL frame-unwind fix

Browse source 
Both bytecode VMs had a latent O(n^2) defect on self-recursive tail
calls invoked from inside let/let*/letrec/letrec*/do bodies. The
self-tail-call op assumed reusing "current env" was safe, but current
env was the innermost let* frame, not the lambda body env. Each iter
pushed a fresh let* frame on top (PUSH_ENV at compile site), the
self-tail-call rebound params into that frame & jumped to ip=0 without
unwinding. Env chain grew linearly with iters; every var lookup walked
O(n) chain; effective O(n^2) behaviour.

Symptom observed 2026-06-14: 156k circ-ops walk hung > 5min instead of
1.4s. K=5 doctrine reducers ran 30+ runaway lumbda procs at 99% CPU
across multiple `make sweep-doctrine` invocations before we tracked
it back to language layer (initially misdiagnosed as K=5 substrate).

Fix: track scope depth at compile time on CodeObj (scope_depth bumped
on PUSH_ENV emit, decremented on POP_ENV emit). Record self_base at
lambda body entry (0 unless internal defines pushed a frame). At
self-tail-call emit, encode pops_needed = scope_depth - self_base in
the op arg. Runtime handler unwinds that many env frames before
rebinding params + jumping to ip=0.

Tree-walker (c/lumbda without --fast) already worked - it walks the
ast & lets recursion clean up frames naturally. Asm tier also fine -
no self-tail-call op, uses different lambda-call convention.

Verification:
  python tier: 571 tests PASS, our 100k let* repro 1.04s wall (was infinite)
  c tier:      205 tests PASS, same repro 0.05s wall (was infinite)
  asm tier:    158 tests PASS (no fix needed, never had the bug)

Portal-resume backwards-compat: pre-fix portals stored OP_SELF_TAIL_CALL
arg as 2-tuple. Deserializer fills pops=0 when 'pops' key is absent,
so an old portal resumes at correct behaviour at the cost of slow walk
on its very next self-tail-call body (no worse than pre-fix).

Memory note saved at reference_lumbda_let_star_in_tail_loop in our
foxhop blackops memory for future agents.
This commit is contained in:
russell@unturf.com 2026-06-14 14:58:30 -04:00
parent 991ef661e3
commit 78fbd906f0
No known key found for this signature in database
3 changed files with 69 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

7
c/lumbda.h
View file

@ -538,6 +538,13 @@ typedef struct CodeObj {
const char *self_name;
Value *self_params;
int self_nparams;
/* 2026-06-14 self-tail-call frame-unwind: tracks env-frame depth at
compile time so OP_SELF_TAIL_CALL pops accumulated let/let-star/letrec/do
frames before reusing our lambda body env. Without this, each iter
let-star frame stayed on the env chain; lookup walked O(n) chain;
effective O(n^2). */
int scope_depth;
int self_base;
} CodeObj;
#define IS_CODE(v) (IS_PTR(v) && obj_type(v) == OBJ_CODE)

26
c/vm.c
View file

@ -28,6 +28,10 @@ int code_emit(CodeObj *c, Opcode op, Value arg) {
c->instrs = (Instruction *)ul_realloc_values(c->instrs, sizeof(Instruction) * c->cap);
c->source_map = (int *)ul_realloc(c->source_map, sizeof(int) * c->cap);
}
/* 2026-06-14 self-tail-call frame-unwind: track env-frame depth so
OP_SELF_TAIL_CALL pops accumulated let/let-star/letrec/do frames. */
if (op == OP_PUSH_ENV) c->scope_depth++;
else if (op == OP_POP_ENV) c->scope_depth--;
int idx = c->count;
c->instrs[idx].op = op;
c->instrs[idx].arg = arg;
@ -134,6 +138,9 @@ CodeObj *bc_lambda(Value *body, int nbody, Value *params, int nparams,
code_emit(inner, OP_BIND, def_names[j]);
}
}
/* 2026-06-14: record baseline depth after internal-defines frame.
Self-tail-call unwinds back to here, not all the way to 0. */
inner->self_base = inner->scope_depth;
bc_body(expanded, n, inner, env, true);
code_emit(inner, OP_RETURN, VAL_NIL);
@ -614,7 +621,11 @@ void bc_compile(Value expr, CodeObj *code, Env *env, bool tail) {
if (tail && IS_SYM(head) && code->self_name && strcmp(sym_name(head), code->self_name) == 0) {
Value *call_args; int nca = value_to_list(args, &call_args);
for (int i = 0; i < nca; i++) bc_compile(call_args[i], code, env, false);
code_emit2(code, OP_SELF_TAIL_CALL, VAL_INT(nca), code->self_nparams);
/* 2026-06-14 frame-unwind: pops accumulated let/let-star/letrec/do
frames before we reuse our lambda body env. arg2 now carries
pops_needed (was self_nparams, redundant since cur_code knows it). */
int pops_needed = code->scope_depth - code->self_base;
code_emit2(code, OP_SELF_TAIL_CALL, VAL_INT(nca), pops_needed);
ul_free(call_args); return;
}
@ -963,11 +974,18 @@ Value vm_exec(CodeObj *code, Env *env) {
case OP_LOOK_SUB1: vs_push(&stack, num_sub(env_lookup(env, arg), VAL_INT(1))); break;
case OP_SELF_TAIL_CALL: {
int nargs = (int)as_int(arg);
int nparams_expected = instr->arg2;
int pops = instr->arg2;
Value *args_arr = stack.data + stack.len - nargs;
/* Rebind in current env */
/* 2026-06-14 frame-unwind: pop let/let-star/letrec/do frames
accumulated since lambda body entry. Otherwise env grows
per iter & every var lookup walks an O(n) chain. */
while (pops > 0 && env->parent) {
env = env->parent;
pops--;
}
/* Rebind in lambda body env (post-unwind) */
if (cur_code->self_params) {
for (int i = 0; i < nargs && i < nparams_expected; i++) {
for (int i = 0; i < nargs && i < cur_code->self_nparams; i++) {
env_set(env, cur_code->self_params[i], args_arr[i]);
}
}

47
lumbda.py
View file

@ -1203,13 +1203,28 @@ _BC_FOLDABLE = {
class CodeObj:
"""Compiled bytecode chunk."""
__slots__ = ('instrs', 'name', 'source_map', 'ic', '_cur_line', '_self_name', '_self_params')
__slots__ = ('instrs', 'name', 'source_map', 'ic', '_cur_line',
'_self_name', '_self_params', '_scope_depth', '_self_base')
def __init__(self, name=None):
self.instrs = []; self.name = name
self.source_map = [] # parallel to instrs: line number or None
self.ic = None # inline cache (populated at runtime)
self._cur_line = None # current source line during compilation
# 2026-06-14 self-tail-call frame-unwind: tracks env-frame depth at
# compile time so OP_SELF_TAIL_CALL can pop accumulated let/let*/
# letrec/do frames before reusing our lambda body env. Without this
# a (let* (...) (loop ...)) inside (let loop ...) bloated env per
# iter — 156k-element walk hung > 5 min instead of completing in
# 1.4s. _self_base records depth at lambda body entry; _scope_depth
# is current depth; pops_needed = depth - base at tail call site.
self._scope_depth = 0
self._self_base = 0
def emit(self, op, arg=None):
# 2026-06-14 self-tail-call frame-unwind: track env-frame depth so
# OP_SELF_TAIL_CALL knows how many let/let*/letrec/do frames sit
# between us & our lambda body env.
if op == OP_PUSH_ENV: self._scope_depth += 1
elif op == OP_POP_ENV: self._scope_depth -= 1
idx = len(self.instrs); self.instrs.append((op, arg))
self.source_map.append(self._cur_line)
return idx
@ -1529,7 +1544,12 @@ def _bc(expr, code, env, tail=False):
if tail and isinstance(head, Symbol) and hasattr(code, '_self_name') and str(head) == code._self_name:
params = code._self_params
for arg in call_args: _bc(arg, code, env)
code.emit(OP_SELF_TAIL_CALL, (len(call_args), tuple(params))); return
# 2026-06-14 frame-unwind: pop accumulated let/let*/letrec/do frames
# before we reuse our lambda body env. Without this each iter's
# let* frame stays on the env chain — env grows linearly with iters
# & every var lookup walks an O(n) chain → effective O(n^2).
pops_needed = code._scope_depth - code._self_base
code.emit(OP_SELF_TAIL_CALL, (len(call_args), tuple(params), pops_needed)); return
# --- Function call ---
_bc(head, code, env)
@ -1569,6 +1589,9 @@ def _bc_lambda(body, params, rest, env, name=None, self_name=None, self_params=N
if def_names:
inner.emit(OP_PUSH_ENV)
for nm in def_names: inner.emit(OP_VOID); inner.emit(OP_BIND, nm)
# 2026-06-14: record baseline depth after any internal-defines frame.
# Our self-tail-call unwind pops back to here, not all the way to 0.
inner._self_base = inner._scope_depth
_bc_body(body_list, inner, env, tail=True)
inner.emit(OP_RETURN)
_peephole(inner)
@ -1832,9 +1855,14 @@ def _vm_loop(instrs, ip, stack, env, frames, vm_id):
c, addr = arg
if _po() != c: ip = addr
elif op == OP_SELF_TAIL_CALL:
n, params = arg
# 2026-06-14: arg now (n_args, params, pops_needed). pops_needed
# unwinds accumulated let/let*/letrec/do frames before we reuse
# our lambda body env — otherwise self-tail-call from inside a
# let* bloats env per iter & every var lookup walks O(n) chain.
n, params, pops = arg
if n: args_ = stack[-n:]; del stack[-n:]
else: args_ = []
for _ in range(pops): env = env.p
b = env.b
for p, a in zip(params, args_): b[p] = a
ip = 0; stack.clear(); continue
@ -1888,9 +1916,12 @@ def _serialize_operand(val):
return {'t': 'closure', 'code': _serialize_code(code),
'params': [str(p) for p in params],
'rest': str(rest) if rest else None}
# OP_SELF_TAIL_CALL: (n_args, params_tuple)
# OP_SELF_TAIL_CALL: (n_args, params_tuple, pops_needed)
if len(val) == 3 and isinstance(val[0], int) and isinstance(val[1], tuple) and isinstance(val[2], int):
return {'t': 'stc', 'n': val[0], 'p': [str(p) for p in val[1]], 'pops': val[2]}
# Backwards-compat: pre-2026-06-14 portals have 2-tuple form.
if len(val) == 2 and isinstance(val[0], int) and isinstance(val[1], tuple):
return {'t': 'stc', 'n': val[0], 'p': [str(p) for p in val[1]]}
return {'t': 'stc', 'n': val[0], 'p': [str(p) for p in val[1]], 'pops': 0}
return {'t': 'repr', 'v': repr(val)}
def _deserialize_operand(data):
@ -1921,7 +1952,8 @@ def _deserialize_operand(data):
rest = S(data['rest']) if data['rest'] else None
return (code, params, rest)
if t == 'stc':
return (data['n'], tuple(S(p) for p in data['p']))
# 2026-06-14: stc now carries pops field; older portals (no pops) → 0.
return (data['n'], tuple(S(p) for p in data['p']), data.get('pops', 0))
return data
def _serialize_code(code):
@ -2632,7 +2664,8 @@ def _jit_transpile(instrs, params, name, has_self_tc):
val = stack.pop() if stack else 'VOID'
stmts.append(('return', val)); ip+=1
elif op == OP_SELF_TAIL_CALL:
tc_n, tc_params = arg
# arg is (n_args, params, pops_needed) since 2026-06-14
tc_n = arg[0]; tc_params = arg[1]
pnames = [str(p) for p in tc_params]
args = [];
for _ in range(tc_n): args.insert(0, stack.pop())