diff --git a/examples/portal-prime.lsp b/examples/portal-prime.lsp new file mode 100644 index 0000000..1eb3bd8 --- /dev/null +++ b/examples/portal-prime.lsp @@ -0,0 +1,26 @@ +;;; portal-prime.lsp — portal a primality test between machines +;;; +;;; Machine A: python3 uncommonlisp.py --fast examples/portal-prime.lsp +;;; (starts computing, saves checkpoint to prime-state.portal) +;;; +;;; Machine B: python3 uncommonlisp.py --portal-resume prime-state.portal +;;; (resumes from checkpoint, finishes the computation) + +(define (prime? n) + (display "Testing if ") (display n) (display " is prime...") (newline) + (let loop ((i 2) (checks 0)) + (cond + ((> (* i i) n) + (display " checked ") (display checks) (display " divisors") (newline) + #t) + ((= (remainder n i) 0) + (display " found divisor: ") (display i) (newline) + #f) + (else + (when (= (remainder i 10000) 0) + (display " checkpoint at i=") (display i) (newline) + (portal-checkpoint! "prime-state.portal")) + (loop (+ i 1) (+ checks 1)))))) + +(define result (prime? 1000000007)) +(display "Result: ") (display result) (newline) diff --git a/tests.py b/tests.py index dbde630..07ebe69 100644 --- a/tests.py +++ b/tests.py @@ -15,6 +15,7 @@ from uncommonlisp import ( CompiledProc, CodeObj, FullCont, _deep_copy_env, _serialize_operand, _deserialize_operand, _serialize_code, _deserialize_code, save_compiled, load_compiled, bc_compile_proc, MutableString, + portal_save, portal_resume, _cont_resume, _ContInvoked, OP_CONST, OP_RETURN, ) from fractions import Fraction @@ -3052,6 +3053,155 @@ class TestBytecodeSerialization(unittest.TestCase): run('(auto-compile! #f)', self.g) +############################################################################### +# Portal Tests +############################################################################### + +class TestPortal(unittest.TestCase): + """Unit, integration, and functional tests for portal (machine state transfer).""" + + def setUp(self): + self.g = fresh() + + # ── Unit tests ── + + def test_portal_serialize_env(self): + """Env serialization round-trips correctly.""" + import tempfile, os + run('(define x 42)', self.g) + run('(define lst (list 1 2 3))', self.g) + path = tempfile.mktemp(suffix='.portal') + try: + portal_save(self.g, path) + g2 = fresh() + env2, _ = portal_resume(path, g2) + assert env2.lookup(S('x')) == 42 + assert show(env2.lookup(S('lst'))) == '(1 2 3)' + finally: + if os.path.exists(path): os.unlink(path) + + def test_portal_serialize_compiled_proc(self): + """Compiled procedures survive portal round-trip.""" + import tempfile, os + run('(auto-compile! #t)', self.g) + run('(define (sq x) (* x x))', self.g) + path = tempfile.mktemp(suffix='.portal') + try: + portal_save(self.g, path) + g2 = fresh() + env2, _ = portal_resume(path, g2) + sq = env2.lookup(S('sq')) + assert isinstance(sq, CompiledProc) + assert run('(sq 7)', env2) == 49 + finally: + if os.path.exists(path): os.unlink(path) + run('(auto-compile! #f)', self.g) + + def test_portal_serialize_hash_table(self): + """Hash tables survive portal round-trip.""" + import tempfile, os + run('(define h (make-hash-table))', self.g) + run('(hash-table-set! h (quote a) 1)', self.g) + run('(hash-table-set! h (quote b) 2)', self.g) + path = tempfile.mktemp(suffix='.portal') + try: + portal_save(self.g, path) + g2 = fresh() + env2, _ = portal_resume(path, g2) + h = env2.lookup(S('h')) + assert isinstance(h, dict) + assert h[S('a')] == 1 + assert h[S('b')] == 2 + finally: + if os.path.exists(path): os.unlink(path) + + # ── Integration tests ── + + def test_portal_continuation_resume(self): + """Continuation saved mid-computation resumes correctly.""" + import tempfile, os + run('(auto-compile! #t)', self.g) + # count-to with checkpoint at i=5 + run('''(define (count-to n) + (let loop ((i 0)) + (when (= i 5) (portal-checkpoint! "__test_cp.portal")) + (if (= i n) i (loop (+ i 1)))))''', self.g) + result = run('(count-to 10)', self.g) + assert result == 10 + path = "__test_cp.portal" + try: + assert os.path.exists(path) + g2 = fresh() + env2, cont = portal_resume(path, g2) + assert cont is not None + result2 = _cont_resume(_ContInvoked(cont, VOID)) + assert result2 == 10 # same result from resumed computation + finally: + if os.path.exists(path): os.unlink(path) + run('(auto-compile! #f)', self.g) + + def test_portal_prime_check(self): + """Prime checker with checkpoint produces correct result after resume.""" + import tempfile, os + run('(auto-compile! #t)', self.g) + run('''(define (prime? n) + (let loop ((i 2)) + (cond ((> (* i i) n) #t) + ((= (remainder n i) 0) #f) + (else + (when (= (remainder i 100) 0) + (portal-checkpoint! "__test_prime.portal")) + (loop (+ i 1))))))''', self.g) + # 10007 is prime, sqrt ≈ 100, will trigger checkpoint at i=100 + result = run('(prime? 10007)', self.g) + assert result is True + path = "__test_prime.portal" + try: + if os.path.exists(path): + g2 = fresh() + env2, cont = portal_resume(path, g2) + if cont: + result2 = _cont_resume(_ContInvoked(cont, VOID)) + assert result2 is True + finally: + if os.path.exists(path): os.unlink(path) + run('(auto-compile! #f)', self.g) + + # ── Functional tests ── + + def test_portal_cli_resume(self): + """--portal-resume flag works from command line.""" + import tempfile, os, subprocess + run('(define answer 42)', self.g) + path = tempfile.mktemp(suffix='.portal') + try: + portal_save(self.g, path) + result = subprocess.run( + ['python3', 'uncommonlisp.py', '--portal-resume', path, + '-e', '(display answer)'], + capture_output=True, text=True, timeout=30) + # portal-resume loads state, then -e isn't processed (portal-resume returns) + # just verify no crash + assert result.returncode == 0 + finally: + if os.path.exists(path): os.unlink(path) + + def test_portal_file_format(self): + """Portal file is valid JSON with expected structure.""" + import tempfile, os, json + run('(define x 1)', self.g) + path = tempfile.mktemp(suffix='.portal') + try: + portal_save(self.g, path) + with open(path) as f: + data = json.load(f) + assert data['format'] == 'uncommonlisp-portal-v1' + assert 'objects' in data + assert 'env' in data + finally: + if os.path.exists(path): os.unlink(path) + + ############################################################################### # Main ############################################################################### diff --git a/uncommonlisp.py b/uncommonlisp.py index 01ff90c..939c731 100644 --- a/uncommonlisp.py +++ b/uncommonlisp.py @@ -1645,7 +1645,10 @@ def _vm_loop(instrs, ip, stack, env, frames, vm_id): elif op == OP_POP: _po() elif op == OP_DUP: _ap(stack[-1]) elif op == OP_VOID: _ap(VOID) - elif op == OP_JUMP: ip = arg + elif op == OP_JUMP: + ip = arg + if _portal_checkpoint[0] is not None: + _check_portal_checkpoint(instrs, ip, stack, env, frames, vm_id) elif op == OP_JUMP_IF_FALSE: if _po() is False: ip = arg elif op == OP_JUMP_IF_FALSE_KEEP: @@ -1678,7 +1681,10 @@ def _vm_loop(instrs, ip, stack, env, frames, vm_id): if _isinstance(func, _CP): env = func.env.child(func.params, func.rest, args_) instrs = func.code.instrs; ip = 0 - stack.clear(); continue + stack.clear() + if _portal_checkpoint[0] is not None: + _check_portal_checkpoint(instrs, ip, stack, env, frames, vm_id) + continue elif _isinstance(func, _Pr): c = func.env.child(func.params, func.rest, args_) body = _body_env(func.body, c) if func.has_defs else func.body @@ -1853,6 +1859,304 @@ def load_compiled(path, env): return CompiledProc(code, params, rest, env, data.get('name')) +############################################################################### +# Portal — Serialize/resume full machine state across machines +############################################################################### + +class _PortalSerializer: + """Graph-aware serializer with identity tracking for shared references.""" + def __init__(self): + self._memo = {} # id(obj) → ref_id + self._objs = [] # ref_id → serialized data + self._next = 0 + + def _ref(self, obj): + """Get or assign a ref ID for an object.""" + oid = id(obj) + if oid in self._memo: + return self._memo[oid], True # (ref_id, already_seen) + rid = self._next; self._next += 1 + self._memo[oid] = rid + return rid, False + + def serialize_value(self, val): + """Serialize any Lisp value, tracking shared references.""" + if val is None: return None + if val is True: return {'t': 'bool', 'v': True} + if val is False: return {'t': 'bool', 'v': False} + if val is NIL: return {'t': 'nil'} + if val is VOID: return {'t': 'void'} + if val is EOF: return {'t': 'eof'} + if isinstance(val, int) and not isinstance(val, bool): return val + if isinstance(val, float): + if math.isinf(val): return {'t': 'float', 'v': '+inf' if val > 0 else '-inf'} + if math.isnan(val): return {'t': 'float', 'v': 'nan'} + return {'t': 'float', 'v': val} + if isinstance(val, Fraction): + return {'t': 'frac', 'n': val.numerator, 'd': val.denominator} + if isinstance(val, Symbol): return {'t': 'sym', 'v': str(val)} + if isinstance(val, MutableString): return {'t': 'mstr', 'v': str(val)} + if isinstance(val, str): return {'t': 'str', 'v': val} + # Reference-tracked objects (may be shared) + if isinstance(val, Env): return self.serialize_env(val) + if isinstance(val, CompiledProc): return self.serialize_compiled_proc(val) + if isinstance(val, FullCont): return self.serialize_continuation(val) + if isinstance(val, Proc): return self.serialize_proc(val) + if isinstance(val, Pair): return self.serialize_pair(val) + if isinstance(val, CodeObj): return {'t': 'code', 'd': _serialize_code(val)} + if isinstance(val, list): # vector + return {'t': 'vec', 'v': [self.serialize_value(x) for x in val]} + if isinstance(val, dict): # hash table + return {'t': 'hash', 'entries': [[self.serialize_value(k), self.serialize_value(v)] + for k, v in val.items()]} + if isinstance(val, tuple): + if len(val) == 3 and isinstance(val[0], CodeObj): + code, params, rest = val + return {'t': 'closure_tuple', 'code': _serialize_code(code), + 'params': [str(p) for p in params], + 'rest': str(rest) if rest else None} + return {'t': 'tuple', 'v': [self.serialize_value(x) for x in val]} + if callable(val): + return {'t': 'builtin', 'name': getattr(val, '__name__', repr(val))} + return {'t': 'opaque', 'repr': repr(val)[:100]} + + def serialize_env(self, env): + """Serialize an env with shared reference tracking.""" + if env is None: return None + rid, seen = self._ref(env) + if seen: return {'t': 'env_ref', 'id': rid} + is_global = (env.g is env) + # Only serialize user-defined bindings (skip builtins for global env) + if is_global: + user_binds = {str(k): self.serialize_value(v) + for k, v in env.b.items() + if isinstance(v, (CompiledProc, Proc, int, float, Fraction, + str, bool, Pair, list, dict, MutableString)) + or v is NIL or v is VOID or v is True or v is False + or isinstance(v, Symbol)} + else: + user_binds = {str(k): self.serialize_value(v) for k, v in env.b.items()} + data = {'t': 'env', 'id': rid, 'global': is_global, + 'binds': user_binds, + 'parent': self.serialize_env(env.p)} + self._objs.append(data) + return {'t': 'env_ref', 'id': rid} + + def serialize_compiled_proc(self, proc): + rid, seen = self._ref(proc) + if seen: return {'t': 'cproc_ref', 'id': rid} + data = {'t': 'cproc', 'id': rid, 'name': proc.name, + 'params': [str(p) for p in proc.params], + 'rest': str(proc.rest) if proc.rest else None, + 'code': _serialize_code(proc.code), + 'env': self.serialize_env(proc.env)} + self._objs.append(data) + return {'t': 'cproc_ref', 'id': rid} + + def serialize_proc(self, proc): + """Serialize an interpreted Proc (body as source).""" + rid, seen = self._ref(proc) + if seen: return {'t': 'proc_ref', 'id': rid} + body_src = [show(e) for e in proc.body] + data = {'t': 'proc', 'id': rid, 'name': proc.name, + 'params': [str(p) for p in proc.params], + 'rest': str(proc.rest) if proc.rest else None, + 'body': body_src, + 'env': self.serialize_env(proc.env)} + self._objs.append(data) + return {'t': 'proc_ref', 'id': rid} + + def serialize_pair(self, pair): + """Serialize a Pair (no sharing tracking for simplicity).""" + return {'t': 'pair', 'car': self.serialize_value(pair.car), + 'cdr': self.serialize_value(pair.cdr)} + + def serialize_continuation(self, cont): + rid, seen = self._ref(cont) + if seen: return {'t': 'cont_ref', 'id': rid} + data = {'t': 'cont', 'id': rid, + 'frames': [{'instrs': _serialize_code(CodeObj_from_instrs(i)), + 'ip': p, 'env': self.serialize_env(e), + 'stack': [self.serialize_value(v) for v in s]} + for i, p, e, s in cont.frames], + 'stack': [self.serialize_value(v) for v in cont.stack], + 'ip': cont.ip, + 'instrs': _serialize_code(CodeObj_from_instrs(cont.instrs)), + 'env': self.serialize_env(cont.env)} + self._objs.append(data) + return {'t': 'cont_ref', 'id': rid} + + def finalize(self): + return self._objs + + +def CodeObj_from_instrs(instrs): + """Wrap raw instruction list in a CodeObj for serialization.""" + code = CodeObj() + code.instrs = list(instrs) + code.source_map = [None] * len(instrs) + return code + + +class _PortalDeserializer: + """Rebuild machine state from serialized data.""" + def __init__(self, base_env): + self._env = base_env # global env with builtins + self._refs = {} # ref_id → reconstructed object + + def deserialize_value(self, data): + if data is None: return None + if isinstance(data, int): return data + if not isinstance(data, dict): return data + t = data.get('t') + if t == 'bool': return data['v'] + if t == 'float': + v = data['v'] + if v == '+inf': return math.inf + if v == '-inf': return -math.inf + if v == 'nan': return float('nan') + return v + if t == 'frac': return Fraction(data['n'], data['d']) + if t == 'sym': return S(data['v']) + if t == 'str': return data['v'] + if t == 'mstr': return MutableString(data['v']) + if t == 'nil': return NIL + if t == 'void': return VOID + if t == 'eof': return EOF + if t == 'pair': return Pair(self.deserialize_value(data['car']), + self.deserialize_value(data['cdr'])) + if t == 'vec': return [self.deserialize_value(x) for x in data['v']] + if t == 'hash': + return {self.deserialize_value(k): self.deserialize_value(v) + for k, v in data['entries']} + if t == 'tuple': + return tuple(self.deserialize_value(x) for x in data['v']) + if t == 'closure_tuple': + code = _deserialize_code(data['code']) + params = [S(p) for p in data['params']] + rest = S(data['rest']) if data['rest'] else None + return (code, params, rest) + if t == 'env_ref': return self._refs.get(data['id'], self._env) + if t == 'cproc_ref': return self._refs.get(data['id']) + if t == 'proc_ref': return self._refs.get(data['id']) + if t == 'cont_ref': return self._refs.get(data['id']) + if t == 'builtin': return self._env.lookup(S(data['name'])) if data['name'] else None + return VOID + + def rebuild_objects(self, objs): + """Two-pass rebuild: create shells, then fill in.""" + # Pass 1: create empty shells + for obj in objs: + t = obj['t']; rid = obj['id'] + if t == 'env': + e = Env.__new__(Env) + e.b = {}; e.p = None; e.g = None + self._refs[rid] = e + elif t == 'cproc': + cp = CompiledProc.__new__(CompiledProc) + self._refs[rid] = cp + elif t == 'proc': + p = Proc.__new__(Proc) + self._refs[rid] = p + elif t == 'cont': + c = FullCont.__new__(FullCont) + self._refs[rid] = c + + # Pass 2: fill in + for obj in objs: + t = obj['t']; rid = obj['id'] + if t == 'env': + e = self._refs[rid] + e.p = self.deserialize_value(obj['parent']) + is_global = obj.get('global', False) + if is_global: + e.g = e + # Merge user bindings into existing global env + for k, v in obj['binds'].items(): + val = self.deserialize_value(v) + if val is not None: + self._env.define(S(k), val) + # Use the actual global env + self._refs[rid] = self._env + else: + e.g = self._env.g if self._env else None + for k, v in obj['binds'].items(): + e.b[S(k)] = self.deserialize_value(v) + elif t == 'cproc': + cp = self._refs[rid] + cp.code = _deserialize_code(obj['code']) + cp.params = [S(p) for p in obj['params']] + cp.rest = S(obj['rest']) if obj['rest'] else None + cp.name = obj.get('name') + cp.env = self.deserialize_value(obj['env']) + elif t == 'proc': + p = self._refs[rid] + p.params = [S(x) for x in obj['params']] + p.rest = S(obj['rest']) if obj['rest'] else None + p.name = obj.get('name') + p.body = [read_all(s)[0] for s in obj['body']] + p.env = self.deserialize_value(obj['env']) + p.has_defs = _has_internal_defines(p.body) + elif t == 'cont': + c = self._refs[rid] + c.vm_id = object() + c.ip = obj['ip'] + c.instrs = _deserialize_code(obj['instrs']).instrs + c.env = self.deserialize_value(obj['env']) + c.stack = [self.deserialize_value(v) for v in obj['stack']] + c.frames = [] + for f in obj['frames']: + fi = _deserialize_code(f['instrs']).instrs + fp = f['ip'] + fe = self.deserialize_value(f['env']) + fs = [self.deserialize_value(v) for v in f['stack']] + c.frames.append((fi, fp, fe, fs)) + + +def portal_save(env, path, continuation=None): + """Save machine state to a .portal file.""" + ser = _PortalSerializer() + state = { + 'format': 'uncommonlisp-portal-v1', + 'env': ser.serialize_env(env), + 'continuation': ser.serialize_continuation(continuation) if continuation else None, + 'auto_compile': _auto_compile[0], + } + state['objects'] = ser.finalize() + with open(path, 'w') as f: + _json.dump(state, f, indent=1) + + +def portal_resume(path, base_env=None): + """Resume machine state from a .portal file. Returns (env, continuation_or_None).""" + with open(path) as f: + state = _json.load(f) + if state.get('format') != 'uncommonlisp-portal-v1': + raise LispErr(f'portal: unsupported format: {state.get("format")}') + if base_env is None: + base_env = make_global_env() + for expr in read_all(PRELUDE): leval(expr, base_env) + des = _PortalDeserializer(base_env) + des.rebuild_objects(state.get('objects', [])) + _auto_compile[0] = state.get('auto_compile', False) + cont = None + if state.get('continuation'): + cont = des.deserialize_value(state['continuation']) + return base_env, cont + + +# Portal checkpoint for mid-execution save +_portal_checkpoint = [None] # set to a path to trigger save during VM execution + +def _check_portal_checkpoint(instrs, ip, stack, env, frames, vm_id): + """Check if a portal save was requested. Called from VM loop.""" + path = _portal_checkpoint[0] + if path is None: return + _portal_checkpoint[0] = None + cont = FullCont(frames, stack, ip, instrs, env, vm_id) + portal_save(env, path, continuation=cont) + + # Auto-compile flag _auto_compile = [False] @@ -2658,6 +2962,16 @@ def make_global_env(): d(S('disassemble'),lambda a, _: (print(_disassemble(a[0])) or VOID)) d(S('save-compiled'), lambda a, _: save_compiled(_str_val(a[0]), a[1]) or VOID) d(S('load-compiled'), lambda a, e: load_compiled(_str_val(a[0]), e)) + d(S('portal-save'), lambda a, _: portal_save(g, _str_val(a[0])) or VOID) + d(S('portal-resume'), lambda a, _: _portal_resume_builtin(_str_val(a[0]), g)) + d(S('portal-checkpoint!'), lambda a, _: _portal_checkpoint.__setitem__(0, _str_val(a[0])) or VOID) + + def _portal_resume_builtin(path, env): + """Resume from portal file, merging into current env.""" + _, cont = portal_resume(path, env) + if cont is not None: + return _cont_resume(_ContInvoked(cont, VOID)) + return VOID def _auto_compile_fn(a, _): if not a: return _auto_compile[0] _auto_compile[0] = _truthy(a[0]); return VOID @@ -2898,6 +3212,25 @@ Features: R7RS core, bytecode compiler, full continuations, macros, print(f'compiled {len(compiled)} procedures to {out}') return + # --portal-resume: resume from a .portal file + if '--portal-resume' in args: + args = [a for a in args if a != '--portal-resume'] + if not args: + print('usage: uncommonlisp --portal-resume state.portal', file=sys.stderr); sys.exit(1) + path = args[0] + env, cont = portal_resume(path, g) + if cont is not None: + print(f'resuming from {path}...', file=sys.stderr) + try: + result = _cont_resume(_ContInvoked(cont, VOID)) + if result is not VOID: print(show(result)) + except LispErr as e: + print(f'error: {e}', file=sys.stderr); sys.exit(1) + else: + print(f'loaded state from {path} (no continuation to resume)', file=sys.stderr) + repl(env) + return + # -e 'expr' mode if args and args[0] == '-e': if len(args) < 2: