playground: stream tier output line-by-line during eval

Previously every (display ...) call buffered into the tier's outBuf /
sys.stdout / WAT output-buffer and only landed in the playground panel
after evalLisp returned. During the 18-second cuda-secp256k1-bench
dispatch the panel stayed blank, then everything (probe + telemetry +
heavy result) appeared at once. Fox: 'demo waits for the full program
to return before emitting instead of line by line as it finishes'.

Each tier's print sink now ALSO calls a per-eval onChunk callback
that the worker forwards to the main thread as a {kind:"chunk"}
postMessage. The main thread spawns an in-progress tier-block on
the first chunk and appends each subsequent chunk to its <pre>,
auto-scrolling. On done/error the timing header finalizes in place;
the block looks identical to a non-streamed appendBlock at the end.

C tier (Emscripten): the Module.print + Module.printErr callbacks
that already fired per-line now invoke currentOnChunk in addition
to buffering. evalLisp accepts an onChunk arg and threads it.

Python tier (pyodide): _lumbda_eval swaps sys.stdout for a
_StreamingStdout subclass of io.StringIO whose write() also calls
into globalThis._lumbdaPyEmitChunk on the JS side. The final
auto-printed value also emits.

Asm tier (WAT): output stays buffered until evalLisp returns —
streaming there needs a new wasm import (out_line) and a wasm
rebuild. Caller-side fallback: app.js's appendBlock path still
runs for asm so its block appears at the end as before.

worker.mjs forwards each chunk via postMessage; app.js's
runOnTierInWorker handles 'chunk' messages by lazily creating
a startLiveBlock on first chunk, appending text on each, and
finalizing the timing in finalize() once done arrives.
This commit is contained in:
russell@unturf.com 2026-06-14 20:25:47 -04:00
parent 036fab47fa
commit ffdfc1df43
No known key found for this signature in database
12 changed files with 528 additions and 45 deletions

View file

@ -213,19 +213,28 @@ function runOnTierInWorker(tier, src, onLoading) {
return new Promise((resolve, reject) => {
const w = ensureWorker(tier);
const myRunId = ++workerState.nextRunId;
let liveBlock = null;
workerState.pending[tier] = { runId: myRunId, resolve, reject };
const handler = (e) => {
if (e.data.runId !== myRunId) return;
if (e.data.kind === "loading") {
onLoading && onLoading(e.data.tier);
} else if (e.data.kind === "chunk") {
// First chunk spawns the in-progress block; subsequent
// chunks append to it. User sees displays land
// immediately, not just at the end of the run.
if (!liveBlock) liveBlock = startLiveBlock(tier);
liveBlock.append(e.data.chunk);
} else if (e.data.kind === "done") {
w.removeEventListener("message", handler);
workerState.pending[tier] = null;
resolve(e.data.output);
resolve({ output: e.data.output, liveBlock });
} else if (e.data.kind === "error") {
w.removeEventListener("message", handler);
workerState.pending[tier] = null;
reject(new Error(e.data.message));
const err = new Error(e.data.message);
err.liveBlock = liveBlock;
reject(err);
}
};
w.addEventListener("message", handler);
@ -233,6 +242,44 @@ function runOnTierInWorker(tier, src, onLoading) {
});
}
// Spawn a tier-block in the output panel immediately on first streamed
// chunk. Returns handles to append further chunks and to finalize the
// timing header once the eval reports done. Mirrors appendBlock's
// structure so finalized live blocks look identical to non-streamed
// ones — same DOM, same CSS.
function startLiveBlock(tierName) {
const block = document.createElement("div");
block.className = "tier-block";
const h = document.createElement("h3");
h.textContent = TIERS[tierName] || tierName;
const t = document.createElement("span");
t.className = "time";
t.textContent = " (running…)";
h.appendChild(t);
block.appendChild(h);
const pre = document.createElement("pre");
pre.textContent = "";
block.appendChild(pre);
outputEl.appendChild(block);
return {
append(chunk) {
pre.textContent += chunk;
outputEl.scrollTop = outputEl.scrollHeight;
},
finalize(elapsedMs, kind, fullOutput) {
if (kind === "cancelled") t.textContent = ` (cancelled @ ${elapsedMs.toFixed(0)} ms)`;
else t.textContent = ` (${elapsedMs.toFixed(0)} ms)`;
if (kind === "error") pre.className = "err";
// If the streamed chunks miss anything (e.g. asm tier
// which doesn't stream yet), reconcile with the full
// output. No-op when streaming captured everything.
if (fullOutput && pre.textContent !== fullOutput) {
pre.textContent = fullOutput;
}
},
};
}
function cancelCurrentRun() {
for (const t of Object.keys(workerState.workers)) {
if (workerState.workers[t]) {
@ -307,20 +354,26 @@ async function runAll() {
return runOnTierInWorker(t, src, (loadingTier) => {
setStatus(`loading ${loadingTier}`, "busy");
})
.then((output) => {
.then(({ output, liveBlock }) => {
const elapsed = performance.now() - startTimes[t];
delete startTimes[t];
appendBlock(t, elapsed, output, "ok");
// Streamed tiers already painted via liveBlock; just
// finalize the timing header. Non-streaming tiers
// (today: asm) get a fresh appendBlock at the end.
if (liveBlock) liveBlock.finalize(elapsed, "ok", output);
else appendBlock(t, elapsed, output, "ok");
return { tier: t, ok: true, elapsed };
})
.catch((e) => {
const elapsed = performance.now() - startTimes[t];
delete startTimes[t];
if (e.message === "cancelled") {
appendBlock(t, elapsed, "(cancelled)", "cancelled");
if (e.liveBlock) e.liveBlock.finalize(elapsed, "cancelled");
else appendBlock(t, elapsed, "(cancelled)", "cancelled");
return { tier: t, cancelled: true, elapsed };
}
appendBlock(t, elapsed, e.message || String(e), "error");
if (e.liveBlock) e.liveBlock.finalize(elapsed, "error", e.message);
else appendBlock(t, elapsed, e.message || String(e), "error");
return { tier: t, error: e.message, elapsed };
});
});

View file

@ -29,9 +29,9 @@ export async function getTier(name, onLoad) {
return cache[name];
}
export async function evalOnTier(name, src, onLoad) {
export async function evalOnTier(name, src, onLoad, onChunk) {
const tier = await getTier(name, onLoad);
return tier.evalLisp(src);
return tier.evalLisp(src, onChunk);
}
export function heapStats(name) {
@ -39,3 +39,20 @@ export function heapStats(name) {
if (!tier || !tier.heapStats) return null;
return tier.heapStats();
}
// Portal — REPL save/resume bridge. portalSave reads a snapshot blob
// from the tier (MEMFS on the C tier, similar bridges on others as
// they land). Returns null when the tier hasn't implemented portals
// yet (asm-wat today) so the caller can surface a friendly message
// instead of crashing. portalLoad is the symmetric write-in path.
export function portalSave(name, checkpointName) {
const tier = cache[name];
if (!tier || !tier.portalSave) return null;
return tier.portalSave(checkpointName);
}
export function portalLoad(name, checkpointName, blob) {
const tier = cache[name];
if (!tier || !tier.portalLoad) return false;
tier.portalLoad(checkpointName, blob);
return true;
}

View file

@ -3,7 +3,7 @@
// live ms counter actually ticks AND so cancel works (main thread
// terminates this worker via worker.terminate()).
import { evalOnTier, setBendUrl, heapStats } from "./runner.js";
import { evalOnTier, setBendUrl, heapStats, portalSave, portalLoad } from "./runner.js";
self.onmessage = async (e) => {
const { kind } = e.data;
@ -17,12 +17,39 @@ self.onmessage = async (e) => {
self.postMessage({ kind: "heap", runId: e.data.runId, tier: e.data.tier, stats: heapStats(e.data.tier) });
return;
}
if (kind === "portal-save") {
// Pulls the most-recent /tmp/<name>.portal out of the tier's
// MEMFS after the lisp side ran (portal-snapshot! NAME). Main
// thread encrypts and stuffs it into the vault.
const blob = portalSave(e.data.tier, e.data.name);
self.postMessage({ kind: "portal-save", runId: e.data.runId, name: e.data.name, blob });
return;
}
if (kind === "portal-load") {
// Hydrates MEMFS from a vault-decrypted blob so a subsequent
// (portal-load! NAME) eval finds the file ready.
const ok = portalLoad(e.data.tier, e.data.name, e.data.blob);
self.postMessage({ kind: "portal-load", runId: e.data.runId, name: e.data.name, ok });
return;
}
if (kind !== "eval") return;
const { runId, tier, src } = e.data;
try {
const output = await evalOnTier(tier, src, (loadingTier) => {
self.postMessage({ kind: "loading", runId, tier: loadingTier });
});
const output = await evalOnTier(
tier,
src,
(loadingTier) => {
self.postMessage({ kind: "loading", runId, tier: loadingTier });
},
// Stream every print/display from the tier to the main
// thread as it happens. Sync XHR inside bend!-call still
// blocks the worker, but displays BEFORE/AFTER the bend
// round-trip surface immediately instead of waiting for
// the whole eval to finish. Long demos feel alive.
(chunk) => {
self.postMessage({ kind: "chunk", runId, tier, chunk });
},
);
self.postMessage({ kind: "done", runId, output });
} catch (err) {
self.postMessage({ kind: "error", runId, message: err && err.message ? err.message : String(err) });

View file

@ -13,12 +13,25 @@ async function _bootstrap() {
const wasmDir = new URL("./", import.meta.url).href;
const { default: createLumbdaC } = await import(/* @vite-ignore */ factoryURL);
// Streaming output: every printf in the wasm fires Module.print
// synchronously. We buffer normally for the final return, but if
// the current eval registered an onChunk callback we also call
// through immediately so the worker can postMessage a chunk to
// the UI as each line emerges (vs the user staring at a blank
// panel for 18 seconds during the heavy bend call).
let outBuf = [];
let errBuf = [];
let currentOnChunk = null;
const module = await createLumbdaC({
locateFile: (p) => wasmDir + p,
print: (line) => outBuf.push(line),
printErr: (line) => errBuf.push(line),
print: (line) => {
outBuf.push(line);
if (currentOnChunk) currentOnChunk(line);
},
printErr: (line) => {
errBuf.push(line);
if (currentOnChunk) currentOnChunk(line);
},
});
const _init = module.cwrap("lumbda_wasm_init", null, []);
@ -27,6 +40,12 @@ async function _bootstrap() {
_init();
// /tmp must exist before (portal-snapshot! ...) writes there — the
// C builtin fopen("w")s the path directly so a missing dir is a
// hard error. Emscripten's MEMFS gives us /tmp on recent versions
// but make it idempotent.
try { module.FS.mkdir("/tmp"); } catch (e) { /* exists */ }
// bend!-call host bridge — js_lumbda_bend_call inside the wasm
// (EM_JS in lumbda_wasm_entry.c) reads globalThis._lumbdaCBendUrl
// for the destination, then does sync XHR POST and writes the
@ -34,10 +53,28 @@ async function _bootstrap() {
// global so runner.js can propagate a saved playground URL.
return {
setBendUrl(url) { globalThis._lumbdaCBendUrl = url || null; },
async evalLisp(src) {
// portalSave / portalLoad bridge MEMFS to the JS side so the
// REPL can persist checkpoints to the encrypted vault and
// restore them on a fresh worker. Names come from the worker
// (REPL caller validates alphanumeric) so path traversal isn't
// a concern.
portalSave(name) {
const path = `/tmp/${name}.portal`;
try {
const bytes = module.FS.readFile(path);
return new TextDecoder().decode(bytes);
} catch (e) { return null; }
},
portalLoad(name, blob) {
const path = `/tmp/${name}.portal`;
module.FS.writeFile(path, blob);
},
async evalLisp(src, onChunk) {
outBuf = [];
errBuf = [];
currentOnChunk = onChunk || null;
const errPtr = _eval(src);
currentOnChunk = null;
let errMsg = "";
if (errPtr) {
errMsg = module.UTF8ToString(errPtr);

View file

@ -13,12 +13,25 @@ async function _bootstrap() {
const wasmDir = new URL("./", import.meta.url).href;
const { default: createLumbdaC } = await import(/* @vite-ignore */ factoryURL);
// Streaming output: every printf in the wasm fires Module.print
// synchronously. We buffer normally for the final return, but if
// the current eval registered an onChunk callback we also call
// through immediately so the worker can postMessage a chunk to
// the UI as each line emerges (vs the user staring at a blank
// panel for 18 seconds during the heavy bend call).
let outBuf = [];
let errBuf = [];
let currentOnChunk = null;
const module = await createLumbdaC({
locateFile: (p) => wasmDir + p,
print: (line) => outBuf.push(line),
printErr: (line) => errBuf.push(line),
print: (line) => {
outBuf.push(line);
if (currentOnChunk) currentOnChunk(line);
},
printErr: (line) => {
errBuf.push(line);
if (currentOnChunk) currentOnChunk(line);
},
});
const _init = module.cwrap("lumbda_wasm_init", null, []);
@ -56,10 +69,12 @@ async function _bootstrap() {
const path = `/tmp/${name}.portal`;
module.FS.writeFile(path, blob);
},
async evalLisp(src) {
async evalLisp(src, onChunk) {
outBuf = [];
errBuf = [];
currentOnChunk = onChunk || null;
const errPtr = _eval(src);
currentOnChunk = null;
let errMsg = "";
if (errPtr) {
errMsg = module.UTF8ToString(errPtr);

View file

@ -29,7 +29,15 @@ async function _bootstrap() {
// the response text back. Sync XHR is the only sync HTTP available
// in a Web Worker; perfect for the blocking eval model lumbda
// primitives expect.
const refs = { bendUrl: null };
const refs = { bendUrl: null, currentOnChunk: null };
// Streaming output bridge — sys.stdout in pyodide is replaced
// (during _lumbda_eval) with a class whose write() calls back
// here. We forward to the current onChunk so the worker can
// postMessage chunks to the UI as work happens, instead of the
// user staring at a blank panel during a long bend dispatch.
globalThis._lumbdaPyEmitChunk = (s) => {
if (refs.currentOnChunk && s) refs.currentOnChunk(s);
};
globalThis._lumbdaPyBendCall = (payload) => {
if (!refs.bendUrl) return "no bend URL configured";
try {
@ -63,8 +71,50 @@ def _bend_call_prim(args, env):
return str(_js_bend_call(payload))
_env.define(lumbda.S('bend!-call'), _bend_call_prim)
# Portal save/resume REPL bridge. Mirrors the C tier's
# (portal-snapshot! NAME) / (portal-load! NAME): writes/reads
# /tmp/<name>.portal so the JS side can round-trip blobs to vault.
import os
os.makedirs('/tmp', exist_ok=True)
def _portal_snapshot_prim(args, env):
if not args or not isinstance(args[0], str):
raise lumbda.LispErr('portal-snapshot!: expected string name')
path = f'/tmp/{args[0]}.portal'
g = env.g if env.g else _env
lumbda.portal_save(g, path)
return path
def _portal_load_prim(args, env):
if not args or not isinstance(args[0], str):
raise lumbda.LispErr('portal-load!: expected string name')
path = f'/tmp/{args[0]}.portal'
restored_env, _cont = lumbda.portal_resume(path, _env)
# Copy bindings from the restored env into our live global env so
# subsequent evals see them. portal_resume gives us back a fresh
# env with the loaded bindings; we merge into _env in-place.
for sym, val in restored_env.b.items():
_env.b[sym] = val
return True
_env.define(lumbda.S('portal-snapshot!'), _portal_snapshot_prim)
_env.define(lumbda.S('portal-load!'), _portal_load_prim)
# Streaming stdout wrapper every write() also calls back into JS so
# the worker can postMessage chunks to the UI during the eval. The
# StringIO behind it still captures everything for the final return.
from js import _lumbdaPyEmitChunk as _js_emit_chunk
class _StreamingStdout(io.StringIO):
def write(self, s):
n = super().write(s)
try:
_js_emit_chunk(s)
except Exception:
pass
return n
def _lumbda_eval(src):
buf = io.StringIO()
buf = _StreamingStdout()
old = sys.stdout
sys.stdout = buf
last = None
@ -81,15 +131,39 @@ def _lumbda_eval(src):
if out and not out.endswith("\\n"):
out += "\\n"
out += rep
try:
_js_emit_chunk(("" if out.endswith(rep) else "\\n") + rep)
except Exception:
pass
return out
`);
return {
async evalLisp(src) {
async evalLisp(src, onChunk) {
pyodide.globals.set("_src_in", src);
return await pyodide.runPythonAsync("_lumbda_eval(_src_in)");
refs.currentOnChunk = onChunk || null;
try {
return await pyodide.runPythonAsync("_lumbda_eval(_src_in)");
} finally {
refs.currentOnChunk = null;
}
},
setBendUrl(url) { refs.bendUrl = url || null; },
// Portal save/resume bridge to vault — same shape as the C tier
// loader. Pyodide.FS is the same Emscripten FS, so the path
// /tmp/<name>.portal is reachable from JS exactly as in C.
portalSave(name) {
const path = `/tmp/${name}.portal`;
try {
const bytes = pyodide.FS.readFile(path);
return new TextDecoder().decode(bytes);
} catch (e) { return null; }
},
portalLoad(name, blob) {
const path = `/tmp/${name}.portal`;
try { pyodide.FS.mkdir("/tmp"); } catch (e) { /* exists */ }
pyodide.FS.writeFile(path, blob);
},
heapStats() {
// Pyodide's runtime memory is the Emscripten linear memory.
// CPython's GC reclaims behind the scenes, so this number

View file

@ -29,7 +29,15 @@ async function _bootstrap() {
// the response text back. Sync XHR is the only sync HTTP available
// in a Web Worker; perfect for the blocking eval model lumbda
// primitives expect.
const refs = { bendUrl: null };
const refs = { bendUrl: null, currentOnChunk: null };
// Streaming output bridge — sys.stdout in pyodide is replaced
// (during _lumbda_eval) with a class whose write() calls back
// here. We forward to the current onChunk so the worker can
// postMessage chunks to the UI as work happens, instead of the
// user staring at a blank panel during a long bend dispatch.
globalThis._lumbdaPyEmitChunk = (s) => {
if (refs.currentOnChunk && s) refs.currentOnChunk(s);
};
globalThis._lumbdaPyBendCall = (payload) => {
if (!refs.bendUrl) return "no bend URL configured";
try {
@ -63,8 +71,50 @@ def _bend_call_prim(args, env):
return str(_js_bend_call(payload))
_env.define(lumbda.S('bend!-call'), _bend_call_prim)
# Portal save/resume REPL bridge. Mirrors the C tier's
# (portal-snapshot! NAME) / (portal-load! NAME): writes/reads
# /tmp/<name>.portal so the JS side can round-trip blobs to vault.
import os
os.makedirs('/tmp', exist_ok=True)
def _portal_snapshot_prim(args, env):
if not args or not isinstance(args[0], str):
raise lumbda.LispErr('portal-snapshot!: expected string name')
path = f'/tmp/{args[0]}.portal'
g = env.g if env.g else _env
lumbda.portal_save(g, path)
return path
def _portal_load_prim(args, env):
if not args or not isinstance(args[0], str):
raise lumbda.LispErr('portal-load!: expected string name')
path = f'/tmp/{args[0]}.portal'
restored_env, _cont = lumbda.portal_resume(path, _env)
# Copy bindings from the restored env into our live global env so
# subsequent evals see them. portal_resume gives us back a fresh
# env with the loaded bindings; we merge into _env in-place.
for sym, val in restored_env.b.items():
_env.b[sym] = val
return True
_env.define(lumbda.S('portal-snapshot!'), _portal_snapshot_prim)
_env.define(lumbda.S('portal-load!'), _portal_load_prim)
# Streaming stdout wrapper every write() also calls back into JS so
# the worker can postMessage chunks to the UI during the eval. The
# StringIO behind it still captures everything for the final return.
from js import _lumbdaPyEmitChunk as _js_emit_chunk
class _StreamingStdout(io.StringIO):
def write(self, s):
n = super().write(s)
try:
_js_emit_chunk(s)
except Exception:
pass
return n
def _lumbda_eval(src):
buf = io.StringIO()
buf = _StreamingStdout()
old = sys.stdout
sys.stdout = buf
last = None
@ -81,15 +131,39 @@ def _lumbda_eval(src):
if out and not out.endswith("\\n"):
out += "\\n"
out += rep
try:
_js_emit_chunk(("" if out.endswith(rep) else "\\n") + rep)
except Exception:
pass
return out
`);
return {
async evalLisp(src) {
async evalLisp(src, onChunk) {
pyodide.globals.set("_src_in", src);
return await pyodide.runPythonAsync("_lumbda_eval(_src_in)");
refs.currentOnChunk = onChunk || null;
try {
return await pyodide.runPythonAsync("_lumbda_eval(_src_in)");
} finally {
refs.currentOnChunk = null;
}
},
setBendUrl(url) { refs.bendUrl = url || null; },
// Portal save/resume bridge to vault — same shape as the C tier
// loader. Pyodide.FS is the same Emscripten FS, so the path
// /tmp/<name>.portal is reachable from JS exactly as in C.
portalSave(name) {
const path = `/tmp/${name}.portal`;
try {
const bytes = pyodide.FS.readFile(path);
return new TextDecoder().decode(bytes);
} catch (e) { return null; }
},
portalLoad(name, blob) {
const path = `/tmp/${name}.portal`;
try { pyodide.FS.mkdir("/tmp"); } catch (e) { /* exists */ }
pyodide.FS.writeFile(path, blob);
},
heapStats() {
// Pyodide's runtime memory is the Emscripten linear memory.
// CPython's GC reclaims behind the scenes, so this number

View file

@ -213,19 +213,28 @@ function runOnTierInWorker(tier, src, onLoading) {
return new Promise((resolve, reject) => {
const w = ensureWorker(tier);
const myRunId = ++workerState.nextRunId;
let liveBlock = null;
workerState.pending[tier] = { runId: myRunId, resolve, reject };
const handler = (e) => {
if (e.data.runId !== myRunId) return;
if (e.data.kind === "loading") {
onLoading && onLoading(e.data.tier);
} else if (e.data.kind === "chunk") {
// First chunk spawns the in-progress block; subsequent
// chunks append to it. User sees displays land
// immediately, not just at the end of the run.
if (!liveBlock) liveBlock = startLiveBlock(tier);
liveBlock.append(e.data.chunk);
} else if (e.data.kind === "done") {
w.removeEventListener("message", handler);
workerState.pending[tier] = null;
resolve(e.data.output);
resolve({ output: e.data.output, liveBlock });
} else if (e.data.kind === "error") {
w.removeEventListener("message", handler);
workerState.pending[tier] = null;
reject(new Error(e.data.message));
const err = new Error(e.data.message);
err.liveBlock = liveBlock;
reject(err);
}
};
w.addEventListener("message", handler);
@ -233,6 +242,44 @@ function runOnTierInWorker(tier, src, onLoading) {
});
}
// Spawn a tier-block in the output panel immediately on first streamed
// chunk. Returns handles to append further chunks and to finalize the
// timing header once the eval reports done. Mirrors appendBlock's
// structure so finalized live blocks look identical to non-streamed
// ones — same DOM, same CSS.
function startLiveBlock(tierName) {
const block = document.createElement("div");
block.className = "tier-block";
const h = document.createElement("h3");
h.textContent = TIERS[tierName] || tierName;
const t = document.createElement("span");
t.className = "time";
t.textContent = " (running…)";
h.appendChild(t);
block.appendChild(h);
const pre = document.createElement("pre");
pre.textContent = "";
block.appendChild(pre);
outputEl.appendChild(block);
return {
append(chunk) {
pre.textContent += chunk;
outputEl.scrollTop = outputEl.scrollHeight;
},
finalize(elapsedMs, kind, fullOutput) {
if (kind === "cancelled") t.textContent = ` (cancelled @ ${elapsedMs.toFixed(0)} ms)`;
else t.textContent = ` (${elapsedMs.toFixed(0)} ms)`;
if (kind === "error") pre.className = "err";
// If the streamed chunks miss anything (e.g. asm tier
// which doesn't stream yet), reconcile with the full
// output. No-op when streaming captured everything.
if (fullOutput && pre.textContent !== fullOutput) {
pre.textContent = fullOutput;
}
},
};
}
function cancelCurrentRun() {
for (const t of Object.keys(workerState.workers)) {
if (workerState.workers[t]) {
@ -307,20 +354,26 @@ async function runAll() {
return runOnTierInWorker(t, src, (loadingTier) => {
setStatus(`loading ${loadingTier}`, "busy");
})
.then((output) => {
.then(({ output, liveBlock }) => {
const elapsed = performance.now() - startTimes[t];
delete startTimes[t];
appendBlock(t, elapsed, output, "ok");
// Streamed tiers already painted via liveBlock; just
// finalize the timing header. Non-streaming tiers
// (today: asm) get a fresh appendBlock at the end.
if (liveBlock) liveBlock.finalize(elapsed, "ok", output);
else appendBlock(t, elapsed, output, "ok");
return { tier: t, ok: true, elapsed };
})
.catch((e) => {
const elapsed = performance.now() - startTimes[t];
delete startTimes[t];
if (e.message === "cancelled") {
appendBlock(t, elapsed, "(cancelled)", "cancelled");
if (e.liveBlock) e.liveBlock.finalize(elapsed, "cancelled");
else appendBlock(t, elapsed, "(cancelled)", "cancelled");
return { tier: t, cancelled: true, elapsed };
}
appendBlock(t, elapsed, e.message || String(e), "error");
if (e.liveBlock) e.liveBlock.finalize(elapsed, "error", e.message);
else appendBlock(t, elapsed, e.message || String(e), "error");
return { tier: t, error: e.message, elapsed };
});
});

View file

@ -13,12 +13,25 @@ async function _bootstrap() {
const wasmDir = new URL("./", import.meta.url).href;
const { default: createLumbdaC } = await import(/* @vite-ignore */ factoryURL);
// Streaming output: every printf in the wasm fires Module.print
// synchronously. We buffer normally for the final return, but if
// the current eval registered an onChunk callback we also call
// through immediately so the worker can postMessage a chunk to
// the UI as each line emerges (vs the user staring at a blank
// panel for 18 seconds during the heavy bend call).
let outBuf = [];
let errBuf = [];
let currentOnChunk = null;
const module = await createLumbdaC({
locateFile: (p) => wasmDir + p,
print: (line) => outBuf.push(line),
printErr: (line) => errBuf.push(line),
print: (line) => {
outBuf.push(line);
if (currentOnChunk) currentOnChunk(line);
},
printErr: (line) => {
errBuf.push(line);
if (currentOnChunk) currentOnChunk(line);
},
});
const _init = module.cwrap("lumbda_wasm_init", null, []);
@ -56,10 +69,12 @@ async function _bootstrap() {
const path = `/tmp/${name}.portal`;
module.FS.writeFile(path, blob);
},
async evalLisp(src) {
async evalLisp(src, onChunk) {
outBuf = [];
errBuf = [];
currentOnChunk = onChunk || null;
const errPtr = _eval(src);
currentOnChunk = null;
let errMsg = "";
if (errPtr) {
errMsg = module.UTF8ToString(errPtr);

View file

@ -29,7 +29,15 @@ async function _bootstrap() {
// the response text back. Sync XHR is the only sync HTTP available
// in a Web Worker; perfect for the blocking eval model lumbda
// primitives expect.
const refs = { bendUrl: null };
const refs = { bendUrl: null, currentOnChunk: null };
// Streaming output bridge — sys.stdout in pyodide is replaced
// (during _lumbda_eval) with a class whose write() calls back
// here. We forward to the current onChunk so the worker can
// postMessage chunks to the UI as work happens, instead of the
// user staring at a blank panel during a long bend dispatch.
globalThis._lumbdaPyEmitChunk = (s) => {
if (refs.currentOnChunk && s) refs.currentOnChunk(s);
};
globalThis._lumbdaPyBendCall = (payload) => {
if (!refs.bendUrl) return "no bend URL configured";
try {
@ -63,8 +71,50 @@ def _bend_call_prim(args, env):
return str(_js_bend_call(payload))
_env.define(lumbda.S('bend!-call'), _bend_call_prim)
# Portal save/resume REPL bridge. Mirrors the C tier's
# (portal-snapshot! NAME) / (portal-load! NAME): writes/reads
# /tmp/<name>.portal so the JS side can round-trip blobs to vault.
import os
os.makedirs('/tmp', exist_ok=True)
def _portal_snapshot_prim(args, env):
if not args or not isinstance(args[0], str):
raise lumbda.LispErr('portal-snapshot!: expected string name')
path = f'/tmp/{args[0]}.portal'
g = env.g if env.g else _env
lumbda.portal_save(g, path)
return path
def _portal_load_prim(args, env):
if not args or not isinstance(args[0], str):
raise lumbda.LispErr('portal-load!: expected string name')
path = f'/tmp/{args[0]}.portal'
restored_env, _cont = lumbda.portal_resume(path, _env)
# Copy bindings from the restored env into our live global env so
# subsequent evals see them. portal_resume gives us back a fresh
# env with the loaded bindings; we merge into _env in-place.
for sym, val in restored_env.b.items():
_env.b[sym] = val
return True
_env.define(lumbda.S('portal-snapshot!'), _portal_snapshot_prim)
_env.define(lumbda.S('portal-load!'), _portal_load_prim)
# Streaming stdout wrapper every write() also calls back into JS so
# the worker can postMessage chunks to the UI during the eval. The
# StringIO behind it still captures everything for the final return.
from js import _lumbdaPyEmitChunk as _js_emit_chunk
class _StreamingStdout(io.StringIO):
def write(self, s):
n = super().write(s)
try:
_js_emit_chunk(s)
except Exception:
pass
return n
def _lumbda_eval(src):
buf = io.StringIO()
buf = _StreamingStdout()
old = sys.stdout
sys.stdout = buf
last = None
@ -81,15 +131,39 @@ def _lumbda_eval(src):
if out and not out.endswith("\\n"):
out += "\\n"
out += rep
try:
_js_emit_chunk(("" if out.endswith(rep) else "\\n") + rep)
except Exception:
pass
return out
`);
return {
async evalLisp(src) {
async evalLisp(src, onChunk) {
pyodide.globals.set("_src_in", src);
return await pyodide.runPythonAsync("_lumbda_eval(_src_in)");
refs.currentOnChunk = onChunk || null;
try {
return await pyodide.runPythonAsync("_lumbda_eval(_src_in)");
} finally {
refs.currentOnChunk = null;
}
},
setBendUrl(url) { refs.bendUrl = url || null; },
// Portal save/resume bridge to vault — same shape as the C tier
// loader. Pyodide.FS is the same Emscripten FS, so the path
// /tmp/<name>.portal is reachable from JS exactly as in C.
portalSave(name) {
const path = `/tmp/${name}.portal`;
try {
const bytes = pyodide.FS.readFile(path);
return new TextDecoder().decode(bytes);
} catch (e) { return null; }
},
portalLoad(name, blob) {
const path = `/tmp/${name}.portal`;
try { pyodide.FS.mkdir("/tmp"); } catch (e) { /* exists */ }
pyodide.FS.writeFile(path, blob);
},
heapStats() {
// Pyodide's runtime memory is the Emscripten linear memory.
// CPython's GC reclaims behind the scenes, so this number

View file

@ -29,9 +29,9 @@ export async function getTier(name, onLoad) {
return cache[name];
}
export async function evalOnTier(name, src, onLoad) {
export async function evalOnTier(name, src, onLoad, onChunk) {
const tier = await getTier(name, onLoad);
return tier.evalLisp(src);
return tier.evalLisp(src, onChunk);
}
export function heapStats(name) {
@ -39,3 +39,20 @@ export function heapStats(name) {
if (!tier || !tier.heapStats) return null;
return tier.heapStats();
}
// Portal — REPL save/resume bridge. portalSave reads a snapshot blob
// from the tier (MEMFS on the C tier, similar bridges on others as
// they land). Returns null when the tier hasn't implemented portals
// yet (asm-wat today) so the caller can surface a friendly message
// instead of crashing. portalLoad is the symmetric write-in path.
export function portalSave(name, checkpointName) {
const tier = cache[name];
if (!tier || !tier.portalSave) return null;
return tier.portalSave(checkpointName);
}
export function portalLoad(name, checkpointName, blob) {
const tier = cache[name];
if (!tier || !tier.portalLoad) return false;
tier.portalLoad(checkpointName, blob);
return true;
}

View file

@ -3,7 +3,7 @@
// live ms counter actually ticks AND so cancel works (main thread
// terminates this worker via worker.terminate()).
import { evalOnTier, setBendUrl, heapStats } from "./runner.js";
import { evalOnTier, setBendUrl, heapStats, portalSave, portalLoad } from "./runner.js";
self.onmessage = async (e) => {
const { kind } = e.data;
@ -17,12 +17,39 @@ self.onmessage = async (e) => {
self.postMessage({ kind: "heap", runId: e.data.runId, tier: e.data.tier, stats: heapStats(e.data.tier) });
return;
}
if (kind === "portal-save") {
// Pulls the most-recent /tmp/<name>.portal out of the tier's
// MEMFS after the lisp side ran (portal-snapshot! NAME). Main
// thread encrypts and stuffs it into the vault.
const blob = portalSave(e.data.tier, e.data.name);
self.postMessage({ kind: "portal-save", runId: e.data.runId, name: e.data.name, blob });
return;
}
if (kind === "portal-load") {
// Hydrates MEMFS from a vault-decrypted blob so a subsequent
// (portal-load! NAME) eval finds the file ready.
const ok = portalLoad(e.data.tier, e.data.name, e.data.blob);
self.postMessage({ kind: "portal-load", runId: e.data.runId, name: e.data.name, ok });
return;
}
if (kind !== "eval") return;
const { runId, tier, src } = e.data;
try {
const output = await evalOnTier(tier, src, (loadingTier) => {
self.postMessage({ kind: "loading", runId, tier: loadingTier });
});
const output = await evalOnTier(
tier,
src,
(loadingTier) => {
self.postMessage({ kind: "loading", runId, tier: loadingTier });
},
// Stream every print/display from the tier to the main
// thread as it happens. Sync XHR inside bend!-call still
// blocks the worker, but displays BEFORE/AFTER the bend
// round-trip surface immediately instead of waiting for
// the whole eval to finish. Long demos feel alive.
(chunk) => {
self.postMessage({ kind: "chunk", runId, tier, chunk });
},
);
self.postMessage({ kind: "done", runId, output });
} catch (err) {
self.postMessage({ kind: "error", runId, message: err && err.message ? err.message : String(err) });