engineering/lumbda
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

wasm asm: stream output line-by-line via new emit_chunk env import

Browse source 
Brings the WAT tier to parity with c-emcc and pyodide for streaming
output during evalLisp. Previously the asm tier buffered everything
in the 0x10000 output region and the JS loader only read the bytes
AFTER lumbda_eval returned — fox saw the bend demo's pre-call
displays sit invisible for 18 s and then appear all at once.

WAT-side changes:

  - New env.emit_chunk(ptr, len) import. Host function forwards the
    slice to the current onChunk callback so the worker can postMessage
    a chunk to the playground panel as work happens.
  - New $flush_start global tracks the offset (relative to 0x10000)
    where the next emit_chunk slice begins. Reset to 0 at the top
    of lumbda_eval alongside $output_len so successive evals don't
    re-emit stale bytes.
  - $out_char now checks for newline (i32.const 10) after the store.
    A newline emits the slice [flush_start, output_len) and advances
    flush_start to the end. Every display call ends up flushing on
    its trailing newline; per-char displays without a newline get
    buffered until the next newline or the eval-end trailing flush.
  - $lumbda_eval ends with a trailing-flush guard so any non-newline-
    terminated content (e.g. print_value's final repr) reaches the
    stream instead of only landing through the final lumbda_output_*
    read.

JS loader:

  - importObj.env.emit_chunk decodes the slice from wasm memory and
    forwards to refs.currentOnChunk.
  - evalLisp(src, onChunk) parameter; sets/clears currentOnChunk
    around the lumbda_eval call. Same shape as c-emcc + pyodide.

Tests: every node test that instantiates the asm wasm directly now
declares a stub emit_chunk() {} alongside its bend_call stub —
unit, integration, functional-cross, parity-cross-tier. Node test
suite passes 23/23 unit; parity probe times out in its full sweep
under our 30s ceiling so it gets run separately.

Quick smoke: (display "line 1") (newline) (display "line 2") (newline)
(display "line 3") emits three chunks via onChunk — "line 1\n",
"line 2\n", "line 3" — and lumbda_output_* still has the full
"line 1\nline 2\nline 3" as before.
This commit is contained in:
russell@unturf.com 2026-06-14 20:35:13 -04:00
parent ffdfc1df43
commit d14c0eb5e4
No known key found for this signature in database
12 changed files with 133 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

17
wasm/asm/lumbda-asm.loader.js
View file

@ -13,7 +13,7 @@ async function _bootstrap() {
// Closure-captured holder so the imported function can see the
// instance's memory after instantiation completes.
const refs = { instance: null, bendUrl: null };
const refs = { instance: null, bendUrl: null, currentOnChunk: null };
const importObj = {
env: {
@ -45,6 +45,16 @@ async function _bootstrap() {
return err.length;
}
},
// emit_chunk(ptr, len) — WAT's out_char calls this on every
// newline (and lumbda_eval calls it once more at the end for
// trailing non-newline content) so the worker can postMessage
// a chunk to the playground panel as work happens, matching
// the C-tier Module.print + pyodide _StreamingStdout streams.
emit_chunk(ptr, len) {
if (!refs.currentOnChunk || len <= 0) return;
const mem = new Uint8Array(refs.instance.exports.memory.buffer, ptr, len);
refs.currentOnChunk(new TextDecoder().decode(mem));
},
},
};
@ -57,16 +67,19 @@ async function _bootstrap() {
const exp = instance.exports;
return {
async evalLisp(src) {
async evalLisp(src, onChunk) {
const srcBytes = enc.encode(src);
const srcPtr = exp.lumbda_source_ptr();
const mem = new Uint8Array(exp.memory.buffer);
mem.set(srcBytes, srcPtr);
refs.currentOnChunk = onChunk || null;
try {
exp.lumbda_eval(srcBytes.length);
} catch (e) {
refs.currentOnChunk = null;
return `error: ${e.message}`;
}
refs.currentOnChunk = null;
const outPtr = exp.lumbda_output_ptr();
const outLen = exp.lumbda_output_len();
return dec.decode(new Uint8Array(exp.memory.buffer, outPtr, outLen));

37
wasm/asm/lumbda.wat
View file

@ -46,11 +46,26 @@
(import "env" "bend_call"
(func $js_bend_call (param i32 i32 i32) (result i32)))
;; emit_chunk(ptr, len) — flush a slice of the output buffer to JS
;; for streaming. Called from out_char whenever a newline is emitted
;; (and from lumbda_eval at the very end for any trailing content
;; without a newline). Loader's host function forwards the bytes to
;; the current onChunk callback so the worker can postMessage the
;; chunk to the playground panel as work happens.
(import "env" "emit_chunk"
(func $js_emit_chunk (param i32 i32)))
;; ─── Memory & exports ──────────────────────────────────────────
(memory (export "memory") 32 4096) ;; 32 pages = 2 MB initial, grow to 256 MB
(global $heap_ptr (mut i32) (i32.const 0x30000))
(global $output_len (mut i32) (i32.const 0))
;; flush_start tracks the offset in the output buffer (relative to
;; 0x10000) where the next emit_chunk should begin. Updated each
;; time we flush; reset to 0 at the top of lumbda_eval alongside
;; output_len. Lets streaming carry just the most-recent line, not
;; the entire eval-so-far buffer.
(global $flush_start (mut i32) (i32.const 0))
(global $source_ptr (mut i32) (i32.const 0x20000))
(global $source_end (mut i32) (i32.const 0x20000))
(global $intern_list (mut i32) (i32.const 4)) ;; NIL initially
@ -1278,7 +1293,16 @@
(i32.store8
(i32.add (i32.const 0x10000) (global.get $output_len))
(local.get $c))
(global.set $output_len (i32.add (global.get $output_len) (i32.const 1))))
(global.set $output_len (i32.add (global.get $output_len) (i32.const 1)))
;; Newline flushes the slice [flush_start, output_len) to JS so
;; the playground panel can render line-by-line during eval
;; instead of waiting for the full evalLisp to return.
(if (i32.eq (local.get $c) (i32.const 10))
(then
(call $js_emit_chunk
(i32.add (i32.const 0x10000) (global.get $flush_start))
(i32.sub (global.get $output_len) (global.get $flush_start)))
(global.set $flush_start (global.get $output_len)))))
(func $out_str (param $ptr i32) (param $len i32)
(local $i i32)
@ -4201,6 +4225,7 @@
(local $val i32)
(call $lumbda_init)
(global.set $output_len (i32.const 0))
(global.set $flush_start (i32.const 0))
(global.set $source_ptr (i32.const 0x20000))
(global.set $source_end (i32.add (i32.const 0x20000) (local.get $src_len)))
(local.set $val (global.get $VOID))
@ -4228,6 +4253,16 @@
(i32.const 10))
(then (call $out_char (i32.const 10))))))
(call $print_value (local.get $val))))
;; Trailing-content flush — print_value's repr doesn't end with a
;; newline, so any bytes past $flush_start would otherwise miss
;; the streaming path and only reach the UI through the final
;; lumbda_output_ptr / lumbda_output_len read.
(if (i32.gt_u (global.get $output_len) (global.get $flush_start))
(then
(call $js_emit_chunk
(i32.add (i32.const 0x10000) (global.get $flush_start))
(i32.sub (global.get $output_len) (global.get $flush_start)))
(global.set $flush_start (global.get $output_len))))
;; Trigger GC when the heap exceeds 60% of available memory. This is
;; the only safe collection point — the eval call stack has unwound,
;; so the only roots are the globals the collector knows about.

17
wasm/dist-repl/asm/lumbda-asm.loader.js
View file

@ -13,7 +13,7 @@ async function _bootstrap() {
// Closure-captured holder so the imported function can see the
// instance's memory after instantiation completes.
const refs = { instance: null, bendUrl: null };
const refs = { instance: null, bendUrl: null, currentOnChunk: null };
const importObj = {
env: {
@ -45,6 +45,16 @@ async function _bootstrap() {
return err.length;
}
},
// emit_chunk(ptr, len) — WAT's out_char calls this on every
// newline (and lumbda_eval calls it once more at the end for
// trailing non-newline content) so the worker can postMessage
// a chunk to the playground panel as work happens, matching
// the C-tier Module.print + pyodide _StreamingStdout streams.
emit_chunk(ptr, len) {
if (!refs.currentOnChunk || len <= 0) return;
const mem = new Uint8Array(refs.instance.exports.memory.buffer, ptr, len);
refs.currentOnChunk(new TextDecoder().decode(mem));
},
},
};
@ -57,16 +67,19 @@ async function _bootstrap() {
const exp = instance.exports;
return {
async evalLisp(src) {
async evalLisp(src, onChunk) {
const srcBytes = enc.encode(src);
const srcPtr = exp.lumbda_source_ptr();
const mem = new Uint8Array(exp.memory.buffer);
mem.set(srcBytes, srcPtr);
refs.currentOnChunk = onChunk || null;
try {
exp.lumbda_eval(srcBytes.length);
} catch (e) {
refs.currentOnChunk = null;
return `error: ${e.message}`;
}
refs.currentOnChunk = null;
const outPtr = exp.lumbda_output_ptr();
const outLen = exp.lumbda_output_len();
return dec.decode(new Uint8Array(exp.memory.buffer, outPtr, outLen));

BIN
wasm/dist-repl/asm/lumbda-asm.wasm
View file

Binary file not shown.

2
wasm/tests/functional-cross.mjs
View file

@ -19,7 +19,7 @@ function countPassFail(out) {
async function runAsm() {
const wasmBytes = fs.readFileSync(path.join(dist, "asm", "lumbda-asm.wasm"));
const importObj = { env: { bend_call(_p, _l, _r) { return 0; } } };
const importObj = { env: { bend_call(_p, _l, _r) { return 0; }, emit_chunk() {} } };
const { instance } = await WebAssembly.instantiate(wasmBytes, importObj);
const exp = instance.exports;
exp.lumbda_init();

2
wasm/tests/integration.mjs
View file

@ -31,7 +31,7 @@ function nativePython(demoPath) {
async function runAsm(src) {
const wasmBytes = fs.readFileSync(path.join(dist, "asm", "lumbda-asm.wasm"));
const importObj = { env: { bend_call(_p, _l, _r) { return 0; } } };
const importObj = { env: { bend_call(_p, _l, _r) { return 0; }, emit_chunk() {} } };
const { instance } = await WebAssembly.instantiate(wasmBytes, importObj);
const exp = instance.exports;
exp.lumbda_init();

2
wasm/tests/parity-cross-tier.mjs
View file

@ -62,7 +62,7 @@ function nativePython(src) {
async function withAsmTier() {
const wasmBytes = fs.readFileSync(path.join(dist, "asm", "lumbda-asm.wasm"));
const importObj = { env: { bend_call() { return 0; } } };
const importObj = { env: { bend_call() { return 0; }, emit_chunk() {} } };
const { instance } = await WebAssembly.instantiate(wasmBytes, importObj);
const exp = instance.exports;
exp.lumbda_init();

35
wasm/tests/unit.mjs
View file

@ -23,7 +23,7 @@ function check(name, cond, detail) {
async function testAsm() {
console.log("── asm tier ──");
const wasmBytes = fs.readFileSync(path.join(dist, "asm", "lumbda-asm.wasm"));
const importObj = { env: { bend_call(_p, _l, _r) { return 0; } } };
const importObj = { env: { bend_call(_p, _l, _r) { return 0; }, emit_chunk() {} } };
const { instance } = await WebAssembly.instantiate(wasmBytes, importObj);
const exp = instance.exports;
exp.lumbda_init();
@ -90,6 +90,39 @@ async function testC() {
// Skip error-path test on C tier: undefined-symbol triggers a long
// setjmp/longjmp chain that the Emscripten runtime executes in finite
// time but our test runner times the whole suite — keep it lean.
// Portal save / resume — REPL backbone. portal-snapshot! writes JSON
// into MEMFS at /tmp/<name>.portal. We round-trip a binding through
// a fresh tier instance to prove the blob is self-contained.
try { m.FS.mkdir("/tmp"); } catch (e) { /* exists */ }
evalLisp("(define round-trip-val 12345)");
evalLisp('(portal-snapshot! "rt")');
const blob = new TextDecoder().decode(m.FS.readFile("/tmp/rt.portal"));
check("portal-snapshot! produces JSON blob",
blob.includes('"lumbda-portal-v1"') && blob.includes('round-trip-val'));
const m2 = await createLumbdaC({
locateFile: (p) => path.join(dist, "c", p),
print: (line) => out.push(line),
printErr: (line) => out.push("ERR: " + line),
});
m2.cwrap("lumbda_wasm_init", null, [])();
const _eval2 = m2.cwrap("lumbda_wasm_eval", "number", ["string"]);
const _free2 = m2.cwrap("lumbda_wasm_free_result", null, ["number"]);
function evalLisp2(src) {
out = [];
const r = _eval2(src);
let errMsg = "";
if (r) { errMsg = m2.UTF8ToString(r); _free2(r); }
return out.join("\n") + (errMsg ? "\n" + errMsg : "");
}
check("fresh tier has no round-trip-val",
evalLisp2("round-trip-val").includes("error"));
try { m2.FS.mkdir("/tmp"); } catch (e) { /* exists */ }
m2.FS.writeFile("/tmp/rt.portal", blob);
evalLisp2('(portal-load! "rt")');
check("portal-load! restores round-trip-val",
evalLisp2("round-trip-val").trim() === "12345");
}
(async () => {

17
www/playground/asm/lumbda-asm.loader.js
View file

@ -13,7 +13,7 @@ async function _bootstrap() {
// Closure-captured holder so the imported function can see the
// instance's memory after instantiation completes.
const refs = { instance: null, bendUrl: null };
const refs = { instance: null, bendUrl: null, currentOnChunk: null };
const importObj = {
env: {
@ -45,6 +45,16 @@ async function _bootstrap() {
return err.length;
}
},
// emit_chunk(ptr, len) — WAT's out_char calls this on every
// newline (and lumbda_eval calls it once more at the end for
// trailing non-newline content) so the worker can postMessage
// a chunk to the playground panel as work happens, matching
// the C-tier Module.print + pyodide _StreamingStdout streams.
emit_chunk(ptr, len) {
if (!refs.currentOnChunk || len <= 0) return;
const mem = new Uint8Array(refs.instance.exports.memory.buffer, ptr, len);
refs.currentOnChunk(new TextDecoder().decode(mem));
},
},
};
@ -57,16 +67,19 @@ async function _bootstrap() {
const exp = instance.exports;
return {
async evalLisp(src) {
async evalLisp(src, onChunk) {
const srcBytes = enc.encode(src);
const srcPtr = exp.lumbda_source_ptr();
const mem = new Uint8Array(exp.memory.buffer);
mem.set(srcBytes, srcPtr);
refs.currentOnChunk = onChunk || null;
try {
exp.lumbda_eval(srcBytes.length);
} catch (e) {
refs.currentOnChunk = null;
return `error: ${e.message}`;
}
refs.currentOnChunk = null;
const outPtr = exp.lumbda_output_ptr();
const outLen = exp.lumbda_output_len();
return dec.decode(new Uint8Array(exp.memory.buffer, outPtr, outLen));

BIN
www/playground/asm/lumbda-asm.wasm
View file

Binary file not shown.

17
www/repl/asm/lumbda-asm.loader.js
View file

@ -13,7 +13,7 @@ async function _bootstrap() {
// Closure-captured holder so the imported function can see the
// instance's memory after instantiation completes.
const refs = { instance: null, bendUrl: null };
const refs = { instance: null, bendUrl: null, currentOnChunk: null };
const importObj = {
env: {
@ -45,6 +45,16 @@ async function _bootstrap() {
return err.length;
}
},
// emit_chunk(ptr, len) — WAT's out_char calls this on every
// newline (and lumbda_eval calls it once more at the end for
// trailing non-newline content) so the worker can postMessage
// a chunk to the playground panel as work happens, matching
// the C-tier Module.print + pyodide _StreamingStdout streams.
emit_chunk(ptr, len) {
if (!refs.currentOnChunk || len <= 0) return;
const mem = new Uint8Array(refs.instance.exports.memory.buffer, ptr, len);
refs.currentOnChunk(new TextDecoder().decode(mem));
},
},
};
@ -57,16 +67,19 @@ async function _bootstrap() {
const exp = instance.exports;
return {
async evalLisp(src) {
async evalLisp(src, onChunk) {
const srcBytes = enc.encode(src);
const srcPtr = exp.lumbda_source_ptr();
const mem = new Uint8Array(exp.memory.buffer);
mem.set(srcBytes, srcPtr);
refs.currentOnChunk = onChunk || null;
try {
exp.lumbda_eval(srcBytes.length);
} catch (e) {
refs.currentOnChunk = null;
return `error: ${e.message}`;
}
refs.currentOnChunk = null;
const outPtr = exp.lumbda_output_ptr();
const outLen = exp.lumbda_output_len();
return dec.decode(new Uint8Array(exp.memory.buffer, outPtr, outLen));

BIN
www/repl/asm/lumbda-asm.wasm
View file

Binary file not shown.