pi_star: code-py-ast@v1 graduates from stub — first non-text canonicalizer

Implements the cross-modality discipline that ticket #000015 spelled
out. Until now only text + claim_lattice + memory carriers existed
in code; the multimodal story was theoretical.

code-py-ast@v1 algorithm:

1. Parse UTF-8 bytes via stdlib ast.parse.
2. Walk the AST; emit deterministic S-expression
   "(NodeType field1=val1 ...)" with sorted fields, lists as
   "[elem0 elem1 ...]", primitives via repr().
3. Source positions (lineno/col_offset) skipped naturally — not in
   ast._fields.

Equivalence classes preserved (verified by tests):

- Whitespace, indentation, blank lines
- Comments
- String quote style ('x' vs "x")
- Operator spacing (1+2 vs 1 + 2)
- Trailing semicolons (x=1;y=2 vs x=1\ny=2)

Equivalence classes kept distinct:

- Identifier names (def foo vs def bar)
- Operator types (Add vs Sub)
- Argument order in calls (f(x,y) vs f(y,x))

Projective, not invertible — canonical bytes are S-expression text,
NOT valid Python. Re-canonicalizing the canonical output is
undefined; idempotency tests run on the original raw input only.

Surface:

- arborist/pi_star/code.py — full implementation, replaces stub
- bench/batteries/base.py — PHASE_1_CARRIERS adds "code"
- bench/batteries/b_5s.py — runner accepts both pi_star_ref
  (cross-modality canonical) and pi_star (Phase 1a legacy) keys;
  backward-compat shim
- bench/fixtures/5s/syntax-code-v1.jsonl — 10 Python parse-pass
  fixtures
- bench/fixtures/5s/semantics-code-v1.jsonl — 12 equivalence-class
  fixtures (whitespace/comments/quote-style/operator collapse;
  identifier/operator/argument-order remain distinct)
- Makefile: bench-5s-code target
- tests/test_pi_star.py: stubs parametrize drops code-py-ast
  (graduated); 11 new tests for code-py-ast@v1 covering equivalence
  classes, distinguishing classes, syntax-error rejection, non-bytes
  rejection, determinism, empty-source handling, equivalence_class_id
  format

Three remaining stubs (logic-kernel, time-series-quantized,
tabular-pinned) keep their NotImplementedError contract.

Full suite: 1221 passed, 36 skipped. Cross-modality discipline now
has actual code-level proof, not just schema metadata.
This commit is contained in:
russell@unturf.com 2026-05-08 08:50:07 -04:00
parent 8c618bbbc8
commit d6f3834ab7
No known key found for this signature in database
7 changed files with 231 additions and 14 deletions

View file

@ -127,7 +127,8 @@ def test_claim_lattice_empty_input_returns_empty_array():
@pytest.mark.parametrize(
"key",
[
"code-py-ast@v1",
# code-py-ast@v1 graduated from stub to real implementation —
# see test_code_py_ast_* below.
"logic-kernel@v1",
"time-series-quantized@v1",
"tabular-pinned@v1",
@ -139,6 +140,100 @@ def test_stubs_raise_not_implemented(key):
pi_star.canonicalize(b"anything")
# --- code-py-ast@v1 (graduated from stub) ----------------------------
def test_code_py_ast_canonicalizes_simple_assign():
pi_star = get("code-py-ast@v1")
out = pi_star.canonicalize(b"x = 1")
assert isinstance(out, bytes)
assert b"Assign" in out
assert b"Constant" in out
def test_code_py_ast_collapses_whitespace_and_comments():
pi_star = get("code-py-ast@v1")
a = pi_star.canonicalize(b"def foo(): pass")
b = pi_star.canonicalize(b"def foo():\n pass # comment")
assert a == b
def test_code_py_ast_collapses_quote_style():
pi_star = get("code-py-ast@v1")
a = pi_star.canonicalize(b"s = 'hello'")
b = pi_star.canonicalize(b's = "hello"')
assert a == b
def test_code_py_ast_collapses_operator_spacing():
pi_star = get("code-py-ast@v1")
a = pi_star.canonicalize(b"y = 1+2")
b = pi_star.canonicalize(b"y = 1 + 2")
assert a == b
def test_code_py_ast_distinguishes_identifiers():
pi_star = get("code-py-ast@v1")
a = pi_star.canonicalize(b"def foo(): pass")
b = pi_star.canonicalize(b"def bar(): pass")
assert a != b
def test_code_py_ast_distinguishes_operators():
pi_star = get("code-py-ast@v1")
a = pi_star.canonicalize(b"y = a + b")
b = pi_star.canonicalize(b"y = a - b")
assert a != b
def test_code_py_ast_distinguishes_argument_order():
pi_star = get("code-py-ast@v1")
a = pi_star.canonicalize(b"f(x, y)")
b = pi_star.canonicalize(b"f(y, x)")
assert a != b
def test_code_py_ast_rejects_non_bytes():
from arborist.pi_star.protocol import PiStarError
pi_star = get("code-py-ast@v1")
with pytest.raises(PiStarError):
pi_star.canonicalize("not bytes") # type: ignore[arg-type]
def test_code_py_ast_rejects_invalid_python():
from arborist.pi_star.protocol import PiStarError
pi_star = get("code-py-ast@v1")
with pytest.raises(PiStarError):
pi_star.canonicalize(b"def foo( malformed")
def test_code_py_ast_deterministic_across_repeated_calls():
pi_star = get("code-py-ast@v1")
src = b"class Foo:\n def bar(self, x):\n return x + 1\n"
a = pi_star.canonicalize(src)
b = pi_star.canonicalize(src)
c = pi_star.canonicalize(src)
assert a == b == c
def test_code_py_ast_handles_empty_source():
pi_star = get("code-py-ast@v1")
out = pi_star.canonicalize(b"")
# Empty source parses to ast.Module(body=[], type_ignores=[]).
assert b"Module" in out
assert b"body=[]" in out
def test_code_py_ast_equivalence_class_id_format():
pi_star = get("code-py-ast@v1")
eid = equivalence_class_id(pi_star, b"x = 1")
assert len(eid) == 64
# Same source twice → same id.
assert eid == equivalence_class_id(pi_star, b"x = 1")
# --- composition --------------------------------------------------