pi_star: code-py-ast@v1 graduates from stub — first non-text canonicalizer
Implements the cross-modality discipline that ticket #000015 spelled out. Until now only text + claim_lattice + memory carriers existed in code; the multimodal story was theoretical. code-py-ast@v1 algorithm: 1. Parse UTF-8 bytes via stdlib ast.parse. 2. Walk the AST; emit deterministic S-expression "(NodeType field1=val1 ...)" with sorted fields, lists as "[elem0 elem1 ...]", primitives via repr(). 3. Source positions (lineno/col_offset) skipped naturally — not in ast._fields. Equivalence classes preserved (verified by tests): - Whitespace, indentation, blank lines - Comments - String quote style ('x' vs "x") - Operator spacing (1+2 vs 1 + 2) - Trailing semicolons (x=1;y=2 vs x=1\ny=2) Equivalence classes kept distinct: - Identifier names (def foo vs def bar) - Operator types (Add vs Sub) - Argument order in calls (f(x,y) vs f(y,x)) Projective, not invertible — canonical bytes are S-expression text, NOT valid Python. Re-canonicalizing the canonical output is undefined; idempotency tests run on the original raw input only. Surface: - arborist/pi_star/code.py — full implementation, replaces stub - bench/batteries/base.py — PHASE_1_CARRIERS adds "code" - bench/batteries/b_5s.py — runner accepts both pi_star_ref (cross-modality canonical) and pi_star (Phase 1a legacy) keys; backward-compat shim - bench/fixtures/5s/syntax-code-v1.jsonl — 10 Python parse-pass fixtures - bench/fixtures/5s/semantics-code-v1.jsonl — 12 equivalence-class fixtures (whitespace/comments/quote-style/operator collapse; identifier/operator/argument-order remain distinct) - Makefile: bench-5s-code target - tests/test_pi_star.py: stubs parametrize drops code-py-ast (graduated); 11 new tests for code-py-ast@v1 covering equivalence classes, distinguishing classes, syntax-error rejection, non-bytes rejection, determinism, empty-source handling, equivalence_class_id format Three remaining stubs (logic-kernel, time-series-quantized, tabular-pinned) keep their NotImplementedError contract. Full suite: 1221 passed, 36 skipped. Cross-modality discipline now has actual code-level proof, not just schema metadata.
This commit is contained in:
parent
8c618bbbc8
commit
d6f3834ab7
7 changed files with 231 additions and 14 deletions
|
|
@ -127,7 +127,8 @@ def test_claim_lattice_empty_input_returns_empty_array():
|
|||
@pytest.mark.parametrize(
|
||||
"key",
|
||||
[
|
||||
"code-py-ast@v1",
|
||||
# code-py-ast@v1 graduated from stub to real implementation —
|
||||
# see test_code_py_ast_* below.
|
||||
"logic-kernel@v1",
|
||||
"time-series-quantized@v1",
|
||||
"tabular-pinned@v1",
|
||||
|
|
@ -139,6 +140,100 @@ def test_stubs_raise_not_implemented(key):
|
|||
pi_star.canonicalize(b"anything")
|
||||
|
||||
|
||||
# --- code-py-ast@v1 (graduated from stub) ----------------------------
|
||||
|
||||
|
||||
def test_code_py_ast_canonicalizes_simple_assign():
|
||||
pi_star = get("code-py-ast@v1")
|
||||
out = pi_star.canonicalize(b"x = 1")
|
||||
assert isinstance(out, bytes)
|
||||
assert b"Assign" in out
|
||||
assert b"Constant" in out
|
||||
|
||||
|
||||
def test_code_py_ast_collapses_whitespace_and_comments():
|
||||
pi_star = get("code-py-ast@v1")
|
||||
a = pi_star.canonicalize(b"def foo(): pass")
|
||||
b = pi_star.canonicalize(b"def foo():\n pass # comment")
|
||||
assert a == b
|
||||
|
||||
|
||||
def test_code_py_ast_collapses_quote_style():
|
||||
pi_star = get("code-py-ast@v1")
|
||||
a = pi_star.canonicalize(b"s = 'hello'")
|
||||
b = pi_star.canonicalize(b's = "hello"')
|
||||
assert a == b
|
||||
|
||||
|
||||
def test_code_py_ast_collapses_operator_spacing():
|
||||
pi_star = get("code-py-ast@v1")
|
||||
a = pi_star.canonicalize(b"y = 1+2")
|
||||
b = pi_star.canonicalize(b"y = 1 + 2")
|
||||
assert a == b
|
||||
|
||||
|
||||
def test_code_py_ast_distinguishes_identifiers():
|
||||
pi_star = get("code-py-ast@v1")
|
||||
a = pi_star.canonicalize(b"def foo(): pass")
|
||||
b = pi_star.canonicalize(b"def bar(): pass")
|
||||
assert a != b
|
||||
|
||||
|
||||
def test_code_py_ast_distinguishes_operators():
|
||||
pi_star = get("code-py-ast@v1")
|
||||
a = pi_star.canonicalize(b"y = a + b")
|
||||
b = pi_star.canonicalize(b"y = a - b")
|
||||
assert a != b
|
||||
|
||||
|
||||
def test_code_py_ast_distinguishes_argument_order():
|
||||
pi_star = get("code-py-ast@v1")
|
||||
a = pi_star.canonicalize(b"f(x, y)")
|
||||
b = pi_star.canonicalize(b"f(y, x)")
|
||||
assert a != b
|
||||
|
||||
|
||||
def test_code_py_ast_rejects_non_bytes():
|
||||
from arborist.pi_star.protocol import PiStarError
|
||||
|
||||
pi_star = get("code-py-ast@v1")
|
||||
with pytest.raises(PiStarError):
|
||||
pi_star.canonicalize("not bytes") # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_code_py_ast_rejects_invalid_python():
|
||||
from arborist.pi_star.protocol import PiStarError
|
||||
|
||||
pi_star = get("code-py-ast@v1")
|
||||
with pytest.raises(PiStarError):
|
||||
pi_star.canonicalize(b"def foo( malformed")
|
||||
|
||||
|
||||
def test_code_py_ast_deterministic_across_repeated_calls():
|
||||
pi_star = get("code-py-ast@v1")
|
||||
src = b"class Foo:\n def bar(self, x):\n return x + 1\n"
|
||||
a = pi_star.canonicalize(src)
|
||||
b = pi_star.canonicalize(src)
|
||||
c = pi_star.canonicalize(src)
|
||||
assert a == b == c
|
||||
|
||||
|
||||
def test_code_py_ast_handles_empty_source():
|
||||
pi_star = get("code-py-ast@v1")
|
||||
out = pi_star.canonicalize(b"")
|
||||
# Empty source parses to ast.Module(body=[], type_ignores=[]).
|
||||
assert b"Module" in out
|
||||
assert b"body=[]" in out
|
||||
|
||||
|
||||
def test_code_py_ast_equivalence_class_id_format():
|
||||
pi_star = get("code-py-ast@v1")
|
||||
eid = equivalence_class_id(pi_star, b"x = 1")
|
||||
assert len(eid) == 64
|
||||
# Same source twice → same id.
|
||||
assert eid == equivalence_class_id(pi_star, b"x = 1")
|
||||
|
||||
|
||||
# --- composition --------------------------------------------------
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue