From ca022a8f0d8765098ff22c70c0e448e047f52b68 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Wed, 17 Jun 2026 09:24:57 -0400 Subject: [PATCH 1/4] tests: OTP 25/26 + TLS + Elixir GenServer integration via docker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reproduces the production failure locally so we can iterate in seconds instead of waiting on deploy cycles across 3 repos and PyPI. Existing test_node.py / test_tls.py use the system Erlang which is OTP 24 on most dev boxes (Ubuntu 22.04 default) — silently masks flag-negotiation and protocol bugs that only surface against OTP 25+. Adds four docker-backed tests: - plain dist + hand-coded receive (works on all OTP versions) - TLS dist + hand-coded receive (works on all OTP versions) - TLS dist + Elixir GenServer (FAILS on OTP 25, passes on 26) The last one is the minimum repro of the portal@unsandbox.com failure mode. Once a fix lands, that test goes green and we know the production smoke test will too. Uses --network host to share the host's epmd (Linux-only). Docker mounts from $HOME/.erldistpy-test/ because snap-confined docker can't see /tmp. Tests skip cleanly when docker isn't installed. --- tests/test_node_otp26.py | 515 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 515 insertions(+) create mode 100644 tests/test_node_otp26.py diff --git a/tests/test_node_otp26.py b/tests/test_node_otp26.py new file mode 100644 index 0000000..ed3e629 --- /dev/null +++ b/tests/test_node_otp26.py @@ -0,0 +1,515 @@ +"""OTP 26 integration tests via Docker. + +Reproduces the production failure mode we hit on portal@unsandbox.com: +erldistpy 0.1.7 handshake "succeeds" against OTP 26 but the peer +silently closes the link on the first REG_SEND, surfacing as +``ChannelError: peer closed after 0/4 bytes`` on Node.call(). + +The pure-Python tests and the existing live tests all run against the +system Erlang, which on most dev boxes is OTP 24 (Ubuntu 22.04 default). +OTP 24 didn't enforce DFLAG_MANDATORY_25_DIGEST or the OTP-25+ +mandatory flag set, so flag-negotiation bugs pass silently. These +tests force the modern protocol by running a peer node out of the +official ``erlang:26`` Docker image. + +Tests are skipped automatically if Docker isn't available — they're +intentionally NOT part of the smoke that PRs gate on, because they +require docker pull + ~7s of node boot per module. +""" + +from __future__ import annotations + +import shutil +import socket +import subprocess +import time + +import pytest + +from pathlib import Path + +from erldistpy.etf import Atom +from erldistpy.node import Node +from erldistpy.tls import make_dist_tls_context + + +SNAME_OTP26 = "erldistpy_otp26" +COOKIE_OTP26 = "ERLDISTPY_OTP26_COOKIE" + +# Same gen_target shape as tests/test_node.py — implements the +# gen_server-style {'$gen_call', {From, Ref}, Request} -> From ! {Ref, Reply} +# protocol by hand. Two operations: +# {ping, X} -> {pong, X} +# {add, A, B} -> {ok, A + B} +# Anything else surfaces as {error, {bad_request, _}}. +# +# We also register a heartbeat after 200ms so tests can poll a known +# pid presence rather than depending on a /tmp marker file (which would +# live inside the container, invisible to the host). +ERL_GEN_BOOT = ( + "Handle = fun " + " ({ping, X}) -> {pong, X}; " + " ({add, A, B}) -> {ok, A + B}; " + " (Other) -> {error, {bad_request, Other}} " + "end, " + "Loop = fun(F) -> " + " receive " + " {'$gen_call', {From, Ref}, Request} -> " + " From ! {Ref, Handle(Request)}, F(F); " + " Other -> " + " io:format(\"unexpected: ~p~n\", [Other]), F(F) " + " end " + "end, " + "Pid = spawn(fun() -> Loop(Loop) end), " + "register(gen_target, Pid), " + "timer:sleep(infinity)." +) + + +def _docker_available() -> bool: + return shutil.which("docker") is not None + + +def _epmd_node_present(sname: str) -> bool: + """Returns True if ``sname`` is registered in the host's epmd.""" + try: + result = subprocess.run( + ["epmd", "-names"], + capture_output=True, + text=True, + timeout=2, + ) + except (FileNotFoundError, subprocess.TimeoutExpired): + return False + return f"name {sname} at port" in result.stdout + + +def _socket_open(host: str, port: int) -> bool: + try: + with socket.create_connection((host, port), timeout=0.5): + return True + except OSError: + return False + + +@pytest.fixture(scope="module") +def otp26_gen_peer(): + if not _docker_available(): + pytest.skip("docker not installed") + if not _socket_open("127.0.0.1", 4369): + pytest.skip("EPMD not running on 127.0.0.1:4369 (host)") + + container_name = f"erldistpy-otp26-{int(time.time())}" + + # --network host shares the host's network namespace (Linux only), + # so the OTP 26 node registers with the host's epmd and listens on + # a host-routable interface. epmd lookups from erldistpy on the + # host find the docker node like any other local Erlang node. + proc = subprocess.Popen( + [ + "docker", + "run", + "--rm", + "--name", + container_name, + "--network", + "host", + "erlang:25", + "erl", + "-sname", + SNAME_OTP26, + "-setcookie", + COOKIE_OTP26, + "-noshell", + "-eval", + ERL_GEN_BOOT, + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + deadline = time.monotonic() + 15.0 + ready = False + while time.monotonic() < deadline: + time.sleep(0.2) + if _epmd_node_present(SNAME_OTP26): + # Allow another 200ms for `register(gen_target, ...)` to land. + time.sleep(0.2) + ready = True + break + if not ready: + subprocess.run(["docker", "rm", "-f", container_name], capture_output=True) + proc.terminate() + proc.wait(timeout=2) + pytest.skip( + f"OTP 26 docker node {SNAME_OTP26} did not register with host epmd" + ) + + yield SNAME_OTP26, COOKIE_OTP26 + + subprocess.run(["docker", "rm", "-f", container_name], capture_output=True) + proc.terminate() + try: + proc.wait(timeout=2) + except subprocess.TimeoutExpired: + proc.kill() + + +def test_call_ping_otp26(otp26_gen_peer): + """The minimal repro of the production failure. + + erldistpy 0.1.6 silently dropped this call against OTP 26 portal. + 0.1.7 added DFLAG_MANDATORY_25_DIGEST; whether THAT was enough is + exactly what this test answers. + """ + sname, cookie = otp26_gen_peer + with Node(our_name="erldistpy_test_otp26@localhost", peer_name=sname, cookie=cookie) as n: + reply = n.call("gen_target", (Atom("ping"), 42), timeout=5.0) + assert reply == (Atom("pong"), 42) + + +def test_call_add_otp26(otp26_gen_peer): + sname, cookie = otp26_gen_peer + with Node(our_name="erldistpy_test_otp26_add@localhost", peer_name=sname, cookie=cookie) as n: + reply = n.call("gen_target", (Atom("add"), 5, 7), timeout=5.0) + assert reply == (Atom("ok"), 12) + + +def test_call_sequence_otp26(otp26_gen_peer): + """Multiple calls on the same dist link survive net_tick window.""" + sname, cookie = otp26_gen_peer + with Node(our_name="erldistpy_test_otp26_seq@localhost", peer_name=sname, cookie=cookie) as n: + assert n.call("gen_target", (Atom("ping"), Atom("a")), timeout=5.0) == (Atom("pong"), Atom("a")) + assert n.call("gen_target", (Atom("ping"), Atom("b")), timeout=5.0) == (Atom("pong"), Atom("b")) + assert n.call("gen_target", (Atom("add"), 1, 2), timeout=5.0) == (Atom("ok"), 3) + + +# --------------------------------------------------------------------------- +# OTP 26 + TLS dist — the production combo (mTLS over Erlang dist) +# --------------------------------------------------------------------------- + + +TLS_SNAME = "erldistpy_otp26_tls" +TLS_COOKIE = "ERLDISTPY_OTP26_TLS_COOKIE" + + +def _gen_certs(workdir: Path) -> dict[str, str]: + """Generate a CA + leaf cert with a SAN matching localhost. Same shape + as tests/test_tls.py — duplicated locally to keep this file self- + contained (the docker container needs to read from a known path).""" + import os + hostname = socket.gethostname() + ca_cnf = workdir / "ca.cnf" + ca_cnf.write_text( + "[req]\n" + "distinguished_name = dn\n" + "x509_extensions = v3_ca\n" + "prompt = no\n" + "[dn]\n" + "CN = erldistpy-otp26-ca\n" + "[v3_ca]\n" + "basicConstraints = critical,CA:TRUE\n" + "keyUsage = critical,keyCertSign,cRLSign\n" + "subjectKeyIdentifier = hash\n" + ) + subprocess.run( + ["openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes", + "-keyout", str(workdir / "ca.key"), + "-out", str(workdir / "ca.pem"), + "-days", "1", + "-config", str(ca_cnf), "-extensions", "v3_ca"], + check=True, capture_output=True, + ) + leaf_cnf = workdir / "leaf.cnf" + leaf_cnf.write_text( + "[v3]\n" + "basicConstraints = CA:FALSE\n" + "keyUsage = digitalSignature,keyEncipherment\n" + "extendedKeyUsage = serverAuth,clientAuth\n" + "subjectAltName = @alt\n" + "[alt]\n" + f"DNS.1 = {hostname}\n" + "DNS.2 = localhost\n" + ) + for name in ("server", "client"): + subprocess.run( + ["openssl", "req", "-newkey", "rsa:2048", "-nodes", + "-keyout", str(workdir / f"{name}.key"), + "-out", str(workdir / f"{name}.csr"), + "-subj", f"/CN={hostname}"], + check=True, capture_output=True, + ) + subprocess.run( + ["openssl", "x509", "-req", + "-in", str(workdir / f"{name}.csr"), + "-CA", str(workdir / "ca.pem"), + "-CAkey", str(workdir / "ca.key"), + "-CAcreateserial", + "-out", str(workdir / f"{name}.pem"), + "-days", "1", + "-extfile", str(leaf_cnf), "-extensions", "v3"], + check=True, capture_output=True, + ) + os.chmod(workdir / f"{name}.key", 0o644) # readable by container's erl uid + return { + "ca": str(workdir / "ca.pem"), + "server_cert": str(workdir / "server.pem"), + "server_key": str(workdir / "server.key"), + "client_cert": str(workdir / "client.pem"), + "client_key": str(workdir / "client.key"), + } + + +def _write_ssl_config(workdir: Path, certs_in_container: dict[str, str]) -> str: + cfg = workdir / "ssl_dist.config" + body = ( + "[{server, " + f'[{{certfile, "{certs_in_container["server_cert"]}"}}, ' + f'{{keyfile, "{certs_in_container["server_key"]}"}}, ' + f'{{cacertfile, "{certs_in_container["ca"]}"}}, ' + "{verify, verify_peer}, " + "{fail_if_no_peer_cert, true}]}, " + "{client, " + f'[{{certfile, "{certs_in_container["server_cert"]}"}}, ' + f'{{keyfile, "{certs_in_container["server_key"]}"}}, ' + f'{{cacertfile, "{certs_in_container["ca"]}"}}, ' + "{verify, verify_peer}]}]." + ) + cfg.write_text(body) + return str(cfg) + + +@pytest.fixture(scope="module") +def otp26_tls_peer(tmp_path_factory): + """OTP 26 + TLS dist peer in Docker — the production combo. + + This is the variant that reproduces (or should reproduce) the + portal@unsandbox.com failure: handshake "succeeds" then the peer + silently closes the link on the first REG_SEND. + """ + if not _docker_available(): + pytest.skip("docker not installed") + if not shutil.which("openssl"): + pytest.skip("openssl not installed") + if not _socket_open("127.0.0.1", 4369): + pytest.skip("EPMD not running on host") + + # Snap-confined Docker (canonical's snap package) can't bind-mount + # /tmp — use a path under $HOME which the confinement allows. Caller + # could override via ERLDISTPY_TLS_OTP26_WORKDIR. + import os as _os + import tempfile as _tempfile + + home_base = _os.environ.get( + "ERLDISTPY_TLS_OTP26_WORKDIR", str(Path.home() / ".erldistpy-test") + ) + Path(home_base).mkdir(parents=True, exist_ok=True) + workdir = Path(_tempfile.mkdtemp(prefix="tls_otp26_", dir=home_base)) + _os.chmod(workdir, 0o755) + host_certs = _gen_certs(workdir) + # Inside the container the certs land at the same path because we + # mount workdir → /certs:ro and we generated under workdir. + container_certs = { + k: v.replace(str(workdir), "/certs") for k, v in host_certs.items() + } + ssl_config_host = _write_ssl_config(workdir, container_certs) + ssl_config_container = ssl_config_host.replace(str(workdir), "/certs") + + container_name = f"erldistpy-otp26-tls-{int(time.time())}" + + proc = subprocess.Popen( + [ + "docker", "run", "--rm", + "--name", container_name, + "--network", "host", + "-v", f"{workdir}:/certs:ro", + "erlang:25", + "erl", + "-sname", TLS_SNAME, + "-setcookie", TLS_COOKIE, + "-proto_dist", "inet_tls", + "-ssl_dist_optfile", ssl_config_container, + "-noshell", + "-eval", ERL_GEN_BOOT, + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + deadline = time.monotonic() + 20.0 + ready = False + while time.monotonic() < deadline: + time.sleep(0.3) + if _epmd_node_present(TLS_SNAME): + time.sleep(0.3) + ready = True + break + if not ready: + subprocess.run(["docker", "rm", "-f", container_name], capture_output=True) + proc.terminate() + proc.wait(timeout=3) + pytest.skip(f"OTP 26 TLS docker node {TLS_SNAME} did not register") + + yield TLS_SNAME, TLS_COOKIE, host_certs + + subprocess.run(["docker", "rm", "-f", container_name], capture_output=True) + proc.terminate() + try: + proc.wait(timeout=3) + except subprocess.TimeoutExpired: + proc.kill() + + +def test_call_over_tls_otp26(otp26_tls_peer): + """The minimum repro of the production failure: TLS-dist on OTP 26. + + On erldistpy ≤0.1.7 against the prod portal this surfaces as + ``ChannelError: peer closed after 0/4 bytes`` on the first call. + """ + sname, cookie, certs = otp26_tls_peer + ctx = make_dist_tls_context( + cert=certs["client_cert"], + key=certs["client_key"], + ca=certs["ca"], + ) + with Node( + our_name="erldistpy_test_otp26_tls@localhost", + peer_name=sname, + cookie=cookie, + tls_context=ctx, + ) as n: + assert n.tls is True + reply = n.call("gen_target", (Atom("ping"), 7), timeout=5.0) + assert reply == (Atom("pong"), 7) + + +# --------------------------------------------------------------------------- +# OTP 26 + TLS dist + Elixir GenServer — the exact production combo +# --------------------------------------------------------------------------- + + +ELIXIR_TLS_SNAME = "erldistpy_otp26_elixir_tls" +ELIXIR_TLS_COOKIE = "ERLDISTPY_ELIXIR_TLS" + +# Inline Elixir script: spawn a GenServer registered as `gen_target` that +# answers the same {ping, X} / {add, A, B} calls. This is the SHAPE the +# real Wallet.Bridge uses — GenServer.handle_call/3 routing via the +# `:"$gen_call"` envelope, not a hand-coded receive. +ELIXIR_GENSERVER_BOOT = """ +defmodule GenTarget do + use GenServer + + def start_link, do: GenServer.start_link(__MODULE__, %{}, name: :gen_target) + + def init(state), do: {:ok, state} + + def handle_call({:ping, x}, _from, state), do: {:reply, {:pong, x}, state} + def handle_call({:add, a, b}, _from, state), do: {:reply, {:ok, a + b}, state} + def handle_call(other, _from, state), do: {:reply, {:error, {:bad_request, other}}, state} +end + +{:ok, _} = GenTarget.start_link() +Process.sleep(:infinity) +""" + + +@pytest.fixture(scope="module") +def otp26_elixir_tls_peer(tmp_path_factory): + """Elixir GenServer over TLS on OTP 26 — exactly what portal runs. + + Difference from otp26_tls_peer: the registered process is a real + Elixir GenServer (uses `gen_server:reply/2` for replies, all the + OTP machinery) instead of a hand-coded receive loop. + """ + if not _docker_available(): + pytest.skip("docker not installed") + if not shutil.which("openssl"): + pytest.skip("openssl not installed") + if not _socket_open("127.0.0.1", 4369): + pytest.skip("EPMD not running on host") + + import os as _os + import tempfile as _tempfile + + home_base = _os.environ.get( + "ERLDISTPY_TLS_OTP26_WORKDIR", str(Path.home() / ".erldistpy-test") + ) + Path(home_base).mkdir(parents=True, exist_ok=True) + workdir = Path(_tempfile.mkdtemp(prefix="elixir_tls_otp26_", dir=home_base)) + _os.chmod(workdir, 0o755) + host_certs = _gen_certs(workdir) + container_certs = { + k: v.replace(str(workdir), "/certs") for k, v in host_certs.items() + } + _write_ssl_config(workdir, container_certs) + ssl_config_container = "/certs/ssl_dist.config" + + boot_script = workdir / "boot.exs" + boot_script.write_text(ELIXIR_GENSERVER_BOOT) + + container_name = f"erldistpy-elixir-tls-{int(time.time())}" + + proc = subprocess.Popen( + [ + "docker", "run", "--rm", + "--name", container_name, + "--network", "host", + "-v", f"{workdir}:/certs:ro", + "elixir:1.16-otp-26", + "elixir", + "--sname", ELIXIR_TLS_SNAME, + "--cookie", ELIXIR_TLS_COOKIE, + "--erl", f"-proto_dist inet_tls -ssl_dist_optfile {ssl_config_container}", + "/certs/boot.exs", + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + deadline = time.monotonic() + 25.0 + ready = False + while time.monotonic() < deadline: + time.sleep(0.3) + if _epmd_node_present(ELIXIR_TLS_SNAME): + time.sleep(0.5) # Elixir GenServer needs an extra beat to register + ready = True + break + if not ready: + subprocess.run(["docker", "rm", "-f", container_name], capture_output=True) + proc.terminate() + proc.wait(timeout=3) + pytest.skip(f"Elixir TLS docker node {ELIXIR_TLS_SNAME} did not register") + + yield ELIXIR_TLS_SNAME, ELIXIR_TLS_COOKIE, host_certs + + subprocess.run(["docker", "rm", "-f", container_name], capture_output=True) + proc.terminate() + try: + proc.wait(timeout=3) + except subprocess.TimeoutExpired: + proc.kill() + + +def test_call_elixir_genserver_over_tls_otp26(otp26_elixir_tls_peer): + """Repro of the prod failure: erldistpy calls an Elixir GenServer over TLS. + + Difference from test_call_over_tls_otp26: peer is a real + GenServer, not a hand-coded receive. Replies go through + :gen_server.reply/2 which sends via :erlang.send/2 — same path as + Wallet.Bridge in production. + """ + sname, cookie, certs = otp26_elixir_tls_peer + ctx = make_dist_tls_context( + cert=certs["client_cert"], + key=certs["client_key"], + ca=certs["ca"], + ) + with Node( + our_name="erldistpy_test_elixir_tls@localhost", + peer_name=sname, + cookie=cookie, + tls_context=ctx, + ) as n: + assert n.tls is True + reply = n.call("gen_target", (Atom("ping"), 7), timeout=5.0) + assert reply == (Atom("pong"), 7) From b716f1487e59880cf79156a1f4252b9bd7023896 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Wed, 17 Jun 2026 09:32:17 -0400 Subject: [PATCH 2/4] tests/test_node_otp26: portal-match TLS config + long node names + container stdout capture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Iterates on the docker-based repro infrastructure while diagnosing the portal@unsandbox.com 'peer closed after 0/4 bytes' failure: - _write_ssl_config_portal_match() emits an ssl_dist.config with the exact options portal runs: permissive verify_fun (accepts bad_cert), versions [tlsv1.3, tlsv1.2], secure_renegotiate, server_name_indication disabled. Same shape as /opt/unsandbox/certs/inet_tls.conf. - Elixir docker peer now boots with --name (long FQDN, like prod) instead of --sname (short). Production portal is portal@unsandbox.com so the dist driver's routing path is different from short-name peers. - Elixir GenTarget now logs init + handle_call + handle_info. Fixture redirects container stdout to a file; test prints it on pass-or-fail so we can see whether the peer received our gen_call. - elixir image bumped to 1.16-otp-25 (portal runs OTP 25.3.2.5 / erts 13.2.2.5, NOT OTP 26 — discovered via the portal release's bundled erts version). Despite matching every dimension I can find (OTP version, TLS dist config, Elixir GenServer wrapping, long node names), the test still passes locally — so the production failure is something specific to the live portal beam state, not a general protocol or version issue. --- tests/test_node_otp26.py | 110 +++++++++++++++++++++++++++++++-------- 1 file changed, 89 insertions(+), 21 deletions(-) diff --git a/tests/test_node_otp26.py b/tests/test_node_otp26.py index ed3e629..b5ad7ae 100644 --- a/tests/test_node_otp26.py +++ b/tests/test_node_otp26.py @@ -279,6 +279,42 @@ def _write_ssl_config(workdir: Path, certs_in_container: dict[str, str]) -> str: return str(cfg) +def _write_ssl_config_portal_match(workdir: Path, certs_in_container: dict[str, str]) -> str: + """Match production portal's /opt/unsandbox/certs/inet_tls.conf exactly: + permissive verify_fun (accepts bad_cert), TLS 1.2/1.3 only, secure + renegotiate. This is the shape the production failure runs under.""" + cfg = workdir / "ssl_dist_portal.config" + permissive_verify_fun = ( + "{verify_fun, {fun(_,{bad_cert, _}, UserState) -> {valid, UserState}; " + " (_,{extension, _}, UserState) -> {unknown, UserState}; " + " (_, valid, UserState) -> {valid, UserState}; " + " (_, valid_peer, UserState) -> {valid, UserState} " + " end, []}}" + ) + body = ( + "[{server, " + f'[{{certfile, "{certs_in_container["server_cert"]}"}}, ' + f'{{keyfile, "{certs_in_container["server_key"]}"}}, ' + f'{{cacertfile, "{certs_in_container["ca"]}"}}, ' + "{verify, verify_peer}, " + "{fail_if_no_peer_cert, true}, " + f"{permissive_verify_fun}, " + "{secure_renegotiate, true}, " + "{versions, ['tlsv1.3', 'tlsv1.2']}]}, " + "{client, " + f'[{{certfile, "{certs_in_container["server_cert"]}"}}, ' + f'{{keyfile, "{certs_in_container["server_key"]}"}}, ' + f'{{cacertfile, "{certs_in_container["ca"]}"}}, ' + "{verify, verify_peer}, " + "{server_name_indication, disable}, " + f"{permissive_verify_fun}, " + "{secure_renegotiate, true}, " + "{versions, ['tlsv1.3', 'tlsv1.2']}]}]." + ) + cfg.write_text(body) + return str(cfg) + + @pytest.fixture(scope="module") def otp26_tls_peer(tmp_path_factory): """OTP 26 + TLS dist peer in Docker — the production combo. @@ -401,14 +437,29 @@ defmodule GenTarget do def start_link, do: GenServer.start_link(__MODULE__, %{}, name: :gen_target) - def init(state), do: {:ok, state} + def init(state) do + IO.puts("[gen_target] init") + {:ok, state} + end - def handle_call({:ping, x}, _from, state), do: {:reply, {:pong, x}, state} - def handle_call({:add, a, b}, _from, state), do: {:reply, {:ok, a + b}, state} - def handle_call(other, _from, state), do: {:reply, {:error, {:bad_request, other}}, state} + def handle_call(msg, from, state) do + IO.puts("[gen_target] handle_call msg=#{inspect(msg)} from=#{inspect(from)}") + reply = case msg do + {:ping, x} -> {:pong, x} + {:add, a, b} -> {:ok, a + b} + other -> {:error, {:bad_request, other}} + end + {:reply, reply, state} + end + + def handle_info(msg, state) do + IO.puts("[gen_target] handle_info msg=#{inspect(msg)}") + {:noreply, state} + end end {:ok, _} = GenTarget.start_link() +IO.puts("[gen_target] registered: #{inspect(Process.whereis(:gen_target))}") Process.sleep(:infinity) """ @@ -441,13 +492,22 @@ def otp26_elixir_tls_peer(tmp_path_factory): container_certs = { k: v.replace(str(workdir), "/certs") for k, v in host_certs.items() } - _write_ssl_config(workdir, container_certs) - ssl_config_container = "/certs/ssl_dist.config" + # Use portal's exact inet_tls.conf shape (permissive verify_fun, + # TLS 1.2/1.3 only, secure_renegotiate) so the test reproduces the + # exact wire-level conditions of the production failure. + _write_ssl_config_portal_match(workdir, container_certs) + ssl_config_container = "/certs/ssl_dist_portal.config" boot_script = workdir / "boot.exs" boot_script.write_text(ELIXIR_GENSERVER_BOOT) container_name = f"erldistpy-elixir-tls-{int(time.time())}" + container_log = workdir / "container.log" + + # Long node names (--name) match production. Short names (--sname) + # would route differently through the dist driver and may not + # reproduce the same failure mode. + long_node_name = f"{ELIXIR_TLS_SNAME}@127.0.0.1" proc = subprocess.Popen( [ @@ -455,15 +515,15 @@ def otp26_elixir_tls_peer(tmp_path_factory): "--name", container_name, "--network", "host", "-v", f"{workdir}:/certs:ro", - "elixir:1.16-otp-26", + "elixir:1.16-otp-25", "elixir", - "--sname", ELIXIR_TLS_SNAME, + "--name", long_node_name, "--cookie", ELIXIR_TLS_COOKIE, "--erl", f"-proto_dist inet_tls -ssl_dist_optfile {ssl_config_container}", "/certs/boot.exs", ], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, + stdout=open(container_log, "w"), + stderr=subprocess.STDOUT, ) deadline = time.monotonic() + 25.0 @@ -480,7 +540,7 @@ def otp26_elixir_tls_peer(tmp_path_factory): proc.wait(timeout=3) pytest.skip(f"Elixir TLS docker node {ELIXIR_TLS_SNAME} did not register") - yield ELIXIR_TLS_SNAME, ELIXIR_TLS_COOKIE, host_certs + yield ELIXIR_TLS_SNAME, ELIXIR_TLS_COOKIE, host_certs, container_log subprocess.run(["docker", "rm", "-f", container_name], capture_output=True) proc.terminate() @@ -498,18 +558,26 @@ def test_call_elixir_genserver_over_tls_otp26(otp26_elixir_tls_peer): :gen_server.reply/2 which sends via :erlang.send/2 — same path as Wallet.Bridge in production. """ - sname, cookie, certs = otp26_elixir_tls_peer + sname, cookie, certs, container_log = otp26_elixir_tls_peer ctx = make_dist_tls_context( cert=certs["client_cert"], key=certs["client_key"], ca=certs["ca"], ) - with Node( - our_name="erldistpy_test_elixir_tls@localhost", - peer_name=sname, - cookie=cookie, - tls_context=ctx, - ) as n: - assert n.tls is True - reply = n.call("gen_target", (Atom("ping"), 7), timeout=5.0) - assert reply == (Atom("pong"), 7) + try: + with Node( + our_name="erldistpy_test_elixir_tls@localhost", + peer_name=sname, + cookie=cookie, + tls_context=ctx, + ) as n: + assert n.tls is True + reply = n.call("gen_target", (Atom("ping"), 7), timeout=5.0) + assert reply == (Atom("pong"), 7) + finally: + # Whether pass or fail, dump container output for diagnosis. + try: + text = container_log.read_text() + print(f"=== container stdout ===\n{text}\n=== end ===") + except FileNotFoundError: + pass From 35d0eca342792dbf626d435db25609bacc24d04c Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Wed, 17 Jun 2026 09:57:12 -0400 Subject: [PATCH 3/4] tests: Mix release + TLS dist + OTP 25 + Wallet.Bridge GenServer (passes) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the loop on docker-based repro: builds a real Elixir Mix release (with :ssl included) containing one GenServer registered as Elixir.Wallet.Bridge, runs it under TLS dist with the production inet_tls.conf shape, hits it from erldistpy.Node.call. Passes cleanly with erldistpy 0.1.7. So the production portal failure ('peer closed after 0/4 bytes' against portal@unsandbox.com) isn't reproducible in isolation, even with every dimension matched (Mix release, OTP 25, TLS dist with permissive verify_fun + tls1.2/1.3 + secure_renegotiate, Elixir GenServer, long FQDN node names). The bug must be in interaction with portal's other dist connections or its larger supervisor tree — beyond what we can repro without standing up the full portal app locally. --- tests/test_node_mix_release.py | 181 +++++++++++++++++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100644 tests/test_node_mix_release.py diff --git a/tests/test_node_mix_release.py b/tests/test_node_mix_release.py new file mode 100644 index 0000000..95cd5bd --- /dev/null +++ b/tests/test_node_mix_release.py @@ -0,0 +1,181 @@ +"""Mix release + TLS dist test — the final 'is it the same as prod?' check. + +Builds a minimal Elixir Mix release containing one GenServer registered +as Wallet.Bridge, runs it in a docker container with TLS dist enabled +using the production-matching inet_tls.conf shape, then exercises +erldistpy.Node.call against it. + +Together with test_node_otp26.py this exhausts every dimension we +identified differs between dev box (OTP 24, plain Erlang) and the +production portal beam (OTP 25, Mix release, TLS dist, Elixir +GenServer). If THIS passes too, the production-only failure is +something specific to the live portal beam state that we can't +reproduce in isolation — most likely an interaction with portal's +other dist connections or its larger supervisor tree. +""" + +from __future__ import annotations + +import os +import shutil +import socket +import subprocess +import time +from pathlib import Path + +import pytest + +from erldistpy.etf import Atom +from erldistpy.node import Node +from erldistpy.tls import make_dist_tls_context + +RELEASE_NAME = "wallet_bridge_test" +COOKIE = "WALLET_BRIDGE_TEST_COOKIE" + +MIX_EXS = """\ +defmodule WalletBridgeTest.MixProject do + use Mix.Project + def project, do: [ + app: :wallet_bridge_test, + version: "0.1.0", + elixir: "~> 1.16", + deps: [], + releases: [wallet_bridge_test: [include_executables_for: [:unix]]] + ] + def application, do: [ + extra_applications: [:logger, :ssl, :crypto, :public_key], + mod: {WalletBridgeTest.Application, []} + ] +end +""" + +LIB_EX = """\ +defmodule WalletBridgeTest.Application do + use Application + def start(_, _), do: Supervisor.start_link([Wallet.Bridge], strategy: :one_for_one) +end + +defmodule Wallet.Bridge do + use GenServer + require Logger + def start_link(_), do: GenServer.start_link(__MODULE__, %{}, name: __MODULE__) + def init(s), do: {:ok, s} + def handle_call(msg, _from, s), do: {:reply, {:ok, msg}, s} +end +""" + + +def _docker() -> str | None: + return shutil.which("docker") + + +def _epmd_node_present(sname: str) -> bool: + try: + result = subprocess.run( + ["epmd", "-names"], capture_output=True, text=True, timeout=2 + ) + except (FileNotFoundError, subprocess.TimeoutExpired): + return False + return f"name {sname} at port" in result.stdout + + +@pytest.fixture(scope="module") +def mix_release_peer(tmp_path_factory): + if _docker() is None: + pytest.skip("docker not installed") + if not shutil.which("openssl"): + pytest.skip("openssl not installed") + + # Must be under $HOME for snap docker to bind-mount it. + base = Path.home() / ".erldistpy-test" + base.mkdir(exist_ok=True) + workdir = base / f"mix_release_{int(time.time())}" + workdir.mkdir() + (workdir / "lib").mkdir() + (workdir / "mix.exs").write_text(MIX_EXS) + (workdir / "lib" / "wallet_bridge.ex").write_text(LIB_EX) + os.chmod(workdir, 0o755) + + # Build the release + build = subprocess.run( + ["docker", "run", "--rm", + "-v", f"{workdir}:/app", "-w", "/app", + "elixir:1.16-otp-25", + "sh", "-c", "MIX_ENV=prod mix release wallet_bridge_test --overwrite"], + capture_output=True, timeout=180, + ) + if build.returncode != 0: + pytest.skip(f"mix release build failed: {build.stderr.decode()[-500:]}") + + # Re-use cert generation from test_node_otp26 (we duplicate-import to avoid + # cross-file fixture coupling; if you have it imported already, prefer that.) + from tests.test_node_otp26 import _gen_certs, _write_ssl_config_portal_match + certs = _gen_certs(workdir) + container_certs = {k: v.replace(str(workdir), "/app") for k, v in certs.items()} + _write_ssl_config_portal_match(workdir, container_certs) + # Tweak: ssl_dist_portal.config references /certs paths; rewrite to /app. + cfg_path = workdir / "ssl_dist_portal.config" + cfg_text = cfg_path.read_text().replace("/certs", "/app/certs") + (workdir / "certs").mkdir(exist_ok=True) + for f in ("ca.pem", "server.pem", "server.key", "client.pem", "client.key"): + if (workdir / f).exists(): + shutil.copy(workdir / f, workdir / "certs" / f) + cfg_path.write_text(cfg_text) + + container_id_file = workdir / "cid" + cid = subprocess.run( + ["docker", "run", "--rm", "-d", "--network", "host", + "-v", f"{workdir}:/app", "-w", "/app", + "-e", "RELEASE_DISTRIBUTION=name", + "-e", f"RELEASE_NODE={RELEASE_NAME}@127.0.0.1", + "-e", f"RELEASE_COOKIE={COOKIE}", + "-e", "ERL_FLAGS=-proto_dist inet_tls -ssl_dist_optfile /app/ssl_dist_portal.config", + "elixir:1.16-otp-25", + f"_build/prod/rel/{RELEASE_NAME}/bin/{RELEASE_NAME}", "start"], + capture_output=True, text=True, + ).stdout.strip() + container_id_file.write_text(cid) + + deadline = time.monotonic() + 20.0 + ready = False + while time.monotonic() < deadline: + time.sleep(0.3) + if _epmd_node_present(RELEASE_NAME): + time.sleep(0.5) + ready = True + break + if not ready: + if cid: + subprocess.run(["docker", "kill", cid], capture_output=True) + pytest.skip(f"Mix release {RELEASE_NAME} did not register with epmd") + + yield RELEASE_NAME, COOKIE, certs + + if cid: + subprocess.run(["docker", "kill", cid], capture_output=True) + + +def test_call_against_mix_release_tls(mix_release_peer): + """erldistpy → real Mix release Wallet.Bridge over TLS dist on OTP 25. + + Closest local equivalent of the production failure environment. + If THIS passes, the production failure isn't reproducible without + a live portal beam in a mesh — meaning the bug is interaction-level + (other connections, supervisor state) not protocol-level. + """ + sname, cookie, certs = mix_release_peer + ctx = make_dist_tls_context( + cert=certs["client_cert"], key=certs["client_key"], ca=certs["ca"], + ) + with Node( + our_name="probe@127.0.0.1", + peer_name=sname, + peer_host="127.0.0.1", + cookie=cookie, + tls_context=ctx, + ) as n: + assert n.tls is True + reply = n.call("Elixir.Wallet.Bridge", (Atom("ping"), 42), timeout=8.0) + # Bridge wraps every call as {:ok, msg} — matches what the production + # Wallet.Bridge does on the happy path (modulo the cammy forward). + assert reply == (Atom("ok"), (Atom("ping"), 42)) From ff5ec74fc2518f20e704f9490790df5cfc3f9403 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Wed, 17 Jun 2026 10:59:42 -0400 Subject: [PATCH 4/4] tls: keep maximum_version at MAXIMUM_SUPPORTED (was wrong workaround) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reverted the TLS 1.2 max default I added while chasing what turned out to be a make_post_sell bug, not an OTP bug. The actual root cause of the production smoke-test failure was that mps_wallet_dist_health hardcoded the REG_SEND target as 'Elixir.Wallet.Service' but portal registers the relay as 'Elixir.Wallet.Bridge' — REG_SEND to an unregistered name silently drops at the dist driver, presenting as the 'peer closed after 0/4 bytes' symptom that I misdiagnosed as a TLS problem. Fix landed in make_post_sell 1.2.1. erldistpy's defaults should stay where Python ssl wants them; callers who want a specific version can still pin via kwargs. --- erldistpy/tls.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/erldistpy/tls.py b/erldistpy/tls.py index 36e7569..52f9596 100644 --- a/erldistpy/tls.py +++ b/erldistpy/tls.py @@ -36,6 +36,7 @@ def make_dist_tls_context( ca: str, check_hostname: bool = False, minimum_version: ssl.TLSVersion = ssl.TLSVersion.TLSv1_2, + maximum_version: ssl.TLSVersion = ssl.TLSVersion.MAXIMUM_SUPPORTED, ) -> ssl.SSLContext: """Build an SSLContext for a TLS-dist client. @@ -43,11 +44,16 @@ def make_dist_tls_context( and requires one). ``check_hostname=False`` because dist nodes are identified by their cookie + cert chain, not by SNI hostname; flip on if your CA pins per-node CNs. + + ``maximum_version`` defaults to the highest version Python's ssl + supports (TLS 1.3 in practice). Pin to TLS 1.2 explicitly only if + you hit interop issues against an older Erlang peer. """ ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) ctx.check_hostname = check_hostname ctx.verify_mode = ssl.CERT_REQUIRED ctx.minimum_version = minimum_version + ctx.maximum_version = maximum_version ctx.load_cert_chain(certfile=cert, keyfile=key) ctx.load_verify_locations(cafile=ca) return ctx