From 271d712237bf91a2f0bbe1cd708f178446b70fd8 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Tue, 16 Jun 2026 12:06:07 -0400 Subject: [PATCH] phase 6: TLS dist via inet_tls_dist make_dist_tls_context() builds an ssl.SSLContext tuned for OTP defaults (verify_peer, mTLS, TLSv1.2 minimum). Node accepts tls_context= and wraps the TCP socket in TLS before the v6 handshake runs. Critical quirk found by experimentation: inet_tls_dist uses {packet, 4} on the SSL socket during the handshake. Plain inet_tcp_dist uses {packet, 2} for handshake then switches to {packet, 4} post-nodeup. handshake() now takes a frame_size= kwarg (2 or 4); Node auto-selects 4 whenever tls_context is supplied. Cert requirements (found by experimentation against Erlang E2E): - CA cert with basicConstraints CA:TRUE - Leaf certs with SAN including the dist hostname (and localhost) - extendedKeyUsage covering both serverAuth and clientAuth Tests: - make_dist_tls_context unit tests - Live: spawn erl -proto_dist inet_tls with SAN-bearing certs, Node.call(gen_target, {ping, 99}) round-trips through the tunnel - Live negative: plaintext connection to TLS-only peer must fail - Live negative: client cert from a different CA must fail 115 tests green across 5 consecutive runs, lint clean. --- .gitignore | 1 + docs/ROADMAP.md | 17 ++- erldistpy/__init__.py | 2 + erldistpy/handshake.py | 39 ++++-- erldistpy/node.py | 26 +++- erldistpy/tls.py | 53 ++++++++ tests/test_tls.py | 296 +++++++++++++++++++++++++++++++++++++++++ 7 files changed, 416 insertions(+), 18 deletions(-) create mode 100644 erldistpy/tls.py create mode 100644 tests/test_tls.py diff --git a/.gitignore b/.gitignore index fd852a8..9389103 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ dist/ *.egg-info/ .coverage htmlcov/ +erl_crash.dump diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index a61650f..1940611 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -79,11 +79,20 @@ if we hit a peer that requires it. responder, drop-on-the-floor against an unknown registered name, Ref uniqueness across the Node lifetime -## Phase 6 — TLS dist +## Phase 6 — TLS dist ✅ -- Wrap the post-EPMD socket in TLS -- Match `inet_tls_dist` config on the Erlang side (cert + key + ca paths) -- Same handshake, just runs inside the TLS tunnel +- `make_dist_tls_context(cert=, key=, ca=)` helper builds an + `ssl.SSLContext` tuned for OTP defaults (verify_peer, TLSv1.2 min) +- `Node(..., tls_context=ctx)` wraps the post-EPMD socket in TLS + before the dist handshake +- Auto-selects `frame_size=4` for the handshake when TLS is in use + (the non-obvious quirk: `inet_tls_dist` uses `{packet, 4}` on the + SSL socket, where plain `inet_tcp_dist` uses `{packet, 2}` during + handshake) +- Live tests: generate CA + SAN-bearing certs with openssl, spawn + `erl -proto_dist inet_tls`, drive a full `Node.call()` through + the tunnel. Negative tests confirm: plaintext connection fails, + wrong-CA client cert fails. ## Phase 7 — unfeed integration diff --git a/erldistpy/__init__.py b/erldistpy/__init__.py index 17812f7..2439f0d 100644 --- a/erldistpy/__init__.py +++ b/erldistpy/__init__.py @@ -13,6 +13,7 @@ from erldistpy.etf import ( ) from erldistpy.handshake import HandshakeError, HandshakeResult, handshake from erldistpy.node import CallProtocolError, CallTimeout, Node, NodeError +from erldistpy.tls import make_dist_tls_context __all__ = [ "Atom", @@ -34,4 +35,5 @@ __all__ = [ "encode", "handshake", "lookup", + "make_dist_tls_context", ] diff --git a/erldistpy/handshake.py b/erldistpy/handshake.py index db634bb..bbe5df0 100644 --- a/erldistpy/handshake.py +++ b/erldistpy/handshake.py @@ -57,12 +57,19 @@ def handshake( our_flags: int = DEFAULT_FLAGS, our_creation: int | None = None, timeout: float = 5.0, + frame_size: int = 2, ) -> HandshakeResult: """Drive ``sock`` through a v6 distribution handshake. ``our_name`` is the full ``name@host`` we are calling as. ``cookie`` is the shared secret with the peer. + ``frame_size`` is the length-prefix size in bytes — 2 for plain TCP + dist (``inet_tcp_dist``, the default), 4 for TLS dist + (``inet_tls_dist``, which sets ``{packet, 4}`` on the SSL socket + before the handshake). Pass ``4`` whenever the socket is wrapped in + TLS; :class:`Node` selects this automatically based on ``tls_context``. + On success the socket is left open and authenticated. On any protocol or auth failure raises :class:`HandshakeError`. """ @@ -70,22 +77,24 @@ def handshake( raise ValueError(f"our_name must look like 'name@host', got {our_name!r}") if not (our_flags & DFLAG_HANDSHAKE_23): raise ValueError("our_flags must include DFLAG_HANDSHAKE_23 (v6 handshake)") + if frame_size not in (2, 4): + raise ValueError(f"frame_size must be 2 or 4, got {frame_size}") if our_creation is None: our_creation = _random_creation() sock.settimeout(timeout) # Step 1: send_name - _send_frame(sock, build_send_name(our_name, our_flags, our_creation)) + _send_frame(sock, build_send_name(our_name, our_flags, our_creation), frame_size) # Step 2: recv_status - status_frame = _recv_frame(sock) + status_frame = _recv_frame(sock, frame_size) status = parse_recv_status(status_frame) if status not in ("ok", "ok_simultaneous"): raise HandshakeError(f"peer rejected handshake: status={status!r}") # Step 3: recv_challenge - challenge_frame = _recv_frame(sock) + challenge_frame = _recv_frame(sock, frame_size) peer_flags, peer_challenge, peer_creation, peer_name = parse_recv_challenge(challenge_frame) if not (peer_flags & DFLAG_HANDSHAKE_23): raise HandshakeError("peer does not speak v6 handshake (no DFLAG_HANDSHAKE_23)") @@ -93,10 +102,10 @@ def handshake( # Step 4: send_challenge_reply with our challenge + MD5(cookie || peer_challenge) our_challenge = _random_challenge() reply_digest = cookie_digest(cookie, peer_challenge) - _send_frame(sock, build_challenge_reply(our_challenge, reply_digest)) + _send_frame(sock, build_challenge_reply(our_challenge, reply_digest), frame_size) # Step 5: recv_challenge_ack — peer must answer with MD5(cookie || our_challenge) - ack_frame = _recv_frame(sock) + ack_frame = _recv_frame(sock, frame_size) ack_digest = parse_challenge_ack(ack_frame) expected = cookie_digest(cookie, our_challenge) if ack_digest != expected: @@ -187,15 +196,21 @@ def cookie_digest(cookie: str, challenge: int) -> bytes: # --------------------------------------------------------------------------- -def _send_frame(sock: socket.socket, body: bytes) -> None: - if len(body) > 0xFFFF: - raise HandshakeError(f"handshake frame too large: {len(body)} bytes") - sock.sendall(struct.pack(">H", len(body)) + body) +def _send_frame(sock: socket.socket, body: bytes, frame_size: int = 2) -> None: + if frame_size == 2: + if len(body) > 0xFFFF: + raise HandshakeError(f"handshake frame too large for 2-byte length: {len(body)} bytes") + sock.sendall(struct.pack(">H", len(body)) + body) + else: + sock.sendall(struct.pack(">I", len(body)) + body) -def _recv_frame(sock: socket.socket) -> bytes: - hdr = _recv_exact(sock, 2) - (length,) = struct.unpack(">H", hdr) +def _recv_frame(sock: socket.socket, frame_size: int = 2) -> bytes: + hdr = _recv_exact(sock, frame_size) + if frame_size == 2: + (length,) = struct.unpack(">H", hdr) + else: + (length,) = struct.unpack(">I", hdr) if length == 0: return b"" return _recv_exact(sock, length) diff --git a/erldistpy/node.py b/erldistpy/node.py index ef32cb5..fe1ec25 100644 --- a/erldistpy/node.py +++ b/erldistpy/node.py @@ -18,6 +18,7 @@ from __future__ import annotations import os import socket +import ssl import struct from erldistpy.channel import ( @@ -59,7 +60,16 @@ class Node: cookie: str, peer_host: str = "localhost", connect_timeout: float = 5.0, + tls_context: ssl.SSLContext | None = None, + tls_server_hostname: str | None = None, ): + """Open a dist connection to the peer. + + If ``tls_context`` is supplied, the TCP socket gets wrapped in + TLS before the v6 handshake runs. EPMD lookup stays in plaintext + (EPMD never negotiates TLS itself). Use + :func:`erldistpy.tls.make_dist_tls_context` to build the context. + """ if "@" not in our_name: raise ValueError(f"our_name must look like 'name@host', got {our_name!r}") self.our_name = our_name @@ -67,21 +77,33 @@ class Node: self.peer_host = peer_host self._cookie = cookie # never logged self._our_atom = Atom(our_name) + self.tls = tls_context is not None info = lookup(peer_name, host=peer_host, timeout=connect_timeout) if info is None: raise NodeError(f"peer node {peer_name!r} not registered with EPMD on {peer_host}") - sock = socket.create_connection((peer_host, info.port), timeout=connect_timeout) + raw_sock = socket.create_connection((peer_host, info.port), timeout=connect_timeout) try: + if tls_context is not None: + sock: socket.socket = tls_context.wrap_socket( + raw_sock, + server_hostname=tls_server_hostname, + ) + else: + sock = raw_sock + # inet_tls_dist sets {packet, 4} on the SSL socket pre-nodeup; + # inet_tcp_dist uses {packet, 2}. Match the wire format. + frame_size = 4 if tls_context is not None else 2 result: HandshakeResult = handshake( sock, our_name=our_name, cookie=cookie, our_flags=DEFAULT_FLAGS, timeout=connect_timeout, + frame_size=frame_size, ) except Exception: - sock.close() + raw_sock.close() raise self._sock = sock self._channel = Channel(sock, recv_timeout=None) diff --git a/erldistpy/tls.py b/erldistpy/tls.py new file mode 100644 index 0000000..36e7569 --- /dev/null +++ b/erldistpy/tls.py @@ -0,0 +1,53 @@ +"""TLS helper for distribution over ``inet_tls_dist``. + +Erlang's ``inet_tls_dist`` runs the distribution protocol inside a +mutual-TLS tunnel. The wire flow is mostly unchanged after the tunnel +exists — EPMD still hands out the listener port in plaintext, then the +TCP connect to that port is immediately wrapped in TLS, then the v6 +handshake runs over the wrapped socket. + +One non-obvious quirk worth knowing: ``inet_tls_dist`` sets +``{packet, 4}`` on the SSL socket pre-nodeup (plain ``inet_tcp_dist`` +uses ``{packet, 2}`` for the handshake then switches to ``{packet, 4}`` +post-nodeup). Our :func:`erldistpy.handshake.handshake` takes a +``frame_size=`` kwarg; :class:`erldistpy.node.Node` passes ``4`` +automatically whenever a ``tls_context`` is supplied. + +This module exposes one helper, :func:`make_dist_tls_context`, which +builds an :class:`ssl.SSLContext` tuned for the OTP server's defaults +(verify_peer, fail_if_no_peer_cert). Pass it into :class:`Node` via the +``tls_context`` kwarg. + +Secrets discipline: ``cert`` / ``key`` / ``ca`` are file *paths*. We +never read PEM contents into Python — :class:`ssl.SSLContext` loads +them through OpenSSL directly. Caller's job to keep ``key`` readable +only by the running process. +""" + +from __future__ import annotations + +import ssl + + +def make_dist_tls_context( + *, + cert: str, + key: str, + ca: str, + check_hostname: bool = False, + minimum_version: ssl.TLSVersion = ssl.TLSVersion.TLSv1_2, +) -> ssl.SSLContext: + """Build an SSLContext for a TLS-dist client. + + Defaults match the OTP server side (which verifies the client cert + and requires one). ``check_hostname=False`` because dist nodes are + identified by their cookie + cert chain, not by SNI hostname; flip + on if your CA pins per-node CNs. + """ + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.check_hostname = check_hostname + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.minimum_version = minimum_version + ctx.load_cert_chain(certfile=cert, keyfile=key) + ctx.load_verify_locations(cafile=ca) + return ctx diff --git a/tests/test_tls.py b/tests/test_tls.py new file mode 100644 index 0000000..d6e6afa --- /dev/null +++ b/tests/test_tls.py @@ -0,0 +1,296 @@ +"""TLS dist tests. + +Unit tests cover the SSLContext builder. Live tests generate a CA + +server/client certs with openssl, spawn ``erl -proto_dist inet_tls``, +and prove a full Node.call() round-trips through the TLS tunnel. + +Two non-obvious requirements found by experimentation: + 1. ``inet_tls_dist`` uses ``{packet, 4}`` on the SSL socket for the + handshake (plain TCP dist uses ``{packet, 2}``). Handshake length + prefix is 4 bytes over TLS. + 2. Erlang validates the peer cert's hostname; certs need a SAN + including the dist node hostname (``hostname`` from ``uname -n``) + and/or ``localhost``. +""" + +from __future__ import annotations + +import os +import shutil +import socket +import ssl +import subprocess +import time +from pathlib import Path + +import pytest + +from erldistpy.epmd import lookup +from erldistpy.etf import Atom +from erldistpy.handshake import HandshakeError +from erldistpy.node import Node +from erldistpy.tls import make_dist_tls_context + +# -------------------------------------------------------------------------- +# Pure +# -------------------------------------------------------------------------- + + +def test_make_dist_tls_context_missing_files(tmp_path): + with pytest.raises((FileNotFoundError, ssl.SSLError)): + make_dist_tls_context( + cert=str(tmp_path / "missing.pem"), + key=str(tmp_path / "missing.key"), + ca=str(tmp_path / "missing-ca.pem"), + ) + + +# -------------------------------------------------------------------------- +# Live — generate certs, spawn erl with inet_tls_dist, call() over TLS +# -------------------------------------------------------------------------- + + +READY_FLAG = "/tmp/erldistpy_test_tls_ready" +SNAME = "erldistpy_tls" +COOKIE = "ERLDISTPY_TLS_COOKIE" + +ERL_GEN_BOOT = ( + "Handle = fun " + " ({ping, X}) -> {pong, X}; " + " (Other) -> {error, Other} " + "end, " + "Loop = fun(F) -> " + " receive " + " {'$gen_call', {From, Ref}, Request} -> " + " From ! {Ref, Handle(Request)}, F(F) " + " end " + "end, " + "Pid = spawn(fun() -> Loop(Loop) end), " + "register(gen_target, Pid), " + f'file:write_file("{READY_FLAG}", "1"), ' + "timer:sleep(infinity)." +) + + +def _epmd_running() -> bool: + try: + with socket.create_connection(("localhost", 4369), timeout=0.5): + return True + except OSError: + return False + + +def _gen_certs(workdir: Path) -> dict[str, str]: + """Generate a CA + server + client cert with SAN matching the hostname. + + Erlang's TLS-dist validates the peer cert's hostname during + handshake; without a matching SAN it rejects the cert chain + (``{bad_cert,hostname_check_failed}``). + """ + hostname = socket.gethostname() + # CA with proper CA:TRUE extension + ca_cnf = workdir / "ca.cnf" + ca_cnf.write_text( + "[req]\n" + "distinguished_name = dn\n" + "x509_extensions = v3_ca\n" + "prompt = no\n" + "[dn]\n" + "CN = erldistpy-test-ca\n" + "[v3_ca]\n" + "basicConstraints = critical,CA:TRUE\n" + "keyUsage = critical,keyCertSign,cRLSign\n" + "subjectKeyIdentifier = hash\n" + ) + subprocess.run( + [ + "openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes", + "-keyout", str(workdir / "ca.key"), + "-out", str(workdir / "ca.pem"), + "-days", "1", + "-config", str(ca_cnf), "-extensions", "v3_ca", + ], + check=True, capture_output=True, + ) + os.chmod(workdir / "ca.key", 0o600) + + # Leaf cert extensions including SAN + leaf_cnf = workdir / "leaf.cnf" + leaf_cnf.write_text( + "[v3]\n" + "basicConstraints = CA:FALSE\n" + "keyUsage = digitalSignature,keyEncipherment\n" + "extendedKeyUsage = serverAuth,clientAuth\n" + "subjectAltName = @alt\n" + "[alt]\n" + f"DNS.1 = {hostname}\n" + "DNS.2 = localhost\n" + ) + + def _signed_cert(name: str) -> None: + subprocess.run( + [ + "openssl", "req", "-newkey", "rsa:2048", "-nodes", + "-keyout", str(workdir / f"{name}.key"), + "-out", str(workdir / f"{name}.csr"), + "-subj", f"/CN={hostname}", + ], + check=True, capture_output=True, + ) + subprocess.run( + [ + "openssl", "x509", "-req", + "-in", str(workdir / f"{name}.csr"), + "-CA", str(workdir / "ca.pem"), + "-CAkey", str(workdir / "ca.key"), + "-CAcreateserial", + "-out", str(workdir / f"{name}.pem"), + "-days", "1", + "-extfile", str(leaf_cnf), "-extensions", "v3", + ], + check=True, capture_output=True, + ) + os.chmod(workdir / f"{name}.key", 0o600) + + _signed_cert("server") + _signed_cert("client") + return { + "ca": str(workdir / "ca.pem"), + "server_cert": str(workdir / "server.pem"), + "server_key": str(workdir / "server.key"), + "client_cert": str(workdir / "client.pem"), + "client_key": str(workdir / "client.key"), + } + + +def _write_ssl_config(workdir: Path, certs: dict[str, str]) -> str: + """Write the ssl_dist_opts.config file Erlang reads.""" + cfg = workdir / "ssl_dist.config" + body = ( + "[{server, " + f'[{{certfile, "{certs["server_cert"]}"}}, ' + f'{{keyfile, "{certs["server_key"]}"}}, ' + f'{{cacertfile, "{certs["ca"]}"}}, ' + "{verify, verify_peer}, " + "{fail_if_no_peer_cert, true}]}, " + "{client, " + f'[{{certfile, "{certs["server_cert"]}"}}, ' + f'{{keyfile, "{certs["server_key"]}"}}, ' + f'{{cacertfile, "{certs["ca"]}"}}, ' + "{verify, verify_peer}]}]." + ) + cfg.write_text(body) + return str(cfg) + + +@pytest.fixture(scope="module") +def tls_peer(tmp_path_factory): + if not shutil.which("erl"): + pytest.skip("erl not installed") + if not shutil.which("openssl"): + pytest.skip("openssl not installed") + if not _epmd_running(): + pytest.skip("EPMD not running") + + if os.path.exists(READY_FLAG): + os.remove(READY_FLAG) + + workdir = tmp_path_factory.mktemp("tls") + certs = _gen_certs(workdir) + ssl_config = _write_ssl_config(workdir, certs) + + proc = subprocess.Popen( + [ + "erl", + "-sname", SNAME, + "-setcookie", COOKIE, + "-proto_dist", "inet_tls", + "-ssl_dist_optfile", ssl_config, + "-noshell", + "-eval", ERL_GEN_BOOT, + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + deadline = time.monotonic() + 8.0 + ready = False + while time.monotonic() < deadline: + time.sleep(0.15) + if lookup(SNAME, timeout=0.5) is None: + continue + if os.path.exists(READY_FLAG): + ready = True + break + if not ready: + proc.terminate() + proc.wait(timeout=3) + pytest.skip("erl TLS-dist node did not become ready") + + yield SNAME, COOKIE, certs + + proc.terminate() + try: + proc.wait(timeout=3) + except subprocess.TimeoutExpired: + proc.kill() + if os.path.exists(READY_FLAG): + os.remove(READY_FLAG) + + +def _client_name() -> str: + return f"erldistpy_tls_client@{socket.gethostname()}" + + +def test_call_over_tls(tls_peer): + sname, cookie, certs = tls_peer + ctx = make_dist_tls_context( + cert=certs["client_cert"], + key=certs["client_key"], + ca=certs["ca"], + ) + with Node( + our_name=_client_name(), + peer_name=sname, + cookie=cookie, + tls_context=ctx, + ) as node: + assert node.tls is True + reply = node.call("gen_target", (Atom("ping"), 99)) + assert reply == (Atom("pong"), 99) + + +def test_plaintext_connection_to_tls_peer_fails(tls_peer): + """Connecting without TLS to a node that requires TLS must fail — + that's the security boundary the tunnel exists to enforce.""" + sname, cookie, _ = tls_peer + with pytest.raises((HandshakeError, OSError, ssl.SSLError, ConnectionError)): + Node( + our_name=_client_name(), + peer_name=sname, + cookie=cookie, + connect_timeout=2.0, + ) + + +def test_wrong_ca_fails(tls_peer, tmp_path): + """A client with a cert signed by a CA the server doesn't trust + must fail to handshake.""" + sname, cookie, certs = tls_peer + # Generate a totally separate CA + client cert in a fresh dir + other = tmp_path / "other" + other.mkdir() + other_certs = _gen_certs(other) + bad_ctx = make_dist_tls_context( + cert=other_certs["client_cert"], + key=other_certs["client_key"], + ca=certs["ca"], # we trust the right CA, but our cert isn't signed by it + ) + with pytest.raises((ssl.SSLError, OSError, ConnectionError, HandshakeError)): + Node( + our_name=_client_name(), + peer_name=sname, + cookie=cookie, + tls_context=bad_ctx, + connect_timeout=3.0, + )