python/erldistpy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

phase 6: TLS dist via inet_tls_dist

Browse source 
make_dist_tls_context() builds an ssl.SSLContext tuned for OTP defaults
(verify_peer, mTLS, TLSv1.2 minimum). Node accepts tls_context= and
wraps the TCP socket in TLS before the v6 handshake runs.

Critical quirk found by experimentation: inet_tls_dist uses {packet, 4}
on the SSL socket during the handshake. Plain inet_tcp_dist uses
{packet, 2} for handshake then switches to {packet, 4} post-nodeup.
handshake() now takes a frame_size= kwarg (2 or 4); Node auto-selects 4
whenever tls_context is supplied.

Cert requirements (found by experimentation against Erlang E2E):
  - CA cert with basicConstraints CA:TRUE
  - Leaf certs with SAN including the dist hostname (and localhost)
  - extendedKeyUsage covering both serverAuth and clientAuth

Tests:
  - make_dist_tls_context unit tests
  - Live: spawn erl -proto_dist inet_tls with SAN-bearing certs,
    Node.call(gen_target, {ping, 99}) round-trips through the tunnel
  - Live negative: plaintext connection to TLS-only peer must fail
  - Live negative: client cert from a different CA must fail

115 tests green across 5 consecutive runs, lint clean.
This commit is contained in:
russell@unturf.com 2026-06-16 12:06:07 -04:00
parent ac4636e00c
commit 271d712237
No known key found for this signature in database
7 changed files with 416 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -10,3 +10,4 @@ dist/
*.egg-info/
.coverage
htmlcov/
erl_crash.dump

17
docs/ROADMAP.md
View file

@ -79,11 +79,20 @@ if we hit a peer that requires it.
responder, drop-on-the-floor against an unknown registered name,
Ref uniqueness across the Node lifetime
## Phase 6 — TLS dist
## Phase 6 — TLS dist ✅
- Wrap the post-EPMD socket in TLS
- Match `inet_tls_dist` config on the Erlang side (cert + key + ca paths)
- Same handshake, just runs inside the TLS tunnel
- `make_dist_tls_context(cert=, key=, ca=)` helper builds an
`ssl.SSLContext` tuned for OTP defaults (verify_peer, TLSv1.2 min)
- `Node(..., tls_context=ctx)` wraps the post-EPMD socket in TLS
before the dist handshake
- Auto-selects `frame_size=4` for the handshake when TLS is in use
(the non-obvious quirk: `inet_tls_dist` uses `{packet, 4}` on the
SSL socket, where plain `inet_tcp_dist` uses `{packet, 2}` during
handshake)
- Live tests: generate CA + SAN-bearing certs with openssl, spawn
`erl -proto_dist inet_tls`, drive a full `Node.call()` through
the tunnel. Negative tests confirm: plaintext connection fails,
wrong-CA client cert fails.
## Phase 7 — unfeed integration

2
erldistpy/__init__.py
View file

@ -13,6 +13,7 @@ from erldistpy.etf import (
)
from erldistpy.handshake import HandshakeError, HandshakeResult, handshake
from erldistpy.node import CallProtocolError, CallTimeout, Node, NodeError
from erldistpy.tls import make_dist_tls_context
__all__ = [
"Atom",
@ -34,4 +35,5 @@ __all__ = [
"encode",
"handshake",
"lookup",
"make_dist_tls_context",
]

33
erldistpy/handshake.py
View file

@ -57,12 +57,19 @@ def handshake(
our_flags: int = DEFAULT_FLAGS,
our_creation: int | None = None,
timeout: float = 5.0,
frame_size: int = 2,
) -> HandshakeResult:
"""Drive ``sock`` through a v6 distribution handshake.
``our_name`` is the full ``name@host`` we are calling as.
``cookie`` is the shared secret with the peer.
``frame_size`` is the length-prefix size in bytes — 2 for plain TCP
dist (``inet_tcp_dist``, the default), 4 for TLS dist
(``inet_tls_dist``, which sets ``{packet, 4}`` on the SSL socket
before the handshake). Pass ``4`` whenever the socket is wrapped in
TLS; :class:`Node` selects this automatically based on ``tls_context``.
On success the socket is left open and authenticated. On any
protocol or auth failure raises :class:`HandshakeError`.
"""
@ -70,22 +77,24 @@ def handshake(
raise ValueError(f"our_name must look like 'name@host', got {our_name!r}")
if not (our_flags & DFLAG_HANDSHAKE_23):
raise ValueError("our_flags must include DFLAG_HANDSHAKE_23 (v6 handshake)")
if frame_size not in (2, 4):
raise ValueError(f"frame_size must be 2 or 4, got {frame_size}")
if our_creation is None:
our_creation = _random_creation()
sock.settimeout(timeout)
# Step 1: send_name
_send_frame(sock, build_send_name(our_name, our_flags, our_creation))
_send_frame(sock, build_send_name(our_name, our_flags, our_creation), frame_size)
# Step 2: recv_status
status_frame = _recv_frame(sock)
status_frame = _recv_frame(sock, frame_size)
status = parse_recv_status(status_frame)
if status not in ("ok", "ok_simultaneous"):
raise HandshakeError(f"peer rejected handshake: status={status!r}")
# Step 3: recv_challenge
challenge_frame = _recv_frame(sock)
challenge_frame = _recv_frame(sock, frame_size)
peer_flags, peer_challenge, peer_creation, peer_name = parse_recv_challenge(challenge_frame)
if not (peer_flags & DFLAG_HANDSHAKE_23):
raise HandshakeError("peer does not speak v6 handshake (no DFLAG_HANDSHAKE_23)")
@ -93,10 +102,10 @@ def handshake(
# Step 4: send_challenge_reply with our challenge + MD5(cookie || peer_challenge)
our_challenge = _random_challenge()
reply_digest = cookie_digest(cookie, peer_challenge)
_send_frame(sock, build_challenge_reply(our_challenge, reply_digest))
_send_frame(sock, build_challenge_reply(our_challenge, reply_digest), frame_size)
# Step 5: recv_challenge_ack — peer must answer with MD5(cookie || our_challenge)
ack_frame = _recv_frame(sock)
ack_frame = _recv_frame(sock, frame_size)
ack_digest = parse_challenge_ack(ack_frame)
expected = cookie_digest(cookie, our_challenge)
if ack_digest != expected:
@ -187,15 +196,21 @@ def cookie_digest(cookie: str, challenge: int) -> bytes:
# ---------------------------------------------------------------------------
def _send_frame(sock: socket.socket, body: bytes) -> None:
def _send_frame(sock: socket.socket, body: bytes, frame_size: int = 2) -> None:
if frame_size == 2:
if len(body) > 0xFFFF:
raise HandshakeError(f"handshake frame too large: {len(body)} bytes")
raise HandshakeError(f"handshake frame too large for 2-byte length: {len(body)} bytes")
sock.sendall(struct.pack(">H", len(body)) + body)
else:
sock.sendall(struct.pack(">I", len(body)) + body)
def _recv_frame(sock: socket.socket) -> bytes:
hdr = _recv_exact(sock, 2)
def _recv_frame(sock: socket.socket, frame_size: int = 2) -> bytes:
hdr = _recv_exact(sock, frame_size)
if frame_size == 2:
(length,) = struct.unpack(">H", hdr)
else:
(length,) = struct.unpack(">I", hdr)
if length == 0:
return b""
return _recv_exact(sock, length)

26
erldistpy/node.py
View file

@ -18,6 +18,7 @@ from __future__ import annotations
import os
import socket
import ssl
import struct
from erldistpy.channel import (
@ -59,7 +60,16 @@ class Node:
cookie: str,
peer_host: str = "localhost",
connect_timeout: float = 5.0,
tls_context: ssl.SSLContext | None = None,
tls_server_hostname: str | None = None,
):
"""Open a dist connection to the peer.
If ``tls_context`` is supplied, the TCP socket gets wrapped in
TLS before the v6 handshake runs. EPMD lookup stays in plaintext
(EPMD never negotiates TLS itself). Use
:func:`erldistpy.tls.make_dist_tls_context` to build the context.
"""
if "@" not in our_name:
raise ValueError(f"our_name must look like 'name@host', got {our_name!r}")
self.our_name = our_name
@ -67,21 +77,33 @@ class Node:
self.peer_host = peer_host
self._cookie = cookie # never logged
self._our_atom = Atom(our_name)
self.tls = tls_context is not None
info = lookup(peer_name, host=peer_host, timeout=connect_timeout)
if info is None:
raise NodeError(f"peer node {peer_name!r} not registered with EPMD on {peer_host}")
sock = socket.create_connection((peer_host, info.port), timeout=connect_timeout)
raw_sock = socket.create_connection((peer_host, info.port), timeout=connect_timeout)
try:
if tls_context is not None:
sock: socket.socket = tls_context.wrap_socket(
raw_sock,
server_hostname=tls_server_hostname,
)
else:
sock = raw_sock
# inet_tls_dist sets {packet, 4} on the SSL socket pre-nodeup;
# inet_tcp_dist uses {packet, 2}. Match the wire format.
frame_size = 4 if tls_context is not None else 2
result: HandshakeResult = handshake(
sock,
our_name=our_name,
cookie=cookie,
our_flags=DEFAULT_FLAGS,
timeout=connect_timeout,
frame_size=frame_size,
)
except Exception:
sock.close()
raw_sock.close()
raise
self._sock = sock
self._channel = Channel(sock, recv_timeout=None)

53
erldistpy/tls.py Normal file
View file

@ -0,0 +1,53 @@
"""TLS helper for distribution over ``inet_tls_dist``.
Erlang's ``inet_tls_dist`` runs the distribution protocol inside a
mutual-TLS tunnel. The wire flow is mostly unchanged after the tunnel
exists — EPMD still hands out the listener port in plaintext, then the
TCP connect to that port is immediately wrapped in TLS, then the v6
handshake runs over the wrapped socket.
One non-obvious quirk worth knowing: ``inet_tls_dist`` sets
``{packet, 4}`` on the SSL socket pre-nodeup (plain ``inet_tcp_dist``
uses ``{packet, 2}`` for the handshake then switches to ``{packet, 4}``
post-nodeup). Our :func:`erldistpy.handshake.handshake` takes a
``frame_size=`` kwarg; :class:`erldistpy.node.Node` passes ``4``
automatically whenever a ``tls_context`` is supplied.
This module exposes one helper, :func:`make_dist_tls_context`, which
builds an :class:`ssl.SSLContext` tuned for the OTP server's defaults
(verify_peer, fail_if_no_peer_cert). Pass it into :class:`Node` via the
``tls_context`` kwarg.
Secrets discipline: ``cert`` / ``key`` / ``ca`` are file *paths*. We
never read PEM contents into Python — :class:`ssl.SSLContext` loads
them through OpenSSL directly. Caller's job to keep ``key`` readable
only by the running process.
"""
from __future__ import annotations
import ssl
def make_dist_tls_context(
*,
cert: str,
key: str,
ca: str,
check_hostname: bool = False,
minimum_version: ssl.TLSVersion = ssl.TLSVersion.TLSv1_2,
) -> ssl.SSLContext:
"""Build an SSLContext for a TLS-dist client.
Defaults match the OTP server side (which verifies the client cert
and requires one). ``check_hostname=False`` because dist nodes are
identified by their cookie + cert chain, not by SNI hostname; flip
on if your CA pins per-node CNs.
"""
ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
ctx.check_hostname = check_hostname
ctx.verify_mode = ssl.CERT_REQUIRED
ctx.minimum_version = minimum_version
ctx.load_cert_chain(certfile=cert, keyfile=key)
ctx.load_verify_locations(cafile=ca)
return ctx

296
tests/test_tls.py Normal file
View file

@ -0,0 +1,296 @@
"""TLS dist tests.
Unit tests cover the SSLContext builder. Live tests generate a CA +
server/client certs with openssl, spawn ``erl -proto_dist inet_tls``,
and prove a full Node.call() round-trips through the TLS tunnel.
Two non-obvious requirements found by experimentation:
1. ``inet_tls_dist`` uses ``{packet, 4}`` on the SSL socket for the
handshake (plain TCP dist uses ``{packet, 2}``). Handshake length
prefix is 4 bytes over TLS.
2. Erlang validates the peer cert's hostname; certs need a SAN
including the dist node hostname (``hostname`` from ``uname -n``)
and/or ``localhost``.
"""
from __future__ import annotations
import os
import shutil
import socket
import ssl
import subprocess
import time
from pathlib import Path
import pytest
from erldistpy.epmd import lookup
from erldistpy.etf import Atom
from erldistpy.handshake import HandshakeError
from erldistpy.node import Node
from erldistpy.tls import make_dist_tls_context
# --------------------------------------------------------------------------
# Pure
# --------------------------------------------------------------------------
def test_make_dist_tls_context_missing_files(tmp_path):
with pytest.raises((FileNotFoundError, ssl.SSLError)):
make_dist_tls_context(
cert=str(tmp_path / "missing.pem"),
key=str(tmp_path / "missing.key"),
ca=str(tmp_path / "missing-ca.pem"),
)
# --------------------------------------------------------------------------
# Live — generate certs, spawn erl with inet_tls_dist, call() over TLS
# --------------------------------------------------------------------------
READY_FLAG = "/tmp/erldistpy_test_tls_ready"
SNAME = "erldistpy_tls"
COOKIE = "ERLDISTPY_TLS_COOKIE"
ERL_GEN_BOOT = (
"Handle = fun "
" ({ping, X}) -> {pong, X}; "
" (Other) -> {error, Other} "
"end, "
"Loop = fun(F) -> "
" receive "
" {'$gen_call', {From, Ref}, Request} -> "
" From ! {Ref, Handle(Request)}, F(F) "
" end "
"end, "
"Pid = spawn(fun() -> Loop(Loop) end), "
"register(gen_target, Pid), "
f'file:write_file("{READY_FLAG}", "1"), '
"timer:sleep(infinity)."
)
def _epmd_running() -> bool:
try:
with socket.create_connection(("localhost", 4369), timeout=0.5):
return True
except OSError:
return False
def _gen_certs(workdir: Path) -> dict[str, str]:
"""Generate a CA + server + client cert with SAN matching the hostname.
Erlang's TLS-dist validates the peer cert's hostname during
handshake; without a matching SAN it rejects the cert chain
(``{bad_cert,hostname_check_failed}``).
"""
hostname = socket.gethostname()
# CA with proper CA:TRUE extension
ca_cnf = workdir / "ca.cnf"
ca_cnf.write_text(
"[req]\n"
"distinguished_name = dn\n"
"x509_extensions = v3_ca\n"
"prompt = no\n"
"[dn]\n"
"CN = erldistpy-test-ca\n"
"[v3_ca]\n"
"basicConstraints = critical,CA:TRUE\n"
"keyUsage = critical,keyCertSign,cRLSign\n"
"subjectKeyIdentifier = hash\n"
)
subprocess.run(
[
"openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes",
"-keyout", str(workdir / "ca.key"),
"-out", str(workdir / "ca.pem"),
"-days", "1",
"-config", str(ca_cnf), "-extensions", "v3_ca",
],
check=True, capture_output=True,
)
os.chmod(workdir / "ca.key", 0o600)
# Leaf cert extensions including SAN
leaf_cnf = workdir / "leaf.cnf"
leaf_cnf.write_text(
"[v3]\n"
"basicConstraints = CA:FALSE\n"
"keyUsage = digitalSignature,keyEncipherment\n"
"extendedKeyUsage = serverAuth,clientAuth\n"
"subjectAltName = @alt\n"
"[alt]\n"
f"DNS.1 = {hostname}\n"
"DNS.2 = localhost\n"
)
def _signed_cert(name: str) -> None:
subprocess.run(
[
"openssl", "req", "-newkey", "rsa:2048", "-nodes",
"-keyout", str(workdir / f"{name}.key"),
"-out", str(workdir / f"{name}.csr"),
"-subj", f"/CN={hostname}",
],
check=True, capture_output=True,
)
subprocess.run(
[
"openssl", "x509", "-req",
"-in", str(workdir / f"{name}.csr"),
"-CA", str(workdir / "ca.pem"),
"-CAkey", str(workdir / "ca.key"),
"-CAcreateserial",
"-out", str(workdir / f"{name}.pem"),
"-days", "1",
"-extfile", str(leaf_cnf), "-extensions", "v3",
],
check=True, capture_output=True,
)
os.chmod(workdir / f"{name}.key", 0o600)
_signed_cert("server")
_signed_cert("client")
return {
"ca": str(workdir / "ca.pem"),
"server_cert": str(workdir / "server.pem"),
"server_key": str(workdir / "server.key"),
"client_cert": str(workdir / "client.pem"),
"client_key": str(workdir / "client.key"),
}
def _write_ssl_config(workdir: Path, certs: dict[str, str]) -> str:
"""Write the ssl_dist_opts.config file Erlang reads."""
cfg = workdir / "ssl_dist.config"
body = (
"[{server, "
f'[{{certfile, "{certs["server_cert"]}"}}, '
f'{{keyfile, "{certs["server_key"]}"}}, '
f'{{cacertfile, "{certs["ca"]}"}}, '
"{verify, verify_peer}, "
"{fail_if_no_peer_cert, true}]}, "
"{client, "
f'[{{certfile, "{certs["server_cert"]}"}}, '
f'{{keyfile, "{certs["server_key"]}"}}, '
f'{{cacertfile, "{certs["ca"]}"}}, '
"{verify, verify_peer}]}]."
)
cfg.write_text(body)
return str(cfg)
@pytest.fixture(scope="module")
def tls_peer(tmp_path_factory):
if not shutil.which("erl"):
pytest.skip("erl not installed")
if not shutil.which("openssl"):
pytest.skip("openssl not installed")
if not _epmd_running():
pytest.skip("EPMD not running")
if os.path.exists(READY_FLAG):
os.remove(READY_FLAG)
workdir = tmp_path_factory.mktemp("tls")
certs = _gen_certs(workdir)
ssl_config = _write_ssl_config(workdir, certs)
proc = subprocess.Popen(
[
"erl",
"-sname", SNAME,
"-setcookie", COOKIE,
"-proto_dist", "inet_tls",
"-ssl_dist_optfile", ssl_config,
"-noshell",
"-eval", ERL_GEN_BOOT,
],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
deadline = time.monotonic() + 8.0
ready = False
while time.monotonic() < deadline:
time.sleep(0.15)
if lookup(SNAME, timeout=0.5) is None:
continue
if os.path.exists(READY_FLAG):
ready = True
break
if not ready:
proc.terminate()
proc.wait(timeout=3)
pytest.skip("erl TLS-dist node did not become ready")
yield SNAME, COOKIE, certs
proc.terminate()
try:
proc.wait(timeout=3)
except subprocess.TimeoutExpired:
proc.kill()
if os.path.exists(READY_FLAG):
os.remove(READY_FLAG)
def _client_name() -> str:
return f"erldistpy_tls_client@{socket.gethostname()}"
def test_call_over_tls(tls_peer):
sname, cookie, certs = tls_peer
ctx = make_dist_tls_context(
cert=certs["client_cert"],
key=certs["client_key"],
ca=certs["ca"],
)
with Node(
our_name=_client_name(),
peer_name=sname,
cookie=cookie,
tls_context=ctx,
) as node:
assert node.tls is True
reply = node.call("gen_target", (Atom("ping"), 99))
assert reply == (Atom("pong"), 99)
def test_plaintext_connection_to_tls_peer_fails(tls_peer):
"""Connecting without TLS to a node that requires TLS must fail —
that's the security boundary the tunnel exists to enforce."""
sname, cookie, _ = tls_peer
with pytest.raises((HandshakeError, OSError, ssl.SSLError, ConnectionError)):
Node(
our_name=_client_name(),
peer_name=sname,
cookie=cookie,
connect_timeout=2.0,
)
def test_wrong_ca_fails(tls_peer, tmp_path):
"""A client with a cert signed by a CA the server doesn't trust
must fail to handshake."""
sname, cookie, certs = tls_peer
# Generate a totally separate CA + client cert in a fresh dir
other = tmp_path / "other"
other.mkdir()
other_certs = _gen_certs(other)
bad_ctx = make_dist_tls_context(
cert=other_certs["client_cert"],
key=other_certs["client_key"],
ca=certs["ca"], # we trust the right CA, but our cert isn't signed by it
)
with pytest.raises((ssl.SSLError, OSError, ConnectionError, HandshakeError)):
Node(
our_name=_client_name(),
peer_name=sname,
cookie=cookie,
tls_context=bad_ctx,
connect_timeout=3.0,
)