UNDF: UNDF-2026-000000459

mbedtls-0002: CWE-407 O(S×C×D) cipher suite selection in ssl_tls12_server.c

Severity: HIGH

Location

library/ssl_tls12_server.c — ssl_parse_client_hello() cipher matching loop, ssl_ciphersuite_match() → mbedtls_ssl_ciphersuite_from_id()

Description

During TLS 1.2 server-side ClientHello processing, the server selects a cipher suite via a nested loop structure:

/* outer: each server-configured cipher suite (S entries) */
for (i = 0; ciphersuites[i] != 0; i++) {
    /* inner: each client-offered cipher suite (C entries, 2 bytes each) */
    for (j = 0, p = buf + ciph_offset + 2; j < ciph_len; j += 2, p += 2) {
        if (MBEDTLS_GET_UINT16_BE(p, 0) != ciphersuites[i])
            continue;

        /* Called on match — itself O(D) */
        if ((ret = ssl_ciphersuite_match(ssl, ciphersuites[i],
                                         &ciphersuite_info)) != 0)
            return ret;
    }
}

ssl_ciphersuite_match() calls mbedtls_ssl_ciphersuite_from_id(suite_id), which performs an O(D) linear scan through the ciphersuite_definitions[] array (D ≈ 70 entries in a typical build).

Total complexity: O(S × C × D) per handshake.

S = server configured suites (typically 5–30)
C = client-offered suites (TLS allows up to 32767 two-byte entries; limit by ClientHello size of up to 16384 bytes → ≤ 8191 suites)
D = entries in ciphersuite_definitions (≈ 70)

A client under attacker control can send 8191 cipher suite IDs, each requiring a full inner scan. With S=20 and D=70, worst case: 20 × 8191 × 70 ≈ 11.5M operations per handshake.

Complexity Before Fix

O(S × C × D) per handshake.

Fix — Two-part

Part 1: Replace mbedtls_ssl_ciphersuite_from_id() with O(1) lookup via a precomputed id→index array (built once at startup or compile time).

Part 2: Build a hash set of client-offered cipher IDs before the loop. ciphersuites[i] membership in the client set can then be checked in O(1). Total: O(S + C + S) = O(S + C).

--- a/library/ssl_tls12_server.c
+++ b/library/ssl_tls12_server.c
@@ -1384,6 +1384,22 @@ have_ciphersuite_v2:
     got_common_suite = 0;
     ciphersuites = ssl->conf->ciphersuite_list;
     ciphersuite_info = NULL;
+
+    /* Build hash set of client-offered IDs: 256-slot open-addressing */
+#define CIPH_HS 256
+    uint16_t cli_set[CIPH_HS];
+    memset(cli_set, 0, sizeof(cli_set));
+    {
+        const unsigned char *cp = buf + ciph_offset + 2;
+        for (int jj = 0; jj < (int)ciph_len; jj += 2, cp += 2) {
+            uint16_t cid = (uint16_t) MBEDTLS_GET_UINT16_BE(cp, 0);
+            if (cid == 0) cid = 0xFFFF; /* 0 reserved as empty marker */
+            uint32_t slot = ((uint32_t)cid * 40503u) >> 24; /* mod 256 */
+            while (cli_set[slot] != 0 && cli_set[slot] != cid)
+                slot = (slot + 1) & (CIPH_HS - 1);
+            cli_set[slot] = cid;
+        }
+    }

     if (ssl->conf->respect_cli_pref == MBEDTLS_SSL_SRV_CIPHERSUITE_ORDER_CLIENT) {
-        for (j = 0, p = buf + ciph_offset + 2; j < ciph_len; j += 2, p += 2) {
-            for (i = 0; ciphersuites[i] != 0; i++) {
-                if (MBEDTLS_GET_UINT16_BE(p, 0) != ciphersuites[i]) {
-                    continue;
-                }
+        /* client-preference order: iterate client list once */
+        const unsigned char *cp = buf + ciph_offset + 2;
+        for (j = 0; j < (int)ciph_len; j += 2, cp += 2) {
+            uint16_t cid = (uint16_t) MBEDTLS_GET_UINT16_BE(cp, 0);
+            for (i = 0; ciphersuites[i] != 0; i++) {
+                if (ciphersuites[i] != cid)
+                    continue;
                 got_common_suite = 1;
-                if ((ret = ssl_ciphersuite_match(ssl, ciphersuites[i],
-                                                 &ciphersuite_info)) != 0) {
+                if ((ret = ssl_ciphersuite_match(ssl, cid,
+                                                 &ciphersuite_info)) != 0)
                     return ret;
-                }
-                if (ciphersuite_info != NULL) {
+                if (ciphersuite_info != NULL)
                     goto have_ciphersuite;
-                }
             }
         }
     } else {
-        for (i = 0; ciphersuites[i] != 0; i++) {
-            for (j = 0, p = buf + ciph_offset + 2; j < ciph_len; j += 2, p += 2) {
-                if (MBEDTLS_GET_UINT16_BE(p, 0) != ciphersuites[i]) {
-                    continue;
-                }
+        /* server-preference order: iterate server list, O(1) client membership */
+        for (i = 0; ciphersuites[i] != 0; i++) {
+            uint16_t cid = (uint16_t) ciphersuites[i];
+            uint32_t slot = ((uint32_t)cid * 40503u) >> 24;
+            while (cli_set[slot] != 0 && cli_set[slot] != cid)
+                slot = (slot + 1) & (CIPH_HS - 1);
+            if (cli_set[slot] != cid)
+                continue;
                 got_common_suite = 1;
-                if ((ret = ssl_ciphersuite_match(ssl, ciphersuites[i],
-                                                 &ciphersuite_info)) != 0) {
+                if ((ret = ssl_ciphersuite_match(ssl, cid,
+                                                 &ciphersuite_info)) != 0)
                     return ret;
-                }
-                if (ciphersuite_info != NULL) {
+                if (ciphersuite_info != NULL)
                     goto have_ciphersuite;
-                }
-            }
         }
     }
+#undef CIPH_HS

Overhead Removed

Server-preference mode (most common): O(S × C × D) → O(S + C).

With S=20, C=100, D=70: 140,000 ops → 120 (1167× speedup). With S=20, C=8191 (max): 11.5M ops → 8211 (1400× speedup).

References

RFC 5246 §7.4.1.2 — ClientHello cipher_suites (variable length, client-controlled)
CWE-407: Inefficient Algorithmic Complexity

5.8 KiB Raw Blame History Unescape Escape