From 8e6b3e58a35e327fb681e9fc580a2481b8cfe6b6 Mon Sep 17 00:00:00 2001
From: "russell@unturf.com" <russell@unturf.com>
Date: Thu, 22 Jan 2026 17:07:52 -0500
Subject: [PATCH] fix: add retry logic with exponential backoff for HTTP 429
 errors in CI tests

When fanning out SDK tests in CI, concurrent executions can hit the API's
concurrency limit (HTTP 429). Instead of immediately failing, the test
runner now retries up to 42 times with exponential backoff (2s, 4s, 8s...
capped at 30s) until the execution completes.
---
 scripts/test-sdk.sh | 52 +++++++++++++++++++++++++++++++++------------
 1 file changed, 38 insertions(+), 14 deletions(-)

diff --git a/scripts/test-sdk.sh b/scripts/test-sdk.sh
index b8402c0..df8ab16 100755
--- a/scripts/test-sdk.sh
+++ b/scripts/test-sdk.sh
@@ -161,31 +161,55 @@ TOTAL_TESTS=0
 FAILURES=0
 
 # Helper to run test and record result
+# Retries on HTTP 429 (rate limiting) with exponential backoff
 run_test() {
     local test_name="$1"
     local test_cmd="$2"
     local success_pattern="$3"
     local output_file="$RESULTS_DIR/${test_name}.txt"
+    local max_retries=42
+    local retry_delay=2
+    local attempt=0
 
     echo -n "Test: $test_name... "
     TOTAL_TESTS=$((TOTAL_TESTS + 1))
 
-    if eval "$test_cmd" > "$output_file" 2>&1; then
-        if [ -n "$success_pattern" ]; then
-            if grep -qiE "$success_pattern" "$output_file"; then
-                TEST_RESULTS["$test_name"]="pass"
-                echo "PASS"
-                return 0
-            fi
-        else
-            # No pattern required, just check non-empty output
-            if [ -s "$output_file" ]; then
-                TEST_RESULTS["$test_name"]="pass"
-                echo "PASS"
-                return 0
+    while [ $attempt -lt $max_retries ]; do
+        if eval "$test_cmd" > "$output_file" 2>&1; then
+            if [ -n "$success_pattern" ]; then
+                if grep -qiE "$success_pattern" "$output_file"; then
+                    TEST_RESULTS["$test_name"]="pass"
+                    echo "PASS"
+                    return 0
+                fi
+            else
+                # No pattern required, just check non-empty output
+                if [ -s "$output_file" ]; then
+                    TEST_RESULTS["$test_name"]="pass"
+                    echo "PASS"
+                    return 0
+                fi
             fi
         fi
-    fi
+
+        # Check for HTTP 429 (rate limiting) - retry with backoff
+        if grep -qiE "HTTP 429|concurrency_limit|too many.*concurrent|rate.limit" "$output_file" 2>/dev/null; then
+            attempt=$((attempt + 1))
+            if [ $attempt -lt $max_retries ]; then
+                echo -n "(429 retry $attempt/$max_retries)... "
+                sleep $retry_delay
+                # Exponential backoff, cap at 30 seconds
+                retry_delay=$((retry_delay * 2))
+                if [ $retry_delay -gt 30 ]; then
+                    retry_delay=30
+                fi
+                continue
+            fi
+        fi
+
+        # Not a 429 error, break out of retry loop
+        break
+    done
 
     # Check for acceptable failures (transient API/sandbox issues)
     if grep -qiE "HTTP 5[0-9][0-9]|server error|internal error" "$output_file" 2>/dev/null; then