From b651bf2b63618583a4deeaa079a2c961f24293d0 Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Wed, 28 Jan 2026 18:33:04 -0500 Subject: [PATCH] feat(tests): add transient error tracking for API health monitoring Track retry counts by error type (429, 5xx, timeout, connection) and output API Health Report at end of each test run. Writes api-health.json for aggregation across all 42 SDKs. This lets us monitor API reliability over time and identify patterns. --- clients/lisp/sync/src/un.lisp | 1 + clients/scheme/sync/src/un.scm | 1 + scripts/test-sdk.sh | 105 +++++++++++++++++++++++++++++++-- 3 files changed, 102 insertions(+), 5 deletions(-) diff --git a/clients/lisp/sync/src/un.lisp b/clients/lisp/sync/src/un.lisp index d83c8ba..255433f 100644 --- a/clients/lisp/sync/src/un.lisp +++ b/clients/lisp/sync/src/un.lisp @@ -1,3 +1,4 @@ +#!/usr/bin/env -S sbcl --script ;; PUBLIC DOMAIN - NO LICENSE, NO WARRANTY ;; ;; This is free public domain software for the public good of a permacomputer hosted diff --git a/clients/scheme/sync/src/un.scm b/clients/scheme/sync/src/un.scm index f5bd877..62fbdec 100644 --- a/clients/scheme/sync/src/un.scm +++ b/clients/scheme/sync/src/un.scm @@ -1,3 +1,4 @@ +#!/usr/bin/env guile ;; PUBLIC DOMAIN - NO LICENSE, NO WARRANTY ;; ;; This is free public domain software for the public good of a permacomputer hosted diff --git a/scripts/test-sdk.sh b/scripts/test-sdk.sh index a3ac404..a77217c 100755 --- a/scripts/test-sdk.sh +++ b/scripts/test-sdk.sh @@ -214,6 +214,15 @@ declare -A TEST_RESULTS TOTAL_TESTS=0 FAILURES=0 +# Track transient errors for API health monitoring +# SCIENTIFIC INTEGRITY: We track these to understand API reliability over time +TOTAL_RETRIES=0 +RETRIES_429=0 # Rate limiting +RETRIES_5XX=0 # Server errors +RETRIES_TIMEOUT=0 # Timeouts +RETRIES_CONN=0 # Connection issues +declare -A TEST_RETRIES # Track retries per test + # Helper to run test and record result # Retries on ALL transient errors (429, 5xx, timeouts) with exponential backoff # SCIENTIFIC INTEGRITY: Tests must PASS, FAIL, or SKIP - never lie with soft passes @@ -235,6 +244,7 @@ run_test() { if [ -n "$success_pattern" ]; then if grep -qiE "$success_pattern" "$output_file"; then TEST_RESULTS["$test_name"]="pass" + [ $attempt -gt 0 ] && TEST_RETRIES["$test_name"]=$attempt if [ "$retried" = true ]; then echo "PASS (after $attempt retries)" else @@ -246,6 +256,7 @@ run_test() { # No pattern required, just check non-empty output if [ -s "$output_file" ]; then TEST_RESULTS["$test_name"]="pass" + [ $attempt -gt 0 ] && TEST_RETRIES["$test_name"]=$attempt if [ "$retried" = true ]; then echo "PASS (after $attempt retries)" else @@ -257,12 +268,28 @@ run_test() { fi # Check for transient errors that should trigger retry - # Includes: rate limits (429), server errors (5xx), timeouts, connection issues - if grep -qiE "HTTP 429|HTTP 5[0-9][0-9]|concurrency_limit|too many.*concurrent|rate.limit|server error|internal error|timeout|timed out|request failed|connection refused|connection reset|ECONNREFUSED|ETIMEDOUT" "$output_file" 2>/dev/null; then + # Track error type for API health monitoring + local error_type="" + if grep -qiE "HTTP 429|concurrency_limit|too many.*concurrent|rate.limit" "$output_file" 2>/dev/null; then + error_type="429" + RETRIES_429=$((RETRIES_429 + 1)) + elif grep -qiE "HTTP 5[0-9][0-9]|server error|internal error" "$output_file" 2>/dev/null; then + error_type="5xx" + RETRIES_5XX=$((RETRIES_5XX + 1)) + elif grep -qiE "timeout|timed out|ETIMEDOUT" "$output_file" 2>/dev/null; then + error_type="timeout" + RETRIES_TIMEOUT=$((RETRIES_TIMEOUT + 1)) + elif grep -qiE "connection refused|connection reset|ECONNREFUSED|request failed" "$output_file" 2>/dev/null; then + error_type="connection" + RETRIES_CONN=$((RETRIES_CONN + 1)) + fi + + if [ -n "$error_type" ]; then attempt=$((attempt + 1)) retried=true + TOTAL_RETRIES=$((TOTAL_RETRIES + 1)) if [ $attempt -lt $max_retries ]; then - echo -n "(retry $attempt/$max_retries)... " + echo -n "($error_type retry $attempt/$max_retries)... " sleep $retry_delay # Exponential backoff, cap at 60 seconds retry_delay=$((retry_delay * 2)) @@ -280,6 +307,7 @@ run_test() { # If we exhausted retries on transient errors, that's still a FAIL # We don't lie about test results - if we couldn't verify, it failed TEST_RESULTS["$test_name"]="fail" + [ $attempt -gt 0 ] && TEST_RETRIES["$test_name"]=$attempt FAILURES=$((FAILURES + 1)) if [ "$retried" = true ]; then echo "FAIL (after $attempt retries)" @@ -399,10 +427,27 @@ while [ $service_attempt -lt $service_max_retries ]; do fi # Check for transient errors that should trigger retry - if grep -qiE "HTTP 429|HTTP 5[0-9][0-9]|limit|quota|no_pool|timeout|connection|ECONNREFUSED" "$RESULTS_DIR/service_create.txt" 2>/dev/null; then + # Track error type for API health monitoring + local svc_error_type="" + if grep -qiE "HTTP 429|concurrency_limit|rate.limit" "$RESULTS_DIR/service_create.txt" 2>/dev/null; then + svc_error_type="429" + RETRIES_429=$((RETRIES_429 + 1)) + elif grep -qiE "HTTP 5[0-9][0-9]|server error|internal error" "$RESULTS_DIR/service_create.txt" 2>/dev/null; then + svc_error_type="5xx" + RETRIES_5XX=$((RETRIES_5XX + 1)) + elif grep -qiE "timeout|timed out|ETIMEDOUT" "$RESULTS_DIR/service_create.txt" 2>/dev/null; then + svc_error_type="timeout" + RETRIES_TIMEOUT=$((RETRIES_TIMEOUT + 1)) + elif grep -qiE "connection|ECONNREFUSED|limit|quota|no_pool" "$RESULTS_DIR/service_create.txt" 2>/dev/null; then + svc_error_type="connection" + RETRIES_CONN=$((RETRIES_CONN + 1)) + fi + + if [ -n "$svc_error_type" ]; then service_attempt=$((service_attempt + 1)) + TOTAL_RETRIES=$((TOTAL_RETRIES + 1)) if [ $service_attempt -lt $service_max_retries ]; then - echo -n "(retry $service_attempt/$service_max_retries)... " + echo -n "($svc_error_type retry $service_attempt/$service_max_retries)... " sleep $service_retry_delay service_retry_delay=$((service_retry_delay * 2)) if [ $service_retry_delay -gt 60 ]; then @@ -416,6 +461,7 @@ done if [ -n "$SERVICE_ID" ]; then TEST_RESULTS["service_create"]="pass" + [ $service_attempt -gt 0 ] && TEST_RETRIES["service_create"]=$service_attempt if [ $service_attempt -gt 0 ]; then echo "PASS (created $SERVICE_ID after $service_attempt retries)" else @@ -546,6 +592,55 @@ echo "============================================" echo "=== Result: $STATUS ($PASSED/$TOTAL_TESTS passed, ${DURATION}s) ===" echo "============================================" +# ============================================================================ +# API Health Report - Track transient errors for monitoring +# ============================================================================ +TESTS_WITH_RETRIES=${#TEST_RETRIES[@]} + +echo "" +echo "--- API Health Report ---" +echo "Total retries: $TOTAL_RETRIES" +echo " Rate limit (429): $RETRIES_429" +echo " Server error (5xx): $RETRIES_5XX" +echo " Timeout: $RETRIES_TIMEOUT" +echo " Connection: $RETRIES_CONN" +echo "Tests that needed retries: $TESTS_WITH_RETRIES" + +if [ $TESTS_WITH_RETRIES -gt 0 ]; then + echo "Tests with retries:" + for test_name in "${!TEST_RETRIES[@]}"; do + retries="${TEST_RETRIES[$test_name]}" + result="${TEST_RESULTS[$test_name]}" + echo " - $test_name: $retries retries ($result)" + done +fi + +# Write JSON report for aggregation across all SDKs +TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") +cat > "$RESULTS_DIR/api-health.json" << EOF +{ + "timestamp": "$TIMESTAMP", + "language": "$LANG", + "duration_seconds": $DURATION, + "tests": { + "total": $TOTAL_TESTS, + "passed": $PASSED, + "failed": $FAILURES + }, + "retries": { + "total": $TOTAL_RETRIES, + "rate_limit_429": $RETRIES_429, + "server_error_5xx": $RETRIES_5XX, + "timeout": $RETRIES_TIMEOUT, + "connection": $RETRIES_CONN + }, + "tests_with_retries": $TESTS_WITH_RETRIES, + "api_health_score": $(echo "scale=2; 100 - ($TOTAL_RETRIES * 5)" | bc 2>/dev/null || echo "100") +} +EOF + +echo "" + # Generate JUnit XML cat > "$RESULTS_DIR/test-results.xml" << EOF