From ab70e8b50851e488ac9ac4174f8ec4c6d75608db Mon Sep 17 00:00:00 2001 From: "russell@unturf.com" Date: Wed, 28 Jan 2026 18:26:23 -0500 Subject: [PATCH] fix(tests): remove soft passes - tests must never lie SCIENTIFIC INTEGRITY: Tests now retry transient errors (429, 5xx, timeouts) with exponential backoff instead of falsely passing. Prior to this fix, ~35% of 'passing' tests were masked failures. Now tests will PASS (verified), FAIL (couldn't verify), or SKIP. --- CLAUDE.md | 65 +++++++++++++- clients/awk/sync/src/un.awk | 1 + clients/groovy/sync/src/un.groovy | 1 + clients/javascript/sync/src/un.js | 1 + clients/julia/sync/src/un.jl | 1 + clients/php/sync/src/un.php | 1 + clients/python/sync/src/un.py | 1 + clients/r/sync/src/un.r | 1 + clients/raku/sync/src/un.raku | 1 + clients/ruby/sync/src/un.rb | 1 + scripts/test-sdk.sh | 139 +++++++++++++++++------------- 11 files changed, 149 insertions(+), 64 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index e1ab865..7e67c91 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,5 +1,64 @@ # Claude AI Instructions for un-inception +## ⚠️ CRITICAL: SCIENTIFIC INTEGRITY - TESTS MUST NEVER LIE + +**Science is the foundation of this project.** Tests exist to tell us the truth about our code. A test that lies is worse than no test at all. + +### The Cardinal Rule + +**If a test cannot verify its assertion, it MUST FAIL or RETRY - never silently pass.** + +```bash +# ❌ FORBIDDEN - Lying about results +if api_returned_500_error; then + echo "PASS (API issue)" # THIS IS A LIE - we didn't verify anything +fi + +# ✅ CORRECT - Retry transient failures +if api_returned_500_error; then + sleep $backoff + retry # Keep trying until we get a real answer +fi + +# ✅ CORRECT - Fail if we can't verify +if api_returned_500_error && max_retries_exceeded; then + echo "FAIL (API unavailable after $max_retries attempts)" +fi +``` + +### Why This Matters + +On 2026-01-28, we discovered our "100% pass rate" was a lie: +- **780 tests "passed"** across 42 languages +- **270 were soft passes** (35%) - masked failures +- The test matrix was telling us everything worked when it didn't + +**Soft passes are scientific fraud.** They: +- Hide real bugs in SDKs +- Give false confidence before releases +- Make debugging harder (you don't know what's actually broken) +- Waste time investigating "new" failures that were always there + +### Test Script Requirements + +1. **Retry ALL transient errors** - HTTP 429, 500, 502, 503, 504, timeouts +2. **Use exponential backoff** - Start at 2s, cap at 60s +3. **Max retries = 10** - Then FAIL, don't fake pass +4. **No soft passes** - If the expected output isn't there, it's a FAIL +5. **Track retry stats** - So we can see API health over time + +### Acceptable Test Outcomes + +| Outcome | When to Use | +|---------|-------------| +| `PASS` | Expected output verified | +| `FAIL` | Expected output not found after retries | +| `SKIP` | Test not applicable (e.g., no QR file for this language) | + +**Never**: `PASS (API issue)`, `PASS (timeout)`, `PASS (sandbox state)` + +--- + ## ⚠️ CRITICAL: NEVER USE RAW LXC COMMANDS **ALWAYS use `un` CLI commands. NEVER use raw `lxc` commands on production.** @@ -157,9 +216,11 @@ done ### CI Test Status -**638 tests passing across 42 languages with 100% pass rate.** +The CI runs the full inception test matrix on every tag release and smart change detection on regular pushes. -The CI runs the full inception test matrix on every tag release and smart change detection on regular pushes. See [docs/INCEPTION-TESTING.md](docs/INCEPTION-TESTING.md) for details. +**Test results must be truthful.** Prior to 2026-01-28, tests used "soft passes" that masked failures - this has been fixed. Tests now retry transient errors and fail honestly if they can't verify. + +See [docs/INCEPTION-TESTING.md](docs/INCEPTION-TESTING.md) for details. ## Directory Structure diff --git a/clients/awk/sync/src/un.awk b/clients/awk/sync/src/un.awk index 8817899..b67b75a 100644 --- a/clients/awk/sync/src/un.awk +++ b/clients/awk/sync/src/un.awk @@ -1,3 +1,4 @@ +#!/usr/bin/env -S awk -f # PUBLIC DOMAIN - NO LICENSE, NO WARRANTY # # This is free public domain software for the public good of a permacomputer hosted diff --git a/clients/groovy/sync/src/un.groovy b/clients/groovy/sync/src/un.groovy index 74c25b8..4259f48 100644 --- a/clients/groovy/sync/src/un.groovy +++ b/clients/groovy/sync/src/un.groovy @@ -1,3 +1,4 @@ +#!/usr/bin/env groovy // PUBLIC DOMAIN - NO LICENSE, NO WARRANTY // // This is free public domain software for the public good of a permacomputer hosted diff --git a/clients/javascript/sync/src/un.js b/clients/javascript/sync/src/un.js index 70390a4..24d2ce4 100644 --- a/clients/javascript/sync/src/un.js +++ b/clients/javascript/sync/src/un.js @@ -1,3 +1,4 @@ +#!/usr/bin/env node /** * PUBLIC DOMAIN - NO LICENSE, NO WARRANTY * diff --git a/clients/julia/sync/src/un.jl b/clients/julia/sync/src/un.jl index 56080f3..005132c 100755 --- a/clients/julia/sync/src/un.jl +++ b/clients/julia/sync/src/un.jl @@ -1,3 +1,4 @@ +#!/usr/bin/env julia # PUBLIC DOMAIN - NO LICENSE, NO WARRANTY # # This is free public domain software for the public good of a permacomputer hosted diff --git a/clients/php/sync/src/un.php b/clients/php/sync/src/un.php index ab08343..679ae62 100644 --- a/clients/php/sync/src/un.php +++ b/clients/php/sync/src/un.php @@ -1,3 +1,4 @@ +#!/usr/bin/env php /dev/null; then + # Check for transient errors that should trigger retry + # Includes: rate limits (429), server errors (5xx), timeouts, connection issues + if grep -qiE "HTTP 429|HTTP 5[0-9][0-9]|concurrency_limit|too many.*concurrent|rate.limit|server error|internal error|timeout|timed out|request failed|connection refused|connection reset|ECONNREFUSED|ETIMEDOUT" "$output_file" 2>/dev/null; then attempt=$((attempt + 1)) + retried=true if [ $attempt -lt $max_retries ]; then - echo -n "(429 retry $attempt/$max_retries)... " + echo -n "(retry $attempt/$max_retries)... " sleep $retry_delay - # Exponential backoff, cap at 30 seconds + # Exponential backoff, cap at 60 seconds retry_delay=$((retry_delay * 2)) - if [ $retry_delay -gt 30 ]; then - retry_delay=30 + if [ $retry_delay -gt 60 ]; then + retry_delay=60 fi continue fi fi - # Not a 429 error, break out of retry loop + # Not a transient error, break out of retry loop break done - # Check for acceptable failures (transient API/sandbox issues) - if grep -qiE "HTTP 5[0-9][0-9]|server error|internal error" "$output_file" 2>/dev/null; then - TEST_RESULTS["$test_name"]="pass" - echo "PASS (API issue)" - return 0 - fi - - if grep -qiE "timeout|timed out|request failed" "$output_file" 2>/dev/null; then - TEST_RESULTS["$test_name"]="pass" - echo "PASS (timeout)" - return 0 - fi - - if grep -qiE "permission denied|not running|unfreeze|frozen" "$output_file" 2>/dev/null; then - TEST_RESULTS["$test_name"]="pass" - echo "PASS (sandbox state)" - return 0 - fi - - if grep -qiE "usage|help|not.*found|no.*file" "$output_file" 2>/dev/null; then - TEST_RESULTS["$test_name"]="pass" - echo "PASS (expected output)" - return 0 - fi - - # If output is empty but command didn't error, the runtime executed successfully - # This handles SDKs that exit cleanly without printing anything - if [ ! -s "$output_file" ]; then - TEST_RESULTS["$test_name"]="pass" - echo "PASS (clean exit)" - return 0 - fi - + # If we exhausted retries on transient errors, that's still a FAIL + # We don't lie about test results - if we couldn't verify, it failed TEST_RESULTS["$test_name"]="fail" FAILURES=$((FAILURES + 1)) - echo "FAIL" - head -3 "$output_file" 2>/dev/null || echo "(no output)" + if [ "$retried" = true ]; then + echo "FAIL (after $attempt retries)" + else + echo "FAIL" + fi + head -5 "$output_file" 2>/dev/null || echo "(no output)" return 0 # Always return success to continue test execution } @@ -393,17 +378,49 @@ run_test "service_list" \ "build/un service --list" \ "unsb-service|no.*service|\[\]|services" -# Test 4.2: Create a service (note: syntax is --name not --create) +# Test 4.2: Create a service with retry logic (note: syntax is --name not --create) +# SCIENTIFIC INTEGRITY: Retry on transient errors, fail if we can't create SERVICE_NAME="test-$LANG-$$" -SERVICE_OUTPUT=$(build/un service --name "$SERVICE_NAME" --bootstrap "echo service-started" 2>&1 || true) -echo "$SERVICE_OUTPUT" > "$RESULTS_DIR/service_create.txt" -SERVICE_ID=$(echo "$SERVICE_OUTPUT" | grep -oE "unsb-service-[a-z0-9-]+" | head -1 || true) +SERVICE_ID="" +service_attempt=0 +service_max_retries=10 +service_retry_delay=2 + +echo -n "Test: service_create... " +TOTAL_TESTS=$((TOTAL_TESTS + 1)) + +while [ $service_attempt -lt $service_max_retries ]; do + SERVICE_OUTPUT=$(build/un service --name "$SERVICE_NAME" --bootstrap "echo service-started" 2>&1 || true) + echo "$SERVICE_OUTPUT" > "$RESULTS_DIR/service_create.txt" + SERVICE_ID=$(echo "$SERVICE_OUTPUT" | grep -oE "unsb-service-[a-z0-9-]+" | head -1 || true) + + if [ -n "$SERVICE_ID" ]; then + break + fi + + # Check for transient errors that should trigger retry + if grep -qiE "HTTP 429|HTTP 5[0-9][0-9]|limit|quota|no_pool|timeout|connection|ECONNREFUSED" "$RESULTS_DIR/service_create.txt" 2>/dev/null; then + service_attempt=$((service_attempt + 1)) + if [ $service_attempt -lt $service_max_retries ]; then + echo -n "(retry $service_attempt/$service_max_retries)... " + sleep $service_retry_delay + service_retry_delay=$((service_retry_delay * 2)) + if [ $service_retry_delay -gt 60 ]; then + service_retry_delay=60 + fi + continue + fi + fi + break +done if [ -n "$SERVICE_ID" ]; then - echo -n "Test: service_create... " - TOTAL_TESTS=$((TOTAL_TESTS + 1)) TEST_RESULTS["service_create"]="pass" - echo "PASS (created $SERVICE_ID)" + if [ $service_attempt -gt 0 ]; then + echo "PASS (created $SERVICE_ID after $service_attempt retries)" + else + echo "PASS (created $SERVICE_ID)" + fi # Test 4.3: Get service info run_test "service_info" \ @@ -425,17 +442,15 @@ if [ -n "$SERVICE_ID" ]; then "build/un service --destroy '$SERVICE_ID'" \ "destroy|deleted|success|$SERVICE_ID" else - echo -n "Test: service_create... " - TOTAL_TESTS=$((TOTAL_TESTS + 1)) - if grep -qiE "HTTP 5|error|limit|quota|no_pool" "$RESULTS_DIR/service_create.txt" 2>/dev/null; then - TEST_RESULTS["service_create"]="pass" - echo "PASS (API limit/error)" + # SCIENTIFIC INTEGRITY: If we couldn't create a service after retries, that's a FAIL + TEST_RESULTS["service_create"]="fail" + FAILURES=$((FAILURES + 1)) + if [ $service_attempt -gt 0 ]; then + echo "FAIL (after $service_attempt retries)" else - TEST_RESULTS["service_create"]="fail" - FAILURES=$((FAILURES + 1)) - echo "FAIL (no service ID)" - head -3 "$RESULTS_DIR/service_create.txt" + echo "FAIL" fi + head -5 "$RESULTS_DIR/service_create.txt" fi echo ""