russell@unturf.com
97fbd55e61
QR tests MUST use native language libraries, not qrencode CLI. If the sandbox doesn't have the library, the test should FAIL honestly so we know what to fix in the sandbox image. Also adds CLAUDE.md rule to prevent future attempts to replace native library QR tests with CLI subprocess calls.
699 lines
27 KiB
Bash
Executable file
699 lines
27 KiB
Bash
Executable file
#!/bin/bash
|
|
# Inception test: Use un (C CLI) to test SDKs AND API endpoints through unsandbox
|
|
#
|
|
# Pattern: build/un → unsandbox API → SDK/code → unsandbox API → result
|
|
#
|
|
# Tests:
|
|
# 1. SDK loads and runs in sandbox
|
|
# 2. Execute endpoint with various flags
|
|
# 3. Languages endpoint
|
|
# 4. Sessions lifecycle (create, list, execute, destroy)
|
|
# 5. Services lifecycle (create, list, info, logs, destroy)
|
|
# 6. Snapshots endpoint
|
|
# 7. Images endpoint
|
|
# 8. Key validation
|
|
# 9. QR code generation (native library or qrencode CLI fallback)
|
|
#
|
|
# Usage: test-sdk.sh LANGUAGE
|
|
|
|
# Don't use set -e - we need to continue on test failures to generate report
|
|
# set -e
|
|
|
|
LANG=${1:-python}
|
|
RESULTS_DIR="test-results-$LANG"
|
|
mkdir -p "$RESULTS_DIR"
|
|
|
|
# Map API language name to SDK file path
|
|
get_sdk_file() {
|
|
case "$1" in
|
|
python) echo "clients/python/sync/src/un.py" ;;
|
|
javascript) echo "clients/javascript/sync/src/un.js" ;;
|
|
typescript) echo "clients/typescript/sync/src/un.ts" ;;
|
|
ruby) echo "clients/ruby/sync/src/un.rb" ;;
|
|
php) echo "clients/php/sync/src/un.php" ;;
|
|
perl) echo "clients/perl/sync/src/un.pl" ;;
|
|
lua) echo "clients/lua/sync/src/un.lua" ;;
|
|
bash) echo "clients/bash/sync/src/un.sh" ;;
|
|
r) echo "clients/r/sync/src/un.r" ;;
|
|
awk) echo "clients/awk/sync/src/un.awk" ;;
|
|
tcl) echo "clients/tcl/sync/src/un.tcl" ;;
|
|
scheme) echo "clients/scheme/sync/src/un.scm" ;;
|
|
commonlisp) echo "clients/lisp/sync/src/un.lisp" ;;
|
|
lisp) echo "clients/lisp/sync/src/un.lisp" ;;
|
|
clojure) echo "clients/clojure/sync/src/un.clj" ;;
|
|
elixir) echo "clients/elixir/sync/src/un.ex" ;;
|
|
erlang) echo "clients/erlang/sync/src/un.erl" ;;
|
|
groovy) echo "clients/groovy/sync/src/un.groovy" ;;
|
|
raku) echo "clients/raku/sync/src/un.raku" ;;
|
|
julia) echo "clients/julia/sync/src/un.jl" ;;
|
|
dart) echo "clients/dart/sync/src/un.dart" ;;
|
|
prolog) echo "clients/prolog/sync/src/un.pro" ;;
|
|
forth) echo "clients/forth/sync/src/un.forth" ;;
|
|
powershell) echo "clients/powershell/sync/src/un.ps1" ;;
|
|
objc) echo "clients/objective-c/sync/src/un.m" ;;
|
|
v) echo "clients/v/sync/src/un.v" ;;
|
|
go) echo "clients/go/sync/src/un.go" ;;
|
|
rust) echo "clients/rust/sync/src/lib.rs" ;;
|
|
c) echo "clients/c/src/un.c" ;;
|
|
cpp) echo "clients/cpp/sync/src/un.cpp" ;;
|
|
java) echo "clients/java/sync/src/Un.java" ;;
|
|
kotlin) echo "clients/kotlin/sync/src/un.kt" ;;
|
|
swift) echo "clients/swift/sync/src/un.swift" ;;
|
|
csharp) echo "clients/csharp/sync/src/Un.cs" ;;
|
|
dotnet) echo "clients/dotnet/sync/src/Un.cs" ;;
|
|
fsharp) echo "clients/fsharp/sync/src/un.fs" ;;
|
|
haskell) echo "clients/haskell/sync/src/un.hs" ;;
|
|
ocaml) echo "clients/ocaml/sync/src/un.ml" ;;
|
|
d) echo "clients/d/sync/src/un.d" ;;
|
|
nim) echo "clients/nim/sync/src/un.nim" ;;
|
|
zig) echo "clients/zig/sync/src/un.zig" ;;
|
|
crystal) echo "clients/crystal/sync/src/un.cr" ;;
|
|
fortran) echo "clients/fortran/sync/src/un.f90" ;;
|
|
cobol) echo "clients/cobol/sync/src/un.cob" ;;
|
|
deno) echo "clients/typescript/sync/src/un.ts" ;;
|
|
*) echo "clients/$1/sync/src/un.NOTFOUND" ;;
|
|
esac
|
|
}
|
|
|
|
# Language-specific "hello world" commands
|
|
get_hello_code() {
|
|
case "$1" in
|
|
python|python3) echo 'print("test-ok")' ;;
|
|
javascript|typescript|deno) echo 'console.log("test-ok")' ;;
|
|
ruby) echo 'puts "test-ok"' ;;
|
|
php) echo '<?php echo "test-ok";' ;;
|
|
perl|raku) echo 'print "test-ok\n";' ;;
|
|
lua) echo 'print("test-ok")' ;;
|
|
bash) echo 'echo "test-ok"' ;;
|
|
r) echo 'cat("test-ok\n")' ;;
|
|
awk) echo 'BEGIN { print "test-ok" }' ;;
|
|
tcl) echo 'puts "test-ok"' ;;
|
|
scheme) echo '(display "test-ok") (newline)' ;;
|
|
commonlisp|lisp) echo '(format t "test-ok~%")' ;;
|
|
clojure) echo '(println "test-ok")' ;;
|
|
elixir) echo 'IO.puts "test-ok"' ;;
|
|
erlang) echo '-module(main). -export([start/0]). start() -> io:format("test-ok~n"), halt().' ;;
|
|
groovy) echo 'println "test-ok"' ;;
|
|
julia) echo 'println("test-ok")' ;;
|
|
prolog) echo ':- write("test-ok"), nl, halt.' ;;
|
|
forth) echo '.( test-ok) cr bye' ;;
|
|
powershell) echo 'Write-Output "test-ok"' ;;
|
|
go) echo 'package main; import "fmt"; func main() { fmt.Println("test-ok") }' ;;
|
|
rust) echo 'fn main() { println!("test-ok"); }' ;;
|
|
c) echo '#include <stdio.h>
|
|
int main() { printf("test-ok\n"); return 0; }' ;;
|
|
cpp) echo '#include <iostream>
|
|
int main() { std::cout << "test-ok" << std::endl; return 0; }' ;;
|
|
java) echo 'public class Main { public static void main(String[] args) { System.out.println("test-ok"); } }' ;;
|
|
kotlin) echo 'fun main() { println("test-ok") }' ;;
|
|
swift) echo 'print("test-ok")' ;;
|
|
csharp) echo 'class P{static void Main(){System.Console.WriteLine("test-ok");}}' ;;
|
|
dotnet) echo 'Console.WriteLine("test-ok");' ;;
|
|
fsharp) echo 'printfn "test-ok"' ;;
|
|
haskell) echo 'main = putStrLn "test-ok"' ;;
|
|
ocaml) echo 'print_endline "test-ok"' ;;
|
|
d) echo 'import std.stdio; void main() { writeln("test-ok"); }' ;;
|
|
nim) echo 'echo "test-ok"' ;;
|
|
zig) echo 'const std = @import("std"); pub fn main() void { std.debug.print("test-ok\n", .{}); }' ;;
|
|
crystal) echo 'puts "test-ok"' ;;
|
|
fortran) echo 'program main
|
|
print *, "test-ok"
|
|
end program main' ;;
|
|
cobol) echo 'IDENTIFICATION DIVISION. PROGRAM-ID. HELLO. PROCEDURE DIVISION. DISPLAY "test-ok". STOP RUN.' ;;
|
|
objc) echo '#import <Foundation/Foundation.h>
|
|
int main() { NSLog(@"test-ok"); return 0; }' ;;
|
|
dart) echo 'void main() { print("test-ok"); }' ;;
|
|
v) echo 'fn main() { println("test-ok") }' ;;
|
|
*) echo 'print("test-ok")' ;;
|
|
esac
|
|
}
|
|
|
|
# Map language to QR code test file
|
|
get_qr_file() {
|
|
case "$1" in
|
|
python) echo "test/qr.py" ;;
|
|
javascript) echo "test/qr.js" ;;
|
|
typescript) echo "test/qr.ts" ;;
|
|
ruby) echo "test/qr.rb" ;;
|
|
php) echo "test/qr.php" ;;
|
|
perl) echo "test/qr.pl" ;;
|
|
lua) echo "test/qr.lua" ;;
|
|
bash) echo "test/qr.sh" ;;
|
|
r) echo "test/qr.r" ;;
|
|
awk) echo "test/qr.awk" ;;
|
|
tcl) echo "test/qr.tcl" ;;
|
|
scheme) echo "test/qr.scm" ;;
|
|
commonlisp) echo "test/qr.lisp" ;;
|
|
lisp) echo "test/qr.lisp" ;;
|
|
clojure) echo "test/qr.clj" ;;
|
|
elixir) echo "test/qr.ex" ;;
|
|
erlang) echo "test/qr.erl" ;;
|
|
groovy) echo "test/qr.groovy" ;;
|
|
raku) echo "test/qr.raku" ;;
|
|
julia) echo "test/qr.jl" ;;
|
|
dart) echo "test/qr.dart" ;;
|
|
prolog) echo "test/qr.pro" ;;
|
|
forth) echo "test/qr.forth" ;;
|
|
powershell) echo "test/qr.ps1" ;;
|
|
objc) echo "test/qr.m" ;;
|
|
v) echo "test/qr.v" ;;
|
|
go) echo "test/qr.go" ;;
|
|
rust) echo "test/qr.rs" ;;
|
|
c) echo "test/qr.c" ;;
|
|
cpp) echo "test/qr.cpp" ;;
|
|
java) echo "test/qr.java" ;;
|
|
kotlin) echo "test/qr.kt" ;;
|
|
csharp) echo "test/qr.cs" ;;
|
|
dotnet) echo "test/qr.cs" ;;
|
|
fsharp) echo "test/qr.fs" ;;
|
|
haskell) echo "test/qr.hs" ;;
|
|
ocaml) echo "test/qr.ml" ;;
|
|
d) echo "test/qr.d" ;;
|
|
nim) echo "test/qr.nim" ;;
|
|
zig) echo "test/qr.zig" ;;
|
|
crystal) echo "test/qr.cr" ;;
|
|
fortran) echo "test/qr.f90" ;;
|
|
cobol) echo "test/qr.cob" ;;
|
|
deno) echo "test/qr.ts" ;;
|
|
*) echo "" ;;
|
|
esac
|
|
}
|
|
|
|
SDK_FILE=$(get_sdk_file "$LANG")
|
|
|
|
echo "=== Functional Tests: $LANG ==="
|
|
echo "SDK: $SDK_FILE"
|
|
echo ""
|
|
|
|
# Check if un CLI exists
|
|
if [ ! -x "build/un" ]; then
|
|
echo "ERROR: build/un not found. Run build-clients.sh first."
|
|
exit 1
|
|
fi
|
|
|
|
# Check if SDK file exists
|
|
if [ ! -f "$SDK_FILE" ]; then
|
|
echo "SKIP: SDK file not found: $SDK_FILE"
|
|
cat > "$RESULTS_DIR/test-results.xml" << EOF
|
|
<?xml version="1.0" encoding="UTF-8"?>
|
|
<testsuites>
|
|
<testsuite name="Functional Test" tests="1" skipped="1">
|
|
<testcase name="$LANG SDK" classname="un.$LANG">
|
|
<skipped message="SDK file not found: $SDK_FILE"/>
|
|
</testcase>
|
|
</testsuite>
|
|
</testsuites>
|
|
EOF
|
|
exit 0
|
|
fi
|
|
|
|
START_TIME=$(date +%s.%N)
|
|
|
|
# Track test results
|
|
declare -A TEST_RESULTS
|
|
TOTAL_TESTS=0
|
|
FAILURES=0
|
|
|
|
# Track transient errors for API health monitoring
|
|
# SCIENTIFIC INTEGRITY: We track these to understand API reliability over time
|
|
TOTAL_RETRIES=0
|
|
RETRIES_429=0 # Rate limiting
|
|
RETRIES_5XX=0 # Server errors
|
|
RETRIES_TIMEOUT=0 # Timeouts
|
|
RETRIES_CONN=0 # Connection issues
|
|
declare -A TEST_RETRIES # Track retries per test
|
|
|
|
# Helper to run test and record result
|
|
# Retries on ALL transient errors (429, 5xx, timeouts) with exponential backoff
|
|
# SCIENTIFIC INTEGRITY: Tests must PASS, FAIL, or SKIP - never lie with soft passes
|
|
run_test() {
|
|
local test_name="$1"
|
|
local test_cmd="$2"
|
|
local success_pattern="$3"
|
|
local output_file="$RESULTS_DIR/${test_name}.txt"
|
|
local max_retries=10
|
|
local retry_delay=2
|
|
local attempt=0
|
|
local retried=false
|
|
|
|
echo -n "Test: $test_name... "
|
|
TOTAL_TESTS=$((TOTAL_TESTS + 1))
|
|
|
|
while [ $attempt -lt $max_retries ]; do
|
|
if eval "$test_cmd" > "$output_file" 2>&1; then
|
|
if [ -n "$success_pattern" ]; then
|
|
if grep -qiE "$success_pattern" "$output_file"; then
|
|
TEST_RESULTS["$test_name"]="pass"
|
|
[ $attempt -gt 0 ] && TEST_RETRIES["$test_name"]=$attempt
|
|
if [ "$retried" = true ]; then
|
|
echo "PASS (after $attempt retries)"
|
|
else
|
|
echo "PASS"
|
|
fi
|
|
return 0
|
|
fi
|
|
else
|
|
# No pattern required, just check non-empty output
|
|
if [ -s "$output_file" ]; then
|
|
TEST_RESULTS["$test_name"]="pass"
|
|
[ $attempt -gt 0 ] && TEST_RETRIES["$test_name"]=$attempt
|
|
if [ "$retried" = true ]; then
|
|
echo "PASS (after $attempt retries)"
|
|
else
|
|
echo "PASS"
|
|
fi
|
|
return 0
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# Check for transient errors that should trigger retry
|
|
# Track error type for API health monitoring
|
|
local error_type=""
|
|
if grep -qiE "HTTP 429|concurrency_limit|too many.*concurrent|rate.limit" "$output_file" 2>/dev/null; then
|
|
error_type="429"
|
|
RETRIES_429=$((RETRIES_429 + 1))
|
|
elif grep -qiE "HTTP 5[0-9][0-9]|server error|internal error|bad gateway|service unavailable|gateway timeout" "$output_file" 2>/dev/null; then
|
|
error_type="5xx"
|
|
RETRIES_5XX=$((RETRIES_5XX + 1))
|
|
elif grep -qiE "timeout|timed out|ETIMEDOUT" "$output_file" 2>/dev/null; then
|
|
error_type="timeout"
|
|
RETRIES_TIMEOUT=$((RETRIES_TIMEOUT + 1))
|
|
elif grep -qiE "connection refused|connection reset|ECONNREFUSED|request failed" "$output_file" 2>/dev/null; then
|
|
error_type="connection"
|
|
RETRIES_CONN=$((RETRIES_CONN + 1))
|
|
fi
|
|
|
|
if [ -n "$error_type" ]; then
|
|
attempt=$((attempt + 1))
|
|
retried=true
|
|
TOTAL_RETRIES=$((TOTAL_RETRIES + 1))
|
|
if [ $attempt -lt $max_retries ]; then
|
|
echo -n "($error_type retry $attempt/$max_retries)... "
|
|
sleep $retry_delay
|
|
# Exponential backoff, cap at 30 seconds
|
|
retry_delay=$((retry_delay * 2))
|
|
if [ $retry_delay -gt 30 ]; then
|
|
retry_delay=30
|
|
fi
|
|
continue
|
|
fi
|
|
else
|
|
# No error detected, but output didn't match expected pattern
|
|
# This can happen when API returns 200 but service isn't ready yet
|
|
# Retry a few times for service operations
|
|
if [ $attempt -lt 5 ]; then
|
|
attempt=$((attempt + 1))
|
|
retried=true
|
|
echo -n "(no-match retry $attempt/5)... "
|
|
sleep 3
|
|
continue
|
|
fi
|
|
fi
|
|
|
|
# Exhausted retries, break out of loop
|
|
break
|
|
done
|
|
|
|
# If we exhausted retries on transient errors, that's still a FAIL
|
|
# We don't lie about test results - if we couldn't verify, it failed
|
|
TEST_RESULTS["$test_name"]="fail"
|
|
[ $attempt -gt 0 ] && TEST_RETRIES["$test_name"]=$attempt
|
|
FAILURES=$((FAILURES + 1))
|
|
if [ "$retried" = true ]; then
|
|
echo "FAIL (after $attempt retries)"
|
|
else
|
|
echo "FAIL"
|
|
fi
|
|
# Show full error context: HTTP status, error messages, and first lines of output
|
|
echo " --- Failure output ($test_name) ---"
|
|
grep -iE "HTTP [0-9]{3}|error|Error|status|failed|refused|timeout" "$output_file" 2>/dev/null | head -5 || true
|
|
echo " --- First 5 lines ---"
|
|
head -5 "$output_file" 2>/dev/null || echo "(no output)"
|
|
echo " ---"
|
|
return 0 # Always return success to continue test execution
|
|
}
|
|
|
|
echo "Running functional tests..."
|
|
echo ""
|
|
|
|
HELLO_CODE=$(get_hello_code "$LANG")
|
|
|
|
# ============================================================================
|
|
# SECTION 1: Execute Endpoint Tests
|
|
# ============================================================================
|
|
echo "--- Execute Endpoint ---"
|
|
|
|
# Test 1.1: Basic code execution
|
|
run_test "exec_basic" \
|
|
"build/un -s '$LANG' '$HELLO_CODE'" \
|
|
"test-ok"
|
|
|
|
# Test 1.2: Execute with environment variable
|
|
run_test "exec_env" \
|
|
"build/un -s python -e TEST_VAR=hello123 'import os; print(os.environ.get(\"TEST_VAR\", \"missing\"))'" \
|
|
"hello123"
|
|
|
|
# Test 1.3: Execute with file upload (files go to /tmp/input/)
|
|
run_test "exec_file" \
|
|
"build/un -s python -f test/fib.py 'import os; print(os.listdir(\"/tmp/input\"))'" \
|
|
"fib"
|
|
|
|
# Test 1.4: Execute with network access (semitrusted)
|
|
run_test "exec_network" \
|
|
"build/un -n semitrusted -s python 'import socket; print(\"network-ok\")'" \
|
|
"network-ok"
|
|
|
|
# Test 1.5: Execute SDK file in sandbox (inception)
|
|
# Runs the SDK file inside unsandbox - tests that the interpreter can load/parse the SDK
|
|
# Expected: help/usage output, or interpreter errors (all acceptable)
|
|
run_test "exec_sdk_inception" \
|
|
"build/un -n semitrusted -e UNSANDBOX_PUBLIC_KEY=\$UNSANDBOX_PUBLIC_KEY -e UNSANDBOX_SECRET_KEY=\$UNSANDBOX_SECRET_KEY '$SDK_FILE' 2>&1 || echo 'attempted'" \
|
|
"usage|help|unsandbox|Usage|error|Error|attempted"
|
|
|
|
echo ""
|
|
|
|
# ============================================================================
|
|
# SECTION 2: Languages Endpoint Tests
|
|
# ============================================================================
|
|
echo "--- Languages Endpoint ---"
|
|
|
|
# Test 2.1: List all languages (one per line)
|
|
run_test "languages_list" \
|
|
"build/un languages" \
|
|
"python"
|
|
|
|
# Test 2.2: Verify our language is in the list
|
|
run_test "languages_contains_$LANG" \
|
|
"build/un languages" \
|
|
"$LANG"
|
|
|
|
echo ""
|
|
|
|
# ============================================================================
|
|
# SECTION 3: Session Lifecycle Tests
|
|
# ============================================================================
|
|
echo "--- Session Lifecycle ---"
|
|
|
|
# Test 3.1: List sessions (might be empty, that's ok)
|
|
# Note: Session creation is interactive, so we can only test listing
|
|
run_test "session_list" \
|
|
"build/un session --list" \
|
|
"unsb-session|no.*session|\[\]|sessions|Session"
|
|
|
|
# If there are existing sessions, test session info on first one
|
|
SESSION_ID=$(build/un session --list 2>/dev/null | grep -oE "unsb-session-[a-z0-9-]+" | head -1 || true)
|
|
if [ -n "$SESSION_ID" ]; then
|
|
# Test 3.2: Get session info
|
|
run_test "session_info" \
|
|
"build/un session --info '$SESSION_ID'" \
|
|
"$SESSION_ID|status|created"
|
|
fi
|
|
|
|
echo ""
|
|
|
|
# ============================================================================
|
|
# SECTION 4: Service Lifecycle Tests
|
|
# ============================================================================
|
|
echo "--- Service Lifecycle ---"
|
|
|
|
# Test 4.1: List services
|
|
run_test "service_list" \
|
|
"build/un service --list" \
|
|
"unsb-service|no.*service|\[\]|services"
|
|
|
|
# Test 4.2: Create a service with retry logic (note: syntax is --name not --create)
|
|
# SCIENTIFIC INTEGRITY: Retry on transient errors, fail if we can't create
|
|
# Use a single name per test run - if 5xx hides a successful create, we detect via 409
|
|
SERVICE_NAME="test-$LANG-$$"
|
|
SERVICE_ID=""
|
|
service_attempt=0
|
|
service_max_retries=5
|
|
service_retry_delay=2
|
|
|
|
echo -n "Test: service_create... "
|
|
TOTAL_TESTS=$((TOTAL_TESTS + 1))
|
|
|
|
while [ $service_attempt -lt $service_max_retries ]; do
|
|
SERVICE_OUTPUT=$(build/un service --name "$SERVICE_NAME" --bootstrap "echo service-started" 2>&1 || true)
|
|
echo "$SERVICE_OUTPUT" > "$RESULTS_DIR/service_create.txt"
|
|
SERVICE_ID=$(echo "$SERVICE_OUTPUT" | grep -oE "unsb-service-[a-z0-9-]+" | head -1 || true)
|
|
|
|
if [ -n "$SERVICE_ID" ]; then
|
|
break
|
|
fi
|
|
|
|
# Check for transient errors that should trigger retry
|
|
# Track error type for API health monitoring
|
|
svc_error_type=""
|
|
if grep -qiE "HTTP 429|concurrency_limit|rate.limit" "$RESULTS_DIR/service_create.txt" 2>/dev/null; then
|
|
svc_error_type="429"
|
|
RETRIES_429=$((RETRIES_429 + 1))
|
|
elif grep -qiE "HTTP 409|already taken|already exists" "$RESULTS_DIR/service_create.txt" 2>/dev/null; then
|
|
# 409 means a previous attempt actually created it - look up the ID
|
|
svc_error_type="409"
|
|
echo -n "(409 - looking up existing)... "
|
|
SVC_LOOKUP=$(build/un service --list 2>/dev/null | grep "$SERVICE_NAME" || true)
|
|
SERVICE_ID=$(echo "$SVC_LOOKUP" | grep -oE "unsb-service-[a-z0-9-]+" | head -1 || true)
|
|
if [ -n "$SERVICE_ID" ]; then
|
|
break
|
|
fi
|
|
elif grep -qiE "HTTP 5[0-9][0-9]|server error|internal error" "$RESULTS_DIR/service_create.txt" 2>/dev/null; then
|
|
svc_error_type="5xx"
|
|
RETRIES_5XX=$((RETRIES_5XX + 1))
|
|
elif grep -qiE "timeout|timed out|ETIMEDOUT" "$RESULTS_DIR/service_create.txt" 2>/dev/null; then
|
|
svc_error_type="timeout"
|
|
RETRIES_TIMEOUT=$((RETRIES_TIMEOUT + 1))
|
|
elif grep -qiE "connection|ECONNREFUSED|limit|quota|no_pool" "$RESULTS_DIR/service_create.txt" 2>/dev/null; then
|
|
svc_error_type="connection"
|
|
RETRIES_CONN=$((RETRIES_CONN + 1))
|
|
fi
|
|
|
|
if [ -n "$svc_error_type" ]; then
|
|
service_attempt=$((service_attempt + 1))
|
|
TOTAL_RETRIES=$((TOTAL_RETRIES + 1))
|
|
if [ $service_attempt -lt $service_max_retries ]; then
|
|
echo -n "($svc_error_type retry $service_attempt/$service_max_retries)... "
|
|
sleep $service_retry_delay
|
|
service_retry_delay=$((service_retry_delay * 2))
|
|
if [ $service_retry_delay -gt 10 ]; then
|
|
service_retry_delay=10
|
|
fi
|
|
continue
|
|
fi
|
|
fi
|
|
break
|
|
done
|
|
|
|
if [ -n "$SERVICE_ID" ]; then
|
|
TEST_RESULTS["service_create"]="pass"
|
|
[ $service_attempt -gt 0 ] && TEST_RETRIES["service_create"]=$service_attempt
|
|
if [ $service_attempt -gt 0 ]; then
|
|
echo "PASS (created $SERVICE_ID after $service_attempt retries)"
|
|
else
|
|
echo "PASS (created $SERVICE_ID)"
|
|
fi
|
|
|
|
# Test 4.3: Get service info
|
|
run_test "service_info" \
|
|
"build/un service --info '$SERVICE_ID'" \
|
|
"$SERVICE_ID|status|name"
|
|
|
|
# Test 4.4: Execute command in service
|
|
run_test "service_exec" \
|
|
"build/un service --execute '$SERVICE_ID' 'echo service-exec-ok'" \
|
|
"service-exec-ok"
|
|
|
|
# Test 4.5: Get service logs
|
|
run_test "service_logs" \
|
|
"build/un service --logs '$SERVICE_ID'" \
|
|
"log|output|service|$SERVICE_ID"
|
|
|
|
# Test 4.6: Destroy service (cleanup)
|
|
# "not found" counts as success - service was already destroyed (e.g., by a 5xx that hid success)
|
|
run_test "service_destroy" \
|
|
"build/un service --destroy '$SERVICE_ID'" \
|
|
"destroy|deleted|success|not found|Not found|$SERVICE_ID"
|
|
else
|
|
# SCIENTIFIC INTEGRITY: If we couldn't create a service after retries, that's a FAIL
|
|
TEST_RESULTS["service_create"]="fail"
|
|
FAILURES=$((FAILURES + 1))
|
|
if [ $service_attempt -gt 0 ]; then
|
|
echo "FAIL (after $service_attempt retries)"
|
|
else
|
|
echo "FAIL"
|
|
fi
|
|
head -5 "$RESULTS_DIR/service_create.txt"
|
|
fi
|
|
|
|
echo ""
|
|
|
|
# ============================================================================
|
|
# SECTION 5: Snapshot Endpoint Tests
|
|
# ============================================================================
|
|
echo "--- Snapshot Endpoint ---"
|
|
|
|
# Test 5.1: List snapshots
|
|
run_test "snapshot_list" \
|
|
"build/un snapshot --list" \
|
|
"unsb-snapshot|no.*snapshot|\[\]|snapshots"
|
|
|
|
echo ""
|
|
|
|
# ============================================================================
|
|
# SECTION 6: Image Endpoint Tests
|
|
# ============================================================================
|
|
echo "--- Image Endpoint ---"
|
|
|
|
# Test 6.1: List images
|
|
run_test "image_list" \
|
|
"build/un image --list" \
|
|
"unsb-image|no.*image|\[\]|images|name"
|
|
|
|
echo ""
|
|
|
|
# ============================================================================
|
|
# SECTION 7: Key Validation Tests
|
|
# ============================================================================
|
|
echo "--- Key Validation ---"
|
|
|
|
# Test 7.1: Validate API key
|
|
run_test "key_validate" \
|
|
"build/un key" \
|
|
"valid|key|account|unsb-pk"
|
|
|
|
echo ""
|
|
|
|
# ============================================================================
|
|
# SECTION 8: SDK-Specific Inception Tests
|
|
# ============================================================================
|
|
echo "--- SDK Inception ($LANG) ---"
|
|
|
|
# Test 8.1: SDK file exists and is readable
|
|
run_test "sdk_exists" \
|
|
"test -f '$SDK_FILE' && wc -l < '$SDK_FILE'" \
|
|
"[0-9]+"
|
|
|
|
# Test 8.2: Run SDK file directly (auto-detect language from extension)
|
|
# Note: Complex SDK files may fail (shebangs, deps, compile errors) - that's ok
|
|
# We accept any output showing the runtime attempted execution
|
|
run_test "sdk_runs" \
|
|
"build/un -n semitrusted -e UNSANDBOX_PUBLIC_KEY=\$UNSANDBOX_PUBLIC_KEY -e UNSANDBOX_SECRET_KEY=\$UNSANDBOX_SECRET_KEY '$SDK_FILE' 2>&1 || echo 'attempted'" \
|
|
"usage|help|unsandbox|un |version|error|Error|compile|undefined|attempted|unexpected|syntax|import|require|module|WARNING|domain|license"
|
|
|
|
echo ""
|
|
|
|
# ============================================================================
|
|
# SECTION 9: QR Code Generation Tests
|
|
# ============================================================================
|
|
echo "--- QR Code Generation ---"
|
|
|
|
QR_FILE=$(get_qr_file "$LANG")
|
|
if [ -n "$QR_FILE" ] && [ -f "$QR_FILE" ]; then
|
|
# Test 9.1: QR code generation produces structured output
|
|
# Pass file path without -s: un CLI reads the file and auto-detects language
|
|
# (with -s, the positional arg is treated as inline code, not a file path)
|
|
# QR tests use native language libraries - if the library isn't in the sandbox,
|
|
# the test FAILS honestly. Fix the sandbox image, don't mask the failure.
|
|
run_test "qr_generate" \
|
|
"build/un '$QR_FILE'" \
|
|
"QR:unsandbox-qr-ok:ROWS:[0-9]+"
|
|
else
|
|
echo "SKIP: No QR test file for $LANG"
|
|
fi
|
|
|
|
echo ""
|
|
|
|
# ============================================================================
|
|
# Results Summary
|
|
# ============================================================================
|
|
END_TIME=$(date +%s.%N)
|
|
DURATION=$(echo "$END_TIME - $START_TIME" | bc 2>/dev/null || echo "0")
|
|
|
|
PASSED=$((TOTAL_TESTS - FAILURES))
|
|
if [ "$FAILURES" -eq 0 ]; then
|
|
STATUS="PASS"
|
|
else
|
|
STATUS="FAIL"
|
|
fi
|
|
|
|
echo "============================================"
|
|
echo "=== Result: $STATUS ($PASSED/$TOTAL_TESTS passed, ${DURATION}s) ==="
|
|
echo "============================================"
|
|
|
|
# ============================================================================
|
|
# API Health Report - Track transient errors for monitoring
|
|
# ============================================================================
|
|
TESTS_WITH_RETRIES=${#TEST_RETRIES[@]}
|
|
|
|
echo ""
|
|
echo "--- API Health Report ---"
|
|
echo "Total retries: $TOTAL_RETRIES"
|
|
echo " Rate limit (429): $RETRIES_429"
|
|
echo " Server error (5xx): $RETRIES_5XX"
|
|
echo " Timeout: $RETRIES_TIMEOUT"
|
|
echo " Connection: $RETRIES_CONN"
|
|
echo "Tests that needed retries: $TESTS_WITH_RETRIES"
|
|
|
|
if [ $TESTS_WITH_RETRIES -gt 0 ]; then
|
|
echo "Tests with retries:"
|
|
for test_name in "${!TEST_RETRIES[@]}"; do
|
|
retries="${TEST_RETRIES[$test_name]}"
|
|
result="${TEST_RESULTS[$test_name]}"
|
|
echo " - $test_name: $retries retries ($result)"
|
|
done
|
|
fi
|
|
|
|
# Write JSON report for aggregation across all SDKs
|
|
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
|
cat > "$RESULTS_DIR/api-health.json" << EOF
|
|
{
|
|
"timestamp": "$TIMESTAMP",
|
|
"language": "$LANG",
|
|
"duration_seconds": $DURATION,
|
|
"tests": {
|
|
"total": $TOTAL_TESTS,
|
|
"passed": $PASSED,
|
|
"failed": $FAILURES
|
|
},
|
|
"retries": {
|
|
"total": $TOTAL_RETRIES,
|
|
"rate_limit_429": $RETRIES_429,
|
|
"server_error_5xx": $RETRIES_5XX,
|
|
"timeout": $RETRIES_TIMEOUT,
|
|
"connection": $RETRIES_CONN
|
|
},
|
|
"tests_with_retries": $TESTS_WITH_RETRIES,
|
|
"api_health_score": $(echo "scale=2; 100 - ($TOTAL_RETRIES * 5)" | bc 2>/dev/null || echo "100")
|
|
}
|
|
EOF
|
|
|
|
echo ""
|
|
|
|
# Generate JUnit XML
|
|
cat > "$RESULTS_DIR/test-results.xml" << EOF
|
|
<?xml version="1.0" encoding="UTF-8"?>
|
|
<testsuites>
|
|
<testsuite name="Functional Test - $LANG" tests="$TOTAL_TESTS" failures="$FAILURES" time="$DURATION">
|
|
EOF
|
|
|
|
for test_name in "${!TEST_RESULTS[@]}"; do
|
|
result="${TEST_RESULTS[$test_name]}"
|
|
cat >> "$RESULTS_DIR/test-results.xml" << EOF
|
|
<testcase name="$test_name" classname="un.$LANG">
|
|
EOF
|
|
if [ "$result" = "pass" ]; then
|
|
echo " <system-out>Test passed</system-out>" >> "$RESULTS_DIR/test-results.xml"
|
|
else
|
|
echo " <failure message=\"Test failed\">See ${test_name}.txt for details</failure>" >> "$RESULTS_DIR/test-results.xml"
|
|
fi
|
|
echo " </testcase>" >> "$RESULTS_DIR/test-results.xml"
|
|
done
|
|
|
|
cat >> "$RESULTS_DIR/test-results.xml" << EOF
|
|
</testsuite>
|
|
</testsuites>
|
|
EOF
|
|
|
|
# Exit with appropriate code
|
|
[ "$STATUS" = "PASS" ] && exit 0 || exit 1
|