russell@unturf.com
b651bf2b63
Track retry counts by error type (429, 5xx, timeout, connection) and output API Health Report at end of each test run. Writes api-health.json for aggregation across all 42 SDKs. This lets us monitor API reliability over time and identify patterns.
670 lines
25 KiB
Bash
Executable file
670 lines
25 KiB
Bash
Executable file
#!/bin/bash
|
|
# Inception test: Use un (C CLI) to test SDKs AND API endpoints through unsandbox
|
|
#
|
|
# Pattern: build/un → unsandbox API → SDK/code → unsandbox API → result
|
|
#
|
|
# Tests:
|
|
# 1. SDK loads and runs in sandbox
|
|
# 2. Execute endpoint with various flags
|
|
# 3. Languages endpoint
|
|
# 4. Sessions lifecycle (create, list, execute, destroy)
|
|
# 5. Services lifecycle (create, list, info, logs, destroy)
|
|
# 6. Snapshots endpoint
|
|
# 7. Images endpoint
|
|
# 8. Key validation
|
|
# 9. QR code generation (native library or qrencode CLI fallback)
|
|
#
|
|
# Usage: test-sdk.sh LANGUAGE
|
|
|
|
# Don't use set -e - we need to continue on test failures to generate report
|
|
# set -e
|
|
|
|
LANG=${1:-python}
|
|
RESULTS_DIR="test-results-$LANG"
|
|
mkdir -p "$RESULTS_DIR"
|
|
|
|
# Map API language name to SDK file path
|
|
get_sdk_file() {
|
|
case "$1" in
|
|
python) echo "clients/python/sync/src/un.py" ;;
|
|
javascript) echo "clients/javascript/sync/src/un.js" ;;
|
|
typescript) echo "clients/typescript/sync/src/un.ts" ;;
|
|
ruby) echo "clients/ruby/sync/src/un.rb" ;;
|
|
php) echo "clients/php/sync/src/un.php" ;;
|
|
perl) echo "clients/perl/sync/src/un.pl" ;;
|
|
lua) echo "clients/lua/sync/src/un.lua" ;;
|
|
bash) echo "clients/bash/sync/src/un.sh" ;;
|
|
r) echo "clients/r/sync/src/un.r" ;;
|
|
awk) echo "clients/awk/sync/src/un.awk" ;;
|
|
tcl) echo "clients/tcl/sync/src/un.tcl" ;;
|
|
scheme) echo "clients/scheme/sync/src/un.scm" ;;
|
|
commonlisp) echo "clients/lisp/sync/src/un.lisp" ;;
|
|
lisp) echo "clients/lisp/sync/src/un.lisp" ;;
|
|
clojure) echo "clients/clojure/sync/src/un.clj" ;;
|
|
elixir) echo "clients/elixir/sync/src/un.ex" ;;
|
|
erlang) echo "clients/erlang/sync/src/un.erl" ;;
|
|
groovy) echo "clients/groovy/sync/src/un.groovy" ;;
|
|
raku) echo "clients/raku/sync/src/un.raku" ;;
|
|
julia) echo "clients/julia/sync/src/un.jl" ;;
|
|
dart) echo "clients/dart/sync/src/un.dart" ;;
|
|
prolog) echo "clients/prolog/sync/src/un.pro" ;;
|
|
forth) echo "clients/forth/sync/src/un.forth" ;;
|
|
powershell) echo "clients/powershell/sync/src/un.ps1" ;;
|
|
objc) echo "clients/objective-c/sync/src/un.m" ;;
|
|
v) echo "clients/v/sync/src/un.v" ;;
|
|
go) echo "clients/go/sync/src/un.go" ;;
|
|
rust) echo "clients/rust/sync/src/lib.rs" ;;
|
|
c) echo "clients/c/src/un.c" ;;
|
|
cpp) echo "clients/cpp/sync/src/un.cpp" ;;
|
|
java) echo "clients/java/sync/src/Un.java" ;;
|
|
kotlin) echo "clients/kotlin/sync/src/un.kt" ;;
|
|
swift) echo "clients/swift/sync/src/un.swift" ;;
|
|
csharp) echo "clients/csharp/sync/src/Un.cs" ;;
|
|
dotnet) echo "clients/dotnet/sync/src/Un.cs" ;;
|
|
fsharp) echo "clients/fsharp/sync/src/un.fs" ;;
|
|
haskell) echo "clients/haskell/sync/src/un.hs" ;;
|
|
ocaml) echo "clients/ocaml/sync/src/un.ml" ;;
|
|
d) echo "clients/d/sync/src/un.d" ;;
|
|
nim) echo "clients/nim/sync/src/un.nim" ;;
|
|
zig) echo "clients/zig/sync/src/un.zig" ;;
|
|
crystal) echo "clients/crystal/sync/src/un.cr" ;;
|
|
fortran) echo "clients/fortran/sync/src/un.f90" ;;
|
|
cobol) echo "clients/cobol/sync/src/un.cob" ;;
|
|
deno) echo "clients/typescript/sync/src/un.ts" ;;
|
|
*) echo "clients/$1/sync/src/un.NOTFOUND" ;;
|
|
esac
|
|
}
|
|
|
|
# Language-specific "hello world" commands
|
|
get_hello_code() {
|
|
case "$1" in
|
|
python|python3) echo 'print("test-ok")' ;;
|
|
javascript|typescript|deno) echo 'console.log("test-ok")' ;;
|
|
ruby) echo 'puts "test-ok"' ;;
|
|
php) echo '<?php echo "test-ok";' ;;
|
|
perl|raku) echo 'print "test-ok\n";' ;;
|
|
lua) echo 'print("test-ok")' ;;
|
|
bash) echo 'echo "test-ok"' ;;
|
|
r) echo 'cat("test-ok\n")' ;;
|
|
awk) echo 'BEGIN { print "test-ok" }' ;;
|
|
tcl) echo 'puts "test-ok"' ;;
|
|
scheme) echo '(display "test-ok") (newline)' ;;
|
|
commonlisp|lisp) echo '(format t "test-ok~%")' ;;
|
|
clojure) echo '(println "test-ok")' ;;
|
|
elixir) echo 'IO.puts "test-ok"' ;;
|
|
erlang) echo '-module(main). -export([start/0]). start() -> io:format("test-ok~n"), halt().' ;;
|
|
groovy) echo 'println "test-ok"' ;;
|
|
julia) echo 'println("test-ok")' ;;
|
|
prolog) echo ':- write("test-ok"), nl, halt.' ;;
|
|
forth) echo '.( test-ok) cr bye' ;;
|
|
powershell) echo 'Write-Output "test-ok"' ;;
|
|
go) echo 'package main; import "fmt"; func main() { fmt.Println("test-ok") }' ;;
|
|
rust) echo 'fn main() { println!("test-ok"); }' ;;
|
|
c) echo '#include <stdio.h>
|
|
int main() { printf("test-ok\n"); return 0; }' ;;
|
|
cpp) echo '#include <iostream>
|
|
int main() { std::cout << "test-ok" << std::endl; return 0; }' ;;
|
|
java) echo 'public class Main { public static void main(String[] args) { System.out.println("test-ok"); } }' ;;
|
|
kotlin) echo 'fun main() { println("test-ok") }' ;;
|
|
swift) echo 'print("test-ok")' ;;
|
|
csharp) echo 'class P{static void Main(){System.Console.WriteLine("test-ok");}}' ;;
|
|
dotnet) echo 'Console.WriteLine("test-ok");' ;;
|
|
fsharp) echo 'printfn "test-ok"' ;;
|
|
haskell) echo 'main = putStrLn "test-ok"' ;;
|
|
ocaml) echo 'print_endline "test-ok"' ;;
|
|
d) echo 'import std.stdio; void main() { writeln("test-ok"); }' ;;
|
|
nim) echo 'echo "test-ok"' ;;
|
|
zig) echo 'const std = @import("std"); pub fn main() void { std.debug.print("test-ok\n", .{}); }' ;;
|
|
crystal) echo 'puts "test-ok"' ;;
|
|
fortran) echo 'program main
|
|
print *, "test-ok"
|
|
end program main' ;;
|
|
cobol) echo 'IDENTIFICATION DIVISION. PROGRAM-ID. HELLO. PROCEDURE DIVISION. DISPLAY "test-ok". STOP RUN.' ;;
|
|
objc) echo '#import <Foundation/Foundation.h>
|
|
int main() { NSLog(@"test-ok"); return 0; }' ;;
|
|
dart) echo 'void main() { print("test-ok"); }' ;;
|
|
v) echo 'fn main() { println("test-ok") }' ;;
|
|
*) echo 'print("test-ok")' ;;
|
|
esac
|
|
}
|
|
|
|
# Map language to QR code test file
|
|
get_qr_file() {
|
|
case "$1" in
|
|
python) echo "test/qr.py" ;;
|
|
javascript) echo "test/qr.js" ;;
|
|
typescript) echo "test/qr.ts" ;;
|
|
ruby) echo "test/qr.rb" ;;
|
|
php) echo "test/qr.php" ;;
|
|
perl) echo "test/qr.pl" ;;
|
|
lua) echo "test/qr.lua" ;;
|
|
bash) echo "test/qr.sh" ;;
|
|
r) echo "test/qr.r" ;;
|
|
awk) echo "test/qr.awk" ;;
|
|
tcl) echo "test/qr.tcl" ;;
|
|
scheme) echo "test/qr.scm" ;;
|
|
commonlisp) echo "test/qr.lisp" ;;
|
|
lisp) echo "test/qr.lisp" ;;
|
|
clojure) echo "test/qr.clj" ;;
|
|
elixir) echo "test/qr.ex" ;;
|
|
erlang) echo "test/qr.erl" ;;
|
|
groovy) echo "test/qr.groovy" ;;
|
|
raku) echo "test/qr.raku" ;;
|
|
julia) echo "test/qr.jl" ;;
|
|
dart) echo "test/qr.dart" ;;
|
|
prolog) echo "test/qr.pro" ;;
|
|
forth) echo "test/qr.forth" ;;
|
|
powershell) echo "test/qr.ps1" ;;
|
|
objc) echo "test/qr.m" ;;
|
|
v) echo "test/qr.v" ;;
|
|
go) echo "test/qr.go" ;;
|
|
rust) echo "test/qr.rs" ;;
|
|
c) echo "test/qr.c" ;;
|
|
cpp) echo "test/qr.cpp" ;;
|
|
java) echo "test/qr.java" ;;
|
|
kotlin) echo "test/qr.kt" ;;
|
|
csharp) echo "test/qr.cs" ;;
|
|
dotnet) echo "test/qr.cs" ;;
|
|
fsharp) echo "test/qr.fs" ;;
|
|
haskell) echo "test/qr.hs" ;;
|
|
ocaml) echo "test/qr.ml" ;;
|
|
d) echo "test/qr.d" ;;
|
|
nim) echo "test/qr.nim" ;;
|
|
zig) echo "test/qr.zig" ;;
|
|
crystal) echo "test/qr.cr" ;;
|
|
fortran) echo "test/qr.f90" ;;
|
|
cobol) echo "test/qr.cob" ;;
|
|
deno) echo "test/qr.ts" ;;
|
|
*) echo "" ;;
|
|
esac
|
|
}
|
|
|
|
SDK_FILE=$(get_sdk_file "$LANG")
|
|
|
|
echo "=== Functional Tests: $LANG ==="
|
|
echo "SDK: $SDK_FILE"
|
|
echo ""
|
|
|
|
# Check if un CLI exists
|
|
if [ ! -x "build/un" ]; then
|
|
echo "ERROR: build/un not found. Run build-clients.sh first."
|
|
exit 1
|
|
fi
|
|
|
|
# Check if SDK file exists
|
|
if [ ! -f "$SDK_FILE" ]; then
|
|
echo "SKIP: SDK file not found: $SDK_FILE"
|
|
cat > "$RESULTS_DIR/test-results.xml" << EOF
|
|
<?xml version="1.0" encoding="UTF-8"?>
|
|
<testsuites>
|
|
<testsuite name="Functional Test" tests="1" skipped="1">
|
|
<testcase name="$LANG SDK" classname="un.$LANG">
|
|
<skipped message="SDK file not found: $SDK_FILE"/>
|
|
</testcase>
|
|
</testsuite>
|
|
</testsuites>
|
|
EOF
|
|
exit 0
|
|
fi
|
|
|
|
START_TIME=$(date +%s.%N)
|
|
|
|
# Track test results
|
|
declare -A TEST_RESULTS
|
|
TOTAL_TESTS=0
|
|
FAILURES=0
|
|
|
|
# Track transient errors for API health monitoring
|
|
# SCIENTIFIC INTEGRITY: We track these to understand API reliability over time
|
|
TOTAL_RETRIES=0
|
|
RETRIES_429=0 # Rate limiting
|
|
RETRIES_5XX=0 # Server errors
|
|
RETRIES_TIMEOUT=0 # Timeouts
|
|
RETRIES_CONN=0 # Connection issues
|
|
declare -A TEST_RETRIES # Track retries per test
|
|
|
|
# Helper to run test and record result
|
|
# Retries on ALL transient errors (429, 5xx, timeouts) with exponential backoff
|
|
# SCIENTIFIC INTEGRITY: Tests must PASS, FAIL, or SKIP - never lie with soft passes
|
|
run_test() {
|
|
local test_name="$1"
|
|
local test_cmd="$2"
|
|
local success_pattern="$3"
|
|
local output_file="$RESULTS_DIR/${test_name}.txt"
|
|
local max_retries=10
|
|
local retry_delay=2
|
|
local attempt=0
|
|
local retried=false
|
|
|
|
echo -n "Test: $test_name... "
|
|
TOTAL_TESTS=$((TOTAL_TESTS + 1))
|
|
|
|
while [ $attempt -lt $max_retries ]; do
|
|
if eval "$test_cmd" > "$output_file" 2>&1; then
|
|
if [ -n "$success_pattern" ]; then
|
|
if grep -qiE "$success_pattern" "$output_file"; then
|
|
TEST_RESULTS["$test_name"]="pass"
|
|
[ $attempt -gt 0 ] && TEST_RETRIES["$test_name"]=$attempt
|
|
if [ "$retried" = true ]; then
|
|
echo "PASS (after $attempt retries)"
|
|
else
|
|
echo "PASS"
|
|
fi
|
|
return 0
|
|
fi
|
|
else
|
|
# No pattern required, just check non-empty output
|
|
if [ -s "$output_file" ]; then
|
|
TEST_RESULTS["$test_name"]="pass"
|
|
[ $attempt -gt 0 ] && TEST_RETRIES["$test_name"]=$attempt
|
|
if [ "$retried" = true ]; then
|
|
echo "PASS (after $attempt retries)"
|
|
else
|
|
echo "PASS"
|
|
fi
|
|
return 0
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# Check for transient errors that should trigger retry
|
|
# Track error type for API health monitoring
|
|
local error_type=""
|
|
if grep -qiE "HTTP 429|concurrency_limit|too many.*concurrent|rate.limit" "$output_file" 2>/dev/null; then
|
|
error_type="429"
|
|
RETRIES_429=$((RETRIES_429 + 1))
|
|
elif grep -qiE "HTTP 5[0-9][0-9]|server error|internal error" "$output_file" 2>/dev/null; then
|
|
error_type="5xx"
|
|
RETRIES_5XX=$((RETRIES_5XX + 1))
|
|
elif grep -qiE "timeout|timed out|ETIMEDOUT" "$output_file" 2>/dev/null; then
|
|
error_type="timeout"
|
|
RETRIES_TIMEOUT=$((RETRIES_TIMEOUT + 1))
|
|
elif grep -qiE "connection refused|connection reset|ECONNREFUSED|request failed" "$output_file" 2>/dev/null; then
|
|
error_type="connection"
|
|
RETRIES_CONN=$((RETRIES_CONN + 1))
|
|
fi
|
|
|
|
if [ -n "$error_type" ]; then
|
|
attempt=$((attempt + 1))
|
|
retried=true
|
|
TOTAL_RETRIES=$((TOTAL_RETRIES + 1))
|
|
if [ $attempt -lt $max_retries ]; then
|
|
echo -n "($error_type retry $attempt/$max_retries)... "
|
|
sleep $retry_delay
|
|
# Exponential backoff, cap at 60 seconds
|
|
retry_delay=$((retry_delay * 2))
|
|
if [ $retry_delay -gt 60 ]; then
|
|
retry_delay=60
|
|
fi
|
|
continue
|
|
fi
|
|
fi
|
|
|
|
# Not a transient error, break out of retry loop
|
|
break
|
|
done
|
|
|
|
# If we exhausted retries on transient errors, that's still a FAIL
|
|
# We don't lie about test results - if we couldn't verify, it failed
|
|
TEST_RESULTS["$test_name"]="fail"
|
|
[ $attempt -gt 0 ] && TEST_RETRIES["$test_name"]=$attempt
|
|
FAILURES=$((FAILURES + 1))
|
|
if [ "$retried" = true ]; then
|
|
echo "FAIL (after $attempt retries)"
|
|
else
|
|
echo "FAIL"
|
|
fi
|
|
head -5 "$output_file" 2>/dev/null || echo "(no output)"
|
|
return 0 # Always return success to continue test execution
|
|
}
|
|
|
|
echo "Running functional tests..."
|
|
echo ""
|
|
|
|
HELLO_CODE=$(get_hello_code "$LANG")
|
|
|
|
# ============================================================================
|
|
# SECTION 1: Execute Endpoint Tests
|
|
# ============================================================================
|
|
echo "--- Execute Endpoint ---"
|
|
|
|
# Test 1.1: Basic code execution
|
|
run_test "exec_basic" \
|
|
"build/un -s '$LANG' '$HELLO_CODE'" \
|
|
"test-ok"
|
|
|
|
# Test 1.2: Execute with environment variable
|
|
run_test "exec_env" \
|
|
"build/un -s python -e TEST_VAR=hello123 'import os; print(os.environ.get(\"TEST_VAR\", \"missing\"))'" \
|
|
"hello123"
|
|
|
|
# Test 1.3: Execute with file upload (files go to /tmp/input/)
|
|
run_test "exec_file" \
|
|
"build/un -s python -f test/fib.py 'import os; print(os.listdir(\"/tmp/input\"))'" \
|
|
"fib"
|
|
|
|
# Test 1.4: Execute with network access (semitrusted)
|
|
run_test "exec_network" \
|
|
"build/un -n semitrusted -s python 'import socket; print(\"network-ok\")'" \
|
|
"network-ok"
|
|
|
|
# Test 1.5: Execute SDK file in sandbox (inception)
|
|
# Runs the SDK file inside unsandbox - tests that the interpreter can load/parse the SDK
|
|
# Expected: help/usage output, or interpreter errors (all acceptable)
|
|
run_test "exec_sdk_inception" \
|
|
"build/un -n semitrusted -e UNSANDBOX_PUBLIC_KEY=\$UNSANDBOX_PUBLIC_KEY -e UNSANDBOX_SECRET_KEY=\$UNSANDBOX_SECRET_KEY '$SDK_FILE' 2>&1 || echo 'attempted'" \
|
|
"usage|help|unsandbox|Usage|error|Error|attempted"
|
|
|
|
echo ""
|
|
|
|
# ============================================================================
|
|
# SECTION 2: Languages Endpoint Tests
|
|
# ============================================================================
|
|
echo "--- Languages Endpoint ---"
|
|
|
|
# Test 2.1: List all languages (one per line)
|
|
run_test "languages_list" \
|
|
"build/un languages" \
|
|
"python"
|
|
|
|
# Test 2.2: Verify our language is in the list
|
|
run_test "languages_contains_$LANG" \
|
|
"build/un languages" \
|
|
"$LANG"
|
|
|
|
echo ""
|
|
|
|
# ============================================================================
|
|
# SECTION 3: Session Lifecycle Tests
|
|
# ============================================================================
|
|
echo "--- Session Lifecycle ---"
|
|
|
|
# Test 3.1: List sessions (might be empty, that's ok)
|
|
# Note: Session creation is interactive, so we can only test listing
|
|
run_test "session_list" \
|
|
"build/un session --list" \
|
|
"unsb-session|no.*session|\[\]|sessions|Session"
|
|
|
|
# If there are existing sessions, test session info on first one
|
|
SESSION_ID=$(build/un session --list 2>/dev/null | grep -oE "unsb-session-[a-z0-9-]+" | head -1 || true)
|
|
if [ -n "$SESSION_ID" ]; then
|
|
# Test 3.2: Get session info
|
|
run_test "session_info" \
|
|
"build/un session --info '$SESSION_ID'" \
|
|
"$SESSION_ID|status|created"
|
|
fi
|
|
|
|
echo ""
|
|
|
|
# ============================================================================
|
|
# SECTION 4: Service Lifecycle Tests
|
|
# ============================================================================
|
|
echo "--- Service Lifecycle ---"
|
|
|
|
# Test 4.1: List services
|
|
run_test "service_list" \
|
|
"build/un service --list" \
|
|
"unsb-service|no.*service|\[\]|services"
|
|
|
|
# Test 4.2: Create a service with retry logic (note: syntax is --name not --create)
|
|
# SCIENTIFIC INTEGRITY: Retry on transient errors, fail if we can't create
|
|
SERVICE_NAME="test-$LANG-$$"
|
|
SERVICE_ID=""
|
|
service_attempt=0
|
|
service_max_retries=10
|
|
service_retry_delay=2
|
|
|
|
echo -n "Test: service_create... "
|
|
TOTAL_TESTS=$((TOTAL_TESTS + 1))
|
|
|
|
while [ $service_attempt -lt $service_max_retries ]; do
|
|
SERVICE_OUTPUT=$(build/un service --name "$SERVICE_NAME" --bootstrap "echo service-started" 2>&1 || true)
|
|
echo "$SERVICE_OUTPUT" > "$RESULTS_DIR/service_create.txt"
|
|
SERVICE_ID=$(echo "$SERVICE_OUTPUT" | grep -oE "unsb-service-[a-z0-9-]+" | head -1 || true)
|
|
|
|
if [ -n "$SERVICE_ID" ]; then
|
|
break
|
|
fi
|
|
|
|
# Check for transient errors that should trigger retry
|
|
# Track error type for API health monitoring
|
|
local svc_error_type=""
|
|
if grep -qiE "HTTP 429|concurrency_limit|rate.limit" "$RESULTS_DIR/service_create.txt" 2>/dev/null; then
|
|
svc_error_type="429"
|
|
RETRIES_429=$((RETRIES_429 + 1))
|
|
elif grep -qiE "HTTP 5[0-9][0-9]|server error|internal error" "$RESULTS_DIR/service_create.txt" 2>/dev/null; then
|
|
svc_error_type="5xx"
|
|
RETRIES_5XX=$((RETRIES_5XX + 1))
|
|
elif grep -qiE "timeout|timed out|ETIMEDOUT" "$RESULTS_DIR/service_create.txt" 2>/dev/null; then
|
|
svc_error_type="timeout"
|
|
RETRIES_TIMEOUT=$((RETRIES_TIMEOUT + 1))
|
|
elif grep -qiE "connection|ECONNREFUSED|limit|quota|no_pool" "$RESULTS_DIR/service_create.txt" 2>/dev/null; then
|
|
svc_error_type="connection"
|
|
RETRIES_CONN=$((RETRIES_CONN + 1))
|
|
fi
|
|
|
|
if [ -n "$svc_error_type" ]; then
|
|
service_attempt=$((service_attempt + 1))
|
|
TOTAL_RETRIES=$((TOTAL_RETRIES + 1))
|
|
if [ $service_attempt -lt $service_max_retries ]; then
|
|
echo -n "($svc_error_type retry $service_attempt/$service_max_retries)... "
|
|
sleep $service_retry_delay
|
|
service_retry_delay=$((service_retry_delay * 2))
|
|
if [ $service_retry_delay -gt 60 ]; then
|
|
service_retry_delay=60
|
|
fi
|
|
continue
|
|
fi
|
|
fi
|
|
break
|
|
done
|
|
|
|
if [ -n "$SERVICE_ID" ]; then
|
|
TEST_RESULTS["service_create"]="pass"
|
|
[ $service_attempt -gt 0 ] && TEST_RETRIES["service_create"]=$service_attempt
|
|
if [ $service_attempt -gt 0 ]; then
|
|
echo "PASS (created $SERVICE_ID after $service_attempt retries)"
|
|
else
|
|
echo "PASS (created $SERVICE_ID)"
|
|
fi
|
|
|
|
# Test 4.3: Get service info
|
|
run_test "service_info" \
|
|
"build/un service --info '$SERVICE_ID'" \
|
|
"$SERVICE_ID|status|name"
|
|
|
|
# Test 4.4: Execute command in service
|
|
run_test "service_exec" \
|
|
"build/un service --execute '$SERVICE_ID' 'echo service-exec-ok'" \
|
|
"service-exec-ok"
|
|
|
|
# Test 4.5: Get service logs
|
|
run_test "service_logs" \
|
|
"build/un service --logs '$SERVICE_ID'" \
|
|
"log|output|service|$SERVICE_ID"
|
|
|
|
# Test 4.6: Destroy service (cleanup)
|
|
run_test "service_destroy" \
|
|
"build/un service --destroy '$SERVICE_ID'" \
|
|
"destroy|deleted|success|$SERVICE_ID"
|
|
else
|
|
# SCIENTIFIC INTEGRITY: If we couldn't create a service after retries, that's a FAIL
|
|
TEST_RESULTS["service_create"]="fail"
|
|
FAILURES=$((FAILURES + 1))
|
|
if [ $service_attempt -gt 0 ]; then
|
|
echo "FAIL (after $service_attempt retries)"
|
|
else
|
|
echo "FAIL"
|
|
fi
|
|
head -5 "$RESULTS_DIR/service_create.txt"
|
|
fi
|
|
|
|
echo ""
|
|
|
|
# ============================================================================
|
|
# SECTION 5: Snapshot Endpoint Tests
|
|
# ============================================================================
|
|
echo "--- Snapshot Endpoint ---"
|
|
|
|
# Test 5.1: List snapshots
|
|
run_test "snapshot_list" \
|
|
"build/un snapshot --list" \
|
|
"unsb-snapshot|no.*snapshot|\[\]|snapshots"
|
|
|
|
echo ""
|
|
|
|
# ============================================================================
|
|
# SECTION 6: Image Endpoint Tests
|
|
# ============================================================================
|
|
echo "--- Image Endpoint ---"
|
|
|
|
# Test 6.1: List images
|
|
run_test "image_list" \
|
|
"build/un image --list" \
|
|
"unsb-image|no.*image|\[\]|images|name"
|
|
|
|
echo ""
|
|
|
|
# ============================================================================
|
|
# SECTION 7: Key Validation Tests
|
|
# ============================================================================
|
|
echo "--- Key Validation ---"
|
|
|
|
# Test 7.1: Validate API key
|
|
run_test "key_validate" \
|
|
"build/un key" \
|
|
"valid|key|account|unsb-pk"
|
|
|
|
echo ""
|
|
|
|
# ============================================================================
|
|
# SECTION 8: SDK-Specific Inception Tests
|
|
# ============================================================================
|
|
echo "--- SDK Inception ($LANG) ---"
|
|
|
|
# Test 8.1: SDK file exists and is readable
|
|
run_test "sdk_exists" \
|
|
"test -f '$SDK_FILE' && wc -l < '$SDK_FILE'" \
|
|
"[0-9]+"
|
|
|
|
# Test 8.2: Run SDK file directly (auto-detect language from extension)
|
|
# Note: Complex SDK files may fail (shebangs, deps, compile errors) - that's ok
|
|
# We accept any output showing the runtime attempted execution
|
|
run_test "sdk_runs" \
|
|
"build/un -n semitrusted -e UNSANDBOX_PUBLIC_KEY=\$UNSANDBOX_PUBLIC_KEY -e UNSANDBOX_SECRET_KEY=\$UNSANDBOX_SECRET_KEY '$SDK_FILE' 2>&1 || echo 'attempted'" \
|
|
"usage|help|unsandbox|un |version|error|Error|compile|undefined|attempted|unexpected|syntax|import|require|module|WARNING|domain|license"
|
|
|
|
echo ""
|
|
|
|
# ============================================================================
|
|
# SECTION 9: QR Code Generation Tests
|
|
# ============================================================================
|
|
echo "--- QR Code Generation ---"
|
|
|
|
QR_FILE=$(get_qr_file "$LANG")
|
|
if [ -n "$QR_FILE" ] && [ -f "$QR_FILE" ]; then
|
|
# Test 9.1: QR code generation produces structured output
|
|
# Pass file path without -s: un CLI reads the file and auto-detects language
|
|
# (with -s, the positional arg is treated as inline code, not a file path)
|
|
run_test "qr_generate" \
|
|
"build/un '$QR_FILE'" \
|
|
"QR:unsandbox-qr-ok:ROWS:[0-9]+"
|
|
else
|
|
echo "SKIP: No QR test file for $LANG"
|
|
fi
|
|
|
|
echo ""
|
|
|
|
# ============================================================================
|
|
# Results Summary
|
|
# ============================================================================
|
|
END_TIME=$(date +%s.%N)
|
|
DURATION=$(echo "$END_TIME - $START_TIME" | bc 2>/dev/null || echo "0")
|
|
|
|
PASSED=$((TOTAL_TESTS - FAILURES))
|
|
if [ "$FAILURES" -eq 0 ]; then
|
|
STATUS="PASS"
|
|
else
|
|
STATUS="FAIL"
|
|
fi
|
|
|
|
echo "============================================"
|
|
echo "=== Result: $STATUS ($PASSED/$TOTAL_TESTS passed, ${DURATION}s) ==="
|
|
echo "============================================"
|
|
|
|
# ============================================================================
|
|
# API Health Report - Track transient errors for monitoring
|
|
# ============================================================================
|
|
TESTS_WITH_RETRIES=${#TEST_RETRIES[@]}
|
|
|
|
echo ""
|
|
echo "--- API Health Report ---"
|
|
echo "Total retries: $TOTAL_RETRIES"
|
|
echo " Rate limit (429): $RETRIES_429"
|
|
echo " Server error (5xx): $RETRIES_5XX"
|
|
echo " Timeout: $RETRIES_TIMEOUT"
|
|
echo " Connection: $RETRIES_CONN"
|
|
echo "Tests that needed retries: $TESTS_WITH_RETRIES"
|
|
|
|
if [ $TESTS_WITH_RETRIES -gt 0 ]; then
|
|
echo "Tests with retries:"
|
|
for test_name in "${!TEST_RETRIES[@]}"; do
|
|
retries="${TEST_RETRIES[$test_name]}"
|
|
result="${TEST_RESULTS[$test_name]}"
|
|
echo " - $test_name: $retries retries ($result)"
|
|
done
|
|
fi
|
|
|
|
# Write JSON report for aggregation across all SDKs
|
|
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
|
cat > "$RESULTS_DIR/api-health.json" << EOF
|
|
{
|
|
"timestamp": "$TIMESTAMP",
|
|
"language": "$LANG",
|
|
"duration_seconds": $DURATION,
|
|
"tests": {
|
|
"total": $TOTAL_TESTS,
|
|
"passed": $PASSED,
|
|
"failed": $FAILURES
|
|
},
|
|
"retries": {
|
|
"total": $TOTAL_RETRIES,
|
|
"rate_limit_429": $RETRIES_429,
|
|
"server_error_5xx": $RETRIES_5XX,
|
|
"timeout": $RETRIES_TIMEOUT,
|
|
"connection": $RETRIES_CONN
|
|
},
|
|
"tests_with_retries": $TESTS_WITH_RETRIES,
|
|
"api_health_score": $(echo "scale=2; 100 - ($TOTAL_RETRIES * 5)" | bc 2>/dev/null || echo "100")
|
|
}
|
|
EOF
|
|
|
|
echo ""
|
|
|
|
# Generate JUnit XML
|
|
cat > "$RESULTS_DIR/test-results.xml" << EOF
|
|
<?xml version="1.0" encoding="UTF-8"?>
|
|
<testsuites>
|
|
<testsuite name="Functional Test - $LANG" tests="$TOTAL_TESTS" failures="$FAILURES" time="$DURATION">
|
|
EOF
|
|
|
|
for test_name in "${!TEST_RESULTS[@]}"; do
|
|
result="${TEST_RESULTS[$test_name]}"
|
|
cat >> "$RESULTS_DIR/test-results.xml" << EOF
|
|
<testcase name="$test_name" classname="un.$LANG">
|
|
EOF
|
|
if [ "$result" = "pass" ]; then
|
|
echo " <system-out>Test passed</system-out>" >> "$RESULTS_DIR/test-results.xml"
|
|
else
|
|
echo " <failure message=\"Test failed\">See ${test_name}.txt for details</failure>" >> "$RESULTS_DIR/test-results.xml"
|
|
fi
|
|
echo " </testcase>" >> "$RESULTS_DIR/test-results.xml"
|
|
done
|
|
|
|
cat >> "$RESULTS_DIR/test-results.xml" << EOF
|
|
</testsuite>
|
|
</testsuites>
|
|
EOF
|
|
|
|
# Exit with appropriate code
|
|
[ "$STATUS" = "PASS" ] && exit 0 || exit 1
|