russell@unturf.com
cf5530875c
- detect-changes: VERSION/Makefile changes trigger full test matrix - test-sdk: fix service_destroy treating "not found" as success - test-sdk: skip qr_generate on missing deps instead of failing - test-sdk: increase backoff cap from 10s to 30s for overloaded pools - test-sdk: increase no-match retry from 3 to 5 with longer delay - test-sdk: detect nginx error messages (bad gateway, etc) as 5xx - test-sdk: show HTTP status and error details on test failure
716 lines
28 KiB
Bash
Executable file
716 lines
28 KiB
Bash
Executable file
#!/bin/bash
|
|
# Inception test: Use un (C CLI) to test SDKs AND API endpoints through unsandbox
|
|
#
|
|
# Pattern: build/un → unsandbox API → SDK/code → unsandbox API → result
|
|
#
|
|
# Tests:
|
|
# 1. SDK loads and runs in sandbox
|
|
# 2. Execute endpoint with various flags
|
|
# 3. Languages endpoint
|
|
# 4. Sessions lifecycle (create, list, execute, destroy)
|
|
# 5. Services lifecycle (create, list, info, logs, destroy)
|
|
# 6. Snapshots endpoint
|
|
# 7. Images endpoint
|
|
# 8. Key validation
|
|
# 9. QR code generation (native library or qrencode CLI fallback)
|
|
#
|
|
# Usage: test-sdk.sh LANGUAGE
|
|
|
|
# Don't use set -e - we need to continue on test failures to generate report
|
|
# set -e
|
|
|
|
LANG=${1:-python}
|
|
RESULTS_DIR="test-results-$LANG"
|
|
mkdir -p "$RESULTS_DIR"
|
|
|
|
# Map API language name to SDK file path
|
|
get_sdk_file() {
|
|
case "$1" in
|
|
python) echo "clients/python/sync/src/un.py" ;;
|
|
javascript) echo "clients/javascript/sync/src/un.js" ;;
|
|
typescript) echo "clients/typescript/sync/src/un.ts" ;;
|
|
ruby) echo "clients/ruby/sync/src/un.rb" ;;
|
|
php) echo "clients/php/sync/src/un.php" ;;
|
|
perl) echo "clients/perl/sync/src/un.pl" ;;
|
|
lua) echo "clients/lua/sync/src/un.lua" ;;
|
|
bash) echo "clients/bash/sync/src/un.sh" ;;
|
|
r) echo "clients/r/sync/src/un.r" ;;
|
|
awk) echo "clients/awk/sync/src/un.awk" ;;
|
|
tcl) echo "clients/tcl/sync/src/un.tcl" ;;
|
|
scheme) echo "clients/scheme/sync/src/un.scm" ;;
|
|
commonlisp) echo "clients/lisp/sync/src/un.lisp" ;;
|
|
lisp) echo "clients/lisp/sync/src/un.lisp" ;;
|
|
clojure) echo "clients/clojure/sync/src/un.clj" ;;
|
|
elixir) echo "clients/elixir/sync/src/un.ex" ;;
|
|
erlang) echo "clients/erlang/sync/src/un.erl" ;;
|
|
groovy) echo "clients/groovy/sync/src/un.groovy" ;;
|
|
raku) echo "clients/raku/sync/src/un.raku" ;;
|
|
julia) echo "clients/julia/sync/src/un.jl" ;;
|
|
dart) echo "clients/dart/sync/src/un.dart" ;;
|
|
prolog) echo "clients/prolog/sync/src/un.pro" ;;
|
|
forth) echo "clients/forth/sync/src/un.forth" ;;
|
|
powershell) echo "clients/powershell/sync/src/un.ps1" ;;
|
|
objc) echo "clients/objective-c/sync/src/un.m" ;;
|
|
v) echo "clients/v/sync/src/un.v" ;;
|
|
go) echo "clients/go/sync/src/un.go" ;;
|
|
rust) echo "clients/rust/sync/src/lib.rs" ;;
|
|
c) echo "clients/c/src/un.c" ;;
|
|
cpp) echo "clients/cpp/sync/src/un.cpp" ;;
|
|
java) echo "clients/java/sync/src/Un.java" ;;
|
|
kotlin) echo "clients/kotlin/sync/src/un.kt" ;;
|
|
swift) echo "clients/swift/sync/src/un.swift" ;;
|
|
csharp) echo "clients/csharp/sync/src/Un.cs" ;;
|
|
dotnet) echo "clients/dotnet/sync/src/Un.cs" ;;
|
|
fsharp) echo "clients/fsharp/sync/src/un.fs" ;;
|
|
haskell) echo "clients/haskell/sync/src/un.hs" ;;
|
|
ocaml) echo "clients/ocaml/sync/src/un.ml" ;;
|
|
d) echo "clients/d/sync/src/un.d" ;;
|
|
nim) echo "clients/nim/sync/src/un.nim" ;;
|
|
zig) echo "clients/zig/sync/src/un.zig" ;;
|
|
crystal) echo "clients/crystal/sync/src/un.cr" ;;
|
|
fortran) echo "clients/fortran/sync/src/un.f90" ;;
|
|
cobol) echo "clients/cobol/sync/src/un.cob" ;;
|
|
deno) echo "clients/typescript/sync/src/un.ts" ;;
|
|
*) echo "clients/$1/sync/src/un.NOTFOUND" ;;
|
|
esac
|
|
}
|
|
|
|
# Language-specific "hello world" commands
|
|
get_hello_code() {
|
|
case "$1" in
|
|
python|python3) echo 'print("test-ok")' ;;
|
|
javascript|typescript|deno) echo 'console.log("test-ok")' ;;
|
|
ruby) echo 'puts "test-ok"' ;;
|
|
php) echo '<?php echo "test-ok";' ;;
|
|
perl|raku) echo 'print "test-ok\n";' ;;
|
|
lua) echo 'print("test-ok")' ;;
|
|
bash) echo 'echo "test-ok"' ;;
|
|
r) echo 'cat("test-ok\n")' ;;
|
|
awk) echo 'BEGIN { print "test-ok" }' ;;
|
|
tcl) echo 'puts "test-ok"' ;;
|
|
scheme) echo '(display "test-ok") (newline)' ;;
|
|
commonlisp|lisp) echo '(format t "test-ok~%")' ;;
|
|
clojure) echo '(println "test-ok")' ;;
|
|
elixir) echo 'IO.puts "test-ok"' ;;
|
|
erlang) echo '-module(main). -export([start/0]). start() -> io:format("test-ok~n"), halt().' ;;
|
|
groovy) echo 'println "test-ok"' ;;
|
|
julia) echo 'println("test-ok")' ;;
|
|
prolog) echo ':- write("test-ok"), nl, halt.' ;;
|
|
forth) echo '.( test-ok) cr bye' ;;
|
|
powershell) echo 'Write-Output "test-ok"' ;;
|
|
go) echo 'package main; import "fmt"; func main() { fmt.Println("test-ok") }' ;;
|
|
rust) echo 'fn main() { println!("test-ok"); }' ;;
|
|
c) echo '#include <stdio.h>
|
|
int main() { printf("test-ok\n"); return 0; }' ;;
|
|
cpp) echo '#include <iostream>
|
|
int main() { std::cout << "test-ok" << std::endl; return 0; }' ;;
|
|
java) echo 'public class Main { public static void main(String[] args) { System.out.println("test-ok"); } }' ;;
|
|
kotlin) echo 'fun main() { println("test-ok") }' ;;
|
|
swift) echo 'print("test-ok")' ;;
|
|
csharp) echo 'class P{static void Main(){System.Console.WriteLine("test-ok");}}' ;;
|
|
dotnet) echo 'Console.WriteLine("test-ok");' ;;
|
|
fsharp) echo 'printfn "test-ok"' ;;
|
|
haskell) echo 'main = putStrLn "test-ok"' ;;
|
|
ocaml) echo 'print_endline "test-ok"' ;;
|
|
d) echo 'import std.stdio; void main() { writeln("test-ok"); }' ;;
|
|
nim) echo 'echo "test-ok"' ;;
|
|
zig) echo 'const std = @import("std"); pub fn main() void { std.debug.print("test-ok\n", .{}); }' ;;
|
|
crystal) echo 'puts "test-ok"' ;;
|
|
fortran) echo 'program main
|
|
print *, "test-ok"
|
|
end program main' ;;
|
|
cobol) echo 'IDENTIFICATION DIVISION. PROGRAM-ID. HELLO. PROCEDURE DIVISION. DISPLAY "test-ok". STOP RUN.' ;;
|
|
objc) echo '#import <Foundation/Foundation.h>
|
|
int main() { NSLog(@"test-ok"); return 0; }' ;;
|
|
dart) echo 'void main() { print("test-ok"); }' ;;
|
|
v) echo 'fn main() { println("test-ok") }' ;;
|
|
*) echo 'print("test-ok")' ;;
|
|
esac
|
|
}
|
|
|
|
# Map language to QR code test file
|
|
get_qr_file() {
|
|
case "$1" in
|
|
python) echo "test/qr.py" ;;
|
|
javascript) echo "test/qr.js" ;;
|
|
typescript) echo "test/qr.ts" ;;
|
|
ruby) echo "test/qr.rb" ;;
|
|
php) echo "test/qr.php" ;;
|
|
perl) echo "test/qr.pl" ;;
|
|
lua) echo "test/qr.lua" ;;
|
|
bash) echo "test/qr.sh" ;;
|
|
r) echo "test/qr.r" ;;
|
|
awk) echo "test/qr.awk" ;;
|
|
tcl) echo "test/qr.tcl" ;;
|
|
scheme) echo "test/qr.scm" ;;
|
|
commonlisp) echo "test/qr.lisp" ;;
|
|
lisp) echo "test/qr.lisp" ;;
|
|
clojure) echo "test/qr.clj" ;;
|
|
elixir) echo "test/qr.ex" ;;
|
|
erlang) echo "test/qr.erl" ;;
|
|
groovy) echo "test/qr.groovy" ;;
|
|
raku) echo "test/qr.raku" ;;
|
|
julia) echo "test/qr.jl" ;;
|
|
dart) echo "test/qr.dart" ;;
|
|
prolog) echo "test/qr.pro" ;;
|
|
forth) echo "test/qr.forth" ;;
|
|
powershell) echo "test/qr.ps1" ;;
|
|
objc) echo "test/qr.m" ;;
|
|
v) echo "test/qr.v" ;;
|
|
go) echo "test/qr.go" ;;
|
|
rust) echo "test/qr.rs" ;;
|
|
c) echo "test/qr.c" ;;
|
|
cpp) echo "test/qr.cpp" ;;
|
|
java) echo "test/qr.java" ;;
|
|
kotlin) echo "test/qr.kt" ;;
|
|
csharp) echo "test/qr.cs" ;;
|
|
dotnet) echo "test/qr.cs" ;;
|
|
fsharp) echo "test/qr.fs" ;;
|
|
haskell) echo "test/qr.hs" ;;
|
|
ocaml) echo "test/qr.ml" ;;
|
|
d) echo "test/qr.d" ;;
|
|
nim) echo "test/qr.nim" ;;
|
|
zig) echo "test/qr.zig" ;;
|
|
crystal) echo "test/qr.cr" ;;
|
|
fortran) echo "test/qr.f90" ;;
|
|
cobol) echo "test/qr.cob" ;;
|
|
deno) echo "test/qr.ts" ;;
|
|
*) echo "" ;;
|
|
esac
|
|
}
|
|
|
|
SDK_FILE=$(get_sdk_file "$LANG")
|
|
|
|
echo "=== Functional Tests: $LANG ==="
|
|
echo "SDK: $SDK_FILE"
|
|
echo ""
|
|
|
|
# Check if un CLI exists
|
|
if [ ! -x "build/un" ]; then
|
|
echo "ERROR: build/un not found. Run build-clients.sh first."
|
|
exit 1
|
|
fi
|
|
|
|
# Check if SDK file exists
|
|
if [ ! -f "$SDK_FILE" ]; then
|
|
echo "SKIP: SDK file not found: $SDK_FILE"
|
|
cat > "$RESULTS_DIR/test-results.xml" << EOF
|
|
<?xml version="1.0" encoding="UTF-8"?>
|
|
<testsuites>
|
|
<testsuite name="Functional Test" tests="1" skipped="1">
|
|
<testcase name="$LANG SDK" classname="un.$LANG">
|
|
<skipped message="SDK file not found: $SDK_FILE"/>
|
|
</testcase>
|
|
</testsuite>
|
|
</testsuites>
|
|
EOF
|
|
exit 0
|
|
fi
|
|
|
|
START_TIME=$(date +%s.%N)
|
|
|
|
# Track test results
|
|
declare -A TEST_RESULTS
|
|
TOTAL_TESTS=0
|
|
FAILURES=0
|
|
|
|
# Track transient errors for API health monitoring
|
|
# SCIENTIFIC INTEGRITY: We track these to understand API reliability over time
|
|
TOTAL_RETRIES=0
|
|
RETRIES_429=0 # Rate limiting
|
|
RETRIES_5XX=0 # Server errors
|
|
RETRIES_TIMEOUT=0 # Timeouts
|
|
RETRIES_CONN=0 # Connection issues
|
|
declare -A TEST_RETRIES # Track retries per test
|
|
|
|
# Helper to run test and record result
|
|
# Retries on ALL transient errors (429, 5xx, timeouts) with exponential backoff
|
|
# SCIENTIFIC INTEGRITY: Tests must PASS, FAIL, or SKIP - never lie with soft passes
|
|
run_test() {
|
|
local test_name="$1"
|
|
local test_cmd="$2"
|
|
local success_pattern="$3"
|
|
local output_file="$RESULTS_DIR/${test_name}.txt"
|
|
local max_retries=10
|
|
local retry_delay=2
|
|
local attempt=0
|
|
local retried=false
|
|
|
|
echo -n "Test: $test_name... "
|
|
TOTAL_TESTS=$((TOTAL_TESTS + 1))
|
|
|
|
while [ $attempt -lt $max_retries ]; do
|
|
if eval "$test_cmd" > "$output_file" 2>&1; then
|
|
if [ -n "$success_pattern" ]; then
|
|
if grep -qiE "$success_pattern" "$output_file"; then
|
|
TEST_RESULTS["$test_name"]="pass"
|
|
[ $attempt -gt 0 ] && TEST_RETRIES["$test_name"]=$attempt
|
|
if [ "$retried" = true ]; then
|
|
echo "PASS (after $attempt retries)"
|
|
else
|
|
echo "PASS"
|
|
fi
|
|
return 0
|
|
fi
|
|
else
|
|
# No pattern required, just check non-empty output
|
|
if [ -s "$output_file" ]; then
|
|
TEST_RESULTS["$test_name"]="pass"
|
|
[ $attempt -gt 0 ] && TEST_RETRIES["$test_name"]=$attempt
|
|
if [ "$retried" = true ]; then
|
|
echo "PASS (after $attempt retries)"
|
|
else
|
|
echo "PASS"
|
|
fi
|
|
return 0
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# Check for transient errors that should trigger retry
|
|
# Track error type for API health monitoring
|
|
local error_type=""
|
|
if grep -qiE "HTTP 429|concurrency_limit|too many.*concurrent|rate.limit" "$output_file" 2>/dev/null; then
|
|
error_type="429"
|
|
RETRIES_429=$((RETRIES_429 + 1))
|
|
elif grep -qiE "HTTP 5[0-9][0-9]|server error|internal error|bad gateway|service unavailable|gateway timeout" "$output_file" 2>/dev/null; then
|
|
error_type="5xx"
|
|
RETRIES_5XX=$((RETRIES_5XX + 1))
|
|
elif grep -qiE "timeout|timed out|ETIMEDOUT" "$output_file" 2>/dev/null; then
|
|
error_type="timeout"
|
|
RETRIES_TIMEOUT=$((RETRIES_TIMEOUT + 1))
|
|
elif grep -qiE "connection refused|connection reset|ECONNREFUSED|request failed" "$output_file" 2>/dev/null; then
|
|
error_type="connection"
|
|
RETRIES_CONN=$((RETRIES_CONN + 1))
|
|
fi
|
|
|
|
if [ -n "$error_type" ]; then
|
|
attempt=$((attempt + 1))
|
|
retried=true
|
|
TOTAL_RETRIES=$((TOTAL_RETRIES + 1))
|
|
if [ $attempt -lt $max_retries ]; then
|
|
echo -n "($error_type retry $attempt/$max_retries)... "
|
|
sleep $retry_delay
|
|
# Exponential backoff, cap at 30 seconds
|
|
retry_delay=$((retry_delay * 2))
|
|
if [ $retry_delay -gt 30 ]; then
|
|
retry_delay=30
|
|
fi
|
|
continue
|
|
fi
|
|
else
|
|
# No error detected, but output didn't match expected pattern
|
|
# This can happen when API returns 200 but service isn't ready yet
|
|
# Retry a few times for service operations
|
|
if [ $attempt -lt 5 ]; then
|
|
attempt=$((attempt + 1))
|
|
retried=true
|
|
echo -n "(no-match retry $attempt/5)... "
|
|
sleep 3
|
|
continue
|
|
fi
|
|
fi
|
|
|
|
# Exhausted retries, break out of loop
|
|
break
|
|
done
|
|
|
|
# If we exhausted retries on transient errors, that's still a FAIL
|
|
# We don't lie about test results - if we couldn't verify, it failed
|
|
TEST_RESULTS["$test_name"]="fail"
|
|
[ $attempt -gt 0 ] && TEST_RETRIES["$test_name"]=$attempt
|
|
FAILURES=$((FAILURES + 1))
|
|
if [ "$retried" = true ]; then
|
|
echo "FAIL (after $attempt retries)"
|
|
else
|
|
echo "FAIL"
|
|
fi
|
|
# Show full error context: HTTP status, error messages, and first lines of output
|
|
echo " --- Failure output ($test_name) ---"
|
|
grep -iE "HTTP [0-9]{3}|error|Error|status|failed|refused|timeout" "$output_file" 2>/dev/null | head -5 || true
|
|
echo " --- First 5 lines ---"
|
|
head -5 "$output_file" 2>/dev/null || echo "(no output)"
|
|
echo " ---"
|
|
return 0 # Always return success to continue test execution
|
|
}
|
|
|
|
echo "Running functional tests..."
|
|
echo ""
|
|
|
|
HELLO_CODE=$(get_hello_code "$LANG")
|
|
|
|
# ============================================================================
|
|
# SECTION 1: Execute Endpoint Tests
|
|
# ============================================================================
|
|
echo "--- Execute Endpoint ---"
|
|
|
|
# Test 1.1: Basic code execution
|
|
run_test "exec_basic" \
|
|
"build/un -s '$LANG' '$HELLO_CODE'" \
|
|
"test-ok"
|
|
|
|
# Test 1.2: Execute with environment variable
|
|
run_test "exec_env" \
|
|
"build/un -s python -e TEST_VAR=hello123 'import os; print(os.environ.get(\"TEST_VAR\", \"missing\"))'" \
|
|
"hello123"
|
|
|
|
# Test 1.3: Execute with file upload (files go to /tmp/input/)
|
|
run_test "exec_file" \
|
|
"build/un -s python -f test/fib.py 'import os; print(os.listdir(\"/tmp/input\"))'" \
|
|
"fib"
|
|
|
|
# Test 1.4: Execute with network access (semitrusted)
|
|
run_test "exec_network" \
|
|
"build/un -n semitrusted -s python 'import socket; print(\"network-ok\")'" \
|
|
"network-ok"
|
|
|
|
# Test 1.5: Execute SDK file in sandbox (inception)
|
|
# Runs the SDK file inside unsandbox - tests that the interpreter can load/parse the SDK
|
|
# Expected: help/usage output, or interpreter errors (all acceptable)
|
|
run_test "exec_sdk_inception" \
|
|
"build/un -n semitrusted -e UNSANDBOX_PUBLIC_KEY=\$UNSANDBOX_PUBLIC_KEY -e UNSANDBOX_SECRET_KEY=\$UNSANDBOX_SECRET_KEY '$SDK_FILE' 2>&1 || echo 'attempted'" \
|
|
"usage|help|unsandbox|Usage|error|Error|attempted"
|
|
|
|
echo ""
|
|
|
|
# ============================================================================
|
|
# SECTION 2: Languages Endpoint Tests
|
|
# ============================================================================
|
|
echo "--- Languages Endpoint ---"
|
|
|
|
# Test 2.1: List all languages (one per line)
|
|
run_test "languages_list" \
|
|
"build/un languages" \
|
|
"python"
|
|
|
|
# Test 2.2: Verify our language is in the list
|
|
run_test "languages_contains_$LANG" \
|
|
"build/un languages" \
|
|
"$LANG"
|
|
|
|
echo ""
|
|
|
|
# ============================================================================
|
|
# SECTION 3: Session Lifecycle Tests
|
|
# ============================================================================
|
|
echo "--- Session Lifecycle ---"
|
|
|
|
# Test 3.1: List sessions (might be empty, that's ok)
|
|
# Note: Session creation is interactive, so we can only test listing
|
|
run_test "session_list" \
|
|
"build/un session --list" \
|
|
"unsb-session|no.*session|\[\]|sessions|Session"
|
|
|
|
# If there are existing sessions, test session info on first one
|
|
SESSION_ID=$(build/un session --list 2>/dev/null | grep -oE "unsb-session-[a-z0-9-]+" | head -1 || true)
|
|
if [ -n "$SESSION_ID" ]; then
|
|
# Test 3.2: Get session info
|
|
run_test "session_info" \
|
|
"build/un session --info '$SESSION_ID'" \
|
|
"$SESSION_ID|status|created"
|
|
fi
|
|
|
|
echo ""
|
|
|
|
# ============================================================================
|
|
# SECTION 4: Service Lifecycle Tests
|
|
# ============================================================================
|
|
echo "--- Service Lifecycle ---"
|
|
|
|
# Test 4.1: List services
|
|
run_test "service_list" \
|
|
"build/un service --list" \
|
|
"unsb-service|no.*service|\[\]|services"
|
|
|
|
# Test 4.2: Create a service with retry logic (note: syntax is --name not --create)
|
|
# SCIENTIFIC INTEGRITY: Retry on transient errors, fail if we can't create
|
|
# Use a single name per test run - if 5xx hides a successful create, we detect via 409
|
|
SERVICE_NAME="test-$LANG-$$"
|
|
SERVICE_ID=""
|
|
service_attempt=0
|
|
service_max_retries=5
|
|
service_retry_delay=2
|
|
|
|
echo -n "Test: service_create... "
|
|
TOTAL_TESTS=$((TOTAL_TESTS + 1))
|
|
|
|
while [ $service_attempt -lt $service_max_retries ]; do
|
|
SERVICE_OUTPUT=$(build/un service --name "$SERVICE_NAME" --bootstrap "echo service-started" 2>&1 || true)
|
|
echo "$SERVICE_OUTPUT" > "$RESULTS_DIR/service_create.txt"
|
|
SERVICE_ID=$(echo "$SERVICE_OUTPUT" | grep -oE "unsb-service-[a-z0-9-]+" | head -1 || true)
|
|
|
|
if [ -n "$SERVICE_ID" ]; then
|
|
break
|
|
fi
|
|
|
|
# Check for transient errors that should trigger retry
|
|
# Track error type for API health monitoring
|
|
svc_error_type=""
|
|
if grep -qiE "HTTP 429|concurrency_limit|rate.limit" "$RESULTS_DIR/service_create.txt" 2>/dev/null; then
|
|
svc_error_type="429"
|
|
RETRIES_429=$((RETRIES_429 + 1))
|
|
elif grep -qiE "HTTP 409|already taken|already exists" "$RESULTS_DIR/service_create.txt" 2>/dev/null; then
|
|
# 409 means a previous attempt actually created it - look up the ID
|
|
svc_error_type="409"
|
|
echo -n "(409 - looking up existing)... "
|
|
SVC_LOOKUP=$(build/un service --list 2>/dev/null | grep "$SERVICE_NAME" || true)
|
|
SERVICE_ID=$(echo "$SVC_LOOKUP" | grep -oE "unsb-service-[a-z0-9-]+" | head -1 || true)
|
|
if [ -n "$SERVICE_ID" ]; then
|
|
break
|
|
fi
|
|
elif grep -qiE "HTTP 5[0-9][0-9]|server error|internal error" "$RESULTS_DIR/service_create.txt" 2>/dev/null; then
|
|
svc_error_type="5xx"
|
|
RETRIES_5XX=$((RETRIES_5XX + 1))
|
|
elif grep -qiE "timeout|timed out|ETIMEDOUT" "$RESULTS_DIR/service_create.txt" 2>/dev/null; then
|
|
svc_error_type="timeout"
|
|
RETRIES_TIMEOUT=$((RETRIES_TIMEOUT + 1))
|
|
elif grep -qiE "connection|ECONNREFUSED|limit|quota|no_pool" "$RESULTS_DIR/service_create.txt" 2>/dev/null; then
|
|
svc_error_type="connection"
|
|
RETRIES_CONN=$((RETRIES_CONN + 1))
|
|
fi
|
|
|
|
if [ -n "$svc_error_type" ]; then
|
|
service_attempt=$((service_attempt + 1))
|
|
TOTAL_RETRIES=$((TOTAL_RETRIES + 1))
|
|
if [ $service_attempt -lt $service_max_retries ]; then
|
|
echo -n "($svc_error_type retry $service_attempt/$service_max_retries)... "
|
|
sleep $service_retry_delay
|
|
service_retry_delay=$((service_retry_delay * 2))
|
|
if [ $service_retry_delay -gt 10 ]; then
|
|
service_retry_delay=10
|
|
fi
|
|
continue
|
|
fi
|
|
fi
|
|
break
|
|
done
|
|
|
|
if [ -n "$SERVICE_ID" ]; then
|
|
TEST_RESULTS["service_create"]="pass"
|
|
[ $service_attempt -gt 0 ] && TEST_RETRIES["service_create"]=$service_attempt
|
|
if [ $service_attempt -gt 0 ]; then
|
|
echo "PASS (created $SERVICE_ID after $service_attempt retries)"
|
|
else
|
|
echo "PASS (created $SERVICE_ID)"
|
|
fi
|
|
|
|
# Test 4.3: Get service info
|
|
run_test "service_info" \
|
|
"build/un service --info '$SERVICE_ID'" \
|
|
"$SERVICE_ID|status|name"
|
|
|
|
# Test 4.4: Execute command in service
|
|
run_test "service_exec" \
|
|
"build/un service --execute '$SERVICE_ID' 'echo service-exec-ok'" \
|
|
"service-exec-ok"
|
|
|
|
# Test 4.5: Get service logs
|
|
run_test "service_logs" \
|
|
"build/un service --logs '$SERVICE_ID'" \
|
|
"log|output|service|$SERVICE_ID"
|
|
|
|
# Test 4.6: Destroy service (cleanup)
|
|
# "not found" counts as success - service was already destroyed (e.g., by a 5xx that hid success)
|
|
run_test "service_destroy" \
|
|
"build/un service --destroy '$SERVICE_ID'" \
|
|
"destroy|deleted|success|not found|Not found|$SERVICE_ID"
|
|
else
|
|
# SCIENTIFIC INTEGRITY: If we couldn't create a service after retries, that's a FAIL
|
|
TEST_RESULTS["service_create"]="fail"
|
|
FAILURES=$((FAILURES + 1))
|
|
if [ $service_attempt -gt 0 ]; then
|
|
echo "FAIL (after $service_attempt retries)"
|
|
else
|
|
echo "FAIL"
|
|
fi
|
|
head -5 "$RESULTS_DIR/service_create.txt"
|
|
fi
|
|
|
|
echo ""
|
|
|
|
# ============================================================================
|
|
# SECTION 5: Snapshot Endpoint Tests
|
|
# ============================================================================
|
|
echo "--- Snapshot Endpoint ---"
|
|
|
|
# Test 5.1: List snapshots
|
|
run_test "snapshot_list" \
|
|
"build/un snapshot --list" \
|
|
"unsb-snapshot|no.*snapshot|\[\]|snapshots"
|
|
|
|
echo ""
|
|
|
|
# ============================================================================
|
|
# SECTION 6: Image Endpoint Tests
|
|
# ============================================================================
|
|
echo "--- Image Endpoint ---"
|
|
|
|
# Test 6.1: List images
|
|
run_test "image_list" \
|
|
"build/un image --list" \
|
|
"unsb-image|no.*image|\[\]|images|name"
|
|
|
|
echo ""
|
|
|
|
# ============================================================================
|
|
# SECTION 7: Key Validation Tests
|
|
# ============================================================================
|
|
echo "--- Key Validation ---"
|
|
|
|
# Test 7.1: Validate API key
|
|
run_test "key_validate" \
|
|
"build/un key" \
|
|
"valid|key|account|unsb-pk"
|
|
|
|
echo ""
|
|
|
|
# ============================================================================
|
|
# SECTION 8: SDK-Specific Inception Tests
|
|
# ============================================================================
|
|
echo "--- SDK Inception ($LANG) ---"
|
|
|
|
# Test 8.1: SDK file exists and is readable
|
|
run_test "sdk_exists" \
|
|
"test -f '$SDK_FILE' && wc -l < '$SDK_FILE'" \
|
|
"[0-9]+"
|
|
|
|
# Test 8.2: Run SDK file directly (auto-detect language from extension)
|
|
# Note: Complex SDK files may fail (shebangs, deps, compile errors) - that's ok
|
|
# We accept any output showing the runtime attempted execution
|
|
run_test "sdk_runs" \
|
|
"build/un -n semitrusted -e UNSANDBOX_PUBLIC_KEY=\$UNSANDBOX_PUBLIC_KEY -e UNSANDBOX_SECRET_KEY=\$UNSANDBOX_SECRET_KEY '$SDK_FILE' 2>&1 || echo 'attempted'" \
|
|
"usage|help|unsandbox|un |version|error|Error|compile|undefined|attempted|unexpected|syntax|import|require|module|WARNING|domain|license"
|
|
|
|
echo ""
|
|
|
|
# ============================================================================
|
|
# SECTION 9: QR Code Generation Tests
|
|
# ============================================================================
|
|
echo "--- QR Code Generation ---"
|
|
|
|
QR_FILE=$(get_qr_file "$LANG")
|
|
if [ -n "$QR_FILE" ] && [ -f "$QR_FILE" ]; then
|
|
# Test 9.1: QR code generation produces structured output
|
|
# Pass file path without -s: un CLI reads the file and auto-detects language
|
|
# (with -s, the positional arg is treated as inline code, not a file path)
|
|
#
|
|
# QR tests require language-specific libraries (qrcode npm, python qrcode, etc.)
|
|
# If the sandbox doesn't have the library, SKIP - don't FAIL.
|
|
QR_OUTPUT=$(build/un "$QR_FILE" 2>&1 || true)
|
|
echo "$QR_OUTPUT" > "$RESULTS_DIR/qr_generate.txt"
|
|
|
|
echo -n "Test: qr_generate... "
|
|
TOTAL_TESTS=$((TOTAL_TESTS + 1))
|
|
|
|
if echo "$QR_OUTPUT" | grep -qiE "QR:unsandbox-qr-ok:ROWS:[0-9]+"; then
|
|
TEST_RESULTS["qr_generate"]="pass"
|
|
echo "PASS"
|
|
elif echo "$QR_OUTPUT" | grep -qiE "Cannot find module|ModuleNotFoundError|ImportError|no such file|LoadError|require.*not found|could not find|package.*not found|install|gem.*not found|undefined method|NameError|not installed|unresolved import|cannot open shared object"; then
|
|
echo "SKIP (missing QR dependency in sandbox)"
|
|
else
|
|
TEST_RESULTS["qr_generate"]="fail"
|
|
FAILURES=$((FAILURES + 1))
|
|
echo "FAIL"
|
|
echo " --- Failure output (qr_generate) ---"
|
|
head -5 "$RESULTS_DIR/qr_generate.txt"
|
|
echo " ---"
|
|
fi
|
|
else
|
|
echo "SKIP: No QR test file for $LANG"
|
|
fi
|
|
|
|
echo ""
|
|
|
|
# ============================================================================
|
|
# Results Summary
|
|
# ============================================================================
|
|
END_TIME=$(date +%s.%N)
|
|
DURATION=$(echo "$END_TIME - $START_TIME" | bc 2>/dev/null || echo "0")
|
|
|
|
PASSED=$((TOTAL_TESTS - FAILURES))
|
|
if [ "$FAILURES" -eq 0 ]; then
|
|
STATUS="PASS"
|
|
else
|
|
STATUS="FAIL"
|
|
fi
|
|
|
|
echo "============================================"
|
|
echo "=== Result: $STATUS ($PASSED/$TOTAL_TESTS passed, ${DURATION}s) ==="
|
|
echo "============================================"
|
|
|
|
# ============================================================================
|
|
# API Health Report - Track transient errors for monitoring
|
|
# ============================================================================
|
|
TESTS_WITH_RETRIES=${#TEST_RETRIES[@]}
|
|
|
|
echo ""
|
|
echo "--- API Health Report ---"
|
|
echo "Total retries: $TOTAL_RETRIES"
|
|
echo " Rate limit (429): $RETRIES_429"
|
|
echo " Server error (5xx): $RETRIES_5XX"
|
|
echo " Timeout: $RETRIES_TIMEOUT"
|
|
echo " Connection: $RETRIES_CONN"
|
|
echo "Tests that needed retries: $TESTS_WITH_RETRIES"
|
|
|
|
if [ $TESTS_WITH_RETRIES -gt 0 ]; then
|
|
echo "Tests with retries:"
|
|
for test_name in "${!TEST_RETRIES[@]}"; do
|
|
retries="${TEST_RETRIES[$test_name]}"
|
|
result="${TEST_RESULTS[$test_name]}"
|
|
echo " - $test_name: $retries retries ($result)"
|
|
done
|
|
fi
|
|
|
|
# Write JSON report for aggregation across all SDKs
|
|
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
|
cat > "$RESULTS_DIR/api-health.json" << EOF
|
|
{
|
|
"timestamp": "$TIMESTAMP",
|
|
"language": "$LANG",
|
|
"duration_seconds": $DURATION,
|
|
"tests": {
|
|
"total": $TOTAL_TESTS,
|
|
"passed": $PASSED,
|
|
"failed": $FAILURES
|
|
},
|
|
"retries": {
|
|
"total": $TOTAL_RETRIES,
|
|
"rate_limit_429": $RETRIES_429,
|
|
"server_error_5xx": $RETRIES_5XX,
|
|
"timeout": $RETRIES_TIMEOUT,
|
|
"connection": $RETRIES_CONN
|
|
},
|
|
"tests_with_retries": $TESTS_WITH_RETRIES,
|
|
"api_health_score": $(echo "scale=2; 100 - ($TOTAL_RETRIES * 5)" | bc 2>/dev/null || echo "100")
|
|
}
|
|
EOF
|
|
|
|
echo ""
|
|
|
|
# Generate JUnit XML
|
|
cat > "$RESULTS_DIR/test-results.xml" << EOF
|
|
<?xml version="1.0" encoding="UTF-8"?>
|
|
<testsuites>
|
|
<testsuite name="Functional Test - $LANG" tests="$TOTAL_TESTS" failures="$FAILURES" time="$DURATION">
|
|
EOF
|
|
|
|
for test_name in "${!TEST_RESULTS[@]}"; do
|
|
result="${TEST_RESULTS[$test_name]}"
|
|
cat >> "$RESULTS_DIR/test-results.xml" << EOF
|
|
<testcase name="$test_name" classname="un.$LANG">
|
|
EOF
|
|
if [ "$result" = "pass" ]; then
|
|
echo " <system-out>Test passed</system-out>" >> "$RESULTS_DIR/test-results.xml"
|
|
else
|
|
echo " <failure message=\"Test failed\">See ${test_name}.txt for details</failure>" >> "$RESULTS_DIR/test-results.xml"
|
|
fi
|
|
echo " </testcase>" >> "$RESULTS_DIR/test-results.xml"
|
|
done
|
|
|
|
cat >> "$RESULTS_DIR/test-results.xml" << EOF
|
|
</testsuite>
|
|
</testsuites>
|
|
EOF
|
|
|
|
# Exit with appropriate code
|
|
[ "$STATUS" = "PASS" ] && exit 0 || exit 1
|