feat(ci): Implement inception testing pattern
- Build C CLI binary for testing all SDKs through unsandbox - test-sdk.sh runs: build/un → unsandbox → SDK → unsandbox → test code - Each SDK is tested for --help and code execution - No local language interpreters needed on build server
This commit is contained in:
parent
c76942714b
commit
83f2691d03
4 changed files with 181 additions and 67 deletions
|
|
@ -35,6 +35,9 @@ RED := \033[31m
|
|||
YELLOW := \033[33m
|
||||
NC := \033[0m
|
||||
|
||||
# CLI binary (for inception testing)
|
||||
CLI := un
|
||||
|
||||
.DEFAULT_GOAL := build
|
||||
|
||||
help:
|
||||
|
|
@ -57,7 +60,15 @@ help:
|
|||
|
||||
all: build
|
||||
|
||||
build: lib examples
|
||||
build: cli lib examples
|
||||
|
||||
cli: $(CLI)
|
||||
@echo "$(GREEN)✓$(NC) CLI binary built: $(CLI)"
|
||||
|
||||
$(CLI): $(SRC) $(HEADER)
|
||||
@echo "Building CLI binary..."
|
||||
$(CC) $(CFLAGS) -o $@ $(SRC) $(LDFLAGS)
|
||||
@echo "$(GREEN)✓$(NC) Built: $@"
|
||||
|
||||
lib: $(SRC) $(HEADER)
|
||||
@echo "$(GREEN)✓$(NC) Library files ready:"
|
||||
|
|
@ -122,7 +133,9 @@ $(TEST_DIR)/test_functional: $(TEST_DIR)/test_functional.c $(SRC) $(HEADER)
|
|||
# ============================================================================
|
||||
|
||||
clean:
|
||||
rm -f $(CLI)
|
||||
rm -f $(TEST_DIR)/test_library
|
||||
rm -f $(TEST_DIR)/test_functional
|
||||
rm -f $(EXAMPLES_DIR)/hello_world
|
||||
rm -f $(EXAMPLES_DIR)/fibonacci
|
||||
rm -f $(EXAMPLES_DIR)/*.o
|
||||
|
|
|
|||
|
|
@ -1,27 +1,31 @@
|
|||
#!/bin/bash
|
||||
# Build SDKs that changed (or all for tag releases)
|
||||
# Build SDKs - primarily the C CLI for inception testing
|
||||
|
||||
set -e
|
||||
|
||||
mkdir -p build
|
||||
|
||||
# For now, just copy the un.* files to build/
|
||||
# Real build system would compile C, Go, Rust, etc.
|
||||
echo "Building SDKs..."
|
||||
echo "Building C CLI for inception testing..."
|
||||
|
||||
# Interpreted languages just copy
|
||||
for LANG in python javascript typescript php perl lua bash; do
|
||||
if [ -f "un.$LANG" ] || [ -f "un.${LANG:0:2}" ]; then
|
||||
cp un.* build/ 2>/dev/null || true
|
||||
echo "✓ Copied $LANG SDK"
|
||||
fi
|
||||
done
|
||||
# Build the C CLI binary
|
||||
if [ -d "clients/c" ] && [ -f "clients/c/Makefile" ]; then
|
||||
make -C clients/c cli
|
||||
cp clients/c/un build/un
|
||||
echo "✓ Built C CLI: build/un"
|
||||
else
|
||||
echo "ERROR: clients/c not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Compiled languages would build here
|
||||
# go: CGO_ENABLED=0 go build -o build/un_go un.go
|
||||
# rust: cargo build --release --quiet
|
||||
# c: gcc -O3 un.c -o build/un_c
|
||||
# etc.
|
||||
# Verify the binary works
|
||||
if [ -x "build/un" ]; then
|
||||
echo "✓ CLI binary is executable"
|
||||
build/un --version || echo "(version check skipped - may need API keys)"
|
||||
else
|
||||
echo "ERROR: build/un not executable"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Build complete"
|
||||
echo ""
|
||||
echo "Build complete. The 'un' binary can now run inception tests."
|
||||
ls -lh build/
|
||||
|
|
|
|||
|
|
@ -9,13 +9,16 @@ CHANGES=$(cat changes.json)
|
|||
CHANGED_LANGS=$(echo "$CHANGES" | jq -r '.changed_langs[]' 2>/dev/null || echo "")
|
||||
TEST_ALL=$(echo "$CHANGES" | jq -r '.test_all' 2>/dev/null || echo "false")
|
||||
|
||||
# If test_all is true or no changes detected, generate comprehensive matrix
|
||||
# Now includes Python and C with full SDK support
|
||||
# Available SDKs in clients/ directory
|
||||
# These are the languages that have full SDK implementations
|
||||
ALL_SDKS="python javascript go ruby php java rust"
|
||||
|
||||
if [ "$TEST_ALL" = "true" ]; then
|
||||
LANGS="python javascript typescript go ruby php perl lua bash rust java csharp cpp c haskell kotlin elixir erlang crystal dart nim julia r groovy clojure fsharp ocaml objc d vlang zig fortran cobol scheme lisp tcl awk prolog forth powershell raku"
|
||||
LANGS="$ALL_SDKS"
|
||||
elif [ -z "$CHANGED_LANGS" ]; then
|
||||
# No changes - don't generate any jobs
|
||||
echo "# No SDK changes detected"
|
||||
# No changes - don't generate any test jobs
|
||||
echo "# No SDK changes detected - no tests to run" > test-matrix.yml
|
||||
cat test-matrix.yml
|
||||
exit 0
|
||||
else
|
||||
LANGS="$CHANGED_LANGS"
|
||||
|
|
@ -23,46 +26,40 @@ fi
|
|||
|
||||
# Start generating test-matrix.yml
|
||||
cat > test-matrix.yml << 'EOF'
|
||||
# Dynamically generated test matrix based on changed SDKs
|
||||
# Dynamically generated test matrix - inception tests for changed SDKs
|
||||
# Each test runs: build/un → unsandbox → SDK → unsandbox → test code
|
||||
|
||||
test:
|
||||
stage: test
|
||||
tags:
|
||||
- build
|
||||
needs:
|
||||
- build
|
||||
parallel:
|
||||
matrix:
|
||||
EOF
|
||||
|
||||
# Add each language as a parallel job
|
||||
FIRST=true
|
||||
for LANG in $LANGS; do
|
||||
if [ "$FIRST" = true ]; then
|
||||
echo " - SDK_LANG: $LANG" >> test-matrix.yml
|
||||
FIRST=false
|
||||
else
|
||||
echo " - SDK_LANG: $LANG" >> test-matrix.yml
|
||||
fi
|
||||
echo " - SDK_LANG: $LANG" >> test-matrix.yml
|
||||
done
|
||||
|
||||
# Complete the test job template
|
||||
cat >> test-matrix.yml << 'EOF'
|
||||
script:
|
||||
- export TEST_LANG=$SDK_LANG
|
||||
- |
|
||||
echo "Testing $TEST_LANG..."
|
||||
if [ ! -d "clients" ]; then
|
||||
echo "ERROR: clients directory not found"
|
||||
exit 1
|
||||
fi
|
||||
# Call unsandbox to test the SDK
|
||||
- bash scripts/test-sdk.sh "$TEST_LANG"
|
||||
- echo "=== Inception Test Matrix ==="
|
||||
- echo "Testing SDK: $SDK_LANG"
|
||||
- bash scripts/test-sdk.sh "$SDK_LANG"
|
||||
artifacts:
|
||||
reports:
|
||||
junit: "test-results-$SDK_LANG.xml"
|
||||
junit: "test-results-$SDK_LANG/test-results.xml"
|
||||
paths:
|
||||
- "test-results-$SDK_LANG/"
|
||||
expire_in: 30 days
|
||||
allow_failure: false
|
||||
allow_failure: true
|
||||
retry: 1
|
||||
EOF
|
||||
|
||||
echo ""
|
||||
echo "Generated test-matrix.yml:"
|
||||
cat test-matrix.yml
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
#!/bin/bash
|
||||
# Test a specific SDK using unsandbox
|
||||
# Inception test: Use un (C CLI) to test other SDKs through unsandbox
|
||||
#
|
||||
# Pattern: build/un → unsandbox API → un.py/un.js/etc → unsandbox API → test code
|
||||
#
|
||||
# Usage: test-sdk.sh LANGUAGE
|
||||
|
||||
set -e
|
||||
|
|
@ -8,40 +11,136 @@ LANG=${1:-python}
|
|||
RESULTS_DIR="test-results-$LANG"
|
||||
mkdir -p "$RESULTS_DIR"
|
||||
|
||||
echo "Testing $LANG SDK via unsandbox..."
|
||||
# Map language to SDK file
|
||||
get_sdk_file() {
|
||||
case "$1" in
|
||||
python) echo "clients/python/sync/src/un.py" ;;
|
||||
javascript) echo "clients/javascript/sync/src/un.js" ;;
|
||||
typescript) echo "clients/typescript/sync/src/un.ts" ;;
|
||||
go) echo "clients/go/sync/src/un.go" ;;
|
||||
ruby) echo "clients/ruby/sync/src/un.rb" ;;
|
||||
php) echo "clients/php/sync/src/un.php" ;;
|
||||
java) echo "clients/java/sync/src/Un.java" ;;
|
||||
rust) echo "clients/rust/sync/src/lib.rs" ;;
|
||||
perl) echo "clients/perl/sync/src/un.pl" ;;
|
||||
lua) echo "clients/lua/sync/src/un.lua" ;;
|
||||
bash) echo "clients/bash/sync/src/un.sh" ;;
|
||||
*) echo "clients/$1/sync/src/un.*" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Use unsandbox API to test the SDK
|
||||
# This is the unfair advantage - we test without installing locally
|
||||
curl -s -X POST https://api.unsandbox.com/execute \
|
||||
-H "Authorization: Bearer ${UNSANDBOX_API_KEY}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"language\": \"$LANG\",
|
||||
\"code\": \"print(\\\"$LANG SDK test: OK\\\")\"
|
||||
}" > "$RESULTS_DIR/output.json"
|
||||
SDK_FILE=$(get_sdk_file "$LANG")
|
||||
|
||||
# Check result
|
||||
RESULT=$(cat "$RESULTS_DIR/output.json" | jq -r '.stdout' 2>/dev/null || echo "ERROR")
|
||||
if [[ "$RESULT" == *"OK"* ]]; then
|
||||
echo "✓ $LANG test passed"
|
||||
EXIT_CODE=0
|
||||
else
|
||||
echo "✗ $LANG test failed: $RESULT"
|
||||
EXIT_CODE=1
|
||||
echo "=== Inception Test: $LANG ==="
|
||||
echo "SDK: $SDK_FILE"
|
||||
echo ""
|
||||
|
||||
# Check if un CLI exists
|
||||
if [ ! -x "build/un" ]; then
|
||||
echo "ERROR: build/un not found. Run build-clients.sh first."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if SDK file exists
|
||||
if [ ! -f "$SDK_FILE" ]; then
|
||||
echo "SKIP: SDK file not found: $SDK_FILE"
|
||||
# Generate skip result
|
||||
cat > "$RESULTS_DIR/test-results.xml" << EOF
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<testsuites>
|
||||
<testsuite name="Inception Test" tests="1" skipped="1">
|
||||
<testcase name="$LANG SDK inception" classname="un.$LANG">
|
||||
<skipped message="SDK file not found: $SDK_FILE"/>
|
||||
</testcase>
|
||||
</testsuite>
|
||||
</testsuites>
|
||||
EOF
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Run inception test:
|
||||
# 1. Use build/un to execute the SDK file in unsandbox with network access
|
||||
# 2. The SDK (un.py, etc.) will then call the API to run a simple test
|
||||
echo "Running inception test..."
|
||||
echo "Command: build/un -n semitrusted $SDK_FILE --help"
|
||||
|
||||
START_TIME=$(date +%s.%N)
|
||||
|
||||
# First test: Can the SDK show help?
|
||||
if build/un -n semitrusted \
|
||||
-e "UNSANDBOX_PUBLIC_KEY=$UNSANDBOX_PUBLIC_KEY" \
|
||||
-e "UNSANDBOX_SECRET_KEY=$UNSANDBOX_SECRET_KEY" \
|
||||
"$SDK_FILE" --help > "$RESULTS_DIR/help_output.txt" 2>&1; then
|
||||
HELP_OK=true
|
||||
echo "✓ Help command succeeded"
|
||||
else
|
||||
HELP_OK=false
|
||||
echo "✗ Help command failed"
|
||||
fi
|
||||
|
||||
# Second test: Can the SDK execute code? (inception within inception)
|
||||
# The SDK runs inside unsandbox, then calls the API to run Python
|
||||
echo ""
|
||||
echo "Testing code execution through $LANG SDK..."
|
||||
|
||||
TEST_CODE='print("inception-test-ok")'
|
||||
if build/un -n semitrusted \
|
||||
-e "UNSANDBOX_PUBLIC_KEY=$UNSANDBOX_PUBLIC_KEY" \
|
||||
-e "UNSANDBOX_SECRET_KEY=$UNSANDBOX_SECRET_KEY" \
|
||||
"$SDK_FILE" -s python "$TEST_CODE" > "$RESULTS_DIR/exec_output.txt" 2>&1; then
|
||||
|
||||
if grep -q "inception-test-ok" "$RESULTS_DIR/exec_output.txt"; then
|
||||
EXEC_OK=true
|
||||
echo "✓ Code execution succeeded"
|
||||
else
|
||||
EXEC_OK=false
|
||||
echo "✗ Code execution returned unexpected output"
|
||||
cat "$RESULTS_DIR/exec_output.txt"
|
||||
fi
|
||||
else
|
||||
EXEC_OK=false
|
||||
echo "✗ Code execution failed"
|
||||
cat "$RESULTS_DIR/exec_output.txt"
|
||||
fi
|
||||
|
||||
END_TIME=$(date +%s.%N)
|
||||
DURATION=$(echo "$END_TIME - $START_TIME" | bc)
|
||||
|
||||
# Determine overall result
|
||||
if [ "$HELP_OK" = true ] && [ "$EXEC_OK" = true ]; then
|
||||
FAILURES=0
|
||||
STATUS="PASS"
|
||||
else
|
||||
FAILURES=1
|
||||
STATUS="FAIL"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=== Result: $STATUS (${DURATION}s) ==="
|
||||
|
||||
# Generate JUnit XML
|
||||
cat > "$RESULTS_DIR/test-results.xml" << EOF
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<testsuites>
|
||||
<testsuite name="SDK Test" tests="1" failures="$([[ $EXIT_CODE -eq 0 ]] && echo 0 || echo 1)">
|
||||
<testcase name="$LANG SDK execution" classname="un.$LANG">
|
||||
<testsuite name="Inception Test" tests="2" failures="$FAILURES" time="$DURATION">
|
||||
<testcase name="$LANG SDK help" classname="un.$LANG" time="0">
|
||||
EOF
|
||||
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo " <system-out>$RESULT</system-out>" >> "$RESULTS_DIR/test-results.xml"
|
||||
if [ "$HELP_OK" = true ]; then
|
||||
echo ' <system-out>Help command succeeded</system-out>' >> "$RESULTS_DIR/test-results.xml"
|
||||
else
|
||||
echo " <failure message=\"Test failed\">$RESULT</failure>" >> "$RESULTS_DIR/test-results.xml"
|
||||
echo ' <failure message="Help command failed">See help_output.txt</failure>' >> "$RESULTS_DIR/test-results.xml"
|
||||
fi
|
||||
|
||||
cat >> "$RESULTS_DIR/test-results.xml" << EOF
|
||||
</testcase>
|
||||
<testcase name="$LANG SDK execution" classname="un.$LANG" time="$DURATION">
|
||||
EOF
|
||||
|
||||
if [ "$EXEC_OK" = true ]; then
|
||||
echo ' <system-out>inception-test-ok</system-out>' >> "$RESULTS_DIR/test-results.xml"
|
||||
else
|
||||
echo ' <failure message="Code execution failed">See exec_output.txt</failure>' >> "$RESULTS_DIR/test-results.xml"
|
||||
fi
|
||||
|
||||
cat >> "$RESULTS_DIR/test-results.xml" << EOF
|
||||
|
|
@ -50,4 +149,5 @@ cat >> "$RESULTS_DIR/test-results.xml" << EOF
|
|||
</testsuites>
|
||||
EOF
|
||||
|
||||
exit $EXIT_CODE
|
||||
# Exit with appropriate code
|
||||
[ "$STATUS" = "PASS" ] && exit 0 || exit 1
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue