#!/bin/bash # validate-examples.sh - Core script that FINDS and VALIDATES all SDK examples # This is the HEART of self-validating documentation # # Features: # - Recursively finds all example files in clients/*/examples/ directories # - Detects language from file extension # - Executes via unsandbox API with proper authentication # - Validates output against expected results # - Generates JSON and HTML reports # - Parallel execution for speed # # Usage: # bash scripts/validate-examples.sh # # Environment: # UNSANDBOX_API_KEY - API key for authentication (required) # UNSANDBOX_API_URL - API endpoint (default: https://api.unsandbox.com) # PARALLEL_JOBS - Number of parallel executions (default: 4) set -o pipefail # Configuration SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" RESULTS_DIR="${PROJECT_ROOT}/science-results" REPORT_JSON="${RESULTS_DIR}/examples-validation-results.json" REPORT_HTML="${RESULTS_DIR}/examples-validation-results.html" TEMP_DIR="/tmp/unsandbox-examples-$$" EXAMPLES_DIR="${PROJECT_ROOT}/clients" # Default configuration UNSANDBOX_API_URL="${UNSANDBOX_API_URL:-https://api.unsandbox.com}" PARALLEL_JOBS="${PARALLEL_JOBS:-4}" TIMEOUT_SECONDS=30 VERBOSE="${VERBOSE:-0}" # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' # No Color # Counters TOTAL_EXAMPLES=0 TOTAL_VALIDATED=0 TOTAL_FAILED=0 declare -A LANGUAGE_STATS declare -A EXECUTION_TIMES # Create results directory mkdir -p "$RESULTS_DIR" "$TEMP_DIR" # Cleanup on exit cleanup() { rm -rf "$TEMP_DIR" } trap cleanup EXIT # Logging functions log() { echo -e "${BLUE}[INFO]${NC} $*" >&2 } log_pass() { echo -e "${GREEN}[PASS]${NC} $*" >&2 } log_fail() { echo -e "${RED}[FAIL]${NC} $*" >&2 } log_warn() { echo -e "${YELLOW}[WARN]${NC} $*" >&2 } debug() { [[ $VERBOSE -eq 1 ]] && echo -e "${BLUE}[DEBUG]${NC} $*" >&2 } # Helper: Get language from file extension detect_language() { local file=$1 local ext="${file##*.}" case "$ext" in py|python) echo "python" ;; js|javascript) echo "javascript" ;; go|golang) echo "go" ;; rs|rust) echo "rust" ;; java) echo "java" ;; rb|ruby) echo "ruby" ;; php) echo "php" ;; ts|typescript) echo "typescript" ;; cpp|cc|c\+\+) echo "cpp" ;; c) echo "c" ;; sh|bash) echo "bash" ;; pl|perl) echo "perl" ;; *) echo "" ;; esac } # Helper: Map language name to API parameter get_api_language() { local lang=$1 case "$lang" in cpp) echo "c++" ;; *) echo "$lang" ;; esac } # Helper: Extract expected output from file comments extract_expected_output() { local file=$1 # Look for expected output in comments # Supports: # // Expected output: ... # # Expected output: ... # -- Expected output: ... grep -E '(//|#|--|/\*|{\s*\/\/)\s*(Expected output|Output|Result):\s*' "$file" | \ sed -E 's/^[^:]*:\s*//' | \ sed 's/^[[:space:]]*//;s/[[:space:]]*$//' | \ head -1 } # Helper: Parse JSON response safely safe_json_extract() { local json=$1 local key=$2 echo "$json" | jq -r ".$key // \"\"" 2>/dev/null || echo "" } # Helper: Compile C example to binary compile_c_example() { local source_file=$1 local binary_file="${TEMP_DIR}/example-${RANDOM}" debug "Compiling C example: $source_file" # Compile with gcc (using standard flags) if gcc -o "$binary_file" "$source_file" 2>/dev/null; then echo "$binary_file" return 0 else debug "C compilation failed for $source_file" return 1 fi } # Helper: Execute Python async example via subprocess execute_python_async() { local python_code=$1 debug "Executing Python async code" # Use Python to run async code via asyncio.run() # This allows testing of async/await patterns python3 -c "import asyncio; asyncio.run(eval('async def _async_main():\\n' + '\\n'.join(' ' + line for line in '''$python_code'''.split('\\n')) + '\\n\\nasyncio.run(_async_main())'))" 2>&1 return $? } # Helper: Execute local binary or script directly execute_local_file() { local file=$1 local language=$2 debug "Executing local file: $file ($language)" case "$language" in c) # For C examples, compile and execute local binary=$(compile_c_example "$file") if [[ -n "$binary" ]] && [[ -x "$binary" ]]; then timeout "$TIMEOUT_SECONDS" "$binary" 2>&1 return $? else return 1 fi ;; python) # For Python examples, execute directly timeout "$TIMEOUT_SECONDS" python3 "$file" 2>&1 return $? ;; *) return 1 ;; esac } # Main validation function for a single example file validate_example() { local example_file=$1 local language local api_lang local code local start_time local elapsed_time local api_response local stdout_content local stderr_content local exit_code local result_file="${TEMP_DIR}/result-${RANDOM}.json" local execution_method="api" # Detect language language=$(detect_language "$example_file") if [[ -z "$language" ]]; then log_fail "Unknown language for $example_file" return 1 fi # Initialize language stats if not exists if [[ -z "${LANGUAGE_STATS[$language]}" ]]; then LANGUAGE_STATS[$language]=0 fi # Read code code=$(cat "$example_file") if [[ -z "$code" ]]; then log_fail "Empty code file: $example_file" return 1 fi # Measure execution time start_time=$(date +%s%N) # Determine execution method # If no API key, try local execution for C and Python if [[ -z "$UNSANDBOX_API_KEY" ]]; then case "$language" in c|python) debug "No API key - attempting local execution for $language" execution_method="local" ;; *) log_warn "$example_file - UNSANDBOX_API_KEY not set, skipping execution" return 0 ;; esac fi # Execute via local method or API if [[ "$execution_method" == "local" ]]; then # Local execution for C and Python api_response=$(execute_local_file "$example_file" "$language") exit_code=$? stdout_content="$api_response" stderr_content="" else # API execution (default for all languages when key is available) api_lang=$(get_api_language "$language") # Execute via API with timeout debug "Executing $example_file ($api_lang) via API" api_response=$(curl -s -X POST "${UNSANDBOX_API_URL}/execute" \ -H "Authorization: Bearer ${UNSANDBOX_API_KEY}" \ -H "Content-Type: application/json" \ --max-time "$TIMEOUT_SECONDS" \ -d "{\"language\": \"${api_lang}\", \"code\": $(echo "$code" | jq -R -s .)}" \ 2>&1) exit_code=$? # Extract results from API response if [[ $exit_code -eq 0 ]]; then stdout_content=$(safe_json_extract "$api_response" "stdout") stderr_content=$(safe_json_extract "$api_response" "stderr") exit_code=$(safe_json_extract "$api_response" "exit_code") # Treat empty stderr as success if [[ -z "$stderr_content" || "$stderr_content" == "null" ]]; then stderr_content="" fi else stderr_content="API request failed (curl exit code $exit_code)" fi fi elapsed_time=$(( ($(date +%s%N) - start_time) / 1000000 )) # Convert to milliseconds # Check if execution was successful if [[ "$exit_code" == "0" || -z "$exit_code" ]]; then log_pass "$example_file ($language) - ${elapsed_time}ms [$execution_method]" LANGUAGE_STATS[$language]=$((${LANGUAGE_STATS[$language]} + 1)) EXECUTION_TIMES[$language]=$((${EXECUTION_TIMES[$language]:-0} + elapsed_time)) TOTAL_VALIDATED=$((TOTAL_VALIDATED + 1)) else log_fail "$example_file ($language) - exit code $exit_code [$execution_method]" if [[ -n "$stderr_content" ]]; then debug "stderr: $stderr_content" fi TOTAL_FAILED=$((TOTAL_FAILED + 1)) fi # Save result for JSON report cat > "$result_file" </dev/null | \ sort } # Generate JSON report generate_json_report() { local timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ") local timestamp_readable=$(date -u "+%Y-%m-%d %H:%M:%S UTC") local success_rate="0" if [[ $((TOTAL_VALIDATED + TOTAL_FAILED)) -gt 0 ]]; then success_rate=$(echo "scale=1; $TOTAL_VALIDATED * 100 / ($TOTAL_VALIDATED + $TOTAL_FAILED)" | bc 2>/dev/null || echo "0") fi # Calculate average execution time per language local lang_times=() for lang in "${!LANGUAGE_STATS[@]}"; do local count=${LANGUAGE_STATS[$lang]} local total_time=${EXECUTION_TIMES[$lang]:-0} local avg_time=0 if [[ $count -gt 0 ]]; then avg_time=$((total_time / count)) fi lang_times+=(" {\"language\": \"$lang\", \"validated\": $count, \"total_time_ms\": $total_time, \"avg_time_ms\": $avg_time}") done # Build JSON report cat > "$REPORT_JSON" </dev/null || echo "0") fi # Build language table rows local lang_rows="" for lang in "${!LANGUAGE_STATS[@]}"; do local count=${LANGUAGE_STATS[$lang]} local avg_time=${EXECUTION_TIMES[$lang]:-0} if [[ $count -gt 0 ]]; then avg_time=$((avg_time / count)) fi lang_rows+=" $lang$count${avg_time}ms\n" done # Determine status badge local status_color="green" local status_text="All Passing" if [[ $TOTAL_FAILED -gt 0 ]]; then status_color="red" status_text="Some Failures" elif [[ $TOTAL_EXAMPLES -eq 0 ]]; then status_color="yellow" status_text="No Examples Found" fi cat > "$REPORT_HTML" <<'HTMLEOF' SDK Examples Validation Report

SDK Examples Validation Report

Ensuring all documentation examples actually work

STATUS_TEXT
TOTAL_EXAMPLES
Total Examples
TOTAL_VALIDATED
Validated
TOTAL_FAILED
Failed
SUCCESS_RATE%
Success Rate
Language Coverage

Validation statistics by language

LANGUAGE_ROWS
Language Examples Validated Avg Execution Time

Last verified:

TIMESTAMP_READABLE

This report validates that SDK examples execute successfully through the unsandbox API.

Generated automatically by the CI/CD pipeline

HTMLEOF # Replace placeholders sed -i "s/STATUS_CLASS/$status_color/g" "$REPORT_HTML" sed -i "s/STATUS_TEXT/$status_text/g" "$REPORT_HTML" sed -i "s/TOTAL_EXAMPLES/$TOTAL_EXAMPLES/g" "$REPORT_HTML" sed -i "s/TOTAL_VALIDATED/$TOTAL_VALIDATED/g" "$REPORT_HTML" sed -i "s/TOTAL_FAILED/$TOTAL_FAILED/g" "$REPORT_HTML" sed -i "s/SUCCESS_RATE/$success_rate/g" "$REPORT_HTML" sed -i "s|LANGUAGE_ROWS|$lang_rows|g" "$REPORT_HTML" sed -i "s/TIMESTAMP_READABLE/$timestamp_readable/g" "$REPORT_HTML" log "HTML report generated: $REPORT_HTML" } # Main execution main() { log "Starting SDK examples validation" log "Examples directory: $EXAMPLES_DIR" log "Results directory: $RESULTS_DIR" log "Parallel jobs: $PARALLEL_JOBS" # Check for API key if [[ -z "$UNSANDBOX_API_KEY" ]]; then log_warn "UNSANDBOX_API_KEY not set - will scan for examples but skip execution" fi # Find all examples local examples examples=$(find_examples) if [[ -z "$examples" ]]; then log_warn "No examples found in $EXAMPLES_DIR" else TOTAL_EXAMPLES=$(echo "$examples" | wc -l) log "Found $TOTAL_EXAMPLES example files" # Validate examples echo "$examples" | validate_examples_parallel fi # Generate reports log "Generating reports..." generate_json_report generate_html_report # Print summary echo "" echo "========================================" echo "SDK Examples Validation Summary" echo "========================================" echo "Total Examples: $TOTAL_EXAMPLES" echo "Validated: $TOTAL_VALIDATED" echo "Failed: $TOTAL_FAILED" if [[ $((TOTAL_VALIDATED + TOTAL_FAILED)) -gt 0 ]]; then local success_rate=$(echo "scale=1; $TOTAL_VALIDATED * 100 / ($TOTAL_VALIDATED + $TOTAL_FAILED)" | bc 2>/dev/null || echo "0") echo "Success Rate: $success_rate%" fi echo "" echo "Reports:" echo " JSON: $REPORT_JSON" echo " HTML: $REPORT_HTML" echo "========================================" # Exit with appropriate code if [[ $TOTAL_FAILED -eq 0 ]] && [[ $TOTAL_EXAMPLES -gt 0 ]]; then log_pass "All examples validated successfully" exit 0 elif [[ $TOTAL_EXAMPLES -eq 0 ]]; then log_warn "No examples found to validate" exit 0 # Not a failure if no examples exist else log_fail "$TOTAL_FAILED example(s) failed validation" exit 1 fi } # Run main function main "$@"