#!/bin/bash # This is free software for the public good of a permacomputer hosted at # permacomputer.com, an always-on computer by the people, for the people. # One which is durable, easy to repair, & distributed like tap water # for machine learning intelligence. # # The permacomputer is community-owned infrastructure optimized around # four values: # # TRUTH First principles, math & science, open source code freely distributed # FREEDOM Voluntary partnerships, freedom from tyranny & corporate control # HARMONY Minimal waste, self-renewing systems with diverse thriving connections # LOVE Be yourself without hurting others, cooperation through natural law # # This software contributes to that vision by enabling code execution across 42+ programming languages through a unified interface, accessible to all. # Code is seeds to sprout on any abandoned technology. # validate-examples.sh - Core script that FINDS and VALIDATES all SDK examples # This is the HEART of self-validating documentation # # Features: # - Recursively finds all example files in clients/*/examples/ directories # - Detects language from file extension # - Executes via unsandbox API with proper authentication # - Validates output against expected results # - Generates JSON and HTML reports # - Parallel execution for speed # # Usage: # bash scripts/validate-examples.sh # # Environment: # UNSANDBOX_PUBLIC_KEY - Public key for HMAC authentication # UNSANDBOX_SECRET_KEY - Secret key for HMAC authentication # UNSANDBOX_API_URL - API endpoint (default: https://api.unsandbox.com) # PARALLEL_JOBS - Number of parallel executions (default: 4) set -o pipefail # Configuration SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" RESULTS_DIR="${PROJECT_ROOT}/science-results" REPORT_JSON="${RESULTS_DIR}/examples-validation-results.json" REPORT_HTML="${RESULTS_DIR}/examples-validation-results.html" TEMP_DIR="/tmp/unsandbox-examples-$$" EXAMPLES_DIR="${PROJECT_ROOT}/clients" # Default configuration UNSANDBOX_API_URL="${UNSANDBOX_API_URL:-https://api.unsandbox.com}" PARALLEL_JOBS="${PARALLEL_JOBS:-4}" TIMEOUT_SECONDS=30 VERBOSE="${VERBOSE:-0}" # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' # No Color # Counters TOTAL_EXAMPLES=0 TOTAL_VALIDATED=0 TOTAL_FAILED=0 # Check if we have valid API credentials has_api_credentials() { [[ -n "$UNSANDBOX_PUBLIC_KEY" && -n "$UNSANDBOX_SECRET_KEY" ]] } # Generate HMAC signature for API request generate_hmac_signature() { local method=$1 local path=$2 local body=$3 local timestamp=$4 local message="${timestamp}:${method}:${path}:${body}" echo -n "$message" | openssl dgst -sha256 -hmac "$UNSANDBOX_SECRET_KEY" | awk '{print $2}' } declare -A LANGUAGE_STATS declare -A EXECUTION_TIMES # Create results directory mkdir -p "$RESULTS_DIR" "$TEMP_DIR" # Cleanup on exit cleanup() { rm -rf "$TEMP_DIR" } trap cleanup EXIT # Logging functions log() { echo -e "${BLUE}[INFO]${NC} $*" >&2 } log_pass() { echo -e "${GREEN}[PASS]${NC} $*" >&2 } log_fail() { echo -e "${RED}[FAIL]${NC} $*" >&2 } log_warn() { echo -e "${YELLOW}[WARN]${NC} $*" >&2 } debug() { [[ $VERBOSE -eq 1 ]] && echo -e "${BLUE}[DEBUG]${NC} $*" >&2 } # Helper: Get language from file extension detect_language() { local file=$1 local ext="${file##*.}" case "$ext" in py|python) echo "python" ;; js|javascript) echo "javascript" ;; go|golang) echo "go" ;; rs|rust) echo "rust" ;; java) echo "java" ;; rb|ruby) echo "ruby" ;; php) echo "php" ;; ts|typescript) echo "typescript" ;; cpp|cc|c\+\+) echo "cpp" ;; c) echo "c" ;; sh|bash) echo "bash" ;; pl|perl) echo "perl" ;; *) echo "" ;; esac } # Helper: Map language name to API parameter get_api_language() { local lang=$1 case "$lang" in cpp) echo "c++" ;; *) echo "$lang" ;; esac } # Helper: Extract expected output from file comments extract_expected_output() { local file=$1 # Look for expected output in comments # Supports: # // Expected output: ... # # Expected output: ... # -- Expected output: ... grep -E '(//|#|--|/\*|{\s*\/\/)\s*(Expected output|Output|Result):\s*' "$file" | \ sed -E 's/^[^:]*:\s*//' | \ sed 's/^[[:space:]]*//;s/[[:space:]]*$//' | \ head -1 } # Helper: Get SDK source directory from example file path # clients/python/sync/examples/foo.py -> clients/python/sync/src/ get_sdk_src_dir() { local example_file=$1 local src_dir # Replace /examples/ with /src/ in the path src_dir=$(echo "$example_file" | sed 's|/examples/.*|/src/|') if [[ -d "$src_dir" ]]; then echo "$src_dir" fi } # Helper: Build input_files JSON array from SDK source directory # Returns JSON array of {filename, content} objects with base64-encoded content build_input_files_json() { local src_dir=$1 local language=$2 local input_files="[" local first=true # Only include relevant source files (skip __pycache__, .pyc, etc.) while IFS= read -r -d '' src_file; do local basename basename=$(basename "$src_file") # Skip compiled/cache files case "$basename" in *.pyc|*.pyo|*.class|*.o) continue ;; esac # Skip __pycache__ directories [[ "$src_file" == *__pycache__* ]] && continue local b64_content b64_content=$(base64 -w0 "$src_file" 2>/dev/null || base64 "$src_file" 2>/dev/null) if [[ "$first" == "true" ]]; then first=false else input_files+="," fi input_files+="{\"filename\":\"$basename\",\"content\":\"$b64_content\"}" done < <(find "$src_dir" -maxdepth 1 -type f -print0 2>/dev/null) input_files+="]" echo "$input_files" } # Helper: Rewrite import paths in example code so they resolve to /tmp/input/ # The API places input_files at /tmp/input/ rewrite_import_paths() { local code=$1 local language=$2 case "$language" in python) # Rewrite sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src")) # to sys.path.insert(0, '/tmp/input') code=$(echo "$code" | sed "s|os\.path\.join(os\.path\.dirname(__file__), \"\.\.\", \"src\")|'/tmp/input'|g") code=$(echo "$code" | sed "s|os\.path\.join(os\.path\.dirname(__file__), '\.\.', 'src')|'/tmp/input'|g") ;; javascript) # Rewrite from '../src/un_async.js' to '/tmp/input/un_async.js' # Rewrite from '../src/un.js' to '/tmp/input/un.js' code=$(echo "$code" | sed "s|from ['\"]\.\.\/src\/\([^'\"]*\)['\"]|from '/tmp/input/\1'|g") # Also handle require('../src/...') code=$(echo "$code" | sed "s|require(['\"]\.\.\/src\/\([^'\"]*\)['\"])|require('/tmp/input/\1')|g") ;; ruby) # Rewrite require_relative '../src/un' to require '/tmp/input/un' code=$(echo "$code" | sed "s|require_relative ['\"]\.\.\/src\/\([^'\"]*\)['\"]|require '/tmp/input/\1'|g") ;; php) # Rewrite __DIR__ . '/../src/un.php' to '/tmp/input/un.php' code=$(echo "$code" | sed "s|__DIR__ \. ['\"]\/\.\.\/src\/\([^'\"]*\)['\"]|'/tmp/input/\1'|g") ;; esac echo "$code" } # Helper: Parse JSON response safely safe_json_extract() { local json=$1 local key=$2 echo "$json" | jq -r ".$key // \"\"" 2>/dev/null || echo "" } # Helper: Compile C example to binary compile_c_example() { local source_file=$1 local binary_file="${TEMP_DIR}/example-${RANDOM}" debug "Compiling C example: $source_file" # Compile with gcc (using standard flags) if gcc -o "$binary_file" "$source_file" 2>/dev/null; then echo "$binary_file" return 0 else debug "C compilation failed for $source_file" return 1 fi } # Helper: Execute Python async example via subprocess execute_python_async() { local python_code=$1 debug "Executing Python async code" # Use Python to run async code via asyncio.run() # This allows testing of async/await patterns python3 -c "import asyncio; asyncio.run(eval('async def _async_main():\\n' + '\\n'.join(' ' + line for line in '''$python_code'''.split('\\n')) + '\\n\\nasyncio.run(_async_main())'))" 2>&1 return $? } # Helper: Execute local binary or script directly execute_local_file() { local file=$1 local language=$2 debug "Executing local file: $file ($language)" case "$language" in c) # For C examples, compile and execute local binary=$(compile_c_example "$file") if [[ -n "$binary" ]] && [[ -x "$binary" ]]; then timeout "$TIMEOUT_SECONDS" "$binary" 2>&1 return $? else return 1 fi ;; python) # For Python examples, execute directly timeout "$TIMEOUT_SECONDS" python3 "$file" 2>&1 return $? ;; *) return 1 ;; esac } # Main validation function for a single example file validate_example() { local example_file=$1 local language local api_lang local code local start_time local elapsed_time local api_response local stdout_content local stderr_content local exit_code local result_file="${TEMP_DIR}/result-${RANDOM}.json" local execution_method="api" # Detect language language=$(detect_language "$example_file") if [[ -z "$language" ]]; then log_fail "Unknown language for $example_file" return 1 fi # Initialize language stats if not exists if [[ -z "${LANGUAGE_STATS[$language]}" ]]; then LANGUAGE_STATS[$language]=0 fi # Read code code=$(cat "$example_file") if [[ -z "$code" ]]; then log_fail "Empty code file: $example_file" return 1 fi # Strip shebang and PHP opening tag for PHP files (API uses -r flag) if [[ "$language" == "php" ]]; then code=$(echo "$code" | sed '1{/^#!/d}' | sed '1{/^/dev/null || echo '?') files" fi # Build JSON body with credentials and optional input_files local body if [[ -n "$input_files_json" && "$input_files_json" != "[]" ]]; then body=$(jq -n \ --arg lang "$api_lang" \ --arg code "$code" \ --arg pk "$UNSANDBOX_PUBLIC_KEY" \ --arg sk "$UNSANDBOX_SECRET_KEY" \ --argjson input_files "$input_files_json" \ '{language: $lang, code: $code, env: {UNSANDBOX_PUBLIC_KEY: $pk, UNSANDBOX_SECRET_KEY: $sk}, input_files: $input_files}') else body=$(jq -n \ --arg lang "$api_lang" \ --arg code "$code" \ --arg pk "$UNSANDBOX_PUBLIC_KEY" \ --arg sk "$UNSANDBOX_SECRET_KEY" \ '{language: $lang, code: $code, env: {UNSANDBOX_PUBLIC_KEY: $pk, UNSANDBOX_SECRET_KEY: $sk}}') fi local timestamp=$(date +%s) local signature=$(generate_hmac_signature "POST" "/execute" "$body" "$timestamp") # Execute via API with timeout - pipe body via stdin to avoid arg length limits debug "Executing $example_file ($api_lang) via API" api_response=$(echo "$body" | curl -s -X POST "${UNSANDBOX_API_URL}/execute" \ -H "Authorization: Bearer ${UNSANDBOX_PUBLIC_KEY}" \ -H "X-Timestamp: ${timestamp}" \ -H "X-Signature: ${signature}" \ -H "Content-Type: application/json" \ --max-time "$TIMEOUT_SECONDS" \ --data @- \ 2>&1) exit_code=$? # Extract results from API response if [[ $exit_code -eq 0 ]]; then stdout_content=$(safe_json_extract "$api_response" "stdout") stderr_content=$(safe_json_extract "$api_response" "stderr") exit_code=$(safe_json_extract "$api_response" "exit_code") # Treat empty stderr as success if [[ -z "$stderr_content" || "$stderr_content" == "null" ]]; then stderr_content="" fi else stderr_content="API request failed (curl exit code $exit_code)" fi fi elapsed_time=$(( ($(date +%s%N) - start_time) / 1000000 )) # Convert to milliseconds # Check if execution was successful if [[ "$exit_code" == "0" || -z "$exit_code" ]]; then log_pass "$example_file ($language) - ${elapsed_time}ms [$execution_method]" LANGUAGE_STATS[$language]=$((${LANGUAGE_STATS[$language]} + 1)) EXECUTION_TIMES[$language]=$((${EXECUTION_TIMES[$language]:-0} + elapsed_time)) TOTAL_VALIDATED=$((TOTAL_VALIDATED + 1)) else log_fail "$example_file ($language) - exit code $exit_code [$execution_method]" # Always show error details for failures if [[ -n "$stderr_content" && "$stderr_content" != "null" ]]; then echo " stderr: ${stderr_content:0:200}" >&2 fi if [[ -n "$stdout_content" && "$stdout_content" != "null" ]]; then echo " stdout: ${stdout_content:0:200}" >&2 fi if [[ -n "$api_response" && "$execution_method" == "api" ]]; then # Check for API error messages local error_msg=$(echo "$api_response" | jq -r '.error // .message // empty' 2>/dev/null) if [[ -n "$error_msg" ]]; then echo " API error: $error_msg" >&2 fi fi if [[ -n "$stderr_content" ]]; then debug "stderr: $stderr_content" fi TOTAL_FAILED=$((TOTAL_FAILED + 1)) fi # Save result for JSON report # Use same pass/fail logic as the log output (empty exit_code = pass) local result_status="pass" if [[ -n "$exit_code" && "$exit_code" != "0" ]]; then result_status="fail" fi local result_exit_code="${exit_code:-0}" # Default to 0 if empty debug "Writing result to $result_file (TEMP_DIR=$TEMP_DIR)" cat > "$result_file" </dev/null || true pids=("${pids[@]:1}") job_count=$((job_count - 1)) fi done # Wait for ALL background jobs to complete (not just tracked pids) # Using bare 'wait' ensures we catch all subprocesses, even those # whose pids were incorrectly removed from the array by wait -n wait # Extra safety margin for filesystem sync sleep 0.5 # Aggregate results from result files (since subshell variables don't propagate) aggregate_results } # Aggregate results from temp files into parent shell variables aggregate_results() { local result_file TOTAL_VALIDATED=0 TOTAL_FAILED=0 # Count result files and their status debug "Looking for result files in $TEMP_DIR" local file_count=$(ls "$TEMP_DIR"/result-*.json 2>/dev/null | wc -l) debug "Found $file_count result files" for result_file in "$TEMP_DIR"/result-*.json; do [[ -f "$result_file" ]] || continue local status status=$(jq -r '.status // "unknown"' "$result_file" 2>/dev/null || echo "unknown") local lang lang=$(jq -r '.language // "unknown"' "$result_file" 2>/dev/null || echo "unknown") local time_ms time_ms=$(jq -r '.execution_time_ms // 0' "$result_file" 2>/dev/null || echo "0") if [[ "$status" == "pass" ]]; then TOTAL_VALIDATED=$((TOTAL_VALIDATED + 1)) LANGUAGE_STATS[$lang]=$((${LANGUAGE_STATS[$lang]:-0} + 1)) EXECUTION_TIMES[$lang]=$((${EXECUTION_TIMES[$lang]:-0} + time_ms)) elif [[ "$status" == "fail" ]]; then TOTAL_FAILED=$((TOTAL_FAILED + 1)) fi done debug "Aggregated: $TOTAL_VALIDATED validated, $TOTAL_FAILED failed" } # Find all example files find_examples() { if [[ ! -d "$EXAMPLES_DIR" ]]; then log_warn "Examples directory not found: $EXAMPLES_DIR" return 1 fi # Find all files in examples directories # Look for common example patterns and extensions # Includes Python (.py), C (.c), JavaScript, Go, Rust, Java, etc. find "$EXAMPLES_DIR" \ -path "*/examples/*" \ \( -type f -name "*.py" -o -name "*.js" -o -name "*.go" -o \ -name "*.rs" -o -name "*.java" -o -name "*.rb" -o -name "*.php" \ -o -name "*.ts" -o -name "*.cpp" -o -name "*.c" -o -name "*.sh" \ -o -name "*.pl" -o -name "*.example" \) 2>/dev/null | \ sort } # Generate JSON report generate_json_report() { local timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ") local timestamp_readable=$(date -u "+%Y-%m-%d %H:%M:%S UTC") local success_rate="0" if [[ $((TOTAL_VALIDATED + TOTAL_FAILED)) -gt 0 ]]; then success_rate=$(echo "scale=1; $TOTAL_VALIDATED * 100 / ($TOTAL_VALIDATED + $TOTAL_FAILED)" | bc 2>/dev/null || echo "0") fi # Calculate average execution time per language local lang_times=() for lang in "${!LANGUAGE_STATS[@]}"; do local count=${LANGUAGE_STATS[$lang]} local total_time=${EXECUTION_TIMES[$lang]:-0} local avg_time=0 if [[ $count -gt 0 ]]; then avg_time=$((total_time / count)) fi lang_times+=(" {\"language\": \"$lang\", \"validated\": $count, \"total_time_ms\": $total_time, \"avg_time_ms\": $avg_time}") done # Build JSON report cat > "$REPORT_JSON" </dev/null || echo "0") fi # Build language table rows local lang_rows="" for lang in "${!LANGUAGE_STATS[@]}"; do local count=${LANGUAGE_STATS[$lang]} local avg_time=${EXECUTION_TIMES[$lang]:-0} if [[ $count -gt 0 ]]; then avg_time=$((avg_time / count)) fi lang_rows+=" $lang$count${avg_time}ms\n" done # Determine status badge local status_color="green" local status_text="All Passing" if [[ $TOTAL_FAILED -gt 0 ]]; then status_color="red" status_text="Some Failures" elif [[ $TOTAL_EXAMPLES -eq 0 ]]; then status_color="yellow" status_text="No Examples Found" fi cat > "$REPORT_HTML" <<'HTMLEOF' SDK Examples Validation Report

SDK Examples Validation Report

Ensuring all documentation examples actually work

STATUS_TEXT
TOTAL_EXAMPLES
Total Examples
TOTAL_VALIDATED
Validated
TOTAL_FAILED
Failed
SUCCESS_RATE%
Success Rate
Language Coverage

Validation statistics by language

LANGUAGE_ROWS
Language Examples Validated Avg Execution Time

Last verified:

TIMESTAMP_READABLE

HTMLEOF # Replace placeholders sed -i "s/STATUS_CLASS/$status_color/g" "$REPORT_HTML" sed -i "s/STATUS_TEXT/$status_text/g" "$REPORT_HTML" sed -i "s/TOTAL_EXAMPLES/$TOTAL_EXAMPLES/g" "$REPORT_HTML" sed -i "s/TOTAL_VALIDATED/$TOTAL_VALIDATED/g" "$REPORT_HTML" sed -i "s/TOTAL_FAILED/$TOTAL_FAILED/g" "$REPORT_HTML" sed -i "s/SUCCESS_RATE/$success_rate/g" "$REPORT_HTML" sed -i "s|LANGUAGE_ROWS|$lang_rows|g" "$REPORT_HTML" sed -i "s/TIMESTAMP_READABLE/$timestamp_readable/g" "$REPORT_HTML" log "HTML report generated: $REPORT_HTML" } # Generate JUnit XML report for CI integration generate_junit_xml() { local junit_file="${PROJECT_ROOT}/science-results.xml" local timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ") local total=$((TOTAL_VALIDATED + TOTAL_FAILED)) cat > "$junit_file" < EOF # Add individual test cases from result files for result_file in "$TEMP_DIR"/result-*.json; do [[ -f "$result_file" ]] || continue local file lang status time_ms exit_code stderr_preview file=$(jq -r '.file // "unknown"' "$result_file" 2>/dev/null) lang=$(jq -r '.language // "unknown"' "$result_file" 2>/dev/null) status=$(jq -r '.status // "unknown"' "$result_file" 2>/dev/null) time_ms=$(jq -r '.execution_time_ms // 0' "$result_file" 2>/dev/null) exit_code=$(jq -r '.exit_code // 0' "$result_file" 2>/dev/null) stderr_preview=$(jq -r '.stderr_preview // ""' "$result_file" 2>/dev/null) # Convert ms to seconds for JUnit local time_sec time_sec=$(echo "scale=3; $time_ms / 1000" | bc 2>/dev/null || echo "0") # XML-escape the file path for use as classname/name local safe_file safe_file=$(echo "$file" | sed 's/&/\&/g; s//\>/g; s/"/\"/g') if [[ "$status" == "pass" ]]; then cat >> "$junit_file" < EOF else local safe_stderr safe_stderr=$(echo "$stderr_preview" | sed 's/&/\&/g; s//\>/g; s/"/\"/g') cat >> "$junit_file" < $safe_stderr EOF fi done cat >> "$junit_file" < EOF log "JUnit XML report generated: $junit_file" } # Main execution main() { log "Starting SDK examples validation" log "Examples directory: $EXAMPLES_DIR" log "Results directory: $RESULTS_DIR" log "Parallel jobs: $PARALLEL_JOBS" # Check for HMAC credentials if has_api_credentials; then log "HMAC credentials detected - examples will execute with API access" else log_warn "No credentials set - examples will run locally (may exit early)" fi # Find all examples local examples examples=$(find_examples) if [[ -z "$examples" ]]; then log_warn "No examples found in $EXAMPLES_DIR" else TOTAL_EXAMPLES=$(echo "$examples" | wc -l) log "Found $TOTAL_EXAMPLES example files" # Validate examples (runs in subshell due to pipe) echo "$examples" | validate_examples_parallel # Re-aggregate in main shell (subshell variables don't propagate) aggregate_results fi # Generate reports log "Generating reports..." generate_json_report generate_html_report generate_junit_xml # Print summary echo "" echo "========================================" echo "SDK Examples Validation Summary" echo "========================================" echo "Total Examples: $TOTAL_EXAMPLES" echo "Validated: $TOTAL_VALIDATED" echo "Failed: $TOTAL_FAILED" if [[ $((TOTAL_VALIDATED + TOTAL_FAILED)) -gt 0 ]]; then local success_rate=$(echo "scale=1; $TOTAL_VALIDATED * 100 / ($TOTAL_VALIDATED + $TOTAL_FAILED)" | bc 2>/dev/null || echo "0") echo "Success Rate: $success_rate%" fi echo "" echo "Reports:" echo " JSON: $REPORT_JSON" echo " HTML: $REPORT_HTML" echo "========================================" # Exit with appropriate code if [[ $TOTAL_FAILED -eq 0 ]] && [[ $TOTAL_EXAMPLES -gt 0 ]]; then log_pass "All examples validated successfully" exit 0 elif [[ $TOTAL_EXAMPLES -eq 0 ]]; then log_warn "No examples found to validate" exit 0 # Not a failure if no examples exist else log_fail "$TOTAL_FAILED example(s) failed validation" exit 1 fi } # Run main function main "$@"