fix: Upload SDK files via input_files for example validation
Examples import SDK libraries (from un import execute_code, etc.) which aren't available in the sandbox. Instead of making examples standalone (which defeats the purpose), upload SDK source files via the API's input_files parameter and rewrite import paths to /tmp/input/. Changes: - validate-examples.sh: detect SDK src dir, base64-encode files into input_files JSON, rewrite Python/JS/Ruby/PHP import paths, pipe request body via stdin to avoid arg length limits - validate-examples.sh: add JUnit XML generation (science-results.xml) - .gitlab-ci.yml: remove allow_failure from science-validate-examples and validate-examples jobs - .gitignore: add science-results/ (CI artifacts, not source) - git rm science-results/ (committed "100% pass" was a lie)
This commit is contained in:
parent
710eb5d9ac
commit
b3ed0f808d
5 changed files with 173 additions and 217 deletions
2
.gitignore
vendored
2
.gitignore
vendored
|
|
@ -40,6 +40,8 @@ Thumbs.db
|
|||
/output/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
science-results/
|
||||
science-results.xml
|
||||
clients/c/un
|
||||
clients/c/examples/fibonacci
|
||||
clients/c/examples/hello_world
|
||||
|
|
|
|||
|
|
@ -168,7 +168,6 @@ science-validate-examples:
|
|||
paths:
|
||||
- science-results/
|
||||
expire_in: 30 days
|
||||
allow_failure: true
|
||||
only:
|
||||
- main
|
||||
- /^\d+\.\d+\.\d+$/
|
||||
|
|
@ -231,7 +230,6 @@ validate-examples:
|
|||
paths:
|
||||
- science-results/
|
||||
expire_in: 30 days
|
||||
allow_failure: true
|
||||
only:
|
||||
- main
|
||||
- /^\d+\.\d+\.\d+$/
|
||||
|
|
|
|||
|
|
@ -1,186 +0,0 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>SDK Examples Validation Report</title>
|
||||
<style>
|
||||
* { margin: 0; padding: 0; box-sizing: border-box; }
|
||||
body {
|
||||
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, sans-serif;
|
||||
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
||||
min-height: 100vh;
|
||||
padding: 2rem;
|
||||
}
|
||||
.container {
|
||||
max-width: 1000px;
|
||||
margin: 0 auto;
|
||||
background: white;
|
||||
border-radius: 12px;
|
||||
box-shadow: 0 20px 60px rgba(0,0,0,0.3);
|
||||
overflow: hidden;
|
||||
}
|
||||
.header {
|
||||
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
||||
color: white;
|
||||
padding: 3rem 2rem;
|
||||
text-align: center;
|
||||
}
|
||||
.header h1 {
|
||||
font-size: 2.5rem;
|
||||
margin-bottom: 0.5rem;
|
||||
}
|
||||
.header p {
|
||||
font-size: 1.1rem;
|
||||
opacity: 0.9;
|
||||
}
|
||||
.status-badge {
|
||||
display: inline-block;
|
||||
padding: 0.5rem 1.5rem;
|
||||
border-radius: 50px;
|
||||
font-weight: 600;
|
||||
margin-top: 1rem;
|
||||
background: rgba(255,255,255,0.2);
|
||||
color: white;
|
||||
border: 2px solid white;
|
||||
}
|
||||
.status-badge.green { background: rgba(76, 175, 80, 0.8); border-color: #4CAF50; }
|
||||
.status-badge.red { background: rgba(244, 67, 54, 0.8); border-color: #F44336; }
|
||||
.status-badge.yellow { background: rgba(255, 193, 7, 0.8); border-color: #FFC107; }
|
||||
.content { padding: 3rem 2rem; }
|
||||
.stats-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
|
||||
gap: 2rem;
|
||||
margin-bottom: 3rem;
|
||||
}
|
||||
.stat-card {
|
||||
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
||||
color: white;
|
||||
padding: 2rem;
|
||||
border-radius: 12px;
|
||||
text-align: center;
|
||||
}
|
||||
.stat-card .value {
|
||||
font-size: 3rem;
|
||||
font-weight: bold;
|
||||
margin-bottom: 0.5rem;
|
||||
}
|
||||
.stat-card .label {
|
||||
font-size: 0.9rem;
|
||||
opacity: 0.9;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 1px;
|
||||
}
|
||||
.success-rate {
|
||||
background: linear-gradient(135deg, #4CAF50 0%, #45a049 100%);
|
||||
}
|
||||
table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
margin-top: 2rem;
|
||||
}
|
||||
table th {
|
||||
background: #f5f5f5;
|
||||
padding: 1rem;
|
||||
text-align: left;
|
||||
font-weight: 600;
|
||||
color: #333;
|
||||
border-bottom: 2px solid #ddd;
|
||||
}
|
||||
table td {
|
||||
padding: 0.8rem 1rem;
|
||||
border-bottom: 1px solid #ddd;
|
||||
}
|
||||
table tr:hover {
|
||||
background: #fafafa;
|
||||
}
|
||||
.footer {
|
||||
background: #f9f9f9;
|
||||
padding: 2rem;
|
||||
text-align: center;
|
||||
color: #666;
|
||||
font-size: 0.9rem;
|
||||
border-top: 1px solid #ddd;
|
||||
}
|
||||
.checkmark { color: #4CAF50; font-weight: bold; }
|
||||
.cross { color: #F44336; font-weight: bold; }
|
||||
.section-title {
|
||||
font-size: 1.5rem;
|
||||
font-weight: 600;
|
||||
margin-top: 2rem;
|
||||
margin-bottom: 1rem;
|
||||
color: #333;
|
||||
}
|
||||
@media (max-width: 600px) {
|
||||
.header h1 { font-size: 1.8rem; }
|
||||
.stats-grid { grid-template-columns: 1fr; }
|
||||
.header { padding: 2rem 1.5rem; }
|
||||
.content { padding: 2rem 1.5rem; }
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<div class="header">
|
||||
<h1>SDK Examples Validation Report</h1>
|
||||
<p>Ensuring all documentation examples actually work</p>
|
||||
<div class="status-badge green">All Passing</div>
|
||||
</div>
|
||||
|
||||
<div class="content">
|
||||
<div class="stats-grid">
|
||||
<div class="stat-card">
|
||||
<div class="value">46</div>
|
||||
<div class="label">Total Examples</div>
|
||||
</div>
|
||||
<div class="stat-card success-rate">
|
||||
<div class="value">45</div>
|
||||
<div class="label">Validated</div>
|
||||
</div>
|
||||
<div class="stat-card">
|
||||
<div class="value">0</div>
|
||||
<div class="label">Failed</div>
|
||||
</div>
|
||||
<div class="stat-card success-rate">
|
||||
<div class="value">100.0%</div>
|
||||
<div class="label">Success Rate</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="section-title">Language Coverage</div>
|
||||
<p style="color: #666; margin-bottom: 1rem;">Validation statistics by language</p>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Language</th>
|
||||
<th>Examples Validated</th>
|
||||
<th>Avg Execution Time</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr><td>java</td><td>6</td><td>171ms</td></tr>
|
||||
<tr><td>ruby</td><td>4</td><td>165ms</td></tr>
|
||||
<tr><td>c</td><td>3</td><td>316ms</td></tr>
|
||||
<tr><td>python</td><td>14</td><td>331ms</td></tr>
|
||||
<tr><td>php</td><td>4</td><td>174ms</td></tr>
|
||||
<tr><td>go</td><td>4</td><td>169ms</td></tr>
|
||||
<tr><td>rust</td><td>4</td><td>168ms</td></tr>
|
||||
<tr><td>javascript</td><td>6</td><td>165ms</td></tr>
|
||||
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<div style="margin-top: 2rem; padding: 1.5rem; background: #e3f2fd; border-left: 4px solid #2196F3; border-radius: 4px;">
|
||||
<p style="color: #1976D2; font-weight: 500; margin-bottom: 0.5rem;">Last verified:</p>
|
||||
<p style="color: #555;">2026-02-14 17:53:13 UTC</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="footer">
|
||||
<p>This report validates that SDK examples execute successfully through the unsandbox API.</p>
|
||||
<p style="margin-top: 0.5rem; color: #999;">Generated automatically by the CI/CD pipeline</p>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
{
|
||||
"report_type": "examples_validation",
|
||||
"timestamp": "2026-02-14T17:53:13Z",
|
||||
"timestamp_readable": "2026-02-14 17:53:13 UTC",
|
||||
"summary": {
|
||||
"total_examples": 46,
|
||||
"total_validated": 45,
|
||||
"total_failed": 0,
|
||||
"success_rate": "100.0%"
|
||||
},
|
||||
"language_stats": [
|
||||
{"language": "java", "validated": 6, "total_time_ms": 1029, "avg_time_ms": 171}, {"language": "ruby", "validated": 4, "total_time_ms": 662, "avg_time_ms": 165}, {"language": "c", "validated": 3, "total_time_ms": 950, "avg_time_ms": 316}, {"language": "python", "validated": 14, "total_time_ms": 4644, "avg_time_ms": 331}, {"language": "php", "validated": 4, "total_time_ms": 699, "avg_time_ms": 174}, {"language": "go", "validated": 4, "total_time_ms": 677, "avg_time_ms": 169}, {"language": "rust", "validated": 4, "total_time_ms": 673, "avg_time_ms": 168}, {"language": "javascript", "validated": 6, "total_time_ms": 992, "avg_time_ms": 165}
|
||||
],
|
||||
"notes": "Examples validated through unsandbox API. Each example executed with 30s timeout."
|
||||
}
|
||||
|
|
@ -143,6 +143,87 @@ extract_expected_output() {
|
|||
head -1
|
||||
}
|
||||
|
||||
# Helper: Get SDK source directory from example file path
|
||||
# clients/python/sync/examples/foo.py -> clients/python/sync/src/
|
||||
get_sdk_src_dir() {
|
||||
local example_file=$1
|
||||
local src_dir
|
||||
# Replace /examples/ with /src/ in the path
|
||||
src_dir=$(echo "$example_file" | sed 's|/examples/.*|/src/|')
|
||||
if [[ -d "$src_dir" ]]; then
|
||||
echo "$src_dir"
|
||||
fi
|
||||
}
|
||||
|
||||
# Helper: Build input_files JSON array from SDK source directory
|
||||
# Returns JSON array of {filename, content} objects with base64-encoded content
|
||||
build_input_files_json() {
|
||||
local src_dir=$1
|
||||
local language=$2
|
||||
local input_files="["
|
||||
local first=true
|
||||
|
||||
# Only include relevant source files (skip __pycache__, .pyc, etc.)
|
||||
while IFS= read -r -d '' src_file; do
|
||||
local basename
|
||||
basename=$(basename "$src_file")
|
||||
|
||||
# Skip compiled/cache files
|
||||
case "$basename" in
|
||||
*.pyc|*.pyo|*.class|*.o) continue ;;
|
||||
esac
|
||||
|
||||
# Skip __pycache__ directories
|
||||
[[ "$src_file" == *__pycache__* ]] && continue
|
||||
|
||||
local b64_content
|
||||
b64_content=$(base64 -w0 "$src_file" 2>/dev/null || base64 "$src_file" 2>/dev/null)
|
||||
|
||||
if [[ "$first" == "true" ]]; then
|
||||
first=false
|
||||
else
|
||||
input_files+=","
|
||||
fi
|
||||
input_files+="{\"filename\":\"$basename\",\"content\":\"$b64_content\"}"
|
||||
done < <(find "$src_dir" -maxdepth 1 -type f -print0 2>/dev/null)
|
||||
|
||||
input_files+="]"
|
||||
echo "$input_files"
|
||||
}
|
||||
|
||||
# Helper: Rewrite import paths in example code so they resolve to /tmp/input/
|
||||
# The API places input_files at /tmp/input/<filename>
|
||||
rewrite_import_paths() {
|
||||
local code=$1
|
||||
local language=$2
|
||||
|
||||
case "$language" in
|
||||
python)
|
||||
# Rewrite sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src"))
|
||||
# to sys.path.insert(0, '/tmp/input')
|
||||
code=$(echo "$code" | sed "s|os\.path\.join(os\.path\.dirname(__file__), \"\.\.\", \"src\")|'/tmp/input'|g")
|
||||
code=$(echo "$code" | sed "s|os\.path\.join(os\.path\.dirname(__file__), '\.\.', 'src')|'/tmp/input'|g")
|
||||
;;
|
||||
javascript)
|
||||
# Rewrite from '../src/un_async.js' to '/tmp/input/un_async.js'
|
||||
# Rewrite from '../src/un.js' to '/tmp/input/un.js'
|
||||
code=$(echo "$code" | sed "s|from ['\"]\.\.\/src\/\([^'\"]*\)['\"]|from '/tmp/input/\1'|g")
|
||||
# Also handle require('../src/...')
|
||||
code=$(echo "$code" | sed "s|require(['\"]\.\.\/src\/\([^'\"]*\)['\"])|require('/tmp/input/\1')|g")
|
||||
;;
|
||||
ruby)
|
||||
# Rewrite require_relative '../src/un' to require '/tmp/input/un'
|
||||
code=$(echo "$code" | sed "s|require_relative ['\"]\.\.\/src\/\([^'\"]*\)['\"]|require '/tmp/input/\1'|g")
|
||||
;;
|
||||
php)
|
||||
# Rewrite __DIR__ . '/../src/un.php' to '/tmp/input/un.php'
|
||||
code=$(echo "$code" | sed "s|__DIR__ \. ['\"]\/\.\.\/src\/\([^'\"]*\)['\"]|'/tmp/input/\1'|g")
|
||||
;;
|
||||
esac
|
||||
|
||||
echo "$code"
|
||||
}
|
||||
|
||||
# Helper: Parse JSON response safely
|
||||
safe_json_extract() {
|
||||
local json=$1
|
||||
|
|
@ -277,31 +358,51 @@ validate_example() {
|
|||
# API execution with HMAC authentication
|
||||
api_lang=$(get_api_language "$language")
|
||||
|
||||
# Build JSON body with credentials
|
||||
# Note: SDK files are NOT included because:
|
||||
# 1. Example files are standalone demonstrations (don't actually import SDK)
|
||||
# 2. Large SDK files (C SDK is 400KB+) cause "argument list too long" errors
|
||||
# 3. Examples pass credentials via env which is the documented pattern
|
||||
# Detect SDK source directory and build input_files if available
|
||||
local sdk_src_dir
|
||||
sdk_src_dir=$(get_sdk_src_dir "$example_file")
|
||||
local input_files_json=""
|
||||
|
||||
if [[ -n "$sdk_src_dir" ]]; then
|
||||
debug "SDK source dir: $sdk_src_dir"
|
||||
# Rewrite import paths in the code to use /tmp/input/
|
||||
code=$(rewrite_import_paths "$code" "$language")
|
||||
# Build input_files JSON with base64-encoded SDK files
|
||||
input_files_json=$(build_input_files_json "$sdk_src_dir" "$language")
|
||||
debug "Built input_files with $(echo "$input_files_json" | jq 'length' 2>/dev/null || echo '?') files"
|
||||
fi
|
||||
|
||||
# Build JSON body with credentials and optional input_files
|
||||
local body
|
||||
if [[ -n "$input_files_json" && "$input_files_json" != "[]" ]]; then
|
||||
body=$(jq -n \
|
||||
--arg lang "$api_lang" \
|
||||
--arg code "$code" \
|
||||
--arg pk "$UNSANDBOX_PUBLIC_KEY" \
|
||||
--arg sk "$UNSANDBOX_SECRET_KEY" \
|
||||
--argjson input_files "$input_files_json" \
|
||||
'{language: $lang, code: $code, env: {UNSANDBOX_PUBLIC_KEY: $pk, UNSANDBOX_SECRET_KEY: $sk}, input_files: $input_files}')
|
||||
else
|
||||
body=$(jq -n \
|
||||
--arg lang "$api_lang" \
|
||||
--arg code "$code" \
|
||||
--arg pk "$UNSANDBOX_PUBLIC_KEY" \
|
||||
--arg sk "$UNSANDBOX_SECRET_KEY" \
|
||||
'{language: $lang, code: $code, env: {UNSANDBOX_PUBLIC_KEY: $pk, UNSANDBOX_SECRET_KEY: $sk}}')
|
||||
fi
|
||||
|
||||
local timestamp=$(date +%s)
|
||||
local signature=$(generate_hmac_signature "POST" "/execute" "$body" "$timestamp")
|
||||
|
||||
# Execute via API with timeout
|
||||
# Execute via API with timeout - pipe body via stdin to avoid arg length limits
|
||||
debug "Executing $example_file ($api_lang) via API"
|
||||
api_response=$(curl -s -X POST "${UNSANDBOX_API_URL}/execute" \
|
||||
api_response=$(echo "$body" | curl -s -X POST "${UNSANDBOX_API_URL}/execute" \
|
||||
-H "Authorization: Bearer ${UNSANDBOX_PUBLIC_KEY}" \
|
||||
-H "X-Timestamp: ${timestamp}" \
|
||||
-H "X-Signature: ${signature}" \
|
||||
-H "Content-Type: application/json" \
|
||||
--max-time "$TIMEOUT_SECONDS" \
|
||||
-d "$body" \
|
||||
--data @- \
|
||||
2>&1)
|
||||
|
||||
exit_code=$?
|
||||
|
|
@ -726,6 +827,61 @@ HTMLEOF
|
|||
log "HTML report generated: $REPORT_HTML"
|
||||
}
|
||||
|
||||
# Generate JUnit XML report for CI integration
|
||||
generate_junit_xml() {
|
||||
local junit_file="${PROJECT_ROOT}/science-results.xml"
|
||||
local timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||||
local total=$((TOTAL_VALIDATED + TOTAL_FAILED))
|
||||
|
||||
cat > "$junit_file" <<EOF
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<testsuites name="SDK Examples Validation" tests="$total" failures="$TOTAL_FAILED" time="0" timestamp="$timestamp">
|
||||
<testsuite name="examples" tests="$total" failures="$TOTAL_FAILED">
|
||||
EOF
|
||||
|
||||
# Add individual test cases from result files
|
||||
for result_file in "$TEMP_DIR"/result-*.json; do
|
||||
[[ -f "$result_file" ]] || continue
|
||||
|
||||
local file lang status time_ms exit_code stderr_preview
|
||||
file=$(jq -r '.file // "unknown"' "$result_file" 2>/dev/null)
|
||||
lang=$(jq -r '.language // "unknown"' "$result_file" 2>/dev/null)
|
||||
status=$(jq -r '.status // "unknown"' "$result_file" 2>/dev/null)
|
||||
time_ms=$(jq -r '.execution_time_ms // 0' "$result_file" 2>/dev/null)
|
||||
exit_code=$(jq -r '.exit_code // 0' "$result_file" 2>/dev/null)
|
||||
stderr_preview=$(jq -r '.stderr_preview // ""' "$result_file" 2>/dev/null)
|
||||
|
||||
# Convert ms to seconds for JUnit
|
||||
local time_sec
|
||||
time_sec=$(echo "scale=3; $time_ms / 1000" | bc 2>/dev/null || echo "0")
|
||||
|
||||
# XML-escape the file path for use as classname/name
|
||||
local safe_file
|
||||
safe_file=$(echo "$file" | sed 's/&/\&/g; s/</\</g; s/>/\>/g; s/"/\"/g')
|
||||
|
||||
if [[ "$status" == "pass" ]]; then
|
||||
cat >> "$junit_file" <<EOF
|
||||
<testcase classname="$lang" name="$safe_file" time="$time_sec" />
|
||||
EOF
|
||||
else
|
||||
local safe_stderr
|
||||
safe_stderr=$(echo "$stderr_preview" | sed 's/&/\&/g; s/</\</g; s/>/\>/g; s/"/\"/g')
|
||||
cat >> "$junit_file" <<EOF
|
||||
<testcase classname="$lang" name="$safe_file" time="$time_sec">
|
||||
<failure message="exit code $exit_code">$safe_stderr</failure>
|
||||
</testcase>
|
||||
EOF
|
||||
fi
|
||||
done
|
||||
|
||||
cat >> "$junit_file" <<EOF
|
||||
</testsuite>
|
||||
</testsuites>
|
||||
EOF
|
||||
|
||||
log "JUnit XML report generated: $junit_file"
|
||||
}
|
||||
|
||||
# Main execution
|
||||
main() {
|
||||
log "Starting SDK examples validation"
|
||||
|
|
@ -761,6 +917,7 @@ main() {
|
|||
log "Generating reports..."
|
||||
generate_json_report
|
||||
generate_html_report
|
||||
generate_junit_xml
|
||||
|
||||
# Print summary
|
||||
echo ""
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue