russell@unturf.com
b3ed0f808d
Examples import SDK libraries (from un import execute_code, etc.) which aren't available in the sandbox. Instead of making examples standalone (which defeats the purpose), upload SDK source files via the API's input_files parameter and rewrite import paths to /tmp/input/. Changes: - validate-examples.sh: detect SDK src dir, base64-encode files into input_files JSON, rewrite Python/JS/Ruby/PHP import paths, pipe request body via stdin to avoid arg length limits - validate-examples.sh: add JUnit XML generation (science-results.xml) - .gitlab-ci.yml: remove allow_failure from science-validate-examples and validate-examples jobs - .gitignore: add science-results/ (CI artifacts, not source) - git rm science-results/ (committed "100% pass" was a lie)
473 lines
16 KiB
YAML
473 lines
16 KiB
YAML
stages:
|
|
- pre
|
|
- build
|
|
- test
|
|
- science
|
|
- validate
|
|
- document
|
|
- report
|
|
|
|
default:
|
|
tags:
|
|
- build
|
|
|
|
# ============================================================================
|
|
# MANUAL TRIGGER: Emergency Cleanup
|
|
# ============================================================================
|
|
# Click play to clean up orphaned test resources (services, sessions, etc.)
|
|
# Runs in parallel for speed - useful when system is hosed
|
|
manual-cleanup:
|
|
stage: pre
|
|
script:
|
|
- echo "=== Emergency Cleanup ==="
|
|
- echo "Building un CLI first..."
|
|
- bash scripts/build-clients.sh
|
|
- echo "Running parallel cleanup..."
|
|
- bash scripts/cleanup-test-resources.sh
|
|
rules:
|
|
- when: manual
|
|
allow_failure: true
|
|
|
|
# ============================================================================
|
|
# MANUAL TRIGGER: Run Full Test Matrix On Demand
|
|
# ============================================================================
|
|
# Click the play button in GitLab UI to run the full 42-language matrix
|
|
# without needing to create a tag. Useful for load testing & pool churn.
|
|
#
|
|
# Shows as a play button ▶️ on every pipeline - click to burn the pool!
|
|
manual-burn-pool:
|
|
stage: test
|
|
script:
|
|
- echo "=== MANUAL POOL BURN ==="
|
|
- echo "Generating full 42-language test matrix..."
|
|
- |
|
|
# Create changes.json to satisfy generate-matrix.sh
|
|
echo '{"test_all": true, "changed_langs": []}' > changes.json
|
|
- FORCE_FULL_MATRIX=true bash scripts/generate-matrix.sh
|
|
- cat test-matrix.yml
|
|
artifacts:
|
|
paths:
|
|
- test-matrix.yml
|
|
- changes.json
|
|
expire_in: 1 hour
|
|
rules:
|
|
- if: $CI_PIPELINE_SOURCE == "push"
|
|
when: manual
|
|
allow_failure: true
|
|
- if: $CI_PIPELINE_SOURCE == "web"
|
|
when: manual
|
|
allow_failure: true
|
|
- if: $CI_COMMIT_TAG
|
|
when: never
|
|
|
|
manual-burn-pool-trigger:
|
|
stage: test
|
|
needs:
|
|
- manual-burn-pool
|
|
resource_group: inception-test-pool
|
|
trigger:
|
|
include:
|
|
- artifact: test-matrix.yml
|
|
job: manual-burn-pool
|
|
strategy: depend
|
|
rules:
|
|
- when: on_success
|
|
|
|
# CI variables UNSANDBOX_PUBLIC_KEY, UNSANDBOX_SECRET_KEY, DEPLOY_KEY
|
|
# are configured in GitLab Settings > CI/CD > Variables
|
|
|
|
# ============================================================================
|
|
# STAGE 1: Detect Changes
|
|
# ============================================================================
|
|
detect-changes:
|
|
stage: pre
|
|
script:
|
|
- bash scripts/detect-changes.sh > changes.json
|
|
- cat changes.json
|
|
artifacts:
|
|
paths:
|
|
- changes.json
|
|
expire_in: 1 hour
|
|
only:
|
|
- main
|
|
- /^\d+\.\d+\.\d+$/
|
|
|
|
# ============================================================================
|
|
# STAGE 2: Generate Dynamic Matrix
|
|
# ============================================================================
|
|
generate-matrix:
|
|
stage: pre
|
|
needs:
|
|
- detect-changes
|
|
script:
|
|
- bash scripts/generate-matrix.sh
|
|
artifacts:
|
|
paths:
|
|
- test-matrix.yml
|
|
expire_in: 1 hour
|
|
only:
|
|
- main
|
|
- /^\d+\.\d+\.\d+$/
|
|
|
|
# ============================================================================
|
|
# STAGE 3: Build SDKs (only changed ones)
|
|
# ============================================================================
|
|
build:
|
|
stage: build
|
|
script:
|
|
- bash scripts/build-clients.sh
|
|
- |
|
|
# Build C examples
|
|
if [ -d "clients/c" ] && [ -f "clients/c/Makefile" ]; then
|
|
echo "Building C examples..."
|
|
make -C clients/c compile-examples || true
|
|
fi
|
|
artifacts:
|
|
paths:
|
|
- build/
|
|
- clients/c/examples/*.o
|
|
expire_in: 1 hour
|
|
only:
|
|
- main
|
|
- /^\d+\.\d+\.\d+$/
|
|
|
|
# ============================================================================
|
|
# STAGE 4: Test Matrix (child pipeline trigger)
|
|
# ============================================================================
|
|
trigger-test-matrix:
|
|
stage: test
|
|
needs:
|
|
- generate-matrix
|
|
- build
|
|
resource_group: inception-test-pool
|
|
trigger:
|
|
include:
|
|
- artifact: test-matrix.yml
|
|
job: generate-matrix
|
|
strategy: depend
|
|
only:
|
|
- main
|
|
- /^\d+\.\d+\.\d+$/
|
|
|
|
# ============================================================================
|
|
# STAGE 5: Science Jobs (Pool Burning)
|
|
# ============================================================================
|
|
science-validate-examples:
|
|
stage: science
|
|
needs:
|
|
- build
|
|
script:
|
|
# Set up Python venv with dependencies for example validation
|
|
- python3 -m venv .venv
|
|
- source .venv/bin/activate
|
|
- pip install --quiet requests aiohttp
|
|
- bash scripts/validate-examples.sh
|
|
artifacts:
|
|
reports:
|
|
junit: science-results.xml
|
|
paths:
|
|
- science-results/
|
|
expire_in: 30 days
|
|
only:
|
|
- main
|
|
- /^\d+\.\d+\.\d+$/
|
|
|
|
science-lint-sdks:
|
|
stage: science
|
|
script:
|
|
- bash scripts/science/lint-all-sdks.sh
|
|
artifacts:
|
|
reports:
|
|
junit: lint-results.xml
|
|
paths:
|
|
- lint-results/
|
|
expire_in: 30 days
|
|
allow_failure: true
|
|
only:
|
|
- main
|
|
- /^\d+\.\d+\.\d+$/
|
|
|
|
science-benchmark-clients:
|
|
stage: science
|
|
script:
|
|
- bash scripts/science/benchmark-clients.sh
|
|
artifacts:
|
|
reports:
|
|
junit: benchmark-results.xml
|
|
paths:
|
|
- benchmark-results/
|
|
expire_in: 30 days
|
|
allow_failure: true
|
|
only:
|
|
- main
|
|
- /^\d+\.\d+\.\d+$/
|
|
|
|
# ============================================================================
|
|
# STAGE 6: Validate Examples (from science job artifacts)
|
|
# ============================================================================
|
|
validate-examples:
|
|
stage: validate
|
|
needs:
|
|
- science-validate-examples
|
|
script:
|
|
- mkdir -p science-results
|
|
- |
|
|
if [ -f science-results/examples-validation-results.json ]; then
|
|
echo "Example validation results found"
|
|
cat science-results/examples-validation-results.json
|
|
|
|
# Extract stats and display summary
|
|
TOTAL=$(jq '.summary.total_examples' science-results/examples-validation-results.json 2>/dev/null || echo "0")
|
|
PASSED=$(jq '.summary.total_validated' science-results/examples-validation-results.json 2>/dev/null || echo "0")
|
|
FAILED=$(jq '.summary.total_failed' science-results/examples-validation-results.json 2>/dev/null || echo "0")
|
|
|
|
echo ""
|
|
echo "Summary: $PASSED/$TOTAL passed, $FAILED failed"
|
|
else
|
|
echo "No example validation results found"
|
|
fi
|
|
artifacts:
|
|
paths:
|
|
- science-results/
|
|
expire_in: 30 days
|
|
only:
|
|
- main
|
|
- /^\d+\.\d+\.\d+$/
|
|
|
|
# ============================================================================
|
|
# STAGE 7: Generate Documentation
|
|
# ============================================================================
|
|
generate-documentation:
|
|
stage: document
|
|
needs:
|
|
- validate-examples
|
|
script:
|
|
- mkdir -p docs
|
|
- bash scripts/generate-docs.sh || true
|
|
- |
|
|
VALIDATION_TIME=$(date +%s)
|
|
CURRENT_TIME=$(date +%s)
|
|
MINUTES_AGO=$((($CURRENT_TIME - $VALIDATION_TIME) / 60))
|
|
echo "Last verified: $MINUTES_AGO minutes ago" > docs/VERIFICATION_TIMESTAMP.txt
|
|
echo "Generated: $(date -u +"%Y-%m-%dT%H:%M:%SZ")" >> docs/VERIFICATION_TIMESTAMP.txt
|
|
- ls -la docs/ || echo "No documentation generated"
|
|
artifacts:
|
|
paths:
|
|
- docs/
|
|
- docs/VERIFICATION_TIMESTAMP.txt
|
|
expire_in: 30 days
|
|
allow_failure: true
|
|
only:
|
|
- main
|
|
- /^\d+\.\d+\.\d+$/
|
|
|
|
# ============================================================================
|
|
# STAGE 8: Report
|
|
# ============================================================================
|
|
report:
|
|
stage: report
|
|
needs:
|
|
- generate-documentation
|
|
script:
|
|
- bash scripts/filter-results.sh
|
|
- |
|
|
if [ -f science-results/examples-validation-results.json ]; then
|
|
echo "Including example validation stats in final report..."
|
|
EXAMPLE_STATS=$(cat science-results/examples-validation-results.json)
|
|
jq --argjson examples "$EXAMPLE_STATS" '.properties += [{name: "example_validation_passed", value: $examples.examples.passed | tostring}, {name: "example_validation_failed", value: $examples.examples.failed | tostring}]' final-report.xml > final-report.xml.tmp && mv final-report.xml.tmp final-report.xml || true
|
|
fi
|
|
- echo "Including documentation generation artifacts..."
|
|
- ls -la docs/ 2>/dev/null | tail -5 || echo "No documentation available"
|
|
artifacts:
|
|
reports:
|
|
junit: final-report.xml
|
|
paths:
|
|
- reports/
|
|
- science-results/examples-validation-results.json
|
|
- docs/
|
|
- final-report.xml
|
|
expire_in: 30 days
|
|
only:
|
|
- main
|
|
- /^\d+\.\d+\.\d+$/
|
|
|
|
# ============================================================================
|
|
# STAGE 8b: Collect Pool Metrics During Tests (Tags Only)
|
|
# ============================================================================
|
|
collect-pool-metrics:
|
|
stage: test
|
|
script:
|
|
- echo "Starting pool metrics collector..."
|
|
- mkdir -p pool-metrics
|
|
- bash scripts/collect-pool-metrics.sh start pool-metrics
|
|
- echo "Collector running, waiting for tests to complete..."
|
|
- |
|
|
# Poll until trigger-test-matrix child pipeline completes (max 20 min)
|
|
# Stop early when child pipeline finishes to avoid skewing averages
|
|
for i in $(seq 1 240); do
|
|
sleep 5
|
|
# Check if child pipeline is done via GitLab API
|
|
CHILD_STATUS=$(curl -s "https://git.unturf.com/api/v4/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/bridges" 2>/dev/null | jq -r '.[0].downstream_pipeline.status // "pending"' 2>/dev/null || echo "pending")
|
|
if [ "$CHILD_STATUS" = "success" ] || [ "$CHILD_STATUS" = "failed" ]; then
|
|
echo "Child pipeline finished with status: $CHILD_STATUS"
|
|
break
|
|
fi
|
|
bash scripts/collect-pool-metrics.sh status pool-metrics || true
|
|
done
|
|
- echo "Stopping collector..."
|
|
- bash scripts/collect-pool-metrics.sh stop pool-metrics
|
|
- echo "Generating pool metrics report..."
|
|
- bash scripts/collect-pool-metrics.sh report pool-metrics
|
|
- cat pool-metrics/pool-metrics.json
|
|
artifacts:
|
|
paths:
|
|
- pool-metrics/
|
|
expire_in: 90 days
|
|
rules:
|
|
- if: '$CI_COMMIT_TAG =~ /^\d+\.\d+\.\d+$/'
|
|
allow_failure: true
|
|
|
|
# ============================================================================
|
|
# STAGE 9: Performance Report (Tags Only)
|
|
# ============================================================================
|
|
perf-report:
|
|
stage: report
|
|
needs:
|
|
- trigger-test-matrix
|
|
- job: build
|
|
artifacts: true
|
|
- job: collect-pool-metrics
|
|
artifacts: true
|
|
optional: true
|
|
variables:
|
|
GIT_STRATEGY: clone
|
|
GIT_DEPTH: 0
|
|
script:
|
|
- echo "========================================"
|
|
- echo "Performance Report Generator"
|
|
- echo "Tag - $CI_COMMIT_TAG"
|
|
- echo "========================================"
|
|
- echo "$CI_COMMIT_TAG" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$' || exit 1
|
|
- echo "Step 0 - Waiting 60s for pipeline data to settle..."
|
|
- sleep 60
|
|
- echo "Step 1 - Setting up git on main branch..."
|
|
- test -z "$DEPLOY_KEY" && echo "WARN - No DEPLOY_KEY, skipping commit" && exit 0 || true
|
|
- mkdir -p ~/.ssh
|
|
- echo "$DEPLOY_KEY" | base64 -d > ~/.ssh/id_ed25519
|
|
- chmod 600 ~/.ssh/id_ed25519
|
|
- ssh-keyscan -p 2222 -H git.unturf.com >> ~/.ssh/known_hosts 2>/dev/null
|
|
- git remote set-url origin "ssh://git@git.unturf.com:2222/${CI_PROJECT_PATH}.git"
|
|
- git config user.email "ci@unturf.com"
|
|
- git config user.name "GitLab CI"
|
|
- git fetch origin main
|
|
- git checkout -B main origin/main
|
|
- echo "Step 2 - Generating performance report..."
|
|
- bash scripts/generate-perf-report.sh "$CI_COMMIT_TAG" || echo "WARN - Report generation failed"
|
|
- echo "Step 3 - Generating performance charts..."
|
|
- test -x build/un && test -f "reports/$CI_COMMIT_TAG/perf.json" && build/un -a -f "reports/$CI_COMMIT_TAG/perf.json" scripts/generate-perf-charts.py || echo "WARN - Charts skipped (build/un or perf.json missing)"
|
|
- mv -f *.png "reports/$CI_COMMIT_TAG/" 2>/dev/null || true
|
|
- echo "Step 4 - Adding pool metrics if available..."
|
|
- test -f pool-metrics/pool-metrics.json && cp pool-metrics/pool-metrics.json "reports/$CI_COMMIT_TAG/" && echo " ✓ Pool metrics added" || echo " (no pool metrics)"
|
|
- test -f pool-metrics/raw-samples.csv && cp pool-metrics/raw-samples.csv "reports/$CI_COMMIT_TAG/" || true
|
|
- test -f pool-metrics/raw-pools.csv && cp pool-metrics/raw-pools.csv "reports/$CI_COMMIT_TAG/" && echo " ✓ Per-pool metrics added" || true
|
|
- echo "Step 5 - Report contents..."
|
|
- ls -la "reports/$CI_COMMIT_TAG/" 2>/dev/null || echo "No report directory"
|
|
- echo "Step 6 - Committing to main..."
|
|
- git add reports/
|
|
- git diff --cached --quiet && echo "No changes to commit" || git commit -m "perf - Add performance report for $CI_COMMIT_TAG [ci skip]"
|
|
- git push origin main || echo "Nothing to push"
|
|
- echo "Done"
|
|
artifacts:
|
|
paths:
|
|
- reports/
|
|
expire_in: 90 days
|
|
when: always
|
|
rules:
|
|
- if: '$CI_COMMIT_TAG =~ /^\d+\.\d+\.\d+$/'
|
|
allow_failure: true
|
|
|
|
# ============================================================================
|
|
# STAGE 9b: Aggregated Performance Analysis (cross-release trends)
|
|
# ============================================================================
|
|
perf-aggregate-report:
|
|
stage: report
|
|
needs:
|
|
- perf-report
|
|
variables:
|
|
GIT_STRATEGY: clone
|
|
GIT_DEPTH: 0
|
|
script:
|
|
- echo "========================================"
|
|
- echo "Aggregated Performance Analysis"
|
|
- echo "========================================"
|
|
- echo "Step 1 - Discovering releases from git tags..."
|
|
- VERSIONS=$(git tag | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' | sort -V)
|
|
- echo "Found tagged releases:"
|
|
- echo "$VERSIONS"
|
|
- |
|
|
if [ -z "$VERSIONS" ]; then
|
|
echo "ERROR - No version tags found"
|
|
exit 1
|
|
fi
|
|
- echo "Step 2 - Copying perf.json files for tagged releases..."
|
|
- for v in $VERSIONS; do if [ -f "reports/$v/perf.json" ]; then cp "reports/$v/perf.json" "perf-$v.json"; echo " ✓ $v"; fi; done
|
|
- |
|
|
FILES=$(ls perf-*.json 2>/dev/null | sed 's/^/-f /' | tr '\n' ' ')
|
|
if [ -z "$FILES" ]; then
|
|
echo "ERROR - No perf JSON files found"
|
|
exit 1
|
|
fi
|
|
- echo "Step 3 - Generating charts via UN..."
|
|
- |
|
|
if [ -x build/un ]; then
|
|
build/un -a $FILES scripts/generate-aggregated-charts.py
|
|
else
|
|
echo "WARN - Charts skipped (build/un missing)"
|
|
fi
|
|
- rm -f perf-*.json
|
|
- mv -f *.png reports/ 2>/dev/null || true
|
|
- echo "Step 4 - Generating markdown report..."
|
|
- python3 scripts/aggregate-performance-reports.py reports -o AGGREGATED-PERFORMANCE.md
|
|
- echo "Step 5 - Setting up git..."
|
|
- |
|
|
if [ -z "$DEPLOY_KEY" ]; then
|
|
echo "WARN - No DEPLOY_KEY, skipping commit"
|
|
exit 0
|
|
fi
|
|
- mkdir -p ~/.ssh
|
|
- echo "$DEPLOY_KEY" | base64 -d > ~/.ssh/id_ed25519
|
|
- chmod 600 ~/.ssh/id_ed25519
|
|
- ssh-keyscan -p 2222 -H git.unturf.com >> ~/.ssh/known_hosts 2>/dev/null
|
|
- git remote set-url origin "ssh://git@git.unturf.com:2222/${CI_PROJECT_PATH}.git"
|
|
- git config user.email "ci@unturf.com"
|
|
- git config user.name "GitLab CI"
|
|
- git fetch origin main
|
|
- echo "Step 6 - Stashing artifact files temporarily..."
|
|
- git add -A
|
|
- git stash push -m "Temporary stash of artifacts and generated files"
|
|
- echo "Step 7 - Checking out main branch..."
|
|
- git checkout main
|
|
- git pull origin main || echo "Already up to date"
|
|
- echo "Step 8 - Restoring artifacts and committing..."
|
|
- git stash pop || echo "No stash to pop"
|
|
- git add reports/ AGGREGATED-PERFORMANCE.md
|
|
- |
|
|
if git diff --cached --quiet; then
|
|
echo "No changes to commit"
|
|
else
|
|
git commit -m "perf: Update aggregated performance analysis [ci skip]"
|
|
fi
|
|
- git push origin main || echo "Nothing to push"
|
|
- echo "Step 9 - Verifying results..."
|
|
- echo "✓ Aggregated report generated with charts"
|
|
- ls -lh AGGREGATED-PERFORMANCE.md
|
|
- ls -lh reports/aggregated-*.png 2>/dev/null || echo " (charts may not have generated)"
|
|
artifacts:
|
|
paths:
|
|
- AGGREGATED-PERFORMANCE.md
|
|
- reports/aggregated-*.png
|
|
expire_in: 90 days
|
|
when: always
|
|
rules:
|
|
- if: '$CI_COMMIT_TAG =~ /^\d+\.\d+\.\d+$/'
|
|
allow_failure: true
|