russell@unturf.com
824a555974
Fixed perf-aggregate-report job failure caused by artifact files conflicting with git checkout. The issue: - Job downloads artifacts from perf-report (reports/4.2.5/*) - Tries to checkout main with 'git checkout -B main origin/main' - Fails because artifact files would be overwritten The solution: 1. Stash all artifact files and generated content 2. Checkout main branch cleanly 3. Pull latest changes from origin/main 4. Pop stash to restore artifacts 5. Add and commit aggregated report This preserves the artifact files while properly syncing with remote main.
365 lines
12 KiB
YAML
365 lines
12 KiB
YAML
stages:
|
|
- pre
|
|
- build
|
|
- test
|
|
- science
|
|
- validate
|
|
- document
|
|
- report
|
|
|
|
default:
|
|
tags:
|
|
- build
|
|
|
|
# CI variables UNSANDBOX_PUBLIC_KEY, UNSANDBOX_SECRET_KEY, DEPLOY_KEY
|
|
# are configured in GitLab Settings > CI/CD > Variables
|
|
|
|
# ============================================================================
|
|
# STAGE 1: Detect Changes
|
|
# ============================================================================
|
|
detect-changes:
|
|
stage: pre
|
|
script:
|
|
- bash scripts/detect-changes.sh > changes.json
|
|
- cat changes.json
|
|
artifacts:
|
|
paths:
|
|
- changes.json
|
|
expire_in: 1 hour
|
|
only:
|
|
- main
|
|
- /^\d+\.\d+\.\d+$/
|
|
|
|
# ============================================================================
|
|
# STAGE 2: Generate Dynamic Matrix
|
|
# ============================================================================
|
|
generate-matrix:
|
|
stage: pre
|
|
needs:
|
|
- detect-changes
|
|
script:
|
|
- bash scripts/generate-matrix.sh
|
|
artifacts:
|
|
paths:
|
|
- test-matrix.yml
|
|
expire_in: 1 hour
|
|
only:
|
|
- main
|
|
- /^\d+\.\d+\.\d+$/
|
|
|
|
# ============================================================================
|
|
# STAGE 3: Build SDKs (only changed ones)
|
|
# ============================================================================
|
|
build:
|
|
stage: build
|
|
script:
|
|
- bash scripts/build-clients.sh
|
|
- |
|
|
# Build C examples
|
|
if [ -d "clients/c" ] && [ -f "clients/c/Makefile" ]; then
|
|
echo "Building C examples..."
|
|
make -C clients/c compile-examples || true
|
|
fi
|
|
artifacts:
|
|
paths:
|
|
- build/
|
|
- clients/c/examples/*.o
|
|
expire_in: 1 hour
|
|
only:
|
|
- main
|
|
- /^\d+\.\d+\.\d+$/
|
|
|
|
# ============================================================================
|
|
# STAGE 4: Test Matrix (child pipeline trigger)
|
|
# ============================================================================
|
|
trigger-test-matrix:
|
|
stage: test
|
|
needs:
|
|
- generate-matrix
|
|
- build
|
|
trigger:
|
|
include:
|
|
- artifact: test-matrix.yml
|
|
job: generate-matrix
|
|
strategy: depend
|
|
only:
|
|
- main
|
|
- /^\d+\.\d+\.\d+$/
|
|
|
|
# ============================================================================
|
|
# STAGE 5: Science Jobs (Pool Burning)
|
|
# ============================================================================
|
|
science-validate-examples:
|
|
stage: science
|
|
needs:
|
|
- build
|
|
script:
|
|
- bash scripts/validate-examples.sh
|
|
artifacts:
|
|
reports:
|
|
junit: science-results.xml
|
|
paths:
|
|
- science-results/
|
|
expire_in: 30 days
|
|
allow_failure: true
|
|
only:
|
|
- main
|
|
- /^\d+\.\d+\.\d+$/
|
|
|
|
science-lint-sdks:
|
|
stage: science
|
|
script:
|
|
- bash scripts/science/lint-all-sdks.sh
|
|
artifacts:
|
|
reports:
|
|
junit: lint-results.xml
|
|
paths:
|
|
- lint-results/
|
|
expire_in: 30 days
|
|
allow_failure: true
|
|
only:
|
|
- main
|
|
- /^\d+\.\d+\.\d+$/
|
|
|
|
science-benchmark-clients:
|
|
stage: science
|
|
script:
|
|
- bash scripts/science/benchmark-clients.sh
|
|
artifacts:
|
|
reports:
|
|
junit: benchmark-results.xml
|
|
paths:
|
|
- benchmark-results/
|
|
expire_in: 30 days
|
|
allow_failure: true
|
|
only:
|
|
- main
|
|
- /^\d+\.\d+\.\d+$/
|
|
|
|
# ============================================================================
|
|
# STAGE 6: Validate Examples (from science job artifacts)
|
|
# ============================================================================
|
|
validate-examples:
|
|
stage: validate
|
|
needs:
|
|
- science-validate-examples
|
|
script:
|
|
- mkdir -p science-results
|
|
- |
|
|
if [ -f science-results/examples-validation-results.json ]; then
|
|
echo "Example validation results found"
|
|
cat science-results/examples-validation-results.json
|
|
|
|
# Extract stats and display summary
|
|
TOTAL=$(jq '.summary.total_examples' science-results/examples-validation-results.json 2>/dev/null || echo "0")
|
|
PASSED=$(jq '.summary.total_validated' science-results/examples-validation-results.json 2>/dev/null || echo "0")
|
|
FAILED=$(jq '.summary.total_failed' science-results/examples-validation-results.json 2>/dev/null || echo "0")
|
|
|
|
echo ""
|
|
echo "Summary: $PASSED/$TOTAL passed, $FAILED failed"
|
|
else
|
|
echo "No example validation results found"
|
|
fi
|
|
artifacts:
|
|
paths:
|
|
- science-results/
|
|
expire_in: 30 days
|
|
allow_failure: true
|
|
only:
|
|
- main
|
|
- /^\d+\.\d+\.\d+$/
|
|
|
|
# ============================================================================
|
|
# STAGE 7: Generate Documentation
|
|
# ============================================================================
|
|
generate-documentation:
|
|
stage: document
|
|
needs:
|
|
- validate-examples
|
|
script:
|
|
- mkdir -p docs
|
|
- bash scripts/generate-docs.sh || true
|
|
- |
|
|
VALIDATION_TIME=$(date +%s)
|
|
CURRENT_TIME=$(date +%s)
|
|
MINUTES_AGO=$((($CURRENT_TIME - $VALIDATION_TIME) / 60))
|
|
echo "Last verified: $MINUTES_AGO minutes ago" > docs/VERIFICATION_TIMESTAMP.txt
|
|
echo "Generated: $(date -u +"%Y-%m-%dT%H:%M:%SZ")" >> docs/VERIFICATION_TIMESTAMP.txt
|
|
- ls -la docs/ || echo "No documentation generated"
|
|
artifacts:
|
|
paths:
|
|
- docs/
|
|
- docs/VERIFICATION_TIMESTAMP.txt
|
|
expire_in: 30 days
|
|
allow_failure: true
|
|
only:
|
|
- main
|
|
- /^\d+\.\d+\.\d+$/
|
|
|
|
# ============================================================================
|
|
# STAGE 8: Report
|
|
# ============================================================================
|
|
report:
|
|
stage: report
|
|
needs:
|
|
- generate-documentation
|
|
script:
|
|
- bash scripts/filter-results.sh
|
|
- |
|
|
if [ -f science-results/examples-validation-results.json ]; then
|
|
echo "Including example validation stats in final report..."
|
|
EXAMPLE_STATS=$(cat science-results/examples-validation-results.json)
|
|
jq --argjson examples "$EXAMPLE_STATS" '.properties += [{name: "example_validation_passed", value: $examples.examples.passed | tostring}, {name: "example_validation_failed", value: $examples.examples.failed | tostring}]' final-report.xml > final-report.xml.tmp && mv final-report.xml.tmp final-report.xml || true
|
|
fi
|
|
- echo "Including documentation generation artifacts..."
|
|
- ls -la docs/ 2>/dev/null | tail -5 || echo "No documentation available"
|
|
artifacts:
|
|
reports:
|
|
junit: final-report.xml
|
|
paths:
|
|
- reports/
|
|
- science-results/examples-validation-results.json
|
|
- docs/
|
|
- final-report.xml
|
|
expire_in: 30 days
|
|
only:
|
|
- main
|
|
- /^\d+\.\d+\.\d+$/
|
|
|
|
# ============================================================================
|
|
# STAGE 9: Performance Report (Tags Only)
|
|
# ============================================================================
|
|
perf-report:
|
|
stage: report
|
|
needs:
|
|
- trigger-test-matrix
|
|
- job: build
|
|
artifacts: true
|
|
variables:
|
|
GIT_STRATEGY: clone
|
|
GIT_DEPTH: 0
|
|
script:
|
|
- echo "========================================"
|
|
- echo "Performance Report Generator"
|
|
- echo "Tag - $CI_COMMIT_TAG"
|
|
- echo "========================================"
|
|
- echo "$CI_COMMIT_TAG" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$' || exit 1
|
|
- echo "Step 0 - Waiting 60s for pipeline data to settle..."
|
|
- sleep 60
|
|
- echo "Step 1 - Setting up git on main branch..."
|
|
- test -z "$DEPLOY_KEY" && echo "WARN - No DEPLOY_KEY, skipping commit" && exit 0 || true
|
|
- mkdir -p ~/.ssh
|
|
- echo "$DEPLOY_KEY" | base64 -d > ~/.ssh/id_ed25519
|
|
- chmod 600 ~/.ssh/id_ed25519
|
|
- ssh-keyscan -p 2222 -H git.unturf.com >> ~/.ssh/known_hosts 2>/dev/null
|
|
- git remote set-url origin "ssh://git@git.unturf.com:2222/${CI_PROJECT_PATH}.git"
|
|
- git config user.email "ci@unturf.com"
|
|
- git config user.name "GitLab CI"
|
|
- git fetch origin main
|
|
- git checkout -B main origin/main
|
|
- echo "Step 2 - Generating performance report..."
|
|
- bash scripts/generate-perf-report.sh "$CI_COMMIT_TAG" || echo "WARN - Report generation failed"
|
|
- echo "Step 3 - Generating performance charts..."
|
|
- test -x build/un && test -f "reports/$CI_COMMIT_TAG/perf.json" && build/un -a -f "reports/$CI_COMMIT_TAG/perf.json" scripts/generate-perf-charts.py || echo "WARN - Charts skipped (build/un or perf.json missing)"
|
|
- mv -f *.png "reports/$CI_COMMIT_TAG/" 2>/dev/null || true
|
|
- echo "Step 4 - Report contents..."
|
|
- ls -la "reports/$CI_COMMIT_TAG/" 2>/dev/null || echo "No report directory"
|
|
- echo "Step 5 - Committing to main..."
|
|
- git add reports/
|
|
- git diff --cached --quiet && echo "No changes to commit" || git commit -m "perf - Add performance report for $CI_COMMIT_TAG [ci skip]"
|
|
- git push origin main || echo "Nothing to push"
|
|
- echo "Done"
|
|
artifacts:
|
|
paths:
|
|
- reports/
|
|
expire_in: 90 days
|
|
when: always
|
|
rules:
|
|
- if: '$CI_COMMIT_TAG =~ /^\d+\.\d+\.\d+$/'
|
|
allow_failure: true
|
|
|
|
# ============================================================================
|
|
# STAGE 9b: Aggregated Performance Analysis (cross-release trends)
|
|
# ============================================================================
|
|
perf-aggregate-report:
|
|
stage: report
|
|
needs:
|
|
- perf-report
|
|
variables:
|
|
GIT_STRATEGY: clone
|
|
GIT_DEPTH: 0
|
|
script:
|
|
- echo "========================================"
|
|
- echo "Aggregated Performance Analysis"
|
|
- echo "========================================"
|
|
- echo "Step 1 - Discovering releases from git tags..."
|
|
- VERSIONS=$(git tag | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' | sort -V)
|
|
- echo "Found tagged releases:"
|
|
- echo "$VERSIONS"
|
|
- |
|
|
if [ -z "$VERSIONS" ]; then
|
|
echo "ERROR - No version tags found"
|
|
exit 1
|
|
fi
|
|
- echo "Step 2 - Copying perf.json files for tagged releases..."
|
|
- for v in $VERSIONS; do if [ -f "reports/$v/perf.json" ]; then cp "reports/$v/perf.json" "perf-$v.json"; echo " ✓ $v"; fi; done
|
|
- |
|
|
FILES=$(ls perf-*.json 2>/dev/null | sed 's/^/-f /' | tr '\n' ' ')
|
|
if [ -z "$FILES" ]; then
|
|
echo "ERROR - No perf JSON files found"
|
|
exit 1
|
|
fi
|
|
- echo "Step 3 - Generating charts via UN..."
|
|
- |
|
|
if [ -x build/un ]; then
|
|
build/un -a $FILES scripts/generate-aggregated-charts.py
|
|
else
|
|
echo "WARN - Charts skipped (build/un missing)"
|
|
fi
|
|
- rm -f perf-*.json
|
|
- mv -f *.png reports/ 2>/dev/null || true
|
|
- echo "Step 4 - Generating markdown report..."
|
|
- python3 scripts/aggregate-performance-reports.py reports AGGREGATED-PERFORMANCE.md
|
|
- echo "Step 5 - Setting up git..."
|
|
- |
|
|
if [ -z "$DEPLOY_KEY" ]; then
|
|
echo "WARN - No DEPLOY_KEY, skipping commit"
|
|
exit 0
|
|
fi
|
|
- mkdir -p ~/.ssh
|
|
- echo "$DEPLOY_KEY" | base64 -d > ~/.ssh/id_ed25519
|
|
- chmod 600 ~/.ssh/id_ed25519
|
|
- ssh-keyscan -p 2222 -H git.unturf.com >> ~/.ssh/known_hosts 2>/dev/null
|
|
- git remote set-url origin "ssh://git@git.unturf.com:2222/${CI_PROJECT_PATH}.git"
|
|
- git config user.email "ci@unturf.com"
|
|
- git config user.name "GitLab CI"
|
|
- git fetch origin main
|
|
- echo "Step 6 - Stashing artifact files temporarily..."
|
|
- git add -A
|
|
- git stash push -m "Temporary stash of artifacts and generated files"
|
|
- echo "Step 7 - Checking out main branch..."
|
|
- git checkout main
|
|
- git pull origin main || echo "Already up to date"
|
|
- echo "Step 8 - Restoring artifacts and committing..."
|
|
- git stash pop || echo "No stash to pop"
|
|
- git add reports/ AGGREGATED-PERFORMANCE.md
|
|
- |
|
|
if git diff --cached --quiet; then
|
|
echo "No changes to commit"
|
|
else
|
|
git commit -m "perf: Update aggregated performance analysis [ci skip]"
|
|
fi
|
|
- git push origin main || echo "Nothing to push"
|
|
- echo "Step 9 - Verifying results..."
|
|
- echo "✓ Aggregated report generated with charts"
|
|
- ls -lh AGGREGATED-PERFORMANCE.md
|
|
- ls -lh reports/aggregated-*.png 2>/dev/null || echo " (charts may not have generated)"
|
|
artifacts:
|
|
paths:
|
|
- AGGREGATED-PERFORMANCE.md
|
|
- reports/aggregated-*.png
|
|
expire_in: 90 days
|
|
when: always
|
|
rules:
|
|
- if: '$CI_COMMIT_TAG =~ /^\d+\.\d+\.\d+$/'
|
|
allow_failure: true
|