diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 16745c4..64db372 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,6 +11,27 @@ default: tags: - build +# ============================================================================ +# MANUAL TRIGGER: Emergency Cleanup +# ============================================================================ +# Click play to clean up orphaned test resources (services, sessions, etc.) +# Runs in parallel for speed - useful when system is hosed +manual-cleanup: + stage: pre + script: + - echo "=== Emergency Cleanup ===" + - echo "Building un CLI first..." + - bash scripts/build-clients.sh + - echo "Running parallel cleanup..." + - bash scripts/cleanup-test-resources.sh + rules: + - if: $CI_PIPELINE_SOURCE == "push" + when: manual + allow_failure: true + - if: $CI_PIPELINE_SOURCE == "web" + when: manual + allow_failure: true + # ============================================================================ # MANUAL TRIGGER: Run Full Test Matrix On Demand # ============================================================================ diff --git a/scripts/cleanup-test-resources.sh b/scripts/cleanup-test-resources.sh new file mode 100755 index 0000000..9b48afe --- /dev/null +++ b/scripts/cleanup-test-resources.sh @@ -0,0 +1,85 @@ +#!/bin/bash +# Fast parallel cleanup of orphaned test resources +# Usage: scripts/cleanup-test-resources.sh [--dry-run] +# +# Cleans up resources matching pattern: test-{lang}-{pid}-{timestamp} +# Runs destroys in parallel for speed (up to 10 concurrent) + +set -e + +DRY_RUN=false +PARALLEL_JOBS=10 + +if [ "$1" = "--dry-run" ]; then + DRY_RUN=true + echo "DRY RUN - no changes will be made" +fi + +# Check for un CLI +UN_CLI="${UN_CLI:-build/un}" +if [ ! -x "$UN_CLI" ]; then + UN_CLI="./build/un" +fi +if [ ! -x "$UN_CLI" ]; then + echo "ERROR: un CLI not found at $UN_CLI" + echo "Run 'make build' first or set UN_CLI=/path/to/un" + exit 1 +fi + +echo "=== Fast Cleanup of Test Resources ===" +echo "Using: $UN_CLI" +echo "Parallel jobs: $PARALLEL_JOBS" +echo "" + +# Function to destroy in parallel +parallel_destroy() { + local resource_type="$1" + local destroy_cmd="$2" + local pattern="$3" + local id_pattern="$4" + + echo "--- Cleaning $resource_type ---" + + # Get list of matching resources + RESOURCES=$($UN_CLI $resource_type --list 2>/dev/null | grep -E "$pattern" || true) + + if [ -z "$RESOURCES" ]; then + echo "No orphaned $resource_type found" + return 0 + fi + + # Extract IDs + IDS=$(echo "$RESOURCES" | grep -oE "$id_pattern" | sort -u || true) + COUNT=$(echo "$IDS" | grep -c . || echo 0) + + echo "Found $COUNT orphaned $resource_type" + + if [ "$DRY_RUN" = true ]; then + echo "$IDS" | head -10 + [ "$COUNT" -gt 10 ] && echo "... and $((COUNT - 10)) more" + return 0 + fi + + # Destroy in parallel using xargs + echo "$IDS" | xargs -P "$PARALLEL_JOBS" -I {} sh -c " + echo 'Destroying: {}' + $UN_CLI $resource_type $destroy_cmd '{}' 2>&1 || echo 'Failed: {}' + " + + echo "$resource_type cleanup complete ($COUNT destroyed)" +} + +# Cleanup services (test-python-12345-1706000000, etc.) +parallel_destroy "service" "--destroy" "test-[a-z]+-[0-9]+" "unsb-service-[a-f0-9]+" + +# Cleanup sessions +parallel_destroy "session" "--kill" "test-[a-z]+-[0-9]+" "unsb-session-[a-f0-9-]+" + +# Cleanup snapshots +parallel_destroy "snapshot" "--delete" "test-[a-z]+-[0-9]+" "unsb-snapshot-[a-f0-9-]+" + +# Cleanup images +parallel_destroy "image" "--delete" "test-[a-z]+-[0-9]+" "unsb-image-[a-f0-9-]+" + +echo "" +echo "=== Cleanup Complete ===" diff --git a/scripts/generate-matrix.sh b/scripts/generate-matrix.sh index 4bace11..2cd9f06 100755 --- a/scripts/generate-matrix.sh +++ b/scripts/generate-matrix.sh @@ -86,69 +86,13 @@ build-cli: # Cleanup orphaned test resources before fan-out to avoid concurrency limits # ONLY targets resources with "test-" in the name to avoid destroying real services +# Uses parallel cleanup script for speed cleanup-services: stage: cleanup needs: - build-cli script: - - echo "=== Cleaning up orphaned test resources ===" - - | - # Cleanup test services (test-python-*, test-javascript-*, etc.) - echo "Checking for orphaned test services..." - SERVICES=$(build/un service --list 2>/dev/null | grep -E "test-[a-z]+-[0-9]+" || true) - if [ -n "$SERVICES" ]; then - echo "$SERVICES" | while read -r line; do - SVC=$(echo "$line" | grep -oE 'unsb-service-[a-f0-9]+' || true) - if [ -n "$SVC" ]; then - echo "Destroying test service: $SVC" - build/un service --destroy "$SVC" || true - fi - done - fi - echo "Service cleanup complete" - - | - # Cleanup test sessions - echo "Checking for orphaned test sessions..." - SESSIONS=$(build/un session --list 2>/dev/null | grep -E "test-[a-z]+-[0-9]+" || true) - if [ -n "$SESSIONS" ]; then - echo "$SESSIONS" | while read -r line; do - SESS=$(echo "$line" | grep -oE 'unsb-session-[a-f0-9-]+' || true) - if [ -n "$SESS" ]; then - echo "Killing test session: $SESS" - build/un session --kill "$SESS" || true - fi - done - fi - echo "Session cleanup complete" - - | - # Cleanup test snapshots - echo "Checking for orphaned test snapshots..." - SNAPSHOTS=$(build/un snapshot --list 2>/dev/null | grep -E "test-[a-z]+-[0-9]+" || true) - if [ -n "$SNAPSHOTS" ]; then - echo "$SNAPSHOTS" | while read -r line; do - SNAP=$(echo "$line" | grep -oE 'unsb-snapshot-[a-f0-9-]+' || true) - if [ -n "$SNAP" ]; then - echo "Deleting test snapshot: $SNAP" - build/un snapshot --delete "$SNAP" || true - fi - done - fi - echo "Snapshot cleanup complete" - - | - # Cleanup test images - echo "Checking for orphaned test images..." - IMAGES=$(build/un image --list 2>/dev/null | grep -E "test-[a-z]+-[0-9]+" || true) - if [ -n "$IMAGES" ]; then - echo "$IMAGES" | while read -r line; do - IMG=$(echo "$line" | grep -oE 'unsb-image-[a-f0-9-]+' || true) - if [ -n "$IMG" ]; then - echo "Deleting test image: $IMG" - build/un image --delete "$IMG" || true - fi - done - fi - echo "Image cleanup complete" - - echo "=== Cleanup complete ===" + - bash scripts/cleanup-test-resources.sh allow_failure: true test: