ci: add fast parallel cleanup + manual trigger

- New scripts/cleanup-test-resources.sh runs 10 parallel destroys
- Manual 'Emergency Cleanup' job in GitLab UI
- Test matrix cleanup now uses parallel script
This commit is contained in:
russell@unturf.com 2026-01-29 12:40:36 -05:00
parent 271cfb71e7
commit 7baae23164
3 changed files with 108 additions and 58 deletions

View file

@ -0,0 +1,85 @@
#!/bin/bash
# Fast parallel cleanup of orphaned test resources
# Usage: scripts/cleanup-test-resources.sh [--dry-run]
#
# Cleans up resources matching pattern: test-{lang}-{pid}-{timestamp}
# Runs destroys in parallel for speed (up to 10 concurrent)
set -e
DRY_RUN=false
PARALLEL_JOBS=10
if [ "$1" = "--dry-run" ]; then
DRY_RUN=true
echo "DRY RUN - no changes will be made"
fi
# Check for un CLI
UN_CLI="${UN_CLI:-build/un}"
if [ ! -x "$UN_CLI" ]; then
UN_CLI="./build/un"
fi
if [ ! -x "$UN_CLI" ]; then
echo "ERROR: un CLI not found at $UN_CLI"
echo "Run 'make build' first or set UN_CLI=/path/to/un"
exit 1
fi
echo "=== Fast Cleanup of Test Resources ==="
echo "Using: $UN_CLI"
echo "Parallel jobs: $PARALLEL_JOBS"
echo ""
# Function to destroy in parallel
parallel_destroy() {
local resource_type="$1"
local destroy_cmd="$2"
local pattern="$3"
local id_pattern="$4"
echo "--- Cleaning $resource_type ---"
# Get list of matching resources
RESOURCES=$($UN_CLI $resource_type --list 2>/dev/null | grep -E "$pattern" || true)
if [ -z "$RESOURCES" ]; then
echo "No orphaned $resource_type found"
return 0
fi
# Extract IDs
IDS=$(echo "$RESOURCES" | grep -oE "$id_pattern" | sort -u || true)
COUNT=$(echo "$IDS" | grep -c . || echo 0)
echo "Found $COUNT orphaned $resource_type"
if [ "$DRY_RUN" = true ]; then
echo "$IDS" | head -10
[ "$COUNT" -gt 10 ] && echo "... and $((COUNT - 10)) more"
return 0
fi
# Destroy in parallel using xargs
echo "$IDS" | xargs -P "$PARALLEL_JOBS" -I {} sh -c "
echo 'Destroying: {}'
$UN_CLI $resource_type $destroy_cmd '{}' 2>&1 || echo 'Failed: {}'
"
echo "$resource_type cleanup complete ($COUNT destroyed)"
}
# Cleanup services (test-python-12345-1706000000, etc.)
parallel_destroy "service" "--destroy" "test-[a-z]+-[0-9]+" "unsb-service-[a-f0-9]+"
# Cleanup sessions
parallel_destroy "session" "--kill" "test-[a-z]+-[0-9]+" "unsb-session-[a-f0-9-]+"
# Cleanup snapshots
parallel_destroy "snapshot" "--delete" "test-[a-z]+-[0-9]+" "unsb-snapshot-[a-f0-9-]+"
# Cleanup images
parallel_destroy "image" "--delete" "test-[a-z]+-[0-9]+" "unsb-image-[a-f0-9-]+"
echo ""
echo "=== Cleanup Complete ==="

View file

@ -86,69 +86,13 @@ build-cli:
# Cleanup orphaned test resources before fan-out to avoid concurrency limits
# ONLY targets resources with "test-" in the name to avoid destroying real services
# Uses parallel cleanup script for speed
cleanup-services:
stage: cleanup
needs:
- build-cli
script:
- echo "=== Cleaning up orphaned test resources ==="
- |
# Cleanup test services (test-python-*, test-javascript-*, etc.)
echo "Checking for orphaned test services..."
SERVICES=$(build/un service --list 2>/dev/null | grep -E "test-[a-z]+-[0-9]+" || true)
if [ -n "$SERVICES" ]; then
echo "$SERVICES" | while read -r line; do
SVC=$(echo "$line" | grep -oE 'unsb-service-[a-f0-9]+' || true)
if [ -n "$SVC" ]; then
echo "Destroying test service: $SVC"
build/un service --destroy "$SVC" || true
fi
done
fi
echo "Service cleanup complete"
- |
# Cleanup test sessions
echo "Checking for orphaned test sessions..."
SESSIONS=$(build/un session --list 2>/dev/null | grep -E "test-[a-z]+-[0-9]+" || true)
if [ -n "$SESSIONS" ]; then
echo "$SESSIONS" | while read -r line; do
SESS=$(echo "$line" | grep -oE 'unsb-session-[a-f0-9-]+' || true)
if [ -n "$SESS" ]; then
echo "Killing test session: $SESS"
build/un session --kill "$SESS" || true
fi
done
fi
echo "Session cleanup complete"
- |
# Cleanup test snapshots
echo "Checking for orphaned test snapshots..."
SNAPSHOTS=$(build/un snapshot --list 2>/dev/null | grep -E "test-[a-z]+-[0-9]+" || true)
if [ -n "$SNAPSHOTS" ]; then
echo "$SNAPSHOTS" | while read -r line; do
SNAP=$(echo "$line" | grep -oE 'unsb-snapshot-[a-f0-9-]+' || true)
if [ -n "$SNAP" ]; then
echo "Deleting test snapshot: $SNAP"
build/un snapshot --delete "$SNAP" || true
fi
done
fi
echo "Snapshot cleanup complete"
- |
# Cleanup test images
echo "Checking for orphaned test images..."
IMAGES=$(build/un image --list 2>/dev/null | grep -E "test-[a-z]+-[0-9]+" || true)
if [ -n "$IMAGES" ]; then
echo "$IMAGES" | while read -r line; do
IMG=$(echo "$line" | grep -oE 'unsb-image-[a-f0-9-]+' || true)
if [ -n "$IMG" ]; then
echo "Deleting test image: $IMG"
build/un image --delete "$IMG" || true
fi
done
fi
echo "Image cleanup complete"
- echo "=== Cleanup complete ==="
- bash scripts/cleanup-test-resources.sh
allow_failure: true
test: