engineering/un-inception
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(cleanup): add retry rounds with backoff for failed destroys

Browse source 
- 3 retry rounds, re-lists remaining resources each round
- 5s/10s/15s backoff between rounds
- Quiet output (only shows Destroyed/Failed per ID)
This commit is contained in:
russell@unturf.com 2026-01-29 12:58:09 -05:00
parent 0620310781
commit 495d916cfc
1 changed files with 67 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

89
scripts/cleanup-test-resources.sh
View file

@ -4,11 +4,13 @@
#
# Cleans up resources matching pattern: test-{lang}-{pid}-{timestamp}
# Runs destroys in parallel for speed (up to 10 concurrent)
# Retries failed destroys up to 3 times with backoff
set -e
DRY_RUN=false
PARALLEL_JOBS=10
MAX_ROUNDS=3
if [ "$1" = "--dry-run" ]; then
DRY_RUN=true
@ -31,7 +33,7 @@ echo "Using: $UN_CLI"
echo "Parallel jobs: $PARALLEL_JOBS"
echo ""
# Function to destroy in parallel
# Function to destroy in parallel with retry rounds
parallel_destroy() {
local resource_type="$1"
local destroy_cmd="$2"
@ -40,33 +42,76 @@ parallel_destroy() {
echo "--- Cleaning $resource_type ---"
# Get list of matching resources
RESOURCES=$($UN_CLI $resource_type --list 2>/dev/null | grep -E "$pattern" || true)
local round=1
local total_destroyed=0
if [ -z "$RESOURCES" ]; then
echo "No orphaned $resource_type found"
return 0
fi
while [ $round -le $MAX_ROUNDS ]; do
# Get list of matching resources
RESOURCES=$($UN_CLI $resource_type --list 2>/dev/null | grep -E "$pattern" || true)
# Extract IDs
IDS=$(echo "$RESOURCES" | grep -oE "$id_pattern" | sort -u || true)
COUNT=$(echo "$IDS" | grep -c . || echo 0)
if [ -z "$RESOURCES" ]; then
if [ $round -eq 1 ]; then
echo "No orphaned $resource_type found"
fi
break
fi
echo "Found $COUNT orphaned $resource_type"
# Extract IDs
IDS=$(echo "$RESOURCES" | grep -oE "$id_pattern" | sort -u || true)
COUNT=$(echo "$IDS" | grep -c . || echo 0)
if [ "$DRY_RUN" = true ]; then
echo "$IDS" | head -10
[ "$COUNT" -gt 10 ] && echo "... and $((COUNT - 10)) more"
return 0
fi
if [ "$COUNT" -eq 0 ]; then
break
fi
# Destroy in parallel using xargs
echo "$IDS" | xargs -P "$PARALLEL_JOBS" -I {} sh -c "
echo 'Destroying: {}'
$UN_CLI $resource_type $destroy_cmd '{}' 2>&1 || echo 'Failed: {}'
"
if [ $round -eq 1 ]; then
echo "Found $COUNT orphaned $resource_type"
else
echo "Round $round: $COUNT remaining"
fi
echo "$resource_type cleanup complete ($COUNT destroyed)"
if [ "$DRY_RUN" = true ]; then
echo "$IDS" | head -10
[ "$COUNT" -gt 10 ] && echo "... and $((COUNT - 10)) more"
return 0
fi
# Destroy in parallel using xargs
# Capture failures by writing failed IDs to a temp file
FAIL_FILE=$(mktemp)
echo "$IDS" | xargs -P "$PARALLEL_JOBS" -I {} sh -c "
if $UN_CLI $resource_type $destroy_cmd '{}' >/dev/null 2>&1; then
echo 'Destroyed: {}'
else
echo '{}' >> '$FAIL_FILE'
fi
"
FAILED_COUNT=0
if [ -s "$FAIL_FILE" ]; then
FAILED_COUNT=$(wc -l < "$FAIL_FILE")
fi
rm -f "$FAIL_FILE"
SUCCEEDED=$((COUNT - FAILED_COUNT))
total_destroyed=$((total_destroyed + SUCCEEDED))
if [ "$FAILED_COUNT" -eq 0 ]; then
break
fi
echo "Round $round: $SUCCEEDED destroyed, $FAILED_COUNT failed"
if [ $round -lt $MAX_ROUNDS ]; then
BACKOFF=$((round * 5))
echo "Waiting ${BACKOFF}s before retry..."
sleep $BACKOFF
fi
round=$((round + 1))
done
echo "$resource_type cleanup complete ($total_destroyed destroyed)"
}
# Cleanup services (test-python-12345-1706000000, etc.)