ci: Cleanup AWS resources better

.github/workflows/ami-release-nix.yml

@@ -253,21 +253,13 @@ jobs:
           SLACK_MESSAGE: 'Building Postgres AMI failed'
           SLACK_FOOTER: ''
 
-      - name: configure aws credentials for cleanup
-        if: ${{ always() }}
+      - name: Configure AWS credentials for cleanup
+        if: always()
         uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1
         with:
           role-to-assume: ${{ secrets.DEV_AWS_ROLE }}
           aws-region: ${{ env.AWS_REGION }}
 
-      - name: Cleanup resources after build
-        if: ${{ always() }}
-        run: |
-          aws ec2 --region "$AWS_REGION" describe-instances --filters "Name=tag:packerExecutionId,Values=${PACKER_EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region "$AWS_REGION" --instance-ids
-          aws ec2 --region "$AWS_REGION" describe-volumes --filters "Name=tag:packerExecutionId,Values=${PACKER_EXECUTION_ID}" "Name=status,Values=available" --query "Volumes[*].VolumeId" --output text | xargs -r -n1 aws ec2 --region "$AWS_REGION" delete-volume --volume-id
-
-      - name: Cleanup resources on build cancellation
-        if: ${{ cancelled() }}
-        run: |
-          aws ec2 --region "$AWS_REGION" describe-instances --filters "Name=tag:packerExecutionId,Values=${PACKER_EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region "$AWS_REGION" --instance-ids
-          aws ec2 --region "$AWS_REGION" describe-volumes --filters "Name=tag:packerExecutionId,Values=${PACKER_EXECUTION_ID}" "Name=status,Values=available" --query "Volumes[*].VolumeId" --output text | xargs -r -n1 aws ec2 --region "$AWS_REGION" delete-volume --volume-id
+      - name: Cleanup Packer resources
+        if: always()
+        run: ci/cleanup-ami-build-resources.sh "${{ matrix.postgres_version }}-${{ matrix.target.arch }}-${{ github.run_id }}"

.github/workflows/cleanup-stale-ec2.yml

@@ -1,4 +1,4 @@
-name: Cleanup Stale EC2 Instances
+name: Cleanup Stale Build Resources
 
 on:
   schedule:
@@ -8,45 +8,29 @@ on:
 
 jobs:
   cleanup:
-    name: Delete stale packer EC2 instances
+    name: Delete stale build resources
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        region: [ap-southeast-1, us-east-1]
+        region:
+          - ap-southeast-1
+          - us-east-1
 
     permissions:
       id-token: write
       contents: read
 
     steps:
+      - name: Checkout Repo
+        uses: supabase/postgres/.github/actions/shared-checkout@HEAD
+
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4
         with:
           role-to-assume: ${{ secrets.DEV_AWS_ROLE }}
           aws-region: ${{ matrix.region }}
 
-      - name: Find and terminate stale instances
+      - name: Cleanup stale AMI build resources
         env:
           AWS_MAX_ATTEMPTS: 6
-        run: |
-          cutoff=$(date -u -d '24 hours ago' '+%Y-%m-%dT%H:%M:%SZ')
-
-          echo "Looking for running packer builder instances launched before ${cutoff}..."
-
-          instance_ids=$(aws ec2 describe-instances \
-            --filters \
-              "Name=tag:appType,Values=postgres" \
-              "Name=tag:creator,Values=packer" \
-              "Name=instance-state-name,Values=running" \
-            --query "Reservations[].Instances[?LaunchTime<'${cutoff}'][].InstanceId" \
-            --output text)
-
-          if [ -z "$instance_ids" ]; then
-            echo "No stale instances found."
-            exit 0
-          fi
-
-          read -r -a instance_id_arr <<< "$instance_ids"
-          echo "Terminating instances: ${instance_id_arr[*]}"
-          aws ec2 terminate-instances --instance-ids "${instance_id_arr[@]}"
-          echo "Done."
+        run: ci/cleanup-stale-ami-build-resources.sh

.github/workflows/qemu-image-build.yml

@@ -145,12 +145,7 @@ jobs:
           SLACK_MESSAGE: 'Building Postgres QEMU artifact failed'
           SLACK_FOOTER: ''
 
-      - name: Cleanup resources after build
-        if: ${{ always() }}
-        run: |
-          aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids
-
-      - name: Cleanup resources on build cancellation
-        if: ${{ cancelled() }}
+      - name: Cleanup Packer resources
+        if: always()
         run: |
           aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids

.github/workflows/testinfra-ami-build.yml

@@ -131,63 +131,16 @@ jobs:
           pip3 install boto3 'boto3-stubs[essential]' ec2instanceconnectcli pytest 'pytest-testinfra[paramiko]' requests
           pytest -vv -s testinfra/test_ami_nix.py
 
-      - name: Cleanup resources on build cancellation
-        if: ${{ cancelled() }}
-        run: |
-          EXECUTION_ID="${{ steps.build-ami.outputs.execution_id }}"
-          INSTANCE_IDS=$(aws ec2 --region "$AWS_REGION" describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text)
-          if [ -n "$INSTANCE_IDS" ]; then
-            echo "Terminating packer build instances: $INSTANCE_IDS"
-            echo "$INSTANCE_IDS" | xargs -r aws ec2 terminate-instances --region "$AWS_REGION" --instance-ids
-          else
-            echo "No packer build instances to clean up"
-          fi
-          VOLUME_IDS=$(aws ec2 --region "$AWS_REGION" describe-volumes --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" "Name=status,Values=available" --query "Volumes[*].VolumeId" --output text)
-          if [ -n "$VOLUME_IDS" ]; then
-            echo "Deleting orphaned packer volumes: $VOLUME_IDS"
-            echo "$VOLUME_IDS" | xargs -r -n1 aws ec2 --region "$AWS_REGION" delete-volume --volume-id
-          else
-            echo "No orphaned packer volumes to clean up"
-          fi
-
-      - name: Cleanup resources after build
-        if: ${{ always() }}
-        run: |
-          EXECUTION_ID="${{ steps.build-ami.outputs.execution_id }}"
-          INSTANCE_IDS=$(aws ec2 --region "$AWS_REGION" describe-instances --filters "Name=tag:testinfra-run-id,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text)
-          if [ -n "$INSTANCE_IDS" ]; then
-            echo "Terminating testinfra instances: $INSTANCE_IDS"
-            echo "$INSTANCE_IDS" | xargs -r aws ec2 terminate-instances --region "$AWS_REGION" --instance-ids || true
-          else
-            echo "No testinfra instances to clean up"
-          fi
-          VOLUME_IDS=$(aws ec2 --region "$AWS_REGION" describe-volumes --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" "Name=status,Values=available" --query "Volumes[*].VolumeId" --output text)
-          if [ -n "$VOLUME_IDS" ]; then
-            echo "Deleting orphaned packer volumes: $VOLUME_IDS"
-            echo "$VOLUME_IDS" | xargs -r -n1 aws ec2 --region "$AWS_REGION" delete-volume --volume-id || true
-          else
-            echo "No orphaned packer volumes to clean up"
-          fi
+      - name: Configure AWS credentials for cleanup
+        if: always()
+        uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1
+        with:
+          role-to-assume: ${{ secrets.DEV_AWS_ROLE }}
+          aws-region: ${{ env.AWS_REGION }}
 
-      - name: Cleanup stage 2 AMI
+      - name: Cleanup Packer resources
         if: always()
-        run: |
-          EXECUTION_ID="${{ steps.build-ami.outputs.execution_id }}"
-          STAGE2_AMI_IDS=$(aws ec2 describe-images \
-            --region "$AWS_REGION" \
-            --owners self \
-            --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" \
-            --query 'Images[*].ImageId' \
-            --output text)
-
-          if [ -n "$STAGE2_AMI_IDS" ]; then
-            for ami_id in $STAGE2_AMI_IDS; do
-              echo "Deregistering stage 2 AMI: $ami_id"
-              aws ec2 deregister-image --region "$AWS_REGION" --image-id "$ami_id" || true
-            done
-          else
-            echo "No stage 2 AMI to clean up"
-          fi
+        run: ci/cleanup-ami-build-resources.sh --delete-amis "${{ matrix.postgres_version }}-${{ matrix.target.arch }}-${{ github.run_id }}"
 
   report-disk-usage:
     needs: test-ami-nix

amazon-amd64-nix.pkr.hcl

@@ -132,22 +132,25 @@ source "amazon-ebssurrogate" "source" {
     creator           = "packer"
     appType           = "postgres"
     packerExecutionId = "${var.packer-execution-id}"
+    supaCreatedAt     = timestamp()
   }
   run_volume_tags = {
     creator           = "packer"
     appType           = "postgres"
     packerExecutionId = "${var.packer-execution-id}"
   }
   snapshot_tags = {
-    creator = "packer"
-    appType = "postgres"
+    creator           = "packer"
+    appType           = "postgres"
+    packerExecutionId = "${var.packer-execution-id}"
   }
   tags = {
-    creator         = "packer"
-    appType         = "postgres"
-    postgresVersion = "${var.postgres-version}-stage1"
-    sourceSha       = "${var.git-head-version}"
-    inputHash       = "${var.input-hash}"
+    creator           = "packer"
+    appType           = "postgres"
+    postgresVersion   = "${var.postgres-version}-stage1"
+    sourceSha         = "${var.git-head-version}"
+    inputHash         = "${var.input-hash}"
+    packerExecutionId = "${var.packer-execution-id}"
   }
 
   communicator = "ssh"

amazon-arm64-nix.pkr.hcl

@@ -132,22 +132,25 @@ source "amazon-ebssurrogate" "source" {
     creator           = "packer"
     appType           = "postgres"
     packerExecutionId = "${var.packer-execution-id}"
+    supaCreatedAt     = timestamp()
   }
   run_volume_tags = {
     creator           = "packer"
     appType           = "postgres"
     packerExecutionId = "${var.packer-execution-id}"
   }
   snapshot_tags = {
-    creator = "packer"
-    appType = "postgres"
+    creator           = "packer"
+    appType           = "postgres"
+    packerExecutionId = "${var.packer-execution-id}"
   }
   tags = {
-    creator         = "packer"
-    appType         = "postgres"
-    postgresVersion = "${var.postgres-version}-stage1"
-    sourceSha       = "${var.git-head-version}"
-    inputHash       = "${var.input-hash}"
+    creator           = "packer"
+    appType           = "postgres"
+    postgresVersion   = "${var.postgres-version}-stage1"
+    sourceSha         = "${var.git-head-version}"
+    inputHash         = "${var.input-hash}"
+    packerExecutionId = "${var.packer-execution-id}"
   }
 
   communicator = "ssh"

ci/cleanup-ami-build-resources.sh

@@ -0,0 +1,139 @@
+#!/usr/bin/env bash
+
+set -uo pipefail
+
+usage() {
+	echo "Usage: $0 [--delete-amis] <build-execution-id>" >&2
+}
+
+amis=false
+if [[ ${1:-} == "--delete-amis" ]]; then
+	amis=true
+	shift
+fi
+
+execution_id=${1:-}
+if [[ -z $execution_id || $# -ne 1 ]]; then
+	usage
+	exit 2
+fi
+
+if [[ -z ${AWS_REGION:-} ]]; then
+	echo "AWS_REGION must be set" >&2
+	exit 2
+fi
+
+failures=0
+
+failed() {
+	echo "Cleanup failed: $*" >&2
+	failures=$((failures + 1))
+}
+
+ids() {
+	local operation=$1
+	local query=$2
+	local output
+	shift 2
+
+	ids=()
+	if ! output=$(aws ec2 "$operation" --filters "$@" --query "$query" --output text); then
+		failed "unable to list resources with $operation using filters: $*"
+		return 1
+	fi
+	read -r -a ids <<<"$output"
+	if ((${#ids[@]})); then
+		return 0
+	fi
+	return 2
+}
+
+echo "Cleaning up AMI build resources for execution $execution_id in $AWS_REGION" >&2
+
+status=Name=instance-state-name,Values=pending,running,stopping,stopped
+
+# Testinfra instances use a separate tag with same value
+tag=Name=tag:testinfra-run-id,Values=$execution_id
+if ids describe-instances Reservations[].Instances[].InstanceId $status "$tag"; then
+	echo "Terminating testinfra instances: ${ids[*]}" >&2
+	aws ec2 terminate-instances --instance-ids "${ids[@]}" >/dev/null || failed "unable to terminate testinfra instances: ${ids[*]}"
+	aws ec2 wait instance-terminated --instance-ids "${ids[@]}" || failed "timed out waiting for testinfra instances to terminate: ${ids[*]}"
+fi
+
+tag=Name=tag:packerExecutionId,Values=$execution_id
+if ids describe-instances Reservations[].Instances[].InstanceId $status "$tag"; then
+	echo "Terminating Packer instances: ${ids[*]}" >&2
+	aws ec2 terminate-instances --instance-ids "${ids[@]}" >/dev/null || failed "unable to terminate Packer instances: ${ids[*]}"
+	aws ec2 wait instance-terminated --instance-ids "${ids[@]}" || failed "timed out waiting for Packer instances to terminate: ${ids[*]}"
+fi
+
+status=Name=status,Values=available
+if ids describe-network-interfaces NetworkInterfaces[].NetworkInterfaceId $status "$tag"; then
+	for id in "${ids[@]}"; do
+		echo "Deleting network interface: $id" >&2
+		aws ec2 delete-network-interface --network-interface-id "$id" || failed "unable to delete network interface: $id"
+	done
+fi
+
+if ids describe-volumes Volumes[].VolumeId $status "$tag"; then
+	for id in "${ids[@]}"; do
+		echo "Deleting volume: $id" >&2
+		aws ec2 delete-volume --volume-id "$id" || failed "unable to delete volume: $id"
+	done
+fi
+
+if ids describe-security-groups SecurityGroups[].GroupId "$tag"; then
+	for id in "${ids[@]}"; do
+		echo "Deleting security group: $id" >&2
+		deleted=false
+		# AWS may return DependencyViolation until terminated instances release their ENIs
+		for _ in {1..6}; do
+			if aws ec2 delete-security-group --group-id "$id"; then
+				deleted=true
+				break
+			fi
+			sleep 10
+		done
+		$deleted || failed "unable to delete security group: $id"
+	done
+fi
+
+if ids describe-key-pairs KeyPairs[].KeyPairId "$tag"; then
+	for id in "${ids[@]}"; do
+		echo "Deleting key pair: $id" >&2
+		aws ec2 delete-key-pair --key-pair-id "$id" || failed "unable to delete key pair: $id"
+	done
+fi
+
+if $amis; then
+	if ids describe-images Images[].ImageId "$tag"; then
+		for id in "${ids[@]}"; do
+			echo "Deregistering AMI and deleting associated snapshots: $id" >&2
+			aws ec2 deregister-image --image-id "$id" --delete-associated-snapshots || failed "unable to deregister AMI and delete associated snapshots: $id"
+		done
+	fi
+fi
+
+# Find any orphaned snapshots from a cancel before the AMI was finalized
+if ids describe-snapshots Snapshots[].SnapshotId "$tag"; then
+	# going to call ids again below which clobbers ids, so copy to new var
+	snapshots=("${ids[@]}")
+	for snapshot in "${snapshots[@]}"; do
+		if ids describe-images Images[].ImageId "Name=block-device-mapping.snapshot-id,Values=$snapshot"; then
+			continue
+		elif (($? == 1)); then
+			# error with aws command, skip for safety
+			continue
+		fi
+
+		echo "Deleting orphaned snapshot: $snapshot" >&2
+		aws ec2 delete-snapshot --snapshot-id "$snapshot" || failed "unable to delete orphaned snapshot: $snapshot"
+	done
+fi
+
+if ((failures)); then
+	echo "Packer cleanup completed with $failures error(s)" >&2
+	exit 1
+fi
+
+echo "Packer cleanup complete" >&2