fix: adaptive retry on aws (#1917)

* fix: adaptive retry on aws

* fix: also Wait longer for instance state transitions

Author: samrose

.github/workflows/testinfra-ami-build.yml

@@ -109,12 +109,18 @@ jobs:
           echo "" >> common-nix.vars.pkr.hcl
 
       - name: Build AMI stage 1
+        env:
+          AWS_MAX_ATTEMPTS: 10
+          AWS_RETRY_MODE: adaptive
         run: |
           GIT_SHA=${{github.sha}}
           nix run github:supabase/postgres/${GIT_SHA}#packer -- init amazon-arm64-nix.pkr.hcl
           nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=" -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=ap-southeast-1" -var 'ami_regions=["ap-southeast-1"]' -var "force-deregister=true" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl
 
       - name: Build AMI stage 2
+        env:
+          AWS_MAX_ATTEMPTS: 10
+          AWS_RETRY_MODE: adaptive
         run: |
           GIT_SHA=${{github.sha}}
           nix run github:supabase/postgres/${GIT_SHA}#packer -- init stage2-nix-psql.pkr.hcl

amazon-arm64-nix.pkr.hcl

@@ -115,6 +115,12 @@ source "amazon-ebssurrogate" "source" {
   #secret_key   = "${var.aws_secret_key}"
   force_deregister = var.force-deregister
 
+  # Increase timeout for instance stop operations to handle large instances
+  aws_polling {
+    delay_seconds = 15
+    max_attempts  = 120  # 120 * 15s = 30 minutes max wait
+  }
+
   # Use latest official ubuntu noble ami owned by Canonical.
   source_ami_filter {
     filters = {

stage2-nix-psql.pkr.hcl

@@ -81,6 +81,11 @@ source "amazon-ebs" "ubuntu" {
 
   associate_public_ip_address = true
 
+  # Increase timeout for instance stop operations to handle large instances
+  aws_polling {
+    delay_seconds = 15
+    max_attempts  = 120  # 120 * 15s = 30 minutes max wait
+  }
 
   ena_support = true