Fix poison pill deadlock: Make retry heap unbounded

.github/workflows/PR-build.yml

@@ -48,7 +48,7 @@ jobs:
         if: needs.changes.outputs.lint == 'true'
         uses: actions/setup-go@v4
         with:
-          go-version: ~1.25
+          go-version: ~1.25.7
           cache: false
 
       - name: Check out code
@@ -105,7 +105,7 @@ jobs:
         if: needs.changes.outputs.build == 'true'
         uses: actions/setup-go@v4
         with:
-          go-version: ~1.25
+          go-version: ~1.25.7
           cache: false
 
       - name: Check out code
@@ -152,7 +152,7 @@ jobs:
         if: needs.changes.outputs.build == 'true'
         uses: actions/setup-go@v4
         with:
-          go-version: ~1.25
+          go-version: ~1.25.7
           cache: false
 
       - name: Check out code

.github/workflows/PR-test.yml

@@ -95,7 +95,7 @@ jobs:
       - name: Set up Go 1.x
         uses: actions/setup-go@v4
         with:
-          go-version: ~1.25
+          go-version: ~1.25.7
 
   StartLocalStack:
     name: 'StartLocalStack'
@@ -138,7 +138,7 @@ jobs:
       - name: Set up Go 1.x
         uses: actions/setup-go@v4
         with:
-          go-version: ~1.25
+          go-version: ~1.25.7
 
       - name: Install jq
         run: sudo apt-get install -y jq

.github/workflows/application-signals-e2e-test.yml

@@ -41,13 +41,20 @@ jobs:
             exit 1
           fi
       - run: |
-          conclusion=$(gh run view ${{ inputs.build_run_id }} --repo $GITHUB_REPOSITORY --json conclusion -q '.conclusion')
-          if [[ $conclusion == "success" ]]; then
-            echo "Run succeeded"
-          else
-            echo "Run failed"
-            exit 1
-          fi
+          for i in {1..6}; do
+            conclusion=$(gh run view ${{ inputs.build_run_id }} --repo $GITHUB_REPOSITORY --json conclusion -q '.conclusion')
+            if [[ "$conclusion" == "success" ]]; then
+              echo "Run succeeded"
+              exit 0
+            elif [[ "$conclusion" == "failure" || "$conclusion" == "cancelled" ]]; then
+              echo "Run failed with: $conclusion"
+              exit 1
+            fi
+            echo "Waiting for workflow to complete (attempt $i)..."
+            sleep 5
+          done
+          echo "Timed out waiting for workflow"
+          exit 1
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 

.github/workflows/build-test-artifacts.yml

@@ -19,14 +19,14 @@ on:
     - cron: '0 11 * * 1,2,3,4,5' # Every day at 11:00 UTC on Monday to Friday
   workflow_dispatch:
     inputs:
-      test-image-before-upload:
-        description: "Run Test on the new container image"
+      run-tests:
+        description: "Run test workflows after build"
         default: true
         type: boolean
   workflow_call:
     inputs:
-      test-image-before-upload:
-        description: "Run Test on the new container image"
+      run-tests:
+        description: "Run test workflows after build"
         default: true
         type: boolean
 
@@ -114,7 +114,7 @@ jobs:
 
   StartIntegrationTests:
     needs: [ BuildAndUploadPackages, BuildAndUploadITAR, BuildAndUploadCN, BuildDocker, BuildDistributor ]
-    if: ${{ github.event_name == 'push' || inputs.test-image-before-upload }}
+    if: ${{ github.event_name == 'push' || inputs.run-tests }}
     runs-on: ubuntu-latest
     permissions:
       actions: write
@@ -126,7 +126,7 @@ jobs:
   StartApplicationSignalsE2ETests:
     needs: [ BuildAndUploadPackages, BuildAndUploadITAR, BuildAndUploadCN, BuildDocker, BuildDistributor ]
     # Workflow only runs against main
-    if: ${{ github.event_name == 'push' || inputs.test-image-before-upload }}
+    if: ${{ github.event_name == 'push' || inputs.run-tests }}
     runs-on: ubuntu-latest
     permissions:
       actions: write
@@ -137,7 +137,7 @@ jobs:
 
   StartEKSE2ETests:
     needs: [ BuildAndUploadPackages, BuildAndUploadITAR, BuildAndUploadCN, BuildDocker, BuildDistributor ]
-    if: ${{ github.event_name == 'push' || inputs.test-image-before-upload }}
+    if: ${{ github.event_name == 'push' || inputs.run-tests }}
     runs-on: ubuntu-latest
     permissions:
       actions: write
@@ -148,7 +148,7 @@ jobs:
 
   StartWorkloadDiscoveryIntegrationTests:
     needs: [ BuildAndUploadPackages, BuildAndUploadITAR, BuildAndUploadCN, BuildDocker, BuildDistributor ]
-    if: ${{ github.event_name == 'push' || inputs.test-image-before-upload }}
+    if: ${{ github.event_name == 'push' || inputs.run-tests }}
     runs-on: ubuntu-latest
     permissions:
       actions: write

.github/workflows/e2e-test.yml

@@ -69,12 +69,16 @@ jobs:
     permissions:
       id-token: write
       contents: read
+      actions: write
     with:
-      test-image-before-upload: false
+      run-tests: false
 
   BuildOperator:
     needs: [GetLatestOperatorCommitSHA]
     uses: aws/amazon-cloudwatch-agent-operator/.github/workflows/build-and-upload.yml@main
+    permissions:
+      id-token: write
+      contents: read
     concurrency:
       group: ${{ github.workflow }}-operator-${{ inputs.operator-branch || 'main' }}
       cancel-in-progress: true
@@ -106,7 +110,7 @@ jobs:
       - name: Set up Go 1.x
         uses: actions/setup-go@v4
         with:
-          go-version: ~1.25
+          go-version: ~1.25.7
 
       - name: SetOutputs
         id: set-outputs
@@ -143,7 +147,7 @@ jobs:
       - name: Set up Go 1.x
         uses: actions/setup-go@v4
         with:
-          go-version: ~1.25
+          go-version: ~1.25.7
 
       - name: Generate matrix
         id: set-matrix
@@ -159,6 +163,9 @@ jobs:
     needs: [ GetLatestOperatorCommitSHA, GenerateTestMatrix, OutputEnvVariables ]
     if: always() && !cancelled() && !contains(needs.*.result, 'failure')
     name: 'EKSE2EJVMTomcatTestHelm'
+    permissions:
+      id-token: write
+      contents: read
     uses: ./.github/workflows/eks-e2e-test.yml
     with:
       terraform_dir: terraform/eks/e2e
@@ -183,6 +190,9 @@ jobs:
     needs: [ GetLatestOperatorCommitSHA, GenerateTestMatrix, OutputEnvVariables ]
     if: always() && !cancelled() && !contains(needs.*.result, 'failure')
     name: 'EKSE2EJVMTomcatTestAddon'
+    permissions:
+      id-token: write
+      contents: read
     uses: ./.github/workflows/eks-e2e-test.yml
     with:
       terraform_dir: terraform/eks/e2e
@@ -207,6 +217,9 @@ jobs:
     needs: [ GetLatestOperatorCommitSHA, GenerateTestMatrix, OutputEnvVariables ]
     if: always() && !cancelled() && !contains(needs.*.result, 'failure')
     name: 'EKSE2EKafkaTestHelm'
+    permissions:
+      id-token: write
+      contents: read
     uses: ./.github/workflows/eks-e2e-test.yml
     with:
       terraform_dir: terraform/eks/e2e
@@ -231,6 +244,9 @@ jobs:
     needs: [ GetLatestOperatorCommitSHA, GenerateTestMatrix, OutputEnvVariables ]
     if: always() && !cancelled() && !contains(needs.*.result, 'failure')
     name: 'EKSE2EKafkaTestAddon'
+    permissions:
+      id-token: write
+      contents: read
     uses: ./.github/workflows/eks-e2e-test.yml
     with:
       terraform_dir: terraform/eks/e2e
@@ -255,6 +271,9 @@ jobs:
     needs: [ GetLatestOperatorCommitSHA, GenerateTestMatrix, OutputEnvVariables ]
     if: always() && !cancelled() && !contains(needs.*.result, 'failure')
     name: 'EKSE2EJMXContainerInsightsTestHelm'
+    permissions:
+      id-token: write
+      contents: read
     uses: ./.github/workflows/eks-e2e-test.yml
     with:
       terraform_dir: terraform/eks/e2e
@@ -279,6 +298,9 @@ jobs:
     needs: [ GetLatestOperatorCommitSHA, GenerateTestMatrix, OutputEnvVariables ]
     if: always() && !cancelled() && !contains(needs.*.result, 'failure')
     name: 'EKSE2EJMXContainerInsightsTestAddon'
+    permissions:
+      id-token: write
+      contents: read
     uses: ./.github/workflows/eks-e2e-test.yml
     with:
       terraform_dir: terraform/eks/e2e
@@ -303,6 +325,9 @@ jobs:
     needs: [ GetLatestOperatorCommitSHA, GenerateTestMatrix, OutputEnvVariables ]
     if: always() && !cancelled() && !contains(needs.*.result, 'failure')
     name: 'EKSE2EJVMTomcatTestHelmIPv6'
+    permissions:
+      id-token: write
+      contents: read
     uses: ./.github/workflows/eks-e2e-test.yml
     with:
       terraform_dir: terraform/eks/e2e
@@ -329,6 +354,9 @@ jobs:
     needs: [ GetLatestOperatorCommitSHA, GenerateTestMatrix, OutputEnvVariables ]
     if: always() && !cancelled() && !contains(needs.*.result, 'failure')
     name: 'EKSE2EJVMTomcatTestAddonIPv6'
+    permissions:
+      id-token: write
+      contents: read
     uses: ./.github/workflows/eks-e2e-test.yml
     with:
       terraform_dir: terraform/eks/e2e
@@ -355,6 +383,9 @@ jobs:
     needs: [ GetLatestOperatorCommitSHA, GenerateTestMatrix, OutputEnvVariables ]
     if: always() && !cancelled() && !contains(needs.*.result, 'failure')
     name: 'EKSE2EKafkaTestHelmIPv6'
+    permissions:
+      id-token: write
+      contents: read
     uses: ./.github/workflows/eks-e2e-test.yml
     with:
       terraform_dir: terraform/eks/e2e
@@ -381,6 +412,9 @@ jobs:
     needs: [ GetLatestOperatorCommitSHA, GenerateTestMatrix, OutputEnvVariables ]
     if: always() && !cancelled() && !contains(needs.*.result, 'failure')
     name: 'EKSE2EKafkaTestAddonIPv6'
+    permissions:
+      id-token: write
+      contents: read
     uses: ./.github/workflows/eks-e2e-test.yml
     with:
       terraform_dir: terraform/eks/e2e
@@ -407,6 +441,9 @@ jobs:
     needs: [ GetLatestOperatorCommitSHA, GenerateTestMatrix, OutputEnvVariables ]
     if: always() && !cancelled() && !contains(needs.*.result, 'failure')
     name: 'EKSE2EJMXContainerInsightsTestHelmIPv6'
+    permissions:
+      id-token: write
+      contents: read
     uses: ./.github/workflows/eks-e2e-test.yml
     with:
       terraform_dir: terraform/eks/e2e
@@ -433,6 +470,9 @@ jobs:
     needs: [ GetLatestOperatorCommitSHA, GenerateTestMatrix, OutputEnvVariables ]
     if: always() && !cancelled() && !contains(needs.*.result, 'failure')
     name: 'EKSE2EJMXContainerInsightsTestAddonIPv6'
+    permissions:
+      id-token: write
+      contents: read
     uses: ./.github/workflows/eks-e2e-test.yml
     with:
       terraform_dir: terraform/eks/e2e

.github/workflows/eks-performance-cluster-addon-install.yml

@@ -122,14 +122,18 @@ jobs:
     permissions:
       id-token: write
       contents: read
+      actions: write
     with:
-      test-image-before-upload: false
+      run-tests: false
 
   # Build and upload operator image to ECR repo
   BuildOperator:
     needs: [ check-trigger, GetLatestOperatorCommitSHA ]
     if: ${{ needs.check-trigger.outputs.should_continue == 'true' }}
     uses: aws/amazon-cloudwatch-agent-operator/.github/workflows/build-and-upload.yml@main
+    permissions:
+      id-token: write
+      contents: read
     concurrency:
       group: ${{ github.workflow }}-operator-${{ inputs.operator-branch || 'main' }}
       cancel-in-progress: true
@@ -173,10 +177,9 @@ jobs:
         run: |
           aws eks update-kubeconfig --name $CLUSTER_NAME --region $AWS_REGION
 
-      # TODO: Revert to using main helm branch when changes from leader-election are merged in
       - name: Clone Helm Charts Repository
         env:
-          HELM_CHARTS_BRANCH: ${{ inputs.helm-charts-branch || 'sky333999/leader-election' }}
+          HELM_CHARTS_BRANCH: ${{ inputs.helm-charts-branch || 'main' }}
         run: |
           rm -rf ./helm-charts
           git clone -b "$HELM_CHARTS_BRANCH" https://github.com/aws-observability/helm-charts.git ./helm-charts

.github/workflows/eks-performance-cluster-tests.yml

@@ -122,7 +122,7 @@ jobs:
       - name: Set up Go 1.x
         uses: actions/setup-go@v4
         with:
-          go-version: ~1.25
+          go-version: ~1.25.7
 
       - uses: actions/checkout@v4
         with:

.github/workflows/integration-test.yml

@@ -57,13 +57,20 @@ jobs:
             exit 1
           fi
       - run: |
-          conclusion=$(gh run view ${{ inputs.build_run_id }} --repo $GITHUB_REPOSITORY --json conclusion -q '.conclusion')
-          if [[ $conclusion == "success" ]]; then
-            echo "Run succeeded"
-          else
-            echo "Run failed"
-            exit 1
-          fi
+          for i in {1..6}; do
+            conclusion=$(gh run view ${{ inputs.build_run_id }} --repo $GITHUB_REPOSITORY --json conclusion -q '.conclusion')
+            if [[ "$conclusion" == "success" ]]; then
+              echo "Run succeeded"
+              exit 0
+            elif [[ "$conclusion" == "failure" || "$conclusion" == "cancelled" ]]; then
+              echo "Run failed with: $conclusion"
+              exit 1
+            fi
+            echo "Waiting for workflow to complete (attempt $i)..."
+            sleep 5
+          done
+          echo "Timed out waiting for workflow"
+          exit 1
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 

.github/workflows/otel-fork-replace.yml

@@ -33,7 +33,7 @@ jobs:
       - name: Set up Go 1.x
         uses: actions/setup-go@v4
         with:
-          go-version: ~1.25
+          go-version: ~1.25.7
           cache: false
       - name: Update OTel fork components version
         id: set-matrix

.github/workflows/test-artifacts.yml

@@ -145,7 +145,7 @@ jobs:
       - name: Set up Go 1.x
         uses: actions/setup-go@v4
         with:
-          go-version: ~1.25
+          go-version: ~1.25.7
 
   GenerateTestMatrix:
     name: 'GenerateTestMatrix'
@@ -183,7 +183,7 @@ jobs:
       - name: Set up Go 1.x
         uses: actions/setup-go@v4
         with:
-          go-version: ~1.25
+          go-version: ~1.25.7
 
       - name: Generate matrix
         id: set-matrix
@@ -303,7 +303,7 @@ jobs:
       - name: Set up Go 1.x
         uses: actions/setup-go@v2
         with:
-          go-version: ~1.25
+          go-version: ~1.25.7
 
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v4

.github/workflows/test-build-docker.yml

@@ -143,7 +143,7 @@ jobs:
       - name: Set up Go 1.x
         uses: actions/setup-go@v4
         with:
-          go-version: ~1.25
+          go-version: ~1.25.7
 
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v4