Merge branch 'next' into ad/misc-kind-fixups

Author: ludamad

.github/workflows/ci3.yml

@@ -120,9 +120,9 @@ jobs:
   # End-to-end tests that target a network deployment.
   # We run this every release (at minimum, nightly), or when explicitly requested.
   # This task runs against a real testnet deployment. This uses resources on GCP (not AWS, thank free credit incentives).
+  # Runs two test sets in parallel.
   ci-network-scenario:
     runs-on: ubuntu-latest
-    timeout-minutes: 540
     # We either run after a release (tag starting with v), or when the ci-network-scenario label is present in a PR.
     # We exclude ci-release-pr test tags (v0.0.1-commit.*) which are only for testing the release process.
     needs: ci
@@ -133,7 +133,7 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha || github.sha }}
 
-      - name: Run Network Deploy
+      - name: Run Network Scenarios
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
@@ -146,7 +146,7 @@ jobs:
           DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
           RUN_ID: ${{ github.run_id }}
-          AWS_SHUTDOWN_TIME: 540 # temporarily increased to 9 hours, todo(revert to 360)
+          AWS_SHUTDOWN_TIME: 360
           NO_SPOT: 1
         run: |
           # For release tags, use the release image; for PRs, omit to build and push to aztecdev
@@ -163,7 +163,7 @@ jobs:
             docker_image=""
           fi
           set -x # print next line
-          ./.github/ci3.sh network-deploy next-scenario "$namespace" "$docker_image"
+          ./.github/ci3.sh network-scenarios next-scenario "$namespace" "$docker_image"
 
       - name: Cleanup network resources
         # Clean up if this is a CI label or nightly.
@@ -176,7 +176,9 @@ jobs:
           GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
           GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
           NO_SPOT: 1
-        run: ./.github/ci3.sh network-teardown next-scenario "$NAMESPACE"
+        run: |
+          ./.github/ci3.sh network-teardown next-scenario "${NAMESPACE}-1" || true
+          ./.github/ci3.sh network-teardown next-scenario "${NAMESPACE}-2" || true
 
       #############
       # Benchmarks

.github/workflows/test-network-scenarios.yml

@@ -1,5 +1,6 @@
 # CI for Aztec Network Scenarios.
 # Manually triggered to deploy and test a network.
+# If test_set is empty, runs both sets in parallel.
 name: Test Network Scenarios
 
 on:
@@ -18,6 +19,11 @@ on:
         required: false
         type: string
         default: next-scenario
+      test_set:
+        description: Test set to run (1, 2, or empty for both in parallel)
+        required: false
+        type: string
+        default: ""
 
 concurrency:
   group: test-network-scenarios-${{ inputs.namespace }}
@@ -26,12 +32,12 @@ concurrency:
 jobs:
   deploy-and-test:
     runs-on: ubuntu-latest
-    timeout-minutes: 540
+    timeout-minutes: 360
     steps:
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
 
-      - name: Run
+      - name: Run Network Scenarios
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
@@ -44,4 +50,25 @@ jobs:
           NO_SPOT: 1
         run: |
           namespace=$(echo "${{ inputs.namespace }}" | sed 's/[^a-z0-9-]/-/g' | cut -c1-20)
-          ./.github/ci3.sh network-deploy "${{ inputs.env_file }}" "$namespace" "${{ inputs.docker_image }}"
+          echo "NAMESPACE=$namespace" >> $GITHUB_ENV
+          # Run scenario tests (both sets in parallel, or specific set if provided)
+          ./.github/ci3.sh network-scenarios "${{ inputs.env_file }}" "$namespace" "${{ inputs.docker_image }}" "${{ inputs.test_set }}"
+
+      - name: Cleanup network resources
+        if: always()
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
+          BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
+          GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
+          GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
+          NO_SPOT: 1
+        run: |
+          test_set="${{ inputs.test_set }}"
+          if [[ -z "$test_set" ]]; then
+            ./.github/ci3.sh network-teardown "${{ inputs.env_file }}" "${NAMESPACE}-1" || true
+            ./.github/ci3.sh network-teardown "${{ inputs.env_file }}" "${NAMESPACE}-2" || true
+          else
+            ./.github/ci3.sh network-teardown "${{ inputs.env_file }}" "${NAMESPACE}-${test_set}" || true
+          fi

bootstrap.sh

@@ -595,11 +595,12 @@ case "$cmd" in
   # NETWORK DEPLOYMENTS WITH BENCHES/TESTS #
   ##########################################
   "ci-network-deploy")
-    # Args: <env_file> <namespace> [docker_image]
+    # Args: <env_file> <namespace> [docker_image] [test_set]
     export CI=1
     env_file="${1:?env_file is required}"
     namespace="${2:?namespace is required}"
     docker_image="${3:-}"
+    test_set="${4:-}"
     build
     # If no docker image provided, build and push to aztecdev
     if [ -z "$docker_image" ]; then
@@ -610,7 +611,7 @@ case "$cmd" in
     export NAMESPACE="$namespace"
     export AZTEC_DOCKER_IMAGE="$docker_image"
     deploy_exit_code=0
-    spartan/bootstrap.sh network_deploy "${env_file}" || deploy_exit_code=$?
+    spartan/bootstrap.sh network_deploy "${env_file}" "$test_set" || deploy_exit_code=$?
     # Merge and upload deploy benchmarks (deploy_network.sh writes to spartan/bench-out/)
     rm -rf bench-out
     mkdir -p bench-out

ci.sh

@@ -27,6 +27,7 @@ function print_usage {
   echo_cmd "grind"                 "Spin up multiple EC2 instances to run parallel full CI runs."
   echo_cmd "merge-queue"           "Spin up several EC2 instances to run the merge-queue jobs."
   echo_cmd "network-deploy"        "Spin up an EC2 instance to deploy a network."
+  echo_cmd "network-scenarios"      "Spin up EC2 instance(s) to run network scenario tests in parallel."
   echo_cmd "network-tests"         "Spin up an EC2 instance to run tests on a network."
   echo_cmd "network-bench"         "Spin up an EC2 instance to run benchmarks on a network."
   echo_cmd "network-teardown"      "Spin up an EC2 instance to teardown a network deployment."
@@ -117,6 +118,29 @@ case "$cmd" in
   ##########################################
   # NETWORK DEPLOYMENTS WITH BENCHES/TESTS #
   ##########################################
+  network-scenarios)
+    # Args: <scenario> <namespace> [docker_image] [test_set]
+    # If test_set provided, run just that set. Otherwise run both in parallel.
+    scenario="${1:?scenario is required}"
+    namespace="${2:?namespace is required}"
+    docker_image="${3:-}"
+    test_set="${4:-}"
+
+    export CI_DASHBOARD="network"
+    run() {
+      local set=$1
+      JOB_ID="x-${namespace}-${set}" INSTANCE_POSTFIX="n-deploy-${set}" \
+        bootstrap_ec2 "./bootstrap.sh ci-network-deploy $scenario ${namespace}-${set} \"$docker_image\" $set"
+    }
+    export -f run
+    export scenario namespace docker_image
+
+    if [[ -n "$test_set" ]]; then
+      run "$test_set"
+    else
+      parallel --jobs 2 --line-buffered ::: 'run 1' 'run 2'
+    fi
+    ;;
   network-deploy)
     # Args: <scenario> <namespace> [docker_image]
     # If docker_image is not provided, ci-network-deploy will build and push to aztecdev.