|
| 1 | +name: Notebook Tests |
| 2 | + |
| 3 | +# --------------------------------------------------------------------------- |
| 4 | +# Runs the changed SageMaker example notebooks on real SageMaker infrastructure |
| 5 | +# when a PR touches any notebook (**/*.ipynb) targeting the NotebookTestEngine |
| 6 | +# integration branch of aws/amazon-sagemaker-examples. Flow: |
| 7 | +# 1. Collaborator check -> collaborators auto-approve; fork / non-collaborator |
| 8 | +# PRs are gated behind the manual-approval environment. |
| 9 | +# 2. Assume an AWS role via GitHub OIDC (no stored AWS keys) and start the |
| 10 | +# `notebook-test-engine` CodeBuild project against this PR's notebooks. |
| 11 | +# 3. Poll the build to completion; this job's pass/fail IS the PR check, and a |
| 12 | +# failing notebook is attached as a downloadable `failing-notebooks` artifact. |
| 13 | +# --------------------------------------------------------------------------- |
| 14 | + |
| 15 | +on: |
| 16 | + pull_request_target: |
| 17 | + branches: |
| 18 | + - NotebookTestEngine |
| 19 | + paths: |
| 20 | + - '**/*.ipynb' |
| 21 | + |
| 22 | +# Only the newest commit of a PR runs; a new push cancels the older run. |
| 23 | +concurrency: |
| 24 | + group: notebook-tests-${{ github.event.pull_request.number || github.head_ref }} |
| 25 | + cancel-in-progress: true |
| 26 | + |
| 27 | +permissions: |
| 28 | + id-token: write # required to mint the GitHub OIDC token for AWS |
| 29 | + contents: read |
| 30 | + |
| 31 | +jobs: |
| 32 | + # 1) Collaborator? -> auto-approve. Otherwise -> manual-approval (gated below). |
| 33 | + collab-check: |
| 34 | + runs-on: ubuntu-latest |
| 35 | + outputs: |
| 36 | + approval-env: ${{ steps.collab-check.outputs.result }} |
| 37 | + steps: |
| 38 | + - name: Collaborator check |
| 39 | + uses: actions/github-script@v7 |
| 40 | + id: collab-check |
| 41 | + with: |
| 42 | + github-token: ${{ secrets.COLLAB_CHECK_TOKEN || github.token }} |
| 43 | + result-encoding: string |
| 44 | + script: | |
| 45 | + try { |
| 46 | + const res = await github.rest.repos.checkCollaborator({ |
| 47 | + owner: context.repo.owner, |
| 48 | + repo: context.repo.repo, |
| 49 | + username: "${{ github.event.pull_request.user.login }}", |
| 50 | + }); |
| 51 | + return res.status == 204 ? "auto-approve" : "manual-approval"; |
| 52 | + } catch (error) { |
| 53 | + return "manual-approval"; |
| 54 | + } |
| 55 | +
|
| 56 | + # 2) Non-collaborators block on the "manual-approval" environment's required |
| 57 | + # reviewers. Collaborators pass straight through the "auto-approve" env. |
| 58 | + wait-for-approval: |
| 59 | + runs-on: ubuntu-latest |
| 60 | + needs: [collab-check] |
| 61 | + environment: ${{ needs.collab-check.outputs.approval-env }} |
| 62 | + steps: |
| 63 | + - run: echo "Approved — starting notebook tests." |
| 64 | + |
| 65 | + # 3) Assume the engine role via OIDC, start the build against the PR's notebooks, |
| 66 | + # and poll to completion. This job's result is the PR check. |
| 67 | + notebook-tests: |
| 68 | + runs-on: ubuntu-latest |
| 69 | + needs: [wait-for-approval] |
| 70 | + # GitHub hard-caps hosted jobs at 360 min (6h). The CodeBuild project can run up |
| 71 | + # to 7h, but a pr_check build tests only the PR's changed notebooks and finishes |
| 72 | + # well within this; 360 is the maximum GitHub allows for a hosted job. |
| 73 | + timeout-minutes: 360 |
| 74 | + steps: |
| 75 | + - name: Configure AWS credentials (OIDC) |
| 76 | + uses: aws-actions/configure-aws-credentials@v4 |
| 77 | + with: |
| 78 | + role-to-assume: ${{ secrets.NOTEBOOK_TEST_ENGINE_ROLE_ARN }} |
| 79 | + aws-region: us-west-2 |
| 80 | + # 8h session (matches the role's maxSessionDuration in the CDK stack) so the |
| 81 | + # poll never expires mid-build. The GitHub job itself is capped at 360 min below. |
| 82 | + role-duration-seconds: 28800 |
| 83 | + |
| 84 | + - name: Start CodeBuild and wait |
| 85 | + id: build |
| 86 | + env: |
| 87 | + PR_NUMBER: ${{ github.event.pull_request.number }} |
| 88 | + PR_SHA: ${{ github.event.pull_request.head.sha }} |
| 89 | + PR_BRANCH: ${{ github.event.pull_request.head.ref }} |
| 90 | + PR_REPO: ${{ github.event.pull_request.head.repo.full_name }} |
| 91 | + PR_REPO_FULL: ${{ github.event.pull_request.base.repo.full_name }} |
| 92 | + PR_ARTIFACTS_BUCKET: notebook-test-engine-pr-674622101542-us-west-2 |
| 93 | + run: | |
| 94 | + set -euo pipefail |
| 95 | + PROJECT=notebook-test-engine |
| 96 | +
|
| 97 | + # Check out the PR head of the 'sdk' secondary source. refs/pull/<N>/head |
| 98 | + # resolves in the BASE repo even for fork PRs. Use the plain ref (not a |
| 99 | + # ^{sha}-pinned commit) so CodeBuild checks out the ref tip; |
| 100 | + # concurrency:cancel-in-progress (above) keeps only the newest commit's run, |
| 101 | + # so the tip is the commit under test. |
| 102 | + SDK_VERSION="refs/pull/${PR_NUMBER}/head" |
| 103 | +
|
| 104 | + BUILD_ID=$(aws codebuild start-build \ |
| 105 | + --project-name "$PROJECT" \ |
| 106 | + --secondary-sources-version-override "[{\"sourceIdentifier\":\"sdk\",\"sourceVersion\":\"${SDK_VERSION}\"}]" \ |
| 107 | + --environment-variables-override \ |
| 108 | + "name=TEST_MODE,value=pr_check,type=PLAINTEXT" \ |
| 109 | + "name=MAX_CONCURRENT,value=18,type=PLAINTEXT" \ |
| 110 | + "name=PR_NUMBER,value=${PR_NUMBER},type=PLAINTEXT" \ |
| 111 | + "name=PR_BRANCH,value=${PR_BRANCH},type=PLAINTEXT" \ |
| 112 | + "name=PR_REPO,value=${PR_REPO},type=PLAINTEXT" \ |
| 113 | + "name=PR_SHA,value=${PR_SHA},type=PLAINTEXT" \ |
| 114 | + "name=PR_REPO_FULL,value=${PR_REPO_FULL},type=PLAINTEXT" \ |
| 115 | + --query 'build.id' --output text) |
| 116 | +
|
| 117 | + echo "Started build: $BUILD_ID" |
| 118 | + echo "Console: https://us-west-2.console.aws.amazon.com/codesuite/codebuild/projects/${PROJECT}/build/${BUILD_ID//:/%3A}/log" |
| 119 | +
|
| 120 | + # Poll until the build leaves IN_PROGRESS. |
| 121 | + STATUS=IN_PROGRESS |
| 122 | + while [ "$STATUS" = "IN_PROGRESS" ]; do |
| 123 | + sleep 300 |
| 124 | + STATUS=$(aws codebuild batch-get-builds --ids "$BUILD_ID" \ |
| 125 | + --query 'builds[0].buildStatus' --output text 2>/dev/null || echo IN_PROGRESS) |
| 126 | + echo "Build status: $STATUS" |
| 127 | + done |
| 128 | +
|
| 129 | + echo "Final build status: $STATUS" |
| 130 | + echo "status=$STATUS" >> "$GITHUB_OUTPUT" |
| 131 | +
|
| 132 | + # On any non-success, pull the engine-rendered failing-notebook HTML/ipynb |
| 133 | + # (written under pr/<PR#>/<run_ts>/) so the next step can attach it to this run. |
| 134 | + if [ "$STATUS" != "SUCCEEDED" ]; then |
| 135 | + echo "Build did not succeed; fetching this run's failing-notebook artifacts..." |
| 136 | + # Only the NEWEST run-timestamp folder -- not every historical render for |
| 137 | + # this PR -- so the artifact holds just this run's notebook(s). |
| 138 | + LATEST=$(aws s3 ls "s3://${PR_ARTIFACTS_BUCKET}/pr/${PR_NUMBER}/" 2>/dev/null | awk '{print $NF}' | grep '/$' | sort | tail -1 || true) |
| 139 | + if [ -n "$LATEST" ]; then |
| 140 | + aws s3 cp --recursive "s3://${PR_ARTIFACTS_BUCKET}/pr/${PR_NUMBER}/${LATEST}" pr-artifacts/ || true |
| 141 | + fi |
| 142 | + fi |
| 143 | +
|
| 144 | + - name: Upload failing notebook(s) |
| 145 | + if: steps.build.outputs.status != 'SUCCEEDED' |
| 146 | + uses: actions/upload-artifact@v4 |
| 147 | + with: |
| 148 | + name: failing-notebooks |
| 149 | + path: pr-artifacts/ |
| 150 | + if-no-files-found: ignore |
| 151 | + |
| 152 | + - name: Reflect build result as the check |
| 153 | + if: always() |
| 154 | + run: | |
| 155 | + echo "CodeBuild final status: ${{ steps.build.outputs.status }}" |
| 156 | + # Anything other than SUCCEEDED (FAILED / FAULT / TIMED_OUT / STOPPED) |
| 157 | + # fails this job -> red PR check. |
| 158 | + [ "${{ steps.build.outputs.status }}" = "SUCCEEDED" ] |
0 commit comments