Skip to content

Commit 72fb2ed

Browse files
lucasjia-awslucasjia-aws
authored
ci: Split CI Health workflow into V2 and V3 (#6067)
Replace the single CI Health workflow with two independent workflows: - ci-health-v2.yml: canaries-v2-master, canaries-v2-release, unit-tests-v2 (python-version matrix), slow-tests-v2, localmode-tests-v2. - ci-health-v3.yml: unit-test-v3 and canaries-v3-master / canaries-v3-release, the latter two driven by a submodule matrix (SUBMODULE env var) so one CodeBuild project runs all four submodules in parallel. Both workflows run once a day at 05:00 UTC (9pm PDT / 10pm PST) instead of every three hours, and keep workflow_dispatch for manual runs. The new canary jobs target the split CodeBuild projects created in SageMakerMLFPySDKInfraCDK.
1 parent cb70bf8 commit 72fb2ed

3 files changed

Lines changed: 92 additions & 36 deletions

File tree

.github/workflows/ci-health.yml renamed to .github/workflows/ci-health-v2.yml

Lines changed: 11 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,15 @@
1-
name: CI Health
1+
name: CI Health V2
22
on:
33
schedule:
4-
- cron: "0 */3 * * *"
4+
# Once a day at 05:00 UTC (9pm PDT / 10pm PST).
5+
- cron: "0 5 * * *"
56
workflow_dispatch:
6-
7+
78
permissions:
89
id-token: write # This is required for requesting the JWT
910

1011
jobs:
11-
canaries-v3:
12+
canaries-v2-master:
1213
runs-on: ubuntu-latest
1314
steps:
1415
- name: Configure AWS Credentials
@@ -17,12 +18,12 @@ jobs:
1718
role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
1819
aws-region: us-west-2
1920
role-duration-seconds: 10800
20-
- name: Run Canaries V3
21+
- name: Run Canaries V2 (master)
2122
uses: aws-actions/aws-codebuild-run-build@v1
2223
with:
23-
project-name: sagemaker-python-sdk-ci-health-canaries-v3
24-
source-version-override: refs/heads/master
25-
canaries-v2:
24+
project-name: sagemaker-python-sdk-ci-health-canaries-v2-master
25+
source-version-override: refs/heads/master-v2
26+
canaries-v2-release:
2627
runs-on: ubuntu-latest
2728
steps:
2829
- name: Configure AWS Credentials
@@ -31,10 +32,10 @@ jobs:
3132
role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
3233
aws-region: us-west-2
3334
role-duration-seconds: 10800
34-
- name: Run Canaries V2
35+
- name: Run Canaries V2 (release)
3536
uses: aws-actions/aws-codebuild-run-build@v1
3637
with:
37-
project-name: sagemaker-python-sdk-ci-health-canaries-v2
38+
project-name: sagemaker-python-sdk-ci-health-canaries-v2-release
3839
source-version-override: refs/heads/master-v2
3940
unit-tests-v2:
4041
runs-on: ubuntu-latest
@@ -86,25 +87,3 @@ jobs:
8687
with:
8788
project-name: sagemaker-python-sdk-ci-health-localmode-tests
8889
source-version-override: refs/heads/master-v2
89-
unit-test-v3:
90-
runs-on: ubuntu-latest
91-
strategy:
92-
fail-fast: false
93-
matrix:
94-
submodule: [sagemaker-core, sagemaker-train, sagemaker-serve, sagemaker-mlops]
95-
steps:
96-
- name: Configure AWS Credentials
97-
uses: aws-actions/configure-aws-credentials@v4
98-
with:
99-
role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
100-
aws-region: us-west-2
101-
role-duration-seconds: 10800
102-
- name: Run Unit Tests V3 for ${{ matrix.submodule }}
103-
uses: aws-actions/aws-codebuild-run-build@v1
104-
with:
105-
project-name: sagemaker-python-sdk-ci-health-unit-test-v3
106-
source-version-override: refs/heads/master
107-
env-vars-for-codebuild: |
108-
SUBMODULE
109-
env:
110-
SUBMODULE: ${{ matrix.submodule }}

‎.github/workflows/ci-health-v3.yml‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
name: CI Health V3
2+
on:
3+
schedule:
4+
# Once a day at 05:00 UTC (9pm PDT / 10pm PST).
5+
- cron: "0 5 * * *"
6+
workflow_dispatch:
7+
8+
permissions:
9+
id-token: write # This is required for requesting the JWT
10+
11+
jobs:
12+
unit-test-v3:
13+
runs-on: ubuntu-latest
14+
strategy:
15+
fail-fast: false
16+
matrix:
17+
submodule: [sagemaker-core, sagemaker-train, sagemaker-serve, sagemaker-mlops]
18+
steps:
19+
- name: Configure AWS Credentials
20+
uses: aws-actions/configure-aws-credentials@v4
21+
with:
22+
role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
23+
aws-region: us-west-2
24+
role-duration-seconds: 10800
25+
- name: Run Unit Tests V3 for ${{ matrix.submodule }}
26+
uses: aws-actions/aws-codebuild-run-build@v1
27+
with:
28+
project-name: sagemaker-python-sdk-ci-health-unit-test-v3
29+
source-version-override: refs/heads/master
30+
env-vars-for-codebuild: |
31+
SUBMODULE
32+
env:
33+
SUBMODULE: ${{ matrix.submodule }}
34+
canaries-v3-master:
35+
runs-on: ubuntu-latest
36+
strategy:
37+
fail-fast: false
38+
matrix:
39+
submodule: [sagemaker-core, sagemaker-train, sagemaker-serve, sagemaker-mlops]
40+
steps:
41+
- name: Configure AWS Credentials
42+
uses: aws-actions/configure-aws-credentials@v4
43+
with:
44+
role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
45+
aws-region: us-west-2
46+
role-duration-seconds: 10800
47+
- name: Run Canaries V3 master for ${{ matrix.submodule }}
48+
uses: aws-actions/aws-codebuild-run-build@v1
49+
with:
50+
project-name: sagemaker-python-sdk-ci-health-canaries-v3-master
51+
source-version-override: refs/heads/master
52+
env-vars-for-codebuild: |
53+
SUBMODULE
54+
env:
55+
SUBMODULE: ${{ matrix.submodule }}
56+
canaries-v3-release:
57+
runs-on: ubuntu-latest
58+
strategy:
59+
fail-fast: false
60+
matrix:
61+
submodule: [sagemaker-core, sagemaker-train, sagemaker-serve, sagemaker-mlops]
62+
steps:
63+
- name: Configure AWS Credentials
64+
uses: aws-actions/configure-aws-credentials@v4
65+
with:
66+
role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
67+
aws-region: us-west-2
68+
role-duration-seconds: 10800
69+
- name: Run Canaries V3 release for ${{ matrix.submodule }}
70+
uses: aws-actions/aws-codebuild-run-build@v1
71+
with:
72+
project-name: sagemaker-python-sdk-ci-health-canaries-v3-release
73+
source-version-override: refs/heads/master
74+
env-vars-for-codebuild: |
75+
SUBMODULE
76+
env:
77+
SUBMODULE: ${{ matrix.submodule }}

‎.github/workflows/gpu-integ-tests.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
name: GPU Integ Tests
22
on:
33
schedule:
4-
# US Pacific (PST, UTC-8): 10:00 PM / 1:00 AM / 4:00 AM -> 06/09/12 UTC.
4+
# US Pacific (PST, UTC-8): 0:00 AM / 3:00 AM / 6:00 AM -> 08/11/14 UTC.
55
# All three fire within the same UTC day so the run-level CloudWatch metric
66
# (GpuIntegRunFailure) aggregates correctly per day.
7-
- cron: "0 6 * * *"
8-
- cron: "0 9 * * *"
9-
- cron: "0 12 * * *"
7+
- cron: "0 8 * * *"
8+
- cron: "0 11 * * *"
9+
- cron: "0 14 * * *"
1010
workflow_dispatch:
1111

1212
permissions:

0 commit comments

Comments
 (0)