Skip to content

Commit 1f65be5

Browse files
emersodbemersodb
authored
Merge pull request #492 from VectorInstitute/dbe/ubuntu_version
Pinning Ubuntu Version
2 parents c1ca946 + 0bc9d32 commit 1f65be5

File tree

4 files changed

+225
-116
lines changed

4 files changed

+225
-116
lines changed

‎.github/workflows/nnunet_smoke_tests.yaml‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
name: nnUNet Smoke Tests
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
pull_request:
8+
branches:
9+
- main
10+
workflow_dispatch:
11+
12+
jobs:
13+
test:
14+
runs-on: ubuntu-22.04
15+
permissions:
16+
contents: read
17+
actions: write
18+
strategy:
19+
matrix:
20+
group: [1, 2, 3, 4]
21+
steps:
22+
- name: Checkout code
23+
uses: actions/checkout@v6
24+
25+
- name: Install uv
26+
uses: astral-sh/setup-uv@v7.1.5
27+
with:
28+
version: "0.9.11"
29+
enable-cache: true
30+
31+
- name: Set up Python
32+
uses: actions/setup-python@v6.1.0
33+
with:
34+
python-version-file: ".python-version"
35+
36+
- name: Display Python version
37+
run: python -c "import sys; print(sys.version)"
38+
39+
- name: Set up cache
40+
uses: actions/cache@v5
41+
id: cached-uv-dependencies
42+
with:
43+
path: .venv
44+
key: venv-${{ runner.os }}-${{ hashFiles('**/uv.lock') }}
45+
46+
- name: Set up file descriptor limit
47+
run: ulimit -n 4096
48+
49+
- name: Install dependencies
50+
run: uv sync --group dev --group test
51+
52+
- name: Run Script
53+
run: uv run pytest --test-group-count=4 --test-group=${{ matrix.group }} -v --cov fl4health --cov-report=xml tests/smoke_tests/test_nnunet_smoke_tests.py
54+
55+
- name: Upload coverage to Codecov
56+
uses: codecov/codecov-action@v5
57+
with:
58+
token: ${{ secrets.CODECOV_TOKEN }}
59+
slug: VectorInstitute/FL4Health
60+
fail_ci_if_error: true
61+
verbose: true
62+
63+
# Deleting some temporary files and useless folders to free up space
64+
# Deleting /usr/share/dotnet should clear ~4GB of space.
65+
# Deleting /usr/local/lib/android should clear ~12GB of space.
66+
- name: Cleanup space (before cache save)
67+
run: |
68+
df -h /dev/root
69+
sudo rm -rf /usr/share/dotnet /usr/local/lib/android
70+
df -h /dev/root
71+
72+
- name: Minimize uv cache
73+
run: uv cache prune --ci

.github/workflows/smoke_tests.yaml renamed to .github/workflows/standard_smoke_tests.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Smoke Tests
1+
name: Standard Smoke Tests
22

33
on:
44
push:
@@ -50,7 +50,7 @@ jobs:
5050
run: uv sync --group dev --group test
5151

5252
- name: Run Script
53-
run: uv run pytest --test-group-count=4 --test-group=${{ matrix.group }} -v --cov fl4health --cov-report=xml tests/smoke_tests/test_smoke_tests.py
53+
run: uv run pytest --test-group-count=4 --test-group=${{ matrix.group }} -v --cov fl4health --cov-report=xml tests/smoke_tests/test_standard_smoke_tests.py
5454

5555
- name: Upload coverage to Codecov
5656
uses: codecov/codecov-action@v5

‎tests/smoke_tests/test_nnunet_smoke_tests.py‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
import asyncio
2+
3+
import pytest
4+
5+
from tests.smoke_tests.run_smoke_test import (
6+
run_smoke_test,
7+
)
8+
9+
10+
# Marks all test coroutines in this module
11+
pytestmark = pytest.mark.asyncio(loop_scope="module")
12+
13+
14+
async def try_running_test_task(task: asyncio.Task) -> None:
15+
"""
16+
Helper for running task.
17+
18+
If an exception is reached, then cancel the task and wait for it to be cleared.
19+
"""
20+
try:
21+
await task
22+
except Exception as e:
23+
task.cancel()
24+
await asyncio.gather(task, return_exceptions=True) # allow time to clean up cancelled task
25+
pytest.fail(f"Smoke test failed due to error. {e}")
26+
27+
28+
def assert_on_done_task(task: asyncio.Task) -> None:
29+
"""
30+
This function takes a done task and makes assert if a result was returned.
31+
32+
If an exception was returned, then it fails the pytest for proper shutdown.
33+
Also, if the task was cancelled, then it cleans up the cancelled tasks so the
34+
next test doesn't get this hangover and fails as a result.
35+
"""
36+
e = task.exception() # handle TimeoutError / CancelledError above this func
37+
# at this point there is either an Exception or a Result and the task wasn't cancelled
38+
if e:
39+
pytest.fail(f"Smoke test execution failed: {e}")
40+
else:
41+
server_errors, client_errors = task.result()
42+
assert len(server_errors) == 0, f"Server metrics check failed. Errors: {server_errors}"
43+
assert len(client_errors) == 0, f"Client metrics check failed. Errors: {client_errors}"
44+
45+
46+
@pytest.mark.smoketest
47+
async def test_nnunet_config_2d(tolerance: float) -> None:
48+
coroutine = run_smoke_test( # By default will use Task04_Hippocampus Dataset
49+
server_python_path="examples.nnunet_example.server",
50+
client_python_path="examples.nnunet_example.client",
51+
config_path="tests/smoke_tests/nnunet_config_2d.yaml",
52+
dataset_path="examples/datasets/nnunet",
53+
tolerance=tolerance,
54+
read_logs_timeout=450,
55+
)
56+
task = asyncio.create_task(coroutine)
57+
await try_running_test_task(task)
58+
assert_on_done_task(task)
59+
60+
61+
@pytest.mark.smoketest
62+
async def test_nnunet_config_3d(tolerance: float) -> None:
63+
coroutine = run_smoke_test( # By default will use Task04_Hippocampus Dataset
64+
server_python_path="examples.nnunet_example.server",
65+
client_python_path="examples.nnunet_example.client",
66+
config_path="tests/smoke_tests/nnunet_config_3d.yaml",
67+
dataset_path="examples/datasets/nnunet",
68+
tolerance=tolerance,
69+
read_logs_timeout=450,
70+
)
71+
task = asyncio.create_task(coroutine)
72+
await try_running_test_task(task)
73+
assert_on_done_task(task)
74+
75+
76+
@pytest.mark.smoketest
77+
async def test_flexible_nnunet_config_2d(tolerance: float) -> None:
78+
coroutine = run_smoke_test( # By default will use Task04_Hippocampus Dataset
79+
server_python_path="examples.nnunet_example.server",
80+
client_python_path="examples.nnunet_example.client_flexible",
81+
config_path="tests/smoke_tests/nnunet_config_2d.yaml",
82+
dataset_path="examples/datasets/nnunet",
83+
tolerance=tolerance,
84+
read_logs_timeout=450,
85+
)
86+
task = asyncio.create_task(coroutine)
87+
await try_running_test_task(task)
88+
assert_on_done_task(task)
89+
90+
91+
@pytest.mark.smoketest
92+
async def test_flexible_nnunet_config_3d(tolerance: float) -> None:
93+
coroutine = run_smoke_test( # By default will use Task04_Hippocampus Dataset
94+
server_python_path="examples.nnunet_example.server",
95+
client_python_path="examples.nnunet_example.client_flexible",
96+
config_path="tests/smoke_tests/nnunet_config_3d.yaml",
97+
dataset_path="examples/datasets/nnunet",
98+
tolerance=tolerance,
99+
read_logs_timeout=450,
100+
)
101+
task = asyncio.create_task(coroutine)
102+
await try_running_test_task(task)
103+
assert_on_done_task(task)
104+
105+
106+
@pytest.mark.smoketest
107+
async def test_ditto_flexible_nnunet_config_2d(tolerance: float) -> None:
108+
coroutine = run_smoke_test( # By default will use Task04_Hippocampus Dataset
109+
server_python_path="examples.nnunet_pfl_example.server",
110+
client_python_path="examples.nnunet_pfl_example.client",
111+
config_path="tests/smoke_tests/nnunet_config_2d.yaml",
112+
dataset_path="examples/datasets/nnunet",
113+
tolerance=tolerance,
114+
read_logs_timeout=450,
115+
)
116+
task = asyncio.create_task(coroutine)
117+
await try_running_test_task(task)
118+
assert_on_done_task(task)
119+
120+
121+
@pytest.mark.smoketest
122+
async def test_nnunet_pfl_mr_mtl_config_3d(tolerance: float) -> None:
123+
coroutine = run_smoke_test( # By default will use Task04_Hippocampus Dataset
124+
server_python_path="examples.nnunet_pfl_example.server",
125+
client_python_path="examples.nnunet_pfl_example.client",
126+
config_path="tests/smoke_tests/nnunet_config_3d.yaml",
127+
dataset_path="examples/datasets/nnunet",
128+
additional_client_args={"--personalized_strategy": "mr_mtl"},
129+
tolerance=tolerance,
130+
read_logs_timeout=450,
131+
)
132+
task = asyncio.create_task(coroutine)
133+
await try_running_test_task(task)
134+
assert_on_done_task(task)
135+
136+
137+
@pytest.mark.smoketest
138+
async def test_nnunet_pfl_mr_mtl_config_2d(tolerance: float) -> None:
139+
coroutine = run_smoke_test( # By default will use Task04_Hippocampus Dataset
140+
server_python_path="examples.nnunet_pfl_example.server",
141+
client_python_path="examples.nnunet_pfl_example.client",
142+
config_path="tests/smoke_tests/nnunet_config_2d.yaml",
143+
dataset_path="examples/datasets/nnunet",
144+
additional_client_args={"--personalized_strategy": "mr_mtl"},
145+
tolerance=tolerance,
146+
read_logs_timeout=450,
147+
)
148+
task = asyncio.create_task(coroutine)
149+
await try_running_test_task(task)
150+
assert_on_done_task(task)

tests/smoke_tests/test_smoke_tests.py renamed to tests/smoke_tests/test_standard_smoke_tests.py

Lines changed: 0 additions & 114 deletions
Original file line numberDiff line numberDiff line change
@@ -75,38 +75,6 @@ async def test_basic_server_client_checkpoint(tolerance: float, tmp_path: Path)
7575
assert_on_done_task(task)
7676

7777

78-
@pytest.mark.smoketest
79-
@pytest.mark.skipif(IN_GITHUB_ACTIONS, reason="Test doesn't work on github.")
80-
async def test_nnunet_config_2d(tolerance: float) -> None:
81-
coroutine = run_smoke_test( # By default will use Task04_Hippocampus Dataset
82-
server_python_path="examples.nnunet_example.server",
83-
client_python_path="examples.nnunet_example.client",
84-
config_path="tests/smoke_tests/nnunet_config_2d.yaml",
85-
dataset_path="examples/datasets/nnunet",
86-
tolerance=tolerance,
87-
read_logs_timeout=450,
88-
)
89-
task = asyncio.create_task(coroutine)
90-
await try_running_test_task(task)
91-
assert_on_done_task(task)
92-
93-
94-
@pytest.mark.smoketest
95-
@pytest.mark.skipif(IN_GITHUB_ACTIONS, reason="Test doesn't work on github.")
96-
async def test_nnunet_config_3d(tolerance: float) -> None:
97-
coroutine = run_smoke_test( # By default will use Task04_Hippocampus Dataset
98-
server_python_path="examples.nnunet_example.server",
99-
client_python_path="examples.nnunet_example.client",
100-
config_path="tests/smoke_tests/nnunet_config_3d.yaml",
101-
dataset_path="examples/datasets/nnunet",
102-
tolerance=tolerance,
103-
read_logs_timeout=450,
104-
)
105-
task = asyncio.create_task(coroutine)
106-
await try_running_test_task(task)
107-
assert_on_done_task(task)
108-
109-
11078
@pytest.mark.smoketest
11179
async def test_scaffold(tolerance: float) -> None:
11280
coroutine = run_smoke_test(
@@ -472,85 +440,3 @@ async def test_gpfl(tolerance: float) -> None:
472440
task = asyncio.create_task(coroutine)
473441
await try_running_test_task(task)
474442
assert_on_done_task(task)
475-
476-
477-
@pytest.mark.smoketest
478-
@pytest.mark.skipif(IN_GITHUB_ACTIONS, reason="Test doesn't work on github.")
479-
async def test_flexible_nnunet_config_2d(tolerance: float) -> None:
480-
coroutine = run_smoke_test( # By default will use Task04_Hippocampus Dataset
481-
server_python_path="examples.nnunet_example.server",
482-
client_python_path="examples.nnunet_example.client_flexible",
483-
config_path="tests/smoke_tests/nnunet_config_2d.yaml",
484-
dataset_path="examples/datasets/nnunet",
485-
tolerance=tolerance,
486-
read_logs_timeout=450,
487-
)
488-
task = asyncio.create_task(coroutine)
489-
await try_running_test_task(task)
490-
assert_on_done_task(task)
491-
492-
493-
@pytest.mark.smoketest
494-
@pytest.mark.skipif(IN_GITHUB_ACTIONS, reason="Test doesn't work on github.")
495-
async def test_flexible_nnunet_config_3d(tolerance: float) -> None:
496-
coroutine = run_smoke_test( # By default will use Task04_Hippocampus Dataset
497-
server_python_path="examples.nnunet_example.server",
498-
client_python_path="examples.nnunet_example.client_flexible",
499-
config_path="tests/smoke_tests/nnunet_config_3d.yaml",
500-
dataset_path="examples/datasets/nnunet",
501-
tolerance=tolerance,
502-
read_logs_timeout=450,
503-
)
504-
task = asyncio.create_task(coroutine)
505-
await try_running_test_task(task)
506-
assert_on_done_task(task)
507-
508-
509-
@pytest.mark.smoketest
510-
@pytest.mark.skipif(IN_GITHUB_ACTIONS, reason="Test doesn't work on github.")
511-
async def test_ditto_flexible_nnunet_config_2d(tolerance: float) -> None:
512-
coroutine = run_smoke_test( # By default will use Task04_Hippocampus Dataset
513-
server_python_path="examples.nnunet_pfl_example.server",
514-
client_python_path="examples.nnunet_pfl_example.client",
515-
config_path="tests/smoke_tests/nnunet_config_2d.yaml",
516-
dataset_path="examples/datasets/nnunet",
517-
tolerance=tolerance,
518-
read_logs_timeout=450,
519-
)
520-
task = asyncio.create_task(coroutine)
521-
await try_running_test_task(task)
522-
assert_on_done_task(task)
523-
524-
525-
@pytest.mark.smoketest
526-
@pytest.mark.skipif(IN_GITHUB_ACTIONS, reason="Test doesn't work on github.")
527-
async def test_nnunet_pfl_mr_mtl_config_3d(tolerance: float) -> None:
528-
coroutine = run_smoke_test( # By default will use Task04_Hippocampus Dataset
529-
server_python_path="examples.nnunet_pfl_example.server",
530-
client_python_path="examples.nnunet_pfl_example.client",
531-
config_path="tests/smoke_tests/nnunet_config_3d.yaml",
532-
dataset_path="examples/datasets/nnunet",
533-
additional_client_args={"--personalized_strategy": "mr_mtl"},
534-
tolerance=tolerance,
535-
read_logs_timeout=450,
536-
)
537-
task = asyncio.create_task(coroutine)
538-
await try_running_test_task(task)
539-
assert_on_done_task(task)
540-
541-
542-
@pytest.mark.smoketest
543-
@pytest.mark.skipif(IN_GITHUB_ACTIONS, reason="Test doesn't work on github.")
544-
async def test_nnunet_pfl_mr_mtl_config_2d(tolerance: float) -> None:
545-
coroutine = run_smoke_test( # By default will use Task04_Hippocampus Dataset
546-
server_python_path="examples.nnunet_pfl_example.server",
547-
client_python_path="examples.nnunet_pfl_example.client",
548-
config_path="tests/smoke_tests/nnunet_config_2d.yaml",
549-
dataset_path="examples/datasets/nnunet",
550-
additional_client_args={"--personalized_strategy": "mr_mtl"},
551-
tolerance=tolerance,
552-
read_logs_timeout=450,
553-
)
554-
task = asyncio.create_task(coroutine)
555-
await try_running_test_task(task)
556-
assert_on_done_task(task)

0 commit comments

Comments
 (0)