TEST-#7441: Correctly skip sanity tests if we don't need them.

.github/workflows/ci.yml

@@ -547,41 +547,61 @@ jobs:
     # The "sanity" tests run on each pull request to test that a subset of the
     # full tests work with the slower engines (ray, dask, and unidist-MPI).
     needs: [lint-flake8, execution-filter, python-filter]
-    if: github.event_name == 'pull_request'
+    # If we don't need to run any sanity tests, the job matrix that we generate
+    # here gives a single job with all the matrix fields empty (that is, os,
+    # execution, etc. are not set, so we treat them as "").
+    # so, if the matrix is going to be empty, we need to skip this job
+    # completely. This bizarre behavior is not in the official documentation,
+    # of GitHub actions matrices, but someone does mention it here:
+    # https://stackoverflow.com/a/77118991
+    if: |
+      github.event_name == 'pull_request' &&
+      (
+        needs.execution-filter.outputs.ray != 'true' ||
+        needs.execution-filter.outputs.dask != 'true' ||
+        needs.execution-filter.outputs.unidist != 'true'
+      )
     strategy:
       matrix:
         os:
           - ubuntu
           - windows
         python-version: [ "${{ needs.python-filter.outputs.python-version }}" ]
-        execution:
-          - name: ray
-            shell-ex: "python -m pytest"
-            # If we're going to run all ray tests because we've detected a
-            # change to the ray engine, we don't need to run these sanity tests
-            # on ray.
-            if: needs.execution-filter.outputs.ray != 'true'
-          - name: dask
-            shell-ex: "python -m pytest"
-            # If we're going to run all dask tests because we've detected a
-            # change to the dask engine, we don't need to run these sanity tests
-            # on dask.
-            if: needs.execution-filter.outputs.dask != 'true'
-          - name: unidist
-            shell-ex: "mpiexec -n 1 -genv AWS_ACCESS_KEY_ID foobar_key -genv AWS_SECRET_ACCESS_KEY foobar_secret python -m pytest"
-            # If we're going to run all unidist tests because we've detected a
-            # change to the unidist engine, we don't need to run these sanity tests
-            # on unidist.
-            if: needs.execution-filter.outputs.unidist != 'true'
+        running-all-ray-tests: [ "${{ needs.execution-filter.outputs.ray }}" ]
+        running-all-dask-tests: [ "${{needs.execution-filter.outputs.dask}}" ]
+        running-all-unidist-tests: [ "${{needs.execution-filter.outputs.unidist}}" ]
+        execution: [ray, dask, unidist]
+        # If we're going to run all ray tests because we've detected a
+        # change to the ray engine, we don't need to run these sanity tests
+        # on ray. Likewise for dask and unidist.
+        exclude:
+          - running-all-ray-tests: 'true'
+            execution: ray
+          - running-all-dask-tests: 'true'
+            execution: dask
+          - running-all-unidist-tests: 'true'
+            execution: unidist
     runs-on: ${{ matrix.os }}-latest
     defaults:
       run:
         shell: bash -l {0}
     env:
-      MODIN_ENGINE: ${{ matrix.execution.name }}
+      MODIN_ENGINE: ${{ matrix.execution }}
       UNIDIST_BACKEND: "mpi"
-      PARALLEL: ${{ matrix.execution.name != 'unidist' && matrix.os != 'windows' && '-n 2' || '' }}
-    name: test-${{ matrix.os }}-sanity (engine ${{ matrix.execution.name }}, python ${{matrix.python-version}})
+      PARALLEL: ${{ matrix.execution != 'unidist' && matrix.os != 'windows' && '-n 2' || '' }}
+      PYTEST_COMMAND: >-
+        ${{
+          (
+            (matrix.execution == 'ray' || matrix.execution == 'dask') &&
+            'python -m pytest'
+          ) ||
+          (
+            matrix.execution == 'unidist' &&
+            'mpiexec -n 1 -genv AWS_ACCESS_KEY_ID foobar_key -genv AWS_SECRET_ACCESS_KEY foobar_secret python -m pytest'
+          ) ||
+          'UNKNOWN_PYTEST_COMMAND'
+        }}
+    name: test-${{ matrix.os }}-sanity (engine ${{ matrix.execution }}, python ${{matrix.python-version}})
     services:
       moto:
         image: ${{ matrix.os != 'windows' && 'motoserver/moto:5.0.13' || '' }}
@@ -594,18 +614,18 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/mamba-env
         with:
-          environment-file: ${{ matrix.os == 'ubuntu' && matrix.execution.name == 'unidist' && 'requirements/env_unidist_linux.yml' || matrix.os == 'windows' && matrix.execution.name == 'unidist' && 'requirements/env_unidist_win.yml' || 'environment-dev.yml' }}
-          activate-environment: ${{ matrix.execution.name == 'unidist' && 'modin_on_unidist' || 'modin' }}
+          environment-file: ${{ matrix.os == 'ubuntu' && matrix.execution == 'unidist' && 'requirements/env_unidist_linux.yml' || matrix.os == 'windows' && matrix.execution == 'unidist' && 'requirements/env_unidist_win.yml' || 'environment-dev.yml' }}
+          activate-environment: ${{ matrix.execution == 'unidist' && 'modin_on_unidist' || 'modin' }}
           python-version: ${{matrix.python-version}}
       - name: Install HDF5
         run: sudo apt update && sudo apt install -y libhdf5-dev
         if: matrix.os != 'windows'
       - name: Limit ray memory
         run: echo "MODIN_MEMORY=1000000000" >> $GITHUB_ENV
-        if: matrix.os != 'windows' && matrix.execution.name == 'ray'
+        if: matrix.os != 'windows' && matrix.execution == 'ray'
       - name: Tell Modin to use existing ray cluster
         run: echo "MODIN_RAY_CLUSTER=True" >> $GITHUB_ENV
-        if: matrix.os == 'windows' && matrix.execution.name == 'ray'
+        if: matrix.os == 'windows' && matrix.execution == 'ray'
       - name: Start local ray cluster
         # Try a few times to start ray to work around
         # https://github.com/modin-project/modin/issues/4562
@@ -614,71 +634,71 @@ jobs:
           timeout_minutes: 5
           max_attempts: 5
           command: ray start --head --port=6379 --object-store-memory=1000000000
-        if: matrix.os == 'windows' && matrix.execution.name == 'ray'
-      - run: MODIN_BENCHMARK_MODE=True ${{ matrix.execution.shell-ex }} modin/tests/pandas/internals/test_benchmark_mode.py
-      - run: ${{ matrix.execution.shell-ex }} $PARALLEL modin/tests/test_partition_api.py
-      - run: ${{ matrix.execution.shell-ex }} modin/tests/pandas/extensions
+        if: matrix.os == 'windows' && matrix.execution == 'ray'
+      - run: MODIN_BENCHMARK_MODE=True $PYTEST_COMMAND modin/tests/pandas/internals/test_benchmark_mode.py
+      - run: $PYTEST_COMMAND $PARALLEL modin/tests/test_partition_api.py
+      - run: $PYTEST_COMMAND modin/tests/pandas/extensions
       - name: xgboost tests
         run: |
           # TODO(https://github.com/modin-project/modin/issues/5194): Uncap xgboost
           # when we use collective instead of rabit.
           mamba install "xgboost>=1.7.1,<2.0.0" scikit-learn -c conda-forge
-          ${{ matrix.execution.shell-ex }} $PARALLEL \
+          $PYTEST_COMMAND $PARALLEL \
                   modin/tests/experimental/xgboost/test_default.py \
                   modin/tests/experimental/xgboost/test_xgboost.py \
                   modin/tests/experimental/xgboost/test_dmatrix.py
         if: matrix.os != 'windows' && needs.execution-filter.outputs.experimental == 'true'
-      - run: ${{ matrix.execution.shell-ex }} $PARALLEL modin/tests/experimental/test_pipeline.py
-        if: matrix.os != 'windows' && matrix.execution.name != 'unidist' && needs.execution-filter.outputs.experimental == 'true'
+      - run: $PYTEST_COMMAND $PARALLEL modin/tests/experimental/test_pipeline.py
+        if: matrix.os != 'windows' && matrix.execution != 'unidist' && needs.execution-filter.outputs.experimental == 'true'
       - name: "test DF: binary, default, iter"
         run: |
-          ${{ matrix.execution.shell-ex }} $PARALLEL \
+          $PYTEST_COMMAND $PARALLEL \
                   modin/tests/pandas/dataframe/test_binary.py \
                   modin/tests/pandas/dataframe/test_default.py \
                   modin/tests/pandas/dataframe/test_iter.py
         if: matrix.os != 'windows'
       - name: "test DF: reduce, udf, window, pickle"
         run: |
-          ${{ matrix.execution.shell-ex }} $PARALLEL \
+          $PYTEST_COMMAND $PARALLEL \
                   modin/tests/pandas/dataframe/test_reduce.py \
                   modin/tests/pandas/dataframe/test_udf.py \
                   modin/tests/pandas/dataframe/test_window.py \
                   modin/tests/pandas/dataframe/test_pickle.py
         if: matrix.os != 'windows'
-      - run: ${{ matrix.execution.shell-ex }} modin/tests/pandas/test_series.py
-        if: matrix.execution.name == 'ray'
-      - run: ${{ matrix.execution.shell-ex }} -m "not exclude_in_sanity" modin/tests/pandas/test_series.py
-        if: matrix.execution.name != 'ray'
-      - run: ${{ matrix.execution.shell-ex }} modin/tests/pandas/dataframe/test_map_metadata.py
-        if: matrix.execution.name == 'ray'
-      - run: ${{ matrix.execution.shell-ex }} -m "not exclude_in_sanity" modin/tests/pandas/dataframe/test_map_metadata.py
-        if: matrix.execution.name != 'ray'
+      - run: $PYTEST_COMMAND modin/tests/pandas/test_series.py
+        if: matrix.execution == 'ray'
+      - run: $PYTEST_COMMAND -m "not exclude_in_sanity" modin/tests/pandas/test_series.py
+        if: matrix.execution != 'ray'
+      - run: $PYTEST_COMMAND modin/tests/pandas/dataframe/test_map_metadata.py
+        if: matrix.execution == 'ray'
+      - run: $PYTEST_COMMAND -m "not exclude_in_sanity" modin/tests/pandas/dataframe/test_map_metadata.py
+        if: matrix.execution != 'ray'
       - name: "test rolling, expanding, reshape, general, concat"
         run: |
-          ${{ matrix.execution.shell-ex }} $PARALLEL \
+          $PYTEST_COMMAND $PARALLEL \
                   modin/tests/pandas/test_rolling.py \
                   modin/tests/pandas/test_expanding.py \
                   modin/tests/pandas/test_reshape.py \
                   modin/tests/pandas/test_general.py \
                   modin/tests/pandas/test_concat.py
         if: matrix.os != 'windows'
-      - run: ${{ matrix.execution.shell-ex }} $PARALLEL modin/tests/numpy
-      - run: ${{ matrix.execution.shell-ex }} -m "not exclude_in_sanity" modin/tests/pandas/test_io.py --verbose
-        if: matrix.execution.name != 'unidist'
+      - run: $PYTEST_COMMAND $PARALLEL modin/tests/numpy
+      - run: $PYTEST_COMMAND -m "not exclude_in_sanity" modin/tests/pandas/test_io.py --verbose
+        if: matrix.execution != 'unidist'
       - uses: nick-fields/retry@v3
         # to avoid issues with non-stable `to_csv` tests for unidist on MPI backend.
         # for details see: https://github.com/modin-project/modin/pull/6776
         with:
           timeout_minutes: 15
           max_attempts: 3
-          command: conda run --no-capture-output -n modin_on_unidist ${{ matrix.execution.shell-ex }} -m "not exclude_in_sanity" modin/tests/pandas/test_io.py --verbose
-        if: matrix.execution.name == 'unidist'
-      - run: ${{ matrix.execution.shell-ex }} modin/tests/experimental/test_io_exp.py
-      - run: ${{ matrix.execution.shell-ex }} $PARALLEL modin/tests/interchange/dataframe_protocol/test_general.py
-      - run: ${{ matrix.execution.shell-ex }} $PARALLEL modin/tests/interchange/dataframe_protocol/pandas/test_protocol.py
+          command: conda run --no-capture-output -n modin_on_unidist $PYTEST_COMMAND -m "not exclude_in_sanity" modin/tests/pandas/test_io.py --verbose
+        if: matrix.execution == 'unidist'
+      - run: $PYTEST_COMMAND modin/tests/experimental/test_io_exp.py
+      - run: $PYTEST_COMMAND $PARALLEL modin/tests/interchange/dataframe_protocol/test_general.py
+      - run: $PYTEST_COMMAND $PARALLEL modin/tests/interchange/dataframe_protocol/pandas/test_protocol.py
       - name: Stop local ray cluster
         run: ray stop
-        if: matrix.os == 'windows' && matrix.execution.name == 'ray'
+        if: matrix.os == 'windows' && matrix.execution == 'ray'
       - uses: ./.github/actions/upload-coverage
 
   test-experimental: