Skip to content
Build and Test

Move GPU CI + benchmarks off self-hosted onto Cirun ephemeral runners #235

Sign in to view logs

Move GPU CI + benchmarks off self-hosted onto Cirun ephemeral runners

Move GPU CI + benchmarks off self-hosted onto Cirun ephemeral runners #235

Workflow file for this run

.github/workflows/build-and-test.yml at accc7c8
name: Build and Test
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
workflow_dispatch:
jobs:
# Detect whether a change touches anything that affects the built SCAMP
# binary or its tests. Used to skip the *paid* ephemeral-GPU job on
# docs-only PRs (the CPU jobs stay unconditional -- they run on free
# GitHub-hosted runners). This is a positive allowlist: anything not
# listed here (docs/**, *.md, LICENSE, .readthedocs.yaml, ...) is treated
# as docs-only. If you add a new top-level path that affects the build,
# add it here or the GPU job will skip it.
changes:
runs-on: ubuntu-latest
permissions:
pull-requests: read
outputs:
code: ${{ steps.filter.outputs.code }}
steps:
- uses: actions/checkout@v4
- uses: dorny/paths-filter@v3
id: filter
with:
filters: |
code:
- 'src/**'
- 'test/**'
- 'cmake/**'
- 'third_party/**'
- '**/CMakeLists.txt'
- '.gitmodules'
- 'setup.py'
- 'MANIFEST.in'
- 'requirements.txt'
- '.github/workflows/build-and-test.yml'
- '.cirun.yml'
build-and-test-cpu-cli:
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
compiler: [g++, clang++, cl]
redistributable: [OFF, ON]
exclude:
- os: ubuntu-latest
compiler: cl
- os: macos-latest
compiler: cl
- os: macos-latest
compiler: g++
- os: windows-latest
compiler: g++
- os: windows-latest
compiler: clang++
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
with:
submodules: 'true'
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.x'
- name: Prep Build
shell: bash
run: |
set -e
echo "CXX=${{ matrix.compiler }}" >> $GITHUB_ENV
echo "SCAMP_ENABLE_BINARY_DISTRIBUTION=${{ matrix.redistributable }}" >> $GITHUB_ENV
- name: Build SCAMP
shell: bash
run: |
set -e
cd $GITHUB_WORKSPACE
mkdir build && cd build
cmake ..
cmake --build . --config Release --parallel 2
if [ -f "./Release/SCAMP.exe" ]; then
cp ./Release/SCAMP.exe .
fi
- name: Basic Tests SCAMP
shell: bash
run: |
set -e
cd $GITHUB_WORKSPACE/test
./run_tests.sh ../build/SCAMP /dev/null "--no_gpu=true --num_cpu_workers=2"
- name: Extended Tests SCAMP
shell: bash
run: |
set -e
python3 -m pip install numpy pandas tqdm
cd $GITHUB_WORKSPACE/test
python3 ./run_tests.py --executable "../build/SCAMP" --extra_args "--no_gpu --num_cpu_workers=2"
build-and-test-arch-emulation:
strategy:
fail-fast: false
matrix:
# TODO(zpzim): Windows is disabled for now because it is flaky.
#os: [ubuntu-latest, windows-latest]
#compiler: [g++, clang++, cl]
os: [ubuntu-latest]
compiler: [g++, clang++]
#exclude:
# - os: ubuntu-latest
# compiler: cl
# - os: windows-latest
# compiler: g++
# - os: windows-latest
# compiler: clang++
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
with:
submodules: 'true'
- name: Setup SDE binaries
# v4.0 broke when Intel reorganized their SDE download page (the setup
# step failed at download). v6.0 (Aug 2026) tracks the new layout, with
# compatible inputs/outputs (sdeVersion + the SDE_PATH env var consumed
# below). Bumped the pinned SDE to 9.58.0 as well, in case the old
# 9.33.0 build was pruned; the emulation test only exercises standard
# arch flags (p4p..spr) that every recent SDE supports.
uses: petarpetrovt/setup-sde@v6.0
with:
sdeVersion: 9.58.0
- name: Prep Build
shell: bash
run: |
set -e
echo "CXX=${{ matrix.compiler }}" >> $GITHUB_ENV
echo "SCAMP_ENABLE_BINARY_DISTRIBUTION=ON" >> $GITHUB_ENV
if [ "${{ matrix.os }}" = "windows-latest" ]; then
SDE_PATH=`cygpath -u $SDE_PATH`
echo $SDE_PATH
echo "SDE_BINARY_PATH=${SDE_PATH}/sde.exe" >> $GITHUB_ENV
else
echo "SDE_BINARY_PATH=${SDE_PATH}/sde64" >> $GITHUB_ENV
fi
- name: Build SCAMP
shell: bash
run: |
set -e
cd $GITHUB_WORKSPACE
mkdir build && cd build
cmake ..
cmake --build . --config Release --parallel 2
if [ -f "./Release/SCAMP.exe" ]; then
cp ./Release/SCAMP.exe .
fi
- name: Intel x86 Emulation Tests
shell: bash
run: |
set -e
cd $GITHUB_WORKSPACE/test
./run_intel_emulation_tests.sh "../build/SCAMP" ${SDE_BINARY_PATH}
build-and-test-cpu-distributed:
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest]
compiler: [g++, clang++]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
with:
submodules: 'recursive'
- name: Prep Build
run: |
echo "CXX=${{ matrix.compiler }}" >> $GITHUB_ENV
if [ "${{ matrix.compiler }}" = "g++" ]; then
echo "CC=gcc" >> $GITHUB_ENV
elif [ "${{ matrix.compiler }}" = "clang++" ]; then
echo "CC=clang" >> $GITHUB_ENV
fi
- name: Build SCAMP Distributed
shell: bash
run: |
set -e
cd $GITHUB_WORKSPACE
mkdir build && cd build
cmake -DBUILD_CLIENT_SERVER=1 ..
cmake --build . --config Release --parallel 2
- name: Distributed Tests SCAMP
shell: bash
run: |
set -e
python3 -m pip install numpy pandas tqdm
cd $GITHUB_WORKSPACE/test
../build/src/distributed/SCAMPserver &
../build/src/distributed/SCAMPclient &
sleep 10
./run_tests.sh ../build/src/distributed/SCAMP_distributed /dev/null
python3 ./run_tests.py --executable ../build/src/distributed/SCAMP_distributed --matrix_sizes --window_sizes 100 --extra_args "--no_gpu --num_cpu_workers=1 --distributed_tile_size=1500"
build-and-test-pyscamp:
name: pyscamp ${{ matrix.os }} py${{ matrix.python-version }} np${{ matrix.numpy }} redistrib${{ matrix.redistributable }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
python-version: ['3.10', '3.12', '3.13']
numpy: ['1', '2']
redistributable: ['OFF', 'ON']
exclude:
# numpy 1.x wheels do not exist for cp313 on PyPI
- python-version: '3.13'
numpy: '1'
# Windows: only test the latest stable config (py3.13 + numpy2), both redistrib values
- os: windows-latest
python-version: '3.10'
- os: windows-latest
python-version: '3.12'
- os: windows-latest
numpy: '1'
# Redistrib ON only tested with py3.13 to keep job count down
- python-version: '3.10'
redistributable: 'ON'
- python-version: '3.12'
redistributable: 'ON'
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
with:
submodules: 'true'
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Build pyscamp
shell: bash
run: |
set -e
echo "SCAMP_ENABLE_BINARY_DISTRIBUTION=${{ matrix.redistributable }}" >> $GITHUB_ENV
python -m pip install --upgrade pip
python -m pip install --upgrade "setuptools>=70" wheel setuptools_scm cmake
if [ "${{ matrix.numpy }}" = "1" ]; then
python -m pip install "numpy<2" pandas tqdm
else
python -m pip install "numpy>=2" pandas tqdm
fi
python -m pip install -v --no-build-isolation .
- name: pyscamp tests
shell: bash
run: |
set -e
cd $GITHUB_WORKSPACE/test
python test_pyscamp.py
python ./run_tests.py --executable pyscamp
build-cuda-versions:
name: cuda-build ${{ matrix.cuda }} ${{ matrix.os }} ${{ matrix.compiler }}
strategy:
fail-fast: false
# We test two CUDA versions chosen to span the breakpoints we care about:
# 12.8.0 (CCCL 2.8.x with the arch-token bug) and 13.0.0 (CCCL 3.0 fix
# plus the sm_110/Blackwell rename). Each Jimver/cuda-toolkit cache
# entry is ~4GB, so the matrix is sized to stay close to GitHub's 10GB
# cache budget per repo. ubuntu+clang++ is included at 13.0.0 only as
# a smoke test for clang as the host compiler; nvcc owns CUDA codegen
# so the marginal coverage of testing it on every version is low.
matrix:
os: [ubuntu-latest, windows-latest]
cuda: ['12.8.0', '13.0.0']
compiler: [g++, clang++, cl]
exclude:
- os: ubuntu-latest
compiler: cl
- os: windows-latest
compiler: g++
- os: windows-latest
compiler: clang++
- cuda: '12.8.0'
compiler: clang++
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
with:
submodules: 'true'
- uses: Jimver/cuda-toolkit@v0.2.30
id: cuda-toolkit
with:
cuda: ${{ matrix.cuda }}
- name: Prep Build
run: echo "CXX=${{ matrix.compiler }}" >> $GITHUB_ENV
- name: Build SCAMP
shell: bash
run: |
set -e
cd $GITHUB_WORKSPACE
mkdir build && cd build
cmake -DFORCE_CUDA=1 -DBUILD_SCAMP_TESTS=ON ..
cmake --build . --config Release --parallel 2
- name: Run C++ unit tests
shell: bash
# The unit-test suite (autotune cache) needs the CUDA toolchain
# to build but does not launch any CUDA kernels at runtime, so it
# runs on the build-only runners that have no NVIDIA driver.
run: |
set -e
cd $GITHUB_WORKSPACE/build
ctest --output-on-failure --build-config Release
build-and-test-cuda:
# Per-PR GPU correctness on an ephemeral A10G (sm_86) spun up by Cirun.
# The label + instance are defined in .cirun.yml at the repo root. The
# instance uses the AWS Deep Learning Base GPU AMI, which ships the NVIDIA
# driver + CUDA toolkit, so this job builds and runs on the same runner
# exactly like the old self-hosted box did -- only the runner is now
# ephemeral (clean state per run) and cloud-hosted.
#
# Gated on the `changes` job so a docs-only PR doesn't spin up a paid GPU
# instance. Skips cleanly (the job is marked skipped, not failed) when no
# build-relevant path changed.
needs: changes
if: needs.changes.outputs.code == 'true'
runs-on: cirun-gpu-a10
steps:
- uses: actions/checkout@v4
with:
submodules: 'true'
- name: Verify GPU + CUDA toolchain
shell: bash
# Fail fast (and legibly) if the runner image lacks a visible GPU or
# nvcc -- the two most common first-run problems on a fresh cloud
# runner. The Deep Learning AMI installs CUDA under /usr/local/cuda;
# put it on PATH for the build + test steps below.
run: |
set -e
echo "/usr/local/cuda/bin" >> $GITHUB_PATH
export PATH="/usr/local/cuda/bin:$PATH"
nvidia-smi
nvcc --version
- name: Build SCAMP with CUDA
shell: bash
run: |
set -e
cd $GITHUB_WORKSPACE
mkdir build && cd build
cmake -DFORCE_CUDA=1 -DBUILD_SCAMP_TESTS=ON ..
cmake --build . --config Release --parallel 4
- name: Basic CUDA Tests SCAMP
shell: bash
run: |
set -e
cd $GITHUB_WORKSPACE/test
./run_tests.sh ../build/SCAMP /dev/null "--num_cpu_workers=0"
- name: Extended Tests SCAMP
shell: bash
run: |
set -e
cd $GITHUB_WORKSPACE/test
python3 ./run_tests.py --executable "../build/SCAMP" --force_gpu --extra_args " --num_cpu_workers=0"
- name: Per-variant CUDA Tests SCAMP
# Exercise every compiled GPU kernel variant against every profile
# type. The extended-tests step above already runs the full
# parameter sweep (windows x tile sizes x input sizes) against the
# autotune-chosen variant; this step trades sweep breadth for
# variant breadth -- one window, two tile sizes, two input sizes
# per variant per profile. SCAMP_FORCE_VARIANT bypasses the
# autotune cache + cold-start default, so each variant gets
# exercised against its DefaultBlockszForPrecision pick (the
# precision-specific cold-start blocksz). The full
# {64,128,256,512} blocksz axis is NOT swept here -- that belongs
# in `SCAMP --autotune`, not the correctness CI.
shell: bash
run: |
set -e
cd $GITHUB_WORKSPACE/test
NUM_VARIANTS=$(../build/SCAMP --list_variants | wc -l)
echo "Detected $NUM_VARIANTS GPU kernel variants:"
../build/SCAMP --list_variants
for i in $(seq 0 $((NUM_VARIANTS - 1))); do
echo
echo "=== Variant v${i} ==="
SCAMP_FORCE_VARIANT=$i python3 ./run_tests.py \
--executable "../build/SCAMP" --force_gpu \
--window_sizes 100 \
--tile_sizes 1024 4096 \
--input_sizes 1500 8000 \
--matrix_sizes 10 \
--extra_args " --num_cpu_workers=0"
done