buildAndTestRyzenAI.yml

# Copyright (C) 2023-2026 Advanced Micro Devices, Inc.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

name: Build and Test with AIE tools on Ryzen AI

permissions:
  contents: read

on:
  # No `push` on main: see buildAndTestRyzenAIWindows.yml. The merge queue tests
  # the exact commit that merging pushes to main, and main is admin-enforced
  # with a required merge queue and no force pushes, so the postsubmit run only
  # ever re-tests an already-tested SHA. The nightly schedule still covers main.
  push:
    branches:
      - test-ryzen-ai
  pull_request:
    types: [opened, synchronize, reopened]
  merge_group:
  schedule:
    # Runs at midnight (00:00) every day
    - cron: '0 0 * * *'

defaults:
  run:
    shell: bash

concurrency:
  # A PR number if a pull request and otherwise the commit hash. This cancels
  # queued and in-progress runs for the same PR (presubmit) or commit
  # (postsubmit).
  group: ci-build-test-ryzenai-${{ github.event.number || github.sha }}-${{ github.event_name }}
  cancel-in-progress: true

env:
  DEBIAN_FRONTEND: noninteractive
  XILINXD_LICENSE_FILE: /opt/xilinx/Xilinx.lic
  VITIS: /opt/ryzen_ai-1.3.0.1/vitis_aie_essentials
  # ccache and lld are auto-detected and enabled by
  # utils/build-mlir-aie-from-wheels.sh (CMAKE_<LANG>_COMPILER_LAUNCHER /
  # LLVM_USE_LINKER), so only the options the script does not set itself are
  # listed here.
  CMAKE_ARGS: |
            -DXRT_ROOT=/opt/xilinx/xrt \
            -DAIE_ENABLE_PYTHON_PASSES=OFF \
            -DAIE_ENABLE_XRT_PYTHON_BINDINGS=ON \
  LIT_OPTS: -sv --time-tests --timeout 600 --show-unsupported --show-excluded

jobs:

  build-and-test-from-source:
    name: Run Tests and Examples on Ryzen AI (Built from Source) (${{ matrix.runner_type }})
    # The goal for this job is to run the full test suite, including programming guide and examples, on a freshly built-from-source MLIR-AIE.
    # This also exercises the build-mlir-aie-from-wheels.sh script, which is the most common developer-facing way to rebuild MLIR-AIE from source.
    runs-on: ${{ matrix.runner_type }}
    # Normal exec is 20-40 min; cap a hang well below GitHub's 360-min default.
    timeout-minutes: 90
    strategy:
      fail-fast: false
      matrix:
        include:
          - runner_type: aie2-4col
            expected_npu: npu1
          - runner_type: aie2p-8col
            expected_npu: npu2
    env:
      AIE_EXPECTED_NPU: ${{ matrix.expected_npu }}
      NPU_CACHE_HOME: ${{ github.workspace }}/iron-cache-${{ matrix.runner_type }}-${{ github.run_id }}
      PIP_CACHE_DIR: ${{ github.workspace }}/.pip-cache-${{ matrix.runner_type }}-${{ github.run_id }}
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
        with:
          submodules: "true"

      - name: Print hostname
        run: hostname

      - name: ccache stats before the build
        run: |
          # These runners are long-lived and shared, so the ccache directory
          # survives between runs while actions/checkout wipes the workspace.
          # Deliberately read-only: no `ccache -z` and no `ccache -M`, because
          # both write global state on a machine other jobs also build on.
          # Print before and after instead; the delta is the per-job figure.
          command -v ccache >/dev/null && ccache -s || echo "ccache not installed"

      - name: Setup virtual environment / prerequisites
        run: |
          source ./utils/env_install.sh aie-venv --dev --extras

          sed -i.bak 's/OUTPUT_TIMEOUT = 10/OUTPUT_TIMEOUT = 100/g' \
            $(python -c 'import site; print(site.getsitepackages()[0])')/jupyter_client/runapp.py

      - name: Check OpenCV availability
        run: |
          if ! pkg-config --exists opencv4; then
            echo "::error::OpenCV not found on runner. Vision tests require OpenCV."
            echo "Install libopencv-dev or ensure pkg-config can find opencv4."
            exit 1
          fi
          echo "OpenCV found: $(pkg-config --modversion opencv4)"

      - name: Run commands
        run: |
          # Some on-device failures are transient rather than real regressions,
          # and dropping an otherwise-green PR out of the merge queue for one is
          # pure noise. Retry once when the failure matches a known transient
          # signature:
          #   1. aie2-4col NPU hardware fault (DRM_IOCTL_AMDXDNA_EXEC_CMD I/O error).
          #   1b. Hardware-context creation failure (DRM_IOCTL_AMDXDNA_CREATE_HWCTX
          #      err=-2). The driver intermittently cannot hand out a context; the
          #      same test passes on a re-run. Seen on both NPU generations, so
          #      unlike the EIO case it is not restricted to aie2-4col.
          #   2. A host/device read race in an on-device test: an "X != X" result
          #      mismatch where the observed and reference values are equal. The
          #      host read an output buffer before the NPU DMA had fully landed;
          #      by the time the value is printed it has settled. A real logic
          #      error produces UNEQUAL values and is therefore never retried.
          # A deterministic regression fails both runs, so the retry only masks
          # nondeterministic failures.
          #
          # The retry reuses the successful build and skips the suites that
          # already fully passed. For a read-race (lit completed the suite) it
          # re-runs only that suite's failed tests -- across ALL lit failure
          # labels (FAIL/TIMEOUT/UNRESOLVED/XPASS), so a real non-FAIL failure is
          # never dropped. For an EIO (which can abort lit mid-suite, leaving
          # tests un-run) it re-runs the failed suite in FULL. Suites the
          # fail-fast pass never reached always run in full. So every test that
          # ran in the baseline still runs, and any deterministic failure fails
          # again. That keeps a single transient from turning a ~35 min job into
          # the observed ~57-75 min tail. If the failed suite or test names can't
          # be recovered it falls back to a full re-run -- never skips silently.
          if [ -z "$STEP_RETRIED" ]; then
            export STEP_RETRIED=1
            log=$(mktemp)
            bash -eo pipefail "$0" 2>&1 | tee "$log" && exit 0
            # Classify the transient. An aie2-4col amdxdna EIO can abort lit
            # mid-suite (leaving tests un-run), so its retry re-runs the failed
            # suite in FULL; the host/device read race is a value mismatch lit
            # still completes, so its retry may narrow to the failed tests.
            transient=""
            if [ "${{ matrix.runner_type }}" = "aie2-4col" ] && \
               grep -qF "DRM_IOCTL_AMDXDNA_EXEC_CMD IOCTL failed (err=-5): Input/output error" "$log"; then
              transient=eio
            elif grep -qF "DRM_IOCTL_AMDXDNA_CREATE_HWCTX IOCTL failed (err=-2): No such file or directory" "$log"; then
              # Same shape as the EIO case -- the abort can leave tests un-run --
              # so it takes the full-suite retry path rather than narrowing.
              transient=hwctx
            elif grep -Pq 'idx [0-9]+: ([0-9]+) != \1\b' "$log"; then
              transient=readrace
            fi
            if [ -n "$transient" ]; then
              echo "::warning::Transient on-device failure ($transient); retrying the step once"
              # Failed suite = last "===== suite: X =====" marker before the abort.
              # || true: a no-match grep exits 1 under `set -eo pipefail`; empty
              # just forces the full re-run below.
              FAILED_SUITE=$(grep -oP '===== suite: \K\S+' "$log" | tail -1 || true)
              # Collect EVERY lit failure label (FAIL/TIMEOUT/UNRESOLVED/XPASS),
              # not just FAIL, so a real non-FAIL failure is never dropped from
              # the retry set; regex-escape the paths for LIT_FILTER.
              RETRY_LIT_FILTER=$(grep -Eo '(FAIL|TIMEOUT|UNRESOLVED|XPASS): [^ ]+ :: [^ ]+' "$log" \
                | awk '{print $NF}' | sort -u \
                | sed 's/[][(){}.^$*+?|\\]/\\&/g' | paste -sd '|' - || true)
              export FAILED_SUITE RETRY_LIT_FILTER STEP_RETRY_TRANSIENT="$transient"
              export STEP_RETRY_PASS=1
              exec bash -eo pipefail "$0"
            fi
            exit 1
          fi
          sudo prlimit -lunlimited --pid $$
          # Newer XRT/amdxdna packages no longer ship setup.sh; source it
          # only when present so the job works on both old and new XRT.
          [ -f /opt/xilinx/xrt/setup.sh ] && source /opt/xilinx/xrt/setup.sh
          source aie-venv/bin/activate

          # -j here to reduce the number of parallel chess jobs.
          # Pick parallelism from the runner's actual RAM rather than its
          # label, so backup runners that share a label but have less memory
          # don't OOM: -j4 for <48GB RAM, -j12 otherwise.
          TOTAL_MEM_GB=$(free -g | awk '/^Mem:/ {print $2}')
          if [ "$TOTAL_MEM_GB" -lt 48 ]; then
            LIT_OPTS="-j4 $LIT_OPTS"
          else
            LIT_OPTS="-j12 $LIT_OPTS"
          fi

          export PATH=$VITIS/bin:$VITIS/aietools/bin:$PATH

          # Build from source, unless this is a narrowed retry: the build
          # already succeeded before the transient device fault, so reuse it.
          if [ -z "$STEP_RETRY_PASS" ]; then
            OPENCV_CMAKE="-DOpenCV_DIR=$(pkg-config --variable=prefix opencv4)/lib/cmake/opencv4"
            export EXTRA_CMAKE_ARGS="-DPython3_EXECUTABLE=$(which python) $CMAKE_ARGS $OPENCV_CMAKE"
            ./utils/build-mlir-aie-from-wheels.sh "" build mlir_aie
          fi

          pushd build

          # Create runner-specific cache directory
          rm -rf $NPU_CACHE_HOME
          mkdir $NPU_CACHE_HOME

          # Set number of contexts to maintain in cache per process
          export XRT_CONTEXT_CACHE_SIZE=2

          # gemm_asymmetric_tile_buffering chess builds peak ~8 GB RSS and
          # ~9 min wall alone on 32 GB; they run serialized via lit's atb_chess
          # group and need a longer timeout, so they get their own invocation.
          GEMM=gemm_asymmetric_tile_buffering
          ATB_LIT_OPTS="-j4 -sv --time-tests --timeout 1200 --show-unsupported --show-excluded"

          # run_suite <name> full|only-failed
          #   full        -- the suite's normal invocation
          #   only-failed -- restrict to RETRY_LIT_FILTER (retry of the failed suite)
          run_suite() {
            case "$1" in
              check-aie)   if [ "$2" = only-failed ]; then LIT_FILTER="$RETRY_LIT_FILTER" ninja check-aie
                           else ninja check-aie; fi ;;
              concurrency) if [ "$2" = only-failed ]; then LIT_FILTER="$RETRY_LIT_FILTER" ninja check-aie-concurrency
                           else ninja check-aie-concurrency; fi ;;
              refs)        if [ "$2" = only-failed ]; then LIT_FILTER="$RETRY_LIT_FILTER" ninja check-reference-designs
                           else LIT_FILTER_OUT="$GEMM" ninja check-reference-designs; fi ;;
              guide)       if [ "$2" = only-failed ]; then LIT_FILTER="$RETRY_LIT_FILTER" ninja check-programming-guide
                           else ninja check-programming-guide; fi ;;
              atb)         if [ "$2" = only-failed ]; then LIT_OPTS="$ATB_LIT_OPTS" LIT_FILTER="$RETRY_LIT_FILTER" ninja check-reference-designs
                           else LIT_OPTS="$ATB_LIT_OPTS" LIT_FILTER="$GEMM" ninja check-reference-designs; fi ;;
            esac
          }

          # gemm_asymmetric_tile_buffering reproduces a paper rather than gating
          # a regression, and is ~6 min of an 18-24 min job that does not shrink
          # on a bigger host. Nightly, plus test-ryzen-ai for on-demand runs.
          SUITES="check-aie concurrency refs guide"
          if [ "$GITHUB_EVENT_NAME" = schedule ] || [ "$GITHUB_REF_NAME" = test-ryzen-ai ]; then
            SUITES="$SUITES atb"
          fi

          # Only treat FAILED_SUITE as valid if it EXACTLY names a known suite
          # (fixed-string match, not a glob) -- else force the full re-run path.
          if ! printf '%s\n' $SUITES | grep -qxF "$FAILED_SUITE"; then
            FAILED_SUITE=""
          fi

          if [ -n "$STEP_RETRY_PASS" ] && [ -n "$FAILED_SUITE" ]; then
            # Narrowed retry: skip the suites that already passed, re-run only the
            # failed tests in the suite that failed, then run in full the suites
            # the first pass's fail-fast never reached.
            reached=""
            for s in $SUITES; do
              if [ -z "$reached" ] && [ "$s" != "$FAILED_SUITE" ]; then
                echo "===== skip already-passed suite: $s ====="
                continue
              fi
              reached=1
              if [ "$s" = "$FAILED_SUITE" ] && [ "$STEP_RETRY_TRANSIENT" = readrace ] \
                 && [ -n "$RETRY_LIT_FILTER" ]; then
                # read-race only: lit completed the suite, so narrowing to the
                # failed tests is coverage-safe. EIO (may have aborted lit) and
                # the empty-filter fallback both drop to the full-suite branch.
                echo "===== suite: $s (retry only failed tests) ====="
                run_suite "$s" only-failed
              else
                echo "===== suite: $s ====="
                run_suite "$s" full
              fi
            done
          else
            # First pass (and the full-re-run fallback): run every suite in order.
            # The marker is printed before each suite so a fail-fast abort leaves
            # the failed suite as the last "===== suite: X =====" line in the log.
            for s in $SUITES; do
              echo "===== suite: $s ====="
              run_suite "$s" full
            done
          fi

          popd
      - name: ccache stats after the build
        if: always()
        run: |
          command -v ccache >/dev/null && ccache -s || echo "ccache not installed"

      - name: Cleanup NPU_CACHE_HOME
        if: always()
        run: |
          rm -rf $NPU_CACHE_HOME
          rm -rf $PIP_CACHE_DIR

  build-and-test-pure-hrx:
    name: Pure HRX Build and Run (XRT-free packaging + libhrx dispatch)
    # This job exercises the XRT-free flow end to end on XDNA2 hardware:
    #   * builds MLIR-AIE from source with the bundled hrx-xclbinutil
    #     (-DAIE_BUILD_HRXXCLBINUTIL=ON) instead of relying on XRT's xclbinutil,
    #   * packages a real NPU .xclbin with that bundled tool
    #     (the aiecc NPU_RUNTIME=hrx path via AIE_XCLBINUTIL), and
    #   * dispatches it on the NPU through libhrx from a pinned HRX release
    #     (no XRT runtime SDK), for both the Python (IRON) and C++ paths -
    #     covering the multi-dispatch chain/runlist path (vector_scalar_add) and
    #     the normal single-dispatch path (vector_vector_add IRON +
    #     vector_reduce_max C++).
    runs-on: aie2p-8col
    timeout-minutes: 30
    env:
      NPU_CACHE_HOME: ${{ github.workspace }}/iron-cache-hrx-${{ github.run_id }}
      PIP_CACHE_DIR: ${{ github.workspace }}/.pip-cache-hrx-${{ github.run_id }}
      # Token for the (private) HRX release repo. fetch-hrx-release.sh passes it
      # to gh / curl; the runner's own gh auth is used when this is empty.
      GH_TOKEN: ${{ secrets.HRX_RELEASE_TOKEN }}
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
        with:
          submodules: "true"

      - name: Print hostname
        run: hostname

      - name: Setup virtual environment / prerequisites
        run: |
          source ./utils/env_install.sh aie-venv --dev --extras

      - name: ccache stats before the build
        run: |
          # These runners are long-lived and shared, so the ccache directory
          # survives between runs while actions/checkout wipes the workspace.
          # Deliberately read-only: no `ccache -z` and no `ccache -M`, because
          # both write global state on a machine other jobs also build on.
          # Print before and after instead; the delta is the per-job figure.
          command -v ccache >/dev/null && ccache -s || echo "ccache not installed"

      - name: Fetch pinned HRX release
        run: |
          # Downloads + checksum-verifies + extracts the pinned libhrx release
          # and writes its env.sh path to $GITHUB_ENV for later steps.
          env_sh=$(bash utils/fetch-hrx-release.sh)
          echo "HRX_RELEASE_ENV=$env_sh" >> "$GITHUB_ENV"

      - name: Build from source (bundled hrx-xclbinutil, no XRT bindings)
        run: |
          sudo prlimit -lunlimited --pid $$
          source aie-venv/bin/activate
          # Pure-HRX build: enable the bundled XRT-free packaging tool and skip
          # the XRT python bindings. find_package(XRT) stays optional, so no XRT
          # SDK is required to configure or build.
          export EXTRA_CMAKE_ARGS="-DPython3_EXECUTABLE=$(which python) \
            -DAIE_ENABLE_PYTHON_PASSES=OFF \
            -DAIE_BUILD_HRXXCLBINUTIL=ON"
          # Installs the built aie tree (bin/python/lib) into ./install so the
          # make-driven examples below can `import aie` and find aiecc.
          ./utils/build-mlir-aie-from-wheels.sh "" build install
          # Sanity: the bundled tool must be installed as `xclbinutil` next to
          # aiecc (that is what the NPU_RUNTIME=hrx packaging path resolves to).
          test -x install/bin/xclbinutil

      - name: Run pure-HRX tests and example on NPU
        run: |
          # Some on-device failures are transient rather than real regressions,
          # and dropping an otherwise-green PR out of the merge queue for one is
          # pure noise. This mirrors the build-and-test-from-source retry (see
          # that job for the full rationale). Retry once when the failure matches
          # a known transient signature:
          #   1. an amdxdna NPU hardware fault (DRM_IOCTL_AMDXDNA_EXEC_CMD I/O error).
          #   2. A host/device read race in an on-device test: an "X != X" result
          #      mismatch where the observed and reference values are equal. The
          #      host read an output buffer before the NPU DMA had fully landed;
          #      by the time the value is printed it has settled. A real logic
          #      error produces UNEQUAL values and is therefore never retried.
          # A deterministic regression fails both runs, so the retry only masks
          # nondeterministic failures. The build is a separate step, so the retry
          # reuses it and skips the suites that already fully passed; a read-race
          # re-runs only the failed suite's failed tests and an EIO re-runs the
          # failed suite in FULL, with any un-reached suite always run in full.
          # If the failed suite or test names can't be recovered it falls back to
          # a full re-run -- never skips silently.
          if [ -z "$STEP_RETRIED" ]; then
            export STEP_RETRIED=1
            log=$(mktemp)
            bash -eo pipefail "$0" 2>&1 | tee "$log" && exit 0
            # Classify the transient. An amdxdna EIO can abort lit mid-suite
            # (leaving tests un-run), so its retry re-runs the failed suite in
            # FULL; the host/device read race is a value mismatch lit still
            # completes, so its retry may narrow to the failed tests.
            transient=""
            if grep -qF "DRM_IOCTL_AMDXDNA_EXEC_CMD IOCTL failed (err=-5): Input/output error" "$log"; then
              transient=eio
            elif grep -Pq 'idx [0-9]+: ([0-9]+) != \1\b' "$log"; then
              transient=readrace
            fi
            if [ -n "$transient" ]; then
              echo "::warning::Transient on-device failure ($transient); retrying the pure-HRX step once"
              # Failed suite = last "===== suite: X =====" marker before the abort.
              # || true: a no-match grep exits 1 under `set -eo pipefail`; empty
              # just forces the full re-run below.
              FAILED_SUITE=$(grep -oP '===== suite: \K\S+' "$log" | tail -1 || true)
              # Collect EVERY lit failure label (FAIL/TIMEOUT/UNRESOLVED/XPASS),
              # not just FAIL, so a real non-FAIL failure is never dropped from
              # the retry set; regex-escape the paths for LIT_FILTER.
              RETRY_LIT_FILTER=$(grep -Eo '(FAIL|TIMEOUT|UNRESOLVED|XPASS): [^ ]+ :: [^ ]+' "$log" \
                | awk '{print $NF}' | sort -u \
                | sed 's/[][(){}.^$*+?|\\]/\\&/g' | paste -sd '|' - || true)
              export FAILED_SUITE RETRY_LIT_FILTER STEP_RETRY_TRANSIENT="$transient"
              export STEP_RETRY_PASS=1
              exec bash -eo pipefail "$0"
            fi
            exit 1
          fi
          sudo prlimit -lunlimited --pid $$
          source aie-venv/bin/activate
          # libhrx + HRX_DIR/HRX_BUILD for both the C++ testbench (FindHRX.cmake)
          # and the Python runtime (hrxruntime/discovery.py).
          source "$HRX_RELEASE_ENV"
          # Expose the installed aie tree (mirrors utils/env_setup.sh without
          # depending on xrt-smi) so the make-driven examples can `import aie`
          # and resolve aiecc + the bundled xclbinutil. libhrx stays on
          # LD_LIBRARY_PATH from the sourced release env.sh above.
          export PATH="$PWD/install/bin:$PATH"
          export PYTHONPATH="$PWD/install/python:$PYTHONPATH"
          export LD_LIBRARY_PATH="$PWD/install/lib:$LD_LIBRARY_PATH"
          # aiecc uses the bundled XRT-free xclbinutil; NPU2=1 selects npu2 so the
          # .xclbin matches this runner. Set before the loop so the retry sees them.
          export AIE_XCLBINUTIL="$PWD/install/bin/xclbinutil"
          export NPU2=1

          rm -rf "$NPU_CACHE_HOME" && mkdir -p "$NPU_CACHE_HOME"

          # run_suite <name> full|only-failed
          #   full        -- the suite's normal invocation
          #   only-failed -- restrict a lit suite to RETRY_LIT_FILTER (retry of the
          #                  failed suite); example suites can't narrow, run full
          run_suite() {
            case "$1" in
              xclbin_sections)
                pushd build
                if [ "$2" = only-failed ]; then LIT_FILTER="$RETRY_LIT_FILTER" ninja check-aie
                else LIT_FILTER='hrx_xclbin_sections' ninja check-aie; fi
                popd ;;
              hrx_python)
                pushd build
                if [ "$2" = only-failed ]; then AIE_HRX_NPU=npu2 LIT_FILTER="$RETRY_LIT_FILTER" ninja check-aie
                else AIE_HRX_NPU=npu2 LIT_FILTER='python/npu/|npu-hrx|test_hrx_runtime_selection' ninja check-aie; fi
                popd ;;
              vector_scalar_add)
                pushd programming_examples/basic/vector_scalar_add
                NPU_RUNTIME=hrx make clean
                NPU_RUNTIME=hrx make all
                NPU_RUNTIME=hrx make run_runlist_hrx
                popd ;;
              vector_vector_add)
                pushd programming_examples/basic/vector_vector_add
                NPU_RUNTIME=hrx python3 vector_vector_add.py
                popd ;;
              vector_reduce_max)
                pushd programming_examples/basic/vector_reduce_max/single_core_designs
                NPU_RUNTIME=hrx make clean
                NPU_RUNTIME=hrx make all
                NPU_RUNTIME=hrx make run
                popd ;;
            esac
          }

          SUITES="xclbin_sections hrx_python vector_scalar_add vector_vector_add vector_reduce_max"

          # Only treat FAILED_SUITE as valid if it EXACTLY names a known suite
          # (fixed-string match, not a glob) -- else force the full re-run path.
          if ! printf '%s\n' $SUITES | grep -qxF "$FAILED_SUITE"; then
            FAILED_SUITE=""
          fi

          if [ -n "$STEP_RETRY_PASS" ] && [ -n "$FAILED_SUITE" ]; then
            # Narrowed retry: skip the suites that already passed, re-run only the
            # failed tests in the suite that failed, then run in full the suites
            # the first pass's fail-fast never reached.
            reached=""
            for s in $SUITES; do
              if [ -z "$reached" ] && [ "$s" != "$FAILED_SUITE" ]; then
                echo "===== skip already-passed suite: $s ====="
                continue
              fi
              reached=1
              if [ "$s" = "$FAILED_SUITE" ] && [ "$STEP_RETRY_TRANSIENT" = readrace ] \
                 && [ -n "$RETRY_LIT_FILTER" ]; then
                # read-race only: lit completed the suite, so narrowing to the
                # failed tests is coverage-safe. EIO (may have aborted lit) and
                # the empty-filter fallback both drop to the full-suite branch.
                echo "===== suite: $s (retry only failed tests) ====="
                run_suite "$s" only-failed
              else
                echo "===== suite: $s ====="
                run_suite "$s" full
              fi
            done
          else
            # First pass (and the full-re-run fallback): run every suite in order.
            # The marker is printed before each suite so a fail-fast abort leaves
            # the failed suite as the last "===== suite: X =====" line in the log.
            for s in $SUITES; do
              echo "===== suite: $s ====="
              run_suite "$s" full
            done
          fi

      - name: ccache stats after the build
        if: always()
        run: |
          command -v ccache >/dev/null && ccache -s || echo "ccache not installed"

      - name: Cleanup NPU_CACHE_HOME
        if: always()
        run: |
          rm -rf $NPU_CACHE_HOME
          rm -rf $PIP_CACHE_DIR

  build-pure-hrx-windows:
    name: Pure HRX Build-Only (Windows, XRT-free packaging + HRX C++ compile)
    # Windows counterpart of build-and-test-pure-hrx, but BUILD-ONLY (no NPU):
    # GitHub-hosted windows-2022 has no XDNA2 device, so this proves the HRX path
    # *compiles* on Windows rather than dispatching:
    #   * the bundled XRT-free hrx-xclbinutil builds + passes its smoke test,
    #   * the pinned HRX Windows release headers (hrx_runtime.h + hrx_amdxdna.h)
    #     compile under MSVC (libhrx now owns the amdxdna XADX packaging, so there
    #     is no separate helper to compile-check), and
    #   * the HRX Python modules import/parse under the Windows interpreter.
    # On-device dispatch stays on the Linux aie2p-8col job above.
    runs-on: windows-2022
    defaults:
      run:
        shell: pwsh
    env:
      # Token for the (private) HRX release repo; empty falls back to anonymous.
      GH_TOKEN: ${{ secrets.HRX_RELEASE_TOKEN }}
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
        with:
          submodules: "true"

      - name: Set up MSVC dev environment
        uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756  # v1
        with:
          arch: x64

      - name: Set up Python
        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1  # v6
        with:
          python-version: "3.12"

      - name: Build and test the bundled hrx-xclbinutil (XRT-free packaging)
        run: |
          # The XRT-free / Boost-free packaging tool aiecc uses on the NPU_RUNTIME=hrx
          # path. Building it here proves the pure-HRX packaging tool compiles on
          # Windows in this tree (the dedicated buildAndTestHrxXclbinutil.yml workflow
          # covers the standalone build/test matrix in more detail).
          #
          # Build-only on a GitHub-hosted runner with no NPU, so the on-device
          # retry signatures don't apply; retry once on any failure to ride out a
          # network/toolchain hiccup during configure or a flaky ctest. A
          # deterministic break fails both runs; wipe the build dir first so a
          # half-written CMake cache can't make the retry a deterministic failure.
          $build = {
            cmake -B build-hrxxclbinutil -S third_party/hrx-xclbinutil -A x64
            if ($LASTEXITCODE -ne 0) { throw "cmake configure failed ($LASTEXITCODE)" }
            cmake --build build-hrxxclbinutil --config Release --parallel
            if ($LASTEXITCODE -ne 0) { throw "cmake build failed ($LASTEXITCODE)" }
            ctest --test-dir build-hrxxclbinutil -C Release --output-on-failure
            if ($LASTEXITCODE -ne 0) { throw "ctest failed ($LASTEXITCODE)" }
          }
          try {
            & $build
          } catch {
            Write-Host "::warning::Transient Windows build/test failure ($_); retrying once"
            Remove-Item -Recurse -Force build-hrxxclbinutil -ErrorAction SilentlyContinue
            & $build
          }

      - name: Fetch pinned HRX Windows release
        run: |
          # Parse the pinned coordinates (Windows asset) from the shared env file.
          $envmap = @{}
          Get-Content utils/hrx-release.env | ForEach-Object {
            if ($_ -match '^\s*([A-Za-z0-9_]+)\s*=\s*"?([^"]*)"?\s*$') {
              $envmap[$matches[1]] = $matches[2]
            }
          }
          $repo  = $envmap['HRX_RELEASE_REPO']
          $tag   = $envmap['HRX_RELEASE_TAG']
          $asset = $envmap['HRX_RELEASE_ASSET_WINDOWS']
          $sha   = $envmap['HRX_RELEASE_SHA256_WINDOWS'].ToLower()
          if (-not $asset) { throw "HRX_RELEASE_ASSET_WINDOWS not set in hrx-release.env" }
          $url = "https://github.com/$repo/releases/download/$tag/$asset"
          Write-Host "Downloading $url"

          $headers = @{}
          if ($env:GH_TOKEN) { $headers['Authorization'] = "Bearer $($env:GH_TOKEN)" }
          for ($attempt = 1; $attempt -le 3; $attempt++) {
            try { Invoke-WebRequest -Uri $url -Headers $headers -OutFile $asset; break }
            catch { if ($attempt -eq 3) { throw }; Start-Sleep -Seconds ($attempt * 5) }
          }

          $actual = (Get-FileHash $asset -Algorithm SHA256).Hash.ToLower()
          if ($actual -ne $sha) {
            throw "checksum mismatch for ${asset}: got $actual, expected $sha"
          }
          Write-Host "Checksum OK ($sha)"

          Expand-Archive -Path $asset -DestinationPath hrx-win -Force
          $hdr = Get-ChildItem -Recurse hrx-win -Filter hrx_runtime.h | Select-Object -First 1
          if (-not $hdr) { throw "hrx_runtime.h not found in the extracted HRX Windows release" }
          "HRX_WIN_INCLUDE=$($hdr.Directory.FullName)" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
          "HRX_WIN_ROOT=$((Resolve-Path hrx-win).Path)" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8

      - name: Compile-check the HRX C ABI headers under MSVC
        shell: cmd
        run: |
          echo #include "hrx_runtime.h"> hrx_probe.c
          echo #include "hrx_amdxdna.h">> hrx_probe.c
          echo int main(void){return 0;}>> hrx_probe.c
          cl /nologo /c /I "%HRX_WIN_INCLUDE%" hrx_probe.c

      - name: Python HRX modules parse under Windows Python
        shell: bash
        run: |
          # Build-only Python proof: the HRX runtime sources parse under the
          # Windows interpreter. (Full import/selection needs the built aie wheel
          # and libhrx discovery, which is exercised on the Linux job.)
          python -m py_compile \
            python/utils/__init__.py \
            python/utils/hostruntime/hrxruntime/__init__.py \
            python/utils/hostruntime/hrxruntime/_bindings.py \
            python/utils/hostruntime/hrxruntime/context.py \
            python/utils/hostruntime/hrxruntime/hostruntime.py \
            python/utils/hostruntime/hrxruntime/discovery.py \
            python/utils/hostruntime/hrxruntime/tensor.py
          echo "HRX Python sources compiled OK on Windows"