test.yml

name: Run Tests

on:
  - push
  - pull_request
  - workflow_call
  - workflow_dispatch

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  test:
    runs-on: ubuntu-latest
    timeout-minutes: 30
    permissions:
      contents: read
      pull-requests: write
    steps:
      - uses: actions/checkout@v6
        with:
          submodules: recursive
      - uses: awalsh128/cache-apt-pkgs-action@2c09a5e66da6c8016428a2172bd76e5e4f14bb17 # v1.5.3
        with:
          packages: sox libsox-dev
      - name: Verify SoX installation
        run: sox --version
      - uses: FedericoCarboni/setup-ffmpeg@583042d32dd1cabb8bd09df03bde06080da5c87c # v2
      - uses: actions/setup-python@v6
        with:
          python-version: "3.10"
          cache: "pip"
      - name: Install dependencies and EveryVoice itself
        run: |
          CUDA_TAG=cpu pip install -r requirements.torch.txt --find-links https://download.pytorch.org/whl/torch_stable.html
          pip install -e .[dev]
      - run: pip freeze
      - run: pip list
      - name: Run tests
        id: tests
        continue-on-error: true
        run: |
          cd everyvoice && coverage run run_tests.py dev
          coverage xml
      - name: Verbose test outputs in case of failure
        if: steps.tests.outcome == 'failure'
        run: |
          cd everyvoice && EVERYVOICE_VERBOSE_TESTS=1 python run_tests.py --verbose dev
          false
      - run: cd everyvoice && coverage report
      - name: Check for logs_and_checkpoints/ and preprocessed/
        id: no-extra-directories
        run: |
          cd everyvoice
          find -type d -name logs_and_checkpoints -or -name preprocessed | grep -v 'tests/data' || true
          [[ $(find -type d -name logs_and_checkpoints -or -name preprocessed | grep --count --invert-match 'tests/data') -eq 0 ]]
      - uses: codecov/codecov-action@v5
        with:
          fail_ci_if_error: false # optional (default = false)
          token: ${{ secrets.CODECOV_TOKEN }}
      - name: Make sure the CLI stays fast
        id: cli-load-time
        run: |
          PYTHONPROFILEIMPORTTIME=1 everyvoice -h 2> importtime.txt > /dev/null
          CLI_LOAD_TIME="$((/usr/bin/time --format=%E everyvoice -h > /dev/null) 2>&1)"
          echo "CLI load time: $CLI_LOAD_TIME" > import-message.txt
          PR_HEAD="${{ github.event.pull_request.head.sha }}"
          [[ $PR_HEAD ]] && echo "Pull Request HEAD: $PR_HEAD" >> import-message.txt
          echo "Imports that take more than 0.1 s:" >> import-message.txt
          grep -E 'cumulative|[0-9]{6} ' importtime.txt >> import-message.txt
          cat import-message.txt
          echo "Full import time log:"
          cat importtime.txt
          if [[ "$CLI_LOAD_TIME" > "0:01.00" ]]; then \
            echo "ERROR: everyvoice --help is too slow."; \
            echo "Please run 'PYTHONPROFILEIMPORTTIME=1 everyvoice -h 2> importtime.txt; tuna importtime.txt' and tuck away expensive imports so that the CLI doesn't load them until it uses them."; \
            false; \
          fi
          if grep -E -q "shared_types|pydantic" importtime.txt; then \
            echo "ERROR: please be careful not to cause shared_types or pydantic to be imported when the CLI just loads. They are expensive imports."; \
            false; \
          fi
      - name: Report help speed in PR
        if: github.event_name == 'pull_request'
        uses: mshick/add-pr-comment@7c1a3a3e5a072270dc21c0639df39ca11e860b2b # v3.5.0
        with:
          preformatted: true
          message-path: import-message.txt

  test-on-windows:
    runs-on: windows-latest
    timeout-minutes: 30
    permissions:
      contents: read
    defaults:
      run:
        shell: bash
    steps:
      - uses: actions/checkout@v6
        with:
          submodules: recursive
      - uses: FedericoCarboni/setup-ffmpeg@583042d32dd1cabb8bd09df03bde06080da5c87c # v2
      - uses: actions/setup-python@v6
        with:
          python-version: "3.10"
          cache: "pip"
      - name: Install SoX from binaries on codeberg.org
        run: |
          mkdir sox
          cd sox
          curl -O https://codeberg.org/sox_ng/sox_ng/releases/download/sox_ng-14.7.1.1/sox_ng-14.7.1.1-win64-exe.zip
          unzip sox_ng-14.7.1.1-win64-exe.zip
          mkdir bin
          cp sox_ng.exe bin/sox.exe
          echo "${{ github.workspace }}/sox/bin" >> $GITHUB_PATH
      - name: Verify SoX installation
        run: sox --version
      - name: Install dependencies and EveryVoice itself
        run: |
          CUDA_TAG=cpu pip install -r requirements.torch.txt --find-links https://download.pytorch.org/whl/torch_stable.html
          pip install -e .[dev]
      - run: pip freeze
      - run: pip list
      - name: Run tests
        id: tests
        run: everyvoice test

  # Run pre-commit and update a PR automatically if possible
  pre-commit:
    runs-on: ubuntu-latest
    timeout-minutes: 15
    permissions:
      contents: write  # for pre-commit-ci/lite-action
    steps:
      - uses: actions/checkout@v6
        with:
          submodules: recursive
      - run: pip install pre-commit
      - uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v47
        id: file_changes
        with:
          # Only run pre-commit on EV files themselves, not on submodules
          # See https://github.com/EveryVoiceTTS/EveryVoice/issues/555
          exclude_submodules: true
      - uses: actions/cache@v5  # enable caching for pre-commit
        with:
          path: ~/.cache/pre-commit
          key: pre-commit-3|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }}
      - name: Custom replacement for pre-commit/action
        # pre-commit/action is not compatible with conda-incubator/setup-miniconda because it sets the shell wrong.
        run: python -m pre_commit run --show-diff-on-failure --color=always --files ${{ steps.file_changes.outputs.all_changed_files }}
      - uses: pre-commit-ci/lite-action@5d6cc0eb514c891a40562a58a8e71576c5c7fb43 # v1.1.0
        if: always()

  # Automated due diligence: Make sure we don't introduce dependencies with incompatible licenses.
  # But: Ignore packages where auto analysis does not work but manual analysis says OK.
  # And: Ignore nvidia-*-cu12 packages used by the GPU back-end: OTHER/PROPRIETARY LICENSE
  # And: llvmlite crashes licensecheck at the moment, but its license is OK so skip it.
  licensecheck:
    runs-on: ubuntu-latest
    timeout-minutes: 15
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@v6
        with:
          submodules: recursive
      - run: pip install licensecheck --no-warn-conflicts
      - name: Run license check overall
        run: |
          licensecheck --requirements-paths pyproject.toml --zero \
            --ignore-packages text-unidecode pympi-ling pyworld pyworld-prebuilt pysdtw audioread anytree gradio hf-gradio \
            --skip-dependencies llvmlite \
            --ignore-licenses OTHER/PROPRIETARY || \
          ! echo "Package(s) listed with an X above is/are potentially a problem. Please review their licenses for compatibility with EveryVoice."
      - name: Make sure the PROPRIETARY packages are the known ones
        run: |
          ! licensecheck --requirements-paths pyproject.toml --format csv 2> /dev/null | grep -v -E 'nvidia-|aiohappyeyeballs' | grep PROPRIETARY || \
          ! echo "Package(s) listed above is/are potentially a problem. Please review their licenses for compatibility with EveryVoice."