test-mlc-core-actions.yaml

name: MLC core actions test
permissions:
  contents: read

on:
  pull_request:
    branches: [ "main", "dev" ]
    paths:
      - '.github/workflows/test-mlc-core-actions.yml'
      - '**'
      - '!**.md'

jobs:
  test_mlc_cli_core_actions:

    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.14", "3.8"]
        os: ["ubuntu-latest", "windows-latest", "macos-latest"]
        exclude:
          - os: windows-latest
          - os: macos-latest

    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v3
      with:
        python-version: ${{ matrix.python-version }}

    - name: Configure git longpaths (Windows)
      if: matrix.os == 'windows-latest'
      run: |
        git config --system core.longpaths true

    - name: Install mlcflow from the pull request's source repository and branch
      run: |
        python -m pip install --upgrade pip
        python -m pip install --ignore-installed --verbose pip setuptools
        python -m pip install .

    - name: Test 1 - pull repo - Pull a forked MLOps repository
      run: |
        GH_MLC_REPO_PATH_FORK="${HOME}/MLC/repos/anandhu-eng@mlperf-automations"
        GH_MLC_REPO_JSON_PATH="${HOME}/MLC/repos/repos.json"
        mlc pull repo anandhu-eng@mlperf-automations  --checkout=dev
        if [ ! -d "${GH_MLC_REPO_PATH_FORK}" ]; then
          echo "Repository folder $GH_MLC_REPO_PATH_FORK not found. Exiting with failure."
          exit 1
        fi
        if [ ! -f "$GH_MLC_REPO_JSON_PATH" ]; then
          echo "File $GH_MLC_REPO_JSON_PATH does not exist. Exiting with failure."
          exit 1
        fi
        if ! grep -q "$GH_MLC_REPO_PATH_FORK" "$GH_MLC_REPO_JSON_PATH"; then
          echo "Path $GH_MLC_REPO_PATH_FORK not found in $GH_MLC_REPO_JSON_PATH. Exiting with failure."
          exit 1
        fi
        CURRENT_BRANCH=$(git -C "$GH_MLC_REPO_PATH_FORK" rev-parse --abbrev-ref HEAD)
        if [ "$CURRENT_BRANCH" != "dev" ]; then
          echo "Expected branch 'dev', but found '$CURRENT_BRANCH'. Exiting with failure."
          exit 1
        fi

    - name: Test 2 - find repo
      run: |
        mlc find repo anandhu-eng@mlperf-automations
        mlc find repo https://github.com/mlcommons/mlperf-automations.git
        mlc find repo 9cf241afa6074c89
        mlc find repo mlcommons@mlperf-automations
        mlc find repo mlcommons@mlperf-automations,9cf241afa6074c89
      
    - name: Test 3 - pull repo - Test conflicting repo scenario
      run: |
        GH_MLC_REPO_PATH="${HOME}/MLC/repos/mlcommons@mlperf-automations"
        GH_MLC_REPO_JSON_PATH="${HOME}/MLC/repos/repos.json"
        mlc pull repo mlcommons@mlperf-automations --checkout=dev
        if [ ! -d "$GH_MLC_REPO_PATH" ]; then
          echo "Repository folder $GH_MLC_REPO_PATH not found. Exiting with failure."
          exit 1
        fi
        if [ ! -f "$GH_MLC_REPO_JSON_PATH" ]; then
          echo "File $GH_MLC_REPO_JSON_PATH does not exist. Exiting with failure."
          exit 1
        fi
        if ! grep -q "$GH_MLC_REPO_PATH" "$GH_MLC_REPO_JSON_PATH"; then
          echo "Path $GH_MLC_REPO_PATH not found in $GH_MLC_REPO_JSON_PATH. Exiting with failure."
          exit 1
        fi
        if ! grep -q "$GH_MLC_REPO_PATH_FORK" "$GH_MLC_REPO_JSON_PATH"; then
          echo "Path $GH_MLC_REPO_PATH_FORK also found in $GH_MLC_REPO_JSON_PATH. This should have been replaced. Exiting with failure."
          exit 1
        fi
        CURRENT_BRANCH=$(git -C "$GH_MLC_REPO_PATH" rev-parse --abbrev-ref HEAD)
        if [ "$CURRENT_BRANCH" != "dev" ]; then
          echo "Expected branch 'dev', but found '$CURRENT_BRANCH'. Exiting with failure."
          exit 1
        fi
        mlc pull repo
    
    - name: Test 4 - list repo - List the existing repositories
      run: |
        mlc list repo

    - name: Test  5 - rm repo - Remove the forked mlperf-automation repo
      run: |
        GH_MLC_REPO_PATH_FORK="${HOME}/MLC/repos/anandhu-eng@mlperf-automations"
        mlc rm repo anandhu-eng@mlperf-automations
        if [ -d "$GH_MLC_REPO_PATH_FORK" ]; then
          echo "Repository folder $GH_MLC_REPO_PATH found. It should ideally be deleted. Exiting with failure."
          exit 1
        fi

    - name: Test 6 - find cache - Cache not present
      run: |
        mlc find cache --tags=detect,os 2>&1 | tee test5.log
        if ! grep -q "No cache entry found for the specified input:" test5.log; then
          exit 1
        fi

    - name: Test 7 - run script - Output being used for testing mlc cache
      run: |
        mlc run script --tags=get,imagenet-aux --quiet
        mlc run script bb2c6dd8c8c64217 --quiet
        mlc run script --tags=get,imagenet-aux,_from.dropbox --quiet

    - name: Test 8 - find cache - More than one cache present
      run: |
        mlc search cache --tags=get,imagenet-aux 2>&1 | tee test7.log
        if grep -q "No cache entry found for the specified tags:" test7.log; then
          exit 1
        fi
      
    - name: Test 9 - show cache - More than one cache present
      run: |
        mlc show cache --tags=get,imagenet-aux 2>&1 | tee test7.log

    - name: Test 10 - rm cache - More than one cache present
      run: |
        mlc rm cache --tags=get,imagenet-aux -f

    - name: Test 11 - cp script - Copy mlc script 
      run: |
        mlc cp script detect-os my-os-detect
        mlc cp script detect-os/ my-os-detect-1

    - name: Test 12 - add repo - Add a new MLC repo 
      run: |
        mlc add repo my-new-repo
        mlc add repo https://github.com/mlcommons/inference
        mlc add repo https://mygit.com/myrepo
     
   
    - name: Test 14 - add script - Add a new MLC script
      run: |
        mlc add script my-script-1 --tags=my,new-tags-1
        mlc add script my-script-2 --tags=my,new-tags-2
        mlc add script my-script-3 --tags=my,new-tags3 --template_tags=detect,os
        mlc add script mlcommons@mlperf-automations:my-script-4 --tags=my,new-tags4 --template_tags=detect,os
      
    - name: Test 15 - mv script - Move/rename an MLC script 
      run: |
        mlc mv script my-script-1 moved-my-script-1
        mlc mv script my-script-2 mlcommons@mlperf-automations:moved-my-script-2 
    
    - name: Test 16 - show script
      run: |
        mlc show script --tags=run-mlperf,inference
        mlc show script 863735b7db8c44fc
        mlc show script detect-os,863735b7db8c44fc
        mlc show script detect-os

    - name: Test 17 - find script
      run: |
        mlc find script --tags=run-mlperf,inference
        mlc find script 863735b7db8c44fc
        mlc find script detect-os,863735b7db8c44fc
        mlc find script detect-os

    - name: Test 18 - rm script
      run: |
        mlc rm script get-ipol-src -f
        mlc rm script --tags=app,image,corner-detection -f
        mlc rm script 63080407db4d4ac4 -f

    - name: Test 19 - list script
      run: |
        mlc list script
    
    - name: Test 20 - list cache
      run: |
        mlc list cache
    
    - name: Test 21 - Test mlc pull repo to checkout based on particular release tag
      run: |
        mlc rm repo mlcommons@mlperf-automations -f
        mlc pull repo mlcommons@mlperf-automations --tag=v1.2.0

    - name: Test 22 - Test silent mode
      run: |
        mlcr detect,cpu -j -s --quiet 
        ! mlcr detect,cpu -j -s --quiet 2>&1 | grep -q INFO
        ! mlcr detect,cpu -j --silent --quiet 2>&1 | grep -q INFO

    - name: Test 23 - Test verbose mode
      run: |
        mlcr detect,cpu -j -v 2>&1 | grep -q DEBUG
        mlcr detect,cpu -j --verbose 2>&1 | grep -q DEBUG
        
    - name: Test 24 - Test recursive mlc pull repo
      run: |
        export MLC_REPOS=$HOME/test
        mlcp https://github.com/GATEOverflow/GO-PDFs
        mlcr detect,os -j

    - name: Test 25 - Test automatic repo pull
      run: |
        mlc rm repo mlcommons@mlperf-automations -f
        mlcr detect,cpu -j

    - name: Test 26 - Test reindex command and verify index files are updated
      run: |
        INDEX_SCRIPT="${HOME}/MLC/repos/index_script.json"
        INDEX_CACHE="${HOME}/MLC/repos/index_cache.json"
        INDEX_EXPERIMENT="${HOME}/MLC/repos/index_experiment.json"
        
        # Store initial modification times
        if [ -f "$INDEX_SCRIPT" ]; then
          BEFORE_SCRIPT=$(stat -c %Y "$INDEX_SCRIPT" 2>/dev/null || stat -f %m "$INDEX_SCRIPT" 2>/dev/null)
        fi
        if [ -f "$INDEX_CACHE" ]; then
          BEFORE_CACHE=$(stat -c %Y "$INDEX_CACHE" 2>/dev/null || stat -f %m "$INDEX_CACHE" 2>/dev/null)
        fi
        if [ -f "$INDEX_EXPERIMENT" ]; then
          BEFORE_EXPERIMENT=$(stat -c %Y "$INDEX_EXPERIMENT" 2>/dev/null || stat -f %m "$INDEX_EXPERIMENT" 2>/dev/null)
        fi
        
        sleep 1
        
        # Test reindex all
        mlc reindex
        
        # Verify all index files were updated
        if [ -f "$INDEX_SCRIPT" ]; then
          AFTER_SCRIPT=$(stat -c %Y "$INDEX_SCRIPT" 2>/dev/null || stat -f %m "$INDEX_SCRIPT" 2>/dev/null)
          if [ "$BEFORE_SCRIPT" = "$AFTER_SCRIPT" ]; then
            echo "index_script.json was not updated after 'mlc reindex'. Exiting with failure."
            exit 1
          fi
        fi
        
        sleep 1
        BEFORE_SCRIPT=$(stat -c %Y "$INDEX_SCRIPT" 2>/dev/null || stat -f %m "$INDEX_SCRIPT" 2>/dev/null)
        
        # Test reindex specific target
        mlc reindex script
        
        AFTER_SCRIPT=$(stat -c %Y "$INDEX_SCRIPT" 2>/dev/null || stat -f %m "$INDEX_SCRIPT" 2>/dev/null)
        if [ "$BEFORE_SCRIPT" = "$AFTER_SCRIPT" ]; then
          echo "index_script.json was not updated after 'mlc reindex script'. Exiting with failure."
          exit 1
        fi
        
        # Test other reindex commands
        mlc reindex all
        mlc reindex cache
        mlc reindex experiment

    - name: Test 27 - Test index handling when script/cache/repo is manually deleted
      run: |
        # Add a test script
        mlc add script test-delete-script --tags=test,delete,temp
        
        # Get the actual script path from find command
        SCRIPT_PATH=$(mlc find script test-delete-script -p 2>&1)
        echo "Script path: $SCRIPT_PATH"
        
        if [ -z "$SCRIPT_PATH" ]; then
          echo "Script was not found after adding. Exiting with failure."
          exit 1
        fi
        
        # Manually delete the script
        if [ -d "$SCRIPT_PATH" ]; then
          rm -rf "$SCRIPT_PATH"
          echo "Manually deleted script at $SCRIPT_PATH"
        else
          echo "Script directory not found at $SCRIPT_PATH. Exiting with failure."
          exit 1
        fi
        
        # Verify the deleted script is no longer in the index
        # The find command will automatically trigger index rebuild and detect the deletion
        FIND_RESULT=$(mlc find script test-delete-script -p 2>/dev/null)
        if [ -n "$FIND_RESULT" ]; then
          echo "ERROR: Deleted script still found in index. Found: $FIND_RESULT"
          exit 1
        fi
        
        echo "Script deletion test passed successfully"
        
        # Test with cache: run a script to create cache, then delete it manually
        mlc run script --tags=detect,os --quiet
        
        # Find and delete a cache entry
        CACHE_PATH=$(mlc find cache --tags=detect,os -p 2>/dev/null | head -1)
        if [ -n "$CACHE_PATH" ] && [ -d "$CACHE_PATH" ]; then
          echo "Found cache at: $CACHE_PATH"
          
          # Get the cache directory name for later verification
          CACHE_DIR_NAME=$(basename "$CACHE_PATH")
          
          rm -rf "$CACHE_PATH"
          echo "Manually deleted cache at $CACHE_PATH"
          
          # Verify the deleted cache is no longer in the index
          # Check if the cache directory name appears in any found caches
          if mlc find cache --tags=detect,os -p 2>/dev/null | grep -q "$CACHE_DIR_NAME"; then
            echo "ERROR: Deleted cache directory still found in index."
            echo "Deleted cache: $CACHE_PATH"
            echo "Found caches:"
            mlc find cache --tags=detect,os -p 2>/dev/null
            exit 1
          fi
          
          echo "Cache deletion test passed successfully"
        else
          echo "No cache found to test deletion"
        fi

  test_mlc_access_core_actions:

    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.14", "3.8"]
        os: ["ubuntu-latest", "windows-latest", "macos-latest"]
        exclude:
          - os: windows-latest
          - os: macos-latest

    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v3
      with:
        python-version: ${{ matrix.python-version }}

    - name: Configure git longpaths (Windows)
      if: matrix.os == 'windows-latest'
      run: |
        git config --system core.longpaths true

    - name: Install mlcflow from the pull request's source repository and branch
      run: |
        python -m pip install --upgrade pip
        python -m pip install --ignore-installed --verbose pip setuptools
        python -m pip install .
    
    - name: Pull a forked mlperf-automations repository
      run: |
        mlc pull repo anandhu-eng@mlperf-automations  --checkout=dev
    
    - name: Run find repo from test-mlc-access.py
      run: |
        cd .github/scripts && python -c "import test_mlc_access as test; test.test_find_repo()"

    - name: Pull MLCOMMONS mlperf-automations repository
      run: |
        mlc pull repo mlcommons@mlperf-automations --checkout=dev

    - name: Test for rm cache - invalid cache entry tags(test succeeds if command fails)
      run: |
        mlc rm cache --tags=sample,invalid,tags

    - name: Test for rm cache when the cache folder is empty(only for mlc rm cache without specifying particular script)
      run: |
        mlc rm cache -f
        mlc rm cache -f

    - name: Run tests from test-mlc-access.py
      run: |
        cd .github/scripts && python -c "import test_mlc_access as test; test.run_tests()"

  test_mlc_help:
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.14", "3.8"]
        os: ["ubuntu-latest", "windows-latest", "macos-latest"]

    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v3
      with:
        python-version: ${{ matrix.python-version }}

    - name: Configure git longpaths (Windows)
      if: matrix.os == 'windows-latest'
      run: |
        git config --system core.longpaths true

    - name: Install mlcflow from the pull request's source repository and branch
      run: |
        python -m pip install --upgrade pip
        python -m pip install --ignore-installed --verbose pip setuptools
        python -m pip install .
    - name: MLC intro
      run: |
        mlc --help
        mlc -h
    - name: MLC scripts - full
      run: |
        mlc script --help
        mlc script --h
    - name: MLC scripts - individual actions
      run: |
        mlc add script --help
        mlc docker script --help
        mlc find script --help
        mlc search script --help
        mlc list script --help
        mlc rm script --help
        mlc run script --help
        mlc show script --help
        mlc test script --help
        mlc add script -h
        mlc docker script -h
        mlc find script -h
        mlc search script -h
        mlc list script -h
        mlc rm script -h
        mlc run script -h
        mlc show script -h
        mlc test script -h
    - name: MLC repos - full
      run: |
        mlc repo --help
        mlc repo -h
    - name: MLC repos - individual actions
      run: |
        mlc add repo --help
        mlc find repo --help
        mlc pull repo --help
        mlc list repo --help
        mlc rm repo --help
        mlc add repo -h
        mlc find repo -h
        mlc pull repo -h
        mlc list repo -h
        mlc rm repo -h
    - name: MLC cache - full
      run: |
        mlc cache --help
        mlc cache -h
    - name: MLC cache - individual actions
      run: |
        mlc find cache --help
        mlc list cache --help
        mlc rm cache --help
        mlc search cache --help
        mlc show cache --help
        mlc find cache -h
        mlc list cache -h
        mlc rm cache -h
        mlc search cache -h
        mlc show cache -h
  
  test_mlc_script_actions:
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.14", "3.8"]
        os: ["ubuntu-latest", "windows-latest", "macos-latest"]
        action: ["mlcr", "mlce", "mlct"]

    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v3
      with:
        python-version: ${{ matrix.python-version }}

    - name: Configure git longpaths (Windows)
      if: matrix.os == 'windows-latest'
      run: |
        git config --system core.longpaths true

    - name: Install mlcflow from the pull request's source repository and branch
      run: |
        python -m pip install --upgrade pip
        python -m pip install --ignore-installed --verbose pip setuptools
        python -m pip install .
    - name: MLC ${{matrix.action}} script
      run: |
        mlcp mlcommons@mlperf-automations --branch=dev
        ${{matrix.action}} detect,cpu