Staging #613
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: notebook-pr | |
| on: | |
| pull_request: | |
| branches: main | |
| paths: "**/*.ipynb" | |
| env: | |
| NB_KERNEL: python | |
| ORG: neuromatch | |
| NMA_REPO: NeuroAI_Course | |
| NMA_MAIN_BRANCH: main | |
| jobs: | |
| setup: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| notebooks: ${{ steps.set-matrix.outputs.notebooks }} | |
| has_notebooks: ${{ steps.set-matrix.outputs.has_notebooks }} | |
| commit_message: ${{ steps.get-commit.outputs.message }} | |
| materials_changed: ${{ steps.materials-check.outputs.materials_changed }} | |
| skip_ci: ${{ steps.check-skip.outputs.skip }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| fetch-depth: 0 | |
| ref: ${{ github.head_ref }} | |
| - name: Get commit message | |
| id: get-commit | |
| run: | | |
| msg=$(git log -1 --pretty=format:"%s") | |
| echo "message=$msg" >> $GITHUB_OUTPUT | |
| - name: Check skip ci | |
| id: check-skip | |
| env: | |
| COMMIT_MESSAGE: ${{ steps.get-commit.outputs.message }} | |
| run: | | |
| if [[ "$COMMIT_MESSAGE" == *"skip ci"* ]]; then | |
| echo "skip=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "skip=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Setup Python environment | |
| if: steps.check-skip.outputs.skip != 'true' | |
| uses: ./.github/actions/setup-python-env | |
| - name: Install additional dependencies | |
| if: steps.check-skip.outputs.skip != 'true' | |
| run: | | |
| pip install jupyter_client==7.3.5 | |
| pip install jinja2==3.1.2 | |
| - name: Setup CI tools | |
| if: steps.check-skip.outputs.skip != 'true' | |
| uses: ./.github/actions/setup-ci-tools | |
| with: | |
| commit-message: ${{ steps.get-commit.outputs.message }} | |
| - name: Setup rendering dependencies | |
| if: steps.check-skip.outputs.skip != 'true' | |
| uses: ./.github/actions/setup-rendering-deps | |
| - name: Get changed files | |
| if: steps.check-skip.outputs.skip != 'true' | |
| id: changed-files | |
| uses: tj-actions/changed-files@v35 | |
| - name: Check if materials.yml changed | |
| id: materials-check | |
| if: steps.check-skip.outputs.skip != 'true' | |
| run: | | |
| if [[ "${{ steps.changed-files.outputs.all_changed_files }}" =~ materials\.yml ]]; then | |
| echo "materials_changed=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "materials_changed=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Set notebook matrix | |
| id: set-matrix | |
| if: steps.check-skip.outputs.skip != 'true' | |
| run: | | |
| # Get filtered notebook list | |
| nbs=$(python ci/select_notebooks.py ${{ steps.changed-files.outputs.all_changed_files }}) | |
| if [ -z "$nbs" ]; then | |
| echo "No notebooks to process" | |
| echo "has_notebooks=false" >> $GITHUB_OUTPUT | |
| echo "notebooks=[]" >> $GITHUB_OUTPUT | |
| else | |
| echo "Notebooks to process: $nbs" | |
| echo "has_notebooks=true" >> $GITHUB_OUTPUT | |
| # Convert space-separated list to JSON array | |
| json_array=$(echo "$nbs" | tr ' ' '\n' | jq -R . | jq -s -c .) | |
| echo "notebooks=$json_array" >> $GITHUB_OUTPUT | |
| fi | |
| process: | |
| needs: setup | |
| if: needs.setup.outputs.skip_ci != 'true' && needs.setup.outputs.has_notebooks == 'true' | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| notebook: ${{ fromJson(needs.setup.outputs.notebooks) }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| fetch-depth: 0 | |
| ref: ${{ github.head_ref }} | |
| - name: Setup Python environment | |
| uses: ./.github/actions/setup-python-env | |
| - name: Install additional dependencies | |
| run: | | |
| pip install jupyter_client==7.3.5 | |
| pip install jinja2==3.1.2 | |
| - name: Setup CI tools | |
| uses: ./.github/actions/setup-ci-tools | |
| with: | |
| commit-message: ${{ needs.setup.outputs.commit_message }} | |
| - name: Setup rendering dependencies | |
| uses: ./.github/actions/setup-rendering-deps | |
| - name: Process notebook | |
| env: | |
| COMMIT_MESSAGE: ${{ needs.setup.outputs.commit_message }} | |
| run: | | |
| echo "Processing: ${{ matrix.notebook }}" | |
| if [[ "$COMMIT_MESSAGE" == *"ci:check"* ]]; then | |
| execflag="--check-execution" | |
| else | |
| execflag="--execute" | |
| fi | |
| python ci/process_notebooks.py "${{ matrix.notebook }}" $execflag | |
| - name: Get tutorial directory | |
| id: get-dir | |
| run: | | |
| nb="${{ matrix.notebook }}" | |
| dir=$(dirname "$nb") | |
| nb_name=$(basename "$nb" .ipynb) | |
| echo "dir=$dir" >> $GITHUB_OUTPUT | |
| echo "nb_name=$nb_name" >> $GITHUB_OUTPUT | |
| # Use ___ as delimiter (won't appear in paths) so we can restore later | |
| safe_dir=$(echo "$dir" | sed 's|/|___|g') | |
| echo "artifact_name=${safe_dir}___${nb_name}" >> $GITHUB_OUTPUT | |
| - name: Upload processed tutorial directory | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: tutorial-${{ steps.get-dir.outputs.artifact_name }} | |
| path: | | |
| ${{ matrix.notebook }} | |
| ${{ steps.get-dir.outputs.dir }}/static/${{ steps.get-dir.outputs.nb_name }}* | |
| ${{ steps.get-dir.outputs.dir }}/solutions/${{ steps.get-dir.outputs.nb_name }}* | |
| ${{ steps.get-dir.outputs.dir }}/student/${{ steps.get-dir.outputs.nb_name }}* | |
| ${{ steps.get-dir.outputs.dir }}/instructor/${{ steps.get-dir.outputs.nb_name }}* | |
| if-no-files-found: ignore | |
| retention-days: 1 | |
| finalize: | |
| needs: [setup, process] | |
| if: always() && needs.setup.outputs.skip_ci != 'true' && needs.setup.outputs.has_notebooks == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| persist-credentials: false | |
| fetch-depth: 0 | |
| ref: ${{ github.head_ref }} | |
| - name: Setup Python environment | |
| uses: ./.github/actions/setup-python-env | |
| - name: Install additional dependencies | |
| run: | | |
| pip install jupyter_client==7.3.5 | |
| pip install jinja2==3.1.2 | |
| - name: Setup CI tools | |
| uses: ./.github/actions/setup-ci-tools | |
| with: | |
| commit-message: ${{ needs.setup.outputs.commit_message }} | |
| - name: Download all artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: artifacts/ | |
| - name: Restore processed files | |
| run: | | |
| echo "Restoring processed files from artifacts..." | |
| # Artifact name format: tutorial-tutorials___W1D5_Microcircuits___W1D5_Tutorial2 | |
| # The path is encoded with ___ as delimiter. Last segment is notebook name, rest is directory. | |
| # upload-artifact strips common path prefixes, so we need to reconstruct the target directory. | |
| for dir in artifacts/tutorial-*; do | |
| if [ -d "$dir" ]; then | |
| # Extract artifact name and parse it to get target directory | |
| artifact_name=$(basename "$dir" | sed 's/^tutorial-//') | |
| # Convert ___ to / to get full path, then extract directory | |
| full_path=$(echo "$artifact_name" | sed 's|___|/|g') | |
| tutorial_dir=$(dirname "$full_path") | |
| echo "Restoring from artifact $(basename "$dir") to: $tutorial_dir" | |
| # Copy all files to the correct tutorial directory | |
| # The artifact contains files with stripped prefixes (e.g., W1D5_Tutorial2.ipynb, student/, solutions/) | |
| mkdir -p "$tutorial_dir" | |
| cp -rv "$dir"/* "$tutorial_dir/" 2>/dev/null || true | |
| fi | |
| done | |
| echo "Restore complete." | |
| - name: Verify exercises | |
| env: | |
| COMMIT_MESSAGE: ${{ needs.setup.outputs.commit_message }} | |
| run: | | |
| nbs='${{ needs.setup.outputs.notebooks }}' | |
| # Convert JSON array to space-separated list | |
| nb_list=$(echo "$nbs" | jq -r '.[]' | tr '\n' ' ') | |
| python ci/verify_exercises.py $nb_list --c "$COMMIT_MESSAGE" | |
| - name: Make PR comment | |
| run: | | |
| nbs='${{ needs.setup.outputs.notebooks }}' | |
| nb_list=$(echo "$nbs" | jq -r '.[]' | tr '\n' ' ') | |
| branch=${{ github.event.pull_request.head.ref }} | |
| python ci/make_pr_comment.py $nb_list --branch $branch --o comment.txt | |
| - name: Update READMEs | |
| if: needs.setup.outputs.materials_changed == 'true' | |
| run: python ci/generate_tutorial_readmes.py | |
| - name: Remove unreferenced derivatives | |
| run: | | |
| python ci/find_unreferenced_content.py > to_remove.txt | |
| if [ -s to_remove.txt ]; then git rm --pathspec-from-file=to_remove.txt; fi | |
| - name: Clean up artifacts directory | |
| run: rm -rf artifacts/ | |
| - name: Commit post-processed files | |
| run: | | |
| git config --local user.email "action@github.com" | |
| git config --local user.name "GitHub Action" | |
| git add '**/*.ipynb' | |
| git add '**/static/*.png' | |
| git add '**/solutions/*.py' | |
| git add '**/student/*.ipynb' | |
| git add '**/instructor/*.ipynb' | |
| git add '**/README.md' | |
| git diff-index --quiet HEAD || git commit -m "Process tutorial notebooks" | |
| - name: Push post-processed files | |
| uses: ad-m/github-push-action@v0.6.0 | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| branch: ${{ github.head_ref }} |