Fix integration tests #38
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Compare Benchmarks | |
| on: | |
| issue_comment: | |
| types: [created] | |
| workflow_dispatch: | |
| inputs: | |
| pr_number: | |
| description: 'PR number to run benchmarks for' | |
| required: true | |
| type: number | |
| concurrency: | |
| group: benchmark-${{ github.event.issue.number || github.event.inputs.pr_number }} | |
| cancel-in-progress: true | |
| jobs: | |
| benchmark: | |
| if: > | |
| (github.event_name == 'issue_comment' | |
| && github.event.issue.pull_request | |
| && contains(github.event.comment.body, '/benchmark') | |
| && (github.event.comment.author_association == 'COLLABORATOR' || github.event.comment.author_association == 'MEMBER' || github.event.comment.author_association == 'OWNER') | |
| ) || github.event_name == 'workflow_dispatch' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| actions: read | |
| statuses: write | |
| env: | |
| CCACHE_DIR: ${{github.workspace}}/build/.ccache | |
| CCACHE_COMPRESS: true | |
| CCACHE_COMPRESSLEVEL: 6 | |
| steps: | |
| - name: Get PR details | |
| id: pr | |
| uses: actions/github-script@v8 | |
| with: | |
| script: | | |
| const pullRequestNumber = context.eventName === 'workflow_dispatch' | |
| ? parseInt('${{ github.event.inputs.pr_number }}') | |
| : context.issue.number; | |
| const { data: pullRequest } = await github.rest.pulls.get({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| pull_number: pullRequestNumber, | |
| }); | |
| const is_fork = pullRequest.head.repo.full_name !== pullRequest.base.repo.full_name; | |
| return { | |
| base_sha: pullRequest.base.sha, | |
| head_sha: pullRequest.head.sha, | |
| head_repo_full_name: pullRequest.head.repo.full_name, | |
| is_fork: is_fork, | |
| pr_number: pullRequestNumber | |
| }; | |
| - name: Create status check | |
| uses: actions/github-script@v8 | |
| with: | |
| script: | | |
| const prData = ${{ steps.pr.outputs.result }}; | |
| await github.rest.repos.createCommitStatus({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| sha: prData.head_sha, | |
| state: 'pending', | |
| target_url: `${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}`, | |
| description: 'Running benchmark comparison...', | |
| context: 'comparing benchmarks' | |
| }); | |
| - name: Checkout base repository | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Add fork as remote and fetch | |
| if: fromJson(steps.pr.outputs.result).is_fork | |
| run: | | |
| git remote add fork https://github.com/${{ fromJson(steps.pr.outputs.result).head_repo_full_name }}.git | |
| git fetch fork | |
| - name: Install Dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y cmake ninja-build libboost-dev libboost-locale-dev libboost-random-dev libboost-regex-dev libeigen3-dev libmimalloc-dev ccache | |
| - name: Prepare ccache timestamp | |
| id: ccache_cache_timestamp | |
| run: | | |
| echo "timestamp=$(date -u +'%Y-%m-%d-%H;%M;%S')" >> $GITHUB_OUTPUT | |
| - name: Setup ccache cache files | |
| uses: actions/cache@v5 | |
| with: | |
| path: ${{github.workspace}}/build/.ccache | |
| key: benchmark-ccache-${{ steps.ccache_cache_timestamp.outputs.timestamp }} | |
| restore-keys: | | |
| benchmark-ccache- | |
| - name: Install Python Dependencies | |
| run: | | |
| python3 -m pip install --upgrade pip | |
| pip install numpy scipy | |
| - name: Run Benchmark Script | |
| working-directory: ${{ github.workspace }} | |
| run: python3 bin/admin/benchmark_compare.py ${{ fromJson(steps.pr.outputs.result).base_sha }} ${{ fromJson(steps.pr.outputs.result).head_sha }} -c ${{ github.workspace }}/build/_deps/googlebenchmark-src/tools/compare.py -o benchmark-comparison.txt | |
| - name: Upload Results | |
| if: always() | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: benchmark-data-${{ github.run_id }} | |
| path: | | |
| ${{ github.workspace }}/${{ fromJson(steps.pr.outputs.result).base_sha }}-results.json | |
| ${{ github.workspace }}/${{ fromJson(steps.pr.outputs.result).head_sha }}-results.json | |
| ${{ github.workspace }}/benchmark-comparison.txt | |
| retention-days: 30 | |
| - name: Post Results | |
| if: success() | |
| uses: actions/github-script@v8 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const prData = ${{ steps.pr.outputs.result }}; | |
| let comparisonResults = ''; | |
| try { | |
| comparisonResults = fs.readFileSync('benchmark-comparison.txt', 'utf8'); | |
| } catch (error) { | |
| comparisonResults = 'Error reading comparison results: ' + error.message; | |
| } | |
| const commentBody = `## Benchmark Comparison Results | |
| \`\`\` | |
| ${comparisonResults} | |
| \`\`\` | |
| [Full benchmark data](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) | |
| `; | |
| github.rest.issues.createComment({ | |
| issue_number: prData.pr_number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: commentBody | |
| }); | |
| - name: Update status check - Success | |
| if: success() | |
| uses: actions/github-script@v8 | |
| with: | |
| script: | | |
| const prData = ${{ steps.pr.outputs.result }}; | |
| await github.rest.repos.createCommitStatus({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| sha: prData.head_sha, | |
| state: 'success', | |
| target_url: `${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}`, | |
| description: 'Benchmark comparison completed successfully', | |
| context: 'comparing benchmarks' | |
| }); | |
| - name: Notify on Failure | |
| if: failure() | |
| uses: actions/github-script@v8 | |
| with: | |
| script: | | |
| const prData = ${{ steps.pr.outputs.result }}; | |
| // Update status check to failed | |
| await github.rest.repos.createCommitStatus({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| sha: prData.head_sha, | |
| state: 'failure', | |
| target_url: `${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}`, | |
| description: 'Benchmark comparison failed', | |
| context: 'comparing benchmarks' | |
| }); | |
| // Post failure comment | |
| github.rest.issues.createComment({ | |
| issue_number: prData.pr_number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: 'Benchmark comparison failed. Check the workflow logs for details.' | |
| }); |