execution-specs/.github/workflows/benchmark.yaml at 67d63710b8ab053a2c07e6d72d3cf102cf9aa7f8 · ethereum/execution-specs · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
name: Benchmarking

on:
  push:
    branches:
      - mainnet
      - "forks/**"
    paths:
      - "tests/benchmark/**"
      - "packages/testing/src/execution_testing/benchmark/**"
      - "packages/testing/src/execution_testing/cli/pytest_commands/plugins/**"
      - ".github/workflows/benchmark.yaml"
  pull_request:
    paths-ignore:
      - "**.md"
      - "LICENSE*"
      - ".gitignore"
      - ".vscode/**"
      - "whitelist.txt"
      - "docs/**"
      - "mkdocs.yml"
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
  cancel-in-progress: ${{ github.ref_name != github.event.repository.default_branch }}

jobs:
  unit-tests:
    name: Benchmark Unit Tests
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          submodules: true

      - uses: ./.github/actions/setup-uv
        with:
          enable-cache: "false"

      - uses: ./.github/actions/build-evm-base
        id: evm-builder
        with:
          type: benchmark

      - name: Run benchmark unit tests
        run: just test-tests-bench
        env:
          EVM_BIN: ${{ steps.evm-builder.outputs.evm-bin }}

  sanity-checks:
    name: ${{ matrix.name }}
    needs: [unit-tests]
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: Benchmark Gas Values
            recipe: bench-gas
          - name: Fixed Opcode Count CLI
            recipe: bench-opcode
          - name: Fixed Opcode Count Config
            recipe: bench-opcode-config
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          submodules: true

      - uses: ./.github/actions/setup-uv
        with:
          enable-cache: "false"

      - name: Install PyPy for bench-gas json_loader verify
        if: matrix.recipe == 'bench-gas'
        shell: bash
        run: uv python install pypy3.11

      - uses: ./.github/actions/build-evm-base
        id: evm-builder
        with:
          type: benchmark

      - name: Run ${{ matrix.name }}
        run: just ${{ matrix.recipe }}
        env:
          EVM_BIN: ${{ steps.evm-builder.outputs.evm-bin }}

  # TODO: Add execute remote tests with --gas-benchmark-values
  # TODO: Add execute remote tests with --fixed-opcode-count

  build-artifact:
    name: Build Benchmark Fixture Artifact
    needs: [sanity-checks] # TODO: Add execute remote jobs when implemented
    if: github.event_name == 'push'
    runs-on: [self-hosted-ghr, size-gigachungus-x64]
    timeout-minutes: 720
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          submodules: true

      - uses: ./.github/actions/build-fixtures
        with:
          release_name: benchmark_fast