dune/.github/workflows/bench.yml at main · ocaml/dune · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
# This file is auto-generated. Edit ci/bench.yml.in instead
name: Build time benchmarks

# Do not run this workflow on pull request since this workflow has permission to modify contents.
on:
  push:
    branches:
      - main

permissions:
  # deployments permission to deploy GitHub pages website
  deployments: write
  # contents permission to update benchmark contents in gh-pages branch
  contents: write

jobs:
  build:
    name: Build
    strategy:
      fail-fast: false
      matrix:
        os:
          - ubuntu-latest
        ocaml-compiler:
          - 5.4.x

    runs-on: ${{ matrix.os }}

    steps:
      - name: Checkout code
        uses: actions/checkout@v6

      - name: Use OCaml ${{ matrix.ocaml-compiler }}
        uses: ocaml/setup-ocaml@v3
        with:
          ocaml-compiler: ${{ matrix.ocaml-compiler }}
          opam-depext: false

      # dune doesn't have any additional dependencies so we can build it right
      # away this makes it possible to see build errors as soon as possible
      - run: opam exec -- make _boot/dune.exe

      - name: Install deps on Unix
        run: |
          opam pin -n . --with-version=3.22.0
          opam install . --deps-only --with-test
          opam exec -- make dev-deps
          # Install hyperfine
          wget https://github.com/sharkdp/hyperfine/releases/download/v1.14.0/hyperfine_1.14.0_amd64.deb
          sudo dpkg -i hyperfine_1.14.0_amd64.deb

      - name: Create watch synthetic benchmark
        working-directory: bench
        run: opam exec -- ../_boot/dune.exe exec ./gen_synthetic_dune_watch.exe -- synthetic-watch

      - name: Run synthetic watch benchmark
        working-directory: bench/synthetic-watch
        run: ../gen-benchmark.sh 'opam exec -- ../run-synthetic-dune-watch.sh ../../_boot/dune.exe' 'opam exec -- ../../_boot/dune.exe build @all' 'synthetic watch build time (warm, ${{ runner.os }})' > synthetic-benchmark-result.json

      - name: Print synthetic watch benchmark results
        working-directory: bench/synthetic-watch
        run: |
          cat bench.json
          cat synthetic-benchmark-result.json

      - name: Store synthetic watch benchmark result
        uses: benchmark-action/github-action-benchmark@v1
        with:
          name: Synthetic Watch Benchmark
          tool: "customSmallerIsBetter"
          output-file-path: bench/synthetic-watch/synthetic-benchmark-result.json
          github-token: ${{ secrets.GITHUB_TOKEN }}
          auto-push: true
          # Ratio indicating how worse the current benchmark result is.
          # 175% means if last build took 40s and current takes >70s, it will trigger an alert
          alert-threshold: "225%"
          fail-on-alert: true
          # Enable alert commit comment
          comment-on-alert: true
          # Mention @jchavarri in the commit comment
          alert-comment-cc-users: '@jchavarri'

      - name: Create synthetic benchmark
        working-directory: bench
        run: opam exec -- ../_boot/dune.exe exec ./gen_synthetic.exe -- -n 2000 synthetic

      - name: Run cold synthetic benchmark
        working-directory: bench/synthetic
        run: ../gen-benchmark.sh 'opam exec -- ../../_boot/dune.exe build @all' 'opam exec -- ../../_boot/dune.exe clean' 'synthetic build time (cold, ${{ runner.os }})' > synthetic-benchmark-result.json

      - name: Print cold synthetic benchmark results
        working-directory: bench/synthetic
        run: |
          cat bench.json
          cat synthetic-benchmark-result.json

      - name: Store cold synthetic benchmark result
        uses: benchmark-action/github-action-benchmark@v1
        with:
          name: Synthetic Benchmark
          tool: "customSmallerIsBetter"
          output-file-path: bench/synthetic/synthetic-benchmark-result.json
          github-token: ${{ secrets.GITHUB_TOKEN }}
          auto-push: true
          # Ratio indicating how worse the current benchmark result is.
          # 150% means if last build took 40s and current takes >60s, it will trigger an alert
          alert-threshold: "225%"
          fail-on-alert: true
          # Enable alert commit comment
          comment-on-alert: true
          # Mention @jchavarri in the commit comment
          alert-comment-cc-users: '@jchavarri'

      - name: Run warm synthetic benchmark
        working-directory: bench/synthetic
        run: ../gen-benchmark.sh 'opam exec -- ../../_boot/dune.exe build @all' 'true' 'synthetic build time (warm, ${{ runner.os }})' > synthetic-benchmark-result.json

      - name: Print warm synthetic benchmark results
        working-directory: bench/synthetic
        run: |
          cat bench.json
          cat synthetic-benchmark-result.json

      - name: Store warm synthetic benchmark result
        uses: benchmark-action/github-action-benchmark@v1
        with:
          name: Synthetic Benchmark
          tool: "customSmallerIsBetter"
          output-file-path: bench/synthetic/synthetic-benchmark-result.json
          github-token: ${{ secrets.GITHUB_TOKEN }}
          auto-push: true
          # Ratio indicating how worse the current benchmark result is.
          # 150% means if last build took 40s and current takes >60s, it will trigger an alert
          alert-threshold: "225%"
          fail-on-alert: true
          # Enable alert commit comment
          comment-on-alert: true
          # Mention @jchavarri in the commit comment
          alert-comment-cc-users: '@jchavarri'