ev-node/.github/workflows/benchmark.yml at main · evstack/ev-node · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
---
name: Benchmarks
permissions: {}
"on":
  push:
    branches:
      - main
  pull_request:
    branches:
      - main
    paths:
      - 'test/e2e/benchmark/**'
      - 'test/e2e/evm_contract_bench_test.go'
      - 'test/e2e/evm_test_common.go'
      - 'test/e2e/sut_helper.go'
      - 'block/internal/executing/**'
      - '.github/workflows/benchmark.yml'
  workflow_dispatch:

jobs:
  evm-benchmark:
    name: EVM Contract Benchmark
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - name: Set up Go
        uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
        with:
          go-version-file: ./go.mod
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
      - name: Install just
        uses: extractions/setup-just@v3
      - name: Build binaries
        run: just build-evm build-da
      - name: Run EVM benchmarks
        run: |
          cd test/e2e && go test -tags evm -bench=. -benchmem -run='^$' \
            -timeout=10m --evm-binary=../../build/evm | tee output.txt
      - name: Run Block Executor benchmarks
        run: |
          go test -bench=BenchmarkProduceBlock -benchmem -run='^$' \
            ./block/internal/executing/... > block_executor_output.txt
      - name: Upload benchmark results
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        with:
          name: evm-benchmark-results
          path: |
            test/e2e/output.txt
            block_executor_output.txt

  spamoor-benchmark:
    name: Spamoor Trace Benchmark
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - name: Set up Go
        uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
        with:
          go-version-file: ./go.mod
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
      - name: Install just
        uses: extractions/setup-just@v3
      - name: Build binaries
        run: just build-evm build-da
      - name: Run Spamoor smoke test
        run: |
          cd test/e2e && BENCH_JSON_OUTPUT=spamoor_bench.json go test -tags evm \
            -run='^TestSpamoorSuite$/^TestSpamoorSmoke$' -v -timeout=15m \
            ./benchmark/ --evm-binary=../../../build/evm
      - name: Upload benchmark results
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        with:
          name: spamoor-benchmark-results
          path: test/e2e/benchmark/spamoor_bench.json

  # TODO: wire up to publish results once additional tests are in place.
  erc20-benchmark:
    name: ERC20 Throughput Benchmark
    runs-on: ubuntu-latest
    timeout-minutes: 20
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - name: Set up Go
        uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
        with:
          go-version-file: ./go.mod
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
      - name: Install just
        uses: extractions/setup-just@v3
      - name: Build binaries
        run: just build-evm build-da
      - name: Run ERC20 throughput test
        run: |
          cd test/e2e && go test -tags evm \
            -run='^TestSpamoorSuite$/^TestERC20Throughput$' -v -timeout=15m \
            ./benchmark/ --evm-binary=../../../build/evm

  # single job to push all results to gh-pages sequentially, avoiding race conditions
  publish-benchmarks:
    name: Publish Benchmark Results
    needs: [evm-benchmark]
    runs-on: ubuntu-latest
    permissions:
      contents: write
      issues: write
      pull-requests: write
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - name: Download EVM benchmark results
        uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
        with:
          name: evm-benchmark-results

      # only update the benchmark baseline on push/dispatch, not on PRs
      - name: Store EVM Contract Roundtrip result
        if: always()
        uses: benchmark-action/github-action-benchmark@a7bc2366eda11037936ea57d811a43b3418d3073 # v1.21.0
        with:
          name: EVM Contract Roundtrip
          tool: 'go'
          output-file-path: test/e2e/output.txt
          auto-push: ${{ github.event_name != 'pull_request' }}
          save-data-file: ${{ github.event_name != 'pull_request' }}
          github-token: ${{ secrets.GITHUB_TOKEN }}
          alert-threshold: '150%'
          fail-on-alert: true
          comment-on-alert: true

      # delete local gh-pages so the next benchmark action step fetches fresh from remote
      - name: Reset local gh-pages branch
        if: always()
        run: git branch -D gh-pages || true

      - name: Store Block Executor result
        if: always()
        uses: benchmark-action/github-action-benchmark@a7bc2366eda11037936ea57d811a43b3418d3073 # v1.21.0
        with:
          name: Block Executor Benchmark
          tool: 'go'
          output-file-path: block_executor_output.txt
          auto-push: ${{ github.event_name != 'pull_request' }}
          save-data-file: ${{ github.event_name != 'pull_request' }}
          github-token: ${{ secrets.GITHUB_TOKEN }}
          alert-threshold: '150%'
          fail-on-alert: true
          comment-on-alert: true