From 0847b99a7dd8d2ed0f937e0842c43b7b205df184 Mon Sep 17 00:00:00 2001 From: elle-j Date: Wed, 26 Nov 2025 11:35:30 +0100 Subject: [PATCH 1/4] Implement initial workflow to run benchmarks. --- .github/workflows/benchmark.yml | 111 ++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 .github/workflows/benchmark.yml diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 00000000..09ffd2bd --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,111 @@ +name: Benchmark + +on: + workflow_run: + workflows: [Test] + types: [completed] + +jobs: + benchmark: + runs-on: ubuntu-24.04 + # TODO: Add condition that the base branch in the PR is main. + if: ${{ github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'pull_request' }} + permissions: + pull-requests: write + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set Up Rust Toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + # Without this it will override our rust flags. + rustflags: "" + + - name: Install Solc + uses: ./.github/actions/get-solc + + - name: Download LLVM + uses: ./.github/actions/get-llvm + with: + target: x86_64-unknown-linux-gnu + + - name: Set LLVM Environment Variables + run: | + echo "LLVM_SYS_181_PREFIX=$(pwd)/llvm-x86_64-unknown-linux-gnu" >> $GITHUB_ENV + + - name: Install Benchmarking Tools + run: | + cargo install cargo-criterion critcmp + + - name: Checkout and Compile Main Branch + run: | + git fetch origin "$GITHUB_DEFAULT_BRANCH":"$GITHUB_DEFAULT_BRANCH" + git checkout -f "$GITHUB_DEFAULT_BRANCH" + make install + + - name: Run Benchmarks on Main Branch + run: | + cargo bench --package resolc --bench compile -- --save-baseline main-resolc + cargo bench --package revive-yul --bench parse -- --save-baseline main-yul-parse + cargo bench --package revive-yul --bench lower -- --save-baseline main-yul-lower + timeout-minutes: 20 + + - name: Checkout and Compile PR Branch + run: | + git checkout -f ${{ github.sha }} + make install + + - name: Run Benchmarks on PR Branch + run: | + cargo bench --package resolc --bench compile -- --save-baseline pr-resolc + cargo bench --package revive-yul --bench parse -- --save-baseline pr-yul-parse + cargo bench --package revive-yul --bench lower -- --save-baseline pr-yul-lower + timeout-minutes: 20 + + - name: Compare Benchmarks + run: | + critcmp main-resolc pr-resolc > benchmarks-resolc + critcmp main-yul-parse pr-yul-parse > benchmarks-yul-parse + critcmp main-yul-lower pr-yul-lower > benchmarks-yul-lower + + - name: Post PR Comment with Benchmark Results + uses: marocchino/sticky-pull-request-comment@v2 + with: + header: benchmark + number: ${{ github.event.workflow_run.pull_requests[0].number }} + message: | + # Benchmarks + + ## Compile E2E + +
+ Details + + ``` + $(cat benchmarks-resolc) + ``` + +
+ + ## Parse Yul + +
+ Details + + ``` + $(cat benchmarks-yul-parse) + ``` + +
+ + ## Lower Yul + +
+ Details + + ``` + $(cat benchmarks-yul-lower) + ``` + +
From 3446aaa2385c34b422ecb8c68aa32f10ba918654 Mon Sep 17 00:00:00 2001 From: elle-j Date: Thu, 27 Nov 2025 17:21:41 +0100 Subject: [PATCH 2/4] Update checkouts and report generation. --- .github/workflows/benchmark.yml | 94 ++++++++++++++++++--------------- 1 file changed, 51 insertions(+), 43 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 09ffd2bd..01e91ccf 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,3 +1,5 @@ +# This workflow will run on the default branch when triggered by a `workflow_run` event. + name: Benchmark on: @@ -8,13 +10,14 @@ on: jobs: benchmark: runs-on: ubuntu-24.04 - # TODO: Add condition that the base branch in the PR is main. if: ${{ github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'pull_request' }} permissions: pull-requests: write steps: - - name: Checkout + - name: Checkout PR Branch uses: actions/checkout@v4 + with: + ref: ${{ github.event.workflow_run.head_sha }} - name: Set Up Rust Toolchain uses: actions-rust-lang/setup-rust-toolchain@v1 @@ -36,76 +39,81 @@ jobs: - name: Install Benchmarking Tools run: | - cargo install cargo-criterion critcmp + cargo install critcmp - name: Checkout and Compile Main Branch run: | - git fetch origin "$GITHUB_DEFAULT_BRANCH":"$GITHUB_DEFAULT_BRANCH" - git checkout -f "$GITHUB_DEFAULT_BRANCH" + git fetch origin main + git checkout -f main make install - name: Run Benchmarks on Main Branch run: | - cargo bench --package resolc --bench compile -- --save-baseline main-resolc - cargo bench --package revive-yul --bench parse -- --save-baseline main-yul-parse - cargo bench --package revive-yul --bench lower -- --save-baseline main-yul-lower + cargo bench --package resolc --bench compile -- --save-baseline main_branch_resolc + cargo bench --package revive-yul --bench parse -- --save-baseline main_branch_yul_parse + cargo bench --package revive-yul --bench lower -- --save-baseline main_branch_yul_lower timeout-minutes: 20 - name: Checkout and Compile PR Branch run: | - git checkout -f ${{ github.sha }} + git checkout -f ${{ github.event.workflow_run.head_sha }} make install - name: Run Benchmarks on PR Branch run: | - cargo bench --package resolc --bench compile -- --save-baseline pr-resolc - cargo bench --package revive-yul --bench parse -- --save-baseline pr-yul-parse - cargo bench --package revive-yul --bench lower -- --save-baseline pr-yul-lower + cargo bench --package resolc --bench compile -- --save-baseline pr_branch_resolc + cargo bench --package revive-yul --bench parse -- --save-baseline pr_branch_yul_parse + cargo bench --package revive-yul --bench lower -- --save-baseline pr_branch_yul_lower timeout-minutes: 20 - name: Compare Benchmarks run: | - critcmp main-resolc pr-resolc > benchmarks-resolc - critcmp main-yul-parse pr-yul-parse > benchmarks-yul-parse - critcmp main-yul-lower pr-yul-lower > benchmarks-yul-lower + critcmp main_branch_resolc pr_branch_resolc > benchmarks_resolc + critcmp main_branch_yul_parse pr_branch_yul_parse > benchmarks_yul_parse + critcmp main_branch_yul_lower pr_branch_yul_lower > benchmarks_yul_lower - - name: Post PR Comment with Benchmark Results - uses: marocchino/sticky-pull-request-comment@v2 - with: - header: benchmark - number: ${{ github.event.workflow_run.pull_requests[0].number }} - message: | - # Benchmarks + - name: Create Report + run: | + cat > BENCHMARK_REPORT.md << EOF + # Benchmarks + + ## Compile E2E - ## Compile E2E +
+ Details -
- Details + \`\`\` + $(cat benchmarks_resolc) + \`\`\` - ``` - $(cat benchmarks-resolc) - ``` +
-
+ ## Parse Yul - ## Parse Yul +
+ Details -
- Details + \`\`\` + $(cat benchmarks_yul_parse) + \`\`\` - ``` - $(cat benchmarks-yul-parse) - ``` +
-
+ ## Lower Yul - ## Lower Yul +
+ Details -
- Details + \`\`\` + $(cat benchmarks_yul_lower) + \`\`\` - ``` - $(cat benchmarks-yul-lower) - ``` +
+ EOF -
+ - name: Post PR Comment with Benchmark Report + uses: marocchino/sticky-pull-request-comment@v2 + with: + header: benchmark + number: ${{ github.event.workflow_run.pull_requests[0].number }} + path: BENCHMARK_REPORT.md From 7cc1921c01aee2297fa292f028f4859cd091ef9f Mon Sep 17 00:00:00 2001 From: elle-j Date: Thu, 27 Nov 2025 21:31:31 +0100 Subject: [PATCH 3/4] Update naming. --- .github/workflows/benchmark.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 01e91ccf..4f22f373 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -49,9 +49,9 @@ jobs: - name: Run Benchmarks on Main Branch run: | - cargo bench --package resolc --bench compile -- --save-baseline main_branch_resolc - cargo bench --package revive-yul --bench parse -- --save-baseline main_branch_yul_parse - cargo bench --package revive-yul --bench lower -- --save-baseline main_branch_yul_lower + cargo bench --package resolc --bench compile -- --save-baseline main_resolc + cargo bench --package revive-yul --bench parse -- --save-baseline main_yul_parse + cargo bench --package revive-yul --bench lower -- --save-baseline main_yul_lower timeout-minutes: 20 - name: Checkout and Compile PR Branch @@ -61,16 +61,16 @@ jobs: - name: Run Benchmarks on PR Branch run: | - cargo bench --package resolc --bench compile -- --save-baseline pr_branch_resolc - cargo bench --package revive-yul --bench parse -- --save-baseline pr_branch_yul_parse - cargo bench --package revive-yul --bench lower -- --save-baseline pr_branch_yul_lower + cargo bench --package resolc --bench compile -- --save-baseline pr_resolc + cargo bench --package revive-yul --bench parse -- --save-baseline pr_yul_parse + cargo bench --package revive-yul --bench lower -- --save-baseline pr_yul_lower timeout-minutes: 20 - name: Compare Benchmarks run: | - critcmp main_branch_resolc pr_branch_resolc > benchmarks_resolc - critcmp main_branch_yul_parse pr_branch_yul_parse > benchmarks_yul_parse - critcmp main_branch_yul_lower pr_branch_yul_lower > benchmarks_yul_lower + critcmp main_resolc pr_resolc > benchmarks_resolc + critcmp main_yul_parse pr_yul_parse > benchmarks_yul_parse + critcmp main_yul_lower pr_yul_lower > benchmarks_yul_lower - name: Create Report run: | @@ -80,7 +80,7 @@ jobs: ## Compile E2E
- Details + Results \`\`\` $(cat benchmarks_resolc) @@ -91,7 +91,7 @@ jobs: ## Parse Yul
- Details + Results \`\`\` $(cat benchmarks_yul_parse) @@ -102,7 +102,7 @@ jobs: ## Lower Yul
- Details + Results \`\`\` $(cat benchmarks_yul_lower) From 6187f76d630de762dfb064b969f3cc6dcd6e844b Mon Sep 17 00:00:00 2001 From: elle-j Date: Tue, 2 Dec 2025 16:38:06 +0100 Subject: [PATCH 4/4] Control concurrency and cancel in-progress runs when re-triggered. --- .github/workflows/benchmark.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 4f22f373..71b5d0d5 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -7,6 +7,10 @@ on: workflows: [Test] types: [completed] +concurrency: + group: ${{ github.workflow }}-${{ github.event.workflow_run.head_branch }} + cancel-in-progress: true + jobs: benchmark: runs-on: ubuntu-24.04