Build Benchmarking

.github/workflows/build-benchmark-manual.yml

@@ -0,0 +1,199 @@
+name: Build Benchmark - Manual
+
+on:
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: 'Branch to benchmark'
+        required: false
+        default: ''
+      base_branch:
+        description: 'Base branch for comparison'
+        required: false
+        default: 'develop'
+      warmups:
+        description: 'Number of warmup iterations'
+        required: false
+        default: '4'
+      iterations:
+        description: 'Number of measured iterations'
+        required: false
+        default: '7'
+
+jobs:
+  benchmark-head:
+    name: Benchmark Head Branch
+    runs-on: ubuntu-24.04
+    timeout-minutes: 120
+
+    steps:
+      - name: Determine branch to benchmark
+        id: determine-branch
+        run: |
+          BRANCH="${{ github.event.inputs.branch }}"
+          if [ -z "$BRANCH" ]; then
+            BRANCH="${{ github.ref_name }}"
+          fi
+          echo "branch=${BRANCH}" >> "$GITHUB_OUTPUT"
+          echo "Benchmarking branch: ${BRANCH}"
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ steps.determine-branch.outputs.branch }}
+          submodules: recursive
+
+      - name: Set up Java
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'adopt'
+          java-version-file: .github/.java-version
+
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v4
+        with:
+          cache-read-only: true
+
+      - name: Install SDKMAN
+        run: |
+          curl -s "https://get.sdkman.io" | bash
+          source "$HOME/.sdkman/bin/sdkman-init.sh"
+          sdk version
+
+      - name: Install Gradle Profiler
+        run: |
+          source "$HOME/.sdkman/bin/sdkman-init.sh"
+          sdk install gradleprofiler
+          gradle-profiler --version
+
+      - name: Warm up build cache
+        run: |
+          ./gradlew assembleInternalDebug --no-daemon
+
+      - name: Create dynamic scenarios file
+        if: github.event.inputs.warmups != '4' || github.event.inputs.iterations != '7'
+        run: |
+          WARMUPS="${{ github.event.inputs.warmups }}"
+          ITERATIONS="${{ github.event.inputs.iterations }}"
+
+          sed -i "s/warm-ups = 4/warm-ups = ${WARMUPS}/g" build_performance.scenarios
+          sed -i "s/iterations = 7/iterations = ${ITERATIONS}/g" build_performance.scenarios
+
+          echo "Updated scenarios file with warmups=${WARMUPS}, iterations=${ITERATIONS}"
+
+      - name: Run Build Benchmarks
+        run: |
+          source "$HOME/.sdkman/bin/sdkman-init.sh"
+          gradle-profiler --benchmark --scenario-file build_performance.scenarios --output-dir profile-out-head
+
+      - name: Upload Head Benchmark Results
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-results-head-${{ github.run_number }}
+          path: profile-out-head/**/*.csv
+          retention-days: 30
+
+  benchmark-base:
+    name: Benchmark Base Branch
+    runs-on: ubuntu-24.04
+    timeout-minutes: 120
+    needs: benchmark-head
+
+    steps:
+      - name: Determine base branch
+        id: determine-base
+        run: |
+          BASE="${{ github.event.inputs.base_branch }}"
+          echo "base=${BASE}" >> "$GITHUB_OUTPUT"
+          echo "Benchmarking base branch: ${BASE}"
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ steps.determine-base.outputs.base }}
+          submodules: recursive
+
+      - name: Set up Java
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'adopt'
+          java-version-file: .github/.java-version
+
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v4
+        with:
+          cache-read-only: true
+
+      - name: Install SDKMAN
+        run: |
+          curl -s "https://get.sdkman.io" | bash
+          source "$HOME/.sdkman/bin/sdkman-init.sh"
+          sdk version
+
+      - name: Install Gradle Profiler
+        run: |
+          source "$HOME/.sdkman/bin/sdkman-init.sh"
+          sdk install gradleprofiler
+          gradle-profiler --version
+
+      - name: Warm up build cache
+        run: |
+          ./gradlew assembleInternalDebug --no-daemon
+
+      - name: Create dynamic scenarios file
+        if: github.event.inputs.warmups != '4' || github.event.inputs.iterations != '7'
+        run: |
+          WARMUPS="${{ github.event.inputs.warmups }}"
+          ITERATIONS="${{ github.event.inputs.iterations }}"
+
+          sed -i "s/warm-ups = 4/warm-ups = ${WARMUPS}/g" build_performance.scenarios
+          sed -i "s/iterations = 7/iterations = ${ITERATIONS}/g" build_performance.scenarios
+
+          echo "Updated scenarios file with warmups=${WARMUPS}, iterations=${ITERATIONS}"
+
+      - name: Run Build Benchmarks
+        run: |
+          source "$HOME/.sdkman/bin/sdkman-init.sh"
+          gradle-profiler --benchmark --scenario-file build_performance.scenarios --output-dir profile-out-base
+
+      - name: Upload Base Benchmark Results
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-results-base-${{ github.run_number }}
+          path: profile-out-base/**/*.csv
+          retention-days: 30
+
+  compare-results:
+    name: Compare Results
+    runs-on: ubuntu-24.04
+    needs: [benchmark-head, benchmark-base]
+    if: always() && needs.benchmark-head.result == 'success' && needs.benchmark-base.result == 'success'
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Download Head Results
+        uses: actions/download-artifact@v4
+        with:
+          name: benchmark-results-head-${{ github.run_number }}
+          path: profile-out-head
+
+      - name: Download Base Results
+        uses: actions/download-artifact@v4
+        with:
+          name: benchmark-results-base-${{ github.run_number }}
+          path: profile-out-base
+
+      - name: Compare Results
+        run: |
+          python3 scripts/benchmark-comparison.py \
+            profile-out-head/benchmark.csv \
+            profile-out-base/benchmark.csv
+
+      - name: Upload Comparison Report
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-comparison-${{ github.run_number }}
+          path: benchmark-result.md
+          retention-days: 30

.github/workflows/build-benchmark-nightly.yml

@@ -0,0 +1,134 @@
+name: Build Benchmark - Nightly
+
+on:
+  push:
+    branches:
+      - feature/mike/ci-build-times
+  schedule:
+    # Run nightly at 2 AM UTC (matching nightly.yml)
+    - cron: '0 2 * * *'
+  workflow_dispatch:
+    # Allow manual triggering
+
+jobs:
+  benchmark-develop:
+    name: Benchmark Develop Branch
+    runs-on: ubuntu-24.04
+    timeout-minutes: 120
+
+    steps:
+      - name: Determine branch
+        id: determine-branch
+        run: |
+          if [ "${{ github.event_name }}" = "push" ]; then
+            echo "branch=${{ github.ref_name }}" >> "$GITHUB_OUTPUT"
+          else
+            echo "branch=develop" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ steps.determine-branch.outputs.branch }}
+          submodules: recursive
+
+      - name: Set up Java
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'adopt'
+          java-version-file: .github/.java-version
+
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v4
+        with:
+          cache-read-only: true
+
+      - name: Install SDKMAN
+        run: |
+          curl -s "https://get.sdkman.io" | bash
+          source "$HOME/.sdkman/bin/sdkman-init.sh"
+          sdk version
+
+      - name: Install Gradle Profiler
+        run: |
+          source "$HOME/.sdkman/bin/sdkman-init.sh"
+          sdk install gradleprofiler
+          gradle-profiler --version
+
+      - name: Run Gradle Build (warm cache)
+        run: |
+          ./gradlew assembleInternalDebug --no-daemon
+
+      - name: Run Build Benchmarks
+        run: |
+          source "$HOME/.sdkman/bin/sdkman-init.sh"
+          gradle-profiler --benchmark --scenario-file build_performance.scenarios --output-dir profile-out
+
+      - name: Upload Benchmark Results
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-results-develop-${{ github.run_number }}
+          path: profile-out/**/*.csv
+          retention-days: 30
+
+      - name: Generate Summary Report
+        run: |
+          python3 scripts/benchmark-comparison.py profile-out/benchmark.csv
+
+      - name: Upload Summary Report
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-summary-${{ github.run_number }}
+          path: benchmark-result.md
+          retention-days: 30
+
+  benchmark-comparison:
+    name: Compare with Previous Results
+    runs-on: ubuntu-24.04
+    needs: benchmark-develop
+    if: success()
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Download Current Results
+        uses: actions/download-artifact@v4
+        with:
+          name: benchmark-results-develop-${{ github.run_number }}
+          path: profile-out-current
+
+      - name: Compute Previous Run Number
+        id: prev-run
+        run: echo "previous=$(( ${{ github.run_number }} - 1 ))" >> "$GITHUB_OUTPUT"
+
+      - name: Download Previous Results
+        uses: actions/download-artifact@v4
+        continue-on-error: true
+        with:
+          name: benchmark-results-develop-${{ steps.prev-run.outputs.previous }}
+          path: profile-out-previous
+
+      - name: Compare Results
+        if: hashFiles('profile-out-previous/benchmark.csv') != ''
+        run: |
+          python3 scripts/benchmark-comparison.py \
+            profile-out-current/benchmark.csv \
+            profile-out-previous/benchmark.csv
+
+      - name: Generate Trend Summary
+        if: hashFiles('profile-out-previous/benchmark.csv') != ''
+        run: |
+          {
+            echo "## Nightly Build Performance Trend"
+            echo ""
+            cat benchmark-result.md
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload Comparison Report
+        if: hashFiles('profile-out-previous/benchmark.csv') != ''
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-comparison-${{ github.run_number }}
+          path: benchmark-result.md
+          retention-days: 30

build_performance.scenarios

@@ -0,0 +1,50 @@
+# Gradle Profiler Build Performance Scenarios
+# This file defines three benchmark scenarios to measure build performance
+
+# Scenario 1: Clean Build
+# Measures cold build performance with no incremental compilation
+clean-build {
+    title = "Clean Build"
+    tasks = ["assembleInternalDebug"]
+    gradle-args = ["--no-build-cache", "--no-configuration-cache"]
+    cleanup-tasks = ["clean"]
+    warm-ups = 4
+    iterations = 7
+
+    # Use daemon for consistency
+    daemon = warm
+}
+
+# Scenario 2: ABI Change
+# Measures incremental build performance when public API changes
+# This triggers recompilation of all dependent modules
+abi-change {
+    title = "ABI Change (Incremental)"
+    tasks = ["assembleInternalDebug"]
+    gradle-args = ["--no-build-cache", "--no-configuration-cache"]
+
+    # Apply ABI change to a core utility class
+    # Gradle profiler will modify a public method signature
+    apply-abi-change-to = "common/common-utils/src/main/java/com/duckduckgo/common/utils/DispatcherProvider.kt"
+
+    warm-ups = 4
+    iterations = 7
+    daemon = warm
+}
+
+# Scenario 3: Non-ABI Change
+# Measures incremental build performance when only implementation changes
+# Only the modified module needs recompilation
+non-abi-change {
+    title = "Non-ABI Change (Incremental)"
+    tasks = ["assembleInternalDebug"]
+    gradle-args = ["--no-build-cache", "--no-configuration-cache"]
+
+    # Apply non-ABI change to the same file
+    # Gradle profiler will modify method implementation without changing signature
+    apply-non-abi-change-to = "common/common-utils/src/main/java/com/duckduckgo/common/utils/DispatcherProvider.kt"
+
+    warm-ups = 4
+    iterations = 7
+    daemon = warm
+}