From 6ce73ebcc353c5eb45e771fa35d440752923b1a8 Mon Sep 17 00:00:00 2001 From: Mike Scamell Date: Mon, 20 Oct 2025 17:22:27 +0200 Subject: [PATCH 01/10] test --- .github/workflows/build-benchmark-manual.yml | 199 +++++++++++++++ .github/workflows/build-benchmark-nightly.yml | 120 +++++++++ build_performance.scenarios | 53 ++++ scripts/benchmark-comparison.py | 232 ++++++++++++++++++ 4 files changed, 604 insertions(+) create mode 100644 .github/workflows/build-benchmark-manual.yml create mode 100644 .github/workflows/build-benchmark-nightly.yml create mode 100644 build_performance.scenarios create mode 100644 scripts/benchmark-comparison.py diff --git a/.github/workflows/build-benchmark-manual.yml b/.github/workflows/build-benchmark-manual.yml new file mode 100644 index 000000000000..6e087adce53d --- /dev/null +++ b/.github/workflows/build-benchmark-manual.yml @@ -0,0 +1,199 @@ +name: Build Benchmark - Manual + +on: + workflow_dispatch: + inputs: + branch: + description: 'Branch to benchmark' + required: false + default: '' + base_branch: + description: 'Base branch for comparison' + required: false + default: 'develop' + warmups: + description: 'Number of warmup iterations' + required: false + default: '4' + iterations: + description: 'Number of measured iterations' + required: false + default: '7' + +jobs: + benchmark-head: + name: Benchmark Head Branch + runs-on: ubuntu-24.04 + timeout-minutes: 120 + + steps: + - name: Determine branch to benchmark + id: determine-branch + run: | + BRANCH="${{ github.event.inputs.branch }}" + if [ -z "$BRANCH" ]; then + BRANCH="${{ github.ref_name }}" + fi + echo "branch=$BRANCH" >> $GITHUB_OUTPUT + echo "Benchmarking branch: $BRANCH" + + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ steps.determine-branch.outputs.branch }} + submodules: recursive + + - name: Set up Java + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '17' + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v4 + with: + cache-read-only: true + + - name: Install SDKMAN + run: | + curl -s "https://get.sdkman.io" | bash + source "$HOME/.sdkman/bin/sdkman-init.sh" + sdk version + + - name: Install Gradle Profiler + run: | + source "$HOME/.sdkman/bin/sdkman-init.sh" + sdk install gradleprofiler + gradle-profiler --version + + - name: Warm up build cache + run: | + ./gradlew assembleInternalDebug --no-daemon + + - name: Create dynamic scenarios file + if: github.event.inputs.warmups != '4' || github.event.inputs.iterations != '7' + run: | + WARMUPS="${{ github.event.inputs.warmups }}" + ITERATIONS="${{ github.event.inputs.iterations }}" + + sed -i "s/warm-ups = 4/warm-ups = $WARMUPS/g" build_performance.scenarios + sed -i "s/iterations = 7/iterations = $ITERATIONS/g" build_performance.scenarios + + echo "Updated scenarios file with warmups=$WARMUPS, iterations=$ITERATIONS" + + - name: Run Build Benchmarks + run: | + source "$HOME/.sdkman/bin/sdkman-init.sh" + gradle-profiler --benchmark --scenario-file build_performance.scenarios --output-dir profile-out-head + + - name: Upload Head Benchmark Results + uses: actions/upload-artifact@v4 + with: + name: benchmark-results-head-${{ github.run_number }} + path: profile-out-head/**/*.csv + retention-days: 30 + + benchmark-base: + name: Benchmark Base Branch + runs-on: ubuntu-24.04 + timeout-minutes: 120 + needs: benchmark-head + + steps: + - name: Determine base branch + id: determine-base + run: | + BASE="${{ github.event.inputs.base_branch }}" + echo "base=$BASE" >> $GITHUB_OUTPUT + echo "Benchmarking base branch: $BASE" + + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ steps.determine-base.outputs.base }} + submodules: recursive + + - name: Set up Java + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '17' + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v4 + with: + cache-read-only: true + + - name: Install SDKMAN + run: | + curl -s "https://get.sdkman.io" | bash + source "$HOME/.sdkman/bin/sdkman-init.sh" + sdk version + + - name: Install Gradle Profiler + run: | + source "$HOME/.sdkman/bin/sdkman-init.sh" + sdk install gradleprofiler + gradle-profiler --version + + - name: Warm up build cache + run: | + ./gradlew assembleInternalDebug --no-daemon + + - name: Create dynamic scenarios file + if: github.event.inputs.warmups != '4' || github.event.inputs.iterations != '7' + run: | + WARMUPS="${{ github.event.inputs.warmups }}" + ITERATIONS="${{ github.event.inputs.iterations }}" + + sed -i "s/warm-ups = 4/warm-ups = $WARMUPS/g" build_performance.scenarios + sed -i "s/iterations = 7/iterations = $ITERATIONS/g" build_performance.scenarios + + echo "Updated scenarios file with warmups=$WARMUPS, iterations=$ITERATIONS" + + - name: Run Build Benchmarks + run: | + source "$HOME/.sdkman/bin/sdkman-init.sh" + gradle-profiler --benchmark --scenario-file build_performance.scenarios --output-dir profile-out-base + + - name: Upload Base Benchmark Results + uses: actions/upload-artifact@v4 + with: + name: benchmark-results-base-${{ github.run_number }} + path: profile-out-base/**/*.csv + retention-days: 30 + + compare-results: + name: Compare Results + runs-on: ubuntu-24.04 + needs: [benchmark-head, benchmark-base] + if: always() && needs.benchmark-head.result == 'success' && needs.benchmark-base.result == 'success' + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Download Head Results + uses: actions/download-artifact@v4 + with: + name: benchmark-results-head-${{ github.run_number }} + path: profile-out-head + + - name: Download Base Results + uses: actions/download-artifact@v4 + with: + name: benchmark-results-base-${{ github.run_number }} + path: profile-out-base + + - name: Compare Results + run: | + python3 scripts/benchmark-comparison.py \ + profile-out-head/benchmark.csv \ + profile-out-base/benchmark.csv + + - name: Upload Comparison Report + uses: actions/upload-artifact@v4 + with: + name: benchmark-comparison-${{ github.run_number }} + path: benchmark-result.md + retention-days: 30 diff --git a/.github/workflows/build-benchmark-nightly.yml b/.github/workflows/build-benchmark-nightly.yml new file mode 100644 index 000000000000..fa9ea00480ee --- /dev/null +++ b/.github/workflows/build-benchmark-nightly.yml @@ -0,0 +1,120 @@ +name: Build Benchmark - Nightly + +on: + schedule: + # Run nightly at 2 AM UTC (matching nightly.yml) + - cron: '0 2 * * *' + workflow_dispatch: + # Allow manual triggering + +jobs: + benchmark-develop: + name: Benchmark Develop Branch + runs-on: ubuntu-24.04 + timeout-minutes: 120 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: develop + submodules: recursive + + - name: Set up Java + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: '17' + + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v4 + with: + cache-read-only: true + + - name: Install SDKMAN + run: | + curl -s "https://get.sdkman.io" | bash + source "$HOME/.sdkman/bin/sdkman-init.sh" + sdk version + + - name: Install Gradle Profiler + run: | + source "$HOME/.sdkman/bin/sdkman-init.sh" + sdk install gradleprofiler + gradle-profiler --version + + - name: Run Gradle Build (warm cache) + run: | + ./gradlew assembleInternalDebug --no-daemon + + - name: Run Build Benchmarks + run: | + source "$HOME/.sdkman/bin/sdkman-init.sh" + gradle-profiler --benchmark --scenario-file build_performance.scenarios --output-dir profile-out + + - name: Upload Benchmark Results + uses: actions/upload-artifact@v4 + with: + name: benchmark-results-develop-${{ github.run_number }} + path: profile-out/**/*.csv + retention-days: 30 + + - name: Generate Summary Report + run: | + python3 scripts/benchmark-comparison.py profile-out/benchmark.csv + + - name: Upload Summary Report + uses: actions/upload-artifact@v4 + with: + name: benchmark-summary-${{ github.run_number }} + path: benchmark-result.md + retention-days: 30 + + benchmark-comparison: + name: Compare with Previous Results + runs-on: ubuntu-24.04 + needs: benchmark-develop + if: success() + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Download Current Results + uses: actions/download-artifact@v4 + with: + name: benchmark-results-develop-${{ github.run_number }} + path: profile-out-current + + - name: Compute Previous Run Number + id: prev-run + run: echo "previous=$(( ${{ github.run_number }} - 1 ))" >> $GITHUB_OUTPUT + + - name: Download Previous Results + uses: actions/download-artifact@v4 + continue-on-error: true + with: + name: benchmark-results-develop-${{ steps.prev-run.outputs.previous }} + path: profile-out-previous + + - name: Compare Results + if: hashFiles('profile-out-previous/benchmark.csv') != '' + run: | + python3 scripts/benchmark-comparison.py \ + profile-out-current/benchmark.csv \ + profile-out-previous/benchmark.csv + + - name: Generate Trend Summary + if: hashFiles('profile-out-previous/benchmark.csv') != '' + run: | + echo "## Nightly Build Performance Trend" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + cat benchmark-result.md >> $GITHUB_STEP_SUMMARY + + - name: Upload Comparison Report + if: hashFiles('profile-out-previous/benchmark.csv') != '' + uses: actions/upload-artifact@v4 + with: + name: benchmark-comparison-${{ github.run_number }} + path: benchmark-result.md + retention-days: 30 diff --git a/build_performance.scenarios b/build_performance.scenarios new file mode 100644 index 000000000000..32c06f2f7a66 --- /dev/null +++ b/build_performance.scenarios @@ -0,0 +1,53 @@ +# Gradle Profiler Build Performance Scenarios +# This file defines three benchmark scenarios to measure build performance + +# Scenario 1: Clean Build +# Measures cold build performance with no incremental compilation +clean-build { + title = "Clean Build" + versions = ["8.14"] + tasks = ["assembleInternalDebug"] + gradle-args = ["--no-build-cache", "--no-configuration-cache"] + cleanup-tasks = ["clean"] + warm-ups = 4 + iterations = 7 + + # Use daemon for consistency + daemon = warm +} + +# Scenario 2: ABI Change +# Measures incremental build performance when public API changes +# This triggers recompilation of all dependent modules +abi-change { + title = "ABI Change (Incremental)" + versions = ["8.14"] + tasks = ["assembleInternalDebug"] + gradle-args = ["--no-build-cache", "--no-configuration-cache"] + + # Apply ABI change to a core utility class + # Gradle profiler will modify a public method signature + apply-abi-change-to = "app/src/main/java/com/duckduckgo/app/global/DispatcherProvider.kt" + + warm-ups = 4 + iterations = 7 + daemon = warm +} + +# Scenario 3: Non-ABI Change +# Measures incremental build performance when only implementation changes +# Only the modified module needs recompilation +non-abi-change { + title = "Non-ABI Change (Incremental)" + versions = ["8.14"] + tasks = ["assembleInternalDebug"] + gradle-args = ["--no-build-cache", "--no-configuration-cache"] + + # Apply non-ABI change to the same file + # Gradle profiler will modify method implementation without changing signature + apply-non-abi-change-to = "app/src/main/java/com/duckduckgo/app/global/DispatcherProvider.kt" + + warm-ups = 4 + iterations = 7 + daemon = warm +} diff --git a/scripts/benchmark-comparison.py b/scripts/benchmark-comparison.py new file mode 100644 index 000000000000..e066f23d567a --- /dev/null +++ b/scripts/benchmark-comparison.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 +""" +Build Benchmark Comparison Script + +Compares build performance metrics from gradle-profiler CSV outputs. +Analyzes clean build, ABI change, and non-ABI change scenarios. +""" + +import csv +import sys +import statistics +from pathlib import Path +from typing import Dict, List, Optional + + +class BenchmarkResult: + """Represents benchmark results for a single scenario.""" + + def __init__(self, scenario: str, mean: float, median: float, std_dev: float, data: List[float]): + self.scenario = scenario + self.mean = mean + self.median = median + self.std_dev = std_dev + self.data = data + self.min = min(data) if data else 0 + self.max = max(data) if data else 0 + + +def parse_csv(csv_path: Path) -> Dict[str, BenchmarkResult]: + """Parse gradle-profiler CSV output and extract metrics per scenario.""" + results = {} + + if not csv_path.exists(): + print(f"Warning: CSV file not found: {csv_path}") + return results + + with open(csv_path, 'r') as f: + reader = csv.DictReader(f) + scenario_data = {} + + for row in reader: + scenario = row.get('Scenario', row.get('scenario', 'unknown')) + execution_time = row.get('Total execution time', row.get('total execution time')) + + if execution_time: + # Convert time to seconds (gradle-profiler outputs in ms) + time_seconds = float(execution_time) / 1000.0 + + if scenario not in scenario_data: + scenario_data[scenario] = [] + scenario_data[scenario].append(time_seconds) + + # Calculate statistics for each scenario + for scenario, data in scenario_data.items(): + if data: + results[scenario] = BenchmarkResult( + scenario=scenario, + mean=statistics.mean(data), + median=statistics.median(data), + std_dev=statistics.stdev(data) if len(data) > 1 else 0, + data=data + ) + + return results + + +def format_time(seconds: float) -> str: + """Format time in seconds to human-readable format.""" + if seconds >= 60: + minutes = int(seconds // 60) + secs = seconds % 60 + return f"{minutes}m {secs:.1f}s" + else: + return f"{seconds:.1f}s" + + +def calculate_percentage_change(base: float, head: float) -> float: + """Calculate percentage change from base to head.""" + if base == 0: + return 0 + return ((head - base) / base) * 100 + + +def generate_comparison_markdown(base_results: Dict[str, BenchmarkResult], + head_results: Dict[str, BenchmarkResult]) -> str: + """Generate markdown comparison table.""" + + if not base_results and not head_results: + return "No benchmark results found." + + # Determine all scenarios + all_scenarios = set(base_results.keys()) | set(head_results.keys()) + scenario_order = ['clean-build', 'abi-change', 'non-abi-change'] + ordered_scenarios = [s for s in scenario_order if s in all_scenarios] + ordered_scenarios.extend([s for s in all_scenarios if s not in scenario_order]) + + md = "# Build Performance Benchmark Results\n\n" + + if not base_results: + md += "## Head Branch Results\n\n" + md += "| Scenario | Mean | Median | Std Dev | Min | Max |\n" + md += "|----------|------|--------|---------|-----|-----|\n" + + for scenario in ordered_scenarios: + if scenario in head_results: + result = head_results[scenario] + md += f"| {result.scenario} | {format_time(result.mean)} | {format_time(result.median)} | " + md += f"±{format_time(result.std_dev)} | {format_time(result.min)} | {format_time(result.max)} |\n" + + elif not head_results: + md += "## Base Branch Results\n\n" + md += "| Scenario | Mean | Median | Std Dev | Min | Max |\n" + md += "|----------|------|--------|---------|-----|-----|\n" + + for scenario in ordered_scenarios: + if scenario in base_results: + result = base_results[scenario] + md += f"| {result.scenario} | {format_time(result.mean)} | {format_time(result.median)} | " + md += f"±{format_time(result.std_dev)} | {format_time(result.min)} | {format_time(result.max)} |\n" + + else: + # Full comparison + md += "## Comparison: Head vs Base\n\n" + md += "| Scenario | Base Mean | Head Mean | Difference | Change | Status |\n" + md += "|----------|-----------|-----------|------------|--------|--------|\n" + + total_regression = 0 + regression_count = 0 + + for scenario in ordered_scenarios: + base = base_results.get(scenario) + head = head_results.get(scenario) + + if base and head: + diff = head.mean - base.mean + pct_change = calculate_percentage_change(base.mean, head.mean) + + # Determine status emoji + if pct_change > 10: + status = "🔴 Regression" + total_regression += pct_change + regression_count += 1 + elif pct_change > 5: + status = "⚠️ Warning" + elif pct_change < -5: + status = "✅ Improvement" + else: + status = "➖ Neutral" + + sign = "+" if diff > 0 else "" + md += f"| {scenario} | {format_time(base.mean)} | {format_time(head.mean)} | " + md += f"{sign}{format_time(diff)} | {pct_change:+.1f}% | {status} |\n" + + # Summary + md += "\n## Summary\n\n" + if regression_count > 0: + avg_regression = total_regression / regression_count + md += f"⚠️ **{regression_count} scenario(s) show significant regression (>10%)**\n" + md += f"Average regression: {avg_regression:.1f}%\n\n" + else: + md += "✅ No significant regressions detected\n\n" + + # Detailed statistics + md += "### Detailed Statistics\n\n" + for scenario in ordered_scenarios: + base = base_results.get(scenario) + head = head_results.get(scenario) + + if base and head: + md += f"#### {scenario}\n\n" + md += f"- **Base**: Mean={format_time(base.mean)}, Median={format_time(base.median)}, " + md += f"StdDev=±{format_time(base.std_dev)}\n" + md += f"- **Head**: Mean={format_time(head.mean)}, Median={format_time(head.median)}, " + md += f"StdDev=±{format_time(head.std_dev)}\n" + md += f"- **Iterations**: {len(head.data)} measured runs\n\n" + + return md + + +def main(): + """Main entry point.""" + if len(sys.argv) < 2: + print("Usage: benchmark-comparison.py [base_csv]") + sys.exit(1) + + head_csv = Path(sys.argv[1]) + base_csv = Path(sys.argv[2]) if len(sys.argv) > 2 else None + + print(f"Parsing head results from: {head_csv}") + head_results = parse_csv(head_csv) + + base_results = {} + if base_csv: + print(f"Parsing base results from: {base_csv}") + base_results = parse_csv(base_csv) + + # Generate comparison + markdown = generate_comparison_markdown(base_results, head_results) + + # Write to output file + output_file = Path("benchmark-result.md") + with open(output_file, 'w') as f: + f.write(markdown) + + print(f"\nResults written to: {output_file}") + + # Also print to stdout for GitHub Actions + print("\n" + markdown) + + # Write to GitHub Step Summary if available + github_step_summary = Path(os.environ.get('GITHUB_STEP_SUMMARY', '')) + if github_step_summary and github_step_summary.parent.exists(): + with open(github_step_summary, 'a') as f: + f.write('\n' + markdown + '\n') + print(f"Results added to GitHub Step Summary") + + # Exit with error code if there are regressions + if base_results and head_results: + for scenario in head_results.keys(): + if scenario in base_results: + pct_change = calculate_percentage_change( + base_results[scenario].mean, + head_results[scenario].mean + ) + if pct_change > 10: + print(f"\n⚠️ Regression detected in {scenario}: {pct_change:+.1f}%") + sys.exit(1) + + +if __name__ == '__main__': + import os + main() From 2286ab6f775248ff73a3f8aace30bace6ebaca5a Mon Sep 17 00:00:00 2001 From: Mike Scamell Date: Mon, 20 Oct 2025 18:14:44 +0200 Subject: [PATCH 02/10] test run --- .github/workflows/build-benchmark-nightly.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-benchmark-nightly.yml b/.github/workflows/build-benchmark-nightly.yml index fa9ea00480ee..4bfd84233fd8 100644 --- a/.github/workflows/build-benchmark-nightly.yml +++ b/.github/workflows/build-benchmark-nightly.yml @@ -1,6 +1,9 @@ name: Build Benchmark - Nightly on: + push: + branches: + - feature/mike/ci-build-times schedule: # Run nightly at 2 AM UTC (matching nightly.yml) - cron: '0 2 * * *' From 6dd99c9c8198489fb89895104f202b70cd195cca Mon Sep 17 00:00:00 2001 From: Mike Scamell Date: Mon, 20 Oct 2025 18:43:17 +0200 Subject: [PATCH 03/10] test on pushed branch --- .github/workflows/build-benchmark-nightly.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-benchmark-nightly.yml b/.github/workflows/build-benchmark-nightly.yml index 4bfd84233fd8..a1739a3ad3d6 100644 --- a/.github/workflows/build-benchmark-nightly.yml +++ b/.github/workflows/build-benchmark-nightly.yml @@ -17,10 +17,19 @@ jobs: timeout-minutes: 120 steps: + - name: Determine branch + id: determine-branch + run: | + if [ "${{ github.event_name }}" == "push" ]; then + echo "branch=${{ github.ref_name }}" >> $GITHUB_OUTPUT + else + echo "branch=develop" >> $GITHUB_OUTPUT + fi + - name: Checkout repository uses: actions/checkout@v4 with: - ref: develop + ref: ${{ steps.determine-branch.outputs.branch }} submodules: recursive - name: Set up Java From cebc901bdf6eb5bb076e847ad93d13d35c78ea4c Mon Sep 17 00:00:00 2001 From: Mike Scamell Date: Tue, 21 Oct 2025 08:43:25 +0200 Subject: [PATCH 04/10] Update DispatcherProvider path in build performance scenarios --- build_performance.scenarios | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_performance.scenarios b/build_performance.scenarios index 32c06f2f7a66..7dc0a22058b6 100644 --- a/build_performance.scenarios +++ b/build_performance.scenarios @@ -27,7 +27,7 @@ abi-change { # Apply ABI change to a core utility class # Gradle profiler will modify a public method signature - apply-abi-change-to = "app/src/main/java/com/duckduckgo/app/global/DispatcherProvider.kt" + apply-abi-change-to = "common/common-utils/src/main/java/com/duckduckgo/common/utils/DispatcherProvider.kt" warm-ups = 4 iterations = 7 @@ -45,7 +45,7 @@ non-abi-change { # Apply non-ABI change to the same file # Gradle profiler will modify method implementation without changing signature - apply-non-abi-change-to = "app/src/main/java/com/duckduckgo/app/global/DispatcherProvider.kt" + apply-non-abi-change-to = "common/common-utils/src/main/java/com/duckduckgo/common/utils/DispatcherProvider.kt" warm-ups = 4 iterations = 7 From 6fcc544b5e49818c467353ade8bea4eaa1e27072 Mon Sep 17 00:00:00 2001 From: Mike Scamell Date: Tue, 21 Oct 2025 10:33:28 +0200 Subject: [PATCH 05/10] Update build performance scenarios to use default version --- build_performance.scenarios | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build_performance.scenarios b/build_performance.scenarios index 7dc0a22058b6..04bb7eba281f 100644 --- a/build_performance.scenarios +++ b/build_performance.scenarios @@ -5,7 +5,7 @@ # Measures cold build performance with no incremental compilation clean-build { title = "Clean Build" - versions = ["8.14"] + versions = ["default"] tasks = ["assembleInternalDebug"] gradle-args = ["--no-build-cache", "--no-configuration-cache"] cleanup-tasks = ["clean"] @@ -21,7 +21,7 @@ clean-build { # This triggers recompilation of all dependent modules abi-change { title = "ABI Change (Incremental)" - versions = ["8.14"] + versions = ["default"] tasks = ["assembleInternalDebug"] gradle-args = ["--no-build-cache", "--no-configuration-cache"] @@ -39,7 +39,7 @@ abi-change { # Only the modified module needs recompilation non-abi-change { title = "Non-ABI Change (Incremental)" - versions = ["8.14"] + versions = ["default"] tasks = ["assembleInternalDebug"] gradle-args = ["--no-build-cache", "--no-configuration-cache"] From 6a54014181c0b32449bb311fb33bee3bdeec2ce7 Mon Sep 17 00:00:00 2001 From: Mike Scamell Date: Tue, 21 Oct 2025 17:11:53 +0200 Subject: [PATCH 06/10] Refactor CI build scripts for improved readability and consistency --- .github/workflows/build-benchmark-manual.yml | 28 +++++++++---------- .github/workflows/build-benchmark-nightly.yml | 20 +++++++------ 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/.github/workflows/build-benchmark-manual.yml b/.github/workflows/build-benchmark-manual.yml index 6e087adce53d..4a785f198aa4 100644 --- a/.github/workflows/build-benchmark-manual.yml +++ b/.github/workflows/build-benchmark-manual.yml @@ -34,8 +34,8 @@ jobs: if [ -z "$BRANCH" ]; then BRANCH="${{ github.ref_name }}" fi - echo "branch=$BRANCH" >> $GITHUB_OUTPUT - echo "Benchmarking branch: $BRANCH" + echo "branch=${BRANCH}" >> "$GITHUB_OUTPUT" + echo "Benchmarking branch: ${BRANCH}" - name: Checkout repository uses: actions/checkout@v4 @@ -46,8 +46,8 @@ jobs: - name: Set up Java uses: actions/setup-java@v4 with: - distribution: 'temurin' - java-version: '17' + distribution: 'adopt' + java-version-file: .github/.java-version - name: Setup Gradle uses: gradle/actions/setup-gradle@v4 @@ -76,10 +76,10 @@ jobs: WARMUPS="${{ github.event.inputs.warmups }}" ITERATIONS="${{ github.event.inputs.iterations }}" - sed -i "s/warm-ups = 4/warm-ups = $WARMUPS/g" build_performance.scenarios - sed -i "s/iterations = 7/iterations = $ITERATIONS/g" build_performance.scenarios + sed -i "s/warm-ups = 4/warm-ups = ${WARMUPS}/g" build_performance.scenarios + sed -i "s/iterations = 7/iterations = ${ITERATIONS}/g" build_performance.scenarios - echo "Updated scenarios file with warmups=$WARMUPS, iterations=$ITERATIONS" + echo "Updated scenarios file with warmups=${WARMUPS}, iterations=${ITERATIONS}" - name: Run Build Benchmarks run: | @@ -104,8 +104,8 @@ jobs: id: determine-base run: | BASE="${{ github.event.inputs.base_branch }}" - echo "base=$BASE" >> $GITHUB_OUTPUT - echo "Benchmarking base branch: $BASE" + echo "base=${BASE}" >> "$GITHUB_OUTPUT" + echo "Benchmarking base branch: ${BASE}" - name: Checkout repository uses: actions/checkout@v4 @@ -116,8 +116,8 @@ jobs: - name: Set up Java uses: actions/setup-java@v4 with: - distribution: 'temurin' - java-version: '17' + distribution: 'adopt' + java-version-file: .github/.java-version - name: Setup Gradle uses: gradle/actions/setup-gradle@v4 @@ -146,10 +146,10 @@ jobs: WARMUPS="${{ github.event.inputs.warmups }}" ITERATIONS="${{ github.event.inputs.iterations }}" - sed -i "s/warm-ups = 4/warm-ups = $WARMUPS/g" build_performance.scenarios - sed -i "s/iterations = 7/iterations = $ITERATIONS/g" build_performance.scenarios + sed -i "s/warm-ups = 4/warm-ups = ${WARMUPS}/g" build_performance.scenarios + sed -i "s/iterations = 7/iterations = ${ITERATIONS}/g" build_performance.scenarios - echo "Updated scenarios file with warmups=$WARMUPS, iterations=$ITERATIONS" + echo "Updated scenarios file with warmups=${WARMUPS}, iterations=${ITERATIONS}" - name: Run Build Benchmarks run: | diff --git a/.github/workflows/build-benchmark-nightly.yml b/.github/workflows/build-benchmark-nightly.yml index a1739a3ad3d6..0ad163ae1eb8 100644 --- a/.github/workflows/build-benchmark-nightly.yml +++ b/.github/workflows/build-benchmark-nightly.yml @@ -20,10 +20,10 @@ jobs: - name: Determine branch id: determine-branch run: | - if [ "${{ github.event_name }}" == "push" ]; then - echo "branch=${{ github.ref_name }}" >> $GITHUB_OUTPUT + if [ "${{ github.event_name }}" = "push" ]; then + echo "branch=${{ github.ref_name }}" >> "$GITHUB_OUTPUT" else - echo "branch=develop" >> $GITHUB_OUTPUT + echo "branch=develop" >> "$GITHUB_OUTPUT" fi - name: Checkout repository @@ -35,8 +35,8 @@ jobs: - name: Set up Java uses: actions/setup-java@v4 with: - distribution: 'temurin' - java-version: '17' + distribution: 'adopt' + java-version-file: .github/.java-version - name: Setup Gradle uses: gradle/actions/setup-gradle@v4 @@ -100,7 +100,7 @@ jobs: - name: Compute Previous Run Number id: prev-run - run: echo "previous=$(( ${{ github.run_number }} - 1 ))" >> $GITHUB_OUTPUT + run: echo "previous=$(( ${{ github.run_number }} - 1 ))" >> "$GITHUB_OUTPUT" - name: Download Previous Results uses: actions/download-artifact@v4 @@ -119,9 +119,11 @@ jobs: - name: Generate Trend Summary if: hashFiles('profile-out-previous/benchmark.csv') != '' run: | - echo "## Nightly Build Performance Trend" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - cat benchmark-result.md >> $GITHUB_STEP_SUMMARY + { + echo "## Nightly Build Performance Trend" + echo "" + cat benchmark-result.md + } >> "$GITHUB_STEP_SUMMARY" - name: Upload Comparison Report if: hashFiles('profile-out-previous/benchmark.csv') != '' From 99830c14a39859e4fbac2ed827895df0e7ae636f Mon Sep 17 00:00:00 2001 From: Mike Scamell Date: Tue, 21 Oct 2025 17:26:42 +0200 Subject: [PATCH 07/10] Temporarily reduce scenarios in build performance tests for workflow validation --- build_performance.scenarios | 67 +++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/build_performance.scenarios b/build_performance.scenarios index 04bb7eba281f..7a4b98396938 100644 --- a/build_performance.scenarios +++ b/build_performance.scenarios @@ -1,4 +1,5 @@ # Gradle Profiler Build Performance Scenarios +# TEMPORARY: Reduced scenarios for workflow testing # This file defines three benchmark scenarios to measure build performance # Scenario 1: Clean Build @@ -9,45 +10,45 @@ clean-build { tasks = ["assembleInternalDebug"] gradle-args = ["--no-build-cache", "--no-configuration-cache"] cleanup-tasks = ["clean"] - warm-ups = 4 - iterations = 7 + warm-ups = 2 + iterations = 3 # Use daemon for consistency daemon = warm } -# Scenario 2: ABI Change +# Scenario 2: ABI Change - DISABLED FOR TESTING # Measures incremental build performance when public API changes # This triggers recompilation of all dependent modules -abi-change { - title = "ABI Change (Incremental)" - versions = ["default"] - tasks = ["assembleInternalDebug"] - gradle-args = ["--no-build-cache", "--no-configuration-cache"] - - # Apply ABI change to a core utility class - # Gradle profiler will modify a public method signature - apply-abi-change-to = "common/common-utils/src/main/java/com/duckduckgo/common/utils/DispatcherProvider.kt" - - warm-ups = 4 - iterations = 7 - daemon = warm -} - -# Scenario 3: Non-ABI Change +#abi-change { +# title = "ABI Change (Incremental)" +# versions = ["default"] +# tasks = ["assembleInternalDebug"] +# gradle-args = ["--no-build-cache", "--no-configuration-cache"] +# +# # Apply ABI change to a core utility class +# # Gradle profiler will modify a public method signature +# apply-abi-change-to = "common/common-utils/src/main/java/com/duckduckgo/common/utils/DispatcherProvider.kt" +# +# warm-ups = 4 +# iterations = 7 +# daemon = warm +#} + +# Scenario 3: Non-ABI Change - DISABLED FOR TESTING # Measures incremental build performance when only implementation changes # Only the modified module needs recompilation -non-abi-change { - title = "Non-ABI Change (Incremental)" - versions = ["default"] - tasks = ["assembleInternalDebug"] - gradle-args = ["--no-build-cache", "--no-configuration-cache"] - - # Apply non-ABI change to the same file - # Gradle profiler will modify method implementation without changing signature - apply-non-abi-change-to = "common/common-utils/src/main/java/com/duckduckgo/common/utils/DispatcherProvider.kt" - - warm-ups = 4 - iterations = 7 - daemon = warm -} +#non-abi-change { +# title = "Non-ABI Change (Incremental)" +# versions = ["default"] +# tasks = ["assembleInternalDebug"] +# gradle-args = ["--no-build-cache", "--no-configuration-cache"] +# +# # Apply non-ABI change to the same file +# # Gradle profiler will modify method implementation without changing signature +# apply-non-abi-change-to = "common/common-utils/src/main/java/com/duckduckgo/common/utils/DispatcherProvider.kt" +# +# warm-ups = 4 +# iterations = 7 +# daemon = warm +#} From f27992fb28c4d2ef3e854f4d2f2a15517acd0339 Mon Sep 17 00:00:00 2001 From: Mike Scamell Date: Tue, 21 Oct 2025 19:08:39 +0200 Subject: [PATCH 08/10] Remove default version references from build performance scenarios --- build_performance.scenarios | 3 --- 1 file changed, 3 deletions(-) diff --git a/build_performance.scenarios b/build_performance.scenarios index 7a4b98396938..87ac62a19a1c 100644 --- a/build_performance.scenarios +++ b/build_performance.scenarios @@ -6,7 +6,6 @@ # Measures cold build performance with no incremental compilation clean-build { title = "Clean Build" - versions = ["default"] tasks = ["assembleInternalDebug"] gradle-args = ["--no-build-cache", "--no-configuration-cache"] cleanup-tasks = ["clean"] @@ -22,7 +21,6 @@ clean-build { # This triggers recompilation of all dependent modules #abi-change { # title = "ABI Change (Incremental)" -# versions = ["default"] # tasks = ["assembleInternalDebug"] # gradle-args = ["--no-build-cache", "--no-configuration-cache"] # @@ -40,7 +38,6 @@ clean-build { # Only the modified module needs recompilation #non-abi-change { # title = "Non-ABI Change (Incremental)" -# versions = ["default"] # tasks = ["assembleInternalDebug"] # gradle-args = ["--no-build-cache", "--no-configuration-cache"] # From c755584084a9d64305c84f770ef7cd66a8c883c6 Mon Sep 17 00:00:00 2001 From: Mike Scamell Date: Wed, 22 Oct 2025 08:42:48 +0200 Subject: [PATCH 09/10] test 2 --- build_performance.scenarios | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_performance.scenarios b/build_performance.scenarios index 87ac62a19a1c..363ca52262bd 100644 --- a/build_performance.scenarios +++ b/build_performance.scenarios @@ -1,5 +1,5 @@ # Gradle Profiler Build Performance Scenarios -# TEMPORARY: Reduced scenarios for workflow testing +# TEMPORARY: Reduced scenarios for workflow testing test 2 # This file defines three benchmark scenarios to measure build performance # Scenario 1: Clean Build From 660f5e2f3e8fdd45d81195d322ba9cd8323f2bfa Mon Sep 17 00:00:00 2001 From: Mike Scamell Date: Thu, 23 Oct 2025 18:31:40 +0200 Subject: [PATCH 10/10] Revert "Temporarily reduce scenarios in build performance tests for workflow validation" This reverts commit 99830c14 --- build_performance.scenarios | 63 ++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 32 deletions(-) diff --git a/build_performance.scenarios b/build_performance.scenarios index 363ca52262bd..ac4cb2db50d1 100644 --- a/build_performance.scenarios +++ b/build_performance.scenarios @@ -1,5 +1,4 @@ # Gradle Profiler Build Performance Scenarios -# TEMPORARY: Reduced scenarios for workflow testing test 2 # This file defines three benchmark scenarios to measure build performance # Scenario 1: Clean Build @@ -9,43 +8,43 @@ clean-build { tasks = ["assembleInternalDebug"] gradle-args = ["--no-build-cache", "--no-configuration-cache"] cleanup-tasks = ["clean"] - warm-ups = 2 - iterations = 3 + warm-ups = 4 + iterations = 7 # Use daemon for consistency daemon = warm } -# Scenario 2: ABI Change - DISABLED FOR TESTING +# Scenario 2: ABI Change # Measures incremental build performance when public API changes # This triggers recompilation of all dependent modules -#abi-change { -# title = "ABI Change (Incremental)" -# tasks = ["assembleInternalDebug"] -# gradle-args = ["--no-build-cache", "--no-configuration-cache"] -# -# # Apply ABI change to a core utility class -# # Gradle profiler will modify a public method signature -# apply-abi-change-to = "common/common-utils/src/main/java/com/duckduckgo/common/utils/DispatcherProvider.kt" -# -# warm-ups = 4 -# iterations = 7 -# daemon = warm -#} - -# Scenario 3: Non-ABI Change - DISABLED FOR TESTING +abi-change { + title = "ABI Change (Incremental)" + tasks = ["assembleInternalDebug"] + gradle-args = ["--no-build-cache", "--no-configuration-cache"] + + # Apply ABI change to a core utility class + # Gradle profiler will modify a public method signature + apply-abi-change-to = "common/common-utils/src/main/java/com/duckduckgo/common/utils/DispatcherProvider.kt" + + warm-ups = 4 + iterations = 7 + daemon = warm +} + +# Scenario 3: Non-ABI Change # Measures incremental build performance when only implementation changes # Only the modified module needs recompilation -#non-abi-change { -# title = "Non-ABI Change (Incremental)" -# tasks = ["assembleInternalDebug"] -# gradle-args = ["--no-build-cache", "--no-configuration-cache"] -# -# # Apply non-ABI change to the same file -# # Gradle profiler will modify method implementation without changing signature -# apply-non-abi-change-to = "common/common-utils/src/main/java/com/duckduckgo/common/utils/DispatcherProvider.kt" -# -# warm-ups = 4 -# iterations = 7 -# daemon = warm -#} +non-abi-change { + title = "Non-ABI Change (Incremental)" + tasks = ["assembleInternalDebug"] + gradle-args = ["--no-build-cache", "--no-configuration-cache"] + + # Apply non-ABI change to the same file + # Gradle profiler will modify method implementation without changing signature + apply-non-abi-change-to = "common/common-utils/src/main/java/com/duckduckgo/common/utils/DispatcherProvider.kt" + + warm-ups = 4 + iterations = 7 + daemon = warm +}