Real-time Benchmark #507
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Real-time Benchmark | |
| on: | |
| schedule: | |
| - cron: "30 0 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| pull_request: | |
| description: 'PR number that is going to be benchmarked (e.g. "1234")' | |
| required: true | |
| type: number | |
| jit: | |
| description: 'Whether JIT is benchmarked' | |
| required: false | |
| default: "0" | |
| type: choice | |
| options: | |
| - "0" | |
| - "1" | |
| instruction_count: | |
| description: 'Whether Valgrind instruction count should be measured' | |
| required: true | |
| default: "0" | |
| type: choice | |
| options: | |
| - "0" | |
| - "1" | |
| run_micro_bench: | |
| description: 'Whether to run the micro_bench.php test' | |
| required: true | |
| default: "0" | |
| type: choice | |
| options: | |
| - "0" | |
| - "1" | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| concurrency: | |
| group: ${{ github.workflow }} | |
| cancel-in-progress: false | |
| jobs: | |
| REAL_TIME_BENCHMARK: | |
| name: REAL_TIME_BENCHMARK | |
| if: github.repository == 'php/php-src' || github.event_name == 'workflow_dispatch' | |
| runs-on: ubuntu-24.04 | |
| env: | |
| REPOSITORY: ${{ github.repository }} | |
| BRANCH: "master" | |
| COMMIT: ${{ github.sha }} | |
| BASELINE_COMMIT: "d5f6e56610c729710073350af318c4ea1b292cfe" | |
| ID: "master" | |
| JIT: "1" | |
| INSTRUCTION_COUNT: "0" | |
| RUN_MICRO_BENCH: "0" | |
| YEAR: "" | |
| steps: | |
| - name: Setup benchmark environment | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| YEAR="$(date '+%Y')" | |
| echo "YEAR=$YEAR" >> $GITHUB_ENV | |
| if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then | |
| PR_INFO=$(gh pr view ${{ inputs.pull_request }} --json headRepositoryOwner,headRepository,headRefName,headRefOid,baseRefName --repo ${{ github.repository }} | jq -r '.headRepositoryOwner.login, .headRepository.name, .headRefName, .headRefOid, .baseRefName') | |
| REPOSITORY="$(echo "$PR_INFO" | sed -n '1p')/$(echo "$PR_INFO" | sed -n '2p')" | |
| echo "REPOSITORY=$REPOSITORY" >> $GITHUB_ENV | |
| BRANCH=$(echo "$PR_INFO" | sed -n '3p') | |
| echo "BRANCH=$BRANCH" >> $GITHUB_ENV | |
| COMMIT=$(echo "$PR_INFO" | sed -n '4p') | |
| echo "COMMIT=$COMMIT" >> $GITHUB_ENV | |
| BASELINE_BRANCH=$(echo "$PR_INFO" | sed -n '5p') | |
| BASELINE_COMMIT=$(gh api /repos/${{ github.repository }}/compare/$BASELINE_BRANCH...$COMMIT --jq '.merge_base_commit.sha') | |
| echo "BASELINE_COMMIT=$BASELINE_COMMIT" >> $GITHUB_ENV | |
| echo "ID=benchmarked" >> $GITHUB_ENV | |
| echo "JIT=${{ inputs.jit }}" >> $GITHUB_ENV | |
| echo "INSTRUCTION_COUNT=${{ inputs.instruction_count }}" >> $GITHUB_ENV | |
| echo "RUN_MICRO_BENCH=${{ inputs.run_micro_bench }}" >> $GITHUB_ENV | |
| fi | |
| - name: Install dependencies | |
| run: | | |
| set -ex | |
| sudo apt-get update | |
| sudo apt-get install gpg | |
| wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg | |
| gpg --no-default-keyring --keyring /usr/share/keyrings/hashicorp-archive-keyring.gpg --fingerprint | |
| echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list | |
| export DEBIAN_FRONTEND=noninteractive | |
| sudo apt-get update -y | |
| sudo apt-get install -y terraform=1.5.7-* | |
| - name: Checkout benchmark suite | |
| uses: actions/checkout@v6 | |
| with: | |
| repository: 'kocsismate/php-version-benchmarks' | |
| ref: 'main' | |
| fetch-depth: 1 | |
| path: 'php-version-benchmarks' | |
| - name: Checkout php-src (benchmarked version) | |
| uses: actions/checkout@v6 | |
| with: | |
| repository: '${{ env.REPOSITORY }}' | |
| ref: '${{ env.COMMIT }}' | |
| fetch-depth: 100 | |
| path: 'php-version-benchmarks/tmp/php_${{ env.ID }}' | |
| - name: Checkout php-src (baseline version) | |
| uses: actions/checkout@v6 | |
| with: | |
| repository: '${{ env.REPOSITORY }}' | |
| ref: '${{ env.BASELINE_COMMIT }}' | |
| fetch-depth: 100 | |
| path: 'php-version-benchmarks/tmp/php_baseline' | |
| - name: Setup benchmark results | |
| run: | | |
| git config --global user.name "Benchmark" | |
| git config --global user.email "benchmark@php.net" | |
| rm -rf ./php-version-benchmarks/docs/results | |
| - name: Checkout benchmark data | |
| if: github.event_name != 'workflow_dispatch' | |
| uses: actions/checkout@v6 | |
| with: | |
| repository: php/real-time-benchmark-data | |
| ssh-key: ${{ secrets.PHP_VERSION_BENCHMARK_RESULTS_DEPLOY_KEY }} | |
| path: 'php-version-benchmarks/docs/results' | |
| - name: Setup PHP config - baseline PHP version | |
| run: | | |
| set -e | |
| BASELINE_SHORT_SHA="$(echo "${{ env.BASELINE_COMMIT }}" | cut -c1-4)" | |
| cat << EOF > ./php-version-benchmarks/config/php/baseline.ini | |
| PHP_NAME="PHP - baseline@$BASELINE_SHORT_SHA" | |
| PHP_ID=php_baseline | |
| PHP_REPO=${{ github.server_url }}/${{ env.REPOSITORY }}.git | |
| PHP_BRANCH=${{ env.BRANCH }} | |
| PHP_COMMIT=${{ env.BASELINE_COMMIT }} | |
| PHP_JIT=0 | |
| EOF | |
| - name: Setup PHP config - baseline PHP version with JIT | |
| if: github.event_name == 'workflow_dispatch' && inputs.jit == '1' | |
| run: | | |
| set -e | |
| BASELINE_SHORT_SHA="$(echo "${{ env.BASELINE_COMMIT }}" | cut -c1-4)" | |
| cat << EOF > ./php-version-benchmarks/config/php/baseline_jit.ini | |
| PHP_NAME="PHP - baseline@$BASELINE_SHORT_SHA (JIT)" | |
| PHP_ID=php_baseline_jit | |
| PHP_REPO=${{ github.server_url }}/${{ env.REPOSITORY }}.git | |
| PHP_BRANCH=${{ env.BRANCH }} | |
| PHP_COMMIT=${{ env.BASELINE_COMMIT }} | |
| PHP_JIT=${{ env.JIT }} | |
| EOF | |
| git clone ./php-version-benchmarks/tmp/php_baseline/ ./php-version-benchmarks/tmp/php_baseline_jit | |
| - name: Setup PHP config - previous PHP version | |
| if: github.event_name != 'workflow_dispatch' | |
| run: | | |
| set -e | |
| DATABASE="./php-version-benchmarks/docs/results/${{ env.YEAR }}/database.tsv" | |
| if [ -f "$DATABASE" ]; then | |
| LAST_RESULT_SHA="$(tail -n 2 "$DATABASE" | head -n 1 | cut -f 6)" | |
| else | |
| YESTERDAY="$(date -d "-2 day 23:59:59" '+%Y-%m-%d %H:%M:%S')" | |
| LAST_RESULT_SHA="$(cd ./php-version-benchmarks/tmp/php_${{ env.ID }}/ && git --no-pager log --until="$YESTERDAY" -n 1 --pretty='%H')" | |
| fi | |
| echo "LAST_RESULT_SHA=$LAST_RESULT_SHA" >> $GITHUB_ENV | |
| cat << EOF > ./php-version-benchmarks/config/php/previous.ini | |
| PHP_NAME="PHP - previous ${{ env.BRANCH }}" | |
| PHP_ID=php_previous | |
| PHP_REPO=${{ github.server_url }}/${{ env.REPOSITORY }}.git | |
| PHP_BRANCH=${{ env.BRANCH }} | |
| PHP_COMMIT=$LAST_RESULT_SHA | |
| PHP_JIT=0 | |
| EOF | |
| - name: Setup PHP config - benchmarked PHP version | |
| run: | | |
| set -e | |
| cat << EOF > ./php-version-benchmarks/config/php/this.ini | |
| PHP_NAME="PHP - ${{ env.BRANCH }}" | |
| PHP_ID=php_${{ env.ID }} | |
| PHP_REPO=${{ github.server_url }}/${{ env.REPOSITORY }}.git | |
| PHP_BRANCH=${{ env.BRANCH }} | |
| PHP_COMMIT=${{ env.COMMIT }} | |
| PHP_JIT=0 | |
| EOF | |
| - name: Setup PHP config - benchmarked PHP version with JIT | |
| if: env.JIT == '1' | |
| run: | | |
| set -e | |
| cat << EOF > ./php-version-benchmarks/config/php/this_jit.ini | |
| PHP_NAME="PHP - ${{ env.BRANCH }} (JIT)" | |
| PHP_ID=php_${{ env.ID }}_jit | |
| PHP_REPO=${{ github.server_url }}/${{ env.REPOSITORY }}.git | |
| PHP_BRANCH=${{ env.BRANCH }} | |
| PHP_COMMIT=${{ env.COMMIT }} | |
| PHP_JIT=${{ env.JIT }} | |
| EOF | |
| git clone ./php-version-benchmarks/tmp/php_${{ env.ID }}/ ./php-version-benchmarks/tmp/php_${{ env.ID }}_jit | |
| - name: Setup test config | |
| run: | | |
| set -e | |
| cp ./php-version-benchmarks/config/test/1_laravel.ini.dist ./php-version-benchmarks/config/test/1_laravel.ini | |
| cp ./php-version-benchmarks/config/test/2_symfony_main.ini.dist ./php-version-benchmarks/config/test/2_symfony_main.ini | |
| cp ./php-version-benchmarks/config/test/4_wordpress.ini.dist ./php-version-benchmarks/config/test/4_wordpress.ini | |
| cp ./php-version-benchmarks/config/test/5_bench.php.ini.dist ./php-version-benchmarks/config/test/5_bench.php.ini | |
| if [ "${{ env.RUN_MICRO_BENCH }}" -eq "1" ]; then | |
| cp ./php-version-benchmarks/config/test/6_micro_bench.php.ini.dist ./php-version-benchmarks/config/test/6_micro_bench.php.ini | |
| fi | |
| - name: Setup infra config | |
| run: | | |
| set -e | |
| cp ./php-version-benchmarks/config/infra/aws/x86_64-metal.ini.dist ./php-version-benchmarks/config/infra/aws/x86_64-metal.ini | |
| ESCAPED_DOCKER_REGISTRY=$(printf '%s\n' "${{ secrets.PHP_VERSION_BENCHMARK_DOCKER_REGISTRY }}" | sed -e 's/[\/&]/\\&/g') | |
| sed -i "s/INFRA_DOCKER_REGISTRY=public.ecr.aws\/abcdefgh/INFRA_DOCKER_REGISTRY=$ESCAPED_DOCKER_REGISTRY/g" ./php-version-benchmarks/config/infra/aws/x86_64-metal.ini | |
| sed -i "s/INFRA_MEASURE_INSTRUCTION_COUNT=0/INFRA_MEASURE_INSTRUCTION_COUNT=${{ env.INSTRUCTION_COUNT }}/g" ./php-version-benchmarks/config/infra/aws/x86_64-metal.ini | |
| cp ./php-version-benchmarks/build/infrastructure/config/aws.tfvars.dist ./php-version-benchmarks/build/infrastructure/config/aws.tfvars | |
| sed -i 's/access_key = ""/access_key = "${{ secrets.PHP_VERSION_BENCHMARK_AWS_ACCESS_KEY }}"/g' ./php-version-benchmarks/build/infrastructure/config/aws.tfvars | |
| sed -i 's/secret_key = ""/secret_key = "${{ secrets.PHP_VERSION_BENCHMARK_AWS_SECRET_KEY }}"/g' ./php-version-benchmarks/build/infrastructure/config/aws.tfvars | |
| sed -i 's/github_token = ""/github_token = "${{ secrets.GITHUB_TOKEN }}"/g' ./php-version-benchmarks/build/infrastructure/config/aws.tfvars | |
| WORKFLOW_RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| sed -i 's|log_url = ""|log_url = "$WORKFLOW_RUN_URL"|g' ./php-version-benchmarks/build/infrastructure/config/aws.tfvars | |
| sed -i 's|artifact_url = ""|artifact_url = "#ARTIFACT_URL#"|g' ./php-version-benchmarks/build/infrastructure/config/aws.tfvars | |
| if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then | |
| BASE_COMMIT="${{ env.BASELINE_COMMIT }}" | |
| else | |
| BASE_COMMIT="${{ env.LAST_RESULT_SHA }}" | |
| fi | |
| CHANGESET_URL="${{ github.server_url }}/${{ github.repository }}/compare/$BASE_COMMIT..${{ env.COMMIT }}" | |
| sed -i 's|extra_title = ""|extra_title = "Changeset"|g' ./php-version-benchmarks/build/infrastructure/config/aws.tfvars | |
| sed -i 's|extra_text = ""|extra_text = "$CHANGESET_URL"|g' ./php-version-benchmarks/build/infrastructure/config/aws.tfvars | |
| - name: Run benchmark | |
| run: ./php-version-benchmarks/benchmark.sh run aws | |
| - name: Store results | |
| if: github.repository == 'php/php-src' && github.event_name != 'workflow_dispatch' | |
| run: | | |
| set -ex | |
| cd ./php-version-benchmarks/docs/results | |
| git pull --autostash | |
| if [ -e ".git/MERGE_HEAD" ]; then | |
| echo "Merging, can't proceed" | |
| exit 1 | |
| fi | |
| git add . | |
| if git diff --cached --quiet; then | |
| exit 1 | |
| fi | |
| git commit -m "Add result for ${{ github.repository }}@${{ github.sha }}" | |
| git push | |
| - name: Upload artifacts | |
| id: upload | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: results | |
| path: | | |
| ./php-version-benchmarks/tmp/results/${{ env.YEAR }}/**/* | |
| retention-days: 30 | |
| - name: Comment results | |
| if: github.event_name == 'workflow_dispatch' | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| cd ./php-version-benchmarks/tmp/php_${{ env.ID }} | |
| NEWEST_RESULT_DIRECTORY=$(ls -td ${{ github.workspace }}/php-version-benchmarks/docs/results/${{ env.YEAR }}/*/ | head -1) | |
| sed -i "s|#ARTIFACT_URL#|${{ steps.upload.outputs.artifact-url }}|g" "${NEWEST_RESULT_DIRECTORY}result.md" | |
| gh pr comment ${{ inputs.pull_request }} --body-file "${NEWEST_RESULT_DIRECTORY}result.md" --repo ${{ github.repository }} | |
| - name: Cleanup | |
| if: always() | |
| run: | | |
| set -ex | |
| rm -rf ./php-version-benchmarks/tmp/ | |
| rm -f ./php-version-benchmarks/build/infrastructure/config/*.tfvars | |
| rm -rf ./php-version-benchmarks/build/infrastructure/aws/.terraform/ | |
| rm -f ./php-version-benchmarks/build/infrastructure/aws/.terraform.lock.hcl | |
| rm -f ./php-version-benchmarks/build/infrastructure/aws/aws.tfplan | |
| rm -f ./php-version-benchmarks/build/infrastructure/aws/terraform.tfstate | |
| rm -f ./php-version-benchmarks/build/infrastructure/aws/terraform.tfstate.backup | |
| rm -f ./php-version-benchmarks/config/infra/aws/*.ini |