From d9e1265d2ca3c15d9aee128dcf58b7fcc42d30d9 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Tue, 26 Aug 2025 15:43:54 +0200 Subject: [PATCH 01/48] Automate SPMI collection for TE benchmarks --- .../templates/run-superpmi-collect-job.yml | 65 ++-- .../templates/superpmi-collect-pipeline.yml | 213 +----------- src/coreclr/scripts/superpmi_aspnet2.py | 311 ++++++++++++++++++ 3 files changed, 350 insertions(+), 239 deletions(-) create mode 100644 src/coreclr/scripts/superpmi_aspnet2.py diff --git a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml index 78ebc0a8089df8..8627f692e90865 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml @@ -97,6 +97,9 @@ jobs: - ${{ if eq(parameters.collectionName, 'realworld') }}: - name: InputDirectory value: '$(Core_Root_Dir)' + - ${{ if eq(parameters.collectionName, 'aspnet2') }}: + - name: InputDirectory + value: '$(Core_Root_Dir)' - ${{ if eq(parameters.collectionName, 'coreclr_tests') }}: - name: InputDirectory value: '$(managedTestArtifactRootFolderPath)' @@ -124,8 +127,9 @@ jobs: displayName: Enable python venv condition: always() - - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_collect_setup.py -payload_directory $(PayloadLocation) -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -arch $(archType) -platform $(osGroup) -mch_file_tag $(MchFileTag) -input_directory $(InputDirectory) -collection_name $(CollectionName) -collection_type $(CollectionType) $(PublicQueuesCLIArg) -max_size 25 # size in MB - displayName: ${{ format('SuperPMI setup ({0})', parameters.osGroup) }} + - ${{ if ne(parameters.collectionName, 'aspnet2') }}: + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_collect_setup.py -payload_directory $(PayloadLocation) -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -arch $(archType) -platform $(osGroup) -mch_file_tag $(MchFileTag) -input_directory $(InputDirectory) -collection_name $(CollectionName) -collection_type $(CollectionType) $(PublicQueuesCLIArg) -max_size 25 # size in MB + displayName: ${{ format('SuperPMI setup ({0})', parameters.osGroup) }} # Create required directories for merged mch collection and superpmi logs - ${{ if ne(parameters.osGroup, 'windows') }}: @@ -139,36 +143,43 @@ jobs: mkdir $(SpmiLogsLocation) displayName: Create directories + # Direct run for aspnet2 collection: execute the repo script and produce the final .mch without Helix + - ${{ if eq(parameters.collectionName, 'aspnet2') }}: + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_aspnet2.py --runtime-bits-path $(Core_Root_Dir) --output $(MergedMchFileLocation)$(CollectionName).$(CollectionType).$(MchFileTag).mch + displayName: Run aspnet2 SuperPMI collection + # Run superpmi collection in helix - - template: /eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml - parameters: - HelixSource: '$(HelixSourcePrefix)/$(Build.Repository.Name)/$(Build.SourceBranch)' # sources must start with pr/, official/, prodcon/, or agent/ - HelixType: 'test/superpmi/$(CollectionName)/$(CollectionType)/$(Architecture)' - HelixAccessToken: $(HelixApiAccessToken) - HelixTargetQueues: $(Queue) - HelixPreCommands: $(HelixPreCommand) - - ${{ if ne(variables['System.TeamProject'], 'internal') }}: - Creator: $(Build.DefinitionName) - - WorkItemTimeout: 4:00 # 4 hours - WorkItemDirectory: '$(WorkItemDirectory)' - CorrelationPayloadDirectory: '$(CorrelationPayloadDirectory)' - ProjectFile: 'superpmi-collect.proj' - BuildConfig: ${{ parameters.buildConfig }} - osGroup: ${{ parameters.osGroup }} - archType: ${{ parameters.archType }} - InputArtifacts: '$(InputArtifacts)' - CollectionType: '$(CollectionType)' - CollectionName: '$(CollectionName)' - continueOnError: true # Run the future step i.e. merge-mch step even if this step fails. + - ${{ if ne(parameters.collectionName, 'aspnet2') }}: + - template: /eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml + parameters: + HelixSource: '$(HelixSourcePrefix)/$(Build.Repository.Name)/$(Build.SourceBranch)' # sources must start with pr/, official/, prodcon/, or agent/ + HelixType: 'test/superpmi/$(CollectionName)/$(CollectionType)/$(Architecture)' + HelixAccessToken: $(HelixApiAccessToken) + HelixTargetQueues: $(Queue) + HelixPreCommands: $(HelixPreCommand) + + ${{ if ne(variables['System.TeamProject'], 'internal') }}: + Creator: $(Build.DefinitionName) + + WorkItemTimeout: 4:00 # 4 hours + WorkItemDirectory: '$(WorkItemDirectory)' + CorrelationPayloadDirectory: '$(CorrelationPayloadDirectory)' + ProjectFile: 'superpmi-collect.proj' + BuildConfig: ${{ parameters.buildConfig }} + osGroup: ${{ parameters.osGroup }} + archType: ${{ parameters.archType }} + InputArtifacts: '$(InputArtifacts)' + CollectionType: '$(CollectionType)' + CollectionName: '$(CollectionName)' + continueOnError: true # Run the future step i.e. merge-mch step even if this step fails. # Always run merge step even if collection of some partition fails so we can store collection # of the partitions that succeeded. If all the partitions fail, merge-mch would fail and we won't # run future steps like uploading superpmi collection. - - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py merge-mch --ci -log_level DEBUG -pattern $(MchFilesLocation)$(CollectionName).$(CollectionType)*.mch -output_mch_path $(MergedMchFileLocation)$(CollectionName).$(CollectionType).$(MchFileTag).mch - displayName: ${{ format('Merge {0}-{1} SuperPMI collections', parameters.collectionName, parameters.collectionType) }} - condition: always() + - ${{ if ne(parameters.collectionName, 'aspnet2') }}: + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py merge-mch --ci -log_level DEBUG -pattern $(MchFilesLocation)$(CollectionName).$(CollectionType)*.mch -output_mch_path $(MergedMchFileLocation)$(CollectionName).$(CollectionType).$(MchFileTag).mch + displayName: ${{ format('Merge {0}-{1} SuperPMI collections', parameters.collectionName, parameters.collectionType) }} + condition: always() # If merge step above fails, then skip "Upload as artifact" and "Upload to Azure storage" - template: /eng/pipelines/common/upload-artifact-step.yml diff --git a/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml b/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml index 2ff40b6c22efb9..7113fd1246db5e 100644 --- a/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml +++ b/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml @@ -136,228 +136,17 @@ extends: jobParameters: testGroup: outerloop - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml - buildConfig: checked - platforms: - - osx_arm64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: outerloop - liveLibrariesBuildConfig: Release - collectionType: pmi - collectionName: libraries - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml buildConfig: checked platforms: - - osx_arm64 - - linux_arm - - linux_arm64 - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: outerloop - liveLibrariesBuildConfig: Release - collectionType: crossgen2 - collectionName: libraries - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml - buildConfig: checked - platforms: - - osx_arm64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: outerloop - liveLibrariesBuildConfig: Release - collectionType: run - collectionName: realworld - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml - buildConfig: checked - platforms: - - osx_arm64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 helixQueueGroup: ci helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: testGroup: outerloop liveLibrariesBuildConfig: Release collectionType: run - collectionName: benchmarks - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml - buildConfig: checked - platforms: - - osx_arm64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: outerloop - liveLibrariesBuildConfig: Release - collectionType: run_pgo - collectionName: benchmarks - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml - buildConfig: checked - platforms: - - osx_arm64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: outerloop - liveLibrariesBuildConfig: Release - collectionType: run_pgo_optrepeat - collectionName: benchmarks - - # - # Collection of coreclr test run - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml - buildConfig: checked - platforms: - - osx_arm64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - helixQueueGroup: superpmi - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: outerloop - liveLibrariesBuildConfig: Release - SuperPmiCollect: true - unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) - - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml - buildConfig: checked - platforms: - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_arm64 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: outerloop - liveLibrariesBuildConfig: Release - collectionType: nativeaot - collectionName: smoke_tests - - # - # Collection of libraries test run: normal - # Libraries Test Run using Release libraries, and Checked CoreCLR - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/run-test-job.yml - buildConfig: Release - platforms: - - osx_arm64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - helixQueueGroup: superpmi - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testScope: innerloop - liveRuntimeBuildConfig: Checked - dependsOnTestBuildConfiguration: Release - dependsOnTestArchitecture: x64 - scenarios: - - normal - SuperPmiCollect: true - SuperPmiCollectionName: libraries_tests - unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked - helixArtifactsName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked - unifiedBuildConfigOverride: checked - # Default timeout is 150 minutes, which is too low for osx-arm64 queue. - timeoutInMinutes: 300 - - # - # Collection of libraries test run: no_tiered_compilation - # Libraries Test Run using Release libraries, and Checked CoreCLR - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/run-test-job.yml - buildConfig: Release - platforms: - - osx_arm64 - - linux_arm - - linux_arm64 - - linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - helixQueueGroup: superpmi - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testScope: innerloop - liveRuntimeBuildConfig: Checked - dependsOnTestBuildConfiguration: Release - dependsOnTestArchitecture: x64 - scenarios: - - no_tiered_compilation - SuperPmiCollect: true - SuperPmiCollectionName: libraries_tests_no_tiered_compilation - unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked - helixArtifactsName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked - unifiedBuildConfigOverride: checked - # Default timeout is 150 minutes, which is too low for osx-arm64 queue. - timeoutInMinutes: 300 + collectionName: aspnet2 diff --git a/src/coreclr/scripts/superpmi_aspnet2.py b/src/coreclr/scripts/superpmi_aspnet2.py new file mode 100644 index 00000000000000..ca2ec81326aef0 --- /dev/null +++ b/src/coreclr/scripts/superpmi_aspnet2.py @@ -0,0 +1,311 @@ +#!/usr/bin/env python3 +import argparse +import os +import socket +import pathlib +import subprocess +import sys +import textwrap +import time +import zipfile +import shutil +import platform +import tempfile +import urllib.request +from pathlib import Path + +########################################################################################################## +# +# This script sets up an environment for running crank-agent and crank-controller +# (https://github.com/dotnet/crank) locally in order to run various ASP.NET benchmarks +# (TechEmpower, OrchardCMS, etc.) and collect SPMI collections using the provided runtime bits. +# The script is cross-platform and does everything locally while requiring only 'git' as a dependency. +# +# Usage example: +# py run.py --runtime-bits-path C:\runtime\artifacts\bin\coreclr\windows.x64.Checked --output aspnet.mch +# +# Prerequisites: +# * git (crank-agent relies on it being available from PATH) +# * python3 (to run this script) +# +########################################################################################################## + +CRANK_PORT = 5010 + + +# Check if a port is listening +def port_is_listening(host: str, port: int, timeout_s: float = 0.5) -> bool: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.settimeout(timeout_s) + try: + s.connect((host, port)) + return True + except Exception: + return False + + +# Wait for the port to be available +def wait_for_port(host: str, port: int, timeout_s: int) -> bool: + start = time.time() + while time.time() - start < timeout_s: + if port_is_listening(host, port): + return True + time.sleep(0.2) + return False + + +# Convert a filename to the appropriate native DLL name, e.g. "clrjit" -> "libclrjit.so" (on Linux) +def native_dll(name: str) -> str: + ext = ".dll" if sys.platform.startswith("win") else (".dylib" if sys.platform == "darwin" else ".so") + prefix = "" if sys.platform.startswith("win") else "lib" + return f"{prefix}{name}{ext}" + + +# Run a command +def run(cmd): + print(f"Running command: {' '.join(map(str, cmd))}") + kwargs = { + "stdin": subprocess.DEVNULL, + "stdout": sys.stdout, + "stderr": subprocess.STDOUT, + } + if os.name == "nt": + kwargs["creationflags"] = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) + else: + kwargs["start_new_session"] = True + proc = subprocess.Popen(cmd, **kwargs) + try: + return proc.wait() + except KeyboardInterrupt: + proc.terminate() + print("Process terminated") + return None + + +# Ensure .NET tools are installed and Localhost.yml is present +def ensure_tools_and_localhost_yaml(workdir: Path, port: int): + data_dir = workdir / "crank_data" + logs_dir = data_dir / "logs" + build_dir = data_dir / "build" + tools_dir = data_dir / "dotnet_tools" + dotnethome_dir = data_dir / "dotnet_home" + localhost_yml = data_dir / "Localhost.yml" + + os.environ['DOTNET_ROOT'] = str(dotnethome_dir) + os.environ['DOTNET_MULTILEVEL_LOOKUP'] = "1" + os.environ["PATH"] = str(tools_dir) + os.pathsep + os.environ.get("PATH", "") + + if not data_dir.exists(): + print("Installing tools ...") + logs_dir.mkdir(parents=True, exist_ok=True) + build_dir.mkdir(parents=True, exist_ok=True) + dotnethome_dir.mkdir(parents=True, exist_ok=True) + tools_dir.mkdir(parents=True, exist_ok=True) + + # Install .NET 8.0 needed for crank and crank-agent via dotnet-install public script. + url = "https://dot.net/v1/dotnet-install." + ("ps1" if platform.system()=="Windows" else "sh") + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, os.path.basename(url)) + urllib.request.urlretrieve(url, path) + if url.endswith(".ps1"): + subprocess.check_call(["powershell","-ExecutionPolicy","Bypass","-File",path, + "-Channel","8.0","-InstallDir", str(dotnethome_dir)]) + else: + os.chmod(path,0o755) + subprocess.check_call([path,"-Channel","8.0","-InstallDir", str(dotnethome_dir)]) + + dotnet_exe = dotnethome_dir / "dotnet" + run([dotnet_exe, "tool", "install", "--tool-path", str(tools_dir), "Microsoft.Crank.Agent", "--version", "0.2.0-*"]) + run([dotnet_exe, "tool", "install", "--tool-path", str(tools_dir), "Microsoft.Crank.Controller", "--version", "0.2.0-*"]) + + # Create a Localhost.yml to define the local environment + yml = textwrap.dedent( +f""" +variables: + applicationAddress: 127.0.0.1 + loadAddress: 127.0.0.1 + applicationPort: {CRANK_PORT} + applicationScheme: http + loadPort: {CRANK_PORT} + serverPort: 5014 + loadScheme: http +profiles: + Localhost: + variables: + serverAddress: "{{{{applicationAddress}}}}" + jobs: + application: + endpoints: + - "{{{{applicationScheme}}}}://{{{{applicationAddress}}}}:{{{{applicationPort}}}}" + load: + endpoints: + - "{{{{loadScheme}}}}://{{{{loadAddress}}}}:{{{{loadPort}}}}" +""") + localhost_yml.write_text(yml, encoding="utf-8") + else: + print("Localhost.yml already present; skipping tool install/scaffold.") + + +# Start the crank-agent +def start_crank_agent(workdir: Path, port: int): + if port_is_listening("127.0.0.1", port): + raise ValueError("Port already in use") + + print("crank-agent is not running yet. Starting...") + logs_dir = workdir / "crank_data" / "logs" + build_dir = workdir / "crank_data" / "build" + dotnethome_dir = workdir / "crank_data" / "dotnethome" + + agent_process = subprocess.Popen( + [ + "crank-agent", + "--url", f"http://*:{port}", + "--log-path", str(logs_dir), + "--build-path", str(build_dir), + "--dotnethome", str(dotnethome_dir), + ] + ) + + print(f"Waiting up to 20s for crank-agent to start ...") + if not wait_for_port("127.0.0.1", port, 20): + print("Warning: crank-agent didn't open the port in time. Proceeding anyway.", file=sys.stderr) + else: + print("crank-agent started.") + return agent_process + +# Build the crank-controller command for execution +def build_crank_command(framework: str, runtime_bits_path: Path, scenario: str): + spmi_shim = native_dll("superpmi-shim-collector") + clrjit = native_dll("clrjit") + coreclr = native_dll("coreclr") + spcorelib = "System.Private.CoreLib.dll" + config_path = Path("crank_data") / "Localhost.yml" + cmd = [ + "crank", + "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/azure.profile.yml", + "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml", + "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/steadystate.profile.yml", + "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/json.benchmarks.yml", + "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/src/BenchmarksApps/Mvc/benchmarks.jwtapi.yml", + "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/orchard.benchmarks.yml", + "--config", "https://raw.githubusercontent.com/dotnet/crank/main/src/Microsoft.Crank.Jobs.Wrk/wrk.yml", + "--config", "https://raw.githubusercontent.com/dotnet/crank/main/src/Microsoft.Crank.Jobs.Bombardier/bombardier.yml", + "--config", str(config_path), + "--profile", "Localhost", + "--application.noGlobalJson", "false", + "--application.framework", framework, + "--application.Channel", "latest", # should be 'edge', but it causes random build failures sometimes. + "--application.options.collectCounters", "false", + "--application.collectDependencies", "false", + "--load.options.reuseBuild", "true", + "--load.variables.duration", "5", # default 15s is not enough for Tier1 promotion + "--load.job", "bombardier", # Bombardier is more cross-platform friendly (wrk is linux only) + "--application.environmentVariables", f"COMPlus_JitName={spmi_shim}", + "--application.environmentVariables", "SuperPMIShimLogPath=.", + "--application.environmentVariables", f"SuperPMIShimPath=./{clrjit}", + "--application.options.fetch", "true", + "--application.options.fetchOutput", scenario + ".crank.zip", + "--application.options.outputFiles", str(runtime_bits_path / spmi_shim), + "--application.options.outputFiles", str(runtime_bits_path / clrjit), + "--application.options.outputFiles", str(runtime_bits_path / coreclr), + "--application.options.outputFiles", str(runtime_bits_path / spcorelib), + "--scenario", scenario + ] + return cmd + + +# Main entry point +def main(): + parser = argparse.ArgumentParser(description="Cross-platform crank runner with 1 retry (kills its own crank-agent).") + parser.add_argument("--runtime-bits-path", required=True, help="Path to built runtime bits.") + parser.add_argument("--framework", default="net10.0", help="Target .NET framework (e.g., net10.0).") + parser.add_argument("--output", help="File path to copy the resulting merged .mch to (expects a file path, not a directory).") + args = parser.parse_args() + workdir = Path.cwd() + runtime_bits_path = Path(args.runtime_bits_path) + + mcs_cmd = runtime_bits_path / ("mcs.exe" if sys.platform == "win32" else "mcs") + if not mcs_cmd.exists(): + print(f"Error: mcs.exe not found at {mcs_cmd}. Ensure runtime bits include mcs.", file=sys.stderr) + sys.exit(2) + + ensure_tools_and_localhost_yaml(workdir, CRANK_PORT) + try: + agent_process = start_crank_agent(workdir, CRANK_PORT) + + # Benchmarks + + print("### Running OrchardCMS benchmark... ###") + run(build_crank_command(framework=args.framework, runtime_bits_path=runtime_bits_path, scenario="about-sqlite")) + + # print("### Running JsonMVC benchmark... ###") + run(build_crank_command(framework=args.framework, runtime_bits_path=runtime_bits_path, scenario="mvc")) + + # print("### Running NoMvcAuth benchmark... ###") + run(build_crank_command(framework=args.framework, runtime_bits_path=runtime_bits_path, scenario="NoMvcAuth")) + + print("Finished running benchmarks.") + finally: + print("Cleaning up...") + agent_process.terminate() + time.sleep(3) + print("Done!") + + # Extract .mc files from zip archives into crank_data/tmp instead of the current directory + tmp_dir = workdir / "crank_data" / "tmp" + if tmp_dir.exists(): + shutil.rmtree(tmp_dir, ignore_errors=True) + tmp_dir.mkdir(parents=True, exist_ok=True) + + produced_mch = False + try: + extracted_count = 0 + for z in pathlib.Path('.').glob('*.crank.zip'): + with zipfile.ZipFile(z) as f: + for name in f.namelist(): + # include .mc files from any path inside the zip + if name.endswith('.mc'): + f.extract(name, str(tmp_dir)) + extracted_count += 1 + + if extracted_count == 0: + print("No .mc files found in zip outputs; skipping merge.") + else: + # Merge all .mc files into crank.mch, scanning recursively from tmp + print(f"Extracted {extracted_count} .mc files; merging into crank.mch ...") + subprocess.run([ + str(mcs_cmd), + "-merge", + "-recursive", + "-dedup", + "-thin", + "crank.mch", + "." + ], check=True, cwd=str(tmp_dir)) + + # Move the produced crank.mch back to the workspace root + shutil.copyfile(tmp_dir / "crank.mch", workdir / "crank.mch") + produced_mch = True + finally: + # Clean up the tmp extraction directory only after the MCH has been produced + if produced_mch: + shutil.rmtree(tmp_dir, ignore_errors=True) + # remove all *.zip files + for z in pathlib.Path('.').glob('*.crank.zip'): + z.unlink(missing_ok=True) + + # Optionally copy the resulting MCH to the specified output file + if args.output: + mch_name = "crank.mch" + out_path = Path(args.output) + if out_path.exists() and out_path.is_dir(): + print(f"Error: --output points to a directory, expected a file path: {out_path}", file=sys.stderr) + sys.exit(2) + # Ensure the destination directory exists + if out_path.parent and not out_path.parent.exists(): + out_path.parent.mkdir(parents=True, exist_ok=True) + print(f"Copying {mch_name} -> {out_path}") + shutil.copyfile(mch_name, out_path) + +if __name__ == "__main__": + main() \ No newline at end of file From 97137f28b3ab5744b60a24d46738d50d09bcb934 Mon Sep 17 00:00:00 2001 From: Egor Bogatov Date: Tue, 26 Aug 2025 21:35:21 +0200 Subject: [PATCH 02/48] Update superpmi_aspnet2.py --- src/coreclr/scripts/superpmi_aspnet2.py | 47 ++++++++++++++++--------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/src/coreclr/scripts/superpmi_aspnet2.py b/src/coreclr/scripts/superpmi_aspnet2.py index ca2ec81326aef0..53d0ed668229fb 100644 --- a/src/coreclr/scripts/superpmi_aspnet2.py +++ b/src/coreclr/scripts/superpmi_aspnet2.py @@ -62,12 +62,13 @@ def native_dll(name: str) -> str: # Run a command -def run(cmd): +def run(cmd, cwd=None): print(f"Running command: {' '.join(map(str, cmd))}") kwargs = { "stdin": subprocess.DEVNULL, "stdout": sys.stdout, "stderr": subprocess.STDOUT, + "cwd": cwd, } if os.name == "nt": kwargs["creationflags"] = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) @@ -115,8 +116,8 @@ def ensure_tools_and_localhost_yaml(workdir: Path, port: int): subprocess.check_call([path,"-Channel","8.0","-InstallDir", str(dotnethome_dir)]) dotnet_exe = dotnethome_dir / "dotnet" - run([dotnet_exe, "tool", "install", "--tool-path", str(tools_dir), "Microsoft.Crank.Agent", "--version", "0.2.0-*"]) - run([dotnet_exe, "tool", "install", "--tool-path", str(tools_dir), "Microsoft.Crank.Controller", "--version", "0.2.0-*"]) + run([dotnet_exe, "tool", "install", "--tool-path", str(tools_dir), "Microsoft.Crank.Agent", "--version", "0.2.0-*"], cwd=dotnethome_dir) + run([dotnet_exe, "tool", "install", "--tool-path", str(tools_dir), "Microsoft.Crank.Controller", "--version", "0.2.0-*"], cwd=dotnethome_dir) # Create a Localhost.yml to define the local environment yml = textwrap.dedent( @@ -154,7 +155,8 @@ def start_crank_agent(workdir: Path, port: int): print("crank-agent is not running yet. Starting...") logs_dir = workdir / "crank_data" / "logs" build_dir = workdir / "crank_data" / "build" - dotnethome_dir = workdir / "crank_data" / "dotnethome" + # Keep naming consistent with ensure_tools_and_localhost_yaml() + dotnethome_dir = workdir / "crank_data" / "dotnet_home" agent_process = subprocess.Popen( [ @@ -174,12 +176,11 @@ def start_crank_agent(workdir: Path, port: int): return agent_process # Build the crank-controller command for execution -def build_crank_command(framework: str, runtime_bits_path: Path, scenario: str): +def build_crank_command(framework: str, runtime_bits_path: Path, scenario: str, config_path: Path): spmi_shim = native_dll("superpmi-shim-collector") clrjit = native_dll("clrjit") coreclr = native_dll("coreclr") spcorelib = "System.Private.CoreLib.dll" - config_path = Path("crank_data") / "Localhost.yml" cmd = [ "crank", "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/azure.profile.yml", @@ -190,7 +191,7 @@ def build_crank_command(framework: str, runtime_bits_path: Path, scenario: str): "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/orchard.benchmarks.yml", "--config", "https://raw.githubusercontent.com/dotnet/crank/main/src/Microsoft.Crank.Jobs.Wrk/wrk.yml", "--config", "https://raw.githubusercontent.com/dotnet/crank/main/src/Microsoft.Crank.Jobs.Bombardier/bombardier.yml", - "--config", str(config_path), + "--config", str(config_path), "--profile", "Localhost", "--application.noGlobalJson", "false", "--application.framework", framework, @@ -221,7 +222,7 @@ def main(): parser.add_argument("--framework", default="net10.0", help="Target .NET framework (e.g., net10.0).") parser.add_argument("--output", help="File path to copy the resulting merged .mch to (expects a file path, not a directory).") args = parser.parse_args() - workdir = Path.cwd() + repo_dir = Path.cwd() runtime_bits_path = Path(args.runtime_bits_path) mcs_cmd = runtime_bits_path / ("mcs.exe" if sys.platform == "win32" else "mcs") @@ -229,20 +230,28 @@ def main(): print(f"Error: mcs.exe not found at {mcs_cmd}. Ensure runtime bits include mcs.", file=sys.stderr) sys.exit(2) - ensure_tools_and_localhost_yaml(workdir, CRANK_PORT) + # Create an isolated temp working directory for crank_data + temp_root = Path(tempfile.mkdtemp(prefix="aspnet2_crank_")) + print(f"Using temp work directory: {temp_root}") + + # Set current working directory to temp_root + os.chdir(temp_root) + + ensure_tools_and_localhost_yaml(temp_root, CRANK_PORT) try: - agent_process = start_crank_agent(workdir, CRANK_PORT) + agent_process = start_crank_agent(temp_root, CRANK_PORT) # Benchmarks print("### Running OrchardCMS benchmark... ###") - run(build_crank_command(framework=args.framework, runtime_bits_path=runtime_bits_path, scenario="about-sqlite")) + config_path = temp_root / "crank_data" / "Localhost.yml" + run(build_crank_command(framework=args.framework, runtime_bits_path=runtime_bits_path, scenario="about-sqlite", config_path=config_path)) # print("### Running JsonMVC benchmark... ###") - run(build_crank_command(framework=args.framework, runtime_bits_path=runtime_bits_path, scenario="mvc")) + run(build_crank_command(framework=args.framework, runtime_bits_path=runtime_bits_path, scenario="mvc", config_path=config_path)) # print("### Running NoMvcAuth benchmark... ###") - run(build_crank_command(framework=args.framework, runtime_bits_path=runtime_bits_path, scenario="NoMvcAuth")) + run(build_crank_command(framework=args.framework, runtime_bits_path=runtime_bits_path, scenario="NoMvcAuth", config_path=config_path)) print("Finished running benchmarks.") finally: @@ -252,7 +261,7 @@ def main(): print("Done!") # Extract .mc files from zip archives into crank_data/tmp instead of the current directory - tmp_dir = workdir / "crank_data" / "tmp" + tmp_dir = temp_root / "crank_data" / "tmp" if tmp_dir.exists(): shutil.rmtree(tmp_dir, ignore_errors=True) tmp_dir.mkdir(parents=True, exist_ok=True) @@ -284,7 +293,7 @@ def main(): ], check=True, cwd=str(tmp_dir)) # Move the produced crank.mch back to the workspace root - shutil.copyfile(tmp_dir / "crank.mch", workdir / "crank.mch") + shutil.copyfile(tmp_dir / "crank.mch", repo_dir / "crank.mch") produced_mch = True finally: # Clean up the tmp extraction directory only after the MCH has been produced @@ -307,5 +316,11 @@ def main(): print(f"Copying {mch_name} -> {out_path}") shutil.copyfile(mch_name, out_path) + # Clean up the temp crank directory at the very end + try: + shutil.rmtree(temp_root, ignore_errors=True) + except Exception: + pass + if __name__ == "__main__": - main() \ No newline at end of file + main() From 4cd9979b363dcc9fb501c231e353d92ce4f8ec24 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Tue, 26 Aug 2025 23:35:10 +0200 Subject: [PATCH 03/48] bunch of fixes --- src/coreclr/scripts/superpmi_aspnet2.py | 126 ++++++++++++++---------- 1 file changed, 75 insertions(+), 51 deletions(-) diff --git a/src/coreclr/scripts/superpmi_aspnet2.py b/src/coreclr/scripts/superpmi_aspnet2.py index 53d0ed668229fb..8119d4a2518cb0 100644 --- a/src/coreclr/scripts/superpmi_aspnet2.py +++ b/src/coreclr/scripts/superpmi_aspnet2.py @@ -22,7 +22,7 @@ # The script is cross-platform and does everything locally while requiring only 'git' as a dependency. # # Usage example: -# py run.py --runtime-bits-path C:\runtime\artifacts\bin\coreclr\windows.x64.Checked --output aspnet.mch +# py superpmi_aspnet2.py --core_root C:\runtime\artifacts\bin\coreclr\windows.x64.Checked --output_mch aspnet.mch # # Prerequisites: # * git (crank-agent relies on it being available from PATH) @@ -218,56 +218,65 @@ def build_crank_command(framework: str, runtime_bits_path: Path, scenario: str, # Main entry point def main(): parser = argparse.ArgumentParser(description="Cross-platform crank runner with 1 retry (kills its own crank-agent).") - parser.add_argument("--runtime-bits-path", required=True, help="Path to built runtime bits.") - parser.add_argument("--framework", default="net10.0", help="Target .NET framework (e.g., net10.0).") - parser.add_argument("--output", help="File path to copy the resulting merged .mch to (expects a file path, not a directory).") + # Renamed args + parser.add_argument("--core_root", required=True, help="Path to built runtime bits (CORE_ROOT).") + parser.add_argument("--tfm", default="net10.0", help="Target Framework Moniker (e.g., net10.0).") + parser.add_argument("--output_mch", required=True, help="File path to copy the resulting merged .mch to (expects a file path, not a directory).") + + # New args + parser.add_argument("--work_dir", help="Optional work directory; if not specified, a temp directory is used.") + parser.add_argument("--no_cleanup", action="store_true", help="If specified, do not clean up temporary files after execution.") args = parser.parse_args() repo_dir = Path.cwd() - runtime_bits_path = Path(args.runtime_bits_path) + runtime_bits_path = Path(args.core_root).expanduser().resolve() + output_mch_path = Path(args.output_mch).expanduser().resolve() mcs_cmd = runtime_bits_path / ("mcs.exe" if sys.platform == "win32" else "mcs") if not mcs_cmd.exists(): print(f"Error: mcs.exe not found at {mcs_cmd}. Ensure runtime bits include mcs.", file=sys.stderr) sys.exit(2) - # Create an isolated temp working directory for crank_data - temp_root = Path(tempfile.mkdtemp(prefix="aspnet2_crank_")) - print(f"Using temp work directory: {temp_root}") + # Create or use working directory for crank_data + created_temp = False + if args.work_dir: + temp_root = Path(args.work_dir).resolve() + temp_root.mkdir(parents=True, exist_ok=True) + print(f"Using work directory: {temp_root}") + else: + temp_root = Path(tempfile.mkdtemp(prefix="aspnet4_crank_")) + created_temp = True + print(f"Using temp work directory: {temp_root}") # Set current working directory to temp_root os.chdir(temp_root) ensure_tools_and_localhost_yaml(temp_root, CRANK_PORT) try: + agent_process = None agent_process = start_crank_agent(temp_root, CRANK_PORT) # Benchmarks - print("### Running OrchardCMS benchmark... ###") config_path = temp_root / "crank_data" / "Localhost.yml" - run(build_crank_command(framework=args.framework, runtime_bits_path=runtime_bits_path, scenario="about-sqlite", config_path=config_path)) + # print("### Running OrchardCMS benchmark... ###") + # run(build_crank_command(framework=args.tfm, runtime_bits_path=runtime_bits_path, scenario="about-sqlite", config_path=config_path)) - # print("### Running JsonMVC benchmark... ###") - run(build_crank_command(framework=args.framework, runtime_bits_path=runtime_bits_path, scenario="mvc", config_path=config_path)) + print("### Running JsonMVC benchmark... ###") + run(build_crank_command(framework=args.tfm, runtime_bits_path=runtime_bits_path, scenario="mvc", config_path=config_path)) - # print("### Running NoMvcAuth benchmark... ###") - run(build_crank_command(framework=args.framework, runtime_bits_path=runtime_bits_path, scenario="NoMvcAuth", config_path=config_path)) + print("### Running NoMvcAuth benchmark... ###") + run(build_crank_command(framework=args.tfm, runtime_bits_path=runtime_bits_path, scenario="NoMvcAuth", config_path=config_path)) - print("Finished running benchmarks.") - finally: - print("Cleaning up...") - agent_process.terminate() - time.sleep(3) - print("Done!") - # Extract .mc files from zip archives into crank_data/tmp instead of the current directory - tmp_dir = temp_root / "crank_data" / "tmp" - if tmp_dir.exists(): - shutil.rmtree(tmp_dir, ignore_errors=True) - tmp_dir.mkdir(parents=True, exist_ok=True) + print("Finished running benchmarks.") - produced_mch = False - try: + # Extract .mc files from zip archives into crank_data/tmp instead of the current directory + print("Extracting .mc files from zip archives...") + tmp_dir = temp_root / "crank_data" / "tmp" + if tmp_dir.exists(): + shutil.rmtree(tmp_dir, ignore_errors=True) + tmp_dir.mkdir(parents=True, exist_ok=True) + produced_mch = False extracted_count = 0 for z in pathlib.Path('.').glob('*.crank.zip'): with zipfile.ZipFile(z) as f: @@ -276,7 +285,9 @@ def main(): if name.endswith('.mc'): f.extract(name, str(tmp_dir)) extracted_count += 1 + z.unlink(missing_ok=True) + # Merge *.mc files into crank.mch if extracted_count == 0: print("No .mc files found in zip outputs; skipping merge.") else: @@ -293,34 +304,47 @@ def main(): ], check=True, cwd=str(tmp_dir)) # Move the produced crank.mch back to the workspace root + print(f"Moving produced crank.mch to {repo_dir / 'crank.mch'}") shutil.copyfile(tmp_dir / "crank.mch", repo_dir / "crank.mch") produced_mch = True + + # Copy the resulting MCH to the specified output file + if args.output_mch: + mch_src = repo_dir / "crank.mch" + if not mch_src.exists(): + print(f"Error: expected MCH not found at {mch_src}", file=sys.stderr) + sys.exit(2) + out_path = output_mch_path + if out_path.exists() and out_path.is_dir(): + print(f"Error: --output_mch points to a directory, expected a file path: {out_path}", file=sys.stderr) + sys.exit(2) + # Ensure the destination directory exists + if out_path.parent and not out_path.parent.exists(): + out_path.parent.mkdir(parents=True, exist_ok=True) + print(f"Copying {mch_src} -> {out_path}") + shutil.copyfile(mch_src, out_path) + + # Clean up the temp crank directory at the very end + if not args.no_cleanup: + # Only delete the working directory if we created a temp one + if 'created_temp' in locals() and created_temp: + try: + shutil.rmtree(temp_root, ignore_errors=True) + except Exception: + pass + finally: - # Clean up the tmp extraction directory only after the MCH has been produced - if produced_mch: - shutil.rmtree(tmp_dir, ignore_errors=True) - # remove all *.zip files - for z in pathlib.Path('.').glob('*.crank.zip'): - z.unlink(missing_ok=True) + print("Cleaning up...") + if 'agent_process' in locals() and agent_process is not None: + agent_process.terminate() + time.sleep(3) + + # Clean up only if not suppressed + if not args.no_cleanup: + if produced_mch: + shutil.rmtree(tmp_dir, ignore_errors=True) - # Optionally copy the resulting MCH to the specified output file - if args.output: - mch_name = "crank.mch" - out_path = Path(args.output) - if out_path.exists() and out_path.is_dir(): - print(f"Error: --output points to a directory, expected a file path: {out_path}", file=sys.stderr) - sys.exit(2) - # Ensure the destination directory exists - if out_path.parent and not out_path.parent.exists(): - out_path.parent.mkdir(parents=True, exist_ok=True) - print(f"Copying {mch_name} -> {out_path}") - shutil.copyfile(mch_name, out_path) - - # Clean up the temp crank directory at the very end - try: - shutil.rmtree(temp_root, ignore_errors=True) - except Exception: - pass + print("Done!") if __name__ == "__main__": main() From be94b36b339a8b20267374c97314bcec1fabf5bb Mon Sep 17 00:00:00 2001 From: EgorBo Date: Tue, 26 Aug 2025 23:36:01 +0200 Subject: [PATCH 04/48] fix yml --- eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml index 8627f692e90865..d96766edbc2c5b 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml @@ -145,7 +145,7 @@ jobs: # Direct run for aspnet2 collection: execute the repo script and produce the final .mch without Helix - ${{ if eq(parameters.collectionName, 'aspnet2') }}: - - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_aspnet2.py --runtime-bits-path $(Core_Root_Dir) --output $(MergedMchFileLocation)$(CollectionName).$(CollectionType).$(MchFileTag).mch + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_aspnet2.py --core_root $(Core_Root_Dir) --output_mch $(MergedMchFileLocation)$(CollectionName).$(CollectionType).$(MchFileTag).mch displayName: Run aspnet2 SuperPMI collection # Run superpmi collection in helix From f22e4cbfa1978f89ae750941aef9344a11251cfc Mon Sep 17 00:00:00 2001 From: EgorBo Date: Wed, 27 Aug 2025 23:42:45 +0200 Subject: [PATCH 05/48] =?UTF-8?q?=D0=B5=D1=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../templates/run-superpmi-collect-job.yml | 64 ++++++++----------- src/coreclr/scripts/superpmi-collect.proj | 25 ++++++-- src/coreclr/scripts/superpmi_aspnet2.py | 7 +- src/coreclr/scripts/superpmi_collect_setup.py | 9 ++- 4 files changed, 62 insertions(+), 43 deletions(-) diff --git a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml index 081cf003410750..7ff0fa4d1d366f 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml @@ -131,9 +131,8 @@ jobs: displayName: Enable python venv condition: always() - - ${{ if ne(parameters.collectionName, 'aspnet2') }}: - - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_collect_setup.py -payload_directory $(PayloadLocation) -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -release_core_root_directory $(Release_Core_Root_Dir) -arch $(archType) -platform $(osGroup) -mch_file_tag $(MchFileTag) -input_directory $(InputDirectory) -collection_name $(CollectionName) -collection_type $(CollectionType) $(PublicQueuesCLIArg) -max_size 25 # size in MB - displayName: ${{ format('SuperPMI setup ({0})', parameters.osGroup) }} + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_collect_setup.py -payload_directory $(PayloadLocation) -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -release_core_root_directory $(Release_Core_Root_Dir) -arch $(archType) -platform $(osGroup) -mch_file_tag $(MchFileTag) -input_directory $(InputDirectory) -collection_name $(CollectionName) -collection_type $(CollectionType) $(PublicQueuesCLIArg) -max_size 25 # size in MB + displayName: ${{ format('SuperPMI setup ({0})', parameters.osGroup) }} # Create required directories for merged mch collection and superpmi logs - ${{ if ne(parameters.osGroup, 'windows') }}: @@ -147,43 +146,36 @@ jobs: mkdir $(SpmiLogsLocation) displayName: Create directories - # Direct run for aspnet2 collection: execute the repo script and produce the final .mch without Helix - - ${{ if eq(parameters.collectionName, 'aspnet2') }}: - - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_aspnet2.py --core_root $(Core_Root_Dir) --output_mch $(MergedMchFileLocation)$(CollectionName).$(CollectionType).$(MchFileTag).mch - displayName: Run aspnet2 SuperPMI collection - # Run superpmi collection in helix - - ${{ if ne(parameters.collectionName, 'aspnet2') }}: - - template: /eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml - parameters: - HelixSource: '$(HelixSourcePrefix)/$(Build.Repository.Name)/$(Build.SourceBranch)' # sources must start with pr/, official/, prodcon/, or agent/ - HelixType: 'test/superpmi/$(CollectionName)/$(CollectionType)/$(Architecture)' - HelixAccessToken: $(HelixApiAccessToken) - HelixTargetQueues: $(Queue) - HelixPreCommands: $(HelixPreCommand) - - ${{ if ne(variables['System.TeamProject'], 'internal') }}: - Creator: $(Build.DefinitionName) - - WorkItemTimeout: 4:00 # 4 hours - WorkItemDirectory: '$(WorkItemDirectory)' - CorrelationPayloadDirectory: '$(CorrelationPayloadDirectory)' - ProjectFile: 'superpmi-collect.proj' - BuildConfig: ${{ parameters.buildConfig }} - osGroup: ${{ parameters.osGroup }} - archType: ${{ parameters.archType }} - InputArtifacts: '$(InputArtifacts)' - CollectionType: '$(CollectionType)' - CollectionName: '$(CollectionName)' - continueOnError: true # Run the future step i.e. merge-mch step even if this step fails. + - template: /eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml + parameters: + HelixSource: '$(HelixSourcePrefix)/$(Build.Repository.Name)/$(Build.SourceBranch)' # sources must start with pr/, official/, prodcon/, or agent/ + HelixType: 'test/superpmi/$(CollectionName)/$(CollectionType)/$(Architecture)' + HelixAccessToken: $(HelixApiAccessToken) + HelixTargetQueues: $(Queue) + HelixPreCommands: $(HelixPreCommand) + + ${{ if ne(variables['System.TeamProject'], 'internal') }}: + Creator: $(Build.DefinitionName) + + WorkItemTimeout: 4:00 # 4 hours + WorkItemDirectory: '$(WorkItemDirectory)' + CorrelationPayloadDirectory: '$(CorrelationPayloadDirectory)' + ProjectFile: 'superpmi-collect.proj' + BuildConfig: ${{ parameters.buildConfig }} + osGroup: ${{ parameters.osGroup }} + archType: ${{ parameters.archType }} + InputArtifacts: '$(InputArtifacts)' + CollectionType: '$(CollectionType)' + CollectionName: '$(CollectionName)' + continueOnError: true # Run the future step i.e. merge-mch step even if this step fails. # Always run merge step even if collection of some partition fails so we can store collection # of the partitions that succeeded. If all the partitions fail, merge-mch would fail and we won't # run future steps like uploading superpmi collection. - - ${{ if ne(parameters.collectionName, 'aspnet2') }}: - - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py merge-mch --ci -log_level DEBUG -pattern $(MchFilesLocation)$(CollectionName).$(CollectionType)*.mch -output_mch_path $(MergedMchFileLocation)$(CollectionName).$(CollectionType).$(MchFileTag).mch - displayName: ${{ format('Merge {0}-{1} SuperPMI collections', parameters.collectionName, parameters.collectionType) }} - condition: always() + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py merge-mch --ci -log_level DEBUG -pattern $(MchFilesLocation)$(CollectionName).$(CollectionType)*.mch -output_mch_path $(MergedMchFileLocation)$(CollectionName).$(CollectionType).$(MchFileTag).mch + displayName: ${{ format('Merge {0}-{1} SuperPMI collections', parameters.collectionName, parameters.collectionType) }} + condition: always() # If merge step above fails, then skip "Upload as artifact" and "Upload to Azure storage" - template: /eng/pipelines/common/upload-artifact-step.yml @@ -240,4 +232,4 @@ jobs: targetPath: $(Build.SourcesDirectory)/artifacts/log artifactName: 'SuperPMI_BuildLogs_$(CollectionName)_$(CollectionType)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_Attempt$(System.JobAttempt)' condition: always() - continueOnError: true + continueOnError: true \ No newline at end of file diff --git a/src/coreclr/scripts/superpmi-collect.proj b/src/coreclr/scripts/superpmi-collect.proj index fba11793645e20..1fab09bcaf64bf 100644 --- a/src/coreclr/scripts/superpmi-collect.proj +++ b/src/coreclr/scripts/superpmi-collect.proj @@ -68,10 +68,10 @@ - + %HELIX_WORKITEM_PAYLOAD%\binaries - + $HELIX_WORKITEM_PAYLOAD/binaries @@ -138,7 +138,7 @@ - + $(Python) $(SuperPMIDirectory)$(FileSeparatorChar)superpmi.py collect --clean -log_level DEBUG --$(CollectionType) $(PmiArguments) $(InputKind) $(AssembliesDirectoryOnHelix) -arch $(Architecture) -build_type $(BuildConfig) -core_root $(SuperPMIDirectory) 2:00 @@ -148,6 +148,11 @@ 3:00 + + $(Python) $(SuperPMIDirectory)$(FileSeparatorChar)superpmi_aspnet2.py --core_root $(SuperPMIDirectory) --output_mch $(OutputMchPath)$(FileSeparatorChar)$(CollectionName).$(CollectionType).$(MchFileTag).mch + 3:00 + + false false @@ -197,7 +202,7 @@ - + @@ -264,7 +269,7 @@ - + $(CollectionName).$(CollectionType).%(HelixWorkItem.PartitionId).$(MchFileTag) $(AssembliesPayload)$(FileSeparatorChar)%(HelixWorkItem.CollectAssemblies) @@ -292,5 +297,15 @@ $(WorkItemTimeout) %(OutputFileName).mch;%(OutputFileName).mch.mct;%(OutputFileName).log + + + + + $(CollectionName).$(CollectionType).0.$(MchFileTag) + $(WorkItemDirectory) + $(WorkItemCommand) + $(WorkItemTimeout) + $(CollectionName).$(CollectionType).$(MchFileTag).mch + diff --git a/src/coreclr/scripts/superpmi_aspnet2.py b/src/coreclr/scripts/superpmi_aspnet2.py index 8119d4a2518cb0..5cfbc05193fcd2 100644 --- a/src/coreclr/scripts/superpmi_aspnet2.py +++ b/src/coreclr/scripts/superpmi_aspnet2.py @@ -217,7 +217,7 @@ def build_crank_command(framework: str, runtime_bits_path: Path, scenario: str, # Main entry point def main(): - parser = argparse.ArgumentParser(description="Cross-platform crank runner with 1 retry (kills its own crank-agent).") + parser = argparse.ArgumentParser(description="Cross-platform crank runner.") # Renamed args parser.add_argument("--core_root", required=True, help="Path to built runtime bits (CORE_ROOT).") parser.add_argument("--tfm", default="net10.0", help="Target Framework Moniker (e.g., net10.0).") @@ -231,6 +231,11 @@ def main(): runtime_bits_path = Path(args.core_root).expanduser().resolve() output_mch_path = Path(args.output_mch).expanduser().resolve() + print("Running the script with the following parameters:") + print(f"--core_root: {runtime_bits_path}") + print(f"--tfm: {args.tfm}") + print(f"--output_mch: {output_mch_path}") + mcs_cmd = runtime_bits_path / ("mcs.exe" if sys.platform == "win32" else "mcs") if not mcs_cmd.exists(): print(f"Error: mcs.exe not found at {mcs_cmd}. Ensure runtime bits include mcs.", file=sys.stderr) diff --git a/src/coreclr/scripts/superpmi_collect_setup.py b/src/coreclr/scripts/superpmi_collect_setup.py index 3a9c87dc076493..232799f79202c0 100644 --- a/src/coreclr/scripts/superpmi_collect_setup.py +++ b/src/coreclr/scripts/superpmi_collect_setup.py @@ -44,7 +44,7 @@ parser = argparse.ArgumentParser(description="description") parser.add_argument("-collection_type", required=True, help="Type of the SPMI collection to be done (nativeaot, crossgen2, pmi, run, run_tiered, run_pgo, run_pgo_optrepeat)") -parser.add_argument("-collection_name", required=True, help="Name of the SPMI collection to be done (e.g., libraries, libraries_tests, coreclr_tests, benchmarks)") +parser.add_argument("-collection_name", required=True, help="Name of the SPMI collection to be done (e.g., libraries, libraries_tests, coreclr_tests, benchmarks, aspnet2)") parser.add_argument("-payload_directory", required=True, help="Path to payload directory to create: subdirectories are created for the correlation payload as well as the per-partition work items") parser.add_argument("-source_directory", required=True, help="Path to source directory") parser.add_argument("-core_root_directory", required=True, help="Path to Core_Root directory") @@ -523,6 +523,10 @@ def main(main_args): jitname = determine_jit_name(coreclr_args.platform, coreclr_args.platform, coreclr_args.arch, coreclr_args.arch) print('Copying checked {} -> {}'.format(jitname, core_root_dst_directory)) copy_files(coreclr_args.core_root_directory, core_root_dst_directory, [os.path.join(coreclr_args.core_root_directory, jitname)]) + elif coreclr_args.collection_name == "aspnet2": + # For aspnet2, use checked runtime bits + print('Copying {} -> {}'.format(coreclr_args.core_root_directory, core_root_dst_directory)) + copy_directory(coreclr_args.core_root_directory, core_root_dst_directory, verbose_output=True, match_func=acceptable_copy) else: print('Copying {} -> {}'.format(coreclr_args.core_root_directory, core_root_dst_directory)) copy_directory(coreclr_args.core_root_directory, core_root_dst_directory, verbose_output=True, match_func=acceptable_copy) @@ -530,6 +534,9 @@ def main(main_args): if coreclr_args.collection_name == "benchmarks" or coreclr_args.collection_name == "realworld": # Setup benchmarks setup_benchmark(workitem_payload_directory, arch) + elif coreclr_args.collection_name == "aspnet2": + # Setup aspnet2 - no special payload setup needed, the script handles everything + pass else: # Setup for pmi/crossgen2/nativeaot runs From e649a4c1fa66779668c47426878a2b919f62a23c Mon Sep 17 00:00:00 2001 From: EgorBo Date: Thu, 28 Aug 2025 00:19:50 +0200 Subject: [PATCH 06/48] f --- src/coreclr/scripts/superpmi-collect.proj | 8 ++++---- src/coreclr/scripts/superpmi_collect_setup.py | 7 +++++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/coreclr/scripts/superpmi-collect.proj b/src/coreclr/scripts/superpmi-collect.proj index 1fab09bcaf64bf..fa0c31a6cf14a6 100644 --- a/src/coreclr/scripts/superpmi-collect.proj +++ b/src/coreclr/scripts/superpmi-collect.proj @@ -149,7 +149,7 @@ - $(Python) $(SuperPMIDirectory)$(FileSeparatorChar)superpmi_aspnet2.py --core_root $(SuperPMIDirectory) --output_mch $(OutputMchPath)$(FileSeparatorChar)$(CollectionName).$(CollectionType).$(MchFileTag).mch + $(Python) $(SuperPMIDirectory)$(FileSeparatorChar)superpmi_aspnet2.py --core_root $(SuperPMIDirectory) 3:00 @@ -300,12 +300,12 @@ - + $(CollectionName).$(CollectionType).0.$(MchFileTag) $(WorkItemDirectory) - $(WorkItemCommand) + $(WorkItemCommand) --output_mch $(OutputFileName).mch $(WorkItemTimeout) - $(CollectionName).$(CollectionType).$(MchFileTag).mch + $(OutputFileName).mch diff --git a/src/coreclr/scripts/superpmi_collect_setup.py b/src/coreclr/scripts/superpmi_collect_setup.py index 232799f79202c0..924f53bdda9d3e 100644 --- a/src/coreclr/scripts/superpmi_collect_setup.py +++ b/src/coreclr/scripts/superpmi_collect_setup.py @@ -525,8 +525,11 @@ def main(main_args): copy_files(coreclr_args.core_root_directory, core_root_dst_directory, [os.path.join(coreclr_args.core_root_directory, jitname)]) elif coreclr_args.collection_name == "aspnet2": # For aspnet2, use checked runtime bits - print('Copying {} -> {}'.format(coreclr_args.core_root_directory, core_root_dst_directory)) - copy_directory(coreclr_args.core_root_directory, core_root_dst_directory, verbose_output=True, match_func=acceptable_copy) + print('Copying {} -> {}'.format(coreclr_args.release_core_root_directory, core_root_dst_directory)) + copy_directory(coreclr_args.release_core_root_directory, core_root_dst_directory, verbose_output=True, match_func=acceptable_copy) + jitname = determine_jit_name(coreclr_args.platform, coreclr_args.platform, coreclr_args.arch, coreclr_args.arch) + print('Copying checked {} -> {}'.format(jitname, core_root_dst_directory)) + copy_files(coreclr_args.core_root_directory, core_root_dst_directory, [os.path.join(coreclr_args.core_root_directory, jitname)]) else: print('Copying {} -> {}'.format(coreclr_args.core_root_directory, core_root_dst_directory)) copy_directory(coreclr_args.core_root_directory, core_root_dst_directory, verbose_output=True, match_func=acceptable_copy) From 25470bf8f55f901024375f1d920a6a638f660297 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Thu, 28 Aug 2025 01:56:22 +0200 Subject: [PATCH 07/48] test --- src/coreclr/scripts/superpmi-collect.proj | 15 +++++++++++---- src/coreclr/scripts/superpmi_collect_setup.py | 2 +- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/coreclr/scripts/superpmi-collect.proj b/src/coreclr/scripts/superpmi-collect.proj index fa0c31a6cf14a6..e29c5099f8cbef 100644 --- a/src/coreclr/scripts/superpmi-collect.proj +++ b/src/coreclr/scripts/superpmi-collect.proj @@ -267,7 +267,14 @@ - + + + + 1 + + + + @@ -300,12 +307,12 @@ - - $(CollectionName).$(CollectionType).0.$(MchFileTag) + + $(CollectionName).$(CollectionType).%(HelixWorkItem.Index).$(MchFileTag) $(WorkItemDirectory) $(WorkItemCommand) --output_mch $(OutputFileName).mch $(WorkItemTimeout) $(OutputFileName).mch - + diff --git a/src/coreclr/scripts/superpmi_collect_setup.py b/src/coreclr/scripts/superpmi_collect_setup.py index 924f53bdda9d3e..e4c6a0de658361 100644 --- a/src/coreclr/scripts/superpmi_collect_setup.py +++ b/src/coreclr/scripts/superpmi_collect_setup.py @@ -539,7 +539,7 @@ def main(main_args): setup_benchmark(workitem_payload_directory, arch) elif coreclr_args.collection_name == "aspnet2": # Setup aspnet2 - no special payload setup needed, the script handles everything - pass + printff("No special setup required for aspnet2") else: # Setup for pmi/crossgen2/nativeaot runs From 68de0993d1e5194fed287d0ebd9ee58599d129ea Mon Sep 17 00:00:00 2001 From: Egor Bogatov Date: Thu, 28 Aug 2025 03:05:22 +0200 Subject: [PATCH 08/48] Update superpmi_collect_setup.py --- src/coreclr/scripts/superpmi_collect_setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/scripts/superpmi_collect_setup.py b/src/coreclr/scripts/superpmi_collect_setup.py index e4c6a0de658361..ec985affc05d0a 100644 --- a/src/coreclr/scripts/superpmi_collect_setup.py +++ b/src/coreclr/scripts/superpmi_collect_setup.py @@ -539,7 +539,7 @@ def main(main_args): setup_benchmark(workitem_payload_directory, arch) elif coreclr_args.collection_name == "aspnet2": # Setup aspnet2 - no special payload setup needed, the script handles everything - printff("No special setup required for aspnet2") + print("No special setup required for aspnet2") else: # Setup for pmi/crossgen2/nativeaot runs From a0edaa050b0ab4440b0ca9d70c8720841d5e7949 Mon Sep 17 00:00:00 2001 From: Egor Bogatov Date: Thu, 28 Aug 2025 07:49:50 +0200 Subject: [PATCH 09/48] Update superpmi_collect_setup.py --- src/coreclr/scripts/superpmi_collect_setup.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/coreclr/scripts/superpmi_collect_setup.py b/src/coreclr/scripts/superpmi_collect_setup.py index ec985affc05d0a..a5fe3876099f9c 100644 --- a/src/coreclr/scripts/superpmi_collect_setup.py +++ b/src/coreclr/scripts/superpmi_collect_setup.py @@ -534,12 +534,9 @@ def main(main_args): print('Copying {} -> {}'.format(coreclr_args.core_root_directory, core_root_dst_directory)) copy_directory(coreclr_args.core_root_directory, core_root_dst_directory, verbose_output=True, match_func=acceptable_copy) - if coreclr_args.collection_name == "benchmarks" or coreclr_args.collection_name == "realworld": + if coreclr_args.collection_name == "benchmarks" or coreclr_args.collection_name == "realworld" or coreclr_args.collection_name == "aspnet2": # Setup benchmarks setup_benchmark(workitem_payload_directory, arch) - elif coreclr_args.collection_name == "aspnet2": - # Setup aspnet2 - no special payload setup needed, the script handles everything - print("No special setup required for aspnet2") else: # Setup for pmi/crossgen2/nativeaot runs From d28f960a8d38a80a6840112869d41bed5ba11b47 Mon Sep 17 00:00:00 2001 From: Egor Bogatov Date: Thu, 28 Aug 2025 10:15:15 +0200 Subject: [PATCH 10/48] Update superpmi-collect-pipeline.yml --- eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml b/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml index 868d1a2838e24e..af7380549895a5 100644 --- a/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml +++ b/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml @@ -261,7 +261,7 @@ extends: jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml buildConfig: checked platforms: - - linux_x64 + - windows_x64 helixQueueGroup: ci helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: From 6516b5fc9574844a62ed3b6995f510f259a564cf Mon Sep 17 00:00:00 2001 From: Egor Bogatov Date: Thu, 28 Aug 2025 10:17:07 +0200 Subject: [PATCH 11/48] Fix placeholder in DownloadFilesFromResults --- src/coreclr/scripts/superpmi-collect.proj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/scripts/superpmi-collect.proj b/src/coreclr/scripts/superpmi-collect.proj index e29c5099f8cbef..fa6e8ba74aaa32 100644 --- a/src/coreclr/scripts/superpmi-collect.proj +++ b/src/coreclr/scripts/superpmi-collect.proj @@ -312,7 +312,7 @@ $(WorkItemDirectory) $(WorkItemCommand) --output_mch $(OutputFileName).mch $(WorkItemTimeout) - $(OutputFileName).mch + %(OutputFileName).mch From 80e1f558e39c0a54fafe31bbd0c5bf64b06a0d7e Mon Sep 17 00:00:00 2001 From: EgorBo Date: Thu, 28 Aug 2025 12:01:00 +0200 Subject: [PATCH 12/48] small fixes --- src/coreclr/scripts/superpmi_aspnet2.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/coreclr/scripts/superpmi_aspnet2.py b/src/coreclr/scripts/superpmi_aspnet2.py index 5cfbc05193fcd2..62e96b8ef9d73c 100644 --- a/src/coreclr/scripts/superpmi_aspnet2.py +++ b/src/coreclr/scripts/superpmi_aspnet2.py @@ -93,7 +93,9 @@ def ensure_tools_and_localhost_yaml(workdir: Path, port: int): localhost_yml = data_dir / "Localhost.yml" os.environ['DOTNET_ROOT'] = str(dotnethome_dir) - os.environ['DOTNET_MULTILEVEL_LOOKUP'] = "1" + os.environ['DOTNET_CLI_TELEMETRY_OPTOUT'] = '1' + os.environ['DOTNET_MULTILEVEL_LOOKUP'] = '0' + os.environ['UseSharedCompilation'] = 'false' os.environ["PATH"] = str(tools_dir) + os.pathsep + os.environ.get("PATH", "") if not data_dir.exists(): @@ -109,7 +111,7 @@ def ensure_tools_and_localhost_yaml(workdir: Path, port: int): path = os.path.join(tmp, os.path.basename(url)) urllib.request.urlretrieve(url, path) if url.endswith(".ps1"): - subprocess.check_call(["powershell","-ExecutionPolicy","Bypass","-File",path, + subprocess.check_call(["powershell.exe","-NoProfile", "-ExecutionPolicy","Bypass","-File",path, "-Channel","8.0","-InstallDir", str(dotnethome_dir)]) else: os.chmod(path,0o755) From dd34996752c8dc4f9687d013514b9f86e1c26f50 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Thu, 28 Aug 2025 12:03:56 +0200 Subject: [PATCH 13/48] find ps1 --- src/coreclr/scripts/superpmi_aspnet2.py | 33 +++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/src/coreclr/scripts/superpmi_aspnet2.py b/src/coreclr/scripts/superpmi_aspnet2.py index 62e96b8ef9d73c..64734e1333d9a8 100644 --- a/src/coreclr/scripts/superpmi_aspnet2.py +++ b/src/coreclr/scripts/superpmi_aspnet2.py @@ -61,6 +61,21 @@ def native_dll(name: str) -> str: return f"{prefix}{name}{ext}" +# Helper function to find available PowerShell executable +def find_powershell(): + """Find available PowerShell executable, preferring pwsh over powershell.exe""" + # Try pwsh first (PowerShell Core, more modern and cross-platform) + if shutil.which("pwsh"): + return "pwsh" + # Fall back to Windows PowerShell + if shutil.which("powershell.exe"): + return "powershell.exe" + # Last resort, try just "powershell" + if shutil.which("powershell"): + return "powershell" + raise FileNotFoundError("No PowerShell executable found. Please install PowerShell Core (pwsh) or Windows PowerShell.") + + # Run a command def run(cmd, cwd=None): print(f"Running command: {' '.join(map(str, cmd))}") @@ -111,8 +126,22 @@ def ensure_tools_and_localhost_yaml(workdir: Path, port: int): path = os.path.join(tmp, os.path.basename(url)) urllib.request.urlretrieve(url, path) if url.endswith(".ps1"): - subprocess.check_call(["powershell.exe","-NoProfile", "-ExecutionPolicy","Bypass","-File",path, - "-Channel","8.0","-InstallDir", str(dotnethome_dir)]) + # Find available PowerShell executable + powershell_exe = find_powershell() + print(f"Using PowerShell executable: {powershell_exe}") + + # Add better error handling and capture output + try: + result = subprocess.run([ + powershell_exe, "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", path, + "-Channel", "8.0", "-InstallDir", str(dotnethome_dir) + ], capture_output=True, text=True, check=True) + print(f"PowerShell output: {result.stdout}") + except subprocess.CalledProcessError as e: + print(f"PowerShell script failed with exit code {e.returncode}") + print(f"Error output: {e.stderr}") + print(f"Standard output: {e.stdout}") + raise else: os.chmod(path,0o755) subprocess.check_call([path,"-Channel","8.0","-InstallDir", str(dotnethome_dir)]) From 8cae35f6d272ad5c2690743a8e379da51e374b36 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Thu, 28 Aug 2025 12:10:30 +0200 Subject: [PATCH 14/48] disable the benchmark for now --- src/coreclr/scripts/superpmi_aspnet2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/scripts/superpmi_aspnet2.py b/src/coreclr/scripts/superpmi_aspnet2.py index 64734e1333d9a8..b1db22f66782dd 100644 --- a/src/coreclr/scripts/superpmi_aspnet2.py +++ b/src/coreclr/scripts/superpmi_aspnet2.py @@ -230,7 +230,7 @@ def build_crank_command(framework: str, runtime_bits_path: Path, scenario: str, "--application.options.collectCounters", "false", "--application.collectDependencies", "false", "--load.options.reuseBuild", "true", - "--load.variables.duration", "5", # default 15s is not enough for Tier1 promotion + "--load.variables.duration", "10", # default 15s is not enough for Tier1 promotion "--load.job", "bombardier", # Bombardier is more cross-platform friendly (wrk is linux only) "--application.environmentVariables", f"COMPlus_JitName={spmi_shim}", "--application.environmentVariables", "SuperPMIShimLogPath=.", @@ -301,7 +301,7 @@ def main(): run(build_crank_command(framework=args.tfm, runtime_bits_path=runtime_bits_path, scenario="mvc", config_path=config_path)) print("### Running NoMvcAuth benchmark... ###") - run(build_crank_command(framework=args.tfm, runtime_bits_path=runtime_bits_path, scenario="NoMvcAuth", config_path=config_path)) + # run(build_crank_command(framework=args.tfm, runtime_bits_path=runtime_bits_path, scenario="NoMvcAuth", config_path=config_path)) print("Finished running benchmarks.") From 4ddd3656661ae10bfdffc116c2e7b36e37282143 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Thu, 28 Aug 2025 12:14:12 +0200 Subject: [PATCH 15/48] enable on all targets --- .../coreclr/templates/superpmi-collect-pipeline.yml | 4 ++++ src/coreclr/scripts/superpmi_collect_setup.py | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml b/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml index af7380549895a5..78871b211a98e5 100644 --- a/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml +++ b/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml @@ -261,7 +261,11 @@ extends: jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml buildConfig: checked platforms: + - osx_arm64 + - linux_arm64 + - linux_x64 - windows_x64 + - windows_arm64 helixQueueGroup: ci helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: diff --git a/src/coreclr/scripts/superpmi_collect_setup.py b/src/coreclr/scripts/superpmi_collect_setup.py index a5fe3876099f9c..969a2afe738585 100644 --- a/src/coreclr/scripts/superpmi_collect_setup.py +++ b/src/coreclr/scripts/superpmi_collect_setup.py @@ -534,9 +534,13 @@ def main(main_args): print('Copying {} -> {}'.format(coreclr_args.core_root_directory, core_root_dst_directory)) copy_directory(coreclr_args.core_root_directory, core_root_dst_directory, verbose_output=True, match_func=acceptable_copy) - if coreclr_args.collection_name == "benchmarks" or coreclr_args.collection_name == "realworld" or coreclr_args.collection_name == "aspnet2": + if coreclr_args.collection_name == "benchmarks" or coreclr_args.collection_name == "realworld": # Setup benchmarks setup_benchmark(workitem_payload_directory, arch) + elif coreclr_args.collection_name == "aspnet2": + # Nothing to prepare for aspnet2, its script is fully self-contained. + # Just make sure workitem_payload_directory directory exists. + os.makedirs(workitem_payload_directory, exist_ok=True) else: # Setup for pmi/crossgen2/nativeaot runs From 11010a64636bea5bd5a053acfe56ebf24cb3cb96 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Thu, 28 Aug 2025 13:41:55 +0200 Subject: [PATCH 16/48] introduce --cli arg --- src/coreclr/scripts/superpmi_aspnet2.py | 110 ++++++++++++------------ 1 file changed, 57 insertions(+), 53 deletions(-) diff --git a/src/coreclr/scripts/superpmi_aspnet2.py b/src/coreclr/scripts/superpmi_aspnet2.py index b1db22f66782dd..d2adb4e2395d67 100644 --- a/src/coreclr/scripts/superpmi_aspnet2.py +++ b/src/coreclr/scripts/superpmi_aspnet2.py @@ -99,7 +99,7 @@ def run(cmd, cwd=None): # Ensure .NET tools are installed and Localhost.yml is present -def ensure_tools_and_localhost_yaml(workdir: Path, port: int): +def ensure_tools_and_localhost_yaml(workdir: Path, port: int, cli=None): data_dir = workdir / "crank_data" logs_dir = data_dir / "logs" build_dir = data_dir / "build" @@ -107,7 +107,9 @@ def ensure_tools_and_localhost_yaml(workdir: Path, port: int): dotnethome_dir = data_dir / "dotnet_home" localhost_yml = data_dir / "Localhost.yml" - os.environ['DOTNET_ROOT'] = str(dotnethome_dir) + # If a CLI path is provided, use it as DOTNET_ROOT; otherwise default to our local dotnet_home + dotnet_root_dir = cli if cli else dotnethome_dir + os.environ['DOTNET_ROOT'] = str(dotnet_root_dir) os.environ['DOTNET_CLI_TELEMETRY_OPTOUT'] = '1' os.environ['DOTNET_MULTILEVEL_LOOKUP'] = '0' os.environ['UseSharedCompilation'] = 'false' @@ -120,33 +122,39 @@ def ensure_tools_and_localhost_yaml(workdir: Path, port: int): dotnethome_dir.mkdir(parents=True, exist_ok=True) tools_dir.mkdir(parents=True, exist_ok=True) - # Install .NET 8.0 needed for crank and crank-agent via dotnet-install public script. - url = "https://dot.net/v1/dotnet-install." + ("ps1" if platform.system()=="Windows" else "sh") - with tempfile.TemporaryDirectory() as tmp: - path = os.path.join(tmp, os.path.basename(url)) - urllib.request.urlretrieve(url, path) - if url.endswith(".ps1"): - # Find available PowerShell executable - powershell_exe = find_powershell() - print(f"Using PowerShell executable: {powershell_exe}") - - # Add better error handling and capture output - try: - result = subprocess.run([ - powershell_exe, "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", path, - "-Channel", "8.0", "-InstallDir", str(dotnethome_dir) - ], capture_output=True, text=True, check=True) - print(f"PowerShell output: {result.stdout}") - except subprocess.CalledProcessError as e: - print(f"PowerShell script failed with exit code {e.returncode}") - print(f"Error output: {e.stderr}") - print(f"Standard output: {e.stdout}") - raise - else: - os.chmod(path,0o755) - subprocess.check_call([path,"-Channel","8.0","-InstallDir", str(dotnethome_dir)]) - - dotnet_exe = dotnethome_dir / "dotnet" + # If a CLI path was provided, skip installing .NET and use that SDK. + if cli is None: + # Install .NET 8.0 needed for crank and crank-agent via dotnet-install public script. + url = "https://dot.net/v1/dotnet-install." + ("ps1" if platform.system()=="Windows" else "sh") + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, os.path.basename(url)) + urllib.request.urlretrieve(url, path) + if url.endswith(".ps1"): + # Find available PowerShell executable + powershell_exe = find_powershell() + print(f"Using PowerShell executable: {powershell_exe}") + + # Add better error handling and capture output + try: + result = subprocess.run([ + powershell_exe, "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", path, + "-Channel", "8.0", "-InstallDir", str(dotnethome_dir) + ], capture_output=True, text=True, check=True) + print(f"PowerShell output: {result.stdout}") + except subprocess.CalledProcessError as e: + print(f"PowerShell script failed with exit code {e.returncode}") + print(f"Error output: {e.stderr}") + print(f"Standard output: {e.stdout}") + raise + else: + os.chmod(path,0o755) + subprocess.check_call([path,"-Channel","8.0","-InstallDir", str(dotnethome_dir)]) + else: + print(f"Using existing .NET SDK at: {dotnet_root_dir}") + + # Determine the dotnet executable to use for installing tools + dotnet_file = "dotnet.exe" if sys.platform == "win32" else "dotnet" + dotnet_exe = dotnet_root_dir / dotnet_file run([dotnet_exe, "tool", "install", "--tool-path", str(tools_dir), "Microsoft.Crank.Agent", "--version", "0.2.0-*"], cwd=dotnethome_dir) run([dotnet_exe, "tool", "install", "--tool-path", str(tools_dir), "Microsoft.Crank.Controller", "--version", "0.2.0-*"], cwd=dotnethome_dir) @@ -186,8 +194,10 @@ def start_crank_agent(workdir: Path, port: int): print("crank-agent is not running yet. Starting...") logs_dir = workdir / "crank_data" / "logs" build_dir = workdir / "crank_data" / "build" - # Keep naming consistent with ensure_tools_and_localhost_yaml() - dotnethome_dir = workdir / "crank_data" / "dotnet_home" + # Prefer DOTNET_ROOT if set (configured by ensure_tools_and_localhost_yaml), + # otherwise fall back to the local dotnet_home + env_dotnet_root = os.environ.get('DOTNET_ROOT') + dotnethome_dir = Path(env_dotnet_root) if env_dotnet_root else (workdir / "crank_data" / "dotnet_home") agent_process = subprocess.Popen( [ @@ -257,15 +267,21 @@ def main(): # New args parser.add_argument("--work_dir", help="Optional work directory; if not specified, a temp directory is used.") parser.add_argument("--no_cleanup", action="store_true", help="If specified, do not clean up temporary files after execution.") + parser.add_argument("--cli", help="Optional path to an existing .NET SDK root; if provided, DOTNET_ROOT will use this and .NET 8.0 will not be installed.") args = parser.parse_args() + repo_dir = Path.cwd() runtime_bits_path = Path(args.core_root).expanduser().resolve() output_mch_path = Path(args.output_mch).expanduser().resolve() + if args.cli: + args.cli = str(Path(args.cli).expanduser().resolve()) print("Running the script with the following parameters:") print(f"--core_root: {runtime_bits_path}") print(f"--tfm: {args.tfm}") print(f"--output_mch: {output_mch_path}") + if args.cli: + print(f"--cli: {args.cli}") mcs_cmd = runtime_bits_path / ("mcs.exe" if sys.platform == "win32" else "mcs") if not mcs_cmd.exists(): @@ -286,14 +302,15 @@ def main(): # Set current working directory to temp_root os.chdir(temp_root) - ensure_tools_and_localhost_yaml(temp_root, CRANK_PORT) + ensure_tools_and_localhost_yaml(temp_root, CRANK_PORT, cli=args.cli) try: agent_process = None agent_process = start_crank_agent(temp_root, CRANK_PORT) # Benchmarks - config_path = temp_root / "crank_data" / "Localhost.yml" + crank_data_path = temp_root / "crank_data" + config_path = crank_data_path / "Localhost.yml" # print("### Running OrchardCMS benchmark... ###") # run(build_crank_command(framework=args.tfm, runtime_bits_path=runtime_bits_path, scenario="about-sqlite", config_path=config_path)) @@ -308,10 +325,6 @@ def main(): # Extract .mc files from zip archives into crank_data/tmp instead of the current directory print("Extracting .mc files from zip archives...") - tmp_dir = temp_root / "crank_data" / "tmp" - if tmp_dir.exists(): - shutil.rmtree(tmp_dir, ignore_errors=True) - tmp_dir.mkdir(parents=True, exist_ok=True) produced_mch = False extracted_count = 0 for z in pathlib.Path('.').glob('*.crank.zip'): @@ -319,7 +332,7 @@ def main(): for name in f.namelist(): # include .mc files from any path inside the zip if name.endswith('.mc'): - f.extract(name, str(tmp_dir)) + f.extract(name, str(crank_data_path)) extracted_count += 1 z.unlink(missing_ok=True) @@ -336,12 +349,12 @@ def main(): "-dedup", "-thin", "crank.mch", - "." - ], check=True, cwd=str(tmp_dir)) + "*.mc" + ], check=True, cwd=str(crank_data_path)) # Move the produced crank.mch back to the workspace root print(f"Moving produced crank.mch to {repo_dir / 'crank.mch'}") - shutil.copyfile(tmp_dir / "crank.mch", repo_dir / "crank.mch") + shutil.copyfile(crank_data_path / "crank.mch", repo_dir / "crank.mch") produced_mch = True # Copy the resulting MCH to the specified output file @@ -359,26 +372,17 @@ def main(): out_path.parent.mkdir(parents=True, exist_ok=True) print(f"Copying {mch_src} -> {out_path}") shutil.copyfile(mch_src, out_path) - - # Clean up the temp crank directory at the very end - if not args.no_cleanup: - # Only delete the working directory if we created a temp one - if 'created_temp' in locals() and created_temp: - try: - shutil.rmtree(temp_root, ignore_errors=True) - except Exception: - pass finally: print("Cleaning up...") if 'agent_process' in locals() and agent_process is not None: agent_process.terminate() - time.sleep(3) + time.sleep(5) # Clean up only if not suppressed if not args.no_cleanup: - if produced_mch: - shutil.rmtree(tmp_dir, ignore_errors=True) + # remove the entire temp_root: + shutil.rmtree(temp_root, ignore_errors=True) print("Done!") From 110259b3b6617cee0f9493ef14a7f07097d64d95 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Thu, 28 Aug 2025 13:58:20 +0200 Subject: [PATCH 17/48] test --- src/coreclr/scripts/superpmi-collect.proj | 6 +++--- src/coreclr/scripts/superpmi_aspnet2.py | 6 +++--- src/coreclr/scripts/superpmi_collect_setup.py | 7 +++++-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/coreclr/scripts/superpmi-collect.proj b/src/coreclr/scripts/superpmi-collect.proj index fa6e8ba74aaa32..e34d1cac507438 100644 --- a/src/coreclr/scripts/superpmi-collect.proj +++ b/src/coreclr/scripts/superpmi-collect.proj @@ -77,10 +77,10 @@ - + %HELIX_WORKITEM_PAYLOAD%\performance - + $HELIX_WORKITEM_PAYLOAD/performance @@ -149,7 +149,7 @@ - $(Python) $(SuperPMIDirectory)$(FileSeparatorChar)superpmi_aspnet2.py --core_root $(SuperPMIDirectory) + $(Python) $(SuperPMIDirectory)$(FileSeparatorChar)superpmi_aspnet2.py --core_root $(SuperPMIDirectory) --cli $(PerformanceDirectory)$(FileSeparatorChar)tools$(FileSeparatorChar)dotnet 3:00 diff --git a/src/coreclr/scripts/superpmi_aspnet2.py b/src/coreclr/scripts/superpmi_aspnet2.py index d2adb4e2395d67..59954de94e7b72 100644 --- a/src/coreclr/scripts/superpmi_aspnet2.py +++ b/src/coreclr/scripts/superpmi_aspnet2.py @@ -108,7 +108,7 @@ def ensure_tools_and_localhost_yaml(workdir: Path, port: int, cli=None): localhost_yml = data_dir / "Localhost.yml" # If a CLI path is provided, use it as DOTNET_ROOT; otherwise default to our local dotnet_home - dotnet_root_dir = cli if cli else dotnethome_dir + dotnet_root_dir = Path(cli) if cli else dotnethome_dir os.environ['DOTNET_ROOT'] = str(dotnet_root_dir) os.environ['DOTNET_CLI_TELEMETRY_OPTOUT'] = '1' os.environ['DOTNET_MULTILEVEL_LOOKUP'] = '0' @@ -155,8 +155,8 @@ def ensure_tools_and_localhost_yaml(workdir: Path, port: int, cli=None): # Determine the dotnet executable to use for installing tools dotnet_file = "dotnet.exe" if sys.platform == "win32" else "dotnet" dotnet_exe = dotnet_root_dir / dotnet_file - run([dotnet_exe, "tool", "install", "--tool-path", str(tools_dir), "Microsoft.Crank.Agent", "--version", "0.2.0-*"], cwd=dotnethome_dir) - run([dotnet_exe, "tool", "install", "--tool-path", str(tools_dir), "Microsoft.Crank.Controller", "--version", "0.2.0-*"], cwd=dotnethome_dir) + run([dotnet_exe, "tool", "install", "--tool-path", str(tools_dir), "Microsoft.Crank.Agent", "--version", "0.2.0-*"]) + run([dotnet_exe, "tool", "install", "--tool-path", str(tools_dir), "Microsoft.Crank.Controller", "--version", "0.2.0-*"]) # Create a Localhost.yml to define the local environment yml = textwrap.dedent( diff --git a/src/coreclr/scripts/superpmi_collect_setup.py b/src/coreclr/scripts/superpmi_collect_setup.py index 969a2afe738585..0369f7f9a128d0 100644 --- a/src/coreclr/scripts/superpmi_collect_setup.py +++ b/src/coreclr/scripts/superpmi_collect_setup.py @@ -414,6 +414,9 @@ def setup_benchmark(workitem_directory, arch): run_command( get_python_name() + [dotnet_install_script, "install", "--channels", "10.0", "--architecture", arch, "--install-dir", dotnet_directory, "--verbose"]) + run_command( + get_python_name() + [dotnet_install_script, "install", "--channels", "8.0", "--architecture", arch, "--install-dir", + dotnet_directory, "--verbose"]) def get_python_name(): @@ -539,8 +542,8 @@ def main(main_args): setup_benchmark(workitem_payload_directory, arch) elif coreclr_args.collection_name == "aspnet2": # Nothing to prepare for aspnet2, its script is fully self-contained. - # Just make sure workitem_payload_directory directory exists. - os.makedirs(workitem_payload_directory, exist_ok=True) + # But we'll reuse the same setup_benchmark as for benchmarks and realworld. + setup_benchmark(workitem_payload_directory, arch) else: # Setup for pmi/crossgen2/nativeaot runs From c6399e04d8da4712ad3887eb31cea653facbd6fc Mon Sep 17 00:00:00 2001 From: EgorBo Date: Thu, 28 Aug 2025 14:11:12 +0200 Subject: [PATCH 18/48] use isExtraPlatformsBuild: true --- eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml b/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml index 78871b211a98e5..1d0d56248efee5 100644 --- a/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml +++ b/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml @@ -261,13 +261,13 @@ extends: jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml buildConfig: checked platforms: - - osx_arm64 - linux_arm64 - linux_x64 - windows_x64 - windows_arm64 helixQueueGroup: ci helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + isExtraPlatformsBuild: true jobParameters: testGroup: outerloop liveLibrariesBuildConfig: Release From caa0ce614e1c639952689017459a43d4230c5b17 Mon Sep 17 00:00:00 2001 From: Egor Bogatov Date: Thu, 28 Aug 2025 14:18:03 +0200 Subject: [PATCH 19/48] Update superpmi-collect-pipeline.yml --- eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml b/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml index 1d0d56248efee5..badfe4bd661517 100644 --- a/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml +++ b/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml @@ -267,7 +267,6 @@ extends: - windows_arm64 helixQueueGroup: ci helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - isExtraPlatformsBuild: true jobParameters: testGroup: outerloop liveLibrariesBuildConfig: Release From 02397b7adaf3da700bee7be555e9ec9495b1cf9a Mon Sep 17 00:00:00 2001 From: EgorBo Date: Thu, 28 Aug 2025 15:45:14 +0200 Subject: [PATCH 20/48] win --- .../templates/superpmi-collect-pipeline.yml | 2 - src/coreclr/scripts/superpmi_aspnet2.py | 39 +++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml b/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml index 1d0d56248efee5..4cf9b047fa29e1 100644 --- a/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml +++ b/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml @@ -261,8 +261,6 @@ extends: jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml buildConfig: checked platforms: - - linux_arm64 - - linux_x64 - windows_x64 - windows_arm64 helixQueueGroup: ci diff --git a/src/coreclr/scripts/superpmi_aspnet2.py b/src/coreclr/scripts/superpmi_aspnet2.py index 59954de94e7b72..cf39fe1ca25208 100644 --- a/src/coreclr/scripts/superpmi_aspnet2.py +++ b/src/coreclr/scripts/superpmi_aspnet2.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 import argparse import os +import re +import requests import socket import pathlib import subprocess @@ -14,6 +16,7 @@ import urllib.request from pathlib import Path + ########################################################################################################## # # This script sets up an environment for running crank-agent and crank-controller @@ -98,6 +101,40 @@ def run(cmd, cwd=None): return None +def get_arch() -> str: + m = platform.machine().lower() + if "arm64" in m: return "arm64" + if m in ("amd64", "x86_64"): return "x64" + raise RuntimeError(f"Unsupported arch: {m}") + + +def download_mingit_windows(dest: str) -> str: + m = {"x64": "64-bit", "arm64": "arm64"} + assets = requests.get("https://api.github.com/repos/git-for-windows/git/releases/latest",timeout=100).json()["assets"] + rx = re.compile(r"^MinGit-.*-(64-bit|arm64)\.zip$", re.I) + asset = next(a for a in assets if rx.match(a["name"]) and m[get_arch()] in a["name"]) + os.makedirs(dest, exist_ok=True); zip_path = os.path.join(dest, asset["name"]) + with requests.get(asset["browser_download_url"], stream=True) as r, open(zip_path,"wb") as f: + for c in r.iter_content(8192): f.write(c) + git_dir = os.path.join(dest,"git"); shutil.rmtree(git_dir, ignore_errors=True) + with zipfile.ZipFile(zip_path) as z: z.extractall(git_dir) + os.remove(zip_path) + return git_dir + + +def ensure_git(dest: Path) -> str: + # if shutil.which("git"): + # print("git found") + # return shutil.which("git") + if not sys.platform == "win32": + raise RuntimeError("Git is not available on this platform") + print("git not found, downloading portable git...") + git_dir = download_mingit_windows(str(dest)) + cmd_path = os.path.join(git_dir,"cmd") + os.environ["PATH"] = cmd_path + os.pathsep + os.environ["PATH"] + return shutil.which("git") + + # Ensure .NET tools are installed and Localhost.yml is present def ensure_tools_and_localhost_yaml(workdir: Path, port: int, cli=None): data_dir = workdir / "crank_data" @@ -122,6 +159,8 @@ def ensure_tools_and_localhost_yaml(workdir: Path, port: int, cli=None): dotnethome_dir.mkdir(parents=True, exist_ok=True) tools_dir.mkdir(parents=True, exist_ok=True) + ensure_git(tools_dir) + # If a CLI path was provided, skip installing .NET and use that SDK. if cli is None: # Install .NET 8.0 needed for crank and crank-agent via dotnet-install public script. From ac05eb6af2f9a40a837f5464680e45adb49c622e Mon Sep 17 00:00:00 2001 From: EgorBo Date: Thu, 28 Aug 2025 16:06:29 +0200 Subject: [PATCH 21/48] install git on AL3 --- src/coreclr/scripts/superpmi_aspnet2.py | 48 +++++++++++++++++++------ 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/src/coreclr/scripts/superpmi_aspnet2.py b/src/coreclr/scripts/superpmi_aspnet2.py index cf39fe1ca25208..0ef0ef1d7eaf2e 100644 --- a/src/coreclr/scripts/superpmi_aspnet2.py +++ b/src/coreclr/scripts/superpmi_aspnet2.py @@ -123,16 +123,44 @@ def download_mingit_windows(dest: str) -> str: def ensure_git(dest: Path) -> str: - # if shutil.which("git"): - # print("git found") - # return shutil.which("git") - if not sys.platform == "win32": - raise RuntimeError("Git is not available on this platform") - print("git not found, downloading portable git...") - git_dir = download_mingit_windows(str(dest)) - cmd_path = os.path.join(git_dir,"cmd") - os.environ["PATH"] = cmd_path + os.pathsep + os.environ["PATH"] - return shutil.which("git") + existing = shutil.which("git") + if existing: + print("git found") + return existing + + # Windows: download portable MinGit + if sys.platform == "win32": + print("git not found, downloading portable git...") + git_dir = download_mingit_windows(str(dest)) + cmd_path = os.path.join(git_dir, "cmd") + os.environ["PATH"] = cmd_path + os.pathsep + os.environ.get("PATH", "") + found = shutil.which("git") + if found: + return found + raise RuntimeError("Failed to make downloaded Git available in PATH") + + # Linux: try installing via tdnf (Azure Linux 3) + if sys.platform.startswith("linux"): + print("git not found, attempting to install via tdnf (Azure Linux 3)...") + tdnf = shutil.which("tdnf") + if tdnf is None: + raise RuntimeError("'tdnf' not found. Please install Git using your package manager or install tdnf.") + + # Assume this script is run under sudo/root on Azure Linux 3 + cmd = [tdnf, "-y", "install", "git"] + + rc = run(cmd) + if rc != 0: + raise RuntimeError(f"Failed to install Git via tdnf (exit code {rc}). Ensure you run this script with sudo/root.") + + found = shutil.which("git") + if found: + print("git installed via tdnf") + return found + raise RuntimeError("Git was installed via tdnf but is still not found in PATH") + + # Other platforms (e.g., macOS) are not handled here explicitly + raise RuntimeError("Git is not available on this platform; please install it and ensure it is on PATH.") # Ensure .NET tools are installed and Localhost.yml is present From fc0cf3d9c204d3d9546a42e87d6511963f49fa87 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Thu, 28 Aug 2025 18:29:10 +0200 Subject: [PATCH 22/48] fix error --- src/coreclr/scripts/superpmi-collect.proj | 2 +- src/coreclr/scripts/superpmi_aspnet2.py | 47 ++++++++++------------- 2 files changed, 21 insertions(+), 28 deletions(-) diff --git a/src/coreclr/scripts/superpmi-collect.proj b/src/coreclr/scripts/superpmi-collect.proj index e34d1cac507438..1d63b4d9065908 100644 --- a/src/coreclr/scripts/superpmi-collect.proj +++ b/src/coreclr/scripts/superpmi-collect.proj @@ -149,7 +149,7 @@ - $(Python) $(SuperPMIDirectory)$(FileSeparatorChar)superpmi_aspnet2.py --core_root $(SuperPMIDirectory) --cli $(PerformanceDirectory)$(FileSeparatorChar)tools$(FileSeparatorChar)dotnet + $(Python) $(SuperPMIDirectory)$(FileSeparatorChar)superpmi_aspnet2.py --core_root $(SuperPMIDirectory) --cli $(PerformanceDirectory)$(FileSeparatorChar)tools$(FileSeparatorChar)dotnet$(FileSeparatorChar)$(Architecture) 3:00 diff --git a/src/coreclr/scripts/superpmi_aspnet2.py b/src/coreclr/scripts/superpmi_aspnet2.py index 0ef0ef1d7eaf2e..2fc6dd9bf2d081 100644 --- a/src/coreclr/scripts/superpmi_aspnet2.py +++ b/src/coreclr/scripts/superpmi_aspnet2.py @@ -64,21 +64,6 @@ def native_dll(name: str) -> str: return f"{prefix}{name}{ext}" -# Helper function to find available PowerShell executable -def find_powershell(): - """Find available PowerShell executable, preferring pwsh over powershell.exe""" - # Try pwsh first (PowerShell Core, more modern and cross-platform) - if shutil.which("pwsh"): - return "pwsh" - # Fall back to Windows PowerShell - if shutil.which("powershell.exe"): - return "powershell.exe" - # Last resort, try just "powershell" - if shutil.which("powershell"): - return "powershell" - raise FileNotFoundError("No PowerShell executable found. Please install PowerShell Core (pwsh) or Windows PowerShell.") - - # Run a command def run(cmd, cwd=None): print(f"Running command: {' '.join(map(str, cmd))}") @@ -101,18 +86,23 @@ def run(cmd, cwd=None): return None -def get_arch() -> str: - m = platform.machine().lower() - if "arm64" in m: return "arm64" - if m in ("amd64", "x86_64"): return "x64" - raise RuntimeError(f"Unsupported arch: {m}") - - def download_mingit_windows(dest: str) -> str: - m = {"x64": "64-bit", "arm64": "arm64"} - assets = requests.get("https://api.github.com/repos/git-for-windows/git/releases/latest",timeout=100).json()["assets"] - rx = re.compile(r"^MinGit-.*-(64-bit|arm64)\.zip$", re.I) - asset = next(a for a in assets if rx.match(a["name"]) and m[get_arch()] in a["name"]) + # Map our arch key to MinGit asset suffix + m = {"x64": "64-bit", "arm64": "arm64", "x86": "32-bit"} + assets = requests.get("https://api.github.com/repos/git-for-windows/git/releases/latest", timeout=100).json()["assets"] + rx = re.compile(r"^MinGit-.*-(32-bit|64-bit|arm64)\.zip$", re.I) + + arch = "x64" + mach = platform.machine().lower() + if "arm64" in mach: + arch = "arm64" + elif mach in ("x86", "i386", "i686"): + arch = "x86" + + try: + asset = next(a for a in assets if rx.match(a["name"]) and m[arch] in a["name"]) + except StopIteration: + raise RuntimeError(f"Unable to find MinGit asset for arch '{arch}'. Available assets: {[a['name'] for a in assets if 'MinGit' in a['name']]}") os.makedirs(dest, exist_ok=True); zip_path = os.path.join(dest, asset["name"]) with requests.get(asset["browser_download_url"], stream=True) as r, open(zip_path,"wb") as f: for c in r.iter_content(8192): f.write(c) @@ -198,7 +188,10 @@ def ensure_tools_and_localhost_yaml(workdir: Path, port: int, cli=None): urllib.request.urlretrieve(url, path) if url.endswith(".ps1"): # Find available PowerShell executable - powershell_exe = find_powershell() + if shutil.which("pwsh"): + powershell_exe = "pwsh" + else: + powershell_exe = "powershell.exe" print(f"Using PowerShell executable: {powershell_exe}") # Add better error handling and capture output From eb724d64be787723c9a93f181c1d00f97f9d17e6 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Thu, 28 Aug 2025 18:29:49 +0200 Subject: [PATCH 23/48] enable linxu --- eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml b/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml index da965b430a1aaf..efb74998d23792 100644 --- a/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml +++ b/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml @@ -263,6 +263,8 @@ extends: platforms: - windows_x64 - windows_arm64 + - linux_arm64 + - linux_x64 helixQueueGroup: ci helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: From 4ccf02303024af351a37ef4d209094072b00d7d0 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Fri, 29 Aug 2025 09:02:17 +0200 Subject: [PATCH 24/48] clean up --- src/coreclr/scripts/superpmi-collect.proj | 9 ++ src/coreclr/scripts/superpmi_aspnet2.py | 105 ++-------------------- 2 files changed, 15 insertions(+), 99 deletions(-) diff --git a/src/coreclr/scripts/superpmi-collect.proj b/src/coreclr/scripts/superpmi-collect.proj index 1d63b4d9065908..8d84cd714b3572 100644 --- a/src/coreclr/scripts/superpmi-collect.proj +++ b/src/coreclr/scripts/superpmi-collect.proj @@ -163,6 +163,15 @@ + + + + + + + + + diff --git a/src/coreclr/scripts/superpmi_aspnet2.py b/src/coreclr/scripts/superpmi_aspnet2.py index 2711c8b736d2dc..8e79a1e8bd3bcf 100644 --- a/src/coreclr/scripts/superpmi_aspnet2.py +++ b/src/coreclr/scripts/superpmi_aspnet2.py @@ -72,6 +72,45 @@ def run(cmd, cwd=None, timeout_seconds=45*60): return None +def download_mingit_windows(dest: str) -> str: + # Map our arch key to MinGit asset suffix + m = {"x64": "64-bit", "arm64": "arm64", "x86": "32-bit"} + assets = requests.get("https://api.github.com/repos/git-for-windows/git/releases/latest", timeout=100).json()["assets"] + rx = re.compile(r"^MinGit-.*-(32-bit|64-bit|arm64)\.zip$", re.I) + + arch = "x64" + mach = platform.machine().lower() + if "arm64" in mach: + arch = "arm64" + elif mach in ("x86", "i386", "i686"): + arch = "x86" + + try: + asset = next(a for a in assets if rx.match(a["name"]) and m[arch] in a["name"]) + except StopIteration: + raise RuntimeError(f"Unable to find MinGit asset for arch '{arch}'. Available assets: {[a['name'] for a in assets if 'MinGit' in a['name']]}") + os.makedirs(dest, exist_ok=True); zip_path = os.path.join(dest, asset["name"]) + with requests.get(asset["browser_download_url"], stream=True) as r, open(zip_path,"wb") as f: + for c in r.iter_content(8192): f.write(c) + git_dir = os.path.join(dest,"git"); shutil.rmtree(git_dir, ignore_errors=True) + with zipfile.ZipFile(zip_path) as z: z.extractall(git_dir) + os.remove(zip_path) + return git_dir + + +def ensure_git(dest: Path) -> str: + existing = shutil.which("git") + if existing: + print("git found") + return + if sys.platform == "win32": + print("git not found, downloading portable git...") + git_dir = download_mingit_windows(str(dest)) + cmd_path = os.path.join(git_dir, "cmd") + os.environ["PATH"] = cmd_path + os.pathsep + os.environ.get("PATH", "") + return + + # Ensure .NET tools are installed and Localhost.yml is present def ensure_tools_and_localhost_yaml(workdir: Path, port: int, cli=None): data_dir = workdir / "crank_data" @@ -81,6 +120,8 @@ def ensure_tools_and_localhost_yaml(workdir: Path, port: int, cli=None): dotnethome_dir = data_dir / "dotnet_home" localhost_yml = data_dir / "Localhost.yml" + ensure_git(tools_dir) + # If a CLI path is provided, use it as DOTNET_ROOT; otherwise default to our local dotnet_home dotnet_root_dir = Path(cli) if cli else dotnethome_dir os.environ['DOTNET_ROOT'] = str(dotnet_root_dir) From 2157b8ef382392b3434b3f231aa381018cabe0b5 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Fri, 29 Aug 2025 15:36:24 +0200 Subject: [PATCH 28/48] clean up --- src/coreclr/scripts/superpmi-collect.proj | 6 +- src/coreclr/scripts/superpmi_aspnet2.py | 67 ++++++----------------- 2 files changed, 22 insertions(+), 51 deletions(-) diff --git a/src/coreclr/scripts/superpmi-collect.proj b/src/coreclr/scripts/superpmi-collect.proj index 1de9b8fccc211c..5e3523043355c6 100644 --- a/src/coreclr/scripts/superpmi-collect.proj +++ b/src/coreclr/scripts/superpmi-collect.proj @@ -173,6 +173,8 @@ + + @@ -323,9 +325,9 @@ $(CollectionName).$(CollectionType).%(HelixWorkItem.Index).$(MchFileTag) $(WorkItemDirectory) - $(WorkItemCommand) --output_mch $(WorkItemDirectory)$(FileSeparatorChar)$(OutputFileName).mch + $(WorkItemCommand) --output_mch $(OutputMchPath)$(FileSeparatorChar)%(OutputFileName).mch $(WorkItemTimeout) - $(WorkItemDirectory)$(FileSeparatorChar)$(OutputFileName).mch + %(OutputFileName).mch diff --git a/src/coreclr/scripts/superpmi_aspnet2.py b/src/coreclr/scripts/superpmi_aspnet2.py index 8e79a1e8bd3bcf..61828c0811b7da 100644 --- a/src/coreclr/scripts/superpmi_aspnet2.py +++ b/src/coreclr/scripts/superpmi_aspnet2.py @@ -42,6 +42,13 @@ def native_dll(name: str) -> str: prefix = "" if sys.platform.startswith("win") else "lib" return f"{prefix}{name}{ext}" +def native_exe(name: str) -> str: + ext = ".exe" if sys.platform.startswith("win") else ".out" + return f"{name}{ext}" + +def native_script(name: str) -> str: + ext = ".ps1" if sys.platform.startswith("win") else ".sh" + return f"{name}{ext}" # Run a command def run(cmd, cwd=None, timeout_seconds=45*60): @@ -72,6 +79,7 @@ def run(cmd, cwd=None, timeout_seconds=45*60): return None +# Temp workaround, will be removed once https://github.com/dotnet/crank/pull/841 lands def download_mingit_windows(dest: str) -> str: # Map our arch key to MinGit asset suffix m = {"x64": "64-bit", "arm64": "arm64", "x86": "32-bit"} @@ -97,7 +105,6 @@ def download_mingit_windows(dest: str) -> str: os.remove(zip_path) return git_dir - def ensure_git(dest: Path) -> str: existing = shutil.which("git") if existing: @@ -111,8 +118,7 @@ def ensure_git(dest: Path) -> str: return -# Ensure .NET tools are installed and Localhost.yml is present -def ensure_tools_and_localhost_yaml(workdir: Path, port: int, cli=None): +def setup_and_run_crank_agent(workdir: Path, port: int, cli=None): data_dir = workdir / "crank_data" logs_dir = data_dir / "logs" build_dir = data_dir / "build" @@ -152,7 +158,6 @@ def ensure_tools_and_localhost_yaml(workdir: Path, port: int, cli=None): powershell_exe = "powershell.exe" print(f"Using PowerShell executable: {powershell_exe}") - # Add better error handling and capture output try: result = subprocess.run([ powershell_exe, "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", path, @@ -171,8 +176,7 @@ def ensure_tools_and_localhost_yaml(workdir: Path, port: int, cli=None): print(f"Using existing .NET SDK at: {dotnet_root_dir}") # Determine the dotnet executable to use for installing tools - dotnet_file = "dotnet.exe" if sys.platform == "win32" else "dotnet" - dotnet_exe = dotnet_root_dir / dotnet_file + dotnet_exe = dotnet_root_dir / native_exe("dotnet") run([dotnet_exe, "tool", "install", "--tool-path", str(tools_dir), "Microsoft.Crank.Agent", "--version", "0.2.0-*"]) run([dotnet_exe, "tool", "install", "--tool-path", str(tools_dir), "Microsoft.Crank.Controller", "--version", "0.2.0-*"]) @@ -202,28 +206,16 @@ def ensure_tools_and_localhost_yaml(workdir: Path, port: int, cli=None): localhost_yml.write_text(yml, encoding="utf-8") else: print("Localhost.yml already present; skipping tool install/scaffold.") - - -# Start the crank-agent -def start_crank_agent(workdir: Path, port: int): print("crank-agent is not running yet. Starting...") - logs_dir = workdir / "crank_data" / "logs" - build_dir = workdir / "crank_data" / "build" - # Prefer DOTNET_ROOT if set (configured by ensure_tools_and_localhost_yaml), - # otherwise fall back to the local dotnet_home - env_dotnet_root = os.environ.get('DOTNET_ROOT') - dotnethome_dir = Path(env_dotnet_root) if env_dotnet_root else (workdir / "crank_data" / "dotnet_home") - agent_process = subprocess.Popen( [ - "crank-agent", + native_exe("crank-agent"), "--url", f"http://*:{port}", "--log-path", str(logs_dir), "--build-path", str(build_dir), "--dotnethome", str(dotnethome_dir), ] ) - print(f"Waiting up to 10s for crank-agent to start ...") time.sleep(10) return agent_process @@ -235,7 +227,7 @@ def build_crank_command(framework: str, runtime_bits_path: Path, scenario: str, coreclr = native_dll("coreclr") spcorelib = "System.Private.CoreLib.dll" cmd = [ - "crank", + native_exe("crank"), "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/azure.profile.yml", "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml", "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/steadystate.profile.yml", @@ -295,9 +287,9 @@ def main(): if args.cli: print(f"--cli: {args.cli}") - mcs_cmd = runtime_bits_path / ("mcs.exe" if sys.platform == "win32" else "mcs") + mcs_cmd = runtime_bits_path / native_exe("mcs") if not mcs_cmd.exists(): - print(f"Error: mcs.exe not found at {mcs_cmd}. Ensure runtime bits include mcs.", file=sys.stderr) + print(f"Error: mcs[.exe] not found at {mcs_cmd}. Ensure runtime bits include mcs.", file=sys.stderr) sys.exit(2) # Create or use working directory for crank_data @@ -314,10 +306,9 @@ def main(): # Set current working directory to temp_root os.chdir(temp_root) - ensure_tools_and_localhost_yaml(temp_root, CRANK_PORT, cli=args.cli) + agent_process = None try: - agent_process = None - agent_process = start_crank_agent(temp_root, CRANK_PORT) + agent_process = setup_and_run_crank_agent(temp_root, CRANK_PORT, cli=args.cli) # Benchmarks @@ -337,7 +328,6 @@ def main(): # Extract .mc files from zip archives into crank_data/tmp instead of the current directory print("Extracting .mc files from zip archives...") - produced_mch = False extracted_count = 0 for z in pathlib.Path('.').glob('*.crank.zip'): with zipfile.ZipFile(z) as f: @@ -361,30 +351,9 @@ def main(): "-recursive", "-dedup", "-thin", - "crank.mch", - "*.mc" + str(output_mch_path), + str(crank_data_path / "*.mc") ], check=True, cwd=str(crank_data_path)) - - # Move the produced crank.mch back to the workspace root - print(f"Moving produced crank.mch to {repo_dir / 'crank.mch'}") - shutil.copyfile(crank_data_path / "crank.mch", repo_dir / "crank.mch") - produced_mch = True - - # Copy the resulting MCH to the specified output file - if args.output_mch: - mch_src = repo_dir / "crank.mch" - if not mch_src.exists(): - print(f"Error: expected MCH not found at {mch_src}", file=sys.stderr) - sys.exit(2) - out_path = output_mch_path - if out_path.exists() and out_path.is_dir(): - print(f"Error: --output_mch points to a directory, expected a file path: {out_path}", file=sys.stderr) - sys.exit(2) - # Ensure the destination directory exists - if out_path.parent and not out_path.parent.exists(): - out_path.parent.mkdir(parents=True, exist_ok=True) - print(f"Copying {mch_src} -> {out_path}") - shutil.copyfile(mch_src, out_path) finally: print("Cleaning up...") From d3ca9f5fec144a7e43576992f2d0bd49a80c23db Mon Sep 17 00:00:00 2001 From: EgorBo Date: Fri, 29 Aug 2025 17:13:48 +0200 Subject: [PATCH 29/48] fix build --- src/coreclr/scripts/superpmi_aspnet2.py | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/src/coreclr/scripts/superpmi_aspnet2.py b/src/coreclr/scripts/superpmi_aspnet2.py index 61828c0811b7da..3628915055b9e8 100644 --- a/src/coreclr/scripts/superpmi_aspnet2.py +++ b/src/coreclr/scripts/superpmi_aspnet2.py @@ -43,7 +43,7 @@ def native_dll(name: str) -> str: return f"{prefix}{name}{ext}" def native_exe(name: str) -> str: - ext = ".exe" if sys.platform.startswith("win") else ".out" + ext = ".exe" if sys.platform.startswith("win") else "" return f"{name}{ext}" def native_script(name: str) -> str: @@ -209,7 +209,7 @@ def setup_and_run_crank_agent(workdir: Path, port: int, cli=None): print("crank-agent is not running yet. Starting...") agent_process = subprocess.Popen( [ - native_exe("crank-agent"), + str(tools_dir / native_exe("crank-agent")), "--url", f"http://*:{port}", "--log-path", str(logs_dir), "--build-path", str(build_dir), @@ -218,16 +218,16 @@ def setup_and_run_crank_agent(workdir: Path, port: int, cli=None): ) print(f"Waiting up to 10s for crank-agent to start ...") time.sleep(10) - return agent_process + return agent_process, tools_dir / native_exe("crank"), localhost_yml # Build the crank-controller command for execution -def build_crank_command(framework: str, runtime_bits_path: Path, scenario: str, config_path: Path): +def build_crank_command(crank_app: Path, framework: str, runtime_bits_path: Path, scenario: str, config_path: Path): spmi_shim = native_dll("superpmi-shim-collector") clrjit = native_dll("clrjit") coreclr = native_dll("coreclr") spcorelib = "System.Private.CoreLib.dll" cmd = [ - native_exe("crank"), + str(crank_app), "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/azure.profile.yml", "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml", "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/steadystate.profile.yml", @@ -308,21 +308,19 @@ def main(): agent_process = None try: - agent_process = setup_and_run_crank_agent(temp_root, CRANK_PORT, cli=args.cli) + agent_process, crank_app_path, config_path = setup_and_run_crank_agent(temp_root, CRANK_PORT, cli=args.cli) # Benchmarks crank_data_path = temp_root / "crank_data" - config_path = crank_data_path / "Localhost.yml" # print("### Running OrchardCMS benchmark... ###") - # run(build_crank_command(framework=args.tfm, runtime_bits_path=runtime_bits_path, scenario="about-sqlite", config_path=config_path)) + # run(build_crank_command(crank_app_path, args.tfm, runtime_bits_path, "about-sqlite", config_path)) print("### Running JsonMVC benchmark... ###") - run(build_crank_command(framework=args.tfm, runtime_bits_path=runtime_bits_path, scenario="mvc", config_path=config_path)) + run(build_crank_command(crank_app_path, args.tfm, runtime_bits_path, "mvc", config_path)) print("### Running NoMvcAuth benchmark... ###") - # run(build_crank_command(framework=args.tfm, runtime_bits_path=runtime_bits_path, scenario="NoMvcAuth", config_path=config_path)) - + # run(build_crank_command(crank_app_path, args.tfm, runtime_bits_path, "NoMvcAuth", config_path)) print("Finished running benchmarks.") From 86ca7e15336e56dda0748533806c74c392a55220 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Fri, 29 Aug 2025 19:38:28 +0200 Subject: [PATCH 30/48] final fix --- src/coreclr/scripts/superpmi_aspnet2.py | 142 ++++++++++-------------- 1 file changed, 57 insertions(+), 85 deletions(-) diff --git a/src/coreclr/scripts/superpmi_aspnet2.py b/src/coreclr/scripts/superpmi_aspnet2.py index 3628915055b9e8..eee47c4b49ef78 100644 --- a/src/coreclr/scripts/superpmi_aspnet2.py +++ b/src/coreclr/scripts/superpmi_aspnet2.py @@ -3,7 +3,6 @@ import os import re import requests -import socket import pathlib import subprocess import sys @@ -34,6 +33,7 @@ ########################################################################################################## CRANK_PORT = 5010 +CRANK_TFM = "8.0" # Convert a filename to the appropriate native DLL name, e.g. "clrjit" -> "libclrjit.so" (on Linux) @@ -42,13 +42,11 @@ def native_dll(name: str) -> str: prefix = "" if sys.platform.startswith("win") else "lib" return f"{prefix}{name}{ext}" + def native_exe(name: str) -> str: ext = ".exe" if sys.platform.startswith("win") else "" return f"{name}{ext}" -def native_script(name: str) -> str: - ext = ".ps1" if sys.platform.startswith("win") else ".sh" - return f"{name}{ext}" # Run a command def run(cmd, cwd=None, timeout_seconds=45*60): @@ -118,70 +116,63 @@ def ensure_git(dest: Path) -> str: return -def setup_and_run_crank_agent(workdir: Path, port: int, cli=None): - data_dir = workdir / "crank_data" - logs_dir = data_dir / "logs" - build_dir = data_dir / "build" - tools_dir = data_dir / "dotnet_tools" - dotnethome_dir = data_dir / "dotnet_home" - localhost_yml = data_dir / "Localhost.yml" - +def setup_and_run_crank_agent(workdir: Path, port: int): + logs_dir = workdir / "logs" + build_dir = workdir / "build" + tools_dir = workdir / "dotnet_tools" + dotnethome_dir = workdir / "dotnet_home" + localhost_yml = workdir / "localhost.yml" ensure_git(tools_dir) - # If a CLI path is provided, use it as DOTNET_ROOT; otherwise default to our local dotnet_home - dotnet_root_dir = Path(cli) if cli else dotnethome_dir + # Always use the local dotnet_home under crank_data + dotnet_root_dir = dotnethome_dir os.environ['DOTNET_ROOT'] = str(dotnet_root_dir) os.environ['DOTNET_CLI_TELEMETRY_OPTOUT'] = '1' os.environ['DOTNET_MULTILEVEL_LOOKUP'] = '0' os.environ['UseSharedCompilation'] = 'false' os.environ["PATH"] = str(tools_dir) + os.pathsep + os.environ.get("PATH", "") - if not data_dir.exists(): - print("Installing tools ...") - logs_dir.mkdir(parents=True, exist_ok=True) - build_dir.mkdir(parents=True, exist_ok=True) - dotnethome_dir.mkdir(parents=True, exist_ok=True) - tools_dir.mkdir(parents=True, exist_ok=True) - - # If a CLI path was provided, skip installing .NET and use that SDK. - if cli is None: - # Install .NET 8.0 needed for crank and crank-agent via dotnet-install public script. - url = "https://dot.net/v1/dotnet-install." + ("ps1" if platform.system()=="Windows" else "sh") - with tempfile.TemporaryDirectory() as tmp: - path = os.path.join(tmp, os.path.basename(url)) - urllib.request.urlretrieve(url, path) - if url.endswith(".ps1"): - # Find available PowerShell executable - if shutil.which("pwsh"): - powershell_exe = "pwsh" - else: - powershell_exe = "powershell.exe" - print(f"Using PowerShell executable: {powershell_exe}") - - try: - result = subprocess.run([ - powershell_exe, "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", path, - "-Channel", "8.0", "-InstallDir", str(dotnethome_dir) - ], capture_output=True, text=True, check=True) - print(f"PowerShell output: {result.stdout}") - except subprocess.CalledProcessError as e: - print(f"PowerShell script failed with exit code {e.returncode}") - print(f"Error output: {e.stderr}") - print(f"Standard output: {e.stdout}") - raise - else: - os.chmod(path,0o755) - subprocess.check_call([path,"-Channel","8.0","-InstallDir", str(dotnethome_dir)]) + print("Installing tools ...") + logs_dir.mkdir(parents=True, exist_ok=True) + build_dir.mkdir(parents=True, exist_ok=True) + dotnethome_dir.mkdir(parents=True, exist_ok=True) + tools_dir.mkdir(parents=True, exist_ok=True) + + # Install .NET SDK needed for crank and crank-agent via dotnet-install public script. + url = "https://dot.net/v1/dotnet-install." + ("ps1" if platform.system()=="Windows" else "sh") + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, os.path.basename(url)) + urllib.request.urlretrieve(url, path) + if url.endswith(".ps1"): + # Find available PowerShell executable + if shutil.which("pwsh"): + powershell_exe = "pwsh" + else: + powershell_exe = "powershell.exe" + print(f"Using PowerShell executable: {powershell_exe}") + + try: + result = subprocess.run([ + powershell_exe, "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", path, + "-Channel", CRANK_TFM, "-InstallDir", str(dotnethome_dir) + ], capture_output=True, text=True, check=True) + print(f"PowerShell output: {result.stdout}") + except subprocess.CalledProcessError as e: + print(f"PowerShell script failed with exit code {e.returncode}") + print(f"Error output: {e.stderr}") + print(f"Standard output: {e.stdout}") + raise else: - print(f"Using existing .NET SDK at: {dotnet_root_dir}") + os.chmod(path,0o755) + subprocess.check_call([path,"-Channel",CRANK_TFM,"-InstallDir", str(dotnethome_dir)]) - # Determine the dotnet executable to use for installing tools - dotnet_exe = dotnet_root_dir / native_exe("dotnet") - run([dotnet_exe, "tool", "install", "--tool-path", str(tools_dir), "Microsoft.Crank.Agent", "--version", "0.2.0-*"]) - run([dotnet_exe, "tool", "install", "--tool-path", str(tools_dir), "Microsoft.Crank.Controller", "--version", "0.2.0-*"]) + # Determine the dotnet executable to use for installing tools + dotnet_exe = dotnet_root_dir / native_exe("dotnet") + run([dotnet_exe, "tool", "install", "--tool-path", str(tools_dir), "Microsoft.Crank.Agent", "--version", "0.2.0-*"]) + run([dotnet_exe, "tool", "install", "--tool-path", str(tools_dir), "Microsoft.Crank.Controller", "--version", "0.2.0-*"]) - # Create a Localhost.yml to define the local environment - yml = textwrap.dedent( + # Create a Localhost.yml to define the local environment + yml = textwrap.dedent( f""" variables: applicationAddress: 127.0.0.1 @@ -203,9 +194,9 @@ def setup_and_run_crank_agent(workdir: Path, port: int, cli=None): endpoints: - "{{{{loadScheme}}}}://{{{{loadAddress}}}}:{{{{loadPort}}}}" """) - localhost_yml.write_text(yml, encoding="utf-8") - else: - print("Localhost.yml already present; skipping tool install/scaffold.") + localhost_yml.write_text(yml, encoding="utf-8") + + print("crank-agent is not running yet. Starting...") agent_process = subprocess.Popen( [ @@ -263,29 +254,20 @@ def build_crank_command(crank_app: Path, framework: str, runtime_bits_path: Path # Main entry point def main(): parser = argparse.ArgumentParser(description="Cross-platform crank runner.") - # Renamed args parser.add_argument("--core_root", required=True, help="Path to built runtime bits (CORE_ROOT).") parser.add_argument("--tfm", default="net10.0", help="Target Framework Moniker (e.g., net10.0).") parser.add_argument("--output_mch", required=True, help="File path to copy the resulting merged .mch to (expects a file path, not a directory).") - - # New args - parser.add_argument("--work_dir", help="Optional work directory; if not specified, a temp directory is used.") parser.add_argument("--no_cleanup", action="store_true", help="If specified, do not clean up temporary files after execution.") - parser.add_argument("--cli", help="Optional path to an existing .NET SDK root; if provided, DOTNET_ROOT will use this and .NET 8.0 will not be installed.") args = parser.parse_args() repo_dir = Path.cwd() runtime_bits_path = Path(args.core_root).expanduser().resolve() output_mch_path = Path(args.output_mch).expanduser().resolve() - if args.cli: - args.cli = str(Path(args.cli).expanduser().resolve()) print("Running the script with the following parameters:") print(f"--core_root: {runtime_bits_path}") print(f"--tfm: {args.tfm}") print(f"--output_mch: {output_mch_path}") - if args.cli: - print(f"--cli: {args.cli}") mcs_cmd = runtime_bits_path / native_exe("mcs") if not mcs_cmd.exists(): @@ -293,26 +275,16 @@ def main(): sys.exit(2) # Create or use working directory for crank_data - created_temp = False - if args.work_dir: - temp_root = Path(args.work_dir).resolve() - temp_root.mkdir(parents=True, exist_ok=True) - print(f"Using work directory: {temp_root}") - else: - temp_root = Path(tempfile.mkdtemp(prefix="aspnet4_crank_")) - created_temp = True - print(f"Using temp work directory: {temp_root}") + temp_root = Path(tempfile.mkdtemp(prefix="aspnet2_")) + print(f"Using temp work directory: {temp_root}") # Set current working directory to temp_root os.chdir(temp_root) agent_process = None try: - agent_process, crank_app_path, config_path = setup_and_run_crank_agent(temp_root, CRANK_PORT, cli=args.cli) - - # Benchmarks + agent_process, crank_app_path, config_path = setup_and_run_crank_agent(temp_root, CRANK_PORT) - crank_data_path = temp_root / "crank_data" # print("### Running OrchardCMS benchmark... ###") # run(build_crank_command(crank_app_path, args.tfm, runtime_bits_path, "about-sqlite", config_path)) @@ -332,7 +304,7 @@ def main(): for name in f.namelist(): # include .mc files from any path inside the zip if name.endswith('.mc'): - f.extract(name, str(crank_data_path)) + f.extract(name, str(temp_root)) extracted_count += 1 z.unlink(missing_ok=True) @@ -350,8 +322,8 @@ def main(): "-dedup", "-thin", str(output_mch_path), - str(crank_data_path / "*.mc") - ], check=True, cwd=str(crank_data_path)) + str(temp_root / "*.mc") + ], check=True, cwd=str(temp_root)) finally: print("Cleaning up...") From 1b2b13f44aae9d07f22b30412b8cb4d03e460027 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Fri, 29 Aug 2025 23:29:12 +0200 Subject: [PATCH 31/48] clean up --- src/coreclr/scripts/superpmi_aspnet2.py | 194 +++++++++++++----------- 1 file changed, 109 insertions(+), 85 deletions(-) diff --git a/src/coreclr/scripts/superpmi_aspnet2.py b/src/coreclr/scripts/superpmi_aspnet2.py index eee47c4b49ef78..173a7a8caf7d5f 100644 --- a/src/coreclr/scripts/superpmi_aspnet2.py +++ b/src/coreclr/scripts/superpmi_aspnet2.py @@ -21,19 +21,14 @@ # This script sets up an environment for running crank-agent and crank-controller # (https://github.com/dotnet/crank) locally in order to run various ASP.NET benchmarks # (TechEmpower, OrchardCMS, etc.) and collect SPMI collections using the provided runtime bits. -# The script is cross-platform and does everything locally while requiring only 'git' as a dependency. # # Usage example: # py superpmi_aspnet2.py --core_root C:\runtime\artifacts\bin\coreclr\windows.x64.Checked --output_mch aspnet.mch # -# Prerequisites: -# * git (crank-agent relies on it being available from PATH) -# * python3 (to run this script) -# ########################################################################################################## CRANK_PORT = 5010 -CRANK_TFM = "8.0" +CRANK_SDK_CHANNEL = "LTS" # Convert a filename to the appropriate native DLL name, e.g. "clrjit" -> "libclrjit.so" (on Linux) @@ -43,6 +38,7 @@ def native_dll(name: str) -> str: return f"{prefix}{name}{ext}" +# Same for executables def native_exe(name: str) -> str: ext = ".exe" if sys.platform.startswith("win") else "" return f"{name}{ext}" @@ -57,6 +53,7 @@ def run(cmd, cwd=None, timeout_seconds=45*60): "stderr": subprocess.STDOUT, "cwd": cwd, } + # It doesn't download the resulting artifacts without this: if os.name == "nt": kwargs["creationflags"] = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) else: @@ -78,31 +75,6 @@ def run(cmd, cwd=None, timeout_seconds=45*60): # Temp workaround, will be removed once https://github.com/dotnet/crank/pull/841 lands -def download_mingit_windows(dest: str) -> str: - # Map our arch key to MinGit asset suffix - m = {"x64": "64-bit", "arm64": "arm64", "x86": "32-bit"} - assets = requests.get("https://api.github.com/repos/git-for-windows/git/releases/latest", timeout=100).json()["assets"] - rx = re.compile(r"^MinGit-.*-(32-bit|64-bit|arm64)\.zip$", re.I) - - arch = "x64" - mach = platform.machine().lower() - if "arm64" in mach: - arch = "arm64" - elif mach in ("x86", "i386", "i686"): - arch = "x86" - - try: - asset = next(a for a in assets if rx.match(a["name"]) and m[arch] in a["name"]) - except StopIteration: - raise RuntimeError(f"Unable to find MinGit asset for arch '{arch}'. Available assets: {[a['name'] for a in assets if 'MinGit' in a['name']]}") - os.makedirs(dest, exist_ok=True); zip_path = os.path.join(dest, asset["name"]) - with requests.get(asset["browser_download_url"], stream=True) as r, open(zip_path,"wb") as f: - for c in r.iter_content(8192): f.write(c) - git_dir = os.path.join(dest,"git"); shutil.rmtree(git_dir, ignore_errors=True) - with zipfile.ZipFile(zip_path) as z: z.extractall(git_dir) - os.remove(zip_path) - return git_dir - def ensure_git(dest: Path) -> str: existing = shutil.which("git") if existing: @@ -110,13 +82,91 @@ def ensure_git(dest: Path) -> str: return if sys.platform == "win32": print("git not found, downloading portable git...") - git_dir = download_mingit_windows(str(dest)) + m = {"x64": "64-bit", "arm64": "arm64", "x86": "32-bit"} + assets = requests.get( + "https://api.github.com/repos/git-for-windows/git/releases/latest", timeout=100 + ).json()["assets"] + rx = re.compile(r"^MinGit-.*-(32-bit|64-bit|arm64)\.zip$", re.I) + arch = "x64" + mach = platform.machine().lower() + if "arm64" in mach: + arch = "arm64" + elif mach in ("x86", "i386", "i686"): + arch = "x86" + try: + asset = next(a for a in assets if rx.match(a["name"]) and m[arch] in a["name"]) + except StopIteration: + raise RuntimeError( + f"Unable to find MinGit asset for arch '{arch}'. Available assets: " + f"{[a['name'] for a in assets if 'MinGit' in a['name']]}" + ) + dest_str = str(dest) + os.makedirs(dest_str, exist_ok=True) + zip_path = os.path.join(dest_str, asset["name"]) + with requests.get(asset["browser_download_url"], stream=True) as r, open(zip_path, "wb") as f: + for c in r.iter_content(8192): + f.write(c) + git_dir = os.path.join(dest_str, "git") + shutil.rmtree(git_dir, ignore_errors=True) + with zipfile.ZipFile(zip_path) as z: + z.extractall(git_dir) + os.remove(zip_path) cmd_path = os.path.join(git_dir, "cmd") os.environ["PATH"] = cmd_path + os.pathsep + os.environ.get("PATH", "") return -def setup_and_run_crank_agent(workdir: Path, port: int): +# Install the .NET SDK using the official dotnet-install script. +def install_dotnet_sdk(channel: str, install_dir: Path) -> None: + url = "https://dot.net/v1/dotnet-install." + ("ps1" if platform.system() == "Windows" else "sh") + retries = 3 + initial_delay_seconds = 5 + last_err = None + for attempt in range(1, retries + 1): + try: + with tempfile.TemporaryDirectory() as tmp: + script_path = os.path.join(tmp, os.path.basename(url)) + print(f"Downloading dotnet-install script (attempt {attempt}/{retries}) ...") + urllib.request.urlretrieve(url, script_path) + + if script_path.endswith(".ps1"): + # Find available PowerShell executable + powershell_exe = "pwsh" if shutil.which("pwsh") else "powershell.exe" + print(f"Using PowerShell executable: {powershell_exe}") + args = [ + powershell_exe, "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", script_path, + "-Channel", channel, "-InstallDir", str(install_dir) + ] + result = subprocess.run(args, capture_output=True, text=True, check=True) + if result.stdout: + print(result.stdout) + else: + os.chmod(script_path, 0o755) + cmd = [script_path, "-Channel", channel, "-InstallDir", str(install_dir)] + subprocess.check_call(cmd) + + # Success, return + return + except subprocess.CalledProcessError as e: + print(f"dotnet-install attempt {attempt}/{retries} failed with exit code {e.returncode}") + if getattr(e, "stdout", None): + print(e.stdout) + if getattr(e, "stderr", None): + print(e.stderr) + last_err = e + except Exception as e: + print(f"dotnet-install attempt {attempt}/{retries} failed: {e}") + last_err = e + + if attempt < retries: + delay = min(initial_delay_seconds * (2 ** (attempt - 1)), 60) + print(f"Retrying in {delay} seconds ...") + time.sleep(delay) + raise RuntimeError(f"Failed to install .NET SDK after {retries} attempts") from last_err + + +# Prepare the environment and run crank-agent +def setup_and_run_crank_agent(workdir: Path): logs_dir = workdir / "logs" build_dir = workdir / "build" tools_dir = workdir / "dotnet_tools" @@ -124,13 +174,11 @@ def setup_and_run_crank_agent(workdir: Path, port: int): localhost_yml = workdir / "localhost.yml" ensure_git(tools_dir) - # Always use the local dotnet_home under crank_data dotnet_root_dir = dotnethome_dir os.environ['DOTNET_ROOT'] = str(dotnet_root_dir) os.environ['DOTNET_CLI_TELEMETRY_OPTOUT'] = '1' os.environ['DOTNET_MULTILEVEL_LOOKUP'] = '0' os.environ['UseSharedCompilation'] = 'false' - os.environ["PATH"] = str(tools_dir) + os.pathsep + os.environ.get("PATH", "") print("Installing tools ...") logs_dir.mkdir(parents=True, exist_ok=True) @@ -139,32 +187,7 @@ def setup_and_run_crank_agent(workdir: Path, port: int): tools_dir.mkdir(parents=True, exist_ok=True) # Install .NET SDK needed for crank and crank-agent via dotnet-install public script. - url = "https://dot.net/v1/dotnet-install." + ("ps1" if platform.system()=="Windows" else "sh") - with tempfile.TemporaryDirectory() as tmp: - path = os.path.join(tmp, os.path.basename(url)) - urllib.request.urlretrieve(url, path) - if url.endswith(".ps1"): - # Find available PowerShell executable - if shutil.which("pwsh"): - powershell_exe = "pwsh" - else: - powershell_exe = "powershell.exe" - print(f"Using PowerShell executable: {powershell_exe}") - - try: - result = subprocess.run([ - powershell_exe, "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", path, - "-Channel", CRANK_TFM, "-InstallDir", str(dotnethome_dir) - ], capture_output=True, text=True, check=True) - print(f"PowerShell output: {result.stdout}") - except subprocess.CalledProcessError as e: - print(f"PowerShell script failed with exit code {e.returncode}") - print(f"Error output: {e.stderr}") - print(f"Standard output: {e.stdout}") - raise - else: - os.chmod(path,0o755) - subprocess.check_call([path,"-Channel",CRANK_TFM,"-InstallDir", str(dotnethome_dir)]) + install_dotnet_sdk(CRANK_SDK_CHANNEL, dotnethome_dir) # Determine the dotnet executable to use for installing tools dotnet_exe = dotnet_root_dir / native_exe("dotnet") @@ -197,22 +220,23 @@ def setup_and_run_crank_agent(workdir: Path, port: int): localhost_yml.write_text(yml, encoding="utf-8") - print("crank-agent is not running yet. Starting...") + print("Starting crank-agent...") agent_process = subprocess.Popen( [ str(tools_dir / native_exe("crank-agent")), - "--url", f"http://*:{port}", + "--url", f"http://*:{CRANK_PORT}", "--log-path", str(logs_dir), "--build-path", str(build_dir), "--dotnethome", str(dotnethome_dir), ] ) - print(f"Waiting up to 10s for crank-agent to start ...") + print(f"Waiting 10s for crank-agent to start ...") time.sleep(10) return agent_process, tools_dir / native_exe("crank"), localhost_yml -# Build the crank-controller command for execution -def build_crank_command(crank_app: Path, framework: str, runtime_bits_path: Path, scenario: str, config_path: Path): + +# Run crank scenario +def run_crank_scenario(crank_app: Path, framework: str, core_root_path: Path, scenario: str, config_path: Path): spmi_shim = native_dll("superpmi-shim-collector") clrjit = native_dll("clrjit") coreclr = native_dll("coreclr") @@ -235,20 +259,20 @@ def build_crank_command(crank_app: Path, framework: str, runtime_bits_path: Path "--application.options.collectCounters", "false", "--application.collectDependencies", "false", "--load.options.reuseBuild", "true", - "--load.variables.duration", "10", # default 15s is not enough for Tier1 promotion + "--load.variables.duration", "30", # default 15s is not enough for Tier1 promotion "--load.job", "bombardier", # Bombardier is more cross-platform friendly (wrk is linux only) "--application.environmentVariables", f"COMPlus_JitName={spmi_shim}", "--application.environmentVariables", "SuperPMIShimLogPath=.", "--application.environmentVariables", f"SuperPMIShimPath=./{clrjit}", "--application.options.fetch", "true", "--application.options.fetchOutput", scenario + ".crank.zip", - "--application.options.outputFiles", str(runtime_bits_path / spmi_shim), - "--application.options.outputFiles", str(runtime_bits_path / clrjit), - "--application.options.outputFiles", str(runtime_bits_path / coreclr), - "--application.options.outputFiles", str(runtime_bits_path / spcorelib), + "--application.options.outputFiles", str(core_root_path / spmi_shim), + "--application.options.outputFiles", str(core_root_path / clrjit), + "--application.options.outputFiles", str(core_root_path / coreclr), + "--application.options.outputFiles", str(core_root_path / spcorelib), "--scenario", scenario ] - return cmd + run(cmd) # Main entry point @@ -260,16 +284,15 @@ def main(): parser.add_argument("--no_cleanup", action="store_true", help="If specified, do not clean up temporary files after execution.") args = parser.parse_args() - repo_dir = Path.cwd() - runtime_bits_path = Path(args.core_root).expanduser().resolve() + core_root_path = Path(args.core_root).expanduser().resolve() output_mch_path = Path(args.output_mch).expanduser().resolve() print("Running the script with the following parameters:") - print(f"--core_root: {runtime_bits_path}") + print(f"--core_root: {core_root_path}") print(f"--tfm: {args.tfm}") print(f"--output_mch: {output_mch_path}") - mcs_cmd = runtime_bits_path / native_exe("mcs") + mcs_cmd = core_root_path / native_exe("mcs") if not mcs_cmd.exists(): print(f"Error: mcs[.exe] not found at {mcs_cmd}. Ensure runtime bits include mcs.", file=sys.stderr) sys.exit(2) @@ -283,16 +306,18 @@ def main(): agent_process = None try: - agent_process, crank_app_path, config_path = setup_and_run_crank_agent(temp_root, CRANK_PORT) + agent_process, crank_app_path, config_path = setup_and_run_crank_agent(temp_root) - # print("### Running OrchardCMS benchmark... ###") - # run(build_crank_command(crank_app_path, args.tfm, runtime_bits_path, "about-sqlite", config_path)) - - print("### Running JsonMVC benchmark... ###") - run(build_crank_command(crank_app_path, args.tfm, runtime_bits_path, "mvc", config_path)) + # Array of scenarios to run + scenarios = [ + "about-sqlite", + "mvc", + "NoMvcAuth" + ] - print("### Running NoMvcAuth benchmark... ###") - # run(build_crank_command(crank_app_path, args.tfm, runtime_bits_path, "NoMvcAuth", config_path)) + for scenario in scenarios: + print(f"### Running {scenario} benchmark... ###") + run_crank_scenario(crank_app_path, args.tfm, core_root_path, scenario, config_path) print("Finished running benchmarks.") @@ -302,7 +327,6 @@ def main(): for z in pathlib.Path('.').glob('*.crank.zip'): with zipfile.ZipFile(z) as f: for name in f.namelist(): - # include .mc files from any path inside the zip if name.endswith('.mc'): f.extract(name, str(temp_root)) extracted_count += 1 From 91a14896238f76a35a714cdb6886c7ba7e4c99ba Mon Sep 17 00:00:00 2001 From: EgorBo Date: Sat, 30 Aug 2025 00:23:27 +0200 Subject: [PATCH 32/48] final --- src/coreclr/scripts/superpmi-collect.proj | 7 +--- src/coreclr/scripts/superpmi_aspnet2.py | 51 ++++++++++++++++------- 2 files changed, 39 insertions(+), 19 deletions(-) diff --git a/src/coreclr/scripts/superpmi-collect.proj b/src/coreclr/scripts/superpmi-collect.proj index 5e3523043355c6..e644169b673339 100644 --- a/src/coreclr/scripts/superpmi-collect.proj +++ b/src/coreclr/scripts/superpmi-collect.proj @@ -284,11 +284,8 @@ - - 1 - - + @@ -322,7 +319,7 @@ - + $(CollectionName).$(CollectionType).%(HelixWorkItem.Index).$(MchFileTag) $(WorkItemDirectory) $(WorkItemCommand) --output_mch $(OutputMchPath)$(FileSeparatorChar)%(OutputFileName).mch diff --git a/src/coreclr/scripts/superpmi_aspnet2.py b/src/coreclr/scripts/superpmi_aspnet2.py index 173a7a8caf7d5f..cdc1e27fd9f54a 100644 --- a/src/coreclr/scripts/superpmi_aspnet2.py +++ b/src/coreclr/scripts/superpmi_aspnet2.py @@ -236,18 +236,25 @@ def setup_and_run_crank_agent(workdir: Path): # Run crank scenario -def run_crank_scenario(crank_app: Path, framework: str, core_root_path: Path, scenario: str, config_path: Path): +def run_crank_scenario(crank_app: Path, framework: str, core_root_path: Path, config_path: Path, *extra_args: str): spmi_shim = native_dll("superpmi-shim-collector") clrjit = native_dll("clrjit") coreclr = native_dll("coreclr") spcorelib = "System.Private.CoreLib.dll" + # Try to infer scenario name from extra args for naming the output zip + scenario_name = None + for i, a in enumerate(extra_args): + if a == "--scenario" and i + 1 < len(extra_args): + scenario_name = str(extra_args[i + 1]) + break + if scenario_name is None: + scenario_name = "scenario" cmd = [ str(crank_app), "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/azure.profile.yml", "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml", "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/steadystate.profile.yml", "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/json.benchmarks.yml", - "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/src/BenchmarksApps/Mvc/benchmarks.jwtapi.yml", "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/orchard.benchmarks.yml", "--config", "https://raw.githubusercontent.com/dotnet/crank/main/src/Microsoft.Crank.Jobs.Wrk/wrk.yml", "--config", "https://raw.githubusercontent.com/dotnet/crank/main/src/Microsoft.Crank.Jobs.Bombardier/bombardier.yml", @@ -265,13 +272,15 @@ def run_crank_scenario(crank_app: Path, framework: str, core_root_path: Path, sc "--application.environmentVariables", "SuperPMIShimLogPath=.", "--application.environmentVariables", f"SuperPMIShimPath=./{clrjit}", "--application.options.fetch", "true", - "--application.options.fetchOutput", scenario + ".crank.zip", + "--application.options.fetchOutput", scenario_name + ".crank.zip", "--application.options.outputFiles", str(core_root_path / spmi_shim), "--application.options.outputFiles", str(core_root_path / clrjit), "--application.options.outputFiles", str(core_root_path / coreclr), "--application.options.outputFiles", str(core_root_path / spcorelib), - "--scenario", scenario ] + # Append any extra scenario-specific arguments + if extra_args: + cmd.extend(extra_args) run(cmd) @@ -308,16 +317,30 @@ def main(): try: agent_process, crank_app_path, config_path = setup_and_run_crank_agent(temp_root) - # Array of scenarios to run - scenarios = [ - "about-sqlite", - "mvc", - "NoMvcAuth" - ] - - for scenario in scenarios: - print(f"### Running {scenario} benchmark... ###") - run_crank_scenario(crank_app_path, args.tfm, core_root_path, scenario, config_path) + print("### Running about-sqlite benchmark... ###") + run_crank_scenario(crank_app_path, args.tfm, core_root_path, config_path, + "--scenario", "about-sqlite", + "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/orchard.benchmarks.yml" + ) + + print("### Running JsonMVC benchmark... ###") + run_crank_scenario(crank_app_path, args.tfm, core_root_path, config_path, + "--scenario", "mvc", + "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/json.benchmarks.yml" + ) + + print("### Running NoMvcAuth benchmark... ###") + run_crank_scenario(crank_app_path, args.tfm, core_root_path, config_path, + "--scenario", "NoMvcAuth", + "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/src/BenchmarksApps/Mvc/benchmarks.jwtapi.yml" + ) + + print("### Running PlatformPlaintext benchmark... ###") + run_crank_scenario(crank_app_path, args.tfm, core_root_path, config_path, + "--scenario", "plaintext", + "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/platform.benchmarks.yml", + "--load.connections", "512" + ) print("Finished running benchmarks.") From 26e09d47b2fbe26907da8715e40b44effc0163bc Mon Sep 17 00:00:00 2001 From: EgorBo Date: Sat, 30 Aug 2025 00:27:05 +0200 Subject: [PATCH 33/48] cleanup --- .../coreclr/templates/superpmi-collect-pipeline.yml | 7 +++++-- src/coreclr/scripts/superpmi_aspnet2.py | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml b/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml index efb74998d23792..033c68cbe44190 100644 --- a/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml +++ b/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml @@ -261,10 +261,13 @@ extends: jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml buildConfig: checked platforms: - - windows_x64 - - windows_arm64 + - osx_arm64 + - linux_arm - linux_arm64 - linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 helixQueueGroup: ci helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml jobParameters: diff --git a/src/coreclr/scripts/superpmi_aspnet2.py b/src/coreclr/scripts/superpmi_aspnet2.py index cdc1e27fd9f54a..546eab907c1058 100644 --- a/src/coreclr/scripts/superpmi_aspnet2.py +++ b/src/coreclr/scripts/superpmi_aspnet2.py @@ -267,6 +267,7 @@ def run_crank_scenario(crank_app: Path, framework: str, core_root_path: Path, co "--application.collectDependencies", "false", "--load.options.reuseBuild", "true", "--load.variables.duration", "30", # default 15s is not enough for Tier1 promotion + "--load.connections", "32", "--load.job", "bombardier", # Bombardier is more cross-platform friendly (wrk is linux only) "--application.environmentVariables", f"COMPlus_JitName={spmi_shim}", "--application.environmentVariables", "SuperPMIShimLogPath=.", @@ -338,8 +339,7 @@ def main(): print("### Running PlatformPlaintext benchmark... ###") run_crank_scenario(crank_app_path, args.tfm, core_root_path, config_path, "--scenario", "plaintext", - "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/platform.benchmarks.yml", - "--load.connections", "512" + "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/platform.benchmarks.yml" ) print("Finished running benchmarks.") From 6c507ac1035d95aff28f507d38bdb56d44b52604 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Sat, 30 Aug 2025 00:33:08 +0200 Subject: [PATCH 34/48] cleanup --- src/coreclr/scripts/superpmi_aspnet2.py | 58 +++++++++++++++---------- 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/src/coreclr/scripts/superpmi_aspnet2.py b/src/coreclr/scripts/superpmi_aspnet2.py index 546eab907c1058..1fc5c073a11136 100644 --- a/src/coreclr/scripts/superpmi_aspnet2.py +++ b/src/coreclr/scripts/superpmi_aspnet2.py @@ -118,6 +118,7 @@ def ensure_git(dest: Path) -> str: # Install the .NET SDK using the official dotnet-install script. def install_dotnet_sdk(channel: str, install_dir: Path) -> None: + print(f"Installing .NET SDK (channel: {channel}, install_dir: {install_dir})") url = "https://dot.net/v1/dotnet-install." + ("ps1" if platform.system() == "Windows" else "sh") retries = 3 initial_delay_seconds = 5 @@ -318,29 +319,40 @@ def main(): try: agent_process, crank_app_path, config_path = setup_and_run_crank_agent(temp_root) - print("### Running about-sqlite benchmark... ###") - run_crank_scenario(crank_app_path, args.tfm, core_root_path, config_path, - "--scenario", "about-sqlite", - "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/orchard.benchmarks.yml" - ) - - print("### Running JsonMVC benchmark... ###") - run_crank_scenario(crank_app_path, args.tfm, core_root_path, config_path, - "--scenario", "mvc", - "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/json.benchmarks.yml" - ) - - print("### Running NoMvcAuth benchmark... ###") - run_crank_scenario(crank_app_path, args.tfm, core_root_path, config_path, - "--scenario", "NoMvcAuth", - "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/src/BenchmarksApps/Mvc/benchmarks.jwtapi.yml" - ) - - print("### Running PlatformPlaintext benchmark... ###") - run_crank_scenario(crank_app_path, args.tfm, core_root_path, config_path, - "--scenario", "plaintext", - "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/platform.benchmarks.yml" - ) + scenarios = [ + # OrchardCMS scenario + ("about-sqlite", + # Extra args: + "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/orchard.benchmarks.yml"), + + # JsonMVC scenario + ("mvc", + # Extra args: + "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/json.benchmarks.yml"), + + # NoMvcAuth scenario + ("NoMvcAuth", + # Extra args: + "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/src/BenchmarksApps/Mvc/benchmarks.jwtapi.yml"), + + # PlatformPlaintext scenario + ("plaintext", + # Extra args: + "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/platform.benchmarks.yml"), + ] + + for entry in scenarios: + scenario_name, *extra = entry + print(f"### Running {scenario_name} benchmark... ###") + run_crank_scenario( + crank_app_path, + args.tfm, + core_root_path, + config_path, + "--scenario", + scenario_name, + *extra, + ) print("Finished running benchmarks.") From 0895c3ef2ab20a771eeb9754a73edfd37a782c67 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Sat, 30 Aug 2025 01:02:18 +0200 Subject: [PATCH 35/48] add "-NonInteractive" --- src/coreclr/scripts/superpmi_aspnet2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/scripts/superpmi_aspnet2.py b/src/coreclr/scripts/superpmi_aspnet2.py index 1fc5c073a11136..ecf85290cd2b38 100644 --- a/src/coreclr/scripts/superpmi_aspnet2.py +++ b/src/coreclr/scripts/superpmi_aspnet2.py @@ -135,7 +135,7 @@ def install_dotnet_sdk(channel: str, install_dir: Path) -> None: powershell_exe = "pwsh" if shutil.which("pwsh") else "powershell.exe" print(f"Using PowerShell executable: {powershell_exe}") args = [ - powershell_exe, "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", script_path, + powershell_exe, "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-File", script_path, "-Channel", channel, "-InstallDir", str(install_dir) ] result = subprocess.run(args, capture_output=True, text=True, check=True) From aa9fca9e5639298ade5f10cd59bb85e8fbd9e4b9 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Sat, 30 Aug 2025 08:38:31 +0200 Subject: [PATCH 36/48] fix install_dotnet_sdk --- .../templates/superpmi-collect-pipeline.yml | 2 +- src/coreclr/scripts/superpmi_aspnet2.py | 68 ++++++------------- 2 files changed, 22 insertions(+), 48 deletions(-) diff --git a/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml b/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml index 033c68cbe44190..aab59031692bc4 100644 --- a/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml +++ b/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml @@ -262,7 +262,7 @@ extends: buildConfig: checked platforms: - osx_arm64 - - linux_arm + # - linux_arm # crank doesn't support arm32 - linux_arm64 - linux_x64 - windows_x64 diff --git a/src/coreclr/scripts/superpmi_aspnet2.py b/src/coreclr/scripts/superpmi_aspnet2.py index ecf85290cd2b38..eb434353c1f6f0 100644 --- a/src/coreclr/scripts/superpmi_aspnet2.py +++ b/src/coreclr/scripts/superpmi_aspnet2.py @@ -118,52 +118,24 @@ def ensure_git(dest: Path) -> str: # Install the .NET SDK using the official dotnet-install script. def install_dotnet_sdk(channel: str, install_dir: Path) -> None: - print(f"Installing .NET SDK (channel: {channel}, install_dir: {install_dir})") - url = "https://dot.net/v1/dotnet-install." + ("ps1" if platform.system() == "Windows" else "sh") - retries = 3 - initial_delay_seconds = 5 - last_err = None - for attempt in range(1, retries + 1): - try: - with tempfile.TemporaryDirectory() as tmp: - script_path = os.path.join(tmp, os.path.basename(url)) - print(f"Downloading dotnet-install script (attempt {attempt}/{retries}) ...") - urllib.request.urlretrieve(url, script_path) - - if script_path.endswith(".ps1"): - # Find available PowerShell executable - powershell_exe = "pwsh" if shutil.which("pwsh") else "powershell.exe" - print(f"Using PowerShell executable: {powershell_exe}") - args = [ - powershell_exe, "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-File", script_path, - "-Channel", channel, "-InstallDir", str(install_dir) - ] - result = subprocess.run(args, capture_output=True, text=True, check=True) - if result.stdout: - print(result.stdout) - else: - os.chmod(script_path, 0o755) - cmd = [script_path, "-Channel", channel, "-InstallDir", str(install_dir)] - subprocess.check_call(cmd) - - # Success, return - return - except subprocess.CalledProcessError as e: - print(f"dotnet-install attempt {attempt}/{retries} failed with exit code {e.returncode}") - if getattr(e, "stdout", None): - print(e.stdout) - if getattr(e, "stderr", None): - print(e.stderr) - last_err = e - except Exception as e: - print(f"dotnet-install attempt {attempt}/{retries} failed: {e}") - last_err = e - - if attempt < retries: - delay = min(initial_delay_seconds * (2 ** (attempt - 1)), 60) - print(f"Retrying in {delay} seconds ...") - time.sleep(delay) - raise RuntimeError(f"Failed to install .NET SDK after {retries} attempts") from last_err + install_dir.mkdir(parents=True, exist_ok=True) + if os.name == "nt": + ch = channel.replace("'", "''") + di = str(install_dir).replace("'", "''") + ps_script = ( + "[System.Net.ServicePointManager]::SecurityProtocol=[System.Net.SecurityProtocolType]::Tls12;" + "Invoke-WebRequest -Uri 'https://dot.net/v1/dotnet-install.ps1' -OutFile 'dotnet-install.ps1';" + f"$DotnetVersion='{ch}';$InstallDir='{di}';" + "& './dotnet-install.ps1' -Channel $DotnetVersion -InstallDir $InstallDir -NoPath" + ) + subprocess.check_call(["powershell.exe","-NoProfile","-ExecutionPolicy", "Bypass","-Command", ps_script]) + else: + with tempfile.TemporaryDirectory() as td: + script_path = Path(td) / "dotnet-install.sh" + with urllib.request.urlopen("https://dot.net/v1/dotnet-install.sh") as resp, open(script_path, "wb") as f: + f.write(resp.read()) + os.chmod(script_path, 0o755) + subprocess.check_call([str(script_path),"--channel", channel,"--install-dir", str(install_dir),"--no-path"]) # Prepare the environment and run crank-agent @@ -188,7 +160,9 @@ def setup_and_run_crank_agent(workdir: Path): tools_dir.mkdir(parents=True, exist_ok=True) # Install .NET SDK needed for crank and crank-agent via dotnet-install public script. - install_dotnet_sdk(CRANK_SDK_CHANNEL, dotnethome_dir) + # it is currently 8.0, but let's be flexible and intall LTS as well. + install_dotnet_sdk("8.0", dotnethome_dir) + # install_dotnet_sdk("LTS", dotnethome_dir) # Determine the dotnet executable to use for installing tools dotnet_exe = dotnet_root_dir / native_exe("dotnet") From cdbc3ac8e34231ec880d255e3f3f34a0389f5452 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Sat, 30 Aug 2025 10:34:22 +0200 Subject: [PATCH 37/48] clean up --- .../coreclr/templates/run-superpmi-collect-job.yml | 2 +- src/coreclr/scripts/superpmi-collect.proj | 13 ++++++------- src/coreclr/scripts/superpmi_collect_setup.py | 12 +----------- 3 files changed, 8 insertions(+), 19 deletions(-) diff --git a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml index 7ff0fa4d1d366f..b9c99e3bf18314 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml @@ -232,4 +232,4 @@ jobs: targetPath: $(Build.SourcesDirectory)/artifacts/log artifactName: 'SuperPMI_BuildLogs_$(CollectionName)_$(CollectionType)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_Attempt$(System.JobAttempt)' condition: always() - continueOnError: true \ No newline at end of file + continueOnError: true diff --git a/src/coreclr/scripts/superpmi-collect.proj b/src/coreclr/scripts/superpmi-collect.proj index e644169b673339..0298bd9e448ee2 100644 --- a/src/coreclr/scripts/superpmi-collect.proj +++ b/src/coreclr/scripts/superpmi-collect.proj @@ -77,10 +77,10 @@ - + %HELIX_WORKITEM_PAYLOAD%\performance - + $HELIX_WORKITEM_PAYLOAD/performance @@ -161,18 +161,17 @@ + + - - - - - + + diff --git a/src/coreclr/scripts/superpmi_collect_setup.py b/src/coreclr/scripts/superpmi_collect_setup.py index 7f69f956e71a53..28a0941f672827 100644 --- a/src/coreclr/scripts/superpmi_collect_setup.py +++ b/src/coreclr/scripts/superpmi_collect_setup.py @@ -414,9 +414,6 @@ def setup_benchmark(workitem_directory, arch): run_command( get_python_name() + [dotnet_install_script, "install", "--channels", "10.0", "--architecture", arch, "--install-dir", dotnet_directory, "--verbose"]) - run_command( - get_python_name() + [dotnet_install_script, "install", "--channels", "8.0", "--architecture", arch, "--install-dir", - dotnet_directory, "--verbose"]) def get_python_name(): @@ -519,20 +516,13 @@ def main(main_args): # Need to accept files without any extension, which is how executable file's names look. acceptable_copy = lambda path: (os.path.basename(path).find(".") == -1) or any(path.endswith(extension) for extension in acceptable_extensions) - if coreclr_args.collection_name == "benchmarks" or coreclr_args.collection_name == "realworld": + if coreclr_args.collection_name == "benchmarks" or coreclr_args.collection_name == "realworld" or coreclr_args.collection_name == "aspnet2": # create a directory with release runtime bits and a checked jit print('Copying {} -> {}'.format(coreclr_args.release_core_root_directory, core_root_dst_directory)) copy_directory(coreclr_args.release_core_root_directory, core_root_dst_directory, verbose_output=True, match_func=acceptable_copy) jitname = determine_jit_name(coreclr_args.platform, coreclr_args.platform, coreclr_args.arch, coreclr_args.arch) print('Copying checked {} -> {}'.format(jitname, core_root_dst_directory)) copy_files(coreclr_args.core_root_directory, core_root_dst_directory, [os.path.join(coreclr_args.core_root_directory, jitname)]) - elif coreclr_args.collection_name == "aspnet2": - # For aspnet2, use checked runtime bits - print('Copying {} -> {}'.format(coreclr_args.release_core_root_directory, core_root_dst_directory)) - copy_directory(coreclr_args.release_core_root_directory, core_root_dst_directory, verbose_output=True, match_func=acceptable_copy) - jitname = determine_jit_name(coreclr_args.platform, coreclr_args.platform, coreclr_args.arch, coreclr_args.arch) - print('Copying checked {} -> {}'.format(jitname, core_root_dst_directory)) - copy_files(coreclr_args.core_root_directory, core_root_dst_directory, [os.path.join(coreclr_args.core_root_directory, jitname)]) else: print('Copying {} -> {}'.format(coreclr_args.core_root_directory, core_root_dst_directory)) copy_directory(coreclr_args.core_root_directory, core_root_dst_directory, verbose_output=True, match_func=acceptable_copy) From b5ab27be36ac7db3d5160fb2adc938a3270d17cf Mon Sep 17 00:00:00 2001 From: EgorBo Date: Sat, 30 Aug 2025 11:23:15 +0200 Subject: [PATCH 38/48] add --work_dir --- .../templates/superpmi-collect-pipeline.yml | 2 -- src/coreclr/scripts/superpmi-collect.proj | 11 ++++++++++- src/coreclr/scripts/superpmi_aspnet2.py | 14 +++++++++----- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml b/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml index aab59031692bc4..78871b211a98e5 100644 --- a/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml +++ b/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml @@ -262,11 +262,9 @@ extends: buildConfig: checked platforms: - osx_arm64 - # - linux_arm # crank doesn't support arm32 - linux_arm64 - linux_x64 - windows_x64 - - windows_x86 - windows_arm64 helixQueueGroup: ci helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml diff --git a/src/coreclr/scripts/superpmi-collect.proj b/src/coreclr/scripts/superpmi-collect.proj index 0298bd9e448ee2..8ef35d4bda9206 100644 --- a/src/coreclr/scripts/superpmi-collect.proj +++ b/src/coreclr/scripts/superpmi-collect.proj @@ -84,6 +84,15 @@ $HELIX_WORKITEM_PAYLOAD/performance + + + + %HELIX_WORKITEM_PAYLOAD% + + + $HELIX_WORKITEM_PAYLOAD + + @@ -149,7 +158,7 @@ - $(Python) $(SuperPMIDirectory)$(FileSeparatorChar)superpmi_aspnet2.py --core_root $(SuperPMIDirectory) + $(Python) $(SuperPMIDirectory)$(FileSeparatorChar)superpmi_aspnet2.py --core_root $(SuperPMIDirectory) --work_dir $(CrankWorkDirectory) 3:00 diff --git a/src/coreclr/scripts/superpmi_aspnet2.py b/src/coreclr/scripts/superpmi_aspnet2.py index eb434353c1f6f0..5dd659dc9eb78a 100644 --- a/src/coreclr/scripts/superpmi_aspnet2.py +++ b/src/coreclr/scripts/superpmi_aspnet2.py @@ -28,8 +28,6 @@ ########################################################################################################## CRANK_PORT = 5010 -CRANK_SDK_CHANNEL = "LTS" - # Convert a filename to the appropriate native DLL name, e.g. "clrjit" -> "libclrjit.so" (on Linux) def native_dll(name: str) -> str: @@ -160,9 +158,7 @@ def setup_and_run_crank_agent(workdir: Path): tools_dir.mkdir(parents=True, exist_ok=True) # Install .NET SDK needed for crank and crank-agent via dotnet-install public script. - # it is currently 8.0, but let's be flexible and intall LTS as well. install_dotnet_sdk("8.0", dotnethome_dir) - # install_dotnet_sdk("LTS", dotnethome_dir) # Determine the dotnet executable to use for installing tools dotnet_exe = dotnet_root_dir / native_exe("dotnet") @@ -267,6 +263,7 @@ def main(): parser.add_argument("--tfm", default="net10.0", help="Target Framework Moniker (e.g., net10.0).") parser.add_argument("--output_mch", required=True, help="File path to copy the resulting merged .mch to (expects a file path, not a directory).") parser.add_argument("--no_cleanup", action="store_true", help="If specified, do not clean up temporary files after execution.") + parser.add_argument("--work_dir", help="Optional path to a directory in which a new working directory will be created. If specified, a new subdirectory with a random name prefixed with 'aspnet2_' will be created inside this directory. Otherwise a system temp directory is used.") args = parser.parse_args() core_root_path = Path(args.core_root).expanduser().resolve() @@ -276,6 +273,8 @@ def main(): print(f"--core_root: {core_root_path}") print(f"--tfm: {args.tfm}") print(f"--output_mch: {output_mch_path}") + if args.work_dir: + print(f"--work_dir: {Path(args.work_dir).expanduser().resolve()}") mcs_cmd = core_root_path / native_exe("mcs") if not mcs_cmd.exists(): @@ -283,7 +282,12 @@ def main(): sys.exit(2) # Create or use working directory for crank_data - temp_root = Path(tempfile.mkdtemp(prefix="aspnet2_")) + if args.work_dir: + work_dir_base = Path(args.work_dir).expanduser().resolve() + work_dir_base.mkdir(parents=True, exist_ok=True) + temp_root = Path(tempfile.mkdtemp(prefix="aspnet2_", dir=str(work_dir_base))) + else: + temp_root = Path(tempfile.mkdtemp(prefix="aspnet2_")) print(f"Using temp work directory: {temp_root}") # Set current working directory to temp_root From 3740390c9267ee207ebf0365e3cd0b0dc934aabe Mon Sep 17 00:00:00 2001 From: EgorBo Date: Sun, 31 Aug 2025 06:55:30 +0200 Subject: [PATCH 39/48] fix windows --- src/coreclr/scripts/superpmi_aspnet2.py | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/src/coreclr/scripts/superpmi_aspnet2.py b/src/coreclr/scripts/superpmi_aspnet2.py index 5dd659dc9eb78a..935f2a4356ae4c 100644 --- a/src/coreclr/scripts/superpmi_aspnet2.py +++ b/src/coreclr/scripts/superpmi_aspnet2.py @@ -262,7 +262,6 @@ def main(): parser.add_argument("--core_root", required=True, help="Path to built runtime bits (CORE_ROOT).") parser.add_argument("--tfm", default="net10.0", help="Target Framework Moniker (e.g., net10.0).") parser.add_argument("--output_mch", required=True, help="File path to copy the resulting merged .mch to (expects a file path, not a directory).") - parser.add_argument("--no_cleanup", action="store_true", help="If specified, do not clean up temporary files after execution.") parser.add_argument("--work_dir", help="Optional path to a directory in which a new working directory will be created. If specified, a new subdirectory with a random name prefixed with 'aspnet2_' will be created inside this directory. Otherwise a system temp directory is used.") args = parser.parse_args() @@ -285,7 +284,7 @@ def main(): if args.work_dir: work_dir_base = Path(args.work_dir).expanduser().resolve() work_dir_base.mkdir(parents=True, exist_ok=True) - temp_root = Path(tempfile.mkdtemp(prefix="aspnet2_", dir=str(work_dir_base))) + temp_root = work_dir_base else: temp_root = Path(tempfile.mkdtemp(prefix="aspnet2_")) print(f"Using temp work directory: {temp_root}") @@ -299,9 +298,9 @@ def main(): scenarios = [ # OrchardCMS scenario - ("about-sqlite", - # Extra args: - "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/orchard.benchmarks.yml"), + # ("about-sqlite", + # # Extra args: + # "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/orchard.benchmarks.yml"), # JsonMVC scenario ("mvc", @@ -366,17 +365,8 @@ def main(): print("Cleaning up...") if 'agent_process' in locals() and agent_process is not None: agent_process.terminate() - time.sleep(5) - - # Clean up only if not suppressed - if not args.no_cleanup: - print(f'Removing temp dir {temp_root}') - # remove the entire temp_root: - shutil.rmtree(temp_root, ignore_errors=True) - else: - print(f'Not removing temp dir {temp_root} due to --no_cleanup') - - print("Done!") + print("Done!") + sys.exit() if __name__ == "__main__": main() From 755a7bf7516958c55f5c196298993b5a35b52102 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Sun, 31 Aug 2025 09:02:29 +0200 Subject: [PATCH 40/48] clean up --- src/coreclr/scripts/superpmi_aspnet2.py | 65 ++++++++++++------------- 1 file changed, 32 insertions(+), 33 deletions(-) diff --git a/src/coreclr/scripts/superpmi_aspnet2.py b/src/coreclr/scripts/superpmi_aspnet2.py index 935f2a4356ae4c..4638b0ce3d34cd 100644 --- a/src/coreclr/scripts/superpmi_aspnet2.py +++ b/src/coreclr/scripts/superpmi_aspnet2.py @@ -16,16 +16,17 @@ from pathlib import Path -########################################################################################################## +################################################################################## # -# This script sets up an environment for running crank-agent and crank-controller -# (https://github.com/dotnet/crank) locally in order to run various ASP.NET benchmarks -# (TechEmpower, OrchardCMS, etc.) and collect SPMI collections using the provided runtime bits. +# This script sets up an environment for running crank-agent and +# crank-controller (https://github.com/dotnet/crank) locally in +# order to run various ASP.NET benchmarks (TechEmpower, OrchardCMS, etc.) +# and collect SPMI collections using the provided runtime bits. # # Usage example: -# py superpmi_aspnet2.py --core_root C:\runtime\artifacts\bin\coreclr\windows.x64.Checked --output_mch aspnet.mch +# python superpmi_aspnet2.py --core_root C:\runtime\artifacts\bin\coreclr\windows.x64.Checked --output_mch aspnet2.mch # -########################################################################################################## +################################################################################## CRANK_PORT = 5010 @@ -73,6 +74,8 @@ def run(cmd, cwd=None, timeout_seconds=45*60): # Temp workaround, will be removed once https://github.com/dotnet/crank/pull/841 lands +# Our Windows Helix machines don't have git installed (and no winget) while crank relies on +# it internally to download benchmarks. def ensure_git(dest: Path) -> str: existing = shutil.which("git") if existing: @@ -114,7 +117,7 @@ def ensure_git(dest: Path) -> str: return -# Install the .NET SDK using the official dotnet-install script. +# Install .NET SDK using the official dotnet-install script. def install_dotnet_sdk(channel: str, install_dir: Path) -> None: install_dir.mkdir(parents=True, exist_ok=True) if os.name == "nt": @@ -159,13 +162,15 @@ def setup_and_run_crank_agent(workdir: Path): # Install .NET SDK needed for crank and crank-agent via dotnet-install public script. install_dotnet_sdk("8.0", dotnethome_dir) + # Be more agile and install the latest LTS version as well in case if crank moves to it + install_dotnet_sdk("LTS", dotnethome_dir) - # Determine the dotnet executable to use for installing tools + # Install crank-agent (runs benchmarks) and crank-controller (or just crank) that schedules them. dotnet_exe = dotnet_root_dir / native_exe("dotnet") run([dotnet_exe, "tool", "install", "--tool-path", str(tools_dir), "Microsoft.Crank.Agent", "--version", "0.2.0-*"]) run([dotnet_exe, "tool", "install", "--tool-path", str(tools_dir), "Microsoft.Crank.Controller", "--version", "0.2.0-*"]) - # Create a Localhost.yml to define the local environment + # Create a Localhost.yml to define the local environment since we can't access the PerfLab. yml = textwrap.dedent( f""" variables: @@ -207,19 +212,11 @@ def setup_and_run_crank_agent(workdir: Path): # Run crank scenario -def run_crank_scenario(crank_app: Path, framework: str, core_root_path: Path, config_path: Path, *extra_args: str): +def run_crank_scenario(crank_app: Path, scenario_name: str, framework: str, core_root_path: Path, config_path: Path, *extra_args: str): spmi_shim = native_dll("superpmi-shim-collector") clrjit = native_dll("clrjit") coreclr = native_dll("coreclr") spcorelib = "System.Private.CoreLib.dll" - # Try to infer scenario name from extra args for naming the output zip - scenario_name = None - for i, a in enumerate(extra_args): - if a == "--scenario" and i + 1 < len(extra_args): - scenario_name = str(extra_args[i + 1]) - break - if scenario_name is None: - scenario_name = "scenario" cmd = [ str(crank_app), "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/azure.profile.yml", @@ -231,6 +228,7 @@ def run_crank_scenario(crank_app: Path, framework: str, core_root_path: Path, co "--config", "https://raw.githubusercontent.com/dotnet/crank/main/src/Microsoft.Crank.Jobs.Bombardier/bombardier.yml", "--config", str(config_path), "--profile", "Localhost", + "--scenario", scenario_name, "--application.noGlobalJson", "false", "--application.framework", framework, "--application.Channel", "latest", # should be 'edge', but it causes random build failures sometimes. @@ -284,23 +282,24 @@ def main(): if args.work_dir: work_dir_base = Path(args.work_dir).expanduser().resolve() work_dir_base.mkdir(parents=True, exist_ok=True) - temp_root = work_dir_base else: - temp_root = Path(tempfile.mkdtemp(prefix="aspnet2_")) - print(f"Using temp work directory: {temp_root}") + # if not specified, use a temp directory + work_dir_base = Path(tempfile.mkdtemp(prefix="aspnet2_")) + print(f"Using temp work directory: {work_dir_base}") - # Set current working directory to temp_root - os.chdir(temp_root) + # Set current working directory to work_dir_base + os.chdir(work_dir_base) agent_process = None try: - agent_process, crank_app_path, config_path = setup_and_run_crank_agent(temp_root) + agent_process, crank_app_path, config_path = setup_and_run_crank_agent(work_dir_base) scenarios = [ # OrchardCMS scenario # ("about-sqlite", # # Extra args: # "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/orchard.benchmarks.yml"), + # TODO: re-enable it, currently it randomly fails with build errors. # JsonMVC scenario ("mvc", @@ -323,11 +322,10 @@ def main(): print(f"### Running {scenario_name} benchmark... ###") run_crank_scenario( crank_app_path, - args.tfm, + scenario_name, + args.tf, core_root_path, config_path, - "--scenario", - scenario_name, *extra, ) @@ -340,13 +338,13 @@ def main(): with zipfile.ZipFile(z) as f: for name in f.namelist(): if name.endswith('.mc'): - f.extract(name, str(temp_root)) + f.extract(name, str(work_dir_base)) extracted_count += 1 z.unlink(missing_ok=True) # Merge *.mc files into crank.mch if extracted_count == 0: - print("No .mc files found in zip outputs; skipping merge.") + print("Error: No .mc files found in zip outputs.") sys.exit(2) else: # Merge all .mc files into crank.mch, scanning recursively from tmp @@ -358,14 +356,15 @@ def main(): "-dedup", "-thin", str(output_mch_path), - str(temp_root / "*.mc") - ], check=True, cwd=str(temp_root)) - + str(work_dir_base / "*.mc") + ], check=True, cwd=str(work_dir_base)) + print(f"Finished merging .mc files into {output_mch_path}") + finally: print("Cleaning up...") if 'agent_process' in locals() and agent_process is not None: agent_process.terminate() - print("Done!") + print("Done.") sys.exit() if __name__ == "__main__": From efee87d8a8dedc1cfed063e3447d8ead8010daaa Mon Sep 17 00:00:00 2001 From: EgorBo Date: Sun, 31 Aug 2025 10:41:25 +0200 Subject: [PATCH 41/48] clean up --- src/coreclr/scripts/superpmi_aspnet2.py | 181 +++++++++++++----------- 1 file changed, 100 insertions(+), 81 deletions(-) diff --git a/src/coreclr/scripts/superpmi_aspnet2.py b/src/coreclr/scripts/superpmi_aspnet2.py index 4638b0ce3d34cd..305a458fd7e827 100644 --- a/src/coreclr/scripts/superpmi_aspnet2.py +++ b/src/coreclr/scripts/superpmi_aspnet2.py @@ -44,33 +44,19 @@ def native_exe(name: str) -> str: # Run a command -def run(cmd, cwd=None, timeout_seconds=45*60): +def run(cmd): print(f"Running command: {' '.join(map(str, cmd))}") - kwargs = { - "stdin": subprocess.DEVNULL, - "stdout": sys.stdout, - "stderr": subprocess.STDOUT, - "cwd": cwd, - } - # It doesn't download the resulting artifacts without this: - if os.name == "nt": - kwargs["creationflags"] = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) + kwargs = {} + if sys.platform == "win32": + kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP else: kwargs["start_new_session"] = True - proc = subprocess.Popen(cmd, **kwargs) + proc = subprocess.Popen(cmd, shell=True, **kwargs) try: - # Wait with a timeout; let the TimeoutExpired propagate so caller can decide what to do. - return proc.wait(timeout=timeout_seconds) - except subprocess.TimeoutExpired: - print(f"Process timed out after {timeout_seconds} seconds. Leaving process running and raising exception.") - raise + proc.wait() except KeyboardInterrupt: - try: - proc.terminate() - except Exception: - pass + proc.terminate() print("Process terminated") - return None # Temp workaround, will be removed once https://github.com/dotnet/crank/pull/841 lands @@ -120,7 +106,7 @@ def ensure_git(dest: Path) -> str: # Install .NET SDK using the official dotnet-install script. def install_dotnet_sdk(channel: str, install_dir: Path) -> None: install_dir.mkdir(parents=True, exist_ok=True) - if os.name == "nt": + if sys.platform == "win32": ch = channel.replace("'", "''") di = str(install_dir).replace("'", "''") ps_script = ( @@ -162,7 +148,7 @@ def setup_and_run_crank_agent(workdir: Path): # Install .NET SDK needed for crank and crank-agent via dotnet-install public script. install_dotnet_sdk("8.0", dotnethome_dir) - # Be more agile and install the latest LTS version as well in case if crank moves to it + # Be more flexible and install the latest LTS version as well in case if crank moves to it install_dotnet_sdk("LTS", dotnethome_dir) # Install crank-agent (runs benchmarks) and crank-controller (or just crank) that schedules them. @@ -197,14 +183,15 @@ def setup_and_run_crank_agent(workdir: Path): print("Starting crank-agent...") - agent_process = subprocess.Popen( + agent_process = subprocess.Popen( [ str(tools_dir / native_exe("crank-agent")), "--url", f"http://*:{CRANK_PORT}", "--log-path", str(logs_dir), "--build-path", str(build_dir), "--dotnethome", str(dotnethome_dir), - ] + ], + shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, stdin=subprocess.DEVNULL ) print(f"Waiting 10s for crank-agent to start ...") time.sleep(10) @@ -212,7 +199,7 @@ def setup_and_run_crank_agent(workdir: Path): # Run crank scenario -def run_crank_scenario(crank_app: Path, scenario_name: str, framework: str, core_root_path: Path, config_path: Path, *extra_args: str): +def run_crank_scenario(crank_app: Path, scenario_name: str, framework: str, core_root_path: Path, config_path: Path, dryrun: bool, *extra_args: str): spmi_shim = native_dll("superpmi-shim-collector") clrjit = native_dll("clrjit") coreclr = native_dll("coreclr") @@ -235,19 +222,26 @@ def run_crank_scenario(crank_app: Path, scenario_name: str, framework: str, core "--application.options.collectCounters", "false", "--application.collectDependencies", "false", "--load.options.reuseBuild", "true", - "--load.variables.duration", "30", # default 15s is not enough for Tier1 promotion + "--load.variables.duration", "10", + "--load.variables.warmup", "10", "--load.connections", "32", "--load.job", "bombardier", # Bombardier is more cross-platform friendly (wrk is linux only) - "--application.environmentVariables", f"COMPlus_JitName={spmi_shim}", - "--application.environmentVariables", "SuperPMIShimLogPath=.", - "--application.environmentVariables", f"SuperPMIShimPath=./{clrjit}", - "--application.options.fetch", "true", - "--application.options.fetchOutput", scenario_name + ".crank.zip", - "--application.options.outputFiles", str(core_root_path / spmi_shim), - "--application.options.outputFiles", str(core_root_path / clrjit), - "--application.options.outputFiles", str(core_root_path / coreclr), - "--application.options.outputFiles", str(core_root_path / spcorelib), ] + + # Only add SPMI collection environment variables and output files if not in dry run mode + if not dryrun: + cmd.extend([ + "--application.environmentVariables", f"COMPlus_JitName={spmi_shim}", + "--application.environmentVariables", "SuperPMIShimLogPath=.", + "--application.environmentVariables", f"SuperPMIShimPath=./{clrjit}", + "--application.options.fetch", "true", + "--application.options.fetchOutput", scenario_name + ".crank.zip", + "--application.options.outputFiles", str(core_root_path / spmi_shim), + "--application.options.outputFiles", str(core_root_path / clrjit), + "--application.options.outputFiles", str(core_root_path / coreclr), + "--application.options.outputFiles", str(core_root_path / spcorelib), + ]) + # Append any extra scenario-specific arguments if extra_args: cmd.extend(extra_args) @@ -257,26 +251,38 @@ def run_crank_scenario(crank_app: Path, scenario_name: str, framework: str, core # Main entry point def main(): parser = argparse.ArgumentParser(description="Cross-platform crank runner.") - parser.add_argument("--core_root", required=True, help="Path to built runtime bits (CORE_ROOT).") + parser.add_argument("--core_root", help="Path to built runtime bits (CORE_ROOT).") parser.add_argument("--tfm", default="net10.0", help="Target Framework Moniker (e.g., net10.0).") - parser.add_argument("--output_mch", required=True, help="File path to copy the resulting merged .mch to (expects a file path, not a directory).") + parser.add_argument("--output_mch", help="File path to copy the resulting merged .mch to (expects a file path, not a directory).") parser.add_argument("--work_dir", help="Optional path to a directory in which a new working directory will be created. If specified, a new subdirectory with a random name prefixed with 'aspnet2_' will be created inside this directory. Otherwise a system temp directory is used.") + parser.add_argument("--dryrun", action="store_true", help="Run benchmarks only without collecting SPMI data or generating .mch files.") args = parser.parse_args() - core_root_path = Path(args.core_root).expanduser().resolve() - output_mch_path = Path(args.output_mch).expanduser().resolve() + # Validate required arguments when not in dry run mode + if not args.dryrun: + if not args.core_root: + parser.error("--core_root is required when not using --dryrun") + if not args.output_mch: + parser.error("--output_mch is required when not using --dryrun") + + core_root_path = Path(args.core_root).expanduser().resolve() if args.core_root else None + output_mch_path = Path(args.output_mch).expanduser().resolve() if args.output_mch else None print("Running the script with the following parameters:") - print(f"--core_root: {core_root_path}") + if args.core_root: + print(f"--core_root: {core_root_path}") print(f"--tfm: {args.tfm}") - print(f"--output_mch: {output_mch_path}") + if args.output_mch: + print(f"--output_mch: {output_mch_path}") if args.work_dir: print(f"--work_dir: {Path(args.work_dir).expanduser().resolve()}") + print(f"--dryrun: {args.dryrun}") - mcs_cmd = core_root_path / native_exe("mcs") - if not mcs_cmd.exists(): - print(f"Error: mcs[.exe] not found at {mcs_cmd}. Ensure runtime bits include mcs.", file=sys.stderr) - sys.exit(2) + if not args.dryrun: + mcs_cmd = core_root_path / native_exe("mcs") + if not mcs_cmd.exists(): + print(f"Error: mcs[.exe] not found at {mcs_cmd}. Ensure runtime bits include mcs.", file=sys.stderr) + sys.exit(2) # Create or use working directory for crank_data if args.work_dir: @@ -307,14 +313,14 @@ def main(): "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/json.benchmarks.yml"), # NoMvcAuth scenario - ("NoMvcAuth", - # Extra args: - "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/src/BenchmarksApps/Mvc/benchmarks.jwtapi.yml"), + # ("NoMvcAuth", + # # Extra args: + # "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/src/BenchmarksApps/Mvc/benchmarks.jwtapi.yml"), - # PlatformPlaintext scenario - ("plaintext", - # Extra args: - "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/platform.benchmarks.yml"), + # # PlatformPlaintext scenario + # ("plaintext", + # # Extra args: + # "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/platform.benchmarks.yml"), ] for entry in scenarios: @@ -323,47 +329,60 @@ def main(): run_crank_scenario( crank_app_path, scenario_name, - args.tf, + args.tfm, core_root_path, config_path, + args.dryrun, *extra, ) print("Finished running benchmarks.") - # Extract .mc files from zip archives into crank_data/tmp instead of the current directory - print("Extracting .mc files from zip archives...") - extracted_count = 0 - for z in pathlib.Path('.').glob('*.crank.zip'): - with zipfile.ZipFile(z) as f: - for name in f.namelist(): - if name.endswith('.mc'): - f.extract(name, str(work_dir_base)) - extracted_count += 1 - z.unlink(missing_ok=True) - - # Merge *.mc files into crank.mch - if extracted_count == 0: - print("Error: No .mc files found in zip outputs.") - sys.exit(2) + # Skip .mc/.mch processing in dry run mode + if not args.dryrun: + # Extract .mc files from zip archives into crank_data/tmp instead of the current directory + print("Extracting .mc files from zip archives...") + extracted_count = 0 + for z in pathlib.Path('.').glob('*.crank.zip'): + with zipfile.ZipFile(z) as f: + for name in f.namelist(): + if name.endswith('.mc'): + f.extract(name, str(work_dir_base)) + extracted_count += 1 + z.unlink(missing_ok=True) + + # Merge *.mc files into crank.mch + if extracted_count == 0: + print("Error: No .mc files found in zip outputs.") + sys.exit(2) + else: + # Merge all .mc files into crank.mch, scanning recursively from tmp + print(f"Extracted {extracted_count} .mc files; merging into crank.mch ...") + run([ + str(mcs_cmd), + "-merge", + "-recursive", + "-dedup", + "-thin", + str(output_mch_path), + str(work_dir_base / "*.mc") + ]) + print(f"Finished merging .mc files into {output_mch_path}") else: - # Merge all .mc files into crank.mch, scanning recursively from tmp - print(f"Extracted {extracted_count} .mc files; merging into crank.mch ...") - subprocess.run([ - str(mcs_cmd), - "-merge", - "-recursive", - "-dedup", - "-thin", - str(output_mch_path), - str(work_dir_base / "*.mc") - ], check=True, cwd=str(work_dir_base)) - print(f"Finished merging .mc files into {output_mch_path}") + print("Dry run mode: Skipping SPMI data collection and .mch file generation.") + except Exception as e: + print(f"Error: {e}", file=sys.stderr) finally: print("Cleaning up...") if 'agent_process' in locals() and agent_process is not None: + print(f"Terminating agent process {agent_process.pid}...") agent_process.terminate() + time.sleep(10) + print(f"Agent process {agent_process.pid} terminated.") + # validate that mch file was created under non-dryrun: + if not args.dryrun and output_mch_path.exists(): + print(f"Successfully created {output_mch_path}") print("Done.") sys.exit() From 30cdc842c108768ae665b8af49e55f6c3c057118 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Sun, 31 Aug 2025 11:49:01 +0200 Subject: [PATCH 42/48] ts --- src/coreclr/scripts/superpmi_aspnet2.py | 36 ++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/src/coreclr/scripts/superpmi_aspnet2.py b/src/coreclr/scripts/superpmi_aspnet2.py index 305a458fd7e827..79a7e3e35cfaeb 100644 --- a/src/coreclr/scripts/superpmi_aspnet2.py +++ b/src/coreclr/scripts/superpmi_aspnet2.py @@ -51,11 +51,19 @@ def run(cmd): kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP else: kwargs["start_new_session"] = True - proc = subprocess.Popen(cmd, shell=True, **kwargs) + + proc = subprocess.Popen(cmd, **kwargs) try: proc.wait() except KeyboardInterrupt: + print("Keyboard interrupt received, terminating process...") proc.terminate() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + print("Process did not terminate gracefully, force killing...") + proc.kill() + proc.wait() print("Process terminated") @@ -191,7 +199,9 @@ def setup_and_run_crank_agent(workdir: Path): "--build-path", str(build_dir), "--dotnethome", str(dotnethome_dir), ], - shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, stdin=subprocess.DEVNULL + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, stdin=subprocess.DEVNULL, + creationflags=subprocess.CREATE_NEW_PROCESS_GROUP if sys.platform == "win32" else 0, + start_new_session=sys.platform != "win32" ) print(f"Waiting 10s for crank-agent to start ...") time.sleep(10) @@ -378,8 +388,26 @@ def main(): if 'agent_process' in locals() and agent_process is not None: print(f"Terminating agent process {agent_process.pid}...") agent_process.terminate() - time.sleep(10) - print(f"Agent process {agent_process.pid} terminated.") + try: + # Wait for process to terminate gracefully + agent_process.wait(timeout=10) + print(f"Agent process {agent_process.pid} terminated gracefully.") + except subprocess.TimeoutExpired: + print(f"Agent process {agent_process.pid} did not terminate gracefully, force killing...") + agent_process.kill() + try: + agent_process.wait(timeout=5) + print(f"Agent process {agent_process.pid} force killed.") + except subprocess.TimeoutExpired: + print(f"Warning: Agent process {agent_process.pid} could not be killed.") + if sys.platform == "win32": + # On Windows, try to kill the entire process tree + try: + subprocess.run(["taskkill", "/F", "/T", "/PID", str(agent_process.pid)], + check=False, capture_output=True) + print(f"Used taskkill to terminate process tree for PID {agent_process.pid}") + except Exception as e: + print(f"Failed to use taskkill: {e}") # validate that mch file was created under non-dryrun: if not args.dryrun and output_mch_path.exists(): print(f"Successfully created {output_mch_path}") From 254a961698c6b5618b9f26b3e47f7c1dc730f4bd Mon Sep 17 00:00:00 2001 From: EgorBo Date: Mon, 1 Sep 2025 00:48:13 +0200 Subject: [PATCH 43/48] fix windows --- src/coreclr/scripts/superpmi_aspnet2.py | 207 ++++++++++++------------ 1 file changed, 99 insertions(+), 108 deletions(-) diff --git a/src/coreclr/scripts/superpmi_aspnet2.py b/src/coreclr/scripts/superpmi_aspnet2.py index 79a7e3e35cfaeb..200204bff34178 100644 --- a/src/coreclr/scripts/superpmi_aspnet2.py +++ b/src/coreclr/scripts/superpmi_aspnet2.py @@ -1,4 +1,21 @@ #!/usr/bin/env python3 +# +# Licensed to the .NET Foundation under one or more agreements. +# The .NET Foundation licenses this file to you under the MIT license. +# +# +# Title: superpmi_aspnet2.py +# +# Notes: +# +# Script to perform the superpmi collection for Techempower Benchmarks +# via "crank" (https://github.com/dotnet/crank). +# +# Usage example: +# +# python superpmi_aspnet2.py --core_root C:\runtime\artifacts\bin\coreclr\windows.x64.Checked --output_mch aspnet2.mch +# + import argparse import os import re @@ -16,20 +33,12 @@ from pathlib import Path -################################################################################## -# -# This script sets up an environment for running crank-agent and -# crank-controller (https://github.com/dotnet/crank) locally in -# order to run various ASP.NET benchmarks (TechEmpower, OrchardCMS, etc.) -# and collect SPMI collections using the provided runtime bits. -# -# Usage example: -# python superpmi_aspnet2.py --core_root C:\runtime\artifacts\bin\coreclr\windows.x64.Checked --output_mch aspnet2.mch -# -################################################################################## - CRANK_PORT = 5010 +# Global variable to track the agent process for cleanup +_crank_agent_process = None + + # Convert a filename to the appropriate native DLL name, e.g. "clrjit" -> "libclrjit.so" (on Linux) def native_dll(name: str) -> str: ext = ".dll" if sys.platform.startswith("win") else (".dylib" if sys.platform == "darwin" else ".so") @@ -46,26 +55,11 @@ def native_exe(name: str) -> str: # Run a command def run(cmd): print(f"Running command: {' '.join(map(str, cmd))}") - kwargs = {} - if sys.platform == "win32": - kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP - else: - kwargs["start_new_session"] = True - - proc = subprocess.Popen(cmd, **kwargs) - try: - proc.wait() - except KeyboardInterrupt: - print("Keyboard interrupt received, terminating process...") - proc.terminate() - try: - proc.wait(timeout=5) - except subprocess.TimeoutExpired: - print("Process did not terminate gracefully, force killing...") - proc.kill() - proc.wait() - print("Process terminated") - + proc = subprocess.Popen(cmd) + output,error = proc.communicate() + if proc.returncode != 0: + print("command failed") + # Temp workaround, will be removed once https://github.com/dotnet/crank/pull/841 lands # Our Windows Helix machines don't have git installed (and no winget) while crank relies on @@ -123,14 +117,14 @@ def install_dotnet_sdk(channel: str, install_dir: Path) -> None: f"$DotnetVersion='{ch}';$InstallDir='{di}';" "& './dotnet-install.ps1' -Channel $DotnetVersion -InstallDir $InstallDir -NoPath" ) - subprocess.check_call(["powershell.exe","-NoProfile","-ExecutionPolicy", "Bypass","-Command", ps_script]) + run(["powershell.exe","-NoProfile","-ExecutionPolicy", "Bypass","-Command", ps_script]) else: with tempfile.TemporaryDirectory() as td: script_path = Path(td) / "dotnet-install.sh" with urllib.request.urlopen("https://dot.net/v1/dotnet-install.sh") as resp, open(script_path, "wb") as f: f.write(resp.read()) os.chmod(script_path, 0o755) - subprocess.check_call([str(script_path),"--channel", channel,"--install-dir", str(install_dir),"--no-path"]) + run([str(script_path),"--channel", channel,"--install-dir", str(install_dir),"--no-path"]) # Prepare the environment and run crank-agent @@ -191,6 +185,16 @@ def setup_and_run_crank_agent(workdir: Path): print("Starting crank-agent...") + # Create process with proper flags for Windows process management + if sys.platform == "win32": + # On Windows, create a new process group and detach from console + creation_flags = subprocess.CREATE_NEW_PROCESS_GROUP | subprocess.DETACHED_PROCESS + start_new_session = False + else: + # On Unix-like systems, start a new session + creation_flags = 0 + start_new_session = True + agent_process = subprocess.Popen( [ str(tools_dir / native_exe("crank-agent")), @@ -200,8 +204,8 @@ def setup_and_run_crank_agent(workdir: Path): "--dotnethome", str(dotnethome_dir), ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, stdin=subprocess.DEVNULL, - creationflags=subprocess.CREATE_NEW_PROCESS_GROUP if sys.platform == "win32" else 0, - start_new_session=sys.platform != "win32" + creationflags=creation_flags, + start_new_session=start_new_session ) print(f"Waiting 10s for crank-agent to start ...") time.sleep(10) @@ -209,7 +213,7 @@ def setup_and_run_crank_agent(workdir: Path): # Run crank scenario -def run_crank_scenario(crank_app: Path, scenario_name: str, framework: str, core_root_path: Path, config_path: Path, dryrun: bool, *extra_args: str): +def run_crank_scenario(crank_app: Path, scenario_name: str, framework: str, work_dir: Path, core_root_path: Path, config_path: Path, dryrun: bool, *extra_args: str): spmi_shim = native_dll("superpmi-shim-collector") clrjit = native_dll("clrjit") coreclr = native_dll("coreclr") @@ -232,9 +236,9 @@ def run_crank_scenario(crank_app: Path, scenario_name: str, framework: str, core "--application.options.collectCounters", "false", "--application.collectDependencies", "false", "--load.options.reuseBuild", "true", - "--load.variables.duration", "10", - "--load.variables.warmup", "10", - "--load.connections", "32", + "--load.variables.duration", "45", + "--load.variables.warmup", "15", + "--load.connections", "64", "--load.job", "bombardier", # Bombardier is more cross-platform friendly (wrk is linux only) ] @@ -242,10 +246,8 @@ def run_crank_scenario(crank_app: Path, scenario_name: str, framework: str, core if not dryrun: cmd.extend([ "--application.environmentVariables", f"COMPlus_JitName={spmi_shim}", - "--application.environmentVariables", "SuperPMIShimLogPath=.", + "--application.environmentVariables", f"SuperPMIShimLogPath={str(work_dir)}", "--application.environmentVariables", f"SuperPMIShimPath=./{clrjit}", - "--application.options.fetch", "true", - "--application.options.fetchOutput", scenario_name + ".crank.zip", "--application.options.outputFiles", str(core_root_path / spmi_shim), "--application.options.outputFiles", str(core_root_path / clrjit), "--application.options.outputFiles", str(core_root_path / coreclr), @@ -306,9 +308,8 @@ def main(): # Set current working directory to work_dir_base os.chdir(work_dir_base) - agent_process = None try: - agent_process, crank_app_path, config_path = setup_and_run_crank_agent(work_dir_base) + _crank_agent_process, crank_app_path, config_path = setup_and_run_crank_agent(work_dir_base) scenarios = [ # OrchardCMS scenario @@ -323,14 +324,14 @@ def main(): "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/json.benchmarks.yml"), # NoMvcAuth scenario - # ("NoMvcAuth", - # # Extra args: - # "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/src/BenchmarksApps/Mvc/benchmarks.jwtapi.yml"), + ("NoMvcAuth", + # Extra args: + "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/src/BenchmarksApps/Mvc/benchmarks.jwtapi.yml"), - # # PlatformPlaintext scenario - # ("plaintext", - # # Extra args: - # "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/platform.benchmarks.yml"), + # PlatformPlaintext scenario + ("plaintext", + # Extra args: + "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/platform.benchmarks.yml"), ] for entry in scenarios: @@ -340,6 +341,7 @@ def main(): crank_app_path, scenario_name, args.tfm, + work_dir_base, core_root_path, config_path, args.dryrun, @@ -348,71 +350,60 @@ def main(): print("Finished running benchmarks.") + except Exception as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) + finally: + print("Cleaning up...") + if _crank_agent_process is not None: + print(f"Terminating agent process {_crank_agent_process.pid}...") + if sys.platform == "win32": + # On Windows, use taskkill first to terminate the entire process tree + try: + result = subprocess.run(["taskkill", "/F", "/T", "/PID", str(_crank_agent_process.pid)]) + except Exception as e: + print(f"Failed to use taskkill: {e}") + + try: + _crank_agent_process.terminate() + _crank_agent_process.wait(timeout=10) + print(f"Agent process {_crank_agent_process.pid} terminated gracefully.") + except Exception as e: + print(f"Error during standard termination: {e}") + + # Clear the global reference + _crank_agent_process = None + # Skip .mc/.mch processing in dry run mode if not args.dryrun: - # Extract .mc files from zip archives into crank_data/tmp instead of the current directory - print("Extracting .mc files from zip archives...") - extracted_count = 0 - for z in pathlib.Path('.').glob('*.crank.zip'): - with zipfile.ZipFile(z) as f: - for name in f.namelist(): - if name.endswith('.mc'): - f.extract(name, str(work_dir_base)) - extracted_count += 1 - z.unlink(missing_ok=True) - - # Merge *.mc files into crank.mch - if extracted_count == 0: - print("Error: No .mc files found in zip outputs.") + # Count number of *.mc files in work_dir_base: + mc_file_count = len(list(work_dir_base.glob("*.mc"))) + if mc_file_count == 0: + print("Error: No .mc files found.") sys.exit(2) - else: - # Merge all .mc files into crank.mch, scanning recursively from tmp - print(f"Extracted {extracted_count} .mc files; merging into crank.mch ...") - run([ - str(mcs_cmd), - "-merge", - "-recursive", - "-dedup", - "-thin", - str(output_mch_path), - str(work_dir_base / "*.mc") - ]) + + print(f"Merging {mc_file_count} .mc files into {output_mch_path}...") + run([ + str(mcs_cmd), + "-merge", + "-recursive", + "-dedup", + "-thin", + str(output_mch_path), + str(work_dir_base / "*.mc") + ]) print(f"Finished merging .mc files into {output_mch_path}") + # delete all .mc files + for mc_file in work_dir_base.glob("*.mc"): + mc_file.unlink() else: print("Dry run mode: Skipping SPMI data collection and .mch file generation.") - - except Exception as e: - print(f"Error: {e}", file=sys.stderr) - finally: - print("Cleaning up...") - if 'agent_process' in locals() and agent_process is not None: - print(f"Terminating agent process {agent_process.pid}...") - agent_process.terminate() - try: - # Wait for process to terminate gracefully - agent_process.wait(timeout=10) - print(f"Agent process {agent_process.pid} terminated gracefully.") - except subprocess.TimeoutExpired: - print(f"Agent process {agent_process.pid} did not terminate gracefully, force killing...") - agent_process.kill() - try: - agent_process.wait(timeout=5) - print(f"Agent process {agent_process.pid} force killed.") - except subprocess.TimeoutExpired: - print(f"Warning: Agent process {agent_process.pid} could not be killed.") - if sys.platform == "win32": - # On Windows, try to kill the entire process tree - try: - subprocess.run(["taskkill", "/F", "/T", "/PID", str(agent_process.pid)], - check=False, capture_output=True) - print(f"Used taskkill to terminate process tree for PID {agent_process.pid}") - except Exception as e: - print(f"Failed to use taskkill: {e}") + # validate that mch file was created under non-dryrun: - if not args.dryrun and output_mch_path.exists(): + if not args.dryrun and output_mch_path and output_mch_path.exists(): print(f"Successfully created {output_mch_path}") print("Done.") - sys.exit() if __name__ == "__main__": main() + sys.exit(0) From 4055e18b67a0d6900460ff2366c82224a7f4812f Mon Sep 17 00:00:00 2001 From: EgorBo Date: Mon, 1 Sep 2025 05:55:21 +0200 Subject: [PATCH 44/48] final --- .../templates/superpmi-collect-pipeline.yml | 228 ++++++++++++++++++ src/coreclr/scripts/superpmi-collect.proj | 2 - src/coreclr/scripts/superpmi_aspnet2.py | 5 +- 3 files changed, 231 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml b/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml index 78871b211a98e5..3f260420a14a0e 100644 --- a/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml +++ b/eng/pipelines/coreclr/templates/superpmi-collect-pipeline.yml @@ -255,6 +255,45 @@ extends: jobParameters: testGroup: outerloop + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml + buildConfig: checked + platforms: + - osx_arm64 + - linux_arm + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + helixQueueGroup: ci + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: outerloop + liveLibrariesBuildConfig: Release + collectionType: pmi + collectionName: libraries + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml + buildConfig: checked + platforms: + - osx_arm64 + - linux_arm + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + helixQueueGroup: ci + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: outerloop + liveLibrariesBuildConfig: Release + collectionType: crossgen2 + collectionName: libraries - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -262,9 +301,198 @@ extends: buildConfig: checked platforms: - osx_arm64 + - linux_arm - linux_arm64 - linux_x64 - windows_x64 + - windows_x86 + - windows_arm64 + helixQueueGroup: ci + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: outerloop + liveLibrariesBuildConfig: Release + collectionType: run + collectionName: realworld + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml + buildConfig: checked + platforms: + - osx_arm64 + - linux_arm + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + helixQueueGroup: ci + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: outerloop + liveLibrariesBuildConfig: Release + collectionType: run + collectionName: benchmarks + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml + buildConfig: checked + platforms: + - osx_arm64 + - linux_arm + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + helixQueueGroup: ci + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: outerloop + liveLibrariesBuildConfig: Release + collectionType: run_pgo + collectionName: benchmarks + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml + buildConfig: checked + platforms: + - osx_arm64 + - linux_arm + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + helixQueueGroup: ci + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: outerloop + liveLibrariesBuildConfig: Release + collectionType: run_pgo_optrepeat + collectionName: benchmarks + + # + # Collection of coreclr test run + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml + buildConfig: checked + platforms: + - osx_arm64 + - linux_arm + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + helixQueueGroup: superpmi + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: outerloop + liveLibrariesBuildConfig: Release + SuperPmiCollect: true + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml + buildConfig: checked + platforms: + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_arm64 + helixQueueGroup: ci + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: outerloop + liveLibrariesBuildConfig: Release + collectionType: nativeaot + collectionName: smoke_tests + + # + # Collection of libraries test run: normal + # Libraries Test Run using Release libraries, and Checked CoreCLR + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/libraries/run-test-job.yml + buildConfig: Release + platforms: + - osx_arm64 + - linux_arm + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + helixQueueGroup: superpmi + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testScope: innerloop + liveRuntimeBuildConfig: Checked + dependsOnTestBuildConfiguration: Release + dependsOnTestArchitecture: x64 + scenarios: + - normal + SuperPmiCollect: true + SuperPmiCollectionName: libraries_tests + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked + helixArtifactsName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked + unifiedBuildConfigOverride: checked + # Default timeout is 150 minutes, which is too low for osx-arm64 queue. + timeoutInMinutes: 300 + + # + # Collection of libraries test run: no_tiered_compilation + # Libraries Test Run using Release libraries, and Checked CoreCLR + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/libraries/run-test-job.yml + buildConfig: Release + platforms: + - osx_arm64 + - linux_arm + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + helixQueueGroup: superpmi + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testScope: innerloop + liveRuntimeBuildConfig: Checked + dependsOnTestBuildConfiguration: Release + dependsOnTestArchitecture: x64 + scenarios: + - no_tiered_compilation + SuperPmiCollect: true + SuperPmiCollectionName: libraries_tests_no_tiered_compilation + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked + helixArtifactsName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked + unifiedBuildConfigOverride: checked + # Default timeout is 150 minutes, which is too low for osx-arm64 queue. + timeoutInMinutes: 300 + + # + # Collection of aspnet/TechEmpower benchmarks (crank-based): aspnet2 + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml + buildConfig: checked + platforms: + - osx_arm64 + - linux_arm64 + # - linux_x64 # currently runs of disk space too often. + - windows_x64 - windows_arm64 helixQueueGroup: ci helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml diff --git a/src/coreclr/scripts/superpmi-collect.proj b/src/coreclr/scripts/superpmi-collect.proj index 8ef35d4bda9206..72972e4d09395f 100644 --- a/src/coreclr/scripts/superpmi-collect.proj +++ b/src/coreclr/scripts/superpmi-collect.proj @@ -172,9 +172,7 @@ - - diff --git a/src/coreclr/scripts/superpmi_aspnet2.py b/src/coreclr/scripts/superpmi_aspnet2.py index 200204bff34178..2e79da3aab6ddb 100644 --- a/src/coreclr/scripts/superpmi_aspnet2.py +++ b/src/coreclr/scripts/superpmi_aspnet2.py @@ -230,10 +230,8 @@ def run_crank_scenario(crank_app: Path, scenario_name: str, framework: str, work "--config", str(config_path), "--profile", "Localhost", "--scenario", scenario_name, - "--application.noGlobalJson", "false", "--application.framework", framework, "--application.Channel", "latest", # should be 'edge', but it causes random build failures sometimes. - "--application.options.collectCounters", "false", "--application.collectDependencies", "false", "--load.options.reuseBuild", "true", "--load.variables.duration", "45", @@ -305,6 +303,9 @@ def main(): work_dir_base = Path(tempfile.mkdtemp(prefix="aspnet2_")) print(f"Using temp work directory: {work_dir_base}") + work_dir_base = work_dir_base / "crank_data" + work_dir_base.mkdir(parents=True, exist_ok=True) + # Set current working directory to work_dir_base os.chdir(work_dir_base) From 96ee4c81b762c6a42cd480b94741c0953086cbbd Mon Sep 17 00:00:00 2001 From: EgorBo Date: Mon, 1 Sep 2025 06:58:06 +0200 Subject: [PATCH 45/48] enable orchardcms --- src/coreclr/scripts/superpmi_aspnet2.py | 35 ++++++++++++++++++------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/src/coreclr/scripts/superpmi_aspnet2.py b/src/coreclr/scripts/superpmi_aspnet2.py index 2e79da3aab6ddb..cca55206485ec5 100644 --- a/src/coreclr/scripts/superpmi_aspnet2.py +++ b/src/coreclr/scripts/superpmi_aspnet2.py @@ -52,14 +52,28 @@ def native_exe(name: str) -> str: return f"{name}{ext}" -# Run a command +# Run a command with retries def run(cmd): + retries = 3 print(f"Running command: {' '.join(map(str, cmd))}") - proc = subprocess.Popen(cmd) - output,error = proc.communicate() - if proc.returncode != 0: - print("command failed") - + attempt = 0 + while True: + try: + proc = subprocess.Popen(cmd) + output, error = proc.communicate() + returncode = proc.returncode + except Exception as e: + print(f"Failed to start command: {e}") + returncode = -1 + if returncode == 0: + return + attempt += 1 + if attempt > retries: + print(f"command failed after {retries} attempts") + break + print(f"Command failed with return code {returncode}, retrying ({attempt}/{retries}) after {delay_seconds}s...") + time.sleep(3) + # Temp workaround, will be removed once https://github.com/dotnet/crank/pull/841 lands # Our Windows Helix machines don't have git installed (and no winget) while crank relies on @@ -232,7 +246,9 @@ def run_crank_scenario(crank_app: Path, scenario_name: str, framework: str, work "--scenario", scenario_name, "--application.framework", framework, "--application.Channel", "latest", # should be 'edge', but it causes random build failures sometimes. + "--application.noGlobalJson", "false", "--application.collectDependencies", "false", + "--application.options.collectCounters", "false", "--load.options.reuseBuild", "true", "--load.variables.duration", "45", "--load.variables.warmup", "15", @@ -314,10 +330,9 @@ def main(): scenarios = [ # OrchardCMS scenario - # ("about-sqlite", - # # Extra args: - # "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/orchard.benchmarks.yml"), - # TODO: re-enable it, currently it randomly fails with build errors. + ("about-sqlite", + # Extra args: + "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/orchard.benchmarks.yml"), # JsonMVC scenario ("mvc", From 810269fd9e7876f39a2e9399d3d1bad9c99d05e5 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Mon, 1 Sep 2025 09:45:15 +0200 Subject: [PATCH 46/48] Final commit --- src/coreclr/scripts/superpmi-collect.proj | 2 +- src/coreclr/scripts/superpmi_aspnet2.py | 58 +++++++++++++++++------ 2 files changed, 44 insertions(+), 16 deletions(-) diff --git a/src/coreclr/scripts/superpmi-collect.proj b/src/coreclr/scripts/superpmi-collect.proj index 72972e4d09395f..6a72dd505a9060 100644 --- a/src/coreclr/scripts/superpmi-collect.proj +++ b/src/coreclr/scripts/superpmi-collect.proj @@ -330,7 +330,7 @@ $(WorkItemDirectory) $(WorkItemCommand) --output_mch $(OutputMchPath)$(FileSeparatorChar)%(OutputFileName).mch $(WorkItemTimeout) - %(OutputFileName).mch + %(OutputFileName).mch;%(OutputFileName).mch.mct diff --git a/src/coreclr/scripts/superpmi_aspnet2.py b/src/coreclr/scripts/superpmi_aspnet2.py index cca55206485ec5..ee5193b0d398bb 100644 --- a/src/coreclr/scripts/superpmi_aspnet2.py +++ b/src/coreclr/scripts/superpmi_aspnet2.py @@ -53,8 +53,7 @@ def native_exe(name: str) -> str: # Run a command with retries -def run(cmd): - retries = 3 +def run_command(cmd, retries=1): print(f"Running command: {' '.join(map(str, cmd))}") attempt = 0 while True: @@ -66,13 +65,14 @@ def run(cmd): print(f"Failed to start command: {e}") returncode = -1 if returncode == 0: - return + return True attempt += 1 if attempt > retries: print(f"command failed after {retries} attempts") break - print(f"Command failed with return code {returncode}, retrying ({attempt}/{retries}) after {delay_seconds}s...") + print(f"Command failed with return code {returncode}") time.sleep(3) + return False # Temp workaround, will be removed once https://github.com/dotnet/crank/pull/841 lands @@ -131,14 +131,14 @@ def install_dotnet_sdk(channel: str, install_dir: Path) -> None: f"$DotnetVersion='{ch}';$InstallDir='{di}';" "& './dotnet-install.ps1' -Channel $DotnetVersion -InstallDir $InstallDir -NoPath" ) - run(["powershell.exe","-NoProfile","-ExecutionPolicy", "Bypass","-Command", ps_script]) + run_command(["powershell.exe","-NoProfile","-ExecutionPolicy", "Bypass","-Command", ps_script], retries=3) else: with tempfile.TemporaryDirectory() as td: script_path = Path(td) / "dotnet-install.sh" with urllib.request.urlopen("https://dot.net/v1/dotnet-install.sh") as resp, open(script_path, "wb") as f: f.write(resp.read()) os.chmod(script_path, 0o755) - run([str(script_path),"--channel", channel,"--install-dir", str(install_dir),"--no-path"]) + run_command([str(script_path),"--channel", channel,"--install-dir", str(install_dir),"--no-path"], retries=3) # Prepare the environment and run crank-agent @@ -169,8 +169,8 @@ def setup_and_run_crank_agent(workdir: Path): # Install crank-agent (runs benchmarks) and crank-controller (or just crank) that schedules them. dotnet_exe = dotnet_root_dir / native_exe("dotnet") - run([dotnet_exe, "tool", "install", "--tool-path", str(tools_dir), "Microsoft.Crank.Agent", "--version", "0.2.0-*"]) - run([dotnet_exe, "tool", "install", "--tool-path", str(tools_dir), "Microsoft.Crank.Controller", "--version", "0.2.0-*"]) + run_command([str(dotnet_exe), "tool", "install", "--tool-path", str(tools_dir), "Microsoft.Crank.Agent", "--version", "0.2.0-*"], retries=3) + run_command([str(dotnet_exe), "tool", "install", "--tool-path", str(tools_dir), "Microsoft.Crank.Controller", "--version", "0.2.0-*"], retries=3) # Create a Localhost.yml to define the local environment since we can't access the PerfLab. yml = textwrap.dedent( @@ -181,7 +181,6 @@ def setup_and_run_crank_agent(workdir: Path): applicationPort: {CRANK_PORT} applicationScheme: http loadPort: {CRANK_PORT} - serverPort: 5014 loadScheme: http profiles: Localhost: @@ -252,16 +251,16 @@ def run_crank_scenario(crank_app: Path, scenario_name: str, framework: str, work "--load.options.reuseBuild", "true", "--load.variables.duration", "45", "--load.variables.warmup", "15", - "--load.connections", "64", "--load.job", "bombardier", # Bombardier is more cross-platform friendly (wrk is linux only) ] # Only add SPMI collection environment variables and output files if not in dry run mode if not dryrun: cmd.extend([ - "--application.environmentVariables", f"COMPlus_JitName={spmi_shim}", + "--application.environmentVariables", f"DOTNET_JitName={spmi_shim}", "--application.environmentVariables", f"SuperPMIShimLogPath={str(work_dir)}", "--application.environmentVariables", f"SuperPMIShimPath=./{clrjit}", + "--application.environmentVariables", "DOTNET_EnableExtraSuperPmiQueries=1", "--application.options.outputFiles", str(core_root_path / spmi_shim), "--application.options.outputFiles", str(core_root_path / clrjit), "--application.options.outputFiles", str(core_root_path / coreclr), @@ -271,7 +270,7 @@ def run_crank_scenario(crank_app: Path, scenario_name: str, framework: str, work # Append any extra scenario-specific arguments if extra_args: cmd.extend(extra_args) - run(cmd) + run_command(cmd, retries=3) # Main entry point @@ -398,20 +397,49 @@ def main(): print("Error: No .mc files found.") sys.exit(2) - print(f"Merging {mc_file_count} .mc files into {output_mch_path}...") - run([ + print(f"Merging {mc_file_count} .mc files...") + + # Merge + run_command([ str(mcs_cmd), "-merge", "-recursive", "-dedup", "-thin", - str(output_mch_path), + "temp.mch", str(work_dir_base / "*.mc") ]) + + # clean + jitlib = str(core_root_path / native_dll("clrjit")) + run_command([str(core_root_path / native_exe("superpmi")), "-v", "ewmi", "-f", "fail.mcl", jitlib, "temp.mch"]) + + # strip + if os.path.isfile("fail.mcl") and os.stat("fail.mcl").st_size != 0: + print("Replay had failures, cleaning...") + run_command([str(mcs_cmd), "-strip", "fail.mcl", "temp.mch", str(output_mch_path)]) + else: + print("Replay was clean...") + shutil.copy2("temp.mch", str(output_mch_path)) + + # index + run_command([str(mcs_cmd), "-toc", str(output_mch_path)]) + + # overall summary + print("Merged summary for " + str(output_mch_path)) + run_command([str(mcs_cmd), "-jitflags", str(output_mch_path)]) + print(f"Finished merging .mc files into {output_mch_path}") + # delete all .mc files for mc_file in work_dir_base.glob("*.mc"): mc_file.unlink() + + # delete temp.mch and fail.mcl if they exist + if os.path.exists("temp.mch"): + os.remove("temp.mch") + if os.path.exists("fail.mcl"): + os.remove("fail.mcl") else: print("Dry run mode: Skipping SPMI data collection and .mch file generation.") From ad31183b0785b885707a1e0674a6c7c539633b1b Mon Sep 17 00:00:00 2001 From: Egor Bogatov Date: Tue, 2 Sep 2025 08:49:44 +0200 Subject: [PATCH 47/48] Update src/coreclr/scripts/superpmi_aspnet2.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/coreclr/scripts/superpmi_aspnet2.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/coreclr/scripts/superpmi_aspnet2.py b/src/coreclr/scripts/superpmi_aspnet2.py index ee5193b0d398bb..8f9aa1fa1d923e 100644 --- a/src/coreclr/scripts/superpmi_aspnet2.py +++ b/src/coreclr/scripts/superpmi_aspnet2.py @@ -58,19 +58,16 @@ def run_command(cmd, retries=1): attempt = 0 while True: try: - proc = subprocess.Popen(cmd) - output, error = proc.communicate() - returncode = proc.returncode + subprocess.run(cmd, check=True) + return True + except subprocess.CalledProcessError as e: + print(f"Command failed with return code {e.returncode}") except Exception as e: print(f"Failed to start command: {e}") - returncode = -1 - if returncode == 0: - return True attempt += 1 if attempt > retries: print(f"command failed after {retries} attempts") break - print(f"Command failed with return code {returncode}") time.sleep(3) return False From d353f5f07ab744c06bbb106448da73d847a2293f Mon Sep 17 00:00:00 2001 From: Egor Bogatov Date: Tue, 2 Sep 2025 12:48:35 +0200 Subject: [PATCH 48/48] Update superpmi_aspnet2.py --- src/coreclr/scripts/superpmi_aspnet2.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/coreclr/scripts/superpmi_aspnet2.py b/src/coreclr/scripts/superpmi_aspnet2.py index 8f9aa1fa1d923e..40ae796f7a6e66 100644 --- a/src/coreclr/scripts/superpmi_aspnet2.py +++ b/src/coreclr/scripts/superpmi_aspnet2.py @@ -233,8 +233,6 @@ def run_crank_scenario(crank_app: Path, scenario_name: str, framework: str, work "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/azure.profile.yml", "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/build/ci.profile.yml", "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/steadystate.profile.yml", - "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/json.benchmarks.yml", - "--config", "https://raw.githubusercontent.com/aspnet/Benchmarks/main/scenarios/orchard.benchmarks.yml", "--config", "https://raw.githubusercontent.com/dotnet/crank/main/src/Microsoft.Crank.Jobs.Wrk/wrk.yml", "--config", "https://raw.githubusercontent.com/dotnet/crank/main/src/Microsoft.Crank.Jobs.Bombardier/bombardier.yml", "--config", str(config_path),