From e9bfed23e0f48d400905b5c44afc9c9074719c05 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 7 Oct 2025 14:15:22 +0000 Subject: [PATCH 1/5] Initial plan From c8800b1c80b57725345d4bf9e775abffcd303b94 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 7 Oct 2025 14:25:09 +0000 Subject: [PATCH 2/5] Add --record option to crank-agent for custom measurements Co-authored-by: sebastienros <1165805+sebastienros@users.noreply.github.com> --- src/Microsoft.Crank.Agent/Startup.cs | 125 ++++++++++++++++++++++++++- 1 file changed, 124 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.Crank.Agent/Startup.cs b/src/Microsoft.Crank.Agent/Startup.cs index c302a6f87..b3976f690 100644 --- a/src/Microsoft.Crank.Agent/Startup.cs +++ b/src/Microsoft.Crank.Agent/Startup.cs @@ -170,9 +170,13 @@ private static CommandOption _certClientId, _certTenantId, _certThumbprint, - _certSniAuth + _certSniAuth, + _recordOption ; + // Stores custom measurements to record for each job + private static readonly Dictionary _customMeasurements = new Dictionary(); + internal static Serilog.Core.Logger Logger { get; private set; } private static readonly string[] _powershellCommands = ["pwsh", "powershell"]; @@ -266,6 +270,7 @@ public static int Main(string[] args) _certPath = app.Option("--cert-path", "Location of the certificate to be used for auth.", CommandOptionType.SingleValue); _certPassword = app.Option("--cert-pwd", "Password of the certificate to be used for auth.", CommandOptionType.SingleValue); _certSniAuth = app.Option("--cert-sni", "Enable subject name / issuer based authentication (SNI).", CommandOptionType.NoValue); + _recordOption = app.Option("-r|--record", "Records a custom measurement for each benchmark. Format: 'name=value'. Can use command substitution with $(command). Can be specified multiple times.", CommandOptionType.MultipleValue); app.OnExecute(() => { @@ -381,6 +386,40 @@ public static int Main(string[] args) } } + // Process custom measurement records + if (_recordOption.HasValue()) + { + foreach (var recordValue in _recordOption.Values) + { + if (string.IsNullOrWhiteSpace(recordValue)) + { + continue; + } + + var separatorIndex = recordValue.IndexOf('='); + if (separatorIndex <= 0) + { + Log.Warning($"Invalid --record format: '{recordValue}'. Expected format: 'name=value'"); + continue; + } + + var name = recordValue.Substring(0, separatorIndex).Trim(); + var value = recordValue.Substring(separatorIndex + 1).Trim(); + + if (string.IsNullOrEmpty(name)) + { + Log.Warning($"Invalid --record format: '{recordValue}'. Name cannot be empty."); + continue; + } + + // Check if value contains command substitution + value = ProcessCommandSubstitution(value); + + _customMeasurements[name] = value; + Log.Info($"Recording custom measurement: {name} = {value}"); + } + } + return Run(url, hostname, dockerHostname).Result; }); @@ -5556,6 +5595,65 @@ private static async Task AOT4Mono(string dotnetSdkVersion, string runtimeVersio captureOutput: true); } + /// + /// Process command substitution in the format $(command) + /// + private static string ProcessCommandSubstitution(string value) + { + if (string.IsNullOrEmpty(value)) + { + return value; + } + + // Check for command substitution pattern $(...) + var pattern = @"\$\(([^)]+)\)"; + var regex = new Regex(pattern); + var match = regex.Match(value); + + if (!match.Success) + { + return value; + } + + try + { + var command = match.Groups[1].Value; + string output; + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + // On Windows, use cmd.exe to execute the command + var result = ProcessUtil.RunAsync("cmd.exe", $"/c {command}", + captureOutput: true, + log: false, + throwOnError: false + ).GetAwaiter().GetResult(); + + output = result.StandardOutput.Trim(); + } + else + { + // On Linux/macOS, use sh + var result = ProcessUtil.RunAsync("/bin/sh", $"-c \"{command}\"", + captureOutput: true, + log: false, + throwOnError: false + ).GetAwaiter().GetResult(); + + output = result.StandardOutput.Trim(); + } + + // Replace the command substitution with the output + value = regex.Replace(value, output); + } + catch (Exception ex) + { + Log.Warning($"Failed to execute command substitution in value '{value}': {ex.Message}"); + } + + return value; + } + private static string ConvertCmd2Arg(string cmd) { cmd.Replace("\"", "\"\""); @@ -5581,6 +5679,31 @@ private static bool MarkAsRunning(string hostname, Job job, Stopwatch stopwatch) }); BenchmarksEventSource.Start(); + // Add custom measurements from --record options + foreach (var customMeasurement in _customMeasurements) + { + job.Measurements.Enqueue(new Measurement + { + Name = customMeasurement.Key, + Timestamp = DateTime.UtcNow, + Value = customMeasurement.Value + }); + + // Also add metadata for the custom measurement + job.Metadata.Enqueue(new MeasurementMetadata + { + Source = "Agent", + Name = customMeasurement.Key, + Aggregate = Operation.First, + Reduce = Operation.First, + Format = "", + LongDescription = $"Custom measurement: {customMeasurement.Key}", + ShortDescription = customMeasurement.Key + }); + + Log.Info($"Added custom measurement to job {job.Id}: {customMeasurement.Key} = {customMeasurement.Value}"); + } + Log.Info($"Running job '{job.Service}' ({job.Id})"); job.Url = ComputeServerUrl(hostname, job); From 76c15455c58b2206d5919d51f59056d00aa6a2cf Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 7 Oct 2025 14:30:24 +0000 Subject: [PATCH 3/5] Add documentation for --record option Co-authored-by: sebastienros <1165805+sebastienros@users.noreply.github.com> --- docs/measurements.md | 69 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/docs/measurements.md b/docs/measurements.md index 5f8feeada..e5e095e00 100644 --- a/docs/measurements.md +++ b/docs/measurements.md @@ -140,3 +140,72 @@ Here is a working example: }] } ``` + +### From the Agent Command Line + +The agent can be configured to record custom measurements for every benchmark it runs using the `--record` (or `-r`) command line option. This is useful for recording system information or configuration details that should be captured for all benchmarks. + +#### Usage + +```bash +crank-agent --record "name=value" +``` + +The option can be specified multiple times to record multiple measurements: + +```bash +crank-agent --record "system/openssl=$(openssl version)" --record "system/kernel=$(uname -r)" +``` + +#### Format + +- The name and value are separated by the first `=` character +- The name portion becomes the measurement name +- The value can be a literal string or include command substitution using `$(command)` syntax + +#### Command Substitution + +When a value contains `$(command)`, the agent will execute the command and use its output as the measurement value: + +**Linux/macOS example:** +```bash +crank-agent --record "system/openssl=$(openssl version)" \ + --record "system/kernel=$(uname -r)" \ + --record "system/hostname=$(hostname)" +``` + +**Windows example:** +```powershell +crank-agent --record "system/dotnet=$(dotnet --version)" ` + --record "system/os=$(systeminfo | findstr /B /C:'OS Name')" +``` + +#### Behavior + +- Custom measurements are automatically added to every job that the agent runs +- Each measurement includes metadata with: + - `Source`: "Agent" + - `Aggregate`: First + - `Reduce`: First + - `ShortDescription`: The measurement name + - `LongDescription`: "Custom measurement: {name}" +- Invalid formats (missing `=` or empty name) will be logged as warnings and skipped +- Command substitution failures are logged as warnings, and the original value is used + +#### Example + +Start an agent that records the OpenSSL version: + +```bash +crank-agent --record "system/openssl=$(openssl version)" +``` + +When this agent runs a benchmark, the measurement will automatically include: + +```json +{ + "name": "system/openssl", + "timestamp": "2024-02-23T13:01:56.12Z", + "value": "OpenSSL 3.0.13 30 Jan 2024" +} +``` From 9f06183291ffc121f502594782e59dba9c49b1b5 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 7 Oct 2025 14:33:46 +0000 Subject: [PATCH 4/5] Check for existing metadata before adding custom measurements Co-authored-by: sebastienros <1165805+sebastienros@users.noreply.github.com> --- src/Microsoft.Crank.Agent/Startup.cs | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/Microsoft.Crank.Agent/Startup.cs b/src/Microsoft.Crank.Agent/Startup.cs index b3976f690..8e4321f21 100644 --- a/src/Microsoft.Crank.Agent/Startup.cs +++ b/src/Microsoft.Crank.Agent/Startup.cs @@ -5689,17 +5689,20 @@ private static bool MarkAsRunning(string hostname, Job job, Stopwatch stopwatch) Value = customMeasurement.Value }); - // Also add metadata for the custom measurement - job.Metadata.Enqueue(new MeasurementMetadata + // Also add metadata for the custom measurement if it doesn't exist + if (!job.Metadata.Any(x => x.Name == customMeasurement.Key)) { - Source = "Agent", - Name = customMeasurement.Key, - Aggregate = Operation.First, - Reduce = Operation.First, - Format = "", - LongDescription = $"Custom measurement: {customMeasurement.Key}", - ShortDescription = customMeasurement.Key - }); + job.Metadata.Enqueue(new MeasurementMetadata + { + Source = "Agent", + Name = customMeasurement.Key, + Aggregate = Operation.First, + Reduce = Operation.First, + Format = "", + LongDescription = $"Custom measurement: {customMeasurement.Key}", + ShortDescription = customMeasurement.Key + }); + } Log.Info($"Added custom measurement to job {job.Id}: {customMeasurement.Key} = {customMeasurement.Value}"); } From 7782a755b721f6e882dc54f53cba368cdc55739d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 7 Oct 2025 14:49:52 +0000 Subject: [PATCH 5/5] Remove command substitution logic - let shell handle it Co-authored-by: sebastienros <1165805+sebastienros@users.noreply.github.com> --- docs/measurements.md | 11 +++-- src/Microsoft.Crank.Agent/Startup.cs | 64 +--------------------------- 2 files changed, 6 insertions(+), 69 deletions(-) diff --git a/docs/measurements.md b/docs/measurements.md index e5e095e00..e86979eb5 100644 --- a/docs/measurements.md +++ b/docs/measurements.md @@ -161,23 +161,23 @@ crank-agent --record "system/openssl=$(openssl version)" --record "system/kernel - The name and value are separated by the first `=` character - The name portion becomes the measurement name -- The value can be a literal string or include command substitution using `$(command)` syntax +- The value can be a literal string #### Command Substitution -When a value contains `$(command)`, the agent will execute the command and use its output as the measurement value: +Command substitution is handled by the shell before the arguments reach the agent. Use your shell's command substitution syntax: -**Linux/macOS example:** +**Linux/macOS example (using bash/sh):** ```bash crank-agent --record "system/openssl=$(openssl version)" \ --record "system/kernel=$(uname -r)" \ --record "system/hostname=$(hostname)" ``` -**Windows example:** +**Windows example (using PowerShell):** ```powershell crank-agent --record "system/dotnet=$(dotnet --version)" ` - --record "system/os=$(systeminfo | findstr /B /C:'OS Name')" + --record "system/os=$(systeminfo | Select-String 'OS Name')" ``` #### Behavior @@ -190,7 +190,6 @@ crank-agent --record "system/dotnet=$(dotnet --version)" ` - `ShortDescription`: The measurement name - `LongDescription`: "Custom measurement: {name}" - Invalid formats (missing `=` or empty name) will be logged as warnings and skipped -- Command substitution failures are logged as warnings, and the original value is used #### Example diff --git a/src/Microsoft.Crank.Agent/Startup.cs b/src/Microsoft.Crank.Agent/Startup.cs index 8e4321f21..859387526 100644 --- a/src/Microsoft.Crank.Agent/Startup.cs +++ b/src/Microsoft.Crank.Agent/Startup.cs @@ -270,7 +270,7 @@ public static int Main(string[] args) _certPath = app.Option("--cert-path", "Location of the certificate to be used for auth.", CommandOptionType.SingleValue); _certPassword = app.Option("--cert-pwd", "Password of the certificate to be used for auth.", CommandOptionType.SingleValue); _certSniAuth = app.Option("--cert-sni", "Enable subject name / issuer based authentication (SNI).", CommandOptionType.NoValue); - _recordOption = app.Option("-r|--record", "Records a custom measurement for each benchmark. Format: 'name=value'. Can use command substitution with $(command). Can be specified multiple times.", CommandOptionType.MultipleValue); + _recordOption = app.Option("-r|--record", "Records a custom measurement for each benchmark. Format: 'name=value'. Can be specified multiple times.", CommandOptionType.MultipleValue); app.OnExecute(() => { @@ -412,9 +412,6 @@ public static int Main(string[] args) continue; } - // Check if value contains command substitution - value = ProcessCommandSubstitution(value); - _customMeasurements[name] = value; Log.Info($"Recording custom measurement: {name} = {value}"); } @@ -5595,65 +5592,6 @@ private static async Task AOT4Mono(string dotnetSdkVersion, string runtimeVersio captureOutput: true); } - /// - /// Process command substitution in the format $(command) - /// - private static string ProcessCommandSubstitution(string value) - { - if (string.IsNullOrEmpty(value)) - { - return value; - } - - // Check for command substitution pattern $(...) - var pattern = @"\$\(([^)]+)\)"; - var regex = new Regex(pattern); - var match = regex.Match(value); - - if (!match.Success) - { - return value; - } - - try - { - var command = match.Groups[1].Value; - string output; - - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - // On Windows, use cmd.exe to execute the command - var result = ProcessUtil.RunAsync("cmd.exe", $"/c {command}", - captureOutput: true, - log: false, - throwOnError: false - ).GetAwaiter().GetResult(); - - output = result.StandardOutput.Trim(); - } - else - { - // On Linux/macOS, use sh - var result = ProcessUtil.RunAsync("/bin/sh", $"-c \"{command}\"", - captureOutput: true, - log: false, - throwOnError: false - ).GetAwaiter().GetResult(); - - output = result.StandardOutput.Trim(); - } - - // Replace the command substitution with the output - value = regex.Replace(value, output); - } - catch (Exception ex) - { - Log.Warning($"Failed to execute command substitution in value '{value}': {ex.Message}"); - } - - return value; - } - private static string ConvertCmd2Arg(string cmd) { cmd.Replace("\"", "\"\"");