[Dynamic Instrumentation] DEBUG-4089 Support file-based debugger probes

tracer/src/Datadog.Trace/Configuration/ConfigurationKeys.Debugger.cs

@@ -179,6 +179,13 @@ internal static class Debugger
             /// </summary>
             /// <seealso cref="DebuggerSettings.CodeOriginMaxUserFrames"/>
             public const string CodeOriginMaxUserFrames = "DD_CODE_ORIGIN_FOR_SPANS_MAX_USER_FRAMES";
+
+            /// <summary>
+            /// Configuration key for setting probes file path to add probes via file during DynamicInstrumentation initialization.
+            /// Default value is <c>empty</c>.
+            /// </summary>
+            /// <seealso cref="DebuggerSettings.ProbeFile"/>
+            public const string DynamicInstrumentationProbeFile = "DD_DYNAMIC_INSTRUMENTATION_PROBE_FILE";
         }
     }
 }

tracer/src/Datadog.Trace/Debugger/Configurations/ConfigurationUpdater.cs

@@ -43,7 +43,13 @@ public List<UpdateResult> AcceptAdded(ProbeConfiguration configuration)
         {
             var result = new List<UpdateResult>();
             var filteredConfiguration = ApplyConfigurationFilters(configuration);
-            var comparer = new ProbeConfigurationComparer(_currentConfiguration, filteredConfiguration);
+
+            // Merge the new filtered configuration with the existing one so that
+            // _currentConfiguration always represents the union of all accepted
+            // configurations from both file and RCM
+            var mergedConfiguration = MergeConfigurations(_currentConfiguration, filteredConfiguration);
+
+            var comparer = new ProbeConfigurationComparer(_currentConfiguration, mergedConfiguration);
 
             if (comparer.HasProbeRelatedChanges)
             {
@@ -55,7 +61,7 @@ public List<UpdateResult> AcceptAdded(ProbeConfiguration configuration)
                 HandleRateLimitChanged(comparer);
             }
 
-            _currentConfiguration = configuration;
+            _currentConfiguration = mergedConfiguration;
 
             return result;
         }
@@ -72,6 +78,30 @@ public void AcceptRemoved(List<RemoteConfigurationPath> paths)
             }
         }
 
+        private static ProbeConfiguration MergeConfigurations(ProbeConfiguration current, ProbeConfiguration incoming)
+        {
+            return new ProbeConfiguration
+            {
+                LogProbes = current.LogProbes
+                                   .Concat(incoming.LogProbes)
+                                   .Distinct()
+                                   .ToArray(),
+                MetricProbes = current.MetricProbes
+                                      .Concat(incoming.MetricProbes)
+                                      .Distinct()
+                                      .ToArray(),
+                SpanProbes = current.SpanProbes
+                                    .Concat(incoming.SpanProbes)
+                                    .Distinct()
+                                    .ToArray(),
+                SpanDecorationProbes = current.SpanDecorationProbes
+                                              .Concat(incoming.SpanDecorationProbes)
+                                              .Distinct()
+                                              .ToArray(),
+                ServiceConfiguration = incoming.ServiceConfiguration ?? current.ServiceConfiguration
+            };
+        }
+
         private ProbeConfiguration ApplyConfigurationFilters(ProbeConfiguration configuration)
         {
             return new ProbeConfiguration()

tracer/src/Datadog.Trace/Debugger/DebuggerManager.cs

@@ -342,7 +342,10 @@ private async Task DebouncedUpdateDynamicInstrumentationAsync(TracerSettings tra
                     Log.Warning("Remote configuration is not available in this environment, so Dynamic Instrumentation cannot be enabled.");
                 }
 
-                return;
+                if (string.IsNullOrEmpty(debuggerSettings.ProbeFile))
+                {
+                    return;
+                }
             }
 
             var requestedDiState = debuggerSettings.DynamicInstrumentationEnabled || debuggerSettings.DynamicSettings.DynamicInstrumentationEnabled == true;

tracer/src/Datadog.Trace/Debugger/DebuggerSettings.cs

@@ -145,6 +145,8 @@ public DebuggerSettings(IConfigurationSource? source, IConfigurationTelemetry te
                                          .Value;
 
             SymbolDatabaseCompressionEnabled = config.WithKeys(ConfigurationKeys.Debugger.SymbolDatabaseCompressionEnabled).AsBool(true);
+
+            ProbeFile = config.WithKeys(ConfigurationKeys.Debugger.DynamicInstrumentationProbeFile).AsString() ?? string.Empty;
         }
 
         internal ImmutableDynamicDebuggerSettings DynamicSettings { get; init; } = new();
@@ -189,6 +191,8 @@ public DebuggerSettings(IConfigurationSource? source, IConfigurationTelemetry te
 
         public int CodeOriginMaxUserFrames { get; }
 
+        public string ProbeFile { get; }
+
         public static DebuggerSettings FromSource(IConfigurationSource source, IConfigurationTelemetry telemetry)
         {
             return new DebuggerSettings(source, telemetry);

tracer/src/Datadog.Trace/Debugger/DynamicInstrumentation.cs

@@ -8,6 +8,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
@@ -24,6 +25,7 @@
 using Datadog.Trace.DogStatsd;
 using Datadog.Trace.Logging;
 using Datadog.Trace.RemoteConfigurationManagement;
+using Datadog.Trace.Vendors.Newtonsoft.Json.Linq;
 using Datadog.Trace.Vendors.Serilog.Events;
 using Datadog.Trace.Vendors.StatsdClient;
 using ProbeInfo = Datadog.Trace.Debugger.Expressions.ProbeInfo;
@@ -49,6 +51,7 @@ internal class DynamicInstrumentation : IDisposable
         private readonly DebuggerSettings _settings;
         private readonly object _instanceLock = new();
         private int _disposeState;
+        private volatile ProbeConfiguration? _fileProbes;
 
         internal DynamicInstrumentation(
             DebuggerSettings settings,
@@ -103,17 +106,44 @@ private async Task InitializeAsync()
         {
             try
             {
-                var isRcmAvailable = await WaitForRcmAvailabilityAsync().ConfigureAwait(false);
-                if (!isRcmAvailable)
+                // Start loading probes from file and checking RCM availability in parallel
+                var fileProbesTask = LoadProbesFromFileAsync();
+                var rcmAvailabilityTask = WaitForRcmAvailabilityAsync();
+
+                var hasFileProbes = false;
+
+                // Always attempt to load probes from file, even if RCM is unavailable
+                var probeConfiguration = await fileProbesTask.ConfigureAwait(false);
+                if (probeConfiguration != null)
                 {
-                    return;
+                    _fileProbes = probeConfiguration;
+                    _configurationUpdater.AcceptAdded(probeConfiguration);
+                    hasFileProbes = (probeConfiguration.LogProbes.Length
+                                   + probeConfiguration.MetricProbes.Length
+                                   + probeConfiguration.SpanProbes.Length
+                                   + probeConfiguration.SpanDecorationProbes.Length) > 0;
                 }
 
-                _subscriptionManager.SubscribeToChanges(_subscription);
-                AppDomain.CurrentDomain.AssemblyLoad += CheckUnboundProbes;
-                StartBackgroundProcess();
-                IsInitialized = true;
-                Log.Information("Dynamic Instrumentation initialization completed successfully");
+                var isRcmAvailable = await rcmAvailabilityTask.ConfigureAwait(false);
+                if (isRcmAvailable)
+                {
+                    _subscriptionManager.SubscribeToChanges(_subscription);
+                }
+
+                // Start background processing and register the assembly load callback if either:
+                // - RCM is available
+                // - There are probes from file
+                if (isRcmAvailable || hasFileProbes)
+                {
+                    AppDomain.CurrentDomain.AssemblyLoad += CheckUnboundProbes;
+                    StartBackgroundProcess();
+                    IsInitialized = true;
+                    Log.Information("Dynamic Instrumentation initialization completed successfully");
+                }
+                else
+                {
+                    Log.Information("Dynamic Instrumentation not initialized because RCM isn't available and no valid probes have loaded from file");
+                }
             }
             catch (OperationCanceledException e)
             {
@@ -474,13 +504,16 @@ private ApplyDetails[] AcceptAddedConfiguration(List<RemoteConfiguration>? confi
                 }
             }
 
+            // Merge file probes with RCM probes (RCM takes precedence on ID conflicts)
+            var currentFileProbes = _fileProbes;
+
             var probeConfiguration = new ProbeConfiguration()
             {
                 ServiceConfiguration = serviceConfig,
-                MetricProbes = metrics.ToArray(),
-                SpanDecorationProbes = spanDecoration.ToArray(),
-                LogProbes = logs.ToArray(),
-                SpanProbes = spans.ToArray()
+                LogProbes = MergeProbes(currentFileProbes?.LogProbes, logs.ToArray()),
+                MetricProbes = MergeProbes(currentFileProbes?.MetricProbes, metrics.ToArray()),
+                SpanProbes = MergeProbes(currentFileProbes?.SpanProbes, spans.ToArray()),
+                SpanDecorationProbes = MergeProbes(currentFileProbes?.SpanDecorationProbes, spanDecoration.ToArray())
             };
 
             try
@@ -636,6 +669,199 @@ void DiscoveryCallback(AgentConfiguration x)
             }
         }
 
+        private async Task<ProbeConfiguration?> LoadProbesFromFileAsync()
+        {
+            if (string.IsNullOrEmpty(_settings.ProbeFile))
+            {
+                return null;
+            }
+
+            try
+            {
+                if (!File.Exists(_settings.ProbeFile))
+                {
+                    Log.Warning("Probe file specified but not found: {ProbeFile}", _settings.ProbeFile);
+                    return null;
+                }
+
+                Log.Information("Loading probes from file: {ProbeFile}", _settings.ProbeFile);
+
+                string fileContent;
+                using (var reader = new StreamReader(_settings.ProbeFile))
+                {
+                    fileContent = await reader.ReadToEndAsync().ConfigureAwait(false);
+                }
+
+                if (string.IsNullOrWhiteSpace(fileContent))
+                {
+                    Log.Debug("Probe file is empty: {ProbeFile}", _settings.ProbeFile);
+                    return null;
+                }
+
+                var jArray = JArray.Parse(fileContent);
+                var logs = new List<LogProbe>();
+                var metrics = new List<MetricProbe>();
+                var spans = new List<SpanProbe>();
+                var spanDecorations = new List<SpanDecorationProbe>();
+
+                foreach (var jToken in jArray)
+                {
+                    var jObject = jToken as JObject;
+                    if (jObject == null)
+                    {
+                        Log.Warning("Invalid probe entry in file, skipping");
+                        continue;
+                    }
+
+                    var typeToken = jObject["type"];
+                    if (typeToken == null)
+                    {
+                        Log.Warning("Probe entry missing 'type' field, skipping");
+                        continue;
+                    }
+
+                    var type = typeToken.ToString();
+                    try
+                    {
+                        switch (type)
+                        {
+                            case "LOG_PROBE":
+                                var logProbe = jObject.ToObject<LogProbe>();
+                                if (logProbe != null)
+                                {
+                                    logs.Add(logProbe);
+                                }
+
+                                break;
+                            case "METRIC_PROBE":
+                                var metricProbe = jObject.ToObject<MetricProbe>();
+                                if (metricProbe != null)
+                                {
+                                    metrics.Add(metricProbe);
+                                }
+
+                                break;
+                            case "SPAN_PROBE":
+                                var spanProbe = jObject.ToObject<SpanProbe>();
+                                if (spanProbe != null)
+                                {
+                                    spans.Add(spanProbe);
+                                }
+
+                                break;
+                            case "SPAN_DECORATION_PROBE":
+                                var spanDecorationProbe = jObject.ToObject<SpanDecorationProbe>();
+                                if (spanDecorationProbe != null)
+                                {
+                                    spanDecorations.Add(spanDecorationProbe);
+                                }
+
+                                break;
+                            default:
+                                Log.Warning("Unknown probe type '{Type}' in file, skipping", type);
+                                break;
+                        }
+                    }
+                    catch (Exception ex)
+                    {
+                        Log.Error(ex, "Failed to deserialize probe of type '{Type}', skipping", type);
+                    }
+                }
+
+                var totalProbes = logs.Count + metrics.Count + spans.Count + spanDecorations.Count;
+                if (totalProbes == 0)
+                {
+                    Log.Warning("No valid probes found in file: {ProbeFile}", _settings.ProbeFile);
+                    return null;
+                }
+
+                // Deduplicate probes within the file by ID
+                var uniqueLogs = DeduplicateProbes(logs);
+                var uniqueMetrics = DeduplicateProbes(metrics);
+                var uniqueSpans = DeduplicateProbes(spans);
+                var uniqueSpanDecorations = DeduplicateProbes(spanDecorations);
+
+                var uniqueCount = uniqueLogs.Length + uniqueMetrics.Length + uniqueSpans.Length + uniqueSpanDecorations.Length;
+                if (uniqueCount < totalProbes)
+                {
+                    Log.Debug("Removed {Count} duplicate probe(s) from file", property: totalProbes - uniqueCount);
+                }
+
+                Log.Information("Successfully loaded {Count} probes from file.", property: uniqueCount);
+
+                return new ProbeConfiguration
+                {
+                    LogProbes = uniqueLogs,
+                    MetricProbes = uniqueMetrics,
+                    SpanProbes = uniqueSpans,
+                    SpanDecorationProbes = uniqueSpanDecorations
+                };
+            }
+            catch (Exception ex)
+            {
+                Log.Error(ex, "Failed to load probes from file: {ProbeFile}", _settings.ProbeFile);
+                return null;
+            }
+        }
+
+        private T[] DeduplicateProbes<T>(List<T> probes)
+            where T : ProbeDefinition
+        {
+            if (probes.Count == 0)
+            {
+                return [];
+            }
+
+            return probes
+                  .GroupBy(p => p.Id)
+                  .Select(g =>
+                   {
+                       if (g.Count() > 1)
+                       {
+                           Log.Warning("Duplicate probe ID '{Id}' found in file, using first occurrence", g.Key);
+                       }
+
+                       return g.First();
+                   })
+                  .ToArray();
+        }
+
+        private T[] MergeProbes<T>(T[]? fileProbes, T[] rcmProbes)
+            where T : ProbeDefinition
+        {
+            if (fileProbes == null || fileProbes.Length == 0)
+            {
+                return rcmProbes;
+            }
+
+            if (rcmProbes.Length == 0)
+            {
+                return fileProbes;
+            }
+
+            // Combine and deduplicate by ID (RCM takes precedence over file if conflict)
+            var mergedProbes = new Dictionary<string, T>(StringComparer.Ordinal);
+
+            // Add file probes first
+            foreach (var probe in fileProbes)
+            {
+                mergedProbes[probe.Id] = probe;
+            }
+
+            // Add/overwrite with RCM probes (RCM wins on conflicts)
+            foreach (var probe in rcmProbes)
+            {
+                if (mergedProbes.ContainsKey(probe.Id))
+                {
+                    Log.Debug("Probe ID '{Id}' exists in both file and RCM, using RCM version", probe.Id);
+                }
+
+                mergedProbes[probe.Id] = probe;
+            }
+
+            return mergedProbes.Values.ToArray();
+        }
+
         public void Dispose()
         {
             // Already disposed