Enhance Docker Compose and Helm detectors for improved parallel processing and file size handling

Author: jpinz

src/Microsoft.ComponentDetection.Detectors/dockercompose/DockerComposeComponentDetector.cs

@@ -40,7 +40,15 @@ public DockerComposeComponentDetector(
 
     public override IEnumerable<string> Categories => [nameof(DetectorClass.DockerCompose)];
 
-    protected override async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary<string, string> detectorArgs, CancellationToken cancellationToken = default)
+    /// <summary>
+    /// Gets or sets a value indicating whether compose files are processed concurrently.
+    /// Each file is parsed independently into its own <see cref="ISingleFileComponentRecorder"/>
+    /// and <see cref="DockerReferenceUtility"/> is stateless, so parsing is thread-safe and
+    /// scales across cores for repositories containing many compose files.
+    /// </summary>
+    protected override bool EnableParallelism { get; set; } = true;
+
+    protected override Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary<string, string> detectorArgs, CancellationToken cancellationToken = default)
     {
         var singleFileComponentRecorder = processRequest.SingleFileComponentRecorder;
         var file = processRequest.ComponentStream;
@@ -49,18 +57,18 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID
         {
             this.Logger.LogInformation("Discovered Docker Compose file: {Location}", file.Location);
 
-            string contents;
+            // Parse directly from the stream; the content is already buffered in memory by
+            // LazyComponentStream, so reading it into an intermediate string only adds an
+            // extra full-file allocation and GC pressure under parallel processing.
+            var yaml = new YamlStream();
             using (var reader = new StreamReader(file.Stream))
             {
-                contents = await reader.ReadToEndAsync(cancellationToken);
+                yaml.Load(reader);
             }
 
-            var yaml = new YamlStream();
-            yaml.Load(new StringReader(contents));
-
             if (yaml.Documents.Count == 0)
             {
-                return;
+                return Task.CompletedTask;
             }
 
             foreach (var document in yaml.Documents)
@@ -75,6 +83,8 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID
         {
             this.Logger.LogError(e, "Failed to parse Docker Compose file: {Location}", file.Location);
         }
+
+        return Task.CompletedTask;
     }
 
     private static YamlMappingNode? GetMappingChild(YamlMappingNode parent, string key)

src/Microsoft.ComponentDetection.Detectors/helm/HelmComponentDetector.cs

@@ -17,6 +17,14 @@ namespace Microsoft.ComponentDetection.Detectors.Helm;
 
 public class HelmComponentDetector : FileComponentDetector, IExperimentalDetector
 {
+    /// <summary>
+    /// Maximum size (in bytes) of a values file the detector will parse. The "*values*" globs
+    /// can match large, non-Helm YAML files whose full-DOM parse dominates worst-case runtime;
+    /// files above this limit are skipped so a single pathological file cannot exhaust the
+    /// detector's time budget.
+    /// </summary>
+    private const long MaxValuesFileSizeBytes = 20 * 1024 * 1024; // 20 MB
+
     public HelmComponentDetector(
         IComponentStreamEnumerableFactory componentStreamEnumerableFactory,
         IObservableDirectoryWalkerFactory walkerFactory,
@@ -41,6 +49,14 @@ public HelmComponentDetector(
 
     public override IEnumerable<string> Categories => [nameof(DetectorClass.Helm)];
 
+    /// <summary>
+    /// Gets or sets a value indicating whether values files are processed concurrently.
+    /// Each file is parsed independently into its own <see cref="ISingleFileComponentRecorder"/>
+    /// and <see cref="DockerReferenceUtility"/> is stateless, so parsing is thread-safe and
+    /// scales across cores for repositories containing many charts.
+    /// </summary>
+    protected override bool EnableParallelism { get; set; } = true;
+
     /// <summary>
     /// Pre-filters scan work to only values files co-located with a Chart.yaml/Chart.yml.
     /// Materializes all matched files, identifies Helm chart directories, then filters.
@@ -65,7 +81,7 @@ protected override async Task<IObservable<ProcessRequest>> OnPrepareDetectionAsy
             .ToObservable();
     }
 
-    protected override async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary<string, string> detectorArgs, CancellationToken cancellationToken = default)
+    protected override Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary<string, string> detectorArgs, CancellationToken cancellationToken = default)
     {
         var file = processRequest.ComponentStream;
 
@@ -74,20 +90,34 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID
         // filename/directory checks are needed.
         try
         {
+            // Check the size before touching ComponentStream so an oversized file is never
+            // buffered into memory. The "*values*" globs can match large, non-Helm YAML files
+            // whose full-DOM parse is the main driver of worst-case (timeout) runtime.
+            var fileInfo = new FileInfo(file.Location);
+            if (fileInfo.Exists && fileInfo.Length > MaxValuesFileSizeBytes)
+            {
+                this.Logger.LogWarning(
+                    "Skipping Helm values file exceeding size limit ({Length} bytes > {Limit} bytes): {Location}",
+                    fileInfo.Length,
+                    MaxValuesFileSizeBytes,
+                    file.Location);
+                return Task.CompletedTask;
+            }
+
             this.Logger.LogInformation("Discovered Helm values file: {Location}", file.Location);
 
-            string contents;
+            // Parse directly from the stream; the content is already buffered in memory by
+            // LazyComponentStream, so reading it into an intermediate string only adds an
+            // extra full-file allocation and GC pressure under parallel processing.
+            var yaml = new YamlStream();
             using (var reader = new StreamReader(file.Stream))
             {
-                contents = await reader.ReadToEndAsync(cancellationToken);
+                yaml.Load(reader);
             }
 
-            var yaml = new YamlStream();
-            yaml.Load(new StringReader(contents));
-
             if (yaml.Documents.Count == 0)
             {
-                return;
+                return Task.CompletedTask;
             }
 
             this.ExtractImageReferencesFromValues(yaml, processRequest.SingleFileComponentRecorder);
@@ -96,6 +126,8 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID
         {
             this.Logger.LogError(e, "Failed to parse Helm file: {Location}", file.Location);
         }
+
+        return Task.CompletedTask;
     }
 
     /// <summary>