Improve detection graph traversal for large result sets

src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs

@@ -7,6 +7,7 @@
 using System.Text;
 using Microsoft.ComponentDetection.Contracts;
 using Microsoft.ComponentDetection.Contracts.BcdeModels;
+using Microsoft.ComponentDetection.Contracts.TypedComponent;
 
 [assembly: InternalsVisibleTo("Microsoft.ComponentDetection.Common.Tests")]
 
@@ -141,6 +142,34 @@ public ICollection<string> GetAncestors(string componentId)
             .ToList();
     }
 
+    public HashSet<TypedComponent> GetAncestorsAsTypedComponents(string componentId, Func<string, TypedComponent> toTypedComponent)
+    {
+        ArgumentNullException.ThrowIfNull(componentId);
+        return this.GetAncestors(componentId)
+            .Select(a => this.componentNodes.TryGetValue(a, out var component) ? component : null)
+            .Where(a => a != null)
+            .Select(a => a.TypedComponent ?? toTypedComponent(a.Id))
+            .ToHashSet(new ComponentComparer());
+    }
+
+    public HashSet<TypedComponent> GetRootsAsTypedComponents(string componentId, Func<string, TypedComponent> toTypedComponent)
+    {
+        ArgumentNullException.ThrowIfNull(componentId);
+        return this.GetExplicitReferencedDependencyIds(componentId)
+            .Select(r => this.componentNodes.TryGetValue(r, out var component) ? component : null)
+            .Where(r => r != null)
+            .Select(r => r.TypedComponent ?? toTypedComponent(r.Id))
+            .ToHashSet(new ComponentComparer());
+    }
+
+    public void FillTypedComponents(Func<string, TypedComponent> toTypedComponent)
+    {
+        foreach (var componentId in this.componentNodes.Values)
+        {
+            componentId.TypedComponent = toTypedComponent(componentId.Id);
+        }
+    }
+
     IEnumerable<string> IDependencyGraph.GetDependenciesForComponent(string componentId)
     {
         return this.GetDependenciesForComponent(componentId).ToImmutableList();
@@ -229,5 +258,7 @@ internal ComponentRefNode()
         internal bool? IsDevelopmentDependency { get; set; }
 
         internal DependencyScope? DependencyScope { get; set; }
+
+        internal TypedComponent TypedComponent { get; set; }
     }
 }

src/Microsoft.ComponentDetection.Common/Telemetry/Records/DependencyGraphTranslationRecord.cs

@@ -0,0 +1,14 @@
+namespace Microsoft.ComponentDetection.Common.Telemetry.Records;
+
+using System;
+
+public class DependencyGraphTranslationRecord : BaseDetectionTelemetryRecord
+{
+    public override string RecordName => "DependencyGraphTranslationRecord";
+
+    public string DetectorId { get; set; }
+
+    public TimeSpan? TimeToAddRoots { get; set; }
+
+    public TimeSpan? TimeToAddAncestors { get; set; }
+}

src/Microsoft.ComponentDetection.Contracts/IComponentRecorder.cs

@@ -1,5 +1,6 @@
 namespace Microsoft.ComponentDetection.Contracts;
 
+using System;
 using System.Collections.Generic;
 using Microsoft.ComponentDetection.Contracts.BcdeModels;
 
@@ -124,4 +125,28 @@ public interface IDependencyGraph
     /// <param name="componentId">The component id to look up ancestors for.</param>
     /// <returns>The componentIds that are ancestors for a given componentId.</returns>
     ICollection<string> GetAncestors(string componentId);
+
+    /// <summary>
+    /// Gets the component IDs of all explicitly referenced components, and converts them to a set of typed components.
+    /// WARNING: Using this method without calling <see cref="FillTypedComponents"/> first will result in a performance hit.
+    /// </summary>
+    /// <param name="componentId">The component to find all roots for.</param>
+    /// <param name="toTypedComponent">Function that converts the component id to the typed component object.</param>
+    /// <returns>Set of TypedComponents containing the roots.</returns>
+    public HashSet<TypedComponent.TypedComponent> GetRootsAsTypedComponents(string componentId, Func<string, TypedComponent.TypedComponent> toTypedComponent);
+
+    /// <summary>
+    /// Gets the component IDs of all ancestors for a given component id, and converts them to a set of typed components.
+    /// WARNING: Using this method without calling <see cref="FillTypedComponents"/> first will result in a performance hit.
+    /// </summary>
+    /// <param name="componentId">The component to find all roots for.</param>
+    /// <param name="toTypedComponent">Function that converts the component id to the typed component object.</param>
+    /// <returns>Set of TypedComponents containing the ancestors.</returns>
+    public HashSet<TypedComponent.TypedComponent> GetAncestorsAsTypedComponents(string componentId, Func<string, TypedComponent.TypedComponent> toTypedComponent);
+
+    /// <summary>
+    /// This operation pre-fills all nodes with the specified typed component, which improves performance for subsequent runs
+    /// of <see cref="GetRootsAsTypedComponents"/> and <see cref="GetAncestorsAsTypedComponents"/>.
+    /// </summary>
+    public void FillTypedComponents(Func<string, TypedComponent.TypedComponent> toTypedComponent);
 }

src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs

@@ -90,11 +90,24 @@ private IEnumerable<DetectedComponent> GatherSetOfDetectedComponentsUnmerged(IEn
             .Where(recorderDetectorPair => recorderDetectorPair.Recorder != null)
             .SelectMany(recorderDetectorPair =>
             {
+                using var record = new DependencyGraphTranslationRecord()
+                {
+                    DetectorId = recorderDetectorPair.Detector.Id,
+                };
+
                 var detector = recorderDetectorPair.Detector;
                 var componentRecorder = recorderDetectorPair.Recorder;
                 var detectedComponents = componentRecorder.GetDetectedComponents();
                 var dependencyGraphsByLocation = componentRecorder.GetDependencyGraphsByLocation();
 
+                foreach (var graph in dependencyGraphsByLocation.Values)
+                {
+                    graph.FillTypedComponents(componentRecorder.GetComponent);
+                }
+
+                var totalTimeToAddRoots = TimeSpan.Zero;
+                var totalTimeToAddAncestors = TimeSpan.Zero;
+
                 // Note that it looks like we are building up detected components functionally, but they are not immutable -- the code is just written
                 //  to look like a pipeline.
                 foreach (var component in detectedComponents)
@@ -118,10 +131,17 @@ private IEnumerable<DetectedComponent> GatherSetOfDetectedComponentsUnmerged(IEn
                         var dependencyGraph = graphKvp.Value;
 
                         // Calculate roots of the component
+                        var rootStartTime = DateTime.UtcNow;
                         this.AddRootsToDetectedComponent(component, dependencyGraph, componentRecorder);
+                        var rootEndTime = DateTime.UtcNow;
+                        totalTimeToAddRoots += rootEndTime - rootStartTime;
 
                         // Calculate Ancestors of the component
+                        var ancestorStartTime = DateTime.UtcNow;
                         this.AddAncestorsToDetectedComponent(component, dependencyGraph, componentRecorder);
+                        var ancestorEndTime = DateTime.UtcNow;
+                        totalTimeToAddAncestors += ancestorEndTime - ancestorStartTime;
+
                         component.DevelopmentDependency = this.MergeDevDependency(component.DevelopmentDependency, dependencyGraph.IsDevelopmentDependency(component.Component.Id));
                         component.DependencyScope = DependencyScopeComparer.GetMergedDependencyScope(component.DependencyScope, dependencyGraph.GetDependencyScope(component.Component.Id));
                         component.DetectedBy = detector;
@@ -151,6 +171,9 @@ private IEnumerable<DetectedComponent> GatherSetOfDetectedComponentsUnmerged(IEn
                     }
                 }
 
+                record.TimeToAddRoots = totalTimeToAddRoots;
+                record.TimeToAddAncestors = totalTimeToAddAncestors;
+
                 return detectedComponents;
             }).ToList();
     }
@@ -223,35 +246,23 @@ private DetectedComponent MergeComponents(IEnumerable<DetectedComponent> enumera
     private void AddRootsToDetectedComponent(DetectedComponent detectedComponent, IDependencyGraph dependencyGraph, IComponentRecorder componentRecorder)
     {
         detectedComponent.DependencyRoots ??= new HashSet<TypedComponent>(new ComponentComparer());
-
         if (dependencyGraph == null)
         {
             return;
         }
 
-        var roots = dependencyGraph.GetExplicitReferencedDependencyIds(detectedComponent.Component.Id);
-
-        foreach (var rootId in roots)
-        {
-            detectedComponent.DependencyRoots.Add(componentRecorder.GetComponent(rootId));
-        }
+        detectedComponent.DependencyRoots.UnionWith(dependencyGraph.GetRootsAsTypedComponents(detectedComponent.Component.Id, componentRecorder.GetComponent));
     }
 
     private void AddAncestorsToDetectedComponent(DetectedComponent detectedComponent, IDependencyGraph dependencyGraph, IComponentRecorder componentRecorder)
     {
         detectedComponent.AncestralDependencyRoots ??= new HashSet<TypedComponent>(new ComponentComparer());
-
         if (dependencyGraph == null)
         {
             return;
         }
 
-        var roots = dependencyGraph.GetAncestors(detectedComponent.Component.Id);
-
-        foreach (var rootId in roots)
-        {
-            detectedComponent.AncestralDependencyRoots.Add(componentRecorder.GetComponent(rootId));
-        }
+        detectedComponent.AncestralDependencyRoots.UnionWith(dependencyGraph.GetAncestorsAsTypedComponents(detectedComponent.Component.Id, componentRecorder.GetComponent));
     }
 
     private HashSet<string> MakeFilePathsRelative(ILogger logger, DirectoryInfo rootDirectory, HashSet<string> filePaths)