From f30099f7f4fca48db33b93561a969be7e4096771 Mon Sep 17 00:00:00 2001 From: Paul Dorsch Date: Fri, 2 May 2025 09:12:42 -0700 Subject: [PATCH 01/12] test --- .../DependencyGraph/ComponentRecorder.cs | 1 + .../Records/DependencyGraphInnerRecord.cs | 12 ++++++++++++ .../Telemetry/Records/DependencyGraphRecord.cs | 8 ++++++++ .../Services/DetectorProcessingService.cs | 1 + .../DefaultGraphTranslationService.cs | 18 +++++++++++++++++- 5 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 src/Microsoft.ComponentDetection.Common/Telemetry/Records/DependencyGraphInnerRecord.cs create mode 100644 src/Microsoft.ComponentDetection.Common/Telemetry/Records/DependencyGraphRecord.cs diff --git a/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs b/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs index 7a9be52bf..7aa74cbc4 100644 --- a/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs +++ b/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs @@ -33,6 +33,7 @@ public TypedComponent GetComponent(string componentId) return this.singleFileRecorders.Select(x => x.GetComponent(componentId)?.Component).FirstOrDefault(x => x != null); } + // TODO: INVESTIGATE SUBSTRATE public IEnumerable GetDetectedComponents() { IEnumerable detectedComponents; diff --git a/src/Microsoft.ComponentDetection.Common/Telemetry/Records/DependencyGraphInnerRecord.cs b/src/Microsoft.ComponentDetection.Common/Telemetry/Records/DependencyGraphInnerRecord.cs new file mode 100644 index 000000000..af5c33c90 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Common/Telemetry/Records/DependencyGraphInnerRecord.cs @@ -0,0 +1,12 @@ +namespace Microsoft.ComponentDetection.Common.Telemetry.Records; + +public class DependencyGraphInnerRecord : BaseDetectionTelemetryRecord +{ + public override string RecordName => "DependencyGraphInnerRecord"; + + public string DetectorId { get; set; } + + public string ComponentId { get; set; } + + public int Count { get; set; } +} diff --git a/src/Microsoft.ComponentDetection.Common/Telemetry/Records/DependencyGraphRecord.cs b/src/Microsoft.ComponentDetection.Common/Telemetry/Records/DependencyGraphRecord.cs new file mode 100644 index 000000000..7d2c71f31 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Common/Telemetry/Records/DependencyGraphRecord.cs @@ -0,0 +1,8 @@ +namespace Microsoft.ComponentDetection.Common.Telemetry.Records; + +public class DependencyGraphRecord : BaseDetectionTelemetryRecord +{ + public override string RecordName => "DependencyGraphRecord"; + + public string DetectorId { get; set; } +} diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs index 92b3154af..9919c3b20 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs @@ -62,6 +62,7 @@ public async Task ProcessDetectorsAsync( await this.experimentService.InitializeAsync(); this.experimentService.RemoveUnwantedExperimentsbyDetectors(detectorRestrictions.DisabledDetectors); + // TODO: INVESTIGATE SUBSTRATE IEnumerable> scanTasks = detectors .Select(async detector => { diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs index ceed83e84..54a8bceac 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs @@ -21,6 +21,7 @@ public class DefaultGraphTranslationService : IGraphTranslationService public DefaultGraphTranslationService(ILogger logger) => this.logger = logger; + // TODO: INVESTIGATE SUBSTRATE public ScanResult GenerateScanResultFromProcessingResult( DetectorProcessingResult detectorProcessingResult, ScanSettings settings, @@ -91,6 +92,11 @@ private IEnumerable GatherSetOfDetectedComponentsUnmerged(IEn .Where(recorderDetectorPair => recorderDetectorPair.Recorder != null) .SelectMany(recorderDetectorPair => { + using var record = new DependencyGraphRecord() + { + DetectorId = recorderDetectorPair.Detector.Id, + }; + var detector = recorderDetectorPair.Detector; var componentRecorder = recorderDetectorPair.Recorder; var detectedComponents = componentRecorder.GetDetectedComponents(); @@ -98,8 +104,16 @@ private IEnumerable GatherSetOfDetectedComponentsUnmerged(IEn // Note that it looks like we are building up detected components functionally, but they are not immutable -- the code is just written // to look like a pipeline. + var count = 0; foreach (var component in detectedComponents) { + using var innerRecord = new DependencyGraphInnerRecord() + { + DetectorId = recorderDetectorPair.Detector.Id, + ComponentId = component.Component.Id, + Count = count++, + }; + // clone custom locations and make them relative to root. var declaredRawFilePaths = component.FilePaths ?? []; var componentCustomLocations = JsonConvert.DeserializeObject>(JsonConvert.SerializeObject(declaredRawFilePaths)); @@ -109,8 +123,10 @@ private IEnumerable GatherSetOfDetectedComponentsUnmerged(IEn component.FilePaths?.Clear(); } + var relevantDependencyGraphs = dependencyGraphsByLocation.Where(x => x.Value.Contains(component.Component.Id)); + // Information about each component is relative to all of the graphs it is present in, so we take all graphs containing a given component and apply the graph data. - foreach (var graphKvp in dependencyGraphsByLocation.Where(x => x.Value.Contains(component.Component.Id))) + foreach (var graphKvp in relevantDependencyGraphs) { var location = graphKvp.Key; var dependencyGraph = graphKvp.Value; From 5405d78affc07ee287dc30542756f94952dc335e Mon Sep 17 00:00:00 2001 From: Paul Dorsch Date: Fri, 2 May 2025 09:22:47 -0700 Subject: [PATCH 02/12] update comments --- .../DependencyGraph/ComponentRecorder.cs | 1 - .../Services/DetectorProcessingService.cs | 1 - .../Services/GraphTranslation/DefaultGraphTranslationService.cs | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs b/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs index e040dc10e..aa72fb234 100644 --- a/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs +++ b/src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs @@ -34,7 +34,6 @@ public TypedComponent GetComponent(string componentId) return this.singleFileRecorders.Values.Select(x => x.GetComponent(componentId)?.Component).FirstOrDefault(x => x != null); } - // TODO: INVESTIGATE SUBSTRATE public IEnumerable GetDetectedComponents() { IEnumerable detectedComponents; diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs index 838c5e49b..248c401d1 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs @@ -62,7 +62,6 @@ public async Task ProcessDetectorsAsync( await this.experimentService.InitializeAsync(); this.experimentService.RemoveUnwantedExperimentsbyDetectors(detectorRestrictions.DisabledDetectors); - // TODO: INVESTIGATE SUBSTRATE IEnumerable> scanTasks = detectors .Select(async detector => { diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs index b374a7d7b..24a44ab18 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs @@ -20,7 +20,6 @@ public class DefaultGraphTranslationService : IGraphTranslationService public DefaultGraphTranslationService(ILogger logger) => this.logger = logger; - // TODO: INVESTIGATE SUBSTRATE public ScanResult GenerateScanResultFromProcessingResult( DetectorProcessingResult detectorProcessingResult, ScanSettings settings, @@ -85,6 +84,7 @@ private void LogComponentScopeTelemetry(List components) }); } + // TODO: INVESTIGATE PERF private IEnumerable GatherSetOfDetectedComponentsUnmerged(IEnumerable<(IComponentDetector Detector, ComponentRecorder Recorder)> recorderDetectorPairs, DirectoryInfo rootDirectory, bool updateLocations) { return recorderDetectorPairs From a9f3ad6156c270a8c32ec18fbd477046efb81b91 Mon Sep 17 00:00:00 2001 From: Paul Dorsch Date: Fri, 2 May 2025 13:59:49 -0700 Subject: [PATCH 03/12] initial VERY rough attempt to cache roots/ancestors --- .../DependencyGraph/DependencyGraph.cs | 36 ++++++++++++++----- .../Records/DependencyGraphInnerRecord.cs | 6 ++++ .../DefaultGraphTranslationService.cs | 13 +++++++ 3 files changed, 47 insertions(+), 8 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs b/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs index cff395fc2..4270acb7b 100644 --- a/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs +++ b/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs @@ -20,9 +20,15 @@ internal class DependencyGraph : IDependencyGraph private readonly bool enableManualTrackingOfExplicitReferences; + private readonly ConcurrentDictionary> rootsCache; + + private readonly ConcurrentDictionary> ancestorsCache; + public DependencyGraph(bool enableManualTrackingOfExplicitReferences) { this.componentNodes = new ConcurrentDictionary(); + this.rootsCache = new ConcurrentDictionary>(); + this.ancestorsCache = new ConcurrentDictionary>(); this.enableManualTrackingOfExplicitReferences = enableManualTrackingOfExplicitReferences; } @@ -80,11 +86,7 @@ public ICollection GetExplicitReferencedDependencyIds(string componentId throw new ArgumentException(string.Format(null, MissingNodeFormat, componentId), paramName: nameof(componentId)); } - IList explicitReferencedDependencyIds = []; - - this.GetExplicitReferencedDependencies(componentRef, explicitReferencedDependencyIds, new HashSet()); - - return explicitReferencedDependencyIds; + return this.GetExplicitReferencedDependencies(componentRef, new HashSet()).ToList(); } /// @@ -156,22 +158,33 @@ bool IDependencyGraph.IsComponentExplicitlyReferenced(string componentId) return this.IsExplicitReferencedDependency(this.componentNodes[componentId]); } - private void GetExplicitReferencedDependencies(ComponentRefNode component, IList explicitReferencedDependencyIds, ISet visited) + private IEnumerable GetExplicitReferencedDependencies(ComponentRefNode component, ISet visited) { + if (this.rootsCache.TryGetValue(component.Id, out var cachedExplicitReferencedDependencyIds)) + { + return cachedExplicitReferencedDependencyIds; + } + if (this.IsExplicitReferencedDependency(component)) { - explicitReferencedDependencyIds.Add(component.Id); + var explicitReferencedDependencyIdsSet = new HashSet() { component.Id }; + this.rootsCache.TryAdd(component.Id, explicitReferencedDependencyIdsSet); + return explicitReferencedDependencyIdsSet; } visited.Add(component.Id); + IEnumerable explicitReferencedDependencyIds = []; foreach (var parentId in component.DependedOnByIds) { if (!visited.Contains(parentId)) { - this.GetExplicitReferencedDependencies(this.componentNodes[parentId], explicitReferencedDependencyIds, visited); + explicitReferencedDependencyIds = explicitReferencedDependencyIds.Concat(this.GetExplicitReferencedDependencies(this.componentNodes[parentId], visited)); } } + + this.rootsCache.TryAdd(component.Id, explicitReferencedDependencyIds.ToHashSet()); + return explicitReferencedDependencyIds; } private bool IsExplicitReferencedDependency(ComponentRefNode component) @@ -198,6 +211,11 @@ private void AddDependency(string componentId, string parentComponentId) private void GetAncestorsRecursive(ComponentRefNode componentRef, IDictionary ancestors, int depth) { + if (this.ancestorsCache.TryGetValue(componentRef.Id, out var cachedAncestors)) + { + return; + } + foreach (var parentId in componentRef.DependedOnByIds) { if (ancestors.ContainsKey(parentId)) @@ -208,6 +226,8 @@ private void GetAncestorsRecursive(ComponentRefNode componentRef, IDictionary "DependencyGraphInnerRecord"; @@ -9,4 +11,8 @@ public class DependencyGraphInnerRecord : BaseDetectionTelemetryRecord public string ComponentId { get; set; } public int Count { get; set; } + + public TimeSpan? TimeToAddRoots { get; set; } + + public TimeSpan? TimeToAddAncestors { get; set; } } diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs index 24a44ab18..91ce168e1 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs @@ -127,6 +127,9 @@ private IEnumerable GatherSetOfDetectedComponentsUnmerged(IEn var relevantDependencyGraphs = dependencyGraphsByLocation.Where(x => x.Value.Contains(component.Component.Id)); + var totalTimeToAddRoots = TimeSpan.Zero; + var totalTimeToAddAncestors = TimeSpan.Zero; + // Information about each component is relative to all of the graphs it is present in, so we take all graphs containing a given component and apply the graph data. foreach (var graphKvp in relevantDependencyGraphs) { @@ -134,10 +137,17 @@ private IEnumerable GatherSetOfDetectedComponentsUnmerged(IEn var dependencyGraph = graphKvp.Value; // Calculate roots of the component + var startTime = DateTime.UtcNow; this.AddRootsToDetectedComponent(component, dependencyGraph, componentRecorder); + var endTime = DateTime.UtcNow; + totalTimeToAddRoots += endTime - startTime; // Calculate Ancestors of the component + var startTime2 = DateTime.UtcNow; this.AddAncestorsToDetectedComponent(component, dependencyGraph, componentRecorder); + var endTime2 = DateTime.UtcNow; + totalTimeToAddAncestors += endTime2 - startTime2; + component.DevelopmentDependency = this.MergeDevDependency(component.DevelopmentDependency, dependencyGraph.IsDevelopmentDependency(component.Component.Id)); component.DependencyScope = DependencyScopeComparer.GetMergedDependencyScope(component.DependencyScope, dependencyGraph.GetDependencyScope(component.Component.Id)); component.DetectedBy = detector; @@ -165,6 +175,9 @@ private IEnumerable GatherSetOfDetectedComponentsUnmerged(IEn } } } + + innerRecord.TimeToAddRoots = totalTimeToAddRoots; + innerRecord.TimeToAddAncestors = totalTimeToAddAncestors; } return detectedComponents; From dee48d55c12434fe1e14b6e4626f2a21be40760c Mon Sep 17 00:00:00 2001 From: Paul Dorsch Date: Tue, 6 May 2025 07:05:00 -0700 Subject: [PATCH 04/12] pre-compute typed components --- .../DependencyGraph/DependencyGraph.cs | 83 +++++++---- .../Records/DependencyGraphFillRootsRecord.cs | 12 ++ .../IComponentRecorder.cs | 7 + .../DefaultGraphTranslationService.cs | 33 ++--- .../DependencyGraphTests.cs | 136 ++++++++++-------- .../pytest_pyresultreport.egg-info/PKG-INFO | 24 ++++ .../SOURCES.txt | 9 ++ .../dependency_links.txt | 1 + .../entry_points.txt | 2 + .../requires.txt | 1 + .../top_level.txt | 1 + 11 files changed, 205 insertions(+), 104 deletions(-) create mode 100644 src/Microsoft.ComponentDetection.Common/Telemetry/Records/DependencyGraphFillRootsRecord.cs create mode 100644 test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/PKG-INFO create mode 100644 test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/SOURCES.txt create mode 100644 test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/dependency_links.txt create mode 100644 test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/entry_points.txt create mode 100644 test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/requires.txt create mode 100644 test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/top_level.txt diff --git a/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs b/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs index 4270acb7b..ad777a5fc 100644 --- a/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs +++ b/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs @@ -7,6 +7,7 @@ using System.Text; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.BcdeModels; +using Microsoft.ComponentDetection.Contracts.TypedComponent; [assembly: InternalsVisibleTo("Microsoft.ComponentDetection.Common.Tests")] @@ -16,19 +17,15 @@ internal class DependencyGraph : IDependencyGraph { private static readonly CompositeFormat MissingNodeFormat = CompositeFormat.Parse(Resources.MissingNodeInDependencyGraph); - private readonly ConcurrentDictionary componentNodes; - private readonly bool enableManualTrackingOfExplicitReferences; - private readonly ConcurrentDictionary> rootsCache; + private readonly ConcurrentDictionary componentNodes; - private readonly ConcurrentDictionary> ancestorsCache; + private Type currentComponentType; public DependencyGraph(bool enableManualTrackingOfExplicitReferences) { this.componentNodes = new ConcurrentDictionary(); - this.rootsCache = new ConcurrentDictionary>(); - this.ancestorsCache = new ConcurrentDictionary>(); this.enableManualTrackingOfExplicitReferences = enableManualTrackingOfExplicitReferences; } @@ -86,7 +83,11 @@ public ICollection GetExplicitReferencedDependencyIds(string componentId throw new ArgumentException(string.Format(null, MissingNodeFormat, componentId), paramName: nameof(componentId)); } - return this.GetExplicitReferencedDependencies(componentRef, new HashSet()).ToList(); + IList explicitReferencedDependencyIds = []; + + this.GetExplicitReferencedDependencies(componentRef, explicitReferencedDependencyIds, new HashSet()); + + return explicitReferencedDependencyIds; } /// @@ -143,6 +144,48 @@ public ICollection GetAncestors(string componentId) .ToList(); } + public HashSet GetAncestorsAsTypedComponents(string componentId, Func toTypedComponent) + { + ArgumentNullException.ThrowIfNull(componentId); + if (this.ShouldFillTypedComponents(toTypedComponent)) + { + this.FillTypedComponents(toTypedComponent); + } + + return this.GetAncestors(componentId) + .Select(c => this.componentNodes[c].TypedComponent) + .ToHashSet(new ComponentComparer()); + } + + public HashSet GetRootsAsTypedComponents(string componentId, Func toTypedComponent) + { + ArgumentNullException.ThrowIfNull(componentId); + if (this.ShouldFillTypedComponents(toTypedComponent)) + { + this.FillTypedComponents(toTypedComponent); + } + + return this.GetExplicitReferencedDependencyIds(componentId) + .Select(c => this.componentNodes[c].TypedComponent) + .ToHashSet(new ComponentComparer()); + } + + public bool ShouldFillTypedComponents(Func toTypedComponent) + { + ArgumentNullException.ThrowIfNull(toTypedComponent); + return toTypedComponent.Method?.ReturnType?.GetType() != this.currentComponentType?.GetType(); + } + + private void FillTypedComponents(Func toTypedComponent) + { + foreach (var componentId in this.componentNodes.Values) + { + componentId.TypedComponent = toTypedComponent(componentId.Id); + } + + this.currentComponentType = toTypedComponent.Method.ReturnType.GetType(); + } + IEnumerable IDependencyGraph.GetDependenciesForComponent(string componentId) { return this.GetDependenciesForComponent(componentId).ToImmutableList(); @@ -158,33 +201,22 @@ bool IDependencyGraph.IsComponentExplicitlyReferenced(string componentId) return this.IsExplicitReferencedDependency(this.componentNodes[componentId]); } - private IEnumerable GetExplicitReferencedDependencies(ComponentRefNode component, ISet visited) + private void GetExplicitReferencedDependencies(ComponentRefNode component, IList explicitReferencedDependencyIds, ISet visited) { - if (this.rootsCache.TryGetValue(component.Id, out var cachedExplicitReferencedDependencyIds)) - { - return cachedExplicitReferencedDependencyIds; - } - if (this.IsExplicitReferencedDependency(component)) { - var explicitReferencedDependencyIdsSet = new HashSet() { component.Id }; - this.rootsCache.TryAdd(component.Id, explicitReferencedDependencyIdsSet); - return explicitReferencedDependencyIdsSet; + explicitReferencedDependencyIds.Add(component.Id); } visited.Add(component.Id); - IEnumerable explicitReferencedDependencyIds = []; foreach (var parentId in component.DependedOnByIds) { if (!visited.Contains(parentId)) { - explicitReferencedDependencyIds = explicitReferencedDependencyIds.Concat(this.GetExplicitReferencedDependencies(this.componentNodes[parentId], visited)); + this.GetExplicitReferencedDependencies(this.componentNodes[parentId], explicitReferencedDependencyIds, visited); } } - - this.rootsCache.TryAdd(component.Id, explicitReferencedDependencyIds.ToHashSet()); - return explicitReferencedDependencyIds; } private bool IsExplicitReferencedDependency(ComponentRefNode component) @@ -211,11 +243,6 @@ private void AddDependency(string componentId, string parentComponentId) private void GetAncestorsRecursive(ComponentRefNode componentRef, IDictionary ancestors, int depth) { - if (this.ancestorsCache.TryGetValue(componentRef.Id, out var cachedAncestors)) - { - return; - } - foreach (var parentId in componentRef.DependedOnByIds) { if (ancestors.ContainsKey(parentId)) @@ -226,8 +253,6 @@ private void GetAncestorsRecursive(ComponentRefNode componentRef, IDictionary "DependencyGraphFillRootsRecord"; + + public string DetectorId { get; set; } + + public int Count { get; set; } + + public int ComponentCount { get; set; } +} diff --git a/src/Microsoft.ComponentDetection.Contracts/IComponentRecorder.cs b/src/Microsoft.ComponentDetection.Contracts/IComponentRecorder.cs index dafaa23f1..3cea9334e 100644 --- a/src/Microsoft.ComponentDetection.Contracts/IComponentRecorder.cs +++ b/src/Microsoft.ComponentDetection.Contracts/IComponentRecorder.cs @@ -1,5 +1,6 @@ namespace Microsoft.ComponentDetection.Contracts; +using System; using System.Collections.Generic; using Microsoft.ComponentDetection.Contracts.BcdeModels; @@ -124,4 +125,10 @@ public interface IDependencyGraph /// The component id to look up ancestors for. /// The componentIds that are ancestors for a given componentId. ICollection GetAncestors(string componentId); + + public HashSet GetRootsAsTypedComponents(string componentId, Func toTypedComponent); + + public HashSet GetAncestorsAsTypedComponents(string componentId, Func toTypedComponent); + + public bool ShouldFillTypedComponents(Func toTypedComponent); } diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs index 91ce168e1..bd3680bd2 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs @@ -101,6 +101,21 @@ private IEnumerable GatherSetOfDetectedComponentsUnmerged(IEn var detectedComponents = componentRecorder.GetDetectedComponents(); var dependencyGraphsByLocation = componentRecorder.GetDependencyGraphsByLocation(); + /* + var count = 0; + foreach (var dependendencyGraph in dependencyGraphsByLocation.Values) + { + using var fillRootsRecord = new DependencyGraphFillRootsRecord() + { + DetectorId = recorderDetectorPair.Detector.Id, + Count = count++, + }; + + dependendencyGraph.FillRoots(componentRecorder.GetComponent); + dependendencyGraph.FillTypedComponents(componentRecorder.GetComponent); + } + */ + // Note that it looks like we are building up detected components functionally, but they are not immutable -- the code is just written // to look like a pipeline. var count = 0; @@ -251,36 +266,22 @@ private DetectedComponent MergeComponents(IEnumerable enumera private void AddRootsToDetectedComponent(DetectedComponent detectedComponent, IDependencyGraph dependencyGraph, IComponentRecorder componentRecorder) { - detectedComponent.DependencyRoots ??= new HashSet(new ComponentComparer()); - if (dependencyGraph == null) { return; } - var roots = dependencyGraph.GetExplicitReferencedDependencyIds(detectedComponent.Component.Id); - - foreach (var rootId in roots) - { - detectedComponent.DependencyRoots.Add(componentRecorder.GetComponent(rootId)); - } + detectedComponent.DependencyRoots = dependencyGraph.GetRootsAsTypedComponents(detectedComponent.Component.Id, componentRecorder.GetComponent); } private void AddAncestorsToDetectedComponent(DetectedComponent detectedComponent, IDependencyGraph dependencyGraph, IComponentRecorder componentRecorder) { - detectedComponent.AncestralDependencyRoots ??= new HashSet(new ComponentComparer()); - if (dependencyGraph == null) { return; } - var roots = dependencyGraph.GetAncestors(detectedComponent.Component.Id); - - foreach (var rootId in roots) - { - detectedComponent.AncestralDependencyRoots.Add(componentRecorder.GetComponent(rootId)); - } + detectedComponent.AncestralDependencyRoots = dependencyGraph.GetAncestorsAsTypedComponents(detectedComponent.Component.Id, componentRecorder.GetComponent); } private HashSet MakeFilePathsRelative(ILogger logger, DirectoryInfo rootDirectory, HashSet filePaths) diff --git a/test/Microsoft.ComponentDetection.Common.Tests/DependencyGraphTests.cs b/test/Microsoft.ComponentDetection.Common.Tests/DependencyGraphTests.cs index 200f03c5e..278d0b5ba 100644 --- a/test/Microsoft.ComponentDetection.Common.Tests/DependencyGraphTests.cs +++ b/test/Microsoft.ComponentDetection.Common.Tests/DependencyGraphTests.cs @@ -2,28 +2,32 @@ namespace Microsoft.ComponentDetection.Common.Tests; using System; using FluentAssertions; +using Microsoft.ComponentDetection.Common.DependencyGraph; using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.TypedComponent; using Microsoft.VisualStudio.TestTools.UnitTesting; [TestClass] public class DependencyGraphTests { - private DependencyGraph.DependencyGraph dependencyGraph; + private DependencyGraph dependencyGraph; + private IComponentRecorder componentRecorder; [TestInitialize] public void TestInitializer() { // Default value of true -- some tests will create their own, though. - this.dependencyGraph = new DependencyGraph.DependencyGraph(true); + this.dependencyGraph = new DependencyGraph(true); + this.componentRecorder = new ComponentRecorder(enableManualTrackingOfExplicitReferences: false); } [TestMethod] public void AddComponent_ParentComponentIdIsPresent_DependencyRelationIsAdded() { - var componentA = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentA" }; - var componentB = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentB" }; - var componentC = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentC" }; - var componentD = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentD" }; + var componentA = new DependencyGraph.ComponentRefNode { Id = "componentA" }; + var componentB = new DependencyGraph.ComponentRefNode { Id = "componentB" }; + var componentC = new DependencyGraph.ComponentRefNode { Id = "componentC" }; + var componentD = new DependencyGraph.ComponentRefNode { Id = "componentD" }; this.dependencyGraph.AddComponent(componentD); this.dependencyGraph.AddComponent(componentB, parentComponentId: componentD.Id); @@ -51,7 +55,7 @@ public void AddComponent_ParentComponentIdIsPresent_DependencyRelationIsAdded() [TestMethod] public void AddComponent_parentComponentIdIsNotPresent_AdditionTakePlaceWithoutThrowing() { - var componentA = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentA", IsExplicitReferencedDependency = true }; + var componentA = new DependencyGraph.ComponentRefNode { Id = "componentA", IsExplicitReferencedDependency = true }; Action action = () => this.dependencyGraph.AddComponent(componentA); action.Should().NotThrow(); @@ -70,15 +74,15 @@ public void AddComponent_ComponentIsNull_ArgumentNullExceptionIsThrow() [TestMethod] public void AddComponent_ComponentHasNoId_ArgumentNullExceptionIsThrow() { - var component = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = null }; + var component = new DependencyGraph.ComponentRefNode { Id = null }; Action action = () => this.dependencyGraph.AddComponent(component); action.Should().Throw(); - component = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = string.Empty }; + component = new DependencyGraph.ComponentRefNode { Id = string.Empty }; action = () => this.dependencyGraph.AddComponent(component); action.Should().Throw(); - component = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = " " }; + component = new DependencyGraph.ComponentRefNode { Id = " " }; action = () => this.dependencyGraph.AddComponent(component); action.Should().Throw(); } @@ -86,7 +90,7 @@ public void AddComponent_ComponentHasNoId_ArgumentNullExceptionIsThrow() [TestMethod] public void AddComponent_ParentComponentWasNotAddedPreviously_ArgumentExceptionIsThrown() { - var componentA = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentA" }; + var componentA = new DependencyGraph.ComponentRefNode { Id = "componentA" }; Action action = () => this.dependencyGraph.AddComponent(componentA, parentComponentId: "nonexistingComponent"); @@ -96,12 +100,12 @@ public void AddComponent_ParentComponentWasNotAddedPreviously_ArgumentExceptionI [TestMethod] public void GetExplicitReferencedDependencyIds_ComponentsWereAddedSpecifyingRoot_RootsAreReturned() { - var componentA = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentA", IsExplicitReferencedDependency = true }; - var componentB = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentB" }; - var componentC = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentC" }; - var componentD = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentD" }; - var componentE = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentE", IsExplicitReferencedDependency = true }; - var componentF = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentF" }; + var componentA = new DependencyGraph.ComponentRefNode { Id = "componentA", IsExplicitReferencedDependency = true }; + var componentB = new DependencyGraph.ComponentRefNode { Id = "componentB" }; + var componentC = new DependencyGraph.ComponentRefNode { Id = "componentC" }; + var componentD = new DependencyGraph.ComponentRefNode { Id = "componentD" }; + var componentE = new DependencyGraph.ComponentRefNode { Id = "componentE", IsExplicitReferencedDependency = true }; + var componentF = new DependencyGraph.ComponentRefNode { Id = "componentF" }; this.dependencyGraph.AddComponent(componentA); this.dependencyGraph.AddComponent(componentB, componentA.Id); @@ -137,10 +141,12 @@ public void GetExplicitReferencedDependencyIds_ComponentsWereAddedSpecifyingRoot } [TestMethod] - public void GetExplicitReferencedDependencyIds_ComponentsWereAddedWithoutSpecifyingRoot_RootsAreEmpty() + [DataRow(true)] + [DataRow(false)] + public void GetExplicitReferencedDependencyIds_ComponentsWereAddedWithoutSpecifyingRoot_RootsAreEmpty(bool shouldPreFillRoots) { - var componentA = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentA" }; - var componentB = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentB" }; + var componentA = new DependencyGraph.ComponentRefNode { Id = "componentA" }; + var componentB = new DependencyGraph.ComponentRefNode { Id = "componentB" }; this.dependencyGraph.AddComponent(componentA); this.dependencyGraph.AddComponent(componentB, componentA.Id); @@ -153,9 +159,11 @@ public void GetExplicitReferencedDependencyIds_ComponentsWereAddedWithoutSpecify } [TestMethod] - public void GetExplicitReferencedDependencyIds_ComponentIsRoot_ARootIsRootOfItSelf() + [DataRow(true)] + [DataRow(false)] + public void GetExplicitReferencedDependencyIds_ComponentIsRoot_ARootIsRootOfItSelf(bool shouldPreFillRoots) { - var componentA = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentA", IsExplicitReferencedDependency = true }; + var componentA = new DependencyGraph.ComponentRefNode { Id = "componentA", IsExplicitReferencedDependency = true }; this.dependencyGraph.AddComponent(componentA); var aRoots = this.dependencyGraph.GetExplicitReferencedDependencyIds(componentA.Id); @@ -164,11 +172,13 @@ public void GetExplicitReferencedDependencyIds_ComponentIsRoot_ARootIsRootOfItSe } [TestMethod] - public void GetExplicitReferencedDependencyIds_RootHasParent_ReturnItselfAndItsParents() + [DataRow(true)] + [DataRow(false)] + public void GetExplicitReferencedDependencyIds_RootHasParent_ReturnItselfAndItsParents(bool shouldPreFillRoots) { - var componentA = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentA", IsExplicitReferencedDependency = true }; - var componentB = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentB", IsExplicitReferencedDependency = true }; - var componentC = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentC", IsExplicitReferencedDependency = true }; + var componentA = new DependencyGraph.ComponentRefNode { Id = "componentA", IsExplicitReferencedDependency = true }; + var componentB = new DependencyGraph.ComponentRefNode { Id = "componentB", IsExplicitReferencedDependency = true }; + var componentC = new DependencyGraph.ComponentRefNode { Id = "componentC", IsExplicitReferencedDependency = true }; this.dependencyGraph.AddComponent(componentA); this.dependencyGraph.AddComponent(componentB, componentA.Id); @@ -191,18 +201,20 @@ public void GetExplicitReferencedDependencyIds_RootHasParent_ReturnItselfAndItsP } [TestMethod] - public void GetExplicitReferencedDependencyIds_InsertionOrderNotAffectedRoots() + [DataRow(true)] + [DataRow(false)] + public void GetExplicitReferencedDependencyIds_InsertionOrderNotAffectedRoots(bool shouldPreFillRoots) { - var componentA = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentA", IsExplicitReferencedDependency = true }; - var componentB = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentB" }; - var componentC = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentC", IsExplicitReferencedDependency = true }; + var componentA = new DependencyGraph.ComponentRefNode { Id = "componentA", IsExplicitReferencedDependency = true }; + var componentB = new DependencyGraph.ComponentRefNode { Id = "componentB" }; + var componentC = new DependencyGraph.ComponentRefNode { Id = "componentC", IsExplicitReferencedDependency = true }; this.dependencyGraph.AddComponent(componentA); this.dependencyGraph.AddComponent(componentB, componentA.Id); this.dependencyGraph.AddComponent(componentC); this.dependencyGraph.AddComponent(componentA, componentC.Id); - componentB = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentB", IsExplicitReferencedDependency = true }; + componentB = new DependencyGraph.ComponentRefNode { Id = "componentB", IsExplicitReferencedDependency = true }; this.dependencyGraph.AddComponent(componentB); var aRoots = this.dependencyGraph.GetExplicitReferencedDependencyIds(componentA.Id); @@ -222,12 +234,14 @@ public void GetExplicitReferencedDependencyIds_InsertionOrderNotAffectedRoots() } [TestMethod] - public void GetExplicitReferencedDependencyIds_UseManualSelectionTurnedOff_ComponentsWithNoParentsAreSelectedAsExplicitReferencedDependencies() + [DataRow(true)] + [DataRow(false)] + public void GetExplicitReferencedDependencyIds_UseManualSelectionTurnedOff_ComponentsWithNoParentsAreSelectedAsExplicitReferencedDependencies(bool shouldPreFillRoots) { - this.dependencyGraph = new DependencyGraph.DependencyGraph(false); - var componentA = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentA" }; - var componentB = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentB" }; - var componentC = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentC" }; + this.dependencyGraph = new DependencyGraph(false); + var componentA = new DependencyGraph.ComponentRefNode { Id = "componentA" }; + var componentB = new DependencyGraph.ComponentRefNode { Id = "componentB" }; + var componentC = new DependencyGraph.ComponentRefNode { Id = "componentC" }; this.dependencyGraph.AddComponent(componentA); this.dependencyGraph.AddComponent(componentB, componentA.Id); @@ -251,12 +265,14 @@ public void GetExplicitReferencedDependencyIds_UseManualSelectionTurnedOff_Compo } [TestMethod] - public void GetExplicitReferencedDependencyIds_UseManualSelectionTurnedOff_PropertyIsExplicitReferencedDependencyIsIgnored() + [DataRow(true)] + [DataRow(false)] + public void GetExplicitReferencedDependencyIds_UseManualSelectionTurnedOff_PropertyIsExplicitReferencedDependencyIsIgnored(bool shouldPreFillRoots) { - this.dependencyGraph = new DependencyGraph.DependencyGraph(false); - var componentA = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentA", IsExplicitReferencedDependency = true }; - var componentB = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentB", IsExplicitReferencedDependency = true }; - var componentC = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentC", IsExplicitReferencedDependency = true }; + this.dependencyGraph = new DependencyGraph(false); + var componentA = new DependencyGraph.ComponentRefNode { Id = "componentA", IsExplicitReferencedDependency = true }; + var componentB = new DependencyGraph.ComponentRefNode { Id = "componentB", IsExplicitReferencedDependency = true }; + var componentC = new DependencyGraph.ComponentRefNode { Id = "componentC", IsExplicitReferencedDependency = true }; this.dependencyGraph.AddComponent(componentA); this.dependencyGraph.AddComponent(componentB, componentA.Id); @@ -302,9 +318,9 @@ public void GetExplicitReferencedDependencyIds_ComponentIdIsNotRegisteredInGraph [TestMethod] public void IsDevelopmentDependency_ReturnsAsExpected() { - var componentA = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentA", IsDevelopmentDependency = true }; - var componentB = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentB", IsDevelopmentDependency = false }; - var componentC = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentC" }; + var componentA = new DependencyGraph.ComponentRefNode { Id = "componentA", IsDevelopmentDependency = true }; + var componentB = new DependencyGraph.ComponentRefNode { Id = "componentB", IsDevelopmentDependency = false }; + var componentC = new DependencyGraph.ComponentRefNode { Id = "componentC" }; this.dependencyGraph.AddComponent(componentA); this.dependencyGraph.AddComponent(componentB, componentA.Id); @@ -319,18 +335,18 @@ public void IsDevelopmentDependency_ReturnsAsExpected() [TestMethod] public void IsDevelopmentDependency_ReturnsAsExpected_AfterMerge() { - var componentA = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentA", IsDevelopmentDependency = true }; - var componentB = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentB", IsDevelopmentDependency = false }; - var componentC = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentC" }; + var componentA = new DependencyGraph.ComponentRefNode { Id = "componentA", IsDevelopmentDependency = true }; + var componentB = new DependencyGraph.ComponentRefNode { Id = "componentB", IsDevelopmentDependency = false }; + var componentC = new DependencyGraph.ComponentRefNode { Id = "componentC" }; this.dependencyGraph.AddComponent(componentA); this.dependencyGraph.AddComponent(componentB, componentA.Id); this.dependencyGraph.AddComponent(componentC); this.dependencyGraph.AddComponent(componentA, componentC.Id); - var componentANewValue = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentA", IsDevelopmentDependency = false }; - var componentBNewValue = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentB", IsDevelopmentDependency = true }; - var componentCNewValue = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentC", IsDevelopmentDependency = true }; + var componentANewValue = new DependencyGraph.ComponentRefNode { Id = "componentA", IsDevelopmentDependency = false }; + var componentBNewValue = new DependencyGraph.ComponentRefNode { Id = "componentB", IsDevelopmentDependency = true }; + var componentCNewValue = new DependencyGraph.ComponentRefNode { Id = "componentC", IsDevelopmentDependency = true }; this.dependencyGraph.AddComponent(componentANewValue); this.dependencyGraph.AddComponent(componentBNewValue); this.dependencyGraph.AddComponent(componentCNewValue); @@ -339,9 +355,9 @@ public void IsDevelopmentDependency_ReturnsAsExpected_AfterMerge() this.dependencyGraph.IsDevelopmentDependency(componentB.Id).Should().Be(false); this.dependencyGraph.IsDevelopmentDependency(componentC.Id).Should().Be(true); - var componentANullValue = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentA" }; - var componentBNullValue = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentB" }; - var componentCNullValue = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentC" }; + var componentANullValue = new DependencyGraph.ComponentRefNode { Id = "componentA" }; + var componentBNullValue = new DependencyGraph.ComponentRefNode { Id = "componentB" }; + var componentCNullValue = new DependencyGraph.ComponentRefNode { Id = "componentC" }; this.dependencyGraph.AddComponent(componentANullValue); this.dependencyGraph.AddComponent(componentBNullValue); this.dependencyGraph.AddComponent(componentCNullValue); @@ -354,9 +370,9 @@ public void IsDevelopmentDependency_ReturnsAsExpected_AfterMerge() [TestMethod] public void GetAncestors_ReturnsAsExpected() { - var componentA = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentA" }; - var componentB = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentB" }; - var componentC = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentC" }; + var componentA = new DependencyGraph.ComponentRefNode { Id = "componentA" }; + var componentB = new DependencyGraph.ComponentRefNode { Id = "componentB" }; + var componentC = new DependencyGraph.ComponentRefNode { Id = "componentC" }; this.dependencyGraph.AddComponent(componentA); this.dependencyGraph.AddComponent(componentB, componentA.Id); @@ -387,10 +403,10 @@ public void GetAncestors_Null_ThrowsException() [TestMethod] public void GetAncestors_Cyclic_ReturnsAsExpected() { - var root = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "root" }; - var componentA = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentA" }; - var componentB = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentB" }; - var componentC = new DependencyGraph.DependencyGraph.ComponentRefNode { Id = "componentC" }; + var root = new DependencyGraph.ComponentRefNode { Id = "root" }; + var componentA = new DependencyGraph.ComponentRefNode { Id = "componentA" }; + var componentB = new DependencyGraph.ComponentRefNode { Id = "componentB" }; + var componentC = new DependencyGraph.ComponentRefNode { Id = "componentC" }; // root -> componentA -> componentB -> componentC --> loops to componentA this.dependencyGraph.AddComponent(root); diff --git a/test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/PKG-INFO b/test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/PKG-INFO new file mode 100644 index 000000000..271114bcc --- /dev/null +++ b/test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/PKG-INFO @@ -0,0 +1,24 @@ +Metadata-Version: 2.4 +Name: pytest_pyresultreport +Version: 0.1.0 +Summary: plugin to modify pytest reports +Author: CG +Maintainer: CG +Classifier: Framework :: Pytest +Classifier: Intended Audience :: Developers +Classifier: Topic :: Software Development :: Testing +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Operating System :: OS Independent +Requires-Python: >=3.7 +Requires-Dist: pytest>=3.7 +Dynamic: author +Dynamic: classifier +Dynamic: maintainer +Dynamic: requires-dist +Dynamic: requires-python +Dynamic: summary diff --git a/test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/SOURCES.txt b/test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/SOURCES.txt new file mode 100644 index 000000000..99dc197d7 --- /dev/null +++ b/test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/SOURCES.txt @@ -0,0 +1,9 @@ +setup.py +src/pytest_pyresultreport/__init__.py +src/pytest_pyresultreport/plugin.py +src/pytest_pyresultreport.egg-info/PKG-INFO +src/pytest_pyresultreport.egg-info/SOURCES.txt +src/pytest_pyresultreport.egg-info/dependency_links.txt +src/pytest_pyresultreport.egg-info/entry_points.txt +src/pytest_pyresultreport.egg-info/requires.txt +src/pytest_pyresultreport.egg-info/top_level.txt \ No newline at end of file diff --git a/test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/dependency_links.txt b/test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/dependency_links.txt new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/entry_points.txt b/test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/entry_points.txt new file mode 100644 index 000000000..68c6cc24e --- /dev/null +++ b/test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/entry_points.txt @@ -0,0 +1,2 @@ +[pytest11] +py_result_report = pytest_pyresultreport.plugin diff --git a/test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/requires.txt b/test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/requires.txt new file mode 100644 index 000000000..b4b839007 --- /dev/null +++ b/test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/requires.txt @@ -0,0 +1 @@ +pytest>=3.7 diff --git a/test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/top_level.txt b/test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/top_level.txt new file mode 100644 index 000000000..03af29338 --- /dev/null +++ b/test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/top_level.txt @@ -0,0 +1 @@ +pytest_pyresultreport From 3e41c9bbbd1204e422ce27b3c44e43ca0fbfd810 Mon Sep 17 00:00:00 2001 From: Paul Dorsch Date: Tue, 6 May 2025 07:05:41 -0700 Subject: [PATCH 05/12] remove --- .../pytest_pyresultreport.egg-info/PKG-INFO | 24 ------------------- .../SOURCES.txt | 9 ------- .../dependency_links.txt | 1 - .../entry_points.txt | 2 -- .../requires.txt | 1 - .../top_level.txt | 1 - 6 files changed, 38 deletions(-) delete mode 100644 test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/PKG-INFO delete mode 100644 test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/SOURCES.txt delete mode 100644 test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/dependency_links.txt delete mode 100644 test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/entry_points.txt delete mode 100644 test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/requires.txt delete mode 100644 test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/top_level.txt diff --git a/test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/PKG-INFO b/test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/PKG-INFO deleted file mode 100644 index 271114bcc..000000000 --- a/test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/PKG-INFO +++ /dev/null @@ -1,24 +0,0 @@ -Metadata-Version: 2.4 -Name: pytest_pyresultreport -Version: 0.1.0 -Summary: plugin to modify pytest reports -Author: CG -Maintainer: CG -Classifier: Framework :: Pytest -Classifier: Intended Audience :: Developers -Classifier: Topic :: Software Development :: Testing -Classifier: Programming Language :: Python :: 3.7 -Classifier: Programming Language :: Python :: 3.8 -Classifier: Programming Language :: Python :: 3.9 -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: Implementation :: CPython -Classifier: Programming Language :: Python :: Implementation :: PyPy -Classifier: Operating System :: OS Independent -Requires-Python: >=3.7 -Requires-Dist: pytest>=3.7 -Dynamic: author -Dynamic: classifier -Dynamic: maintainer -Dynamic: requires-dist -Dynamic: requires-python -Dynamic: summary diff --git a/test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/SOURCES.txt b/test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/SOURCES.txt deleted file mode 100644 index 99dc197d7..000000000 --- a/test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/SOURCES.txt +++ /dev/null @@ -1,9 +0,0 @@ -setup.py -src/pytest_pyresultreport/__init__.py -src/pytest_pyresultreport/plugin.py -src/pytest_pyresultreport.egg-info/PKG-INFO -src/pytest_pyresultreport.egg-info/SOURCES.txt -src/pytest_pyresultreport.egg-info/dependency_links.txt -src/pytest_pyresultreport.egg-info/entry_points.txt -src/pytest_pyresultreport.egg-info/requires.txt -src/pytest_pyresultreport.egg-info/top_level.txt \ No newline at end of file diff --git a/test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/dependency_links.txt b/test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/dependency_links.txt deleted file mode 100644 index 8b1378917..000000000 --- a/test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/entry_points.txt b/test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/entry_points.txt deleted file mode 100644 index 68c6cc24e..000000000 --- a/test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/entry_points.txt +++ /dev/null @@ -1,2 +0,0 @@ -[pytest11] -py_result_report = pytest_pyresultreport.plugin diff --git a/test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/requires.txt b/test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/requires.txt deleted file mode 100644 index b4b839007..000000000 --- a/test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/requires.txt +++ /dev/null @@ -1 +0,0 @@ -pytest>=3.7 diff --git a/test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/top_level.txt b/test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/top_level.txt deleted file mode 100644 index 03af29338..000000000 --- a/test/Microsoft.ComponentDetection.VerificationTests/resources/pip/pytestresultpkg/python/src/pytest_pyresultreport.egg-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -pytest_pyresultreport From 0a7f1d4f754ad4349636e4ac93ca18645fcfc827 Mon Sep 17 00:00:00 2001 From: Paul Dorsch Date: Tue, 6 May 2025 07:41:43 -0700 Subject: [PATCH 06/12] use loop --- .../DependencyGraph/DependencyGraph.cs | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs b/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs index ad777a5fc..52dc9f123 100644 --- a/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs +++ b/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs @@ -152,9 +152,16 @@ public HashSet GetAncestorsAsTypedComponents(string componentId, this.FillTypedComponents(toTypedComponent); } - return this.GetAncestors(componentId) - .Select(c => this.componentNodes[c].TypedComponent) - .ToHashSet(new ComponentComparer()); + var ancestorSet = new HashSet(new ComponentComparer()); + var ancestors = this.GetAncestors(componentId); + foreach (var ancestor in ancestors) + { + var component = this.componentNodes[ancestor]; + component.TypedComponent ??= toTypedComponent(ancestor); + ancestorSet.Add(component.TypedComponent); + } + + return ancestorSet; } public HashSet GetRootsAsTypedComponents(string componentId, Func toTypedComponent) @@ -165,9 +172,16 @@ public HashSet GetRootsAsTypedComponents(string componentId, Fun this.FillTypedComponents(toTypedComponent); } - return this.GetExplicitReferencedDependencyIds(componentId) - .Select(c => this.componentNodes[c].TypedComponent) - .ToHashSet(new ComponentComparer()); + var rootSet = new HashSet(new ComponentComparer()); + var roots = this.GetExplicitReferencedDependencyIds(componentId); + foreach (var root in roots) + { + var component = this.componentNodes[root]; + component.TypedComponent ??= toTypedComponent(root); + rootSet.Add(component.TypedComponent); + } + + return rootSet; } public bool ShouldFillTypedComponents(Func toTypedComponent) From a7305b79b4bdfacde8810be11e902f2cec42cb43 Mon Sep 17 00:00:00 2001 From: Paul Dorsch Date: Tue, 6 May 2025 09:15:57 -0700 Subject: [PATCH 07/12] clarity and testing --- .../Records/DependencyGraphFillRootsRecord.cs | 12 --- .../Records/DependencyGraphRecord.cs | 8 -- ...cs => DependencyGraphTranslationRecord.cs} | 8 +- .../DefaultGraphTranslationService.cs | 54 ++++-------- .../DependencyGraphTests.cs | 84 +++++++++++-------- 5 files changed, 69 insertions(+), 97 deletions(-) delete mode 100644 src/Microsoft.ComponentDetection.Common/Telemetry/Records/DependencyGraphFillRootsRecord.cs delete mode 100644 src/Microsoft.ComponentDetection.Common/Telemetry/Records/DependencyGraphRecord.cs rename src/Microsoft.ComponentDetection.Common/Telemetry/Records/{DependencyGraphInnerRecord.cs => DependencyGraphTranslationRecord.cs} (51%) diff --git a/src/Microsoft.ComponentDetection.Common/Telemetry/Records/DependencyGraphFillRootsRecord.cs b/src/Microsoft.ComponentDetection.Common/Telemetry/Records/DependencyGraphFillRootsRecord.cs deleted file mode 100644 index 5942045dc..000000000 --- a/src/Microsoft.ComponentDetection.Common/Telemetry/Records/DependencyGraphFillRootsRecord.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace Microsoft.ComponentDetection.Common.Telemetry.Records; - -public class DependencyGraphFillRootsRecord : BaseDetectionTelemetryRecord -{ - public override string RecordName => "DependencyGraphFillRootsRecord"; - - public string DetectorId { get; set; } - - public int Count { get; set; } - - public int ComponentCount { get; set; } -} diff --git a/src/Microsoft.ComponentDetection.Common/Telemetry/Records/DependencyGraphRecord.cs b/src/Microsoft.ComponentDetection.Common/Telemetry/Records/DependencyGraphRecord.cs deleted file mode 100644 index 7d2c71f31..000000000 --- a/src/Microsoft.ComponentDetection.Common/Telemetry/Records/DependencyGraphRecord.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Microsoft.ComponentDetection.Common.Telemetry.Records; - -public class DependencyGraphRecord : BaseDetectionTelemetryRecord -{ - public override string RecordName => "DependencyGraphRecord"; - - public string DetectorId { get; set; } -} diff --git a/src/Microsoft.ComponentDetection.Common/Telemetry/Records/DependencyGraphInnerRecord.cs b/src/Microsoft.ComponentDetection.Common/Telemetry/Records/DependencyGraphTranslationRecord.cs similarity index 51% rename from src/Microsoft.ComponentDetection.Common/Telemetry/Records/DependencyGraphInnerRecord.cs rename to src/Microsoft.ComponentDetection.Common/Telemetry/Records/DependencyGraphTranslationRecord.cs index 82f843dad..2a4ddae93 100644 --- a/src/Microsoft.ComponentDetection.Common/Telemetry/Records/DependencyGraphInnerRecord.cs +++ b/src/Microsoft.ComponentDetection.Common/Telemetry/Records/DependencyGraphTranslationRecord.cs @@ -2,16 +2,12 @@ namespace Microsoft.ComponentDetection.Common.Telemetry.Records; using System; -public class DependencyGraphInnerRecord : BaseDetectionTelemetryRecord +public class DependencyGraphTranslationRecord : BaseDetectionTelemetryRecord { - public override string RecordName => "DependencyGraphInnerRecord"; + public override string RecordName => "DependencyGraphTranslationRecord"; public string DetectorId { get; set; } - public string ComponentId { get; set; } - - public int Count { get; set; } - public TimeSpan? TimeToAddRoots { get; set; } public TimeSpan? TimeToAddAncestors { get; set; } diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs index bd3680bd2..149b4e734 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs @@ -84,14 +84,13 @@ private void LogComponentScopeTelemetry(List components) }); } - // TODO: INVESTIGATE PERF private IEnumerable GatherSetOfDetectedComponentsUnmerged(IEnumerable<(IComponentDetector Detector, ComponentRecorder Recorder)> recorderDetectorPairs, DirectoryInfo rootDirectory, bool updateLocations) { return recorderDetectorPairs .Where(recorderDetectorPair => recorderDetectorPair.Recorder != null) .SelectMany(recorderDetectorPair => { - using var record = new DependencyGraphRecord() + using var record = new DependencyGraphTranslationRecord() { DetectorId = recorderDetectorPair.Detector.Id, }; @@ -101,33 +100,13 @@ private IEnumerable GatherSetOfDetectedComponentsUnmerged(IEn var detectedComponents = componentRecorder.GetDetectedComponents(); var dependencyGraphsByLocation = componentRecorder.GetDependencyGraphsByLocation(); - /* - var count = 0; - foreach (var dependendencyGraph in dependencyGraphsByLocation.Values) - { - using var fillRootsRecord = new DependencyGraphFillRootsRecord() - { - DetectorId = recorderDetectorPair.Detector.Id, - Count = count++, - }; - - dependendencyGraph.FillRoots(componentRecorder.GetComponent); - dependendencyGraph.FillTypedComponents(componentRecorder.GetComponent); - } - */ + var totalTimeToAddRoots = TimeSpan.Zero; + var totalTimeToAddAncestors = TimeSpan.Zero; // Note that it looks like we are building up detected components functionally, but they are not immutable -- the code is just written // to look like a pipeline. - var count = 0; foreach (var component in detectedComponents) { - using var innerRecord = new DependencyGraphInnerRecord() - { - DetectorId = recorderDetectorPair.Detector.Id, - ComponentId = component.Component.Id, - Count = count++, - }; - // clone custom locations and make them relative to root. var componentCustomLocations = component.FilePaths ?? []; @@ -142,9 +121,6 @@ private IEnumerable GatherSetOfDetectedComponentsUnmerged(IEn var relevantDependencyGraphs = dependencyGraphsByLocation.Where(x => x.Value.Contains(component.Component.Id)); - var totalTimeToAddRoots = TimeSpan.Zero; - var totalTimeToAddAncestors = TimeSpan.Zero; - // Information about each component is relative to all of the graphs it is present in, so we take all graphs containing a given component and apply the graph data. foreach (var graphKvp in relevantDependencyGraphs) { @@ -152,16 +128,16 @@ private IEnumerable GatherSetOfDetectedComponentsUnmerged(IEn var dependencyGraph = graphKvp.Value; // Calculate roots of the component - var startTime = DateTime.UtcNow; + var rootStartTime = DateTime.UtcNow; this.AddRootsToDetectedComponent(component, dependencyGraph, componentRecorder); - var endTime = DateTime.UtcNow; - totalTimeToAddRoots += endTime - startTime; + var rootEndTime = DateTime.UtcNow; + totalTimeToAddRoots += rootEndTime - rootStartTime; // Calculate Ancestors of the component - var startTime2 = DateTime.UtcNow; + var ancestorStartTime = DateTime.UtcNow; this.AddAncestorsToDetectedComponent(component, dependencyGraph, componentRecorder); - var endTime2 = DateTime.UtcNow; - totalTimeToAddAncestors += endTime2 - startTime2; + var ancestorEndTime = DateTime.UtcNow; + totalTimeToAddAncestors += ancestorEndTime - ancestorStartTime; component.DevelopmentDependency = this.MergeDevDependency(component.DevelopmentDependency, dependencyGraph.IsDevelopmentDependency(component.Component.Id)); component.DependencyScope = DependencyScopeComparer.GetMergedDependencyScope(component.DependencyScope, dependencyGraph.GetDependencyScope(component.Component.Id)); @@ -190,11 +166,11 @@ private IEnumerable GatherSetOfDetectedComponentsUnmerged(IEn } } } - - innerRecord.TimeToAddRoots = totalTimeToAddRoots; - innerRecord.TimeToAddAncestors = totalTimeToAddAncestors; } + record.TimeToAddRoots = totalTimeToAddRoots; + record.TimeToAddAncestors = totalTimeToAddAncestors; + return detectedComponents; }).ToList(); } @@ -266,22 +242,24 @@ private DetectedComponent MergeComponents(IEnumerable enumera private void AddRootsToDetectedComponent(DetectedComponent detectedComponent, IDependencyGraph dependencyGraph, IComponentRecorder componentRecorder) { + detectedComponent.DependencyRoots ??= new HashSet(new ComponentComparer()); if (dependencyGraph == null) { return; } - detectedComponent.DependencyRoots = dependencyGraph.GetRootsAsTypedComponents(detectedComponent.Component.Id, componentRecorder.GetComponent); + detectedComponent.DependencyRoots.UnionWith(dependencyGraph.GetRootsAsTypedComponents(detectedComponent.Component.Id, componentRecorder.GetComponent)); } private void AddAncestorsToDetectedComponent(DetectedComponent detectedComponent, IDependencyGraph dependencyGraph, IComponentRecorder componentRecorder) { + detectedComponent.AncestralDependencyRoots ??= new HashSet(new ComponentComparer()); if (dependencyGraph == null) { return; } - detectedComponent.AncestralDependencyRoots = dependencyGraph.GetAncestorsAsTypedComponents(detectedComponent.Component.Id, componentRecorder.GetComponent); + detectedComponent.AncestralDependencyRoots.UnionWith(dependencyGraph.GetAncestorsAsTypedComponents(detectedComponent.Component.Id, componentRecorder.GetComponent)); } private HashSet MakeFilePathsRelative(ILogger logger, DirectoryInfo rootDirectory, HashSet filePaths) diff --git a/test/Microsoft.ComponentDetection.Common.Tests/DependencyGraphTests.cs b/test/Microsoft.ComponentDetection.Common.Tests/DependencyGraphTests.cs index 278d0b5ba..1d33a16bd 100644 --- a/test/Microsoft.ComponentDetection.Common.Tests/DependencyGraphTests.cs +++ b/test/Microsoft.ComponentDetection.Common.Tests/DependencyGraphTests.cs @@ -1,6 +1,8 @@ namespace Microsoft.ComponentDetection.Common.Tests; using System; +using System.Collections.Generic; +using System.Linq; using FluentAssertions; using Microsoft.ComponentDetection.Common.DependencyGraph; using Microsoft.ComponentDetection.Contracts; @@ -143,7 +145,7 @@ public void GetExplicitReferencedDependencyIds_ComponentsWereAddedSpecifyingRoot [TestMethod] [DataRow(true)] [DataRow(false)] - public void GetExplicitReferencedDependencyIds_ComponentsWereAddedWithoutSpecifyingRoot_RootsAreEmpty(bool shouldPreFillRoots) + public void GetExplicitReferencedDependencyIds_ComponentsWereAddedWithoutSpecifyingRoot_RootsAreEmpty(bool shouldUseTypedComponents) { var componentA = new DependencyGraph.ComponentRefNode { Id = "componentA" }; var componentB = new DependencyGraph.ComponentRefNode { Id = "componentB" }; @@ -151,22 +153,22 @@ public void GetExplicitReferencedDependencyIds_ComponentsWereAddedWithoutSpecify this.dependencyGraph.AddComponent(componentA); this.dependencyGraph.AddComponent(componentB, componentA.Id); - var rootsForComponentA = this.dependencyGraph.GetExplicitReferencedDependencyIds(componentA.Id); + var rootsForComponentA = this.GetExplicitReferencedDependencyIds(componentA.Id, shouldUseTypedComponents); rootsForComponentA.Should().BeEmpty(); - var rootsForComponentB = this.dependencyGraph.GetExplicitReferencedDependencyIds(componentB.Id); + var rootsForComponentB = this.GetExplicitReferencedDependencyIds(componentB.Id, shouldUseTypedComponents); rootsForComponentB.Should().BeEmpty(); } [TestMethod] [DataRow(true)] [DataRow(false)] - public void GetExplicitReferencedDependencyIds_ComponentIsRoot_ARootIsRootOfItSelf(bool shouldPreFillRoots) + public void GetExplicitReferencedDependencyIds_ComponentIsRoot_ARootIsRootOfItSelf(bool shouldUseTypedComponents) { var componentA = new DependencyGraph.ComponentRefNode { Id = "componentA", IsExplicitReferencedDependency = true }; this.dependencyGraph.AddComponent(componentA); - var aRoots = this.dependencyGraph.GetExplicitReferencedDependencyIds(componentA.Id); + var aRoots = this.GetExplicitReferencedDependencyIds(componentA.Id, shouldUseTypedComponents); aRoots.Should().ContainSingle(); aRoots.Should().Contain(componentA.Id); } @@ -174,7 +176,7 @@ public void GetExplicitReferencedDependencyIds_ComponentIsRoot_ARootIsRootOfItSe [TestMethod] [DataRow(true)] [DataRow(false)] - public void GetExplicitReferencedDependencyIds_RootHasParent_ReturnItselfAndItsParents(bool shouldPreFillRoots) + public void GetExplicitReferencedDependencyIds_RootHasParent_ReturnItselfAndItsParents(bool shouldUseTypedComponents) { var componentA = new DependencyGraph.ComponentRefNode { Id = "componentA", IsExplicitReferencedDependency = true }; var componentB = new DependencyGraph.ComponentRefNode { Id = "componentB", IsExplicitReferencedDependency = true }; @@ -184,16 +186,16 @@ public void GetExplicitReferencedDependencyIds_RootHasParent_ReturnItselfAndItsP this.dependencyGraph.AddComponent(componentB, componentA.Id); this.dependencyGraph.AddComponent(componentC, componentB.Id); - var aRoots = this.dependencyGraph.GetExplicitReferencedDependencyIds(componentA.Id); + var aRoots = this.GetExplicitReferencedDependencyIds(componentA.Id, shouldUseTypedComponents); aRoots.Should().ContainSingle(); aRoots.Should().Contain(componentA.Id); - var bRoots = this.dependencyGraph.GetExplicitReferencedDependencyIds(componentB.Id); + var bRoots = this.GetExplicitReferencedDependencyIds(componentB.Id, shouldUseTypedComponents); bRoots.Should().HaveCount(2); bRoots.Should().Contain(componentA.Id); bRoots.Should().Contain(componentB.Id); - var cRoots = this.dependencyGraph.GetExplicitReferencedDependencyIds(componentC.Id); + var cRoots = this.GetExplicitReferencedDependencyIds(componentC.Id, shouldUseTypedComponents); cRoots.Should().HaveCount(3); cRoots.Should().Contain(componentA.Id); cRoots.Should().Contain(componentB.Id); @@ -203,7 +205,7 @@ public void GetExplicitReferencedDependencyIds_RootHasParent_ReturnItselfAndItsP [TestMethod] [DataRow(true)] [DataRow(false)] - public void GetExplicitReferencedDependencyIds_InsertionOrderNotAffectedRoots(bool shouldPreFillRoots) + public void GetExplicitReferencedDependencyIds_InsertionOrderNotAffectedRoots(bool shouldUseTypedComponents) { var componentA = new DependencyGraph.ComponentRefNode { Id = "componentA", IsExplicitReferencedDependency = true }; var componentB = new DependencyGraph.ComponentRefNode { Id = "componentB" }; @@ -217,18 +219,18 @@ public void GetExplicitReferencedDependencyIds_InsertionOrderNotAffectedRoots(bo componentB = new DependencyGraph.ComponentRefNode { Id = "componentB", IsExplicitReferencedDependency = true }; this.dependencyGraph.AddComponent(componentB); - var aRoots = this.dependencyGraph.GetExplicitReferencedDependencyIds(componentA.Id); + var aRoots = this.GetExplicitReferencedDependencyIds(componentA.Id, shouldUseTypedComponents); aRoots.Should().HaveCount(2); aRoots.Should().Contain(componentA.Id); aRoots.Should().Contain(componentC.Id); - var bRoots = this.dependencyGraph.GetExplicitReferencedDependencyIds(componentB.Id); + var bRoots = this.GetExplicitReferencedDependencyIds(componentB.Id, shouldUseTypedComponents); bRoots.Should().HaveCount(3); bRoots.Should().Contain(componentA.Id); bRoots.Should().Contain(componentB.Id); bRoots.Should().Contain(componentC.Id); - var cRoots = this.dependencyGraph.GetExplicitReferencedDependencyIds(componentC.Id); + var cRoots = this.GetExplicitReferencedDependencyIds(componentC.Id, shouldUseTypedComponents); cRoots.Should().ContainSingle(); cRoots.Should().Contain(componentC.Id); } @@ -236,7 +238,7 @@ public void GetExplicitReferencedDependencyIds_InsertionOrderNotAffectedRoots(bo [TestMethod] [DataRow(true)] [DataRow(false)] - public void GetExplicitReferencedDependencyIds_UseManualSelectionTurnedOff_ComponentsWithNoParentsAreSelectedAsExplicitReferencedDependencies(bool shouldPreFillRoots) + public void GetExplicitReferencedDependencyIds_UseManualSelectionTurnedOff_ComponentsWithNoParentsAreSelectedAsExplicitReferencedDependencies(bool shouldUseTypedComponents) { this.dependencyGraph = new DependencyGraph(false); var componentA = new DependencyGraph.ComponentRefNode { Id = "componentA" }; @@ -248,17 +250,17 @@ public void GetExplicitReferencedDependencyIds_UseManualSelectionTurnedOff_Compo this.dependencyGraph.AddComponent(componentC); this.dependencyGraph.AddComponent(componentA, componentC.Id); - var aRoots = this.dependencyGraph.GetExplicitReferencedDependencyIds(componentA.Id); + var aRoots = this.GetExplicitReferencedDependencyIds(componentA.Id, shouldUseTypedComponents); aRoots.Should().ContainSingle(); aRoots.Should().Contain(componentC.Id); ((IDependencyGraph)this.dependencyGraph).IsComponentExplicitlyReferenced(componentA.Id).Should().BeFalse(); - var bRoots = this.dependencyGraph.GetExplicitReferencedDependencyIds(componentB.Id); + var bRoots = this.GetExplicitReferencedDependencyIds(componentB.Id, shouldUseTypedComponents); bRoots.Should().ContainSingle(); bRoots.Should().Contain(componentC.Id); ((IDependencyGraph)this.dependencyGraph).IsComponentExplicitlyReferenced(componentB.Id).Should().BeFalse(); - var cRoots = this.dependencyGraph.GetExplicitReferencedDependencyIds(componentC.Id); + var cRoots = this.GetExplicitReferencedDependencyIds(componentC.Id, shouldUseTypedComponents); cRoots.Should().ContainSingle(); cRoots.Should().Contain(componentC.Id); ((IDependencyGraph)this.dependencyGraph).IsComponentExplicitlyReferenced(componentC.Id).Should().BeTrue(); @@ -267,7 +269,7 @@ public void GetExplicitReferencedDependencyIds_UseManualSelectionTurnedOff_Compo [TestMethod] [DataRow(true)] [DataRow(false)] - public void GetExplicitReferencedDependencyIds_UseManualSelectionTurnedOff_PropertyIsExplicitReferencedDependencyIsIgnored(bool shouldPreFillRoots) + public void GetExplicitReferencedDependencyIds_UseManualSelectionTurnedOff_PropertyIsExplicitReferencedDependencyIsIgnored(bool shouldUseTypedComponents) { this.dependencyGraph = new DependencyGraph(false); var componentA = new DependencyGraph.ComponentRefNode { Id = "componentA", IsExplicitReferencedDependency = true }; @@ -279,17 +281,17 @@ public void GetExplicitReferencedDependencyIds_UseManualSelectionTurnedOff_Prope this.dependencyGraph.AddComponent(componentC); this.dependencyGraph.AddComponent(componentA, componentC.Id); - var aRoots = this.dependencyGraph.GetExplicitReferencedDependencyIds(componentA.Id); + var aRoots = this.GetExplicitReferencedDependencyIds(componentA.Id, shouldUseTypedComponents); aRoots.Should().ContainSingle(); aRoots.Should().Contain(componentC.Id); ((IDependencyGraph)this.dependencyGraph).IsComponentExplicitlyReferenced(componentA.Id).Should().BeFalse(); - var bRoots = this.dependencyGraph.GetExplicitReferencedDependencyIds(componentB.Id); + var bRoots = this.GetExplicitReferencedDependencyIds(componentB.Id, shouldUseTypedComponents); bRoots.Should().ContainSingle(); bRoots.Should().Contain(componentC.Id); ((IDependencyGraph)this.dependencyGraph).IsComponentExplicitlyReferenced(componentB.Id).Should().BeFalse(); - var cRoots = this.dependencyGraph.GetExplicitReferencedDependencyIds(componentC.Id); + var cRoots = this.GetExplicitReferencedDependencyIds(componentC.Id, shouldUseTypedComponents); cRoots.Should().ContainSingle(); cRoots.Should().Contain(componentC.Id); ((IDependencyGraph)this.dependencyGraph).IsComponentExplicitlyReferenced(componentC.Id).Should().BeTrue(); @@ -368,7 +370,9 @@ public void IsDevelopmentDependency_ReturnsAsExpected_AfterMerge() } [TestMethod] - public void GetAncestors_ReturnsAsExpected() + [DataRow(true)] + [DataRow(false)] + public void GetAncestors_ReturnsAsExpected(bool shouldUseTypedComponents) { var componentA = new DependencyGraph.ComponentRefNode { Id = "componentA" }; var componentB = new DependencyGraph.ComponentRefNode { Id = "componentB" }; @@ -378,30 +382,34 @@ public void GetAncestors_ReturnsAsExpected() this.dependencyGraph.AddComponent(componentB, componentA.Id); this.dependencyGraph.AddComponent(componentC, componentB.Id); - var ancestors = this.dependencyGraph.GetAncestors(componentC.Id); + var ancestors = this.GetAncestors(componentC.Id, shouldUseTypedComponents); ancestors.Should().HaveCount(2); ancestors.Should().Contain(componentA.Id); ancestors.Should().Contain(componentB.Id); - ancestors = this.dependencyGraph.GetAncestors(componentB.Id); + ancestors = this.GetAncestors(componentB.Id, shouldUseTypedComponents); ancestors.Should().ContainSingle(); ancestors.Should().Contain(componentA.Id); - ancestors = this.dependencyGraph.GetAncestors(componentA.Id); + ancestors = this.GetAncestors(componentA.Id, shouldUseTypedComponents); ancestors.Should().BeEmpty(); - ancestors = this.dependencyGraph.GetAncestors("test"); + ancestors = this.GetAncestors("test", shouldUseTypedComponents); ancestors.Should().BeEmpty(); } [TestMethod] - public void GetAncestors_Null_ThrowsException() + [DataRow(true)] + [DataRow(false)] + public void GetAncestors_Null_ThrowsException(bool shouldUseTypedComponents) { - this.dependencyGraph.Invoking(d => d.GetAncestors(null)).Should().Throw(); + this.Invoking(d => d.GetAncestors(null, shouldUseTypedComponents)).Should().Throw(); } [TestMethod] - public void GetAncestors_Cyclic_ReturnsAsExpected() + [DataRow(true)] + [DataRow(false)] + public void GetAncestors_Cyclic_ReturnsAsExpected(bool shouldUseTypedComponents) { var root = new DependencyGraph.ComponentRefNode { Id = "root" }; var componentA = new DependencyGraph.ComponentRefNode { Id = "componentA" }; @@ -415,25 +423,35 @@ public void GetAncestors_Cyclic_ReturnsAsExpected() this.dependencyGraph.AddComponent(componentC, componentB.Id); this.dependencyGraph.AddComponent(componentA, componentC.Id); - var ancestors = this.dependencyGraph.GetAncestors(componentC.Id); + var ancestors = this.GetAncestors(componentC.Id, shouldUseTypedComponents); ancestors.Should().HaveCount(3); ancestors.Should().Contain(root.Id); ancestors.Should().Contain(componentA.Id); ancestors.Should().Contain(componentB.Id); - ancestors = this.dependencyGraph.GetAncestors(componentA.Id); + ancestors = this.GetAncestors(componentA.Id, shouldUseTypedComponents); ancestors.Should().HaveCount(3); ancestors.Should().Contain(root.Id); ancestors.Should().Contain(componentB.Id); ancestors.Should().Contain(componentC.Id); - ancestors = this.dependencyGraph.GetAncestors(componentB.Id); + ancestors = this.GetAncestors(componentB.Id, shouldUseTypedComponents); ancestors.Should().HaveCount(3); ancestors.Should().Contain(root.Id); ancestors.Should().Contain(componentC.Id); ancestors.Should().Contain(componentA.Id); - ancestors = this.dependencyGraph.GetAncestors(root.Id); + ancestors = this.GetAncestors(root.Id, shouldUseTypedComponents); ancestors.Should().BeEmpty(); } + + private IEnumerable GetExplicitReferencedDependencyIds(string componentId, bool shouldUseTypedComponents) => + shouldUseTypedComponents + ? this.dependencyGraph.GetRootsAsTypedComponents(componentId, id => new NuGetComponent(id, "1.0.0")).Select(x => ((NuGetComponent)x).Name) + : this.dependencyGraph.GetExplicitReferencedDependencyIds(componentId); + + private ICollection GetAncestors(string componentId, bool shouldUseTypedComponents) => + shouldUseTypedComponents + ? this.dependencyGraph.GetAncestorsAsTypedComponents(componentId, id => new NuGetComponent(id, "1.0.0")).Select(x => ((NuGetComponent)x).Name).ToList() + : this.dependencyGraph.GetAncestors(componentId); } From f537d20e96dceadb312a13d9abb6c5f20532f63a Mon Sep 17 00:00:00 2001 From: Paul Dorsch Date: Tue, 6 May 2025 11:05:57 -0700 Subject: [PATCH 08/12] manually use typed component fill isntead of type logic --- .../DependencyGraph/DependencyGraph.cs | 22 +------------------ .../IComponentRecorder.cs | 18 ++++++++++++++- .../DefaultGraphTranslationService.cs | 5 +++++ .../DependencyGraphTests.cs | 14 ++++++++---- 4 files changed, 33 insertions(+), 26 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs b/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs index 52dc9f123..9c37715be 100644 --- a/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs +++ b/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs @@ -21,8 +21,6 @@ internal class DependencyGraph : IDependencyGraph private readonly ConcurrentDictionary componentNodes; - private Type currentComponentType; - public DependencyGraph(bool enableManualTrackingOfExplicitReferences) { this.componentNodes = new ConcurrentDictionary(); @@ -147,11 +145,6 @@ public ICollection GetAncestors(string componentId) public HashSet GetAncestorsAsTypedComponents(string componentId, Func toTypedComponent) { ArgumentNullException.ThrowIfNull(componentId); - if (this.ShouldFillTypedComponents(toTypedComponent)) - { - this.FillTypedComponents(toTypedComponent); - } - var ancestorSet = new HashSet(new ComponentComparer()); var ancestors = this.GetAncestors(componentId); foreach (var ancestor in ancestors) @@ -167,11 +160,6 @@ public HashSet GetAncestorsAsTypedComponents(string componentId, public HashSet GetRootsAsTypedComponents(string componentId, Func toTypedComponent) { ArgumentNullException.ThrowIfNull(componentId); - if (this.ShouldFillTypedComponents(toTypedComponent)) - { - this.FillTypedComponents(toTypedComponent); - } - var rootSet = new HashSet(new ComponentComparer()); var roots = this.GetExplicitReferencedDependencyIds(componentId); foreach (var root in roots) @@ -184,20 +172,12 @@ public HashSet GetRootsAsTypedComponents(string componentId, Fun return rootSet; } - public bool ShouldFillTypedComponents(Func toTypedComponent) - { - ArgumentNullException.ThrowIfNull(toTypedComponent); - return toTypedComponent.Method?.ReturnType?.GetType() != this.currentComponentType?.GetType(); - } - - private void FillTypedComponents(Func toTypedComponent) + public void FillTypedComponents(Func toTypedComponent) { foreach (var componentId in this.componentNodes.Values) { componentId.TypedComponent = toTypedComponent(componentId.Id); } - - this.currentComponentType = toTypedComponent.Method.ReturnType.GetType(); } IEnumerable IDependencyGraph.GetDependenciesForComponent(string componentId) diff --git a/src/Microsoft.ComponentDetection.Contracts/IComponentRecorder.cs b/src/Microsoft.ComponentDetection.Contracts/IComponentRecorder.cs index 3cea9334e..945a48cad 100644 --- a/src/Microsoft.ComponentDetection.Contracts/IComponentRecorder.cs +++ b/src/Microsoft.ComponentDetection.Contracts/IComponentRecorder.cs @@ -126,9 +126,25 @@ public interface IDependencyGraph /// The componentIds that are ancestors for a given componentId. ICollection GetAncestors(string componentId); + /// + /// Gets the component IDs of all explicitly referenced components, and converts them to a set of typed components. + /// + /// The component to find all roots for. + /// Function that converts the component id to the typed component object. + /// Set of TypedComponents containing the roots. public HashSet GetRootsAsTypedComponents(string componentId, Func toTypedComponent); + /// + /// Gets the component IDs of all ancestors for a given component id, and converts them to a set of typed components. + /// + /// The component to find all roots for. + /// Function that converts the component id to the typed component object. + /// Set of TypedComponents containing the ancestors. public HashSet GetAncestorsAsTypedComponents(string componentId, Func toTypedComponent); - public bool ShouldFillTypedComponents(Func toTypedComponent); + /// + /// This operation pre-fills all nodes with the specified typed component, which improves performance for subsequent runs + /// of and . + /// + public void FillTypedComponents(Func toTypedComponent); } diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs index 149b4e734..80823ff57 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs @@ -100,6 +100,11 @@ private IEnumerable GatherSetOfDetectedComponentsUnmerged(IEn var detectedComponents = componentRecorder.GetDetectedComponents(); var dependencyGraphsByLocation = componentRecorder.GetDependencyGraphsByLocation(); + foreach (var graph in dependencyGraphsByLocation.Values) + { + graph.FillTypedComponents(componentRecorder.GetComponent); + } + var totalTimeToAddRoots = TimeSpan.Zero; var totalTimeToAddAncestors = TimeSpan.Zero; diff --git a/test/Microsoft.ComponentDetection.Common.Tests/DependencyGraphTests.cs b/test/Microsoft.ComponentDetection.Common.Tests/DependencyGraphTests.cs index 1d33a16bd..334a92ff5 100644 --- a/test/Microsoft.ComponentDetection.Common.Tests/DependencyGraphTests.cs +++ b/test/Microsoft.ComponentDetection.Common.Tests/DependencyGraphTests.cs @@ -445,13 +445,19 @@ public void GetAncestors_Cyclic_ReturnsAsExpected(bool shouldUseTypedComponents) ancestors.Should().BeEmpty(); } - private IEnumerable GetExplicitReferencedDependencyIds(string componentId, bool shouldUseTypedComponents) => - shouldUseTypedComponents + private IEnumerable GetExplicitReferencedDependencyIds(string componentId, bool shouldUseTypedComponents) + { + this.dependencyGraph.FillTypedComponents(id => new NuGetComponent(id, "1.0.0")); + return shouldUseTypedComponents ? this.dependencyGraph.GetRootsAsTypedComponents(componentId, id => new NuGetComponent(id, "1.0.0")).Select(x => ((NuGetComponent)x).Name) : this.dependencyGraph.GetExplicitReferencedDependencyIds(componentId); + } - private ICollection GetAncestors(string componentId, bool shouldUseTypedComponents) => - shouldUseTypedComponents + private ICollection GetAncestors(string componentId, bool shouldUseTypedComponents) + { + this.dependencyGraph.FillTypedComponents(id => new NuGetComponent(id, "1.0.0")); + return shouldUseTypedComponents ? this.dependencyGraph.GetAncestorsAsTypedComponents(componentId, id => new NuGetComponent(id, "1.0.0")).Select(x => ((NuGetComponent)x).Name).ToList() : this.dependencyGraph.GetAncestors(componentId); + } } From 3dcf8ec952499363bc199a8afd8d6450035997a8 Mon Sep 17 00:00:00 2001 From: Paul Dorsch Date: Tue, 6 May 2025 11:09:22 -0700 Subject: [PATCH 09/12] update comments --- .../IComponentRecorder.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.ComponentDetection.Contracts/IComponentRecorder.cs b/src/Microsoft.ComponentDetection.Contracts/IComponentRecorder.cs index 945a48cad..a0ee3ecfe 100644 --- a/src/Microsoft.ComponentDetection.Contracts/IComponentRecorder.cs +++ b/src/Microsoft.ComponentDetection.Contracts/IComponentRecorder.cs @@ -128,6 +128,7 @@ public interface IDependencyGraph /// /// Gets the component IDs of all explicitly referenced components, and converts them to a set of typed components. + /// WARNING: Using this method without calling first will result in a performance hit. /// /// The component to find all roots for. /// Function that converts the component id to the typed component object. @@ -136,6 +137,7 @@ public interface IDependencyGraph /// /// Gets the component IDs of all ancestors for a given component id, and converts them to a set of typed components. + /// WARNING: Using this method without calling first will result in a performance hit. /// /// The component to find all roots for. /// Function that converts the component id to the typed component object. @@ -144,7 +146,7 @@ public interface IDependencyGraph /// /// This operation pre-fills all nodes with the specified typed component, which improves performance for subsequent runs - /// of and . + /// of and . /// public void FillTypedComponents(Func toTypedComponent); } From e52705888b9cc9d299073f19bd9b0131323975f4 Mon Sep 17 00:00:00 2001 From: Paul Dorsch Date: Tue, 6 May 2025 11:13:13 -0700 Subject: [PATCH 10/12] styling --- .../DependencyGraph/DependencyGraph.cs | 4 ++-- .../GraphTranslation/DefaultGraphTranslationService.cs | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs b/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs index 9c37715be..0fbf2bea7 100644 --- a/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs +++ b/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs @@ -17,10 +17,10 @@ internal class DependencyGraph : IDependencyGraph { private static readonly CompositeFormat MissingNodeFormat = CompositeFormat.Parse(Resources.MissingNodeInDependencyGraph); - private readonly bool enableManualTrackingOfExplicitReferences; - private readonly ConcurrentDictionary componentNodes; + private readonly bool enableManualTrackingOfExplicitReferences; + public DependencyGraph(bool enableManualTrackingOfExplicitReferences) { this.componentNodes = new ConcurrentDictionary(); diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs index 80823ff57..788b76181 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Services/GraphTranslation/DefaultGraphTranslationService.cs @@ -124,10 +124,8 @@ private IEnumerable GatherSetOfDetectedComponentsUnmerged(IEn } } - var relevantDependencyGraphs = dependencyGraphsByLocation.Where(x => x.Value.Contains(component.Component.Id)); - // Information about each component is relative to all of the graphs it is present in, so we take all graphs containing a given component and apply the graph data. - foreach (var graphKvp in relevantDependencyGraphs) + foreach (var graphKvp in dependencyGraphsByLocation.Where(x => x.Value.Contains(component.Component.Id))) { var location = graphKvp.Key; var dependencyGraph = graphKvp.Value; From f685d44674de7369d2ae509188ab5e4fddaff6bf Mon Sep 17 00:00:00 2001 From: Paul Dorsch Date: Tue, 6 May 2025 11:48:41 -0700 Subject: [PATCH 11/12] linq --- .../DependencyGraph/DependencyGraph.cs | 26 +++++-------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs b/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs index 0fbf2bea7..e54fe30a7 100644 --- a/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs +++ b/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs @@ -145,31 +145,17 @@ public ICollection GetAncestors(string componentId) public HashSet GetAncestorsAsTypedComponents(string componentId, Func toTypedComponent) { ArgumentNullException.ThrowIfNull(componentId); - var ancestorSet = new HashSet(new ComponentComparer()); - var ancestors = this.GetAncestors(componentId); - foreach (var ancestor in ancestors) - { - var component = this.componentNodes[ancestor]; - component.TypedComponent ??= toTypedComponent(ancestor); - ancestorSet.Add(component.TypedComponent); - } - - return ancestorSet; + return this.GetAncestors(componentId) + .Select(a => this.componentNodes[a].TypedComponent ?? toTypedComponent(a)) + .ToHashSet(new ComponentComparer()); } public HashSet GetRootsAsTypedComponents(string componentId, Func toTypedComponent) { ArgumentNullException.ThrowIfNull(componentId); - var rootSet = new HashSet(new ComponentComparer()); - var roots = this.GetExplicitReferencedDependencyIds(componentId); - foreach (var root in roots) - { - var component = this.componentNodes[root]; - component.TypedComponent ??= toTypedComponent(root); - rootSet.Add(component.TypedComponent); - } - - return rootSet; + return this.GetExplicitReferencedDependencyIds(componentId) + .Select(r => this.componentNodes[r].TypedComponent ?? toTypedComponent(r)) + .ToHashSet(new ComponentComparer()); } public void FillTypedComponents(Func toTypedComponent) From 0e5f861e478af37232cfa2770fa0b9acfbca7c29 Mon Sep 17 00:00:00 2001 From: Paul Dorsch Date: Tue, 6 May 2025 12:14:08 -0700 Subject: [PATCH 12/12] confirm compoennt nodes exist --- .../DependencyGraph/DependencyGraph.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs b/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs index e54fe30a7..b5ca144f3 100644 --- a/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs +++ b/src/Microsoft.ComponentDetection.Common/DependencyGraph/DependencyGraph.cs @@ -146,7 +146,9 @@ public HashSet GetAncestorsAsTypedComponents(string componentId, { ArgumentNullException.ThrowIfNull(componentId); return this.GetAncestors(componentId) - .Select(a => this.componentNodes[a].TypedComponent ?? toTypedComponent(a)) + .Select(a => this.componentNodes.TryGetValue(a, out var component) ? component : null) + .Where(a => a != null) + .Select(a => a.TypedComponent ?? toTypedComponent(a.Id)) .ToHashSet(new ComponentComparer()); } @@ -154,7 +156,9 @@ public HashSet GetRootsAsTypedComponents(string componentId, Fun { ArgumentNullException.ThrowIfNull(componentId); return this.GetExplicitReferencedDependencyIds(componentId) - .Select(r => this.componentNodes[r].TypedComponent ?? toTypedComponent(r)) + .Select(r => this.componentNodes.TryGetValue(r, out var component) ? component : null) + .Where(r => r != null) + .Select(r => r.TypedComponent ?? toTypedComponent(r.Id)) .ToHashSet(new ComponentComparer()); }