Skip to content

Commit 37d0131

Browse files
jpinzjpinz
authored
Merge branch 'main' into copilot/refactor-docker-reference-to-container-image-refer
2 parents 08e8438 + 52155a6 commit 37d0131

27 files changed

Lines changed: 1008 additions & 280 deletions

‎Directory.Packages.props‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
<PackageVersion Include="Microsoft.Extensions.Logging" Version="8.0.1" />
1818
<PackageVersion Include="Microsoft.SourceLink.GitHub" Version="8.0.0" />
1919
<PackageVersion Include="Microsoft.VisualStudio.Threading.Analyzers" Version="17.14.15" />
20-
<PackageVersion Include="DotNet.Glob" Version="2.1.1" />
20+
<PackageVersion Include="Microsoft.Extensions.FileSystemGlobbing" Version="8.0.0" />
2121
<PackageVersion Include="MinVer" Version="7.0.0" />
2222
<PackageVersion Include="Moq" Version="4.18.4" />
2323
<PackageVersion Include="morelinq" Version="4.4.0" />

‎src/Microsoft.ComponentDetection.Common/DependencyGraph/ComponentRecorder.cs‎

Lines changed: 119 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -34,56 +34,142 @@ public TypedComponent GetComponent(string componentId)
3434

3535
public IEnumerable<DetectedComponent> GetDetectedComponents()
3636
{
37-
IEnumerable<DetectedComponent> detectedComponents;
3837
if (this.singleFileRecorders == null)
3938
{
4039
return [];
4140
}
4241

43-
detectedComponents = this.singleFileRecorders.Values
44-
.SelectMany(singleFileRecorder => singleFileRecorder.GetDetectedComponents().Values)
45-
.GroupBy(x => x.Component.Id)
46-
.Select(grouping =>
47-
{
48-
// We pick a winner here -- any stateful props could get lost at this point.
49-
var winningDetectedComponent = grouping.First();
42+
var allComponents = this.singleFileRecorders.Values
43+
.SelectMany(singleFileRecorder => singleFileRecorder.GetDetectedComponents().Values);
5044

51-
HashSet<string> mergedLicenses = null;
52-
HashSet<ActorInfo> mergedSuppliers = null;
45+
// When both rich and bare entries exist for the same BaseId, rich entries are used as merge targets for bare entries.
46+
var reconciledComponents = new List<DetectedComponent>();
5347

54-
foreach (var component in grouping.Skip(1))
55-
{
56-
winningDetectedComponent.ContainerDetailIds.UnionWith(component.ContainerDetailIds);
48+
foreach (var baseIdGroup in allComponents.GroupBy(x => x.Component.BaseId))
49+
{
50+
var richEntries = new List<DetectedComponent>();
51+
var bareEntries = new List<DetectedComponent>();
5752

58-
// Defensive: merge in case different file recorders set different values for the same component.
59-
if (component.LicensesConcluded != null)
60-
{
61-
mergedLicenses ??= new HashSet<string>(winningDetectedComponent.LicensesConcluded ?? [], StringComparer.OrdinalIgnoreCase);
62-
mergedLicenses.UnionWith(component.LicensesConcluded);
63-
}
53+
// Sub-group by full Id first: merge duplicates of the same Id (existing behavior).
54+
foreach (var idGroup in baseIdGroup.GroupBy(x => x.Component.Id))
55+
{
56+
var merged = MergeDetectedComponentGroup(idGroup);
6457

65-
if (component.Suppliers != null)
66-
{
67-
mergedSuppliers ??= new HashSet<ActorInfo>(winningDetectedComponent.Suppliers ?? []);
68-
mergedSuppliers.UnionWith(component.Suppliers);
69-
}
58+
if (merged.Component.Id == merged.Component.BaseId)
59+
{
60+
bareEntries.Add(merged);
7061
}
71-
72-
if (mergedLicenses != null)
62+
else
7363
{
74-
winningDetectedComponent.LicensesConcluded = mergedLicenses.Where(x => x != null).OrderBy(x => x, StringComparer.OrdinalIgnoreCase).ToList();
64+
richEntries.Add(merged);
7565
}
66+
}
7667

77-
if (mergedSuppliers != null)
68+
if (richEntries.Count > 0 && bareEntries.Count > 0)
69+
{
70+
// Merge each bare entry's metadata into every rich entry, then drop the bare.
71+
foreach (var bare in bareEntries)
7872
{
79-
winningDetectedComponent.Suppliers = mergedSuppliers.Where(s => s != null).OrderBy(s => s.Name).ThenBy(s => s.Type).ToList();
73+
foreach (var rich in richEntries)
74+
{
75+
MergeComponentMetadata(source: bare, target: rich);
76+
}
8077
}
8178

82-
return winningDetectedComponent;
83-
})
84-
.ToArray();
79+
reconciledComponents.AddRange(richEntries);
80+
}
81+
else
82+
{
83+
// No conflict: either all rich (different Ids kept separate) or all bare.
84+
reconciledComponents.AddRange(richEntries);
85+
reconciledComponents.AddRange(bareEntries);
86+
}
87+
}
88+
89+
return reconciledComponents.ToArray();
90+
}
91+
92+
/// <summary>
93+
/// Merges component-level metadata from <paramref name="source"/> into <paramref name="target"/>.
94+
/// </summary>
95+
private static void MergeComponentMetadata(DetectedComponent source, DetectedComponent target)
96+
{
97+
target.ContainerDetailIds.UnionWith(source.ContainerDetailIds);
98+
99+
foreach (var kvp in source.ContainerLayerIds)
100+
{
101+
if (target.ContainerLayerIds.TryGetValue(kvp.Key, out var existingLayers))
102+
{
103+
target.ContainerLayerIds[kvp.Key] = existingLayers.Union(kvp.Value).ToList();
104+
}
105+
else
106+
{
107+
target.ContainerLayerIds[kvp.Key] = kvp.Value.ToList();
108+
}
109+
}
110+
111+
target.LicensesConcluded = MergeAndNormalizeLicenses(target.LicensesConcluded, source.LicensesConcluded);
112+
target.Suppliers = MergeAndNormalizeSuppliers(target.Suppliers, source.Suppliers);
113+
}
114+
115+
private static IList<string> MergeAndNormalizeLicenses(IList<string> target, IList<string> source)
116+
{
117+
if (target == null && source == null)
118+
{
119+
return null;
120+
}
121+
122+
var merged = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
123+
124+
if (target != null)
125+
{
126+
merged.UnionWith(target.Where(x => x != null));
127+
}
128+
129+
if (source != null)
130+
{
131+
merged.UnionWith(source.Where(x => x != null));
132+
}
133+
134+
return merged.OrderBy(x => x, StringComparer.OrdinalIgnoreCase).ToList();
135+
}
136+
137+
private static IList<ActorInfo> MergeAndNormalizeSuppliers(IList<ActorInfo> target, IList<ActorInfo> source)
138+
{
139+
if (target == null && source == null)
140+
{
141+
return null;
142+
}
143+
144+
var merged = new HashSet<ActorInfo>();
145+
146+
if (target != null)
147+
{
148+
merged.UnionWith(target.Where(s => s != null));
149+
}
150+
151+
if (source != null)
152+
{
153+
merged.UnionWith(source.Where(s => s != null));
154+
}
155+
156+
return merged.OrderBy(s => s.Name).ThenBy(s => s.Type).ToList();
157+
}
158+
159+
/// <summary>
160+
/// Merges a group of <see cref="DetectedComponent"/>s that share the same <see cref="TypedComponent.Id"/>
161+
/// into a single entry.
162+
/// </summary>
163+
private static DetectedComponent MergeDetectedComponentGroup(IEnumerable<DetectedComponent> grouping)
164+
{
165+
var winner = grouping.First();
166+
167+
foreach (var component in grouping.Skip(1))
168+
{
169+
MergeComponentMetadata(source: component, target: winner);
170+
}
85171

86-
return detectedComponents;
172+
return winner;
87173
}
88174

89175
public IEnumerable<string> GetSkippedComponents()

‎src/Microsoft.ComponentDetection.Common/DockerService.cs‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -216,8 +216,22 @@ public async Task<ContainerDetails> InspectImageAsync(string image, Cancellation
216216
}
217217
finally
218218
{
219-
// Best-effort container cleanup; RemoveContainerAsync already handles not-found.
220-
await RemoveContainerAsync(container.ID, CancellationToken.None);
219+
// Best-effort container cleanup with a bounded timeout.
220+
// RemoveContainerAsync already handles not-found, but we must guard against
221+
// the Docker daemon hanging on container removal (e.g. when the container
222+
// process is stuck), which would block the detector indefinitely.
223+
using var removeCts = new CancellationTokenSource(TimeSpan.FromSeconds(30));
224+
try
225+
{
226+
await RemoveContainerAsync(container.ID, removeCts.Token);
227+
}
228+
catch (Exception ex)
229+
{
230+
this.logger.LogWarning(
231+
ex,
232+
"Failed to remove container {ContainerId}; abandoning cleanup",
233+
container.ID);
234+
}
221235
}
222236
}
223237

‎src/Microsoft.ComponentDetection.Common/FastDirectoryWalkerFactory.cs‎

Lines changed: 40 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -39,15 +39,12 @@ public IObservable<FileSystemInfo> GetDirectoryScanner(DirectoryInfo root, Concu
3939
return Task.CompletedTask;
4040
}
4141

42-
PatternMatchingUtility.FilePatternMatcher fileIsMatch = null;
42+
PatternMatchingUtility.CompiledMatcher fileIsMatch = null;
43+
var patternsArray = filePatterns?.ToArray();
4344

44-
if (filePatterns == null || !filePatterns.Any())
45+
if (patternsArray is { Length: > 0 })
4546
{
46-
fileIsMatch = span => true;
47-
}
48-
else
49-
{
50-
fileIsMatch = PatternMatchingUtility.GetFilePatternMatcher(filePatterns);
47+
fileIsMatch = PatternMatchingUtility.Compile(patternsArray);
5148
}
5249

5350
var sw = Stopwatch.StartNew();
@@ -100,7 +97,7 @@ public IObservable<FileSystemInfo> GetDirectoryScanner(DirectoryInfo root, Concu
10097
{
10198
ShouldIncludePredicate = (ref FileSystemEntry entry) =>
10299
{
103-
if (!entry.IsDirectory && fileIsMatch(entry.FileName))
100+
if (!entry.IsDirectory && (fileIsMatch == null || fileIsMatch.IsMatch(entry.FileName)))
104101
{
105102
return true;
106103
}
@@ -210,46 +207,26 @@ public void Initialize(DirectoryInfo root, ExcludeDirectoryPredicate directoryEx
210207

211208
public IObservable<FileSystemInfo> Subscribe(DirectoryInfo root, IEnumerable<string> patterns)
212209
{
213-
var patternArray = patterns.ToArray();
214-
215-
if (this.pendingScans.TryGetValue(root, out var scannerObservable))
216-
{
217-
this.logger.LogDebug("Logging patterns {Patterns} for {Root}", string.Join(":", patterns), root.FullName);
218-
219-
var inner = scannerObservable.Value.Where(fsi =>
220-
{
221-
if (fsi is FileInfo fi)
222-
{
223-
return this.MatchesAnyPattern(fi, patternArray);
224-
}
225-
else
226-
{
227-
return true;
228-
}
229-
});
230-
231-
return inner;
232-
}
233-
234-
throw new InvalidOperationException("Subscribe called without initializing scanner");
210+
var patternsArray = patterns as string[] ?? patterns.ToArray();
211+
var compiled = PatternMatchingUtility.Compile(patternsArray);
212+
return this.Subscribe(root, patternsArray, compiled);
235213
}
236214

237215
public IObservable<ProcessRequest> GetFilteredComponentStreamObservable(DirectoryInfo root, IEnumerable<string> patterns, IComponentRecorder componentRecorder)
238216
{
239-
var observable = this.Subscribe(root, patterns).OfType<FileInfo>().SelectMany(f => patterns.Select(sp => new
240-
{
241-
SearchPattern = sp,
242-
File = f,
243-
})).Where(x =>
244-
{
245-
var searchPattern = x.SearchPattern;
246-
var fileName = x.File.Name;
217+
var patternsArray = patterns as string[] ?? patterns.ToArray();
218+
var compiled = PatternMatchingUtility.Compile(patternsArray);
247219

248-
return this.pathUtilityService.MatchesPattern(searchPattern, fileName);
249-
}).Where(x => x.File.Exists)
220+
var observable = this.Subscribe(root, patternsArray, compiled).OfType<FileInfo>()
221+
.Select(f => new
222+
{
223+
File = f,
224+
MatchedPattern = compiled.GetMatchingPattern(f.Name),
225+
})
226+
.Where(x => x.MatchedPattern != null && x.File.Exists)
250227
.Select(x =>
251228
{
252-
var lazyComponentStream = new LazyComponentStream(x.File, x.SearchPattern, this.logger);
229+
var lazyComponentStream = new LazyComponentStream(x.File, x.MatchedPattern, this.logger);
253230
return new ProcessRequest
254231
{
255232
ComponentStream = lazyComponentStream,
@@ -280,14 +257,31 @@ private FileSystemInfo Transform(ref FileSystemEntry entry)
280257
return entry.ToFileSystemInfo();
281258
}
282259

283-
private IObservable<FileSystemInfo> CreateDirectoryWalker(DirectoryInfo di, ExcludeDirectoryPredicate directoryExclusionPredicate, int minimumConnectionCount, IEnumerable<string> filePatterns)
260+
private IObservable<FileSystemInfo> Subscribe(DirectoryInfo root, string[] patterns, PatternMatchingUtility.CompiledMatcher compiled)
284261
{
285-
return this.GetDirectoryScanner(di, new ConcurrentDictionary<string, bool>(), directoryExclusionPredicate, filePatterns, true).Replay() // Returns a replay subject which will republish anything found to new subscribers.
286-
.AutoConnect(minimumConnectionCount); // Specifies that this connectable observable should start when minimumConnectionCount subscribe.
262+
if (this.pendingScans.TryGetValue(root, out var scannerObservable))
263+
{
264+
this.logger.LogDebug("Logging patterns {Patterns} for {Root}", string.Join(":", patterns), root.FullName);
265+
266+
var inner = scannerObservable.Value.Where(fsi =>
267+
{
268+
if (fsi is FileInfo fi)
269+
{
270+
return compiled.IsMatch(fi.Name.AsSpan());
271+
}
272+
273+
return true;
274+
});
275+
276+
return inner;
277+
}
278+
279+
throw new InvalidOperationException("Subscribe called without initializing scanner");
287280
}
288281

289-
private bool MatchesAnyPattern(FileInfo fi, params string[] searchPatterns)
282+
private IObservable<FileSystemInfo> CreateDirectoryWalker(DirectoryInfo di, ExcludeDirectoryPredicate directoryExclusionPredicate, int minimumConnectionCount, IEnumerable<string> filePatterns)
290283
{
291-
return searchPatterns != null && searchPatterns.Any(sp => this.pathUtilityService.MatchesPattern(sp, fi.Name));
284+
return this.GetDirectoryScanner(di, new ConcurrentDictionary<string, bool>(), directoryExclusionPredicate, filePatterns, true).Replay() // Returns a replay subject which will republish anything found to new subscribers.
285+
.AutoConnect(minimumConnectionCount); // Specifies that this connectable observable should start when minimumConnectionCount subscribe.
292286
}
293287
}

‎src/Microsoft.ComponentDetection.Common/PathUtilityService.cs‎

Lines changed: 3 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ namespace Microsoft.ComponentDetection.Common;
33

44
using System;
55
using System.IO;
6-
using System.IO.Enumeration;
76
using Microsoft.ComponentDetection.Contracts;
87
using Microsoft.Extensions.Logging;
98

@@ -22,21 +21,6 @@ internal class PathUtilityService : IPathUtilityService
2221

2322
public PathUtilityService(ILogger<PathUtilityService> logger) => this.logger = logger;
2423

25-
public static bool MatchesPattern(string searchPattern, ref FileSystemEntry fse)
26-
{
27-
if (searchPattern.StartsWith('*') && fse.FileName.EndsWith(searchPattern.AsSpan()[1..], StringComparison.OrdinalIgnoreCase))
28-
{
29-
return true;
30-
}
31-
32-
if (searchPattern.EndsWith('*') && fse.FileName.StartsWith(searchPattern.AsSpan()[..^1], StringComparison.OrdinalIgnoreCase))
33-
{
34-
return true;
35-
}
36-
37-
return fse.FileName.Equals(searchPattern.AsSpan(), StringComparison.OrdinalIgnoreCase);
38-
}
39-
4024
public string GetParentDirectory(string path) => Path.GetDirectoryName(path);
4125

4226
public bool IsFileBelowAnother(string aboveFilePath, string belowFilePath)
@@ -48,21 +32,6 @@ public bool IsFileBelowAnother(string aboveFilePath, string belowFilePath)
4832
return (aboveDirectoryPath.Length != belowDirectoryPath.Length) && belowDirectoryPath.StartsWith(aboveDirectoryPath);
4933
}
5034

51-
public bool MatchesPattern(string searchPattern, string fileName)
52-
{
53-
if (searchPattern.StartsWith('*') && fileName.EndsWith(searchPattern[1..], StringComparison.OrdinalIgnoreCase))
54-
{
55-
return true;
56-
}
57-
58-
if (searchPattern.EndsWith('*') && fileName.StartsWith(searchPattern[..^1], StringComparison.OrdinalIgnoreCase))
59-
{
60-
return true;
61-
}
62-
63-
return searchPattern.Equals(fileName, StringComparison.OrdinalIgnoreCase);
64-
}
65-
6635
public string ResolvePhysicalPath(string path)
6736
{
6837
var directoryInfo = new DirectoryInfo(path);
@@ -75,6 +44,9 @@ public string ResolvePhysicalPath(string path)
7544
return fileInfo.Exists ? this.ResolvePathFromInfo(fileInfo) : null;
7645
}
7746

47+
[Obsolete("Use PatternMatchingUtility.MatchesPattern instead.")]
48+
public bool MatchesPattern(string pattern, string fileName) => PatternMatchingUtility.MatchesPattern(pattern, fileName);
49+
7850
private string ResolvePathFromInfo(FileSystemInfo info) => info.LinkTarget ?? info.FullName;
7951

8052
public string NormalizePath(string path)

0 commit comments

Comments
 (0)