Replace DotNet.Glob with Microsoft.Extensions.FileSystemGlobbing (#1767)

JamieMagee · Copilot · web-flow · commit e05ea39ccfde · 2026-04-07T09:55:25.000-07:00
* Replace DotNet.Glob with Microsoft.Extensions.FileSystemGlobbing Fixes #201. DotNet.Glob v2.1.1 throws IndexOutOfRangeException on ** patterns (e.g. **/samples/**). Replace it with Microsoft.Extensions.FileSystemGlobbing in all three call sites: - DetectorProcessingService: directory exclusion via --DirectoryExclusionList - YarnLockComponentDetector: workspace pattern matching - RustSbomDetector: Cargo workspace include/exclude rules FileSystemGlobbing's ** does not match zero trailing segments, so **/dir/** patterns get a companion **/dir pattern added in the directory exclusion predicate. Paths are normalized to forward slashes before matching, which replaces the DotNet.Glob-specific backslash escaping workaround. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Address PR review feedback - Respect allowWindowsPaths flag: skip patterns containing backslashes when the flag is false, restoring the original behavior where backslash-based patterns don't match on non-Windows platforms - Remove unused rootPath variable in YarnLockComponentDetector - Fix stale XML doc on AddGlobRule (was claiming OS-dependent case sensitivity, but the code always uses OrdinalIgnoreCase) - Add test for trailing ** companion pattern workaround - Fix absolute path handling in directory exclusion predicate by stripping the root prefix before matching Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
diff --git a/Directory.Packages.props b/Directory.Packages.props
@@ -17,7 +17,7 @@
     <PackageVersion Include="Microsoft.Extensions.Logging" Version="8.0.1" />
     <PackageVersion Include="Microsoft.SourceLink.GitHub" Version="8.0.0" />
     <PackageVersion Include="Microsoft.VisualStudio.Threading.Analyzers" Version="17.14.15" />
-    <PackageVersion Include="DotNet.Glob" Version="2.1.1" />
+    <PackageVersion Include="Microsoft.Extensions.FileSystemGlobbing" Version="8.0.0" />
     <PackageVersion Include="MinVer" Version="7.0.0" />
     <PackageVersion Include="Moq" Version="4.18.4" />
     <PackageVersion Include="morelinq" Version="4.4.0" />
diff --git a/src/Microsoft.ComponentDetection.Detectors/Microsoft.ComponentDetection.Detectors.csproj b/src/Microsoft.ComponentDetection.Detectors/Microsoft.ComponentDetection.Detectors.csproj
@@ -9,7 +9,7 @@
     </ItemGroup>
 
     <ItemGroup>
-        <PackageReference Include="DotNet.Glob" />
+        <PackageReference Include="Microsoft.Extensions.FileSystemGlobbing" />
         <PackageReference Include="Microsoft.Extensions.Logging" />
         <PackageReference Include="morelinq" />
         <PackageReference Include="NuGet.ProjectModel" />
diff --git a/src/Microsoft.ComponentDetection.Detectors/rust/RustSbomDetector.cs b/src/Microsoft.ComponentDetection.Detectors/rust/RustSbomDetector.cs
@@ -9,11 +9,11 @@ namespace Microsoft.ComponentDetection.Detectors.Rust;
 using System.Reactive.Threading.Tasks;
 using System.Threading;
 using System.Threading.Tasks;
-using global::DotNet.Globbing;
 using Microsoft.ComponentDetection.Common.Telemetry.Records;
 using Microsoft.ComponentDetection.Contracts;
 using Microsoft.ComponentDetection.Contracts.Internal;
 using Microsoft.ComponentDetection.Contracts.TypedComponent;
+using Microsoft.Extensions.FileSystemGlobbing;
 using Microsoft.Extensions.Logging;
 using Tomlyn;
 using Tomlyn.Model;
@@ -408,18 +408,18 @@ private bool ShouldSkip(string directory, FileKind fileKind, string fullPath)
 
             var relativePath = this.GetRelativePath(rule.Root, normalizedDir);
 
-            // Match against include globs
-            var matchesInclude = rule.IncludeGlobs.Any(g =>
-                g.IsMatch(relativePath) || g.IsMatch(normalizedFullPath));
+            // Match against include globs (try both relative and full path)
+            var matchesInclude = rule.IncludeMatcher.Match(relativePath).HasMatches
+                || rule.IncludeMatcher.Match(normalizedFullPath).HasMatches;
 
             if (!matchesInclude)
             {
                 continue;
             }
 
             // Match against exclude globs
-            var matchesExclude = rule.ExcludeGlobs.Any(g =>
-                g.IsMatch(relativePath) || g.IsMatch(normalizedFullPath));
+            var matchesExclude = rule.ExcludeMatcher.Match(relativePath).HasMatches
+                || rule.ExcludeMatcher.Match(normalizedFullPath).HasMatches;
 
             if (matchesExclude)
             {
@@ -441,7 +441,7 @@ private bool ShouldSkip(string directory, FileKind fileKind, string fullPath)
     /// <param name="excludes">Collection of glob patterns to exclude workspace members (e.g., "examples/*", "tests/*").</param>
     /// <remarks>
     /// This method normalizes all paths and patterns for cross-platform compatibility.
-    /// On Windows, patterns are evaluated case-insensitively, while on other platforms they are case-sensitive.
+    /// Patterns are always evaluated case-insensitively.
     /// The glob rule is used to determine whether files in descendant directories should be skipped during detection.
     /// </remarks>
     private void AddGlobRule(string root, IEnumerable<string> includes, IEnumerable<string> excludes)
@@ -450,35 +450,27 @@ private void AddGlobRule(string root, IEnumerable<string> includes, IEnumerable<
         var includesList = includes?.ToList() ?? [];
         var excludesList = excludes?.ToList() ?? [];
 
-        var globOptions = new GlobOptions
-        {
-            Evaluation = new EvaluationOptions
-            {
-                CaseInsensitive = true,
-            },
-        };
-
-        var includeGlobs = new List<Glob>();
+        var includeMatcher = new Matcher(StringComparison.OrdinalIgnoreCase);
         foreach (var pattern in includesList)
         {
             var normalizedPattern = this.pathUtilityService.NormalizePath(pattern);
-            includeGlobs.Add(Glob.Parse(normalizedPattern, globOptions));
+            includeMatcher.AddInclude(normalizedPattern);
         }
 
-        var excludeGlobs = new List<Glob>();
+        var excludeMatcher = new Matcher(StringComparison.OrdinalIgnoreCase);
         foreach (var pattern in excludesList)
         {
             var normalizedPattern = this.pathUtilityService.NormalizePath(pattern);
-            excludeGlobs.Add(Glob.Parse(normalizedPattern, globOptions));
+            excludeMatcher.AddInclude(normalizedPattern);
         }
 
         var rule = new GlobRule
         {
             Root = normalizedRoot,
             Includes = includesList,
             Excludes = excludesList,
-            IncludeGlobs = includeGlobs,
-            ExcludeGlobs = excludeGlobs,
+            IncludeMatcher = includeMatcher,
+            ExcludeMatcher = excludeMatcher,
         };
 
         this.visitedGlobRules.Add(rule);
@@ -774,8 +766,8 @@ private class GlobRule
 
         public List<string> Excludes { get; set; }
 
-        public List<Glob> IncludeGlobs { get; set; }
+        public Matcher IncludeMatcher { get; set; }
 
-        public List<Glob> ExcludeGlobs { get; set; }
+        public Matcher ExcludeMatcher { get; set; }
     }
 }
diff --git a/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/yarn/YarnLockComponentDetector.cs
@@ -8,11 +8,11 @@ namespace Microsoft.ComponentDetection.Detectors.Yarn;
 using System.Runtime.InteropServices;
 using System.Threading;
 using System.Threading.Tasks;
-using global::DotNet.Globbing;
 using Microsoft.ComponentDetection.Contracts;
 using Microsoft.ComponentDetection.Contracts.Internal;
 using Microsoft.ComponentDetection.Contracts.TypedComponent;
 using Microsoft.ComponentDetection.Detectors.Npm;
+using Microsoft.Extensions.FileSystemGlobbing;
 using Microsoft.Extensions.Logging;
 
 public class YarnLockComponentDetector : FileComponentDetector
@@ -259,21 +259,24 @@ private bool TryReadPeerPackageJsonRequestsAsYarnEntries(ISingleFileComponentRec
 
     private void GetWorkspaceDependencies(IList<string> yarnWorkspaces, DirectoryInfo root, IDictionary<string, IDictionary<string, bool>> dependencies, IDictionary<string, string> workspaceDependencyVsLocationMap)
     {
-        var ignoreCase = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
-
-        var globOptions = new GlobOptions()
-        {
-            Evaluation = new EvaluationOptions()
-            {
-                CaseInsensitive = ignoreCase,
-            },
-        };
+        var comparison = RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
+            ? StringComparison.OrdinalIgnoreCase
+            : StringComparison.Ordinal;
 
         foreach (var workspacePattern in yarnWorkspaces)
         {
-            var glob = Glob.Parse($"{root.FullName.Replace('\\', '/')}/{workspacePattern}/package.json", globOptions);
+            var matcher = new Matcher(comparison);
+            matcher.AddInclude($"{workspacePattern}/package.json");
 
-            var componentStreams = this.ComponentStreamEnumerableFactory.GetComponentStreams(root, (file) => glob.IsMatch(file.FullName.Replace('\\', '/')), null, true);
+            var componentStreams = this.ComponentStreamEnumerableFactory.GetComponentStreams(
+                root,
+                (file) =>
+                {
+                    var relativePath = Path.GetRelativePath(root.FullName, file.FullName).Replace('\\', '/');
+                    return matcher.Match(relativePath).HasMatches;
+                },
+                null,
+                true);
 
             foreach (var stream in componentStreams)
             {
diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Microsoft.ComponentDetection.Orchestrator.csproj b/src/Microsoft.ComponentDetection.Orchestrator/Microsoft.ComponentDetection.Orchestrator.csproj
@@ -9,7 +9,7 @@
 
     <ItemGroup>
         <PackageReference Include="CommandLineParser" />
-        <PackageReference Include="DotNet.Glob" />
+        <PackageReference Include="Microsoft.Extensions.FileSystemGlobbing" />
         <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" />
         <PackageReference Include="Microsoft.Extensions.Logging" />
         <PackageReference Include="Newtonsoft.Json" />
diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs b/src/Microsoft.ComponentDetection.Orchestrator/Services/DetectorProcessingService.cs
@@ -10,14 +10,14 @@ namespace Microsoft.ComponentDetection.Orchestrator.Services;
 using System.Text.Json;
 using System.Threading;
 using System.Threading.Tasks;
-using DotNet.Globbing;
 using Microsoft.ComponentDetection.Common;
 using Microsoft.ComponentDetection.Common.DependencyGraph;
 using Microsoft.ComponentDetection.Common.Telemetry.Records;
 using Microsoft.ComponentDetection.Contracts;
 using Microsoft.ComponentDetection.Contracts.BcdeModels;
 using Microsoft.ComponentDetection.Orchestrator.Commands;
 using Microsoft.ComponentDetection.Orchestrator.Experiments;
+using Microsoft.Extensions.FileSystemGlobbing;
 using Microsoft.Extensions.Logging;
 using Spectre.Console;
 using static System.Environment;
@@ -249,35 +249,49 @@ public ExcludeDirectoryPredicate GenerateDirectoryExclusionPredicate(string orig
             };
         }
 
-        var minimatchers = new Dictionary<string, Glob>();
+        var comparison = ignoreCase ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal;
+        var matcher = new Matcher(comparison);
 
-        var globOptions = new GlobOptions()
+        foreach (var directoryExclusion in directoryExclusionList)
         {
-            Evaluation = new EvaluationOptions()
+            if (!allowWindowsPaths && directoryExclusion.Contains('\\'))
             {
-                CaseInsensitive = ignoreCase,
-            },
-        };
+                this.logger.LogDebug("Skipping directory exclusion pattern {Pattern} because it contains backslashes and Windows-style paths are not enabled.", directoryExclusion);
+                continue;
+            }
 
-        foreach (var directoryExclusion in directoryExclusionList)
-        {
-            minimatchers.Add(directoryExclusion, Glob.Parse(allowWindowsPaths ? directoryExclusion : /* [] escapes special chars */ directoryExclusion.Replace("\\", "[\\]"), globOptions));
+            var pattern = directoryExclusion.Replace('\\', '/');
+            matcher.AddInclude(pattern);
+
+            // FileSystemGlobbing's ** does not match zero trailing segments,
+            // so **/dir/** won't match "dir" itself. Add **/dir to cover that case.
+            if (pattern.EndsWith("/**"))
+            {
+                matcher.AddInclude(pattern[..^3]);
+            }
         }
 
         return (name, directoryName) =>
         {
-            var path = Path.Combine(directoryName.ToString(), name.ToString());
+            var path = Path.Combine(directoryName.ToString(), name.ToString()).Replace('\\', '/');
 
-            return minimatchers.Any(minimatcherKeyValue =>
+            // FileSystemGlobbing requires relative paths for matching.
+            // Strip the leading slash (or drive letter on Windows) so that
+            // patterns like **/dir/** can match against the full directory path.
+            var relativePath = path.StartsWith('/') ? path[1..] : path;
+            if (relativePath.Length > 1 && relativePath[1] == ':')
             {
-                if (minimatcherKeyValue.Value.IsMatch(path))
-                {
-                    this.logger.LogDebug("Excluding folder {Path} because it matched glob {Glob}.", path, minimatcherKeyValue.Key);
-                    return true;
-                }
+                // Windows drive letter, e.g. "C:/foo" → "foo"
+                relativePath = relativePath[3..];
+            }
 
-                return false;
-            });
+            if (matcher.Match(relativePath).HasMatches)
+            {
+                this.logger.LogDebug("Excluding folder {Path} because it matched a directory exclusion glob.", path);
+                return true;
+            }
+
+            return false;
         };
     }
 
diff --git a/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs b/test/Microsoft.ComponentDetection.Orchestrator.Tests/Services/DetectorProcessingServiceTests.cs
@@ -419,6 +419,38 @@ public void GenerateDirectoryExclusionPredicate_IgnoreCaseAndAllowWindowsPathsWo
         exclusionPredicate(dn, dp).Should().BeTrue();
     }
 
+    [TestMethod]
+    public void GenerateDirectoryExclusionPredicate_TrailingDoubleStarMatchesDirectoryItself()
+    {
+        // FileSystemGlobbing's ** does not match zero trailing segments,
+        // so the implementation adds a companion pattern (**/dir) alongside **/dir/**.
+        // This test verifies that the directory itself is excluded, not just its children.
+        var args = new ScanSettings
+        {
+            SourceDirectory = new DirectoryInfo(this.isWin ? @"C:\project" : "/tmp/project"),
+            DetectorArgs = new Dictionary<string, string>(),
+            DirectoryExclusionList = ["**/Source/**"],
+        };
+
+        var exclusionPredicate = this.serviceUnderTest.GenerateDirectoryExclusionPredicate(
+            args.SourceDirectory.FullName,
+            args.DirectoryExclusionList,
+            args.DirectoryExclusionListObsolete,
+            allowWindowsPaths: true,
+            ignoreCase: true);
+
+        // The directory itself (no trailing segment) should be excluded
+        var projectPath = this.isWin ? @"C:\project" : "/tmp/project";
+        exclusionPredicate("Source", projectPath).Should().BeTrue();
+
+        // A child under the directory should also be excluded
+        var sourcePath = this.isWin ? @"C:\project\Source" : "/tmp/project/Source";
+        exclusionPredicate("child", sourcePath).Should().BeTrue();
+
+        // An unrelated directory should not be excluded
+        exclusionPredicate("Other", projectPath).Should().BeFalse();
+    }
+
     [TestMethod]
     public async Task ProcessDetectorsAsync_DirectoryExclusionPredicateWorksAsExpectedForObsoleteAsync()
     {
