C#: Use feed management for more package download flows.

Author: michaelnebel

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs

@@ -142,7 +142,7 @@ private IEnumerable<string> GetFeedsFromFolder(string folderPath) =>
             GetFeeds(() => dotnet.GetNugetFeedsFromFolder(folderPath));
 
 
-        private IEnumerable<string> GetFeedsFromNugetConfig(string nugetConfigPath) =>
+        public IEnumerable<string> GetFeedsFromNugetConfig(string nugetConfigPath) =>
             GetFeeds(() => dotnet.GetNugetFeeds(nugetConfigPath));
 
         public string FeedsToRestoreArgument(IEnumerable<string> feeds, string sourceArgumentPrefix)
@@ -164,6 +164,20 @@ public string FeedsToRestoreArgument(IEnumerable<string> feeds, string sourceArg
             return feedArgs.ToString();
         }
 
+        private IEnumerable<string> FeedsToUseAux(HashSet<string> feedsToConsider)
+        {
+            if (HasPrivateRegistryFeeds)
+            {
+                feedsToConsider.UnionWith(privateRegistryFeeds);
+            }
+
+            var feedsToUse = CheckNugetFeedResponsiveness
+                ? feedsToConsider.Where(ReachableFeeds.Contains)
+                : feedsToConsider;
+
+            return feedsToUse;
+        }
+
         /// <summary>
         /// Constructs the list of NuGet sources to use for this restore.
         /// (1) Use the feeds we get from `dotnet nuget list source`
@@ -177,16 +191,19 @@ public IEnumerable<string> FeedsToUse(string path)
             var folder = GetDirectoryName(path);
             var feedsToConsider = folder is not null ? GetFeedsFromFolder(folder).ToHashSet() : new HashSet<string>();
 
-            if (HasPrivateRegistryFeeds)
-            {
-                feedsToConsider.UnionWith(privateRegistryFeeds);
-            }
+            return FeedsToUseAux(feedsToConsider);
+        }
 
-            var feedsToUse = CheckNugetFeedResponsiveness
-                ? feedsToConsider.Where(ReachableFeeds.Contains)
-                : feedsToConsider;
+        public IEnumerable<string> FeedsToUseFromConfig(string config)
+        {
+            var feedsToConsider = GetFeedsFromNugetConfig(config).ToHashSet();
 
-            return feedsToUse;
+            return FeedsToUseAux(feedsToConsider);
+        }
+
+        public string FeedsToDotnetRestoreArgument(IEnumerable<string> feeds)
+        {
+            return FeedsToRestoreArgument(feeds, "-s");
         }
 
         /// <summary>
@@ -206,7 +223,7 @@ public IEnumerable<string> FeedsToUse(string path)
 
             var feedsToUse = FeedsToUse(path);
 
-            return FeedsToRestoreArgument(feedsToUse, "-s");
+            return FeedsToDotnetRestoreArgument(feedsToUse);
         }
 
         private (int initialTimeout, int tryCount) GetFeedRequestSettings(bool isFallback)

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs

@@ -350,10 +350,19 @@ private void RestoreProjects(IEnumerable<string> projects, out ConcurrentBag<Dep
             }
 
             logger.LogInfo($"Found {notYetDownloadedPackages.Count} packages that are not yet restored");
-            using var tempDir = new TemporaryDirectory(ComputeTempDirectoryPath("nugetconfig"), "generated nuget config", logger);
-            var nugetConfig = fallbackNugetFeeds is null
-                ? GetNugetConfig()
-                : CreateFallbackNugetConfig(fallbackNugetFeeds, tempDir.DirInfo.FullName);
+
+            IEnumerable<string> feeds = [];
+            if (fallbackNugetFeeds is not null)
+            {
+                feeds = fallbackNugetFeeds;
+            }
+            else if (GetNugetConfig() is string config)
+            {
+                var configFeeds = feedManager.GetFeedsFromNugetConfig(config);
+                feeds = feedManager.FeedsToUseFromConfig(config);
+            }
+
+            var nugetSources = feedManager.FeedsToDotnetRestoreArgument(feeds);
 
             compilationInfoContainer.CompilationInfos.Add(("Fallback nuget restore", notYetDownloadedPackages.Count.ToString()));
 
@@ -362,7 +371,7 @@ private void RestoreProjects(IEnumerable<string> projects, out ConcurrentBag<Dep
 
             Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = DependencyManager.Threads }, package =>
             {
-                var success = TryRestorePackageManually(package.Name, nugetConfig, package.PackageReferenceSource, tryWithoutNugetConfig: fallbackNugetFeeds is null);
+                var success = TryRestorePackageManually(package.Name, nugetSources, package.PackageReferenceSource, tryWithoutNugetConfig: fallbackNugetFeeds is null);
                 if (!success)
                 {
                     return;
@@ -379,27 +388,6 @@ private void RestoreProjects(IEnumerable<string> projects, out ConcurrentBag<Dep
             return missingPackageDirectory.DirInfo.FullName;
         }
 
-        private string? CreateFallbackNugetConfig(IEnumerable<string> fallbackNugetFeeds, string folderPath)
-        {
-            var sb = new StringBuilder();
-            fallbackNugetFeeds.ForEach((feed, index) => sb.AppendLine($"<add key=\"feed{index}\" value=\"{feed}\" />"));
-
-            var nugetConfigPath = Path.Join(folderPath, "nuget.config");
-            logger.LogInfo($"Creating fallback nuget.config file {nugetConfigPath}.");
-            File.WriteAllText(nugetConfigPath,
-                $"""
-                <?xml version="1.0" encoding="utf-8"?>
-                <configuration>
-                    <packageSources>
-                        <clear />
-                {sb}
-                    </packageSources>
-                </configuration>
-                """);
-
-            return nugetConfigPath;
-        }
-
         private string? GetNugetConfig()
         {
             var nugetConfigs = fileProvider.NugetConfigs;
@@ -415,6 +403,7 @@ private void RestoreProjects(IEnumerable<string> projects, out ConcurrentBag<Dep
             }
             else
             {
+                // TODO: This could cause non-deterministic behaviror, if the first config file is different between runs.
                 nugetConfig = nugetConfigs.FirstOrDefault();
             }
 
@@ -489,7 +478,7 @@ private static IEnumerable<string> GetRestoredPackageDirectoryNames(DirectoryInf
                 .Select(d => Path.GetFileName(d).ToLowerInvariant());
         }
 
-        private bool TryRestorePackageManually(string package, string? nugetConfig = null, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj,
+        private bool TryRestorePackageManually(string package, string? nugetSources = null, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj,
             bool tryWithoutNugetConfig = true, bool tryPrereleaseVersion = true)
         {
             logger.LogInfo($"Restoring package {package}...");
@@ -512,17 +501,17 @@ private bool TryRestorePackageManually(string package, string? nugetConfig = nul
                 return false;
             }
 
-            var res = TryRestorePackageManually(package, nugetConfig, tempDir, tryPrereleaseVersion);
+            var res = TryRestorePackageManually(package, nugetSources, tempDir, tryPrereleaseVersion);
             if (res.Success)
             {
                 return true;
             }
 
-            if (tryWithoutNugetConfig && res.HasNugetPackageSourceError && nugetConfig is not null)
+            if (tryWithoutNugetConfig && res.HasNugetPackageSourceError && nugetSources is not null && !feedManager.CheckNugetFeedResponsiveness)
             {
                 logger.LogDebug($"Trying to restore '{package}' without nuget.config.");
                 // Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
-                res = TryRestorePackageManually(package, nugetConfig: null, tempDir, tryPrereleaseVersion);
+                res = TryRestorePackageManually(package, nugetSources: null, tempDir, tryPrereleaseVersion);
                 if (res.Success)
                 {
                     return true;
@@ -533,16 +522,16 @@ private bool TryRestorePackageManually(string package, string? nugetConfig = nul
             return false;
         }
 
-        private RestoreResult TryRestorePackageManually(string package, string? nugetConfig, TemporaryDirectory tempDir, bool tryPrereleaseVersion)
+        private RestoreResult TryRestorePackageManually(string package, string? nugetSources, TemporaryDirectory tempDir, bool tryPrereleaseVersion)
         {
-            var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig, ForceReevaluation: true));
+            var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, NugetSources: nugetSources, ForceReevaluation: true));
 
             if (!res.Success && tryPrereleaseVersion && res.HasNugetNoStablePackageVersionError)
             {
                 logger.LogDebug($"Failed to restore nuget package {package} because no stable version was found.");
                 TryChangePackageVersion(tempDir.DirInfo, "*-*");
 
-                res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig, ForceReevaluation: true));
+                res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, NugetSources: nugetSources, ForceReevaluation: true));
                 if (!res.Success)
                 {
                     TryChangePackageVersion(tempDir.DirInfo, "*");