C#: Support replaces-base via the DependabotProxy.

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependabotProxy.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Immutable;
 using System.Collections.Generic;
 using System.IO;
 using System.Security.Cryptography.X509Certificates;
@@ -14,13 +15,51 @@ public class DependabotProxy : IDependabotProxy
         /// <summary>
         /// Represents configurations for package registries.
         /// </summary>
-        /// <param name="Type">The type of package registry.</param>
-        /// <param name="URL">The URL of the package registry.</param>
-        public record class RegistryConfig(string Type, string URL);
+        public class RegistryConfig
+        {
+            /// <summary>
+            /// The type of the package registry.
+            /// </summary>
+            public string? Type { get; init; }
+
+            /// <summary>
+            /// The URL of the package registry.
+            /// </summary>
+            public string? Url { get; init; }
+
+            /// <summary>
+            /// A boolean indicating whether this registry replaces the base registry.
+            /// </summary>
+            [JsonProperty("replaces-base")]
+            public bool ReplacesBase { get; init; } = false;
+        };
 
         public string Address { get; }
 
-        public HashSet<string> RegistryURLs { get; } = [];
+        /// <summary>
+        /// A dictionary mapping registry URLs to a boolean indicating whether they replace the base registry.
+        /// </summary>
+        private readonly Dictionary<string, bool> registryMapping = [];
+
+        private ImmutableHashSet<string>? registryURLs;
+        /// <summary>
+        /// Gets the set of registry URLs that have been configured as part of the organization-level
+        /// private registry configuration. This includes all registries, regardless of whether they replace
+        /// the default feeds.
+        /// </summary>
+        public ImmutableHashSet<string> RegistryURLs =>
+            registryURLs ??= registryMapping.Keys.ToImmutableHashSet();
+
+        private ImmutableHashSet<string>? registryBaseURLs;
+        /// <summary>
+        /// Gets the set of registry URLs that have been configured as part of the organization-level
+        /// private registry configuration and that replace the default registry. This is a subset of
+        /// <see cref="RegistryURLs"/>.
+        /// If non-empty, the set should be used as a replacement for the default registry during
+        /// package resolution.
+        /// </summary>
+        public ImmutableHashSet<string> RegistryBaseURLs =>
+            registryBaseURLs ??= registryMapping.Where(kvp => kvp.Value).Select(kvp => kvp.Key).ToImmutableHashSet();
 
         public string? CertificatePath { get; private set; }
 
@@ -56,16 +95,28 @@ private DependabotProxy(IDependabotProxyConfiguration config, ILogger logger, Te
                     {
                         foreach (RegistryConfig registry in array)
                         {
+                            if (string.IsNullOrWhiteSpace(registry.Url))
+                            {
+                                logger.LogError("Ignoring registry with empty URL.");
+                                continue;
+                            }
+
+                            if (string.IsNullOrWhiteSpace(registry.Type))
+                            {
+                                logger.LogError($"Ignoring registry at '{registry.Url}' since it has no type.");
+                                continue;
+                            }
+
                             // The array contains all configured private registries, not just ones for C#.
                             // We ignore the non-C# ones here.
                             if (!registry.Type.Equals("nuget_feed"))
                             {
-                                logger.LogDebug($"Ignoring registry at '{registry.URL}' since it is not of type 'nuget_feed'.");
+                                logger.LogDebug($"Ignoring registry at '{registry.Url}' since it is not of type 'nuget_feed'.");
                                 continue;
                             }
 
-                            logger.LogInfo($"Found private registry at '{registry.URL}'");
-                            RegistryURLs.Add(registry.URL);
+                            logger.LogInfo($"Found private registry at '{registry.Url}'");
+                            registryMapping.AddOrUpdateToLatest(registry.Url, registry.ReplacesBase);
                         }
                     }
                 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs

@@ -137,7 +137,7 @@ public bool Exec(List<string> execArgs)
 
         private static readonly IReadOnlyList<string> nugetListSourceCommandArgs = ["nuget", "list", "source", "--format", "Short"];
 
-        public IList<string> GetNugetFeeds(string nugetConfig)
+        public IList<string> GetNugetFeedsFromConfig(string nugetConfig)
         {
             logger.LogInfo($"Getting NuGet feeds from '{nugetConfig}'...");
             return GetResultList([.. nugetListSourceCommandArgs, "--configfile", nugetConfig]);

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs

@@ -56,7 +56,6 @@ internal static class EnvironmentVariableNames
 
         /// <summary>
         /// Specifies the NuGet feeds to use for fallback NuGet dependency fetching. The value is a space-separated list of feed URLs.
-        /// The default value is `https://api.nuget.org/v3/index.json`.
         /// </summary>
         public const string FallbackNugetFeeds = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_FALLBACK";
 

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs

@@ -10,13 +10,17 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 {
     internal sealed partial class FeedManager : IDisposable
     {
-        internal const string PublicNugetOrgFeed = "https://api.nuget.org/v3/index.json";
+        private const string PublicNugetOrg = "nuget.org";
+        private const string PublicDotNugetOrg = $".{PublicNugetOrg}";
+        internal const string PublicApiNugetOrgFeed = $"https://api{PublicDotNugetOrg}/v3/index.json";
 
         private readonly ILogger logger;
         private readonly IDotNet dotnet;
         private readonly IFileProvider fileProvider;
         private readonly DependencyDirectory emptyPackageDirectory;
         private readonly ImmutableHashSet<string> privateRegistryFeeds;
+        private readonly bool hasPrivateRegistryBaseFeeds;
+        private readonly ImmutableHashSet<string> privateRegistryBaseFeeds;
         private readonly IFeedManagerIO feedManagerIo;
 
         /// <summary>
@@ -72,14 +76,33 @@ internal sealed partial class FeedManager : IDisposable
         /// </summary>
         public ImmutableHashSet<string> ReachableFallbackFeeds => lazyReachableFallbackFeeds.Value;
 
+        private readonly Lazy<ImmutableHashSet<string>> lazyReachableDefaultFeeds;
+
+        /// <summary>
+        /// Gets the list of default NuGet feeds that are configured in the environment.
+        /// This is either the public NuGet feed or a set of feeds specified by the environment.
+        /// </summary>
+        public ImmutableHashSet<string> DefaultFeeds { get; init; }
+
+        /// <summary>
+        /// Gets the list of reachable default NuGet feeds.
+        /// </summary>
+        public ImmutableHashSet<string> ReachableDefaultFeeds => lazyReachableDefaultFeeds.Value;
+
         public FeedManager(ILogger logger, IDotNet dotnet, IDependabotProxy? dependabotProxy, IFileProvider fileProvider, IFeedManagerIO feedManagerIo)
         {
             this.logger = logger;
             this.dotnet = dotnet;
             this.fileProvider = fileProvider;
             this.feedManagerIo = feedManagerIo;
-            privateRegistryFeeds = dependabotProxy?.RegistryURLs.ToImmutableHashSet() ?? [];
+            privateRegistryFeeds = dependabotProxy?.RegistryURLs ?? [];
             HasPrivateRegistryFeeds = privateRegistryFeeds.Count > 0;
+            privateRegistryBaseFeeds = dependabotProxy?.RegistryBaseURLs ?? [];
+            hasPrivateRegistryBaseFeeds = privateRegistryBaseFeeds.Count > 0;
+
+            DefaultFeeds = hasPrivateRegistryBaseFeeds
+                ? privateRegistryBaseFeeds
+                : [PublicApiNugetOrgFeed];
             emptyPackageDirectory = new DependencyDirectory("empty", "empty package", logger);
 
             lazyExplicitFeeds = new Lazy<ImmutableHashSet<string>>(GetExplicitFeeds);
@@ -96,13 +119,28 @@ public FeedManager(ILogger logger, IDotNet dotnet, IDependabotProxy? dependabotP
                 var reachableFallbackFeeds = GetReachableFallbackNugetFeeds();
                 return reachableFallbackFeeds.ToImmutableHashSet();
             });
+            lazyReachableDefaultFeeds = new Lazy<ImmutableHashSet<string>>(() => CheckSpecifiedFeeds(DefaultFeeds));
         }
 
         public FeedManager(ILogger logger, IDotNet dotnet, IDependabotProxy? dependabotProxy, IFileProvider fileProvider)
             : this(logger, dotnet, dependabotProxy, fileProvider, new FeedManagerIO(logger, dependabotProxy))
         {
         }
 
+        private bool IsNugetOrgFeed(string url)
+        {
+            try
+            {
+                var uri = new Uri(url);
+                return uri.Host.EndsWith(PublicDotNugetOrg, StringComparison.InvariantCultureIgnoreCase) ||
+                    string.Equals(uri.Host, PublicNugetOrg, StringComparison.InvariantCultureIgnoreCase);
+            }
+            catch (UriFormatException)
+            {
+                return false;
+            }
+        }
+
         private IEnumerable<string> GetFeeds(Func<IList<string>> getNugetFeeds)
         {
             var results = getNugetFeeds();
@@ -124,18 +162,26 @@ private IEnumerable<string> GetFeeds(Func<IList<string>> getNugetFeeds)
                     continue;
                 }
 
-                if (!string.IsNullOrWhiteSpace(url))
+                if (hasPrivateRegistryBaseFeeds && IsNugetOrgFeed(url))
                 {
-                    yield return url;
+                    // Use private registry base feeds.
+                    foreach (var feed in privateRegistryBaseFeeds)
+                    {
+                        logger.LogDebug($"Using private registry base feed '{feed}'.");
+                        yield return feed;
+                    }
+                    continue;
                 }
+
+                yield return url;
             }
         }
 
         private IEnumerable<string> GetFeedsFromFolder(string folderPath) =>
             GetFeeds(() => dotnet.GetNugetFeedsFromFolder(folderPath));
 
         private IEnumerable<string> GetFeedsFromNugetConfig(string nugetConfigPath) =>
-            GetFeeds(() => dotnet.GetNugetFeeds(nugetConfigPath));
+            GetFeeds(() => dotnet.GetNugetFeedsFromConfig(nugetConfigPath));
 
         /// <summary>
         /// Constructs the NuGet sources argument for the restore command based on the given feeds.
@@ -266,22 +312,6 @@ private ImmutableHashSet<string> CheckSpecifiedFeeds(ImmutableHashSet<string> fe
             return reachable.Union(feeds.Where(feed => excludedFeeds.Contains(feed))).ToImmutableHashSet();
         }
 
-        /// <summary>
-        /// Return true if the default NuGet feed is reachable, false otherwise.
-        /// If the reachability check is disabled, this method will always return true.
-        /// </summary>
-        /// <returns>True if the default NuGet feed is reachable, false otherwise.</returns>
-        public bool IsDefaultFeedReachable()
-        {
-            if (CheckNugetFeedResponsiveness)
-            {
-                var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback: false);
-                return feedManagerIo.IsFeedReachable(PublicNugetOrgFeed, initialTimeout, tryCount);
-            }
-
-            return true;
-        }
-
         /// <summary>
         /// Tests which of the feeds given by <paramref name="feedsToCheck"/> are reachable.
         /// </summary>
@@ -315,8 +345,8 @@ private List<string> GetReachableFallbackNugetFeeds()
             var fallbackFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.FallbackNugetFeeds).ToHashSet();
             if (fallbackFeeds.Count == 0)
             {
-                fallbackFeeds.Add(PublicNugetOrgFeed);
-                logger.LogInfo($"No fallback NuGet feeds specified. Adding default feed: {PublicNugetOrgFeed}");
+                fallbackFeeds.UnionWith(DefaultFeeds);
+                logger.LogInfo($"No fallback NuGet feeds specified. Adding default feeds: {string.Join(", ", DefaultFeeds.OrderBy(f => f))}");
 
                 var shouldAddNugetConfigFeeds = EnvironmentVariables.GetBooleanOptOut(EnvironmentVariableNames.AddNugetConfigFeedsToFallback);
                 logger.LogInfo($"Adding feeds from nuget.config to fallback restore: {shouldAddNugetConfigFeeds}");
@@ -329,6 +359,10 @@ private List<string> GetReachableFallbackNugetFeeds()
                     logger.LogInfo($"Using NuGet feeds from nuget.config files as fallback feeds: {string.Join(", ", ExplicitFeeds.OrderBy(f => f))}");
                 }
             }
+            else
+            {
+                logger.LogInfo($"Using fallback NuGet feeds from environment variable '{EnvironmentVariableNames.FallbackNugetFeeds}'.");
+            }
 
             return GetReachableNuGetFeeds(fallbackFeeds, isFallback: true);
         }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDependabotProxy.cs

@@ -1,5 +1,5 @@
 using System;
-using System.Collections.Generic;
+using System.Collections.Immutable;
 using System.Security.Cryptography.X509Certificates;
 
 namespace Semmle.Extraction.CSharp.DependencyFetching
@@ -14,7 +14,12 @@ public interface IDependabotProxy : IDisposable
         /// <summary>
         /// The URLs of package registries that are configured for the proxy.
         /// </summary>
-        HashSet<string> RegistryURLs { get; }
+        ImmutableHashSet<string> RegistryURLs { get; }
+
+        /// <summary>
+        /// The URLs of package registries that replace the base registry.
+        /// </summary>
+        ImmutableHashSet<string> RegistryBaseURLs { get; }
 
         /// <summary>
         /// The path to the temporary file where the certificate is stored.

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs

@@ -13,7 +13,7 @@ public interface IDotNet
         IList<string> GetListedRuntimes();
         IList<string> GetListedSdks();
         bool Exec(List<string> execArgs);
-        IList<string> GetNugetFeeds(string nugetConfig);
+        IList<string> GetNugetFeedsFromConfig(string nugetConfig);
         IList<string> GetNugetFeedsFromFolder(string folderPath);
     }
 

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs

@@ -460,7 +460,7 @@ private bool TryRestorePackageManually(string package, List<string> nugetSources
                 return true;
             }
 
-            if (!feedManager.CheckNugetFeedResponsiveness && res.HasNugetPackageSourceError && nugetSources.Count > 0)
+            if (!feedManager.CheckNugetFeedResponsiveness && !feedManager.HasPrivateRegistryFeeds && res.HasNugetPackageSourceError && nugetSources.Count > 0)
             {
                 logger.LogDebug($"Trying to restore '{package}' without explicitly providing NuGet sources.");
                 // Restore could not be completed because the listed source is unavailable. Try without an explicit restore source argument.

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs

@@ -67,10 +67,6 @@ private class NugetExeWrapper : IPackagesConfigRestore
 
             private bool IsWindows => SystemBuildActions.Instance.IsWindows();
 
-            private bool? isDefaultFeedReachable;
-            private bool IsDefaultFeedReachable =>
-                isDefaultFeedReachable ??= feedManager.IsDefaultFeedReachable();
-
             /// <summary>
             /// Create the package manager for a specified source tree.
             /// </summary>
@@ -169,15 +165,18 @@ private bool TryRestoreNugetPackage(string packagesConfig)
 
                 List<string> sourcesArgument = [];
                 var feedsToUse = feedManager.FeedsToUse(packagesConfig).ToList();
-                var useDefaultFeed = feedsToUse.Count == 0 && IsDefaultFeedReachable;
+                var defaultFeeds = feedManager.CheckNugetFeedResponsiveness
+                    ? feedManager.ReachableDefaultFeeds
+                    : feedManager.DefaultFeeds;
+                var useDefaultFeeds = feedsToUse.Count == 0 && defaultFeeds.Count > 0;
 
                 // Explicitly construct the sources to be used for the restore command when checking feed
-                // responsiveness, using private registries, or falling back to nuget.org.
-                if (feedManager.CheckNugetFeedResponsiveness || feedManager.HasPrivateRegistryFeeds || useDefaultFeed)
+                // responsiveness, using private registries, or falling back to default feeds.
+                if (feedManager.CheckNugetFeedResponsiveness || feedManager.HasPrivateRegistryFeeds || useDefaultFeeds)
                 {
-                    if (useDefaultFeed)
+                    if (useDefaultFeeds)
                     {
-                        feedsToUse.Add(FeedManager.PublicNugetOrgFeed);
+                        feedsToUse.AddRange(defaultFeeds);
                     }
                     var restoreFeeds = feedManager.RestoreFeeds(feedsToUse);
                     sourcesArgument = restoreFeeds.SelectMany<string, string>(feed => ["-Source", feed]).ToList();