diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependabotProxy.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependabotProxy.cs index 3bf843d3fa2c..8229a3da1375 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependabotProxy.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependabotProxy.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Immutable; using System.Collections.Generic; using System.IO; using System.Security.Cryptography.X509Certificates; @@ -14,13 +15,51 @@ public class DependabotProxy : IDependabotProxy /// /// Represents configurations for package registries. /// - /// The type of package registry. - /// The URL of the package registry. - public record class RegistryConfig(string Type, string URL); + public class RegistryConfig + { + /// + /// The type of the package registry. + /// + public string? Type { get; init; } + + /// + /// The URL of the package registry. + /// + public string? Url { get; init; } + + /// + /// A boolean indicating whether this registry replaces the base registry. + /// + [JsonProperty("replaces-base")] + public bool ReplacesBase { get; init; } = false; + }; public string Address { get; } - public HashSet RegistryURLs { get; } = []; + /// + /// A dictionary mapping registry URLs to a boolean indicating whether they replace the base registry. + /// + private readonly Dictionary registryMapping = []; + + private ImmutableHashSet? registryURLs; + /// + /// Gets the set of registry URLs that have been configured as part of the organization-level + /// private registry configuration. This includes all registries, regardless of whether they replace + /// the default feeds. + /// + public ImmutableHashSet RegistryURLs => + registryURLs ??= registryMapping.Keys.ToImmutableHashSet(); + + private ImmutableHashSet? registryBaseURLs; + /// + /// Gets the set of registry URLs that have been configured as part of the organization-level + /// private registry configuration and that replace the default registry. This is a subset of + /// . + /// If non-empty, the set should be used as a replacement for the default registry during + /// package resolution. + /// + public ImmutableHashSet RegistryBaseURLs => + registryBaseURLs ??= registryMapping.Where(kvp => kvp.Value).Select(kvp => kvp.Key).ToImmutableHashSet(); public string? CertificatePath { get; private set; } @@ -56,16 +95,28 @@ private DependabotProxy(IDependabotProxyConfiguration config, ILogger logger, Te { foreach (RegistryConfig registry in array) { + if (string.IsNullOrWhiteSpace(registry.Url)) + { + logger.LogError("Ignoring registry with empty URL."); + continue; + } + + if (string.IsNullOrWhiteSpace(registry.Type)) + { + logger.LogError($"Ignoring registry at '{registry.Url}' since it has no type."); + continue; + } + // The array contains all configured private registries, not just ones for C#. // We ignore the non-C# ones here. if (!registry.Type.Equals("nuget_feed")) { - logger.LogDebug($"Ignoring registry at '{registry.URL}' since it is not of type 'nuget_feed'."); + logger.LogDebug($"Ignoring registry at '{registry.Url}' since it is not of type 'nuget_feed'."); continue; } - logger.LogInfo($"Found private registry at '{registry.URL}'"); - RegistryURLs.Add(registry.URL); + logger.LogInfo($"Found private registry at '{registry.Url}'"); + registryMapping.AddOrUpdateToLatest(registry.Url, registry.ReplacesBase); } } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs index 9958fbce4e71..340d86eb3914 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs @@ -137,7 +137,7 @@ public bool Exec(List execArgs) private static readonly IReadOnlyList nugetListSourceCommandArgs = ["nuget", "list", "source", "--format", "Short"]; - public IList GetNugetFeeds(string nugetConfig) + public IList GetNugetFeedsFromConfig(string nugetConfig) { logger.LogInfo($"Getting NuGet feeds from '{nugetConfig}'..."); return GetResultList([.. nugetListSourceCommandArgs, "--configfile", nugetConfig]); diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs index b1134ad21e24..94a87037cdbc 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs @@ -56,7 +56,6 @@ internal static class EnvironmentVariableNames /// /// Specifies the NuGet feeds to use for fallback NuGet dependency fetching. The value is a space-separated list of feed URLs. - /// The default value is `https://api.nuget.org/v3/index.json`. /// public const string FallbackNugetFeeds = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_FALLBACK"; diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs index 6c4593f3400c..aeca9c6b2a13 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs @@ -10,13 +10,17 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { internal sealed partial class FeedManager : IDisposable { - internal const string PublicNugetOrgFeed = "https://api.nuget.org/v3/index.json"; + private const string PublicNugetOrg = "nuget.org"; + private const string PublicDotNugetOrg = $".{PublicNugetOrg}"; + internal const string PublicApiNugetOrgFeed = $"https://api{PublicDotNugetOrg}/v3/index.json"; private readonly ILogger logger; private readonly IDotNet dotnet; private readonly IFileProvider fileProvider; private readonly DependencyDirectory emptyPackageDirectory; private readonly ImmutableHashSet privateRegistryFeeds; + private readonly bool hasPrivateRegistryBaseFeeds; + private readonly ImmutableHashSet privateRegistryBaseFeeds; private readonly IFeedManagerIO feedManagerIo; /// @@ -72,14 +76,33 @@ internal sealed partial class FeedManager : IDisposable /// public ImmutableHashSet ReachableFallbackFeeds => lazyReachableFallbackFeeds.Value; + private readonly Lazy> lazyReachableDefaultFeeds; + + /// + /// Gets the list of default NuGet feeds that are configured in the environment. + /// This is either the public NuGet feed or a set of feeds specified by the environment. + /// + public ImmutableHashSet DefaultFeeds { get; init; } + + /// + /// Gets the list of reachable default NuGet feeds. + /// + public ImmutableHashSet ReachableDefaultFeeds => lazyReachableDefaultFeeds.Value; + public FeedManager(ILogger logger, IDotNet dotnet, IDependabotProxy? dependabotProxy, IFileProvider fileProvider, IFeedManagerIO feedManagerIo) { this.logger = logger; this.dotnet = dotnet; this.fileProvider = fileProvider; this.feedManagerIo = feedManagerIo; - privateRegistryFeeds = dependabotProxy?.RegistryURLs.ToImmutableHashSet() ?? []; + privateRegistryFeeds = dependabotProxy?.RegistryURLs ?? []; HasPrivateRegistryFeeds = privateRegistryFeeds.Count > 0; + privateRegistryBaseFeeds = dependabotProxy?.RegistryBaseURLs ?? []; + hasPrivateRegistryBaseFeeds = privateRegistryBaseFeeds.Count > 0; + + DefaultFeeds = hasPrivateRegistryBaseFeeds + ? privateRegistryBaseFeeds + : [PublicApiNugetOrgFeed]; emptyPackageDirectory = new DependencyDirectory("empty", "empty package", logger); lazyExplicitFeeds = new Lazy>(GetExplicitFeeds); @@ -96,6 +119,7 @@ public FeedManager(ILogger logger, IDotNet dotnet, IDependabotProxy? dependabotP var reachableFallbackFeeds = GetReachableFallbackNugetFeeds(); return reachableFallbackFeeds.ToImmutableHashSet(); }); + lazyReachableDefaultFeeds = new Lazy>(() => CheckSpecifiedFeeds(DefaultFeeds)); } public FeedManager(ILogger logger, IDotNet dotnet, IDependabotProxy? dependabotProxy, IFileProvider fileProvider) @@ -103,6 +127,20 @@ public FeedManager(ILogger logger, IDotNet dotnet, IDependabotProxy? dependabotP { } + private bool IsNugetOrgFeed(string url) + { + try + { + var uri = new Uri(url); + return uri.Host.EndsWith(PublicDotNugetOrg, StringComparison.InvariantCultureIgnoreCase) || + string.Equals(uri.Host, PublicNugetOrg, StringComparison.InvariantCultureIgnoreCase); + } + catch (UriFormatException) + { + return false; + } + } + private IEnumerable GetFeeds(Func> getNugetFeeds) { var results = getNugetFeeds(); @@ -124,10 +162,18 @@ private IEnumerable GetFeeds(Func> getNugetFeeds) continue; } - if (!string.IsNullOrWhiteSpace(url)) + if (hasPrivateRegistryBaseFeeds && IsNugetOrgFeed(url)) { - yield return url; + // Use private registry base feeds. + foreach (var feed in privateRegistryBaseFeeds) + { + logger.LogDebug($"Using private registry base feed '{feed}'."); + yield return feed; + } + continue; } + + yield return url; } } @@ -135,7 +181,7 @@ private IEnumerable GetFeedsFromFolder(string folderPath) => GetFeeds(() => dotnet.GetNugetFeedsFromFolder(folderPath)); private IEnumerable GetFeedsFromNugetConfig(string nugetConfigPath) => - GetFeeds(() => dotnet.GetNugetFeeds(nugetConfigPath)); + GetFeeds(() => dotnet.GetNugetFeedsFromConfig(nugetConfigPath)); /// /// Constructs the NuGet sources argument for the restore command based on the given feeds. @@ -266,22 +312,6 @@ private ImmutableHashSet CheckSpecifiedFeeds(ImmutableHashSet fe return reachable.Union(feeds.Where(feed => excludedFeeds.Contains(feed))).ToImmutableHashSet(); } - /// - /// Return true if the default NuGet feed is reachable, false otherwise. - /// If the reachability check is disabled, this method will always return true. - /// - /// True if the default NuGet feed is reachable, false otherwise. - public bool IsDefaultFeedReachable() - { - if (CheckNugetFeedResponsiveness) - { - var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback: false); - return feedManagerIo.IsFeedReachable(PublicNugetOrgFeed, initialTimeout, tryCount); - } - - return true; - } - /// /// Tests which of the feeds given by are reachable. /// @@ -315,8 +345,8 @@ private List GetReachableFallbackNugetFeeds() var fallbackFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.FallbackNugetFeeds).ToHashSet(); if (fallbackFeeds.Count == 0) { - fallbackFeeds.Add(PublicNugetOrgFeed); - logger.LogInfo($"No fallback NuGet feeds specified. Adding default feed: {PublicNugetOrgFeed}"); + fallbackFeeds.UnionWith(DefaultFeeds); + logger.LogInfo($"No fallback NuGet feeds specified. Adding default feeds: {string.Join(", ", DefaultFeeds.OrderBy(f => f))}"); var shouldAddNugetConfigFeeds = EnvironmentVariables.GetBooleanOptOut(EnvironmentVariableNames.AddNugetConfigFeedsToFallback); logger.LogInfo($"Adding feeds from nuget.config to fallback restore: {shouldAddNugetConfigFeeds}"); @@ -329,6 +359,10 @@ private List GetReachableFallbackNugetFeeds() logger.LogInfo($"Using NuGet feeds from nuget.config files as fallback feeds: {string.Join(", ", ExplicitFeeds.OrderBy(f => f))}"); } } + else + { + logger.LogInfo($"Using fallback NuGet feeds from environment variable '{EnvironmentVariableNames.FallbackNugetFeeds}'."); + } return GetReachableNuGetFeeds(fallbackFeeds, isFallback: true); } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDependabotProxy.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDependabotProxy.cs index 37a11900fddf..aafaf851e356 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDependabotProxy.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDependabotProxy.cs @@ -1,5 +1,5 @@ using System; -using System.Collections.Generic; +using System.Collections.Immutable; using System.Security.Cryptography.X509Certificates; namespace Semmle.Extraction.CSharp.DependencyFetching @@ -14,7 +14,12 @@ public interface IDependabotProxy : IDisposable /// /// The URLs of package registries that are configured for the proxy. /// - HashSet RegistryURLs { get; } + ImmutableHashSet RegistryURLs { get; } + + /// + /// The URLs of package registries that replace the base registry. + /// + ImmutableHashSet RegistryBaseURLs { get; } /// /// The path to the temporary file where the certificate is stored. diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs index 0e93fa92813a..06186f1a28d3 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs @@ -13,7 +13,7 @@ public interface IDotNet IList GetListedRuntimes(); IList GetListedSdks(); bool Exec(List execArgs); - IList GetNugetFeeds(string nugetConfig); + IList GetNugetFeedsFromConfig(string nugetConfig); IList GetNugetFeedsFromFolder(string folderPath); } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs index 85d6056d7218..f62105f2b482 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs @@ -460,7 +460,7 @@ private bool TryRestorePackageManually(string package, List nugetSources return true; } - if (!feedManager.CheckNugetFeedResponsiveness && res.HasNugetPackageSourceError && nugetSources.Count > 0) + if (!feedManager.CheckNugetFeedResponsiveness && !feedManager.HasPrivateRegistryFeeds && res.HasNugetPackageSourceError && nugetSources.Count > 0) { logger.LogDebug($"Trying to restore '{package}' without explicitly providing NuGet sources."); // Restore could not be completed because the listed source is unavailable. Try without an explicit restore source argument. diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs index d4403bb955ef..861622ca4c02 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs @@ -67,10 +67,6 @@ private class NugetExeWrapper : IPackagesConfigRestore private bool IsWindows => SystemBuildActions.Instance.IsWindows(); - private bool? isDefaultFeedReachable; - private bool IsDefaultFeedReachable => - isDefaultFeedReachable ??= feedManager.IsDefaultFeedReachable(); - /// /// Create the package manager for a specified source tree. /// @@ -169,15 +165,18 @@ private bool TryRestoreNugetPackage(string packagesConfig) List sourcesArgument = []; var feedsToUse = feedManager.FeedsToUse(packagesConfig).ToList(); - var useDefaultFeed = feedsToUse.Count == 0 && IsDefaultFeedReachable; + var defaultFeeds = feedManager.CheckNugetFeedResponsiveness + ? feedManager.ReachableDefaultFeeds + : feedManager.DefaultFeeds; + var useDefaultFeeds = feedsToUse.Count == 0 && defaultFeeds.Count > 0; // Explicitly construct the sources to be used for the restore command when checking feed - // responsiveness, using private registries, or falling back to nuget.org. - if (feedManager.CheckNugetFeedResponsiveness || feedManager.HasPrivateRegistryFeeds || useDefaultFeed) + // responsiveness, using private registries, or falling back to default feeds. + if (feedManager.CheckNugetFeedResponsiveness || feedManager.HasPrivateRegistryFeeds || useDefaultFeeds) { - if (useDefaultFeed) + if (useDefaultFeeds) { - feedsToUse.Add(FeedManager.PublicNugetOrgFeed); + feedsToUse.AddRange(defaultFeeds); } var restoreFeeds = feedManager.RestoreFeeds(feedsToUse); sourcesArgument = restoreFeeds.SelectMany(feed => ["-Source", feed]).ToList(); diff --git a/csharp/extractor/Semmle.Extraction.Tests/DependabotProxy.cs b/csharp/extractor/Semmle.Extraction.Tests/DependabotProxy.cs index 9c8c762f5989..71c3943fe8fe 100644 --- a/csharp/extractor/Semmle.Extraction.Tests/DependabotProxy.cs +++ b/csharp/extractor/Semmle.Extraction.Tests/DependabotProxy.cs @@ -28,8 +28,12 @@ private static TemporaryDirectory MakeTemporaryDirectory() return new TemporaryDirectory(tmp, "testing", new LoggerStub()); } + /// + /// The purpose of this test is to verify that the registry proxy correctly handles the case where the port is not specified. + /// In this case, the registry proxy should not be created. + /// [Fact] - public void TestDependabotProxyCreation1() + public void TestDependabotProxyNoPort() { // Setup var config = new DependabotConfigurationStub @@ -46,8 +50,12 @@ public void TestDependabotProxyCreation1() Assert.Null(proxy); } + /// + /// The purpose of this test is to verify that the registry proxy correctly handles the case where the host is not specified. + /// In this case, the registry proxy should not be created. + /// [Fact] - public void TestDependabotProxyCreation2() + public void TestDependabotProxyNoHost() { // Setup var config = new DependabotConfigurationStub @@ -96,6 +104,10 @@ public void TestDependabotProxyCreation2() -----END CERTIFICATE----- """; + /// + /// The purpose of this test is to verify that the registry proxy correctly handles the case + /// where the port, host, and certificate are specified. + /// [Fact] public void TestDependabotProxyCertificate() { @@ -118,8 +130,13 @@ public void TestDependabotProxyCertificate() Assert.NotNull(proxy.CertificatePath); } + /// + /// The purpose of this test is to verify that the registry proxy correctly handles the case + /// where the RegistryURLs environment variable is not a valid JSON list. + /// In this case, the registry proxy should be created, but the list of private registries should be empty. + /// [Fact] - public void TestDependabotRegistryUrls1() + public void TestDependabotRegistryUrlsParseError() { // Setup var config = new DependabotConfigurationStub @@ -135,11 +152,17 @@ public void TestDependabotRegistryUrls1() // Verify Assert.NotNull(proxy); - Assert.Equal([], proxy.RegistryURLs); + Assert.Empty(proxy.RegistryURLs); + Assert.Empty(proxy.RegistryBaseURLs); } + /// + /// The purpose of this test is to verify that the registry proxy correctly handles the case + /// where the RegistryURLs environment variable is a valid JSON list with a single entry. + /// In this case, the registry proxy should be created, and the list of private registries should contain the single entry. + /// [Fact] - public void TestDependabotRegistryUrls2() + public void TestDependabotRegistryUrlsSingle() { // Setup var config = new DependabotConfigurationStub @@ -158,8 +181,16 @@ public void TestDependabotRegistryUrls2() Assert.Equal([ "https://nuget.pkg.github.com/org/index.json" ], proxy.RegistryURLs); + Assert.Empty(proxy.RegistryBaseURLs); } + /// + /// The purpose of this test is to verify that the registry proxy correctly handles the case + /// where the RegistryURLs environment variable is a valid JSON list with multiple entries, but only one of them + /// is of type "nuget_feed", which is relevant for C#. + /// In this case, the registry proxy should be created, and the list of private registries should + /// contain only the entry of type "nuget_feed". + /// [Fact] public void TestDependabotRegistryUrls3() { @@ -180,6 +211,40 @@ public void TestDependabotRegistryUrls3() Assert.Equal([ "https://example.com/org/index.json" ], proxy.RegistryURLs); + Assert.Empty(proxy.RegistryBaseURLs); + } + + /// + /// The purpose of this test is to verify that the registry proxy correctly handles the case + /// where the RegistryURLs environment variable is a valid JSON list with multiple entries and one of them + /// is configured to replace the base feeds. + /// In this case, the registry proxy should be created, and the list of private registries should contain all + /// entries, while the list of base registries should contain only the entry that replaces the base feeds. + /// + [Fact] + public void TestDependabotRegistryUrlsReplacesBase() + { + // Setup + var config = new DependabotConfigurationStub + { + Port = "8080", + Host = "localhost", + RegistryURLs = "[ { \"type\": \"nuget_feed\", \"url\": \"https://example.com/org/index.json\", \"replaces-base\": true }, { \"type\": \"nuget_feed\", \"url\": \"https://example2.com/org/index.json\", \"replaces-base\": false } ]" + }; + + // Execute + using var tempWorkingDirectory = MakeTemporaryDirectory(); + using var proxy = DependabotProxy.Make(config, new LoggerStub(), new DiagnosticsWriterStub(), tempWorkingDirectory); + + // Verify + Assert.NotNull(proxy); + Assert.Equal([ + "https://example.com/org/index.json", + "https://example2.com/org/index.json" + ], proxy.RegistryURLs); + Assert.Equal([ + "https://example.com/org/index.json", + ], proxy.RegistryBaseURLs); } } } diff --git a/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs b/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs index 77e88a58443a..1d393a418ca9 100644 --- a/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs +++ b/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs @@ -285,7 +285,7 @@ public void TestNugetFeeds() var dotnet = MakeDotnet(dotnetCliInvoker); // Execute - dotnet.GetNugetFeeds("abc"); + dotnet.GetNugetFeedsFromConfig("abc"); // Verify var lastArgs = dotnetCliInvoker.GetLastArgs(); diff --git a/csharp/extractor/Semmle.Extraction.Tests/DotNetStub.cs b/csharp/extractor/Semmle.Extraction.Tests/DotNetStub.cs index 119e39fd0974..12a131380e41 100644 --- a/csharp/extractor/Semmle.Extraction.Tests/DotNetStub.cs +++ b/csharp/extractor/Semmle.Extraction.Tests/DotNetStub.cs @@ -30,7 +30,7 @@ public DotNetStub(IList runtimes, IList sdks, IList nuge public bool Exec(List execArgs) => true; - public IList GetNugetFeeds(string nugetConfig) => nugetFeedsFromConfig; + public IList GetNugetFeedsFromConfig(string nugetConfig) => nugetFeedsFromConfig; public IList GetNugetFeedsFromFolder(string folderPath) => nugetFeedsFromFolder; } diff --git a/csharp/extractor/Semmle.Extraction.Tests/FeedManager.cs b/csharp/extractor/Semmle.Extraction.Tests/FeedManager.cs index f70efdb4cdcc..2fdecd570954 100644 --- a/csharp/extractor/Semmle.Extraction.Tests/FeedManager.cs +++ b/csharp/extractor/Semmle.Extraction.Tests/FeedManager.cs @@ -1,8 +1,10 @@ using Xunit; using System; using System.Collections.Generic; +using System.Collections.Immutable; using System.IO; using System.Linq; +using System.Security.Cryptography.X509Certificates; using Semmle.Extraction.CSharp.DependencyFetching; namespace Semmle.Extraction.Tests @@ -10,9 +12,21 @@ namespace Semmle.Extraction.Tests public class DependabotProxyStub : IDependabotProxy { public string Address { get; } = ""; - public HashSet RegistryURLs { get; } = ["https://example.com/registry1", "https://example.com/registry2"]; + public ImmutableHashSet RegistryURLs { get; } = ["https://example.com/registry1", "https://example.com/registry2"]; + public ImmutableHashSet RegistryBaseURLs { get; } = []; public string? CertificatePath { get; } = null; - public System.Security.Cryptography.X509Certificates.X509Certificate2? Certificate { get; } = null; + public X509Certificate2? Certificate { get; } = null; + + public void Dispose() { } + } + + public class DependabotProxyStubWithBaseUrls : IDependabotProxy + { + public string Address { get; } = ""; + public ImmutableHashSet RegistryURLs { get; } = ["https://example.com/registry1", "https://example.com/registry2", "https://example.com/base1", "https://example.com/base2"]; + public ImmutableHashSet RegistryBaseURLs { get; } = ["https://example.com/base1", "https://example.com/base2"]; + public string? CertificatePath { get; } = null; + public X509Certificate2? Certificate { get; } = null; public void Dispose() { } } @@ -54,6 +68,11 @@ public class FileProviderStub : IFileProvider public ICollection Resources { get; } = new List(); } + /// + /// The purpose of this test class is to verify the behavior of the FeedManager class. + /// The tests use stub implementations of the FeedManager's dependencies to control the behavior of the FeedManager + /// and verify its behavior. + /// public class FeedManagerTests { private static FeedManager MakeFeedManager() @@ -66,6 +85,12 @@ private static FeedManager MakeFeedManager() return new FeedManager(logger, dotnet, dependabotProxy, fileProvider, feedManagerIo); } + /// + /// Verify that `FeedManager` correctly computes the explicit feeds using feeds discovered in nuget.config files and + /// private registries. + /// See the initialization of `DotNetStub` and `DependabotProxyStub` in `MakeFeedManager` for the feeds configured + /// to be returned and classified as explicit feeds. + /// [Fact] public void TestExplicitFeeds() { @@ -83,6 +108,11 @@ public void TestExplicitFeeds() ], actualFeeds); } + /// + /// Verify that `FeedManager` correctly computes the inherited feeds using feeds discovered from the environment. + /// See the initialization of `DotNetStub` in `MakeFeedManager` for the feeds configured + /// to be returned and classified as inherited feeds. + /// [Fact] public void TestInheritedFeeds() { @@ -99,6 +129,12 @@ public void TestInheritedFeeds() ], inherited); } + /// + /// Verify that `FeedManager` correctly computes all feeds using feeds discovered in nuget.config files, private registries, + /// and the environment. + /// See the initialization of `DotNetStub` and `DependabotProxyStub` in `MakeFeedManager` for the feeds configured + /// to be returned and included in all feeds. + /// [Fact] public void TestAllFeeds() { @@ -118,6 +154,12 @@ public void TestAllFeeds() ], all); } + /// + /// Verify that `FeedManager` correctly computes the reachable feeds using feeds discovered in + /// nuget.config files, private registries, and the environment. + /// See the initialization of `FeedManagerIOStub` in `MakeFeedManager` for the feeds configured as unreachable + /// and therefore filtered out of the reachable feeds. + /// [Fact] public void TestReachableFeeds() { @@ -135,6 +177,12 @@ public void TestReachableFeeds() ], reachableFeeds); } + /// + /// Verify that `FeedManager` correctly computes the reachable explicit feeds using feeds discovered in + /// nuget.config files and private registries. + /// See the initialization of `FeedManagerIOStub` in `MakeFeedManager` for the feeds configured as unreachable + /// and therefore filtered out of the reachable explicit feeds. + /// [Fact] public void TestReachableExplicitFeeds() { @@ -151,6 +199,12 @@ public void TestReachableExplicitFeeds() ], reachableFeeds); } + /// + /// Verify that `FeedManager` correctly computes the reachable fallback feeds using feeds discovered in + /// nuget.config files and the default NuGet.org feed. + /// See the initialization of `FeedManagerIOStub` in `MakeFeedManager` for the feeds configured as unreachable + /// and therefore filtered out of the reachable fallback feeds. + /// [Fact] public void TestReachableFallbackFeeds() { @@ -168,6 +222,12 @@ public void TestReachableFallbackFeeds() ], reachableFallback); } + /// + /// Verify that `FeedManager` correctly computes the feeds to use for a given packages.config file from feeds discovered + /// in private registries and the environment. + /// See the initialization of `DotNetStub` in `MakeFeedManager` for the feeds configured + /// to be returned and selected for use. + /// [Fact] public void TestFeedsToUse() { @@ -183,5 +243,130 @@ public void TestFeedsToUse() "https://feed.from/folder1" ], feedsToUse); } + + /// + /// Verify that `FeedManager` correctly computes the default feeds and reachable default feeds + /// when no private registries are configured. + /// + [Fact] + public void TestDefaultFeedsNugetOrg() + { + // Setup + var feedManager = MakeFeedManager(); + + // Execute + var defaultFeeds = feedManager.DefaultFeeds; + var reachableDefault = feedManager.ReachableDefaultFeeds; + + // Verify + Assert.Equal([ + "https://api.nuget.org/v3/index.json" + ], defaultFeeds); + Assert.Equal([ + "https://api.nuget.org/v3/index.json" + ], reachableDefault); + } + + /// + /// Verify that `FeedManager` correctly computes the default feeds and reachable default feeds + /// when private registries are configured to replace the default feeds. + /// See the initialization of `DependabotProxyStubWithBaseUrls` for the feeds configured to replace the default feeds. + /// + [Fact] + public void TestDefaultFeedsPrivateRegistries() + { + // Setup + var logger = new LoggerStub(); + var dotnet = new DotNetStub([], [], [], []); + var dependabotProxy = new DependabotProxyStubWithBaseUrls(); + var fileProvider = new FileProviderStub(); + var feedManagerIo = new FeedManagerIOStub(["https://example.com/registry2", "https://example.com/base1"]); + var feedManager = new FeedManager(logger, dotnet, dependabotProxy, fileProvider, feedManagerIo); + + // Execute + var defaultFeeds = feedManager.DefaultFeeds; + var reachableDefault = feedManager.ReachableDefaultFeeds; + var reachableFallback = feedManager.ReachableFallbackFeeds; + + // Verify + Assert.Equal([ + "https://example.com/base1", + "https://example.com/base2" + ], defaultFeeds); + Assert.Equal([ + "https://example.com/base2" + ], reachableDefault); + Assert.Equal([ + "https://example.com/registry1", + "https://example.com/base2" + ], reachableFallback); + } + + /// + /// Verify that `FeedManager` correctly computes all feeds when https://api.nuget.org/v3/index.json is not replaced + /// by a private registry because no private registry is configured to replace the base feed. + /// + [Fact] + public void TestNugetOrgNotReplaced() + { + // Setup + var logger = new LoggerStub(); + var dotnet = new DotNetStub([], [], [], ["E https://api.nuget.org/v3/index.json"]); + var dependabotProxy = new DependabotProxyStub(); + var fileProvider = new FileProviderStub(); + var feedManagerIo = new FeedManagerIOStub(["https://example.com/registry2", "https://example.com/base1"]); + var feedManager = new FeedManager(logger, dotnet, dependabotProxy, fileProvider, feedManagerIo); + + // Execute + var explicitFeeds = feedManager.ExplicitFeeds; + var allFeeds = feedManager.AllFeeds; + + // Verify + Assert.Equal([ + "https://example.com/registry1", + "https://example.com/registry2", + ], explicitFeeds); + Assert.Equal([ + "https://example.com/registry1", + "https://example.com/registry2", + "https://api.nuget.org/v3/index.json" + ], allFeeds); + + } + + /// + /// Verify that `FeedManager` correctly computes the explicit and all feeds when https://api.nuget.org/v3/index.json and + /// related NuGet.org URLs are replaced by private registries configured to replace the base feeds. + /// See the initialization of `DependabotProxyStubWithBaseUrls` for the feeds configured as default replacements. + /// + [Fact] + public void TestNugetOrgReplacement() + { + // Setup + var logger = new LoggerStub(); + var dotnet = new DotNetStub([], [], ["E https://www.nuget.org/api/v2/"], ["E https://api.nuget.org/v3/index.json"]); + var dependabotProxy = new DependabotProxyStubWithBaseUrls(); + var fileProvider = new FileProviderStub(); + var feedManagerIo = new FeedManagerIOStub(["https://example.com/registry2", "https://example.com/base1"]); + var feedManager = new FeedManager(logger, dotnet, dependabotProxy, fileProvider, feedManagerIo); + + // Execute + var explicitFeeds = feedManager.ExplicitFeeds; + var allFeeds = feedManager.AllFeeds; + + // Verify + Assert.Equal([ + "https://example.com/base1", + "https://example.com/base2", + "https://example.com/registry1", + "https://example.com/registry2" + ], explicitFeeds); + Assert.Equal([ + "https://example.com/base1", + "https://example.com/base2", + "https://example.com/registry1", + "https://example.com/registry2", + ], allFeeds); + } } } diff --git a/csharp/ql/lib/change-notes/2026-09-03-replaces-base.md b/csharp/ql/lib/change-notes/2026-09-03-replaces-base.md new file mode 100644 index 000000000000..2a80f4a89eab --- /dev/null +++ b/csharp/ql/lib/change-notes/2026-09-03-replaces-base.md @@ -0,0 +1,4 @@ +--- +category: minorAnalysis +--- +* Private NuGet registries for which the "Replaces base" option is enabled in the organization-level private registry configuration now replace default NuGet feeds whenever dependencies are downloaded, including when default NuGet feeds are configured explicitly for a project.