From cf896f243fb8c83d1d0951ca84cccb987bf584d0 Mon Sep 17 00:00:00 2001 From: Michael Nebel Date: Fri, 19 Jun 2026 15:29:16 +0200 Subject: [PATCH] C#: Re-factor more the feed related methods into the FeedManager. --- .../FeedManager.cs | 220 +++++++++++++++++ .../NugetPackageRestorer.cs | 229 +----------------- .../PackagesConfigRestorer.cs | 2 +- 3 files changed, 227 insertions(+), 224 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs index 62a0a7b911b..562fe9afdc7 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs @@ -2,8 +2,13 @@ using System; using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; +using System.Net; +using System.Net.Http; +using System.Security.Cryptography.X509Certificates; using System.Text; using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; using Semmle.Util; using Semmle.Util.Logging; @@ -11,8 +16,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { internal sealed partial class FeedManager : IDisposable { + internal const string PublicNugetOrgFeed = "https://api.nuget.org/v3/index.json"; + private readonly ILogger logger; private readonly IDotNet dotnet; + private readonly DependabotProxy? dependabotProxy; private readonly DependencyDirectory emptyPackageDirectory; public ImmutableHashSet PrivateRegistryFeeds { get; } @@ -23,6 +31,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { this.logger = logger; this.dotnet = dotnet; + this.dependabotProxy = dependabotProxy; PrivateRegistryFeeds = dependabotProxy?.RegistryURLs.ToImmutableHashSet() ?? []; HasPrivateRegistryFeeds = PrivateRegistryFeeds.Count > 0; emptyPackageDirectory = new DependencyDirectory("empty", "empty package", logger); @@ -114,6 +123,217 @@ namespace Semmle.Extraction.CSharp.DependencyFetching return FeedsToRestoreArgument(feedsToUse); } + private (int initialTimeout, int tryCount) GetFeedRequestSettings(bool isFallback) + { + int timeoutMilliSeconds = isFallback && int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessInitialTimeoutForFallback), out timeoutMilliSeconds) + ? timeoutMilliSeconds + : int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessInitialTimeout), out timeoutMilliSeconds) + ? timeoutMilliSeconds + : 1000; + logger.LogDebug($"Initial timeout for NuGet feed reachability check is {timeoutMilliSeconds}ms."); + + int tryCount = isFallback && int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessRequestCountForFallback), out tryCount) + ? tryCount + : int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessRequestCount), out tryCount) + ? tryCount + : 4; + logger.LogDebug($"Number of tries for NuGet feed reachability check is {tryCount}."); + + return (timeoutMilliSeconds, tryCount); + } + + private static async Task ExecuteGetRequest(string address, HttpClient httpClient, CancellationToken cancellationToken) + { + return await httpClient.GetAsync(address, HttpCompletionOption.ResponseHeadersRead, cancellationToken); + } + + private bool IsFeedReachable(string feed, int timeoutMilliSeconds, int tryCount, out bool isTimeout) + { + logger.LogInfo($"Checking if NuGet feed '{feed}' is reachable..."); + + // Configure the HttpClient to be aware of the Dependabot Proxy, if used. + HttpClientHandler httpClientHandler = new(); + if (dependabotProxy != null) + { + httpClientHandler.Proxy = new WebProxy(dependabotProxy.Address); + + if (dependabotProxy.Certificate != null) + { + httpClientHandler.ServerCertificateCustomValidationCallback = (message, cert, chain, _) => + { + if (chain is null || cert is null) + { + var msg = cert is null && chain is null + ? "certificate and chain" + : chain is null + ? "chain" + : "certificate"; + logger.LogWarning($"Dependabot proxy certificate validation failed due to missing {msg}"); + return false; + } + chain.ChainPolicy.TrustMode = X509ChainTrustMode.CustomRootTrust; + chain.ChainPolicy.CustomTrustStore.Add(dependabotProxy.Certificate); + return chain.Build(cert); + }; + } + } + + using HttpClient client = new(httpClientHandler); + + isTimeout = false; + + for (var i = 0; i < tryCount; i++) + { + using var cts = new CancellationTokenSource(); + cts.CancelAfter(timeoutMilliSeconds); + try + { + logger.LogInfo($"Attempt {i + 1}/{tryCount} to reach NuGet feed '{feed}'."); + using var response = ExecuteGetRequest(feed, client, cts.Token).GetAwaiter().GetResult(); + response.EnsureSuccessStatusCode(); + logger.LogInfo($"Querying NuGet feed '{feed}' succeeded."); + return true; + } + catch (Exception exc) + { + if (exc is TaskCanceledException tce && + tce.CancellationToken == cts.Token && + cts.Token.IsCancellationRequested) + { + logger.LogInfo($"Didn't receive answer from NuGet feed '{feed}' in {timeoutMilliSeconds}ms."); + timeoutMilliSeconds *= 2; + continue; + } + + logger.LogInfo($"Querying NuGet feed '{feed}' failed. The reason for the failure: {exc.Message}"); + return false; + } + } + + logger.LogWarning($"Didn't receive answer from NuGet feed '{feed}'. Tried it {tryCount} times."); + isTimeout = true; + return false; + } + + /// + /// Retrieves a list of excluded NuGet feeds from the corresponding environment variable. + /// + private HashSet GetExcludedFeeds() + { + var excludedFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.ExcludedNugetFeedsFromResponsivenessCheck) + .ToHashSet(); + + if (excludedFeeds.Count > 0) + { + logger.LogInfo($"Excluded NuGet feeds from responsiveness check: {string.Join(", ", excludedFeeds.OrderBy(f => f))}"); + } + + return excludedFeeds; + } + + /// + /// Checks that we can connect to the specified NuGet feeds. + /// + /// The set of package feeds to check. + /// The list of feeds that were reachable. + /// + /// True if there is a timeout when trying to reach the feeds (excluding any feeds that are configured + /// to be excluded from the check) or false otherwise. + /// + public bool CheckSpecifiedFeeds(HashSet feeds, out HashSet reachableFeeds) + { + // Exclude any feeds from the feed check that are configured by the corresponding environment variable. + // These feeds are always assumed to be reachable. + var excludedFeeds = GetExcludedFeeds(); + + HashSet feedsToCheck = feeds.Where(feed => + { + if (excludedFeeds.Contains(feed)) + { + logger.LogInfo($"Not checking reachability of NuGet feed '{feed}' as it is in the list of excluded feeds."); + return false; + } + return true; + }).ToHashSet(); + + reachableFeeds = GetReachableNuGetFeeds(feedsToCheck, isFallback: false, out var isTimeout).ToHashSet(); + + // Always consider feeds excluded for the reachability check as reachable. + reachableFeeds.UnionWith(feeds.Where(feed => excludedFeeds.Contains(feed))); + + return isTimeout; + } + + public bool IsDefaultFeedReachable() + { + if (CheckNugetFeedResponsiveness) + { + var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback: false); + return IsFeedReachable(PublicNugetOrgFeed, initialTimeout, tryCount, out var _); + } + + return true; + } + + /// + /// Tests which of the feeds given by are reachable. + /// + /// The feeds to check. + /// Whether the feeds are fallback feeds or not. + /// Whether a timeout occurred while checking the feeds. + /// The list of feeds that could be reached. + public List GetReachableNuGetFeeds(HashSet feedsToCheck, bool isFallback, out bool isTimeout) + { + var fallbackStr = isFallback ? "fallback " : ""; + logger.LogInfo($"Checking {fallbackStr}NuGet feed reachability on feeds: {string.Join(", ", feedsToCheck.OrderBy(f => f))}"); + + var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback); + var timeout = false; + var reachableFeeds = feedsToCheck + .Where(feed => + { + var reachable = IsFeedReachable(feed, initialTimeout, tryCount, out var feedTimeout); + timeout |= feedTimeout; + return reachable; + }) + .ToList(); + + if (reachableFeeds.Count == 0) + { + logger.LogWarning($"No {fallbackStr}NuGet feeds are reachable."); + } + else + { + logger.LogInfo($"Reachable {fallbackStr}NuGet feeds: {string.Join(", ", reachableFeeds.OrderBy(f => f))}"); + } + + isTimeout = timeout; + return reachableFeeds; + } + + public List GetReachableFallbackNugetFeeds(HashSet? feedsFromNugetConfigs) + { + var fallbackFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.FallbackNugetFeeds).ToHashSet(); + if (fallbackFeeds.Count == 0) + { + fallbackFeeds.Add(PublicNugetOrgFeed); + logger.LogInfo($"No fallback NuGet feeds specified. Adding default feed: {PublicNugetOrgFeed}"); + + var shouldAddNugetConfigFeeds = EnvironmentVariables.GetBooleanOptOut(EnvironmentVariableNames.AddNugetConfigFeedsToFallback); + logger.LogInfo($"Adding feeds from nuget.config to fallback restore: {shouldAddNugetConfigFeeds}"); + + if (shouldAddNugetConfigFeeds && feedsFromNugetConfigs?.Count > 0) + { + // There are some feeds in `feedsFromNugetConfigs` that have already been checked for reachability, we could skip those. + // But we might use different responsiveness testing settings when we try them in the fallback logic, so checking them again is safer. + fallbackFeeds.UnionWith(feedsFromNugetConfigs); + logger.LogInfo($"Using NuGet feeds from nuget.config files as fallback feeds: {string.Join(", ", feedsFromNugetConfigs.OrderBy(f => f))}"); + } + } + + return GetReachableNuGetFeeds(fallbackFeeds, isFallback: true, out var _); + } + [GeneratedRegex(@"^E\s(.*)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)] private static partial Regex EnabledNugetFeed(); diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs index 69ca0c3649d..fca913d776c 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs @@ -4,9 +4,6 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.IO; using System.Linq; -using System.Net; -using System.Net.Http; -using System.Security.Cryptography.X509Certificates; using System.Text; using System.Text.RegularExpressions; using System.Threading; @@ -19,8 +16,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { internal sealed partial class NugetPackageRestorer : IDisposable { - internal const string PublicNugetOrgFeed = "https://api.nuget.org/v3/index.json"; - private readonly FileProvider fileProvider; private readonly FileContent fileContent; private readonly IDotNet dotnet; @@ -136,7 +131,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching compilationInfoContainer.CompilationInfos.Add(("Inherited NuGet feed count", inheritedFeeds.Count.ToString())); } - var timeout = CheckSpecifiedFeeds(explicitFeeds, out var reachableExplicitFeeds); + var timeout = feedManager.CheckSpecifiedFeeds(explicitFeeds, out var reachableExplicitFeeds); reachableFeeds.UnionWith(reachableExplicitFeeds); var allExplicitReachable = explicitFeeds.Count == reachableExplicitFeeds.Count; @@ -153,11 +148,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } // Inherited feeds should only be used, if they are indeed reachable (as they may be environment specific). - CheckSpecifiedFeeds(inheritedFeeds, out var reachableInheritedFeeds); + feedManager.CheckSpecifiedFeeds(inheritedFeeds, out var reachableInheritedFeeds); reachableFeeds.UnionWith(reachableInheritedFeeds); } - using (var packagesConfigRestore = PackagesConfigRestoreFactory.Create(fileProvider, legacyPackageDirectory, logger, IsDefaultFeedReachable)) + using (var packagesConfigRestore = PackagesConfigRestoreFactory.Create(fileProvider, legacyPackageDirectory, logger, feedManager.IsDefaultFeedReachable)) { var count = packagesConfigRestore.InstallPackages(); @@ -222,79 +217,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching return assemblyLookupLocations; } - /// - /// Tests which of the feeds given by are reachable. - /// - /// The feeds to check. - /// Whether the feeds are fallback feeds or not. - /// Whether a timeout occurred while checking the feeds. - /// The list of feeds that could be reached. - private List GetReachableNuGetFeeds(HashSet feedsToCheck, bool isFallback, out bool isTimeout) - { - var fallbackStr = isFallback ? "fallback " : ""; - logger.LogInfo($"Checking {fallbackStr}NuGet feed reachability on feeds: {string.Join(", ", feedsToCheck.OrderBy(f => f))}"); - - var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback); - var timeout = false; - var reachableFeeds = feedsToCheck - .Where(feed => - { - var reachable = IsFeedReachable(feed, initialTimeout, tryCount, out var feedTimeout); - timeout |= feedTimeout; - return reachable; - }) - .ToList(); - - if (reachableFeeds.Count == 0) - { - logger.LogWarning($"No {fallbackStr}NuGet feeds are reachable."); - } - else - { - logger.LogInfo($"Reachable {fallbackStr}NuGet feeds: {string.Join(", ", reachableFeeds.OrderBy(f => f))}"); - } - - isTimeout = timeout; - return reachableFeeds; - } - - private bool IsDefaultFeedReachable() - { - if (feedManager.CheckNugetFeedResponsiveness) - { - var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback: false); - return IsFeedReachable(PublicNugetOrgFeed, initialTimeout, tryCount, out var _); - } - - return true; - } - - private List GetReachableFallbackNugetFeeds(HashSet? feedsFromNugetConfigs) - { - var fallbackFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.FallbackNugetFeeds).ToHashSet(); - if (fallbackFeeds.Count == 0) - { - fallbackFeeds.Add(PublicNugetOrgFeed); - logger.LogInfo($"No fallback NuGet feeds specified. Adding default feed: {PublicNugetOrgFeed}"); - - var shouldAddNugetConfigFeeds = EnvironmentVariables.GetBooleanOptOut(EnvironmentVariableNames.AddNugetConfigFeedsToFallback); - logger.LogInfo($"Adding feeds from nuget.config to fallback restore: {shouldAddNugetConfigFeeds}"); - - if (shouldAddNugetConfigFeeds && feedsFromNugetConfigs?.Count > 0) - { - // There are some feeds in `feedsFromNugetConfigs` that have already been checked for reachability, we could skip those. - // But we might use different responsiveness testing settings when we try them in the fallback logic, so checking them again is safer. - fallbackFeeds.UnionWith(feedsFromNugetConfigs); - logger.LogInfo($"Using NuGet feeds from nuget.config files as fallback feeds: {string.Join(", ", feedsFromNugetConfigs.OrderBy(f => f))}"); - } - } - - var reachableFallbackFeeds = GetReachableNuGetFeeds(fallbackFeeds, isFallback: true, out var _); - - compilationInfoContainer.CompilationInfos.Add(("Reachable fallback NuGet feed count", reachableFallbackFeeds.Count.ToString())); - - return reachableFallbackFeeds; - } /// /// Executes `dotnet restore` on all solution files in solutions. @@ -395,7 +317,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching private AssemblyLookupLocation? DownloadMissingPackagesFromSpecificFeeds(IEnumerable usedPackageNames, HashSet? feedsFromNugetConfigs) { - var reachableFallbackFeeds = GetReachableFallbackNugetFeeds(feedsFromNugetConfigs); + var reachableFallbackFeeds = feedManager.GetReachableFallbackNugetFeeds(feedsFromNugetConfigs); + compilationInfoContainer.CompilationInfos.Add(("Reachable fallback NuGet feed count", reachableFallbackFeeds.Count.ToString())); + if (reachableFallbackFeeds.Count > 0) { return DownloadMissingPackages(usedPackageNames, fallbackNugetFeeds: reachableFallbackFeeds); @@ -681,147 +605,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } } - private static async Task ExecuteGetRequest(string address, HttpClient httpClient, CancellationToken cancellationToken) - { - return await httpClient.GetAsync(address, HttpCompletionOption.ResponseHeadersRead, cancellationToken); - } - - private bool IsFeedReachable(string feed, int timeoutMilliSeconds, int tryCount, out bool isTimeout) - { - logger.LogInfo($"Checking if NuGet feed '{feed}' is reachable..."); - - // Configure the HttpClient to be aware of the Dependabot Proxy, if used. - HttpClientHandler httpClientHandler = new(); - if (dependabotProxy != null) - { - httpClientHandler.Proxy = new WebProxy(dependabotProxy.Address); - - if (dependabotProxy.Certificate != null) - { - httpClientHandler.ServerCertificateCustomValidationCallback = (message, cert, chain, _) => - { - if (chain is null || cert is null) - { - var msg = cert is null && chain is null - ? "certificate and chain" - : chain is null - ? "chain" - : "certificate"; - logger.LogWarning($"Dependabot proxy certificate validation failed due to missing {msg}"); - return false; - } - chain.ChainPolicy.TrustMode = X509ChainTrustMode.CustomRootTrust; - chain.ChainPolicy.CustomTrustStore.Add(dependabotProxy.Certificate); - return chain.Build(cert); - }; - } - } - - using HttpClient client = new(httpClientHandler); - - isTimeout = false; - - for (var i = 0; i < tryCount; i++) - { - using var cts = new CancellationTokenSource(); - cts.CancelAfter(timeoutMilliSeconds); - try - { - logger.LogInfo($"Attempt {i + 1}/{tryCount} to reach NuGet feed '{feed}'."); - using var response = ExecuteGetRequest(feed, client, cts.Token).GetAwaiter().GetResult(); - response.EnsureSuccessStatusCode(); - logger.LogInfo($"Querying NuGet feed '{feed}' succeeded."); - return true; - } - catch (Exception exc) - { - if (exc is TaskCanceledException tce && - tce.CancellationToken == cts.Token && - cts.Token.IsCancellationRequested) - { - logger.LogInfo($"Didn't receive answer from NuGet feed '{feed}' in {timeoutMilliSeconds}ms."); - timeoutMilliSeconds *= 2; - continue; - } - - logger.LogInfo($"Querying NuGet feed '{feed}' failed. The reason for the failure: {exc.Message}"); - return false; - } - } - - logger.LogWarning($"Didn't receive answer from NuGet feed '{feed}'. Tried it {tryCount} times."); - isTimeout = true; - return false; - } - - private (int initialTimeout, int tryCount) GetFeedRequestSettings(bool isFallback) - { - int timeoutMilliSeconds = isFallback && int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessInitialTimeoutForFallback), out timeoutMilliSeconds) - ? timeoutMilliSeconds - : int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessInitialTimeout), out timeoutMilliSeconds) - ? timeoutMilliSeconds - : 1000; - logger.LogDebug($"Initial timeout for NuGet feed reachability check is {timeoutMilliSeconds}ms."); - - int tryCount = isFallback && int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessRequestCountForFallback), out tryCount) - ? tryCount - : int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessRequestCount), out tryCount) - ? tryCount - : 4; - logger.LogDebug($"Number of tries for NuGet feed reachability check is {tryCount}."); - - return (timeoutMilliSeconds, tryCount); - } - - /// - /// Retrieves a list of excluded NuGet feeds from the corresponding environment variable. - /// - private HashSet GetExcludedFeeds() - { - var excludedFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.ExcludedNugetFeedsFromResponsivenessCheck) - .ToHashSet(); - - if (excludedFeeds.Count > 0) - { - logger.LogInfo($"Excluded NuGet feeds from responsiveness check: {string.Join(", ", excludedFeeds.OrderBy(f => f))}"); - } - - return excludedFeeds; - } - - /// - /// Checks that we can connect to the specified NuGet feeds. - /// - /// The set of package feeds to check. - /// The list of feeds that were reachable. - /// - /// True if there is a timeout when trying to reach the feeds (excluding any feeds that are configured - /// to be excluded from the check) or false otherwise. - /// - private bool CheckSpecifiedFeeds(HashSet feeds, out HashSet reachableFeeds) - { - // Exclude any feeds from the feed check that are configured by the corresponding environment variable. - // These feeds are always assumed to be reachable. - var excludedFeeds = GetExcludedFeeds(); - - HashSet feedsToCheck = feeds.Where(feed => - { - if (excludedFeeds.Contains(feed)) - { - logger.LogInfo($"Not checking reachability of NuGet feed '{feed}' as it is in the list of excluded feeds."); - return false; - } - return true; - }).ToHashSet(); - - reachableFeeds = GetReachableNuGetFeeds(feedsToCheck, isFallback: false, out var isTimeout).ToHashSet(); - - // Always consider feeds excluded for the reachability check as reachable. - reachableFeeds.UnionWith(feeds.Where(feed => excludedFeeds.Contains(feed))); - - return isTimeout; - } - /// /// If is `false`, logs this and emits a diagnostic. /// Adds a `CompilationInfos` entry either way. diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs index a4fca7e2c84..51cd2755578 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs @@ -302,7 +302,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching private void AddDefaultPackageSource(string nugetConfig) { logger.LogInfo("Adding default package source..."); - RunMonoNugetCommand($"sources add -Name DefaultNugetOrg -Source {NugetPackageRestorer.PublicNugetOrgFeed} -ConfigFile \"{nugetConfig}\"", out _); + RunMonoNugetCommand($"sources add -Name DefaultNugetOrg -Source {FeedManager.PublicNugetOrgFeed} -ConfigFile \"{nugetConfig}\"", out _); } public void Dispose()