hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-06-24 14:17:05 +02:00
Code Issues Packages Projects Releases Wiki Activity

C#: Re-factor more the feed related methods into the FeedManager.

Browse Source
This commit is contained in:
Michael Nebel
2026-06-19 15:29:16 +02:00
parent bae2d18446
commit cf896f243f
3 changed files with 227 additions and 224 deletions
Download Patch File Download Diff File Expand all files Collapse all files

220
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FeedManager.cs
View File

@@ -2,8 +2,13 @@ using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using Semmle.Util;
using Semmle.Util.Logging;
@@ -11,8 +16,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal sealed partial class FeedManager : IDisposable
{
internal const string PublicNugetOrgFeed = "https://api.nuget.org/v3/index.json";
private readonly ILogger logger;
private readonly IDotNet dotnet;
private readonly DependabotProxy? dependabotProxy;
private readonly DependencyDirectory emptyPackageDirectory;
public ImmutableHashSet<string> PrivateRegistryFeeds { get; }
@@ -23,6 +31,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
this.logger = logger;
this.dotnet = dotnet;
this.dependabotProxy = dependabotProxy;
PrivateRegistryFeeds = dependabotProxy?.RegistryURLs.ToImmutableHashSet() ?? [];
HasPrivateRegistryFeeds = PrivateRegistryFeeds.Count > 0;
emptyPackageDirectory = new DependencyDirectory("empty", "empty package", logger);
@@ -114,6 +123,217 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
return FeedsToRestoreArgument(feedsToUse);
}
private (int initialTimeout, int tryCount) GetFeedRequestSettings(bool isFallback)
{
int timeoutMilliSeconds = isFallback && int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessInitialTimeoutForFallback), out timeoutMilliSeconds)
? timeoutMilliSeconds
: int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessInitialTimeout), out timeoutMilliSeconds)
? timeoutMilliSeconds
: 1000;
logger.LogDebug($"Initial timeout for NuGet feed reachability check is {timeoutMilliSeconds}ms.");
int tryCount = isFallback && int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessRequestCountForFallback), out tryCount)
? tryCount
: int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessRequestCount), out tryCount)
? tryCount
: 4;
logger.LogDebug($"Number of tries for NuGet feed reachability check is {tryCount}.");
return (timeoutMilliSeconds, tryCount);
}
private static async Task<HttpResponseMessage> ExecuteGetRequest(string address, HttpClient httpClient, CancellationToken cancellationToken)
{
return await httpClient.GetAsync(address, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
}
private bool IsFeedReachable(string feed, int timeoutMilliSeconds, int tryCount, out bool isTimeout)
{
logger.LogInfo($"Checking if NuGet feed '{feed}' is reachable...");
// Configure the HttpClient to be aware of the Dependabot Proxy, if used.
HttpClientHandler httpClientHandler = new();
if (dependabotProxy != null)
{
httpClientHandler.Proxy = new WebProxy(dependabotProxy.Address);
if (dependabotProxy.Certificate != null)
{
httpClientHandler.ServerCertificateCustomValidationCallback = (message, cert, chain, _) =>
{
if (chain is null || cert is null)
{
var msg = cert is null && chain is null
? "certificate and chain"
: chain is null
? "chain"
: "certificate";
logger.LogWarning($"Dependabot proxy certificate validation failed due to missing {msg}");
return false;
}
chain.ChainPolicy.TrustMode = X509ChainTrustMode.CustomRootTrust;
chain.ChainPolicy.CustomTrustStore.Add(dependabotProxy.Certificate);
return chain.Build(cert);
};
}
}
using HttpClient client = new(httpClientHandler);
isTimeout = false;
for (var i = 0; i < tryCount; i++)
{
using var cts = new CancellationTokenSource();
cts.CancelAfter(timeoutMilliSeconds);
try
{
logger.LogInfo($"Attempt {i + 1}/{tryCount} to reach NuGet feed '{feed}'.");
using var response = ExecuteGetRequest(feed, client, cts.Token).GetAwaiter().GetResult();
response.EnsureSuccessStatusCode();
logger.LogInfo($"Querying NuGet feed '{feed}' succeeded.");
return true;
}
catch (Exception exc)
{
if (exc is TaskCanceledException tce &&
tce.CancellationToken == cts.Token &&
cts.Token.IsCancellationRequested)
{
logger.LogInfo($"Didn't receive answer from NuGet feed '{feed}' in {timeoutMilliSeconds}ms.");
timeoutMilliSeconds *= 2;
continue;
}
logger.LogInfo($"Querying NuGet feed '{feed}' failed. The reason for the failure: {exc.Message}");
return false;
}
}
logger.LogWarning($"Didn't receive answer from NuGet feed '{feed}'. Tried it {tryCount} times.");
isTimeout = true;
return false;
}
/// <summary>
/// Retrieves a list of excluded NuGet feeds from the corresponding environment variable.
/// </summary>
private HashSet<string> GetExcludedFeeds()
{
var excludedFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.ExcludedNugetFeedsFromResponsivenessCheck)
.ToHashSet();
if (excludedFeeds.Count > 0)
{
logger.LogInfo($"Excluded NuGet feeds from responsiveness check: {string.Join(", ", excludedFeeds.OrderBy(f => f))}");
}
return excludedFeeds;
}
/// <summary>
/// Checks that we can connect to the specified NuGet feeds.
/// </summary>
/// <param name="feeds">The set of package feeds to check.</param>
/// <param name="reachableFeeds">The list of feeds that were reachable.</param>
/// <returns>
/// True if there is a timeout when trying to reach the feeds (excluding any feeds that are configured
/// to be excluded from the check) or false otherwise.
/// </returns>
public bool CheckSpecifiedFeeds(HashSet<string> feeds, out HashSet<string> reachableFeeds)
{
// Exclude any feeds from the feed check that are configured by the corresponding environment variable.
// These feeds are always assumed to be reachable.
var excludedFeeds = GetExcludedFeeds();
HashSet<string> feedsToCheck = feeds.Where(feed =>
{
if (excludedFeeds.Contains(feed))
{
logger.LogInfo($"Not checking reachability of NuGet feed '{feed}' as it is in the list of excluded feeds.");
return false;
}
return true;
}).ToHashSet();
reachableFeeds = GetReachableNuGetFeeds(feedsToCheck, isFallback: false, out var isTimeout).ToHashSet();
// Always consider feeds excluded for the reachability check as reachable.
reachableFeeds.UnionWith(feeds.Where(feed => excludedFeeds.Contains(feed)));
return isTimeout;
}
public bool IsDefaultFeedReachable()
{
if (CheckNugetFeedResponsiveness)
{
var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback: false);
return IsFeedReachable(PublicNugetOrgFeed, initialTimeout, tryCount, out var _);
}
return true;
}
/// <summary>
/// Tests which of the feeds given by <paramref name="feedsToCheck"/> are reachable.
/// </summary>
/// <param name="feedsToCheck">The feeds to check.</param>
/// <param name="isFallback">Whether the feeds are fallback feeds or not.</param>
/// <param name="isTimeout">Whether a timeout occurred while checking the feeds.</param>
/// <returns>The list of feeds that could be reached.</returns>
public List<string> GetReachableNuGetFeeds(HashSet<string> feedsToCheck, bool isFallback, out bool isTimeout)
{
var fallbackStr = isFallback ? "fallback " : "";
logger.LogInfo($"Checking {fallbackStr}NuGet feed reachability on feeds: {string.Join(", ", feedsToCheck.OrderBy(f => f))}");
var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback);
var timeout = false;
var reachableFeeds = feedsToCheck
.Where(feed =>
{
var reachable = IsFeedReachable(feed, initialTimeout, tryCount, out var feedTimeout);
timeout |= feedTimeout;
return reachable;
})
.ToList();
if (reachableFeeds.Count == 0)
{
logger.LogWarning($"No {fallbackStr}NuGet feeds are reachable.");
}
else
{
logger.LogInfo($"Reachable {fallbackStr}NuGet feeds: {string.Join(", ", reachableFeeds.OrderBy(f => f))}");
}
isTimeout = timeout;
return reachableFeeds;
}
public List<string> GetReachableFallbackNugetFeeds(HashSet<string>? feedsFromNugetConfigs)
{
var fallbackFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.FallbackNugetFeeds).ToHashSet();
if (fallbackFeeds.Count == 0)
{
fallbackFeeds.Add(PublicNugetOrgFeed);
logger.LogInfo($"No fallback NuGet feeds specified. Adding default feed: {PublicNugetOrgFeed}");
var shouldAddNugetConfigFeeds = EnvironmentVariables.GetBooleanOptOut(EnvironmentVariableNames.AddNugetConfigFeedsToFallback);
logger.LogInfo($"Adding feeds from nuget.config to fallback restore: {shouldAddNugetConfigFeeds}");
if (shouldAddNugetConfigFeeds && feedsFromNugetConfigs?.Count > 0)
{
// There are some feeds in `feedsFromNugetConfigs` that have already been checked for reachability, we could skip those.
// But we might use different responsiveness testing settings when we try them in the fallback logic, so checking them again is safer.
fallbackFeeds.UnionWith(feedsFromNugetConfigs);
logger.LogInfo($"Using NuGet feeds from nuget.config files as fallback feeds: {string.Join(", ", feedsFromNugetConfigs.OrderBy(f => f))}");
}
}
return GetReachableNuGetFeeds(fallbackFeeds, isFallback: true, out var _);
}
[GeneratedRegex(@"^E\s(.*)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
private static partial Regex EnabledNugetFeed();

229
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs
View File

@@ -4,9 +4,6 @@ using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
@@ -19,8 +16,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal sealed partial class NugetPackageRestorer : IDisposable
{
internal const string PublicNugetOrgFeed = "https://api.nuget.org/v3/index.json";
private readonly FileProvider fileProvider;
private readonly FileContent fileContent;
private readonly IDotNet dotnet;
@@ -136,7 +131,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
compilationInfoContainer.CompilationInfos.Add(("Inherited NuGet feed count", inheritedFeeds.Count.ToString()));
}
var timeout = CheckSpecifiedFeeds(explicitFeeds, out var reachableExplicitFeeds);
var timeout = feedManager.CheckSpecifiedFeeds(explicitFeeds, out var reachableExplicitFeeds);
reachableFeeds.UnionWith(reachableExplicitFeeds);
var allExplicitReachable = explicitFeeds.Count == reachableExplicitFeeds.Count;
@@ -153,11 +148,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
// Inherited feeds should only be used, if they are indeed reachable (as they may be environment specific).
CheckSpecifiedFeeds(inheritedFeeds, out var reachableInheritedFeeds);
feedManager.CheckSpecifiedFeeds(inheritedFeeds, out var reachableInheritedFeeds);
reachableFeeds.UnionWith(reachableInheritedFeeds);
}
using (var packagesConfigRestore = PackagesConfigRestoreFactory.Create(fileProvider, legacyPackageDirectory, logger, IsDefaultFeedReachable))
using (var packagesConfigRestore = PackagesConfigRestoreFactory.Create(fileProvider, legacyPackageDirectory, logger, feedManager.IsDefaultFeedReachable))
{
var count = packagesConfigRestore.InstallPackages();
@@ -222,79 +217,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
return assemblyLookupLocations;
}
/// <summary>
/// Tests which of the feeds given by <paramref name="feedsToCheck"/> are reachable.
/// </summary>
/// <param name="feedsToCheck">The feeds to check.</param>
/// <param name="isFallback">Whether the feeds are fallback feeds or not.</param>
/// <param name="isTimeout">Whether a timeout occurred while checking the feeds.</param>
/// <returns>The list of feeds that could be reached.</returns>
private List<string> GetReachableNuGetFeeds(HashSet<string> feedsToCheck, bool isFallback, out bool isTimeout)
{
var fallbackStr = isFallback ? "fallback " : "";
logger.LogInfo($"Checking {fallbackStr}NuGet feed reachability on feeds: {string.Join(", ", feedsToCheck.OrderBy(f => f))}");
var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback);
var timeout = false;
var reachableFeeds = feedsToCheck
.Where(feed =>
{
var reachable = IsFeedReachable(feed, initialTimeout, tryCount, out var feedTimeout);
timeout |= feedTimeout;
return reachable;
})
.ToList();
if (reachableFeeds.Count == 0)
{
logger.LogWarning($"No {fallbackStr}NuGet feeds are reachable.");
}
else
{
logger.LogInfo($"Reachable {fallbackStr}NuGet feeds: {string.Join(", ", reachableFeeds.OrderBy(f => f))}");
}
isTimeout = timeout;
return reachableFeeds;
}
private bool IsDefaultFeedReachable()
{
if (feedManager.CheckNugetFeedResponsiveness)
{
var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback: false);
return IsFeedReachable(PublicNugetOrgFeed, initialTimeout, tryCount, out var _);
}
return true;
}
private List<string> GetReachableFallbackNugetFeeds(HashSet<string>? feedsFromNugetConfigs)
{
var fallbackFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.FallbackNugetFeeds).ToHashSet();
if (fallbackFeeds.Count == 0)
{
fallbackFeeds.Add(PublicNugetOrgFeed);
logger.LogInfo($"No fallback NuGet feeds specified. Adding default feed: {PublicNugetOrgFeed}");
var shouldAddNugetConfigFeeds = EnvironmentVariables.GetBooleanOptOut(EnvironmentVariableNames.AddNugetConfigFeedsToFallback);
logger.LogInfo($"Adding feeds from nuget.config to fallback restore: {shouldAddNugetConfigFeeds}");
if (shouldAddNugetConfigFeeds && feedsFromNugetConfigs?.Count > 0)
{
// There are some feeds in `feedsFromNugetConfigs` that have already been checked for reachability, we could skip those.
// But we might use different responsiveness testing settings when we try them in the fallback logic, so checking them again is safer.
fallbackFeeds.UnionWith(feedsFromNugetConfigs);
logger.LogInfo($"Using NuGet feeds from nuget.config files as fallback feeds: {string.Join(", ", feedsFromNugetConfigs.OrderBy(f => f))}");
}
}
var reachableFallbackFeeds = GetReachableNuGetFeeds(fallbackFeeds, isFallback: true, out var _);
compilationInfoContainer.CompilationInfos.Add(("Reachable fallback NuGet feed count", reachableFallbackFeeds.Count.ToString()));
return reachableFallbackFeeds;
}
/// <summary>
/// Executes `dotnet restore` on all solution files in solutions.
@@ -395,7 +317,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private AssemblyLookupLocation? DownloadMissingPackagesFromSpecificFeeds(IEnumerable<string> usedPackageNames, HashSet<string>? feedsFromNugetConfigs)
{
var reachableFallbackFeeds = GetReachableFallbackNugetFeeds(feedsFromNugetConfigs);
var reachableFallbackFeeds = feedManager.GetReachableFallbackNugetFeeds(feedsFromNugetConfigs);
compilationInfoContainer.CompilationInfos.Add(("Reachable fallback NuGet feed count", reachableFallbackFeeds.Count.ToString()));
if (reachableFallbackFeeds.Count > 0)
{
return DownloadMissingPackages(usedPackageNames, fallbackNugetFeeds: reachableFallbackFeeds);
@@ -681,147 +605,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
}
private static async Task<HttpResponseMessage> ExecuteGetRequest(string address, HttpClient httpClient, CancellationToken cancellationToken)
{
return await httpClient.GetAsync(address, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
}
private bool IsFeedReachable(string feed, int timeoutMilliSeconds, int tryCount, out bool isTimeout)
{
logger.LogInfo($"Checking if NuGet feed '{feed}' is reachable...");
// Configure the HttpClient to be aware of the Dependabot Proxy, if used.
HttpClientHandler httpClientHandler = new();
if (dependabotProxy != null)
{
httpClientHandler.Proxy = new WebProxy(dependabotProxy.Address);
if (dependabotProxy.Certificate != null)
{
httpClientHandler.ServerCertificateCustomValidationCallback = (message, cert, chain, _) =>
{
if (chain is null || cert is null)
{
var msg = cert is null && chain is null
? "certificate and chain"
: chain is null
? "chain"
: "certificate";
logger.LogWarning($"Dependabot proxy certificate validation failed due to missing {msg}");
return false;
}
chain.ChainPolicy.TrustMode = X509ChainTrustMode.CustomRootTrust;
chain.ChainPolicy.CustomTrustStore.Add(dependabotProxy.Certificate);
return chain.Build(cert);
};
}
}
using HttpClient client = new(httpClientHandler);
isTimeout = false;
for (var i = 0; i < tryCount; i++)
{
using var cts = new CancellationTokenSource();
cts.CancelAfter(timeoutMilliSeconds);
try
{
logger.LogInfo($"Attempt {i + 1}/{tryCount} to reach NuGet feed '{feed}'.");
using var response = ExecuteGetRequest(feed, client, cts.Token).GetAwaiter().GetResult();
response.EnsureSuccessStatusCode();
logger.LogInfo($"Querying NuGet feed '{feed}' succeeded.");
return true;
}
catch (Exception exc)
{
if (exc is TaskCanceledException tce &&
tce.CancellationToken == cts.Token &&
cts.Token.IsCancellationRequested)
{
logger.LogInfo($"Didn't receive answer from NuGet feed '{feed}' in {timeoutMilliSeconds}ms.");
timeoutMilliSeconds *= 2;
continue;
}
logger.LogInfo($"Querying NuGet feed '{feed}' failed. The reason for the failure: {exc.Message}");
return false;
}
}
logger.LogWarning($"Didn't receive answer from NuGet feed '{feed}'. Tried it {tryCount} times.");
isTimeout = true;
return false;
}
private (int initialTimeout, int tryCount) GetFeedRequestSettings(bool isFallback)
{
int timeoutMilliSeconds = isFallback && int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessInitialTimeoutForFallback), out timeoutMilliSeconds)
? timeoutMilliSeconds
: int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessInitialTimeout), out timeoutMilliSeconds)
? timeoutMilliSeconds
: 1000;
logger.LogDebug($"Initial timeout for NuGet feed reachability check is {timeoutMilliSeconds}ms.");
int tryCount = isFallback && int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessRequestCountForFallback), out tryCount)
? tryCount
: int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessRequestCount), out tryCount)
? tryCount
: 4;
logger.LogDebug($"Number of tries for NuGet feed reachability check is {tryCount}.");
return (timeoutMilliSeconds, tryCount);
}
/// <summary>
/// Retrieves a list of excluded NuGet feeds from the corresponding environment variable.
/// </summary>
private HashSet<string> GetExcludedFeeds()
{
var excludedFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.ExcludedNugetFeedsFromResponsivenessCheck)
.ToHashSet();
if (excludedFeeds.Count > 0)
{
logger.LogInfo($"Excluded NuGet feeds from responsiveness check: {string.Join(", ", excludedFeeds.OrderBy(f => f))}");
}
return excludedFeeds;
}
/// <summary>
/// Checks that we can connect to the specified NuGet feeds.
/// </summary>
/// <param name="feeds">The set of package feeds to check.</param>
/// <param name="reachableFeeds">The list of feeds that were reachable.</param>
/// <returns>
/// True if there is a timeout when trying to reach the feeds (excluding any feeds that are configured
/// to be excluded from the check) or false otherwise.
/// </returns>
private bool CheckSpecifiedFeeds(HashSet<string> feeds, out HashSet<string> reachableFeeds)
{
// Exclude any feeds from the feed check that are configured by the corresponding environment variable.
// These feeds are always assumed to be reachable.
var excludedFeeds = GetExcludedFeeds();
HashSet<string> feedsToCheck = feeds.Where(feed =>
{
if (excludedFeeds.Contains(feed))
{
logger.LogInfo($"Not checking reachability of NuGet feed '{feed}' as it is in the list of excluded feeds.");
return false;
}
return true;
}).ToHashSet();
reachableFeeds = GetReachableNuGetFeeds(feedsToCheck, isFallback: false, out var isTimeout).ToHashSet();
// Always consider feeds excluded for the reachability check as reachable.
reachableFeeds.UnionWith(feeds.Where(feed => excludedFeeds.Contains(feed)));
return isTimeout;
}
/// <summary>
/// If <paramref name="allFeedsReachable"/> is `false`, logs this and emits a diagnostic.
/// Adds a `CompilationInfos` entry either way.

2
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/PackagesConfigRestorer.cs
View File

@@ -302,7 +302,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private void AddDefaultPackageSource(string nugetConfig)
{
logger.LogInfo("Adding default package source...");
RunMonoNugetCommand($"sources add -Name DefaultNugetOrg -Source {NugetPackageRestorer.PublicNugetOrgFeed} -ConfigFile \"{nugetConfig}\"", out _);
RunMonoNugetCommand($"sources add -Name DefaultNugetOrg -Source {FeedManager.PublicNugetOrgFeed} -ConfigFile \"{nugetConfig}\"", out _);
}
public void Dispose()