hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-04-26 17:25:19 +02:00
Code Issues Packages Projects Releases Wiki Activity

Fix failing test

Browse Source
This commit is contained in:
Tamas Vajk
2024-04-10 11:27:59 +02:00
parent 0f7fc90fe0
commit 8d0856f97e
4 changed files with 68 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs
View File

@@ -98,11 +98,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
fallbackFeeds.Add(PublicNugetFeed);
}
logger.LogInfo("Checking fallback Nuget feed reachability");
var reachableFallbackFeeds = fallbackFeeds.Where(feed => IsFeedReachable(feed)).ToList();
logger.LogInfo($"Checking fallback Nuget feed reachability on feeds: {string.Join(", ", fallbackFeeds.OrderBy(f => f))}");
var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback: true);
var reachableFallbackFeeds = fallbackFeeds.Where(feed => IsFeedReachable(feed, initialTimeout, tryCount, allowExceptions: false)).ToList();
if (reachableFallbackFeeds.Count == 0)
{
logger.LogWarning("No fallback Nuget feeds are reachable. Skipping fallback Nuget package restoration.");
logger.LogWarning("No fallback Nuget feeds are reachable.");
}
else
{
logger.LogInfo($"Reachable fallback Nuget feeds: {string.Join(", ", reachableFallbackFeeds.OrderBy(f => f))}");
}
return reachableFallbackFeeds;
@@ -183,11 +188,15 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var reachableFallbackFeeds = GetReachableFallbackNugetFeeds();
if (reachableFallbackFeeds.Count > 0)
{
DownloadMissingPackages(allNonBinaryFiles, dllLocations, withNugetConfig: false, fallbackNugetFeeds: reachableFallbackFeeds);
DownloadMissingPackages(allNonBinaryFiles, dllLocations, withNugetConfigFromSrc: false, fallbackNugetFeeds: reachableFallbackFeeds);
}
else
{
logger.LogWarning("Skipping download of missing packages from specific feeds as no fallback Nuget feeds are reachable.");
}
}
private void DownloadMissingPackages(List<FileInfo> allFiles, HashSet<AssemblyLookupLocation> dllLocations, bool withNugetConfig = true, IEnumerable<string>? fallbackNugetFeeds = null)
private void DownloadMissingPackages(List<FileInfo> allFiles, HashSet<AssemblyLookupLocation> dllLocations, bool withNugetConfigFromSrc = true, IEnumerable<string>? fallbackNugetFeeds = null)
{
var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(packageDirectory.DirInfo);
var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames();
@@ -221,7 +230,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
logger.LogInfo($"Found {notYetDownloadedPackages.Count} packages that are not yet restored");
using var tempDir = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName, "nugetconfig"));
var nugetConfig = withNugetConfig
var nugetConfig = withNugetConfigFromSrc
? GetNugetConfig(allFiles)
: CreateFallbackNugetConfig(fallbackNugetFeeds, tempDir.DirInfo.FullName);
@@ -232,7 +241,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = threads }, package =>
{
var success = TryRestorePackageManually(package.Name, nugetConfig, package.PackageReferenceSource);
var success = TryRestorePackageManually(package.Name, nugetConfig, package.PackageReferenceSource, tryWithoutNugetConfig: withNugetConfigFromSrc);
if (!success)
{
return;
@@ -254,6 +263,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (fallbackNugetFeeds is null)
{
// We're not overriding the inherited Nuget feeds
logger.LogInfo("No fallback Nuget feeds provided. Not creating a fallback nuget.config file.");
return null;
}
@@ -365,7 +375,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
.Select(d => Path.GetFileName(d).ToLowerInvariant());
}
private bool TryRestorePackageManually(string package, string? nugetConfig, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj)
private bool TryRestorePackageManually(string package, string? nugetConfig = null, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj, bool tryWithoutNugetConfig = true)
{
logger.LogInfo($"Restoring package {package}...");
using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package, "missingpackages_workingdir"));
@@ -389,7 +399,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig));
if (!res.Success)
{
if (res.HasNugetPackageSourceError && nugetConfig is not null)
if (tryWithoutNugetConfig && res.HasNugetPackageSourceError && nugetConfig is not null)
{
// Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true));
@@ -452,16 +462,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
}
private bool IsFeedReachable(string feed)
private bool IsFeedReachable(string feed, int timeoutMilliSeconds, int tryCount, bool allowExceptions = true)
{
logger.LogInfo($"Checking if Nuget feed '{feed}' is reachable...");
using HttpClient client = new();
int timeoutMilliSeconds = int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessInitialTimeout), out timeoutMilliSeconds)
? timeoutMilliSeconds
: 1000;
int tryCount = int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessRequestCount), out tryCount)
? tryCount
: 4;
for (var i = 0; i < tryCount; i++)
{
@@ -470,6 +474,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
try
{
ExecuteGetRequest(feed, client, cts.Token).GetAwaiter().GetResult();
logger.LogInfo($"Querying Nuget feed '{feed}' succeeded.");
return true;
}
catch (Exception exc)
@@ -478,14 +483,15 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
tce.CancellationToken == cts.Token &&
cts.Token.IsCancellationRequested)
{
logger.LogWarning($"Didn't receive answer from Nuget feed '{feed}' in {timeoutMilliSeconds}ms.");
logger.LogInfo($"Didn't receive answer from Nuget feed '{feed}' in {timeoutMilliSeconds}ms.");
timeoutMilliSeconds *= 2;
continue;
}
// We're only interested in timeouts.
logger.LogWarning($"Querying Nuget feed '{feed}' failed: {exc}");
return true;
var start = allowExceptions ? "Considering" : "Not considering";
logger.LogInfo($"Querying Nuget feed '{feed}' failed in a timely manner. {start} the feed for use. The reason for the failure: {exc.Message}");
return allowExceptions;
}
}
@@ -493,6 +499,25 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
return false;
}
private (int initialTimeout, int tryCount) GetFeedRequestSettings(bool isFallback)
{
int timeoutMilliSeconds = isFallback && int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessInitialTimeoutForFallback), out timeoutMilliSeconds)
? timeoutMilliSeconds
: int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessInitialTimeout), out timeoutMilliSeconds)
? timeoutMilliSeconds
: 1000;
logger.LogDebug($"Initial timeout for Nuget feed reachability check is {timeoutMilliSeconds}ms.");
int tryCount = isFallback && int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessRequestCountForFallback), out tryCount)
? tryCount
: int.TryParse(Environment.GetEnvironmentVariable(EnvironmentVariableNames.NugetFeedResponsivenessRequestCount), out tryCount)
? tryCount
: 4;
logger.LogDebug($"Number of tries for Nuget feed reachability check is {tryCount}.");
return (timeoutMilliSeconds, tryCount);
}
private bool CheckFeeds(List<FileInfo> allFiles)
{
logger.LogInfo("Checking Nuget feeds...");
@@ -507,7 +532,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
logger.LogInfo($"Excluded Nuget feeds from responsiveness check: {string.Join(", ", excludedFeeds.OrderBy(f => f))}");
}
var allFeedsReachable = explicitFeeds.All(feed => excludedFeeds.Contains(feed) || IsFeedReachable(feed));
var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback: false);
var allFeedsReachable = explicitFeeds.All(feed => excludedFeeds.Contains(feed) || IsFeedReachable(feed, initialTimeout, tryCount));
if (!allFeedsReachable)
{
logger.LogWarning("Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.");

2
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
View File

@@ -339,7 +339,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
logger.LogInfo("No .NET Desktop Runtime location found. Attempting to restore the .NET Framework reference assemblies manually.");
if (TryRestorePackageManually(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies, null))
if (TryRestorePackageManually(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies))
{
runtimeLocation = GetPackageDirectory(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies, missingPackageDirectory);
}

14
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs
View File

@@ -33,10 +33,22 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
public const string NugetFeedResponsivenessInitialTimeout = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_TIMEOUT";
/// <summary>
/// Specifies how many requests to make to the NuGet feed to check its responsiveness.
/// Specifies the timeout (as an integer) in milliseconds for the initial check of fallback NuGet feeds responsiveness. The value is then doubled for each subsequent check.
/// This is primarily used in testing.
/// </summary>
internal const string NugetFeedResponsivenessInitialTimeoutForFallback = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_FALLBACK_TIMEOUT";
/// <summary>
/// Specifies how many requests to make to the NuGet feeds to check their responsiveness.
/// </summary>
public const string NugetFeedResponsivenessRequestCount = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_LIMIT";
/// <summary>
/// Specifies how many requests to make to the fallback NuGet feeds to check their responsiveness.
/// This is primarily used in testing.
/// </summary>
internal const string NugetFeedResponsivenessRequestCountForFallback = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_FALLBACK_LIMIT";
/// <summary>
/// Specifies the NuGet feeds to use for fallback Nuget dependency fetching. The value is a space-separated list of feed URLs.
/// The default value is `https://api.nuget.org/v3/index.json`.