hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-17 01:03:14 +01:00
Code Issues Packages Projects Releases Wiki Activity

C#: Validate all nuget feeds to respond in reasonable time

Browse Source
This commit is contained in:
Tamas Vajk
2024-04-04 14:26:13 +02:00
parent e42639852c
commit 9aa85f2d13
8 changed files with 208 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

191
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs
View File

@@ -2,7 +2,9 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using Semmle.Util;
@@ -14,6 +16,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
try
{
var checkNugetFeedResponsiveness = EnvironmentVariables.GetBoolean(EnvironmentVariableNames.CheckNugetFeedResponsiveness);
if (checkNugetFeedResponsiveness && !CheckFeeds(allNonBinaryFiles))
{
DownloadMissingPackages(allNonBinaryFiles, dllPaths, withNugetConfig: false);
return;
}
using (var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, logger))
{
var count = nuget.InstallPackages();
@@ -139,7 +148,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString()));
}
private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPaths)
private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPaths, bool withNugetConfig = true)
{
var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(packageDirectory.DirInfo);
var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames();
@@ -172,30 +181,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
logger.LogInfo($"Found {notYetDownloadedPackages.Count} packages that are not yet restored");
var nugetConfigs = allFiles.SelectFileNamesByName("nuget.config").ToArray();
string? nugetConfig = null;
if (nugetConfigs.Length > 1)
{
logger.LogInfo($"Found multiple nuget.config files: {string.Join(", ", nugetConfigs)}.");
nugetConfig = allFiles
.SelectRootFiles(sourceDir)
.SelectFileNamesByName("nuget.config")
.FirstOrDefault();
if (nugetConfig == null)
{
logger.LogInfo("Could not find a top-level nuget.config file.");
}
}
else
{
nugetConfig = nugetConfigs.FirstOrDefault();
}
if (nugetConfig != null)
{
logger.LogInfo($"Using nuget.config file {nugetConfig}.");
}
var nugetConfig = withNugetConfig
? GetNugetConfig(allFiles)
: null;
CompilationInfos.Add(("Fallback nuget restore", notYetDownloadedPackages.Count.ToString()));
@@ -221,6 +209,37 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
dllPaths.Add(missingPackageDirectory.DirInfo.FullName);
}
private string[] GetAllNugetConfigs(List<FileInfo> allFiles) => allFiles.SelectFileNamesByName("nuget.config").ToArray();
private string? GetNugetConfig(List<FileInfo> allFiles)
{
var nugetConfigs = GetAllNugetConfigs(allFiles);
string? nugetConfig;
if (nugetConfigs.Length > 1)
{
logger.LogInfo($"Found multiple nuget.config files: {string.Join(", ", nugetConfigs)}.");
nugetConfig = allFiles
.SelectRootFiles(sourceDir)
.SelectFileNamesByName("nuget.config")
.FirstOrDefault();
if (nugetConfig == null)
{
logger.LogInfo("Could not find a top-level nuget.config file.");
}
}
else
{
nugetConfig = nugetConfigs.FirstOrDefault();
}
if (nugetConfig != null)
{
logger.LogInfo($"Using nuget.config file {nugetConfig}.");
}
return nugetConfig;
}
private void LogAllUnusedPackages(DependencyContainer dependencies)
{
var allPackageDirectories = GetAllPackageDirectories();
@@ -279,9 +298,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
.Select(d => Path.GetFileName(d).ToLowerInvariant());
}
[GeneratedRegex(@"<TargetFramework>.*</TargetFramework>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
private static partial Regex TargetFramework();
private bool TryRestorePackageManually(string package, string? nugetConfig, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj)
{
logger.LogInfo($"Restoring package {package}...");
@@ -358,7 +374,126 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
}
private static async Task ExecuteGetRequest(string address, HttpClient httpClient, CancellationToken cancellationToken)
{
using var stream = await httpClient.GetStreamAsync(address, cancellationToken);
var buffer = new byte[1024];
int bytesRead;
while ((bytesRead = stream.Read(buffer, 0, buffer.Length)) > 0)
{
// do nothing
}
}
private bool IsFeedReachable(string feed)
{
using HttpClient client = new();
var timeoutSeconds = 1;
var tryCount = 4;
for (var i = 0; i < tryCount; i++)
{
using var cts = new CancellationTokenSource();
cts.CancelAfter(timeoutSeconds * 1000);
try
{
ExecuteGetRequest(feed, client, cts.Token).GetAwaiter().GetResult();
return true;
}
catch (Exception exc)
{
if (exc is TaskCanceledException tce &&
tce.CancellationToken == cts.Token &&
cts.Token.IsCancellationRequested)
{
logger.LogWarning($"Didn't receive answer from Nuget feed '{feed}' in {timeoutSeconds} seconds.");
timeoutSeconds *= 2;
continue;
}
// We're only interested in timeouts.
logger.LogWarning($"Querying Nuget feed '{feed}' failed: {exc}");
return true;
}
}
logger.LogError($"Didn't receive answer from Nuget feed '{feed}'. Tried it {tryCount} times.");
return false;
}
private bool CheckFeeds(List<FileInfo> allFiles)
{
logger.LogInfo("Checking Nuget feeds...");
var feeds = GetAllFeeds(allFiles);
var excludedFeeds = Environment.GetEnvironmentVariable(EnvironmentVariableNames.ExcludedNugetFeedsFromResponsivenessCheck)
?.Split(Path.PathSeparator, StringSplitOptions.RemoveEmptyEntries)
.ToHashSet() ?? [];
if (excludedFeeds.Count > 0)
{
logger.LogInfo($"Excluded feeds from responsiveness check: {string.Join(", ", excludedFeeds)}");
}
var allFeedsReachable = feeds.All(feed => excludedFeeds.Contains(feed) || IsFeedReachable(feed));
if (!allFeedsReachable)
{
diagnosticsWriter.AddEntry(new DiagnosticMessage(
Language.CSharp,
"buildless/unreachable-feed",
"Found unreachable Nuget feed in C# analysis with build-mode 'none'",
visibility: new DiagnosticMessage.TspVisibility(statusPage: true, cliSummaryTable: true, telemetry: true),
markdownMessage: "Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.",
severity: DiagnosticMessage.TspSeverity.Warning
));
}
CompilationInfos.Add(("All Nuget feeds reachable", allFeedsReachable ? "1" : "0"));
return allFeedsReachable;
}
private IEnumerable<string> GetFeeds(string nugetConfig)
{
logger.LogInfo($"Getting Nuget feeds from '{nugetConfig}'...");
var results = dotnet.GetNugetFeeds(nugetConfig);
var regex = EnabledNugetFeed();
foreach (var result in results)
{
var match = regex.Match(result);
if (!match.Success)
{
logger.LogError($"Failed to parse feed from '{result}'");
continue;
}
var url = match.Groups[1].Value;
if (!url.StartsWith("https://", StringComparison.InvariantCultureIgnoreCase) &&
!url.StartsWith("http://", StringComparison.InvariantCultureIgnoreCase))
{
logger.LogInfo($"Skipping feed '{url}' as it is not a valid URL.");
continue;
}
yield return url;
}
}
private HashSet<string> GetAllFeeds(List<FileInfo> allFiles)
{
var nugetConfigs = GetAllNugetConfigs(allFiles);
var feeds = nugetConfigs
.SelectMany(nf => GetFeeds(nf))
.Where(str => !string.IsNullOrWhiteSpace(str))
.ToHashSet();
return feeds;
}
[GeneratedRegex(@"<TargetFramework>.*</TargetFramework>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
private static partial Regex TargetFramework();
[GeneratedRegex(@"^(.+)\.(\d+\.\d+\.\d+(-(.+))?)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
private static partial Regex LegacyNugetPackage();
[GeneratedRegex(@"^E (.*)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
private static partial Regex EnabledNugetFeed();
}
}

9
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
View File

@@ -20,6 +20,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
private readonly AssemblyCache assemblyCache;
private readonly ILogger logger;
private readonly IDiagnosticsWriter diagnosticsWriter;
// Only used as a set, but ConcurrentDictionary is the only concurrent set in .NET.
private readonly IDictionary<string, bool> usedReferences = new ConcurrentDictionary<string, bool>();
@@ -52,6 +53,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var startTime = DateTime.Now;
this.logger = logger;
this.diagnosticsWriter = new DiagnosticsStream(Path.Combine(
Environment.GetEnvironmentVariable(EnvironmentVariableNames.DiagnosticDir) ?? "",
$"dependency-manager-{DateTime.UtcNow:yyyyMMddHHmm}-{Environment.ProcessId}.jsonc"));
this.sourceDir = new DirectoryInfo(srcDir);
packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName, "packages"));
@@ -177,8 +181,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var frameworkLocations = new HashSet<string>();
var frameworkReferences = Environment.GetEnvironmentVariable(EnvironmentVariableNames.DotnetFrameworkReferences);
var frameworkReferencesUseSubfolders = Environment.GetEnvironmentVariable(EnvironmentVariableNames.DotnetFrameworkReferencesUseSubfolders);
_ = bool.TryParse(frameworkReferencesUseSubfolders, out var useSubfolders);
var useSubfolders = EnvironmentVariables.GetBoolean(EnvironmentVariableNames.DotnetFrameworkReferencesUseSubfolders);
if (!string.IsNullOrWhiteSpace(frameworkReferences))
{
RemoveFrameworkNugetPackages(dllPaths);
@@ -740,6 +743,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
Dispose(tempWorkingDirectory, "temporary working");
}
diagnosticsWriter?.Dispose();
}
}
}

17
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs
View File

@@ -16,12 +16,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
public partial class DotNet : IDotNet
{
private readonly IDotNetCliInvoker dotnetCliInvoker;
private readonly ILogger logger;
private readonly TemporaryDirectory? tempWorkingDirectory;
private DotNet(IDotNetCliInvoker dotnetCliInvoker, ILogger logger, TemporaryDirectory? tempWorkingDirectory = null)
{
this.tempWorkingDirectory = tempWorkingDirectory;
this.dotnetCliInvoker = dotnetCliInvoker;
this.logger = logger;
Info();
}
@@ -89,17 +91,18 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
return dotnetCliInvoker.RunCommand(args);
}
public IList<string> GetListedRuntimes() => GetListed("--list-runtimes");
public IList<string> GetListedRuntimes() => GetResultList("--list-runtimes");
public IList<string> GetListedSdks() => GetListed("--list-sdks");
public IList<string> GetListedSdks() => GetResultList("--list-sdks");
private IList<string> GetListed(string args)
private IList<string> GetResultList(string args)
{
if (dotnetCliInvoker.RunCommand(args, out var artifacts))
if (dotnetCliInvoker.RunCommand(args, out var results))
{
return artifacts;
return results;
}
return new List<string>();
logger.LogWarning($"Running 'dotnet {args}' failed.");
return [];
}
public bool Exec(string execArgs)
@@ -108,6 +111,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
return dotnetCliInvoker.RunCommand(args);
}
public IList<string> GetNugetFeeds(string nugetConfig) => GetResultList($"nuget list source --format Short --configfile \"{nugetConfig}\"");
// The version number should be kept in sync with the version .NET version used for building the application.
public const string LatestDotNetSdkVersion = "8.0.101";

15
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs
View File

@@ -16,5 +16,20 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// Controls whether to use framework dependencies from subfolders.
/// </summary>
public const string DotnetFrameworkReferencesUseSubfolders = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_DOTNET_FRAMEWORK_REFERENCES_USE_SUBFOLDERS";
/// <summary>
/// Controls whether to check the responsiveness of NuGet feeds.
/// </summary>
public const string CheckNugetFeedResponsiveness = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK";
/// <summary>
/// Specifies the NuGet feeds to exclude from the responsiveness check.
/// </summary>
public const string ExcludedNugetFeedsFromResponsivenessCheck = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_EXCLUDED_FROM_CHECK";
/// <summary>
/// Specifies the location of the diagnostic directory.
/// </summary>
public const string DiagnosticDir = "CODEQL_EXTRACTOR_CSHARP_DIAGNOSTIC_DIR";
}
}

1
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs
View File

@@ -13,6 +13,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
IList<string> GetListedRuntimes();
IList<string> GetListedSdks();
bool Exec(string execArgs);
IList<string> GetNugetFeeds(string nugetConfig);
}
public record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, string? PathToNugetConfig = null, bool ForceReevaluation = false);

2
csharp/extractor/Semmle.Extraction.Tests/Runtime.cs
View File

@@ -26,6 +26,8 @@ namespace Semmle.Extraction.Tests
public IList<string> GetListedSdks() => sdks;
public bool Exec(string execArgs) => true;
public IList<string> GetNugetFeeds(string nugetConfig) => [];
}
public class RuntimeTests

7
csharp/extractor/Semmle.Util/EnvironmentVariables.cs
View File

@@ -27,5 +27,12 @@ namespace Semmle.Util
}
return threads;
}
public static bool GetBoolean(string name)
{
var env = Environment.GetEnvironmentVariable(name);
var _ = bool.TryParse(env, out var value);
return value;
}
}
}

5
csharp/extractor/Semmle.Util/FileUtils.cs
View File

@@ -102,8 +102,7 @@ namespace Semmle.Util
private static async Task DownloadFileAsync(string address, string filename)
{
using var httpClient = new HttpClient();
using var request = new HttpRequestMessage(HttpMethod.Get, address);
using var contentStream = await (await httpClient.SendAsync(request)).Content.ReadAsStreamAsync();
using var contentStream = await httpClient.GetStreamAsync(address);
using var stream = new FileStream(filename, FileMode.Create, FileAccess.Write, FileShare.None, 4096, true);
await contentStream.CopyToAsync(stream);
}
@@ -112,7 +111,7 @@ namespace Semmle.Util
/// Downloads the file at <paramref name="address"/> to <paramref name="fileName"/>.
/// </summary>
public static void DownloadFile(string address, string fileName) =>
DownloadFileAsync(address, fileName).Wait();
DownloadFileAsync(address, fileName).GetAwaiter().GetResult();
public static string NestPaths(ILogger logger, string? outerpath, string innerpath)
{