diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs index 8e0be8ab141..cdf4f5dcf4a 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs @@ -2,7 +2,9 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Net.Http; using System.Text.RegularExpressions; +using System.Threading; using System.Threading.Tasks; using Semmle.Util; @@ -14,6 +16,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { try { + var checkNugetFeedResponsiveness = EnvironmentVariables.GetBoolean(EnvironmentVariableNames.CheckNugetFeedResponsiveness); + if (checkNugetFeedResponsiveness && !CheckFeeds(allNonBinaryFiles)) + { + DownloadMissingPackages(allNonBinaryFiles, dllPaths, withNugetConfig: false); + return; + } + using (var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, logger)) { var count = nuget.InstallPackages(); @@ -139,7 +148,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString())); } - private void DownloadMissingPackages(List allFiles, ISet dllPaths) + private void DownloadMissingPackages(List allFiles, ISet dllPaths, bool withNugetConfig = true) { var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(packageDirectory.DirInfo); var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames(); @@ -172,30 +181,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } logger.LogInfo($"Found {notYetDownloadedPackages.Count} packages that are not yet restored"); - - var nugetConfigs = allFiles.SelectFileNamesByName("nuget.config").ToArray(); - string? nugetConfig = null; - if (nugetConfigs.Length > 1) - { - logger.LogInfo($"Found multiple nuget.config files: {string.Join(", ", nugetConfigs)}."); - nugetConfig = allFiles - .SelectRootFiles(sourceDir) - .SelectFileNamesByName("nuget.config") - .FirstOrDefault(); - if (nugetConfig == null) - { - logger.LogInfo("Could not find a top-level nuget.config file."); - } - } - else - { - nugetConfig = nugetConfigs.FirstOrDefault(); - } - - if (nugetConfig != null) - { - logger.LogInfo($"Using nuget.config file {nugetConfig}."); - } + var nugetConfig = withNugetConfig + ? GetNugetConfig(allFiles) + : null; CompilationInfos.Add(("Fallback nuget restore", notYetDownloadedPackages.Count.ToString())); @@ -221,6 +209,37 @@ namespace Semmle.Extraction.CSharp.DependencyFetching dllPaths.Add(missingPackageDirectory.DirInfo.FullName); } + private string[] GetAllNugetConfigs(List allFiles) => allFiles.SelectFileNamesByName("nuget.config").ToArray(); + + private string? GetNugetConfig(List allFiles) + { + var nugetConfigs = GetAllNugetConfigs(allFiles); + string? nugetConfig; + if (nugetConfigs.Length > 1) + { + logger.LogInfo($"Found multiple nuget.config files: {string.Join(", ", nugetConfigs)}."); + nugetConfig = allFiles + .SelectRootFiles(sourceDir) + .SelectFileNamesByName("nuget.config") + .FirstOrDefault(); + if (nugetConfig == null) + { + logger.LogInfo("Could not find a top-level nuget.config file."); + } + } + else + { + nugetConfig = nugetConfigs.FirstOrDefault(); + } + + if (nugetConfig != null) + { + logger.LogInfo($"Using nuget.config file {nugetConfig}."); + } + + return nugetConfig; + } + private void LogAllUnusedPackages(DependencyContainer dependencies) { var allPackageDirectories = GetAllPackageDirectories(); @@ -279,9 +298,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching .Select(d => Path.GetFileName(d).ToLowerInvariant()); } - [GeneratedRegex(@".*", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)] - private static partial Regex TargetFramework(); - private bool TryRestorePackageManually(string package, string? nugetConfig, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj) { logger.LogInfo($"Restoring package {package}..."); @@ -358,7 +374,126 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } } + private static async Task ExecuteGetRequest(string address, HttpClient httpClient, CancellationToken cancellationToken) + { + using var stream = await httpClient.GetStreamAsync(address, cancellationToken); + var buffer = new byte[1024]; + int bytesRead; + while ((bytesRead = stream.Read(buffer, 0, buffer.Length)) > 0) + { + // do nothing + } + } + + private bool IsFeedReachable(string feed) + { + using HttpClient client = new(); + var timeoutSeconds = 1; + var tryCount = 4; + + for (var i = 0; i < tryCount; i++) + { + using var cts = new CancellationTokenSource(); + cts.CancelAfter(timeoutSeconds * 1000); + try + { + ExecuteGetRequest(feed, client, cts.Token).GetAwaiter().GetResult(); + return true; + } + catch (Exception exc) + { + if (exc is TaskCanceledException tce && + tce.CancellationToken == cts.Token && + cts.Token.IsCancellationRequested) + { + logger.LogWarning($"Didn't receive answer from Nuget feed '{feed}' in {timeoutSeconds} seconds."); + timeoutSeconds *= 2; + continue; + } + + // We're only interested in timeouts. + logger.LogWarning($"Querying Nuget feed '{feed}' failed: {exc}"); + return true; + } + } + + logger.LogError($"Didn't receive answer from Nuget feed '{feed}'. Tried it {tryCount} times."); + return false; + } + + private bool CheckFeeds(List allFiles) + { + logger.LogInfo("Checking Nuget feeds..."); + var feeds = GetAllFeeds(allFiles); + + var excludedFeeds = Environment.GetEnvironmentVariable(EnvironmentVariableNames.ExcludedNugetFeedsFromResponsivenessCheck) + ?.Split(Path.PathSeparator, StringSplitOptions.RemoveEmptyEntries) + .ToHashSet() ?? []; + + if (excludedFeeds.Count > 0) + { + logger.LogInfo($"Excluded feeds from responsiveness check: {string.Join(", ", excludedFeeds)}"); + } + + var allFeedsReachable = feeds.All(feed => excludedFeeds.Contains(feed) || IsFeedReachable(feed)); + if (!allFeedsReachable) + { + diagnosticsWriter.AddEntry(new DiagnosticMessage( + Language.CSharp, + "buildless/unreachable-feed", + "Found unreachable Nuget feed in C# analysis with build-mode 'none'", + visibility: new DiagnosticMessage.TspVisibility(statusPage: true, cliSummaryTable: true, telemetry: true), + markdownMessage: "Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.", + severity: DiagnosticMessage.TspSeverity.Warning + )); + } + CompilationInfos.Add(("All Nuget feeds reachable", allFeedsReachable ? "1" : "0")); + return allFeedsReachable; + } + + private IEnumerable GetFeeds(string nugetConfig) + { + logger.LogInfo($"Getting Nuget feeds from '{nugetConfig}'..."); + var results = dotnet.GetNugetFeeds(nugetConfig); + var regex = EnabledNugetFeed(); + foreach (var result in results) + { + var match = regex.Match(result); + if (!match.Success) + { + logger.LogError($"Failed to parse feed from '{result}'"); + continue; + } + + var url = match.Groups[1].Value; + if (!url.StartsWith("https://", StringComparison.InvariantCultureIgnoreCase) && + !url.StartsWith("http://", StringComparison.InvariantCultureIgnoreCase)) + { + logger.LogInfo($"Skipping feed '{url}' as it is not a valid URL."); + continue; + } + + yield return url; + } + } + + private HashSet GetAllFeeds(List allFiles) + { + var nugetConfigs = GetAllNugetConfigs(allFiles); + var feeds = nugetConfigs + .SelectMany(nf => GetFeeds(nf)) + .Where(str => !string.IsNullOrWhiteSpace(str)) + .ToHashSet(); + return feeds; + } + + [GeneratedRegex(@".*", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)] + private static partial Regex TargetFramework(); + [GeneratedRegex(@"^(.+)\.(\d+\.\d+\.\d+(-(.+))?)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)] private static partial Regex LegacyNugetPackage(); + + [GeneratedRegex(@"^E (.*)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)] + private static partial Regex EnabledNugetFeed(); } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs index 04716a64db4..2fc77d004f2 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs @@ -20,6 +20,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { private readonly AssemblyCache assemblyCache; private readonly ILogger logger; + private readonly IDiagnosticsWriter diagnosticsWriter; // Only used as a set, but ConcurrentDictionary is the only concurrent set in .NET. private readonly IDictionary usedReferences = new ConcurrentDictionary(); @@ -52,6 +53,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching var startTime = DateTime.Now; this.logger = logger; + this.diagnosticsWriter = new DiagnosticsStream(Path.Combine( + Environment.GetEnvironmentVariable(EnvironmentVariableNames.DiagnosticDir) ?? "", + $"dependency-manager-{DateTime.UtcNow:yyyyMMddHHmm}-{Environment.ProcessId}.jsonc")); this.sourceDir = new DirectoryInfo(srcDir); packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName, "packages")); @@ -177,8 +181,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching var frameworkLocations = new HashSet(); var frameworkReferences = Environment.GetEnvironmentVariable(EnvironmentVariableNames.DotnetFrameworkReferences); - var frameworkReferencesUseSubfolders = Environment.GetEnvironmentVariable(EnvironmentVariableNames.DotnetFrameworkReferencesUseSubfolders); - _ = bool.TryParse(frameworkReferencesUseSubfolders, out var useSubfolders); + var useSubfolders = EnvironmentVariables.GetBoolean(EnvironmentVariableNames.DotnetFrameworkReferencesUseSubfolders); if (!string.IsNullOrWhiteSpace(frameworkReferences)) { RemoveFrameworkNugetPackages(dllPaths); @@ -740,6 +743,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { Dispose(tempWorkingDirectory, "temporary working"); } + + diagnosticsWriter?.Dispose(); } } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs index b132d1884f9..c57958845f2 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs @@ -16,12 +16,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching public partial class DotNet : IDotNet { private readonly IDotNetCliInvoker dotnetCliInvoker; + private readonly ILogger logger; private readonly TemporaryDirectory? tempWorkingDirectory; private DotNet(IDotNetCliInvoker dotnetCliInvoker, ILogger logger, TemporaryDirectory? tempWorkingDirectory = null) { this.tempWorkingDirectory = tempWorkingDirectory; this.dotnetCliInvoker = dotnetCliInvoker; + this.logger = logger; Info(); } @@ -89,17 +91,18 @@ namespace Semmle.Extraction.CSharp.DependencyFetching return dotnetCliInvoker.RunCommand(args); } - public IList GetListedRuntimes() => GetListed("--list-runtimes"); + public IList GetListedRuntimes() => GetResultList("--list-runtimes"); - public IList GetListedSdks() => GetListed("--list-sdks"); + public IList GetListedSdks() => GetResultList("--list-sdks"); - private IList GetListed(string args) + private IList GetResultList(string args) { - if (dotnetCliInvoker.RunCommand(args, out var artifacts)) + if (dotnetCliInvoker.RunCommand(args, out var results)) { - return artifacts; + return results; } - return new List(); + logger.LogWarning($"Running 'dotnet {args}' failed."); + return []; } public bool Exec(string execArgs) @@ -108,6 +111,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching return dotnetCliInvoker.RunCommand(args); } + public IList GetNugetFeeds(string nugetConfig) => GetResultList($"nuget list source --format Short --configfile \"{nugetConfig}\""); + // The version number should be kept in sync with the version .NET version used for building the application. public const string LatestDotNetSdkVersion = "8.0.101"; diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs index 65a4664e83e..2d36319042a 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs @@ -16,5 +16,20 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// Controls whether to use framework dependencies from subfolders. /// public const string DotnetFrameworkReferencesUseSubfolders = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_DOTNET_FRAMEWORK_REFERENCES_USE_SUBFOLDERS"; + + /// + /// Controls whether to check the responsiveness of NuGet feeds. + /// + public const string CheckNugetFeedResponsiveness = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK"; + + /// + /// Specifies the NuGet feeds to exclude from the responsiveness check. + /// + public const string ExcludedNugetFeedsFromResponsivenessCheck = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_EXCLUDED_FROM_CHECK"; + + /// + /// Specifies the location of the diagnostic directory. + /// + public const string DiagnosticDir = "CODEQL_EXTRACTOR_CSHARP_DIAGNOSTIC_DIR"; } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs index d66135c1644..d97fc7d6441 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs @@ -13,6 +13,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching IList GetListedRuntimes(); IList GetListedSdks(); bool Exec(string execArgs); + IList GetNugetFeeds(string nugetConfig); } public record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, string? PathToNugetConfig = null, bool ForceReevaluation = false); diff --git a/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs b/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs index 17bc477bde8..2daf8244d97 100644 --- a/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs +++ b/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs @@ -26,6 +26,8 @@ namespace Semmle.Extraction.Tests public IList GetListedSdks() => sdks; public bool Exec(string execArgs) => true; + + public IList GetNugetFeeds(string nugetConfig) => []; } public class RuntimeTests diff --git a/csharp/extractor/Semmle.Util/EnvironmentVariables.cs b/csharp/extractor/Semmle.Util/EnvironmentVariables.cs index c96aa16357c..72ba9224669 100644 --- a/csharp/extractor/Semmle.Util/EnvironmentVariables.cs +++ b/csharp/extractor/Semmle.Util/EnvironmentVariables.cs @@ -27,5 +27,12 @@ namespace Semmle.Util } return threads; } + + public static bool GetBoolean(string name) + { + var env = Environment.GetEnvironmentVariable(name); + var _ = bool.TryParse(env, out var value); + return value; + } } } diff --git a/csharp/extractor/Semmle.Util/FileUtils.cs b/csharp/extractor/Semmle.Util/FileUtils.cs index 094c4da3338..4a22877e3c1 100644 --- a/csharp/extractor/Semmle.Util/FileUtils.cs +++ b/csharp/extractor/Semmle.Util/FileUtils.cs @@ -102,8 +102,7 @@ namespace Semmle.Util private static async Task DownloadFileAsync(string address, string filename) { using var httpClient = new HttpClient(); - using var request = new HttpRequestMessage(HttpMethod.Get, address); - using var contentStream = await (await httpClient.SendAsync(request)).Content.ReadAsStreamAsync(); + using var contentStream = await httpClient.GetStreamAsync(address); using var stream = new FileStream(filename, FileMode.Create, FileAccess.Write, FileShare.None, 4096, true); await contentStream.CopyToAsync(stream); } @@ -112,7 +111,7 @@ namespace Semmle.Util /// Downloads the file at to . /// public static void DownloadFile(string address, string fileName) => - DownloadFileAsync(address, fileName).Wait(); + DownloadFileAsync(address, fileName).GetAwaiter().GetResult(); public static string NestPaths(ILogger logger, string? outerpath, string innerpath) {