From 0f7fc90fe03fe1e3fb1c059429ee7cbef04532b7 Mon Sep 17 00:00:00 2001 From: Tamas Vajk Date: Tue, 9 Apr 2024 15:01:20 +0200 Subject: [PATCH] C#: Check fallback nuget feeds before trying to use them in the fallback restore process --- .../DependencyManager.Nuget.cs | 149 +++++++++++++++--- .../DotNet.cs | 10 +- .../DotNetCliInvoker.cs | 22 ++- .../EnvironmentVariableNames.cs | 6 + .../IDotNet.cs | 1 + .../IDotNetCliInvoker.cs | 6 + .../NugetPackages.cs | 2 +- .../Semmle.Extraction.Tests/DotNet.cs | 7 + .../Semmle.Extraction.Tests/Runtime.cs | 2 + .../Semmle.Util/EnvironmentVariables.cs | 6 + 10 files changed, 180 insertions(+), 31 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs index cf9c7253b17..e65a4c10f6b 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; using System.Net.Http; +using System.Text; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; @@ -14,12 +15,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { private void RestoreNugetPackages(List allNonBinaryFiles, IEnumerable allProjects, IEnumerable allSolutions, HashSet dllLocations) { + var checkNugetFeedResponsiveness = EnvironmentVariables.GetBoolean(EnvironmentVariableNames.CheckNugetFeedResponsiveness); try { - var checkNugetFeedResponsiveness = EnvironmentVariables.GetBoolean(EnvironmentVariableNames.CheckNugetFeedResponsiveness); if (checkNugetFeedResponsiveness && !CheckFeeds(allNonBinaryFiles)) { - DownloadMissingPackages(allNonBinaryFiles, dllLocations, withNugetConfig: false); + // todo: we could also check the reachability of the inherited nuget feeds, but to use those in the fallback we would need to handle authentication too. + DownloadMissingPackagesFromSpecificFeeds(allNonBinaryFiles, dllLocations); return; } @@ -75,7 +77,35 @@ namespace Semmle.Extraction.CSharp.DependencyFetching dllLocations.UnionWith(paths.Select(p => new AssemblyLookupLocation(p))); LogAllUnusedPackages(dependencies); - DownloadMissingPackages(allNonBinaryFiles, dllLocations); + + if (checkNugetFeedResponsiveness) + { + DownloadMissingPackagesFromSpecificFeeds(allNonBinaryFiles, dllLocations); + } + else + { + DownloadMissingPackages(allNonBinaryFiles, dllLocations); + } + } + + internal const string PublicNugetFeed = "https://api.nuget.org/v3/index.json"; + + private List GetReachableFallbackNugetFeeds() + { + var fallbackFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.FallbackNugetFeeds).ToHashSet(); + if (fallbackFeeds.Count == 0) + { + fallbackFeeds.Add(PublicNugetFeed); + } + + logger.LogInfo("Checking fallback Nuget feed reachability"); + var reachableFallbackFeeds = fallbackFeeds.Where(feed => IsFeedReachable(feed)).ToList(); + if (reachableFallbackFeeds.Count == 0) + { + logger.LogWarning("No fallback Nuget feeds are reachable. Skipping fallback Nuget package restoration."); + } + + return reachableFallbackFeeds; } /// @@ -148,7 +178,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString())); } - private void DownloadMissingPackages(List allFiles, ISet dllLocations, bool withNugetConfig = true) + private void DownloadMissingPackagesFromSpecificFeeds(List allNonBinaryFiles, HashSet dllLocations) + { + var reachableFallbackFeeds = GetReachableFallbackNugetFeeds(); + if (reachableFallbackFeeds.Count > 0) + { + DownloadMissingPackages(allNonBinaryFiles, dllLocations, withNugetConfig: false, fallbackNugetFeeds: reachableFallbackFeeds); + } + } + + private void DownloadMissingPackages(List allFiles, HashSet dllLocations, bool withNugetConfig = true, IEnumerable? fallbackNugetFeeds = null) { var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(packageDirectory.DirInfo); var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames(); @@ -181,9 +220,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } logger.LogInfo($"Found {notYetDownloadedPackages.Count} packages that are not yet restored"); + using var tempDir = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName, "nugetconfig")); var nugetConfig = withNugetConfig ? GetNugetConfig(allFiles) - : null; + : CreateFallbackNugetConfig(fallbackNugetFeeds, tempDir.DirInfo.FullName); CompilationInfos.Add(("Fallback nuget restore", notYetDownloadedPackages.Count.ToString())); @@ -209,6 +249,33 @@ namespace Semmle.Extraction.CSharp.DependencyFetching dllLocations.Add(missingPackageDirectory.DirInfo.FullName); } + private string? CreateFallbackNugetConfig(IEnumerable? fallbackNugetFeeds, string folderPath) + { + if (fallbackNugetFeeds is null) + { + // We're not overriding the inherited Nuget feeds + return null; + } + + var sb = new StringBuilder(); + fallbackNugetFeeds.ForEach((feed, index) => sb.AppendLine($"")); + + var nugetConfigPath = Path.Combine(folderPath, "nuget.config"); + logger.LogInfo($"Creating fallback nuget.config file {nugetConfigPath}."); + File.WriteAllText(nugetConfigPath, + $""" + + + + + {sb} + + + """); + + return nugetConfigPath; + } + private string[] GetAllNugetConfigs(List allFiles) => allFiles.SelectFileNamesByName("nuget.config").ToArray(); private string? GetNugetConfig(List allFiles) @@ -429,10 +496,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching private bool CheckFeeds(List allFiles) { logger.LogInfo("Checking Nuget feeds..."); - var feeds = GetAllFeeds(allFiles); + var (explicitFeeds, allFeeds) = GetAllFeeds(allFiles); + var inheritedFeeds = allFeeds.Except(explicitFeeds).ToHashSet(); - var excludedFeeds = Environment.GetEnvironmentVariable(EnvironmentVariableNames.ExcludedNugetFeedsFromResponsivenessCheck) - ?.Split(" ", StringSplitOptions.RemoveEmptyEntries) + var excludedFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.ExcludedNugetFeedsFromResponsivenessCheck) .ToHashSet() ?? []; if (excludedFeeds.Count > 0) @@ -440,7 +507,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching logger.LogInfo($"Excluded Nuget feeds from responsiveness check: {string.Join(", ", excludedFeeds.OrderBy(f => f))}"); } - var allFeedsReachable = feeds.All(feed => excludedFeeds.Contains(feed) || IsFeedReachable(feed)); + var allFeedsReachable = explicitFeeds.All(feed => excludedFeeds.Contains(feed) || IsFeedReachable(feed)); if (!allFeedsReachable) { logger.LogWarning("Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis."); @@ -454,13 +521,19 @@ namespace Semmle.Extraction.CSharp.DependencyFetching )); } CompilationInfos.Add(("All Nuget feeds reachable", allFeedsReachable ? "1" : "0")); + + if (inheritedFeeds.Count > 0) + { + logger.LogInfo($"Inherited Nuget feeds: {string.Join(", ", inheritedFeeds.OrderBy(f => f))}"); + CompilationInfos.Add(("Inherited Nuget feed count", inheritedFeeds.Count.ToString())); + } + return allFeedsReachable; } - private IEnumerable GetFeeds(string nugetConfig) + private IEnumerable GetFeeds(Func> getNugetFeeds) { - logger.LogInfo($"Getting Nuget feeds from '{nugetConfig}'..."); - var results = dotnet.GetNugetFeeds(nugetConfig); + var results = getNugetFeeds(); var regex = EnabledNugetFeed(); foreach (var result in results) { @@ -479,27 +552,63 @@ namespace Semmle.Extraction.CSharp.DependencyFetching continue; } - yield return url; + if (!string.IsNullOrWhiteSpace(url)) + { + yield return url; + } } } - private HashSet GetAllFeeds(List allFiles) + private (HashSet, HashSet) GetAllFeeds(List allFiles) { + IList GetNugetFeeds(string nugetConfig) + { + logger.LogInfo($"Getting Nuget feeds from '{nugetConfig}'..."); + return dotnet.GetNugetFeeds(nugetConfig); + } + + IList GetNugetFeedsFromFolder(string folderPath) + { + logger.LogInfo($"Getting Nuget feeds in folder '{folderPath}'..."); + return dotnet.GetNugetFeedsFromFolder(folderPath); + } + var nugetConfigs = GetAllNugetConfigs(allFiles); - var feeds = nugetConfigs - .SelectMany(GetFeeds) - .Where(str => !string.IsNullOrWhiteSpace(str)) + var explicitFeeds = nugetConfigs + .SelectMany(config => GetFeeds(() => GetNugetFeeds(config))) .ToHashSet(); - if (feeds.Count > 0) + if (explicitFeeds.Count > 0) { - logger.LogInfo($"Found {feeds.Count} Nuget feeds in nuget.config files: {string.Join(", ", feeds.OrderBy(f => f))}"); + logger.LogInfo($"Found {explicitFeeds.Count} Nuget feeds in nuget.config files: {string.Join(", ", explicitFeeds.OrderBy(f => f))}"); } else { logger.LogDebug("No Nuget feeds found in nuget.config files."); } - return feeds; + + // todo: this could be improved. + // We don't have to get the feeds from each of the folders from below, it would be enought to check the folders that recursively contain the others. + var allFeeds = nugetConfigs + .Select(config => + { + try + { + return new FileInfo(config).Directory?.FullName; + } + catch (Exception exc) + { + logger.LogWarning($"Failed to get directory of '{config}': {exc}"); + } + return null; + }) + .Where(folder => folder != null) + .SelectMany(folder => GetFeeds(() => GetNugetFeedsFromFolder(folder!))) + .ToHashSet(); + + logger.LogInfo($"Found {allFeeds.Count} Nuget feeds (with inherited ones) in nuget.config files: {string.Join(", ", allFeeds.OrderBy(f => f))}"); + + return (explicitFeeds, allFeeds); } [GeneratedRegex(@".*", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)] diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs index c57958845f2..bda99f8541b 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs @@ -95,9 +95,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching public IList GetListedSdks() => GetResultList("--list-sdks"); - private IList GetResultList(string args) + private IList GetResultList(string args, string? workingDirectory = null) { - if (dotnetCliInvoker.RunCommand(args, out var results)) + if (dotnetCliInvoker.RunCommand(args, workingDirectory, out var results)) { return results; } @@ -111,7 +111,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching return dotnetCliInvoker.RunCommand(args); } - public IList GetNugetFeeds(string nugetConfig) => GetResultList($"nuget list source --format Short --configfile \"{nugetConfig}\""); + private const string nugetListSourceCommand = "nuget list source --format Short"; + + public IList GetNugetFeeds(string nugetConfig) => GetResultList($"{nugetListSourceCommand} --configfile \"{nugetConfig}\""); + + public IList GetNugetFeedsFromFolder(string folderPath) => GetResultList(nugetListSourceCommand, folderPath); // The version number should be kept in sync with the version .NET version used for building the application. public const string LatestDotNetSdkVersion = "8.0.101"; diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs index d7278723784..737c73b635e 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs @@ -21,7 +21,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching this.Exec = exec; } - private ProcessStartInfo MakeDotnetStartInfo(string args) + private ProcessStartInfo MakeDotnetStartInfo(string args, string? workingDirectory) { var startInfo = new ProcessStartInfo(Exec, args) { @@ -29,6 +29,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching RedirectStandardOutput = true, RedirectStandardError = true }; + if (!string.IsNullOrWhiteSpace(workingDirectory)) + { + startInfo.WorkingDirectory = workingDirectory; + } // Set the .NET CLI language to English to avoid localized output. startInfo.EnvironmentVariables["DOTNET_CLI_UI_LANGUAGE"] = "en"; startInfo.EnvironmentVariables["MSBUILDDISABLENODEREUSE"] = "1"; @@ -36,26 +40,30 @@ namespace Semmle.Extraction.CSharp.DependencyFetching return startInfo; } - private bool RunCommandAux(string args, out IList output) + private bool RunCommandAux(string args, string? workingDirectory, out IList output) { - logger.LogInfo($"Running {Exec} {args}"); - var pi = MakeDotnetStartInfo(args); + var dirLog = string.IsNullOrWhiteSpace(workingDirectory) ? "" : $" in {workingDirectory}"; + logger.LogInfo($"Running {Exec} {args}{dirLog}"); + var pi = MakeDotnetStartInfo(args, workingDirectory); var threadId = Environment.CurrentManagedThreadId; void onOut(string s) => logger.LogInfo(s, threadId); void onError(string s) => logger.LogError(s, threadId); var exitCode = pi.ReadOutput(out output, onOut, onError); if (exitCode != 0) { - logger.LogError($"Command {Exec} {args} failed with exit code {exitCode}"); + logger.LogError($"Command {Exec} {args}{dirLog} failed with exit code {exitCode}"); return false; } return true; } public bool RunCommand(string args) => - RunCommandAux(args, out _); + RunCommandAux(args, null, out _); public bool RunCommand(string args, out IList output) => - RunCommandAux(args, out output); + RunCommandAux(args, null, out output); + + public bool RunCommand(string args, string? workingDirectory, out IList output) => + RunCommandAux(args, workingDirectory, out output); } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs index 9141dc0bf74..3eaa0b031bf 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs @@ -37,6 +37,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// public const string NugetFeedResponsivenessRequestCount = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_LIMIT"; + /// + /// Specifies the NuGet feeds to use for fallback Nuget dependency fetching. The value is a space-separated list of feed URLs. + /// The default value is `https://api.nuget.org/v3/index.json`. + /// + public const string FallbackNugetFeeds = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_FALLBACK"; + /// /// Specifies the location of the diagnostic directory. /// diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs index d97fc7d6441..7126ac388ea 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs @@ -14,6 +14,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching IList GetListedSdks(); bool Exec(string execArgs); IList GetNugetFeeds(string nugetConfig); + IList GetNugetFeedsFromFolder(string folderPath); } public record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, string? PathToNugetConfig = null, bool ForceReevaluation = false); diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNetCliInvoker.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNetCliInvoker.cs index aad4a28a401..0e35c0574ae 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNetCliInvoker.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNetCliInvoker.cs @@ -19,5 +19,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// The output of the command is returned in `output`. /// bool RunCommand(string args, out IList output); + + /// + /// Execute `dotnet ` in `` and return true if the command succeeded, otherwise false. + /// The output of the command is returned in `output`. + /// + bool RunCommand(string args, string? workingDirectory, out IList output); } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackages.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackages.cs index 2183daf09cc..4926b64acd3 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackages.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackages.cs @@ -243,7 +243,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching private void AddDefaultPackageSource(string nugetConfig) { logger.LogInfo("Adding default package source..."); - RunMonoNugetCommand($"sources add -Name DefaultNugetOrg -Source https://api.nuget.org/v3/index.json -ConfigFile \"{nugetConfig}\"", out var _); + RunMonoNugetCommand($"sources add -Name DefaultNugetOrg -Source {DependencyManager.PublicNugetFeed} -ConfigFile \"{nugetConfig}\"", out var _); } public void Dispose() diff --git a/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs b/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs index 289d9efbba4..29961f37a1e 100644 --- a/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs +++ b/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs @@ -10,6 +10,7 @@ namespace Semmle.Extraction.Tests { private readonly IList output; private string lastArgs = ""; + public string WorkingDirectory { get; private set; } = ""; public bool Success { get; set; } = true; public DotNetCliInvokerStub(IList output) @@ -32,6 +33,12 @@ namespace Semmle.Extraction.Tests return Success; } + public bool RunCommand(string args, string? workingDirectory, out IList output) + { + WorkingDirectory = workingDirectory ?? ""; + return RunCommand(args, out output); + } + public string GetLastArgs() => lastArgs; } diff --git a/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs b/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs index 2daf8244d97..0cbd853e736 100644 --- a/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs +++ b/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs @@ -28,6 +28,8 @@ namespace Semmle.Extraction.Tests public bool Exec(string execArgs) => true; public IList GetNugetFeeds(string nugetConfig) => []; + + public IList GetNugetFeedsFromFolder(string folderPath) => []; } public class RuntimeTests diff --git a/csharp/extractor/Semmle.Util/EnvironmentVariables.cs b/csharp/extractor/Semmle.Util/EnvironmentVariables.cs index 72ba9224669..c8c272e94ad 100644 --- a/csharp/extractor/Semmle.Util/EnvironmentVariables.cs +++ b/csharp/extractor/Semmle.Util/EnvironmentVariables.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Globalization; using System.Numerics; @@ -34,5 +35,10 @@ namespace Semmle.Util var _ = bool.TryParse(env, out var value); return value; } + + public static IEnumerable GetURLs(string name) + { + return Environment.GetEnvironmentVariable(name)?.Split(" ", StringSplitOptions.RemoveEmptyEntries) ?? []; + } } }