hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-16 16:53:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

C#: Check fallback nuget feeds before trying to use them in the fallback restore process

Browse Source
This commit is contained in:
Tamas Vajk
2024-04-09 15:01:20 +02:00
parent 161f586510
commit 0f7fc90fe0
10 changed files with 180 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

149
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs
View File

@@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
@@ -14,12 +15,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
private void RestoreNugetPackages(List<FileInfo> allNonBinaryFiles, IEnumerable<string> allProjects, IEnumerable<string> allSolutions, HashSet<AssemblyLookupLocation> dllLocations)
{
var checkNugetFeedResponsiveness = EnvironmentVariables.GetBoolean(EnvironmentVariableNames.CheckNugetFeedResponsiveness);
try
{
var checkNugetFeedResponsiveness = EnvironmentVariables.GetBoolean(EnvironmentVariableNames.CheckNugetFeedResponsiveness);
if (checkNugetFeedResponsiveness && !CheckFeeds(allNonBinaryFiles))
{
DownloadMissingPackages(allNonBinaryFiles, dllLocations, withNugetConfig: false);
// todo: we could also check the reachability of the inherited nuget feeds, but to use those in the fallback we would need to handle authentication too.
DownloadMissingPackagesFromSpecificFeeds(allNonBinaryFiles, dllLocations);
return;
}
@@ -75,7 +77,35 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
dllLocations.UnionWith(paths.Select(p => new AssemblyLookupLocation(p)));
LogAllUnusedPackages(dependencies);
DownloadMissingPackages(allNonBinaryFiles, dllLocations);
if (checkNugetFeedResponsiveness)
{
DownloadMissingPackagesFromSpecificFeeds(allNonBinaryFiles, dllLocations);
}
else
{
DownloadMissingPackages(allNonBinaryFiles, dllLocations);
}
}
internal const string PublicNugetFeed = "https://api.nuget.org/v3/index.json";
private List<string> GetReachableFallbackNugetFeeds()
{
var fallbackFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.FallbackNugetFeeds).ToHashSet();
if (fallbackFeeds.Count == 0)
{
fallbackFeeds.Add(PublicNugetFeed);
}
logger.LogInfo("Checking fallback Nuget feed reachability");
var reachableFallbackFeeds = fallbackFeeds.Where(feed => IsFeedReachable(feed)).ToList();
if (reachableFallbackFeeds.Count == 0)
{
logger.LogWarning("No fallback Nuget feeds are reachable. Skipping fallback Nuget package restoration.");
}
return reachableFallbackFeeds;
}
/// <summary>
@@ -148,7 +178,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString()));
}
private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<AssemblyLookupLocation> dllLocations, bool withNugetConfig = true)
private void DownloadMissingPackagesFromSpecificFeeds(List<FileInfo> allNonBinaryFiles, HashSet<AssemblyLookupLocation> dllLocations)
{
var reachableFallbackFeeds = GetReachableFallbackNugetFeeds();
if (reachableFallbackFeeds.Count > 0)
{
DownloadMissingPackages(allNonBinaryFiles, dllLocations, withNugetConfig: false, fallbackNugetFeeds: reachableFallbackFeeds);
}
}
private void DownloadMissingPackages(List<FileInfo> allFiles, HashSet<AssemblyLookupLocation> dllLocations, bool withNugetConfig = true, IEnumerable<string>? fallbackNugetFeeds = null)
{
var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(packageDirectory.DirInfo);
var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames();
@@ -181,9 +220,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
logger.LogInfo($"Found {notYetDownloadedPackages.Count} packages that are not yet restored");
using var tempDir = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName, "nugetconfig"));
var nugetConfig = withNugetConfig
? GetNugetConfig(allFiles)
: null;
: CreateFallbackNugetConfig(fallbackNugetFeeds, tempDir.DirInfo.FullName);
CompilationInfos.Add(("Fallback nuget restore", notYetDownloadedPackages.Count.ToString()));
@@ -209,6 +249,33 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
dllLocations.Add(missingPackageDirectory.DirInfo.FullName);
}
private string? CreateFallbackNugetConfig(IEnumerable<string>? fallbackNugetFeeds, string folderPath)
{
if (fallbackNugetFeeds is null)
{
// We're not overriding the inherited Nuget feeds
return null;
}
var sb = new StringBuilder();
fallbackNugetFeeds.ForEach((feed, index) => sb.AppendLine($"<add key=\"feed{index}\" value=\"{feed}\" />"));
var nugetConfigPath = Path.Combine(folderPath, "nuget.config");
logger.LogInfo($"Creating fallback nuget.config file {nugetConfigPath}.");
File.WriteAllText(nugetConfigPath,
$"""
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<packageSources>
<clear />
{sb}
</packageSources>
</configuration>
""");
return nugetConfigPath;
}
private string[] GetAllNugetConfigs(List<FileInfo> allFiles) => allFiles.SelectFileNamesByName("nuget.config").ToArray();
private string? GetNugetConfig(List<FileInfo> allFiles)
@@ -429,10 +496,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private bool CheckFeeds(List<FileInfo> allFiles)
{
logger.LogInfo("Checking Nuget feeds...");
var feeds = GetAllFeeds(allFiles);
var (explicitFeeds, allFeeds) = GetAllFeeds(allFiles);
var inheritedFeeds = allFeeds.Except(explicitFeeds).ToHashSet();
var excludedFeeds = Environment.GetEnvironmentVariable(EnvironmentVariableNames.ExcludedNugetFeedsFromResponsivenessCheck)
?.Split(" ", StringSplitOptions.RemoveEmptyEntries)
var excludedFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.ExcludedNugetFeedsFromResponsivenessCheck)
.ToHashSet() ?? [];
if (excludedFeeds.Count > 0)
@@ -440,7 +507,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
logger.LogInfo($"Excluded Nuget feeds from responsiveness check: {string.Join(", ", excludedFeeds.OrderBy(f => f))}");
}
var allFeedsReachable = feeds.All(feed => excludedFeeds.Contains(feed) || IsFeedReachable(feed));
var allFeedsReachable = explicitFeeds.All(feed => excludedFeeds.Contains(feed) || IsFeedReachable(feed));
if (!allFeedsReachable)
{
logger.LogWarning("Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.");
@@ -454,13 +521,19 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
));
}
CompilationInfos.Add(("All Nuget feeds reachable", allFeedsReachable ? "1" : "0"));
if (inheritedFeeds.Count > 0)
{
logger.LogInfo($"Inherited Nuget feeds: {string.Join(", ", inheritedFeeds.OrderBy(f => f))}");
CompilationInfos.Add(("Inherited Nuget feed count", inheritedFeeds.Count.ToString()));
}
return allFeedsReachable;
}
private IEnumerable<string> GetFeeds(string nugetConfig)
private IEnumerable<string> GetFeeds(Func<IList<string>> getNugetFeeds)
{
logger.LogInfo($"Getting Nuget feeds from '{nugetConfig}'...");
var results = dotnet.GetNugetFeeds(nugetConfig);
var results = getNugetFeeds();
var regex = EnabledNugetFeed();
foreach (var result in results)
{
@@ -479,27 +552,63 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
continue;
}
yield return url;
if (!string.IsNullOrWhiteSpace(url))
{
yield return url;
}
}
}
private HashSet<string> GetAllFeeds(List<FileInfo> allFiles)
private (HashSet<string>, HashSet<string>) GetAllFeeds(List<FileInfo> allFiles)
{
IList<string> GetNugetFeeds(string nugetConfig)
{
logger.LogInfo($"Getting Nuget feeds from '{nugetConfig}'...");
return dotnet.GetNugetFeeds(nugetConfig);
}
IList<string> GetNugetFeedsFromFolder(string folderPath)
{
logger.LogInfo($"Getting Nuget feeds in folder '{folderPath}'...");
return dotnet.GetNugetFeedsFromFolder(folderPath);
}
var nugetConfigs = GetAllNugetConfigs(allFiles);
var feeds = nugetConfigs
.SelectMany(GetFeeds)
.Where(str => !string.IsNullOrWhiteSpace(str))
var explicitFeeds = nugetConfigs
.SelectMany(config => GetFeeds(() => GetNugetFeeds(config)))
.ToHashSet();
if (feeds.Count > 0)
if (explicitFeeds.Count > 0)
{
logger.LogInfo($"Found {feeds.Count} Nuget feeds in nuget.config files: {string.Join(", ", feeds.OrderBy(f => f))}");
logger.LogInfo($"Found {explicitFeeds.Count} Nuget feeds in nuget.config files: {string.Join(", ", explicitFeeds.OrderBy(f => f))}");
}
else
{
logger.LogDebug("No Nuget feeds found in nuget.config files.");
}
return feeds;
// todo: this could be improved.
// We don't have to get the feeds from each of the folders from below, it would be enought to check the folders that recursively contain the others.
var allFeeds = nugetConfigs
.Select(config =>
{
try
{
return new FileInfo(config).Directory?.FullName;
}
catch (Exception exc)
{
logger.LogWarning($"Failed to get directory of '{config}': {exc}");
}
return null;
})
.Where(folder => folder != null)
.SelectMany(folder => GetFeeds(() => GetNugetFeedsFromFolder(folder!)))
.ToHashSet();
logger.LogInfo($"Found {allFeeds.Count} Nuget feeds (with inherited ones) in nuget.config files: {string.Join(", ", allFeeds.OrderBy(f => f))}");
return (explicitFeeds, allFeeds);
}
[GeneratedRegex(@"<TargetFramework>.*</TargetFramework>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]

10
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs
View File

@@ -95,9 +95,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
public IList<string> GetListedSdks() => GetResultList("--list-sdks");
private IList<string> GetResultList(string args)
private IList<string> GetResultList(string args, string? workingDirectory = null)
{
if (dotnetCliInvoker.RunCommand(args, out var results))
if (dotnetCliInvoker.RunCommand(args, workingDirectory, out var results))
{
return results;
}
@@ -111,7 +111,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
return dotnetCliInvoker.RunCommand(args);
}
public IList<string> GetNugetFeeds(string nugetConfig) => GetResultList($"nuget list source --format Short --configfile \"{nugetConfig}\"");
private const string nugetListSourceCommand = "nuget list source --format Short";
public IList<string> GetNugetFeeds(string nugetConfig) => GetResultList($"{nugetListSourceCommand} --configfile \"{nugetConfig}\"");
public IList<string> GetNugetFeedsFromFolder(string folderPath) => GetResultList(nugetListSourceCommand, folderPath);
// The version number should be kept in sync with the version .NET version used for building the application.
public const string LatestDotNetSdkVersion = "8.0.101";

22
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs
View File

@@ -21,7 +21,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
this.Exec = exec;
}
private ProcessStartInfo MakeDotnetStartInfo(string args)
private ProcessStartInfo MakeDotnetStartInfo(string args, string? workingDirectory)
{
var startInfo = new ProcessStartInfo(Exec, args)
{
@@ -29,6 +29,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
RedirectStandardOutput = true,
RedirectStandardError = true
};
if (!string.IsNullOrWhiteSpace(workingDirectory))
{
startInfo.WorkingDirectory = workingDirectory;
}
// Set the .NET CLI language to English to avoid localized output.
startInfo.EnvironmentVariables["DOTNET_CLI_UI_LANGUAGE"] = "en";
startInfo.EnvironmentVariables["MSBUILDDISABLENODEREUSE"] = "1";
@@ -36,26 +40,30 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
return startInfo;
}
private bool RunCommandAux(string args, out IList<string> output)
private bool RunCommandAux(string args, string? workingDirectory, out IList<string> output)
{
logger.LogInfo($"Running {Exec} {args}");
var pi = MakeDotnetStartInfo(args);
var dirLog = string.IsNullOrWhiteSpace(workingDirectory) ? "" : $" in {workingDirectory}";
logger.LogInfo($"Running {Exec} {args}{dirLog}");
var pi = MakeDotnetStartInfo(args, workingDirectory);
var threadId = Environment.CurrentManagedThreadId;
void onOut(string s) => logger.LogInfo(s, threadId);
void onError(string s) => logger.LogError(s, threadId);
var exitCode = pi.ReadOutput(out output, onOut, onError);
if (exitCode != 0)
{
logger.LogError($"Command {Exec} {args} failed with exit code {exitCode}");
logger.LogError($"Command {Exec} {args}{dirLog} failed with exit code {exitCode}");
return false;
}
return true;
}
public bool RunCommand(string args) =>
RunCommandAux(args, out _);
RunCommandAux(args, null, out _);
public bool RunCommand(string args, out IList<string> output) =>
RunCommandAux(args, out output);
RunCommandAux(args, null, out output);
public bool RunCommand(string args, string? workingDirectory, out IList<string> output) =>
RunCommandAux(args, workingDirectory, out output);
}
}

6
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs
View File

@@ -37,6 +37,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// </summary>
public const string NugetFeedResponsivenessRequestCount = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_LIMIT";
/// <summary>
/// Specifies the NuGet feeds to use for fallback Nuget dependency fetching. The value is a space-separated list of feed URLs.
/// The default value is `https://api.nuget.org/v3/index.json`.
/// </summary>
public const string FallbackNugetFeeds = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_FALLBACK";
/// <summary>
/// Specifies the location of the diagnostic directory.
/// </summary>

1
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs
View File

@@ -14,6 +14,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
IList<string> GetListedSdks();
bool Exec(string execArgs);
IList<string> GetNugetFeeds(string nugetConfig);
IList<string> GetNugetFeedsFromFolder(string folderPath);
}
public record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, string? PathToNugetConfig = null, bool ForceReevaluation = false);

6
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNetCliInvoker.cs
View File

@@ -19,5 +19,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// The output of the command is returned in `output`.
/// </summary>
bool RunCommand(string args, out IList<string> output);
/// <summary>
/// Execute `dotnet <args>` in `<workingDirectory>` and return true if the command succeeded, otherwise false.
/// The output of the command is returned in `output`.
/// </summary>
bool RunCommand(string args, string? workingDirectory, out IList<string> output);
}
}

2
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackages.cs
View File

@@ -243,7 +243,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private void AddDefaultPackageSource(string nugetConfig)
{
logger.LogInfo("Adding default package source...");
RunMonoNugetCommand($"sources add -Name DefaultNugetOrg -Source https://api.nuget.org/v3/index.json -ConfigFile \"{nugetConfig}\"", out var _);
RunMonoNugetCommand($"sources add -Name DefaultNugetOrg -Source {DependencyManager.PublicNugetFeed} -ConfigFile \"{nugetConfig}\"", out var _);
}
public void Dispose()

7
csharp/extractor/Semmle.Extraction.Tests/DotNet.cs
View File

@@ -10,6 +10,7 @@ namespace Semmle.Extraction.Tests
{
private readonly IList<string> output;
private string lastArgs = "";
public string WorkingDirectory { get; private set; } = "";
public bool Success { get; set; } = true;
public DotNetCliInvokerStub(IList<string> output)
@@ -32,6 +33,12 @@ namespace Semmle.Extraction.Tests
return Success;
}
public bool RunCommand(string args, string? workingDirectory, out IList<string> output)
{
WorkingDirectory = workingDirectory ?? "";
return RunCommand(args, out output);
}
public string GetLastArgs() => lastArgs;
}

2
csharp/extractor/Semmle.Extraction.Tests/Runtime.cs
View File

@@ -28,6 +28,8 @@ namespace Semmle.Extraction.Tests
public bool Exec(string execArgs) => true;
public IList<string> GetNugetFeeds(string nugetConfig) => [];
public IList<string> GetNugetFeedsFromFolder(string folderPath) => [];
}
public class RuntimeTests

6
csharp/extractor/Semmle.Util/EnvironmentVariables.cs
View File

@@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Numerics;
@@ -34,5 +35,10 @@ namespace Semmle.Util
var _ = bool.TryParse(env, out var value);
return value;
}
public static IEnumerable<string> GetURLs(string name)
{
return Environment.GetEnvironmentVariable(name)?.Split(" ", StringSplitOptions.RemoveEmptyEntries) ?? [];
}
}
}