hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-16 16:53:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

Refactor dotnet restore calls

Browse Source
This commit is contained in:
Tamas Vajk
2024-01-24 13:27:06 +01:00
parent d742cd3e44
commit 30095e3179
5 changed files with 70 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
View File

@@ -677,9 +677,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var projects = solutions.SelectMany(solution =>
{
logger.LogInfo($"Restoring solution {solution}...");
var success = dotnet.RestoreSolutionToDirectory(solution, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: true, out var restoredProjects, out var a);
assetFiles.AddRange(a);
return restoredProjects;
var res = dotnet.Restore(new(solution, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
assetFiles.AddRange(res.AssetsFilePaths);
return res.RestoredProjects;
});
assets = assetFiles;
return projects;
@@ -697,8 +697,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, project =>
{
logger.LogInfo($"Restoring project {project}...");
var success = dotnet.RestoreProjectToDirectory(project, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: true, out var a, out var _);
assetFiles.AddRange(a);
var res = dotnet.Restore(new(project, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
assetFiles.AddRange(res.AssetsFilePaths);
});
assets = assetFiles;
}
@@ -757,18 +757,18 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
return;
}
success = dotnet.RestoreProjectToDirectory(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: false, out var _, out var outputLines, pathToNugetConfig: nugetConfig);
if (!success)
var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig));
if (!res.Success)
{
if (outputLines?.Any(s => s.Contains("NU1301")) == true)
if (res.HasNugetPackageSourceError)
{
// Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
success = dotnet.RestoreProjectToDirectory(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: false, out var _, out var _, pathToNugetConfig: null, force: true);
res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true));
}
// TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package.
if (!success)
if (!res.Success)
{
logger.LogInfo($"Failed to restore nuget package {package}");
}

48
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs
View File

@@ -41,11 +41,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
}
private string GetRestoreArgs(string projectOrSolutionFile, string packageDirectory, bool forceDotnetRefAssemblyFetching)
private string GetRestoreArgs(RestoreSettings restoreSettings)
{
var args = $"restore --no-dependencies \"{projectOrSolutionFile}\" --packages \"{packageDirectory}\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal";
var args = $"restore --no-dependencies \"{restoreSettings.File}\" --packages \"{restoreSettings.PackageDirectory}\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal";
if (forceDotnetRefAssemblyFetching)
if (restoreSettings.ForceDotnetRefAssemblyFetching)
{
// Ugly hack: we set the TargetFrameworkRootPath and NetCoreTargetingPackRoot properties to an empty folder:
var path = ".empty";
@@ -58,46 +58,24 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
args += $" /p:TargetFrameworkRootPath=\"{path}\" /p:NetCoreTargetingPackRoot=\"{path}\"";
}
return args;
}
private static IEnumerable<string> GetFirstGroupOnMatch(Regex regex, IEnumerable<string> lines) =>
lines
.Select(line => regex.Match(line))
.Where(match => match.Success)
.Select(match => match.Groups[1].Value);
private static IEnumerable<string> GetAssetsFilePaths(IEnumerable<string> lines) =>
GetFirstGroupOnMatch(AssetsFileRegex(), lines);
private static IEnumerable<string> GetRestoredProjects(IEnumerable<string> lines) =>
GetFirstGroupOnMatch(RestoredProjectRegex(), lines);
public bool RestoreProjectToDirectory(string projectFile, string packageDirectory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> assets, out IList<string> outputLines, string? pathToNugetConfig = null, bool force = false)
{
var args = GetRestoreArgs(projectFile, packageDirectory, forceDotnetRefAssemblyFetching);
if (pathToNugetConfig != null)
if (restoreSettings.PathToNugetConfig != null)
{
args += $" --configfile \"{pathToNugetConfig}\"";
args += $" --configfile \"{restoreSettings.PathToNugetConfig}\"";
}
if (force)
if (restoreSettings.ForceReevaluation)
{
args += " --force";
}
var success = dotnetCliInvoker.RunCommand(args, out outputLines);
assets = success ? GetAssetsFilePaths(outputLines) : Array.Empty<string>();
return success;
return args;
}
public bool RestoreSolutionToDirectory(string solutionFile, string packageDirectory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> projects, out IEnumerable<string> assets)
public RestoreResult Restore(RestoreSettings restoreSettings)
{
var args = GetRestoreArgs(solutionFile, packageDirectory, forceDotnetRefAssemblyFetching);
var args = GetRestoreArgs(restoreSettings);
var success = dotnetCliInvoker.RunCommand(args, out var output);
projects = success ? GetRestoredProjects(output) : Array.Empty<string>();
assets = success ? GetAssetsFilePaths(output) : Array.Empty<string>();
return success;
return new(success, output);
}
public bool New(string folder)
@@ -130,11 +108,5 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var args = $"exec {execArgs}";
return dotnetCliInvoker.RunCommand(args);
}
[GeneratedRegex("Restored\\s+(.+\\.csproj)", RegexOptions.Compiled)]
private static partial Regex RestoredProjectRegex();
[GeneratedRegex("[Assets\\sfile\\shas\\snot\\schanged.\\sSkipping\\sassets\\sfile\\swriting.|Writing\\sassets\\sfile\\sto\\sdisk.]\\sPath:\\s(.+)", RegexOptions.Compiled)]
private static partial Regex AssetsFileRegex();
}
}

32
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs
View File

@@ -1,15 +1,43 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal interface IDotNet
{
bool RestoreProjectToDirectory(string project, string directory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> assets, out IList<string> outputLines, string? pathToNugetConfig = null, bool force = false);
bool RestoreSolutionToDirectory(string solutionFile, string packageDirectory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> projects, out IEnumerable<string> assets);
RestoreResult Restore(RestoreSettings restoreSettings);
bool New(string folder);
bool AddPackage(string folder, string package);
IList<string> GetListedRuntimes();
IList<string> GetListedSdks();
bool Exec(string execArgs);
}
internal record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, string? PathToNugetConfig = null, bool ForceReevaluation = false);
internal partial record class RestoreResult(bool Success, IList<string> Output)
{
private readonly Lazy<IEnumerable<string>> assetsFilePaths = new(() => GetFirstGroupOnMatch(AssetsFileRegex(), Output));
public IEnumerable<string> AssetsFilePaths => Success ? assetsFilePaths.Value : Array.Empty<string>();
private readonly Lazy<IEnumerable<string>> restoredProjects = new(() => GetFirstGroupOnMatch(RestoredProjectRegex(), Output));
public IEnumerable<string> RestoredProjects => Success ? restoredProjects.Value : Array.Empty<string>();
private readonly Lazy<bool> hasNugetPackageSourceError = new(() => Output.Any(s => s.Contains("NU1301")));
public bool HasNugetPackageSourceError => hasNugetPackageSourceError.Value;
private static IEnumerable<string> GetFirstGroupOnMatch(Regex regex, IEnumerable<string> lines) =>
lines
.Select(line => regex.Match(line))
.Where(match => match.Success)
.Select(match => match.Groups[1].Value);
[GeneratedRegex("Restored\\s+(.+\\.csproj)", RegexOptions.Compiled)]
private static partial Regex RestoredProjectRegex();
[GeneratedRegex("[Assets\\sfile\\shas\\snot\\schanged.\\sSkipping\\sassets\\sfile\\swriting.|Writing\\sassets\\sfile\\sto\\sdisk.]\\sPath:\\s(.+)", RegexOptions.Compiled)]
private static partial Regex AssetsFileRegex();
}
}

38
csharp/extractor/Semmle.Extraction.Tests/DotNet.cs
View File

@@ -101,7 +101,7 @@ namespace Semmle.Extraction.Tests
var dotnet = MakeDotnet(dotnetCliInvoker);
// Execute
dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages", false, out var assets, out var _);
dotnet.Restore(new("myproject.csproj", "mypackages", false));
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();
@@ -116,14 +116,14 @@ namespace Semmle.Extraction.Tests
var dotnet = MakeDotnet(dotnetCliInvoker);
// Execute
dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages", false, out var assets, out var _, pathToNugetConfig: "myconfig.config");
var res = dotnet.Restore(new("myproject.csproj", "mypackages", false, "myconfig.config"));
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();
Assert.Equal("restore --no-dependencies \"myproject.csproj\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal --configfile \"myconfig.config\"", lastArgs);
Assert.Equal(2, assets.Count());
Assert.Contains("/path/to/project.assets.json", assets);
Assert.Contains("/path/to/project2.assets.json", assets);
Assert.Equal(2, res.AssetsFilePaths.Count());
Assert.Contains("/path/to/project.assets.json", res.AssetsFilePaths);
Assert.Contains("/path/to/project2.assets.json", res.AssetsFilePaths);
}
[Fact]
@@ -134,14 +134,14 @@ namespace Semmle.Extraction.Tests
var dotnet = MakeDotnet(dotnetCliInvoker);
// Execute
dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages", false, out var assets, out var _, pathToNugetConfig: "myconfig.config", force: true);
var res = dotnet.Restore(new("myproject.csproj", "mypackages", false, "myconfig.config", true));
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();
Assert.Equal("restore --no-dependencies \"myproject.csproj\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal --configfile \"myconfig.config\" --force", lastArgs);
Assert.Equal(2, assets.Count());
Assert.Contains("/path/to/project.assets.json", assets);
Assert.Contains("/path/to/project2.assets.json", assets);
Assert.Equal(2, res.AssetsFilePaths.Count());
Assert.Contains("/path/to/project.assets.json", res.AssetsFilePaths);
Assert.Contains("/path/to/project2.assets.json", res.AssetsFilePaths);
}
[Fact]
@@ -152,17 +152,17 @@ namespace Semmle.Extraction.Tests
var dotnet = MakeDotnet(dotnetCliInvoker);
// Execute
dotnet.RestoreSolutionToDirectory("mysolution.sln", "mypackages", false, out var projects, out var assets);
var res = dotnet.Restore(new("mysolution.sln", "mypackages", false));
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();
Assert.Equal("restore --no-dependencies \"mysolution.sln\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal", lastArgs);
Assert.Equal(2, projects.Count());
Assert.Contains("/path/to/project.csproj", projects);
Assert.Contains("/path/to/project2.csproj", projects);
Assert.Equal(2, assets.Count());
Assert.Contains("/path/to/project.assets.json", assets);
Assert.Contains("/path/to/project2.assets.json", assets);
Assert.Equal(2, res.RestoredProjects.Count());
Assert.Contains("/path/to/project.csproj", res.RestoredProjects);
Assert.Contains("/path/to/project2.csproj", res.RestoredProjects);
Assert.Equal(2, res.AssetsFilePaths.Count());
Assert.Contains("/path/to/project.assets.json", res.AssetsFilePaths);
Assert.Contains("/path/to/project2.assets.json", res.AssetsFilePaths);
}
[Fact]
@@ -174,13 +174,13 @@ namespace Semmle.Extraction.Tests
dotnetCliInvoker.Success = false;
// Execute
dotnet.RestoreSolutionToDirectory("mysolution.sln", "mypackages", false, out var projects, out var assets);
var res = dotnet.Restore(new("mysolution.sln", "mypackages", false));
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();
Assert.Equal("restore --no-dependencies \"mysolution.sln\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal", lastArgs);
Assert.Empty(projects);
Assert.Empty(assets);
Assert.Empty(res.RestoredProjects);
Assert.Empty(res.AssetsFilePaths);
}
[Fact]

14
csharp/extractor/Semmle.Extraction.Tests/Runtime.cs
View File

@@ -19,19 +19,7 @@ namespace Semmle.Extraction.Tests
public bool New(string folder) => true;
public bool RestoreProjectToDirectory(string project, string directory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> assets, out IList<string> outputLines, string? pathToNugetConfig = null, bool force = false)
{
assets = Array.Empty<string>();
outputLines = Array.Empty<string>();
return true;
}
public bool RestoreSolutionToDirectory(string solution, string directory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> projects, out IEnumerable<string> assets)
{
projects = Array.Empty<string>();
assets = Array.Empty<string>();
return true;
}
public RestoreResult Restore(RestoreSettings restoreSettings) => new(true, Array.Empty<string>());
public IList<string> GetListedRuntimes() => runtimes;