hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-16 16:53:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

C#: Parallelize restore logic of missing packages

Browse Source
This commit is contained in:
Tamas Vajk
2023-09-18 12:32:44 +02:00
parent fc3bc95147
commit dfd7f1e78b
5 changed files with 59 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
View File

@@ -499,29 +499,38 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var alreadyDownloadedPackages = Directory.GetDirectories(packageDirectory.DirInfo.FullName)
.Select(d => Path.GetFileName(d).ToLowerInvariant());
var notYetDownloadedPackages = fileContent.AllPackages.Except(alreadyDownloadedPackages);
foreach (var package in notYetDownloadedPackages)
var stdoutLines = notYetDownloadedPackages
.AsParallel()
.WithDegreeOfParallelism(options.Threads)
.Select(package =>
{
progressMonitor.NugetInstall(package);
using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package));
var success = dotnet.New(tempDir.DirInfo.FullName, out var stdout1);
if (!success)
{
return new[] { stdout1 };
}
success = dotnet.AddPackage(tempDir.DirInfo.FullName, package, out var stdout2);
if (!success)
{
return new[] { stdout1, stdout2 };
}
success = RestoreProject(tempDir.DirInfo.FullName, out var stdout3, nugetConfig);
// TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package.
if (!success)
{
progressMonitor.FailedToRestoreNugetPackage(package);
}
return new[] { stdout1, stdout2, stdout3 };
})
.ToList();
foreach (var line in stdoutLines.SelectMany(l => l))
{
progressMonitor.NugetInstall(package);
using var tempDir = new TemporaryDirectory(GetTemporaryWorkingDirectory(package));
var success = dotnet.New(tempDir.DirInfo.FullName);
if (!success)
{
continue;
}
success = dotnet.AddPackage(tempDir.DirInfo.FullName, package);
if (!success)
{
continue;
}
success = RestoreProject(tempDir.DirInfo.FullName, out var stdout, nugetConfig);
Console.WriteLine(stdout);
// TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package.
if (!success)
{
progressMonitor.FailedToRestoreNugetPackage(package);
}
Console.WriteLine(line);
}
}

22
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs
View File

@@ -37,6 +37,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
}
private bool RunCommand(string args, out string stdout)
{
var success = dotnetCliInvoker.RunCommand(args, out var output);
stdout = string.Join("\n", output);
return success;
}
private static string GetRestoreArgs(string projectOrSolutionFile, string packageDirectory) =>
$"restore --no-dependencies \"{projectOrSolutionFile}\" --packages \"{packageDirectory}\" /p:DisableImplicitNuGetFallbackFolder=true";
@@ -47,9 +54,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
args += $" --configfile \"{pathToNugetConfig}\"";
}
var success = dotnetCliInvoker.RunCommand(args, out var output);
stdout = string.Join("\n", output);
return success;
return RunCommand(args, out stdout);
}
public bool RestoreSolutionToDirectory(string solutionFile, string packageDirectory, out IEnumerable<string> projects)
@@ -70,16 +76,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
return false;
}
public bool New(string folder)
public bool New(string folder, out string stdout)
{
var args = $"new console --no-restore --output \"{folder}\"";
return dotnetCliInvoker.RunCommand(args);
return RunCommand(args, out stdout);
}
public bool AddPackage(string folder, string package)
public bool AddPackage(string folder, string package, out string stdout)
{
var args = $"add \"{folder}\" package \"{package}\" --no-restore";
return dotnetCliInvoker.RunCommand(args);
return RunCommand(args, out stdout);
}
public IList<string> GetListedRuntimes() => GetListed("--list-runtimes", "runtime");
@@ -88,7 +94,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private IList<string> GetListed(string args, string artifact)
{
if (dotnetCliInvoker.RunCommand(args, out var artifacts))
if (dotnetCliInvoker.RunCommand(args, out IList<string> artifacts))
{
progressMonitor.LogInfo($"Found {artifact}s: {string.Join("\n", artifacts)}");
return artifacts;

4
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs
View File

@@ -6,8 +6,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
bool RestoreProjectToDirectory(string project, string directory, out string stdout, string? pathToNugetConfig = null);
bool RestoreSolutionToDirectory(string solutionFile, string packageDirectory, out IEnumerable<string> projects);
bool New(string folder);
bool AddPackage(string folder, string package);
bool New(string folder, out string stdout);
bool AddPackage(string folder, string package, out string stdout);
IList<string> GetListedRuntimes();
IList<string> GetListedSdks();
bool Exec(string execArgs);

4
csharp/extractor/Semmle.Extraction.Tests/DotNet.cs
View File

@@ -164,7 +164,7 @@ namespace Semmle.Extraction.Tests
var dotnet = MakeDotnet(dotnetCliInvoker);
// Execute
dotnet.New("myfolder");
dotnet.New("myfolder", out var _);
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();
@@ -179,7 +179,7 @@ namespace Semmle.Extraction.Tests
var dotnet = MakeDotnet(dotnetCliInvoker);
// Execute
dotnet.AddPackage("myfolder", "mypackage");
dotnet.AddPackage("myfolder", "mypackage", out var _);
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();

12
csharp/extractor/Semmle.Extraction.Tests/Runtime.cs
View File

@@ -15,9 +15,17 @@ namespace Semmle.Extraction.Tests
this.runtimes = runtimes;
this.sdks = sdks;
}
public bool AddPackage(string folder, string package) => true;
public bool AddPackage(string folder, string package, out string stdout)
{
stdout = "";
return true;
}
public bool New(string folder) => true;
public bool New(string folder, out string stdout)
{
stdout = "";
return true;
}
public bool RestoreProjectToDirectory(string project, string directory, out string stdout, string? pathToNugetConfig = null)
{