diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs index 6644e003872..8f2572871f4 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs @@ -103,8 +103,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching progressMonitor.MissingNuGet(); } - Restore(solutions); - Restore(allProjects); + var restoredProjects = RestoreSolutions(solutions); + var projects = allProjects.Except(restoredProjects); + RestoreProjects(projects); DownloadMissingPackages(allFiles); } @@ -351,16 +352,26 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } - private bool Restore(string target, string? pathToNugetConfig = null) => - dotnet.RestoreToDirectory(target, packageDirectory.DirInfo.FullName, pathToNugetConfig); + private bool RestoreProject(string project, string? pathToNugetConfig = null) => + dotnet.RestoreProjectToDirectory(project, packageDirectory.DirInfo.FullName, pathToNugetConfig); - private void Restore(IEnumerable targets, string? pathToNugetConfig = null) - { - foreach (var target in targets) - { - Restore(target, pathToNugetConfig); - } - } + private bool RestoreSolution(string solution, out IList projects) => + dotnet.RestoreSolutionToDirectory(solution, packageDirectory.DirInfo.FullName, out projects); + + /// + /// Executes `dotnet restore` on all solution files in solutions. + /// Returns a list of projects that are up to date with respect to restore. + /// + /// A list of paths to solution files. + private IEnumerable RestoreSolutions(IEnumerable solutions) => + solutions.SelectMany(solution => + { + RestoreSolution(solution, out var restoredProjects); + return restoredProjects; + }); + + private void RestoreProjects(IEnumerable projects) => + Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, project => RestoreProject(project)); private void DownloadMissingPackages(List allFiles) { @@ -401,10 +412,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching continue; } - success = Restore(tempDir.DirInfo.FullName, nugetConfig); + success = RestoreProject(tempDir.DirInfo.FullName, nugetConfig); // TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package. - if (!success) { progressMonitor.FailedToRestoreNugetPackage(package); diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs index 35c541a195f..4adf7fa1c5d 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs @@ -2,6 +2,8 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.Linq; +using System.Text.RegularExpressions; using Semmle.Util; namespace Semmle.Extraction.CSharp.DependencyFetching @@ -9,7 +11,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// /// Utilities to run the "dotnet" command. /// - internal class DotNet : IDotNet + internal partial class DotNet : IDotNet { private readonly ProgressMonitor progressMonitor; private readonly string dotnet; @@ -41,7 +43,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching private bool RunCommand(string args) { progressMonitor.RunningProcess($"{dotnet} {args}"); - using var proc = Process.Start(this.MakeDotnetStartInfo(args, redirectStandardOutput: false)); + using var proc = Process.Start(MakeDotnetStartInfo(args, redirectStandardOutput: false)); proc?.WaitForExit(); var exitCode = proc?.ExitCode ?? -1; if (exitCode != 0) @@ -52,14 +54,51 @@ namespace Semmle.Extraction.CSharp.DependencyFetching return true; } - public bool RestoreToDirectory(string projectOrSolutionFile, string packageDirectory, string? pathToNugetConfig = null) + private bool RunCommand(string args, out IList output) { - var args = $"restore --no-dependencies \"{projectOrSolutionFile}\" --packages \"{packageDirectory}\" /p:DisableImplicitNuGetFallbackFolder=true"; + progressMonitor.RunningProcess($"{dotnet} {args}"); + var pi = MakeDotnetStartInfo(args, redirectStandardOutput: true); + var exitCode = pi.ReadOutput(out output); + if (exitCode != 0) + { + progressMonitor.CommandFailed(dotnet, args, exitCode); + return false; + } + return true; + } + + private static string GetRestoreArgs(string projectOrSolutionFile, string packageDirectory) => + $"restore --no-dependencies \"{projectOrSolutionFile}\" --packages \"{packageDirectory}\" /p:DisableImplicitNuGetFallbackFolder=true"; + + public bool RestoreProjectToDirectory(string projectFile, string packageDirectory, string? pathToNugetConfig = null) + { + var args = GetRestoreArgs(projectFile, packageDirectory); if (pathToNugetConfig != null) + { args += $" --configfile \"{pathToNugetConfig}\""; + } return RunCommand(args); } + public bool RestoreSolutionToDirectory(string solutionFile, string packageDirectory, out IList projects) + { + var args = GetRestoreArgs(solutionFile, packageDirectory); + args += " --verbosity normal"; + if (RunCommand(args, out var output)) + { + var regex = RestoreProjectRegex(); + projects = output + .Select(line => regex.Match(line)) + .Where(match => match.Success) + .Select(match => match.Groups[1].Value) + .ToList(); + return true; + } + + projects = new List(); + return false; + } + public bool New(string folder) { var args = $"new console --no-restore --output \"{folder}\""; @@ -78,16 +117,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching private IList GetListed(string args, string artifact) { - progressMonitor.RunningProcess($"{dotnet} {args}"); - var pi = this.MakeDotnetStartInfo(args, redirectStandardOutput: true); - var exitCode = pi.ReadOutput(out var artifacts); - if (exitCode != 0) + if (RunCommand(args, out var artifacts)) { - progressMonitor.CommandFailed(dotnet, args, exitCode); - return new List(); + progressMonitor.LogInfo($"Found {artifact}s: {string.Join("\n", artifacts)}"); + return artifacts; } - progressMonitor.LogInfo($"Found {artifact}s: {string.Join("\n", artifacts)}"); - return artifacts; + return new List(); } public bool Exec(string execArgs) @@ -95,5 +130,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching var args = $"exec {execArgs}"; return RunCommand(args); } + + [GeneratedRegex("Restored\\s+(.+\\.csproj)", RegexOptions.Compiled)] + private static partial Regex RestoreProjectRegex(); } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs index daee32b6cc4..9a3f8b5ec64 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs @@ -4,7 +4,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { internal interface IDotNet { - bool RestoreToDirectory(string project, string directory, string? pathToNugetConfig = null); + bool RestoreProjectToDirectory(string project, string directory, string? pathToNugetConfig = null); + bool RestoreSolutionToDirectory(string solution, string directory, out IList projects); bool New(string folder); bool AddPackage(string folder, string package); IList GetListedRuntimes(); diff --git a/csharp/extractor/Semmle.Extraction.Tests/FileContent.cs b/csharp/extractor/Semmle.Extraction.Tests/FileContent.cs index 131096769e1..00a7238103e 100644 --- a/csharp/extractor/Semmle.Extraction.Tests/FileContent.cs +++ b/csharp/extractor/Semmle.Extraction.Tests/FileContent.cs @@ -1,6 +1,5 @@ using Xunit; using System.Collections.Generic; -using System.Linq; using Semmle.Util.Logging; using Semmle.Extraction.CSharp.DependencyFetching; diff --git a/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs b/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs index 78e2270a883..75b3f98b688 100644 --- a/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs +++ b/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs @@ -18,7 +18,13 @@ namespace Semmle.Extraction.Tests public bool New(string folder) => true; - public bool RestoreToDirectory(string project, string directory, string? pathToNugetConfig = null) => true; + public bool RestoreProjectToDirectory(string project, string directory, string? pathToNugetConfig = null) => true; + + public bool RestoreSolutionToDirectory(string solution, string directory, out IList projects) + { + projects = new List(); + return true; + } public IList GetListedRuntimes() => runtimes;