C#: Avoid explicitly restoring the projects in the restored solution files.

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -103,8 +103,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                     progressMonitor.MissingNuGet();
                 }
 
-                Restore(solutions);
+                var restoredProjects = RestoreSolutions(solutions);
-                Restore(allProjects);
+                var projects = allProjects.Except(restoredProjects);
+                RestoreProjects(projects);
                 DownloadMissingPackages(allFiles);
             }
 
@@ -351,16 +352,26 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 
         }
 
-        private bool Restore(string target, string? pathToNugetConfig = null) =>
+        private bool RestoreProject(string project, string? pathToNugetConfig = null) =>
-            dotnet.RestoreToDirectory(target, packageDirectory.DirInfo.FullName, pathToNugetConfig);
+            dotnet.RestoreProjectToDirectory(project, packageDirectory.DirInfo.FullName, pathToNugetConfig);
 
-        private void Restore(IEnumerable<string> targets, string? pathToNugetConfig = null)
+        private bool RestoreSolution(string solution, out IList<string> projects) =>
+            dotnet.RestoreSolutionToDirectory(solution, packageDirectory.DirInfo.FullName, out projects);
+
+        /// <summary>
+        /// Executes `dotnet restore` on all solution files in solutions.
+        /// Returns a list of projects that are up to date with respect to restore.
+        /// </summary>
+        /// <param name="solutions">A list of paths to solution files.</param>
+        private IEnumerable<string> RestoreSolutions(IEnumerable<string> solutions) =>
+            solutions.SelectMany(solution =>
                 {
-            foreach (var target in targets)
+                    RestoreSolution(solution, out var restoredProjects);
-            {
+                    return restoredProjects;
-                Restore(target, pathToNugetConfig);
+                });
-            }
+
-        }
+        private void RestoreProjects(IEnumerable<string> projects) =>
+            Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, project => RestoreProject(project));
 
         private void DownloadMissingPackages(List<FileInfo> allFiles)
         {
@@ -401,10 +412,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                     continue;
                 }
 
-                success = Restore(tempDir.DirInfo.FullName, nugetConfig);
+                success = RestoreProject(tempDir.DirInfo.FullName, nugetConfig);
 
                 // TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package.
-
                 if (!success)
                 {
                     progressMonitor.FailedToRestoreNugetPackage(package);

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs

@@ -2,6 +2,8 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
+using System.Linq;
+using System.Text.RegularExpressions;
 using Semmle.Util;
 
 namespace Semmle.Extraction.CSharp.DependencyFetching
@@ -9,7 +11,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
     /// <summary>
     /// Utilities to run the "dotnet" command.
     /// </summary>
-    internal class DotNet : IDotNet
+    internal partial class DotNet : IDotNet
     {
         private readonly ProgressMonitor progressMonitor;
         private readonly string dotnet;
@@ -41,7 +43,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         private bool RunCommand(string args)
         {
             progressMonitor.RunningProcess($"{dotnet} {args}");
-            using var proc = Process.Start(this.MakeDotnetStartInfo(args, redirectStandardOutput: false));
+            using var proc = Process.Start(MakeDotnetStartInfo(args, redirectStandardOutput: false));
             proc?.WaitForExit();
             var exitCode = proc?.ExitCode ?? -1;
             if (exitCode != 0)
@@ -52,14 +54,51 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             return true;
         }
 
-        public bool RestoreToDirectory(string projectOrSolutionFile, string packageDirectory, string? pathToNugetConfig = null)
+        private bool RunCommand(string args, out IList<string> output)
         {
-            var args = $"restore --no-dependencies \"{projectOrSolutionFile}\" --packages \"{packageDirectory}\" /p:DisableImplicitNuGetFallbackFolder=true";
+            progressMonitor.RunningProcess($"{dotnet} {args}");
+            var pi = MakeDotnetStartInfo(args, redirectStandardOutput: true);
+            var exitCode = pi.ReadOutput(out output);
+            if (exitCode != 0)
+            {
+                progressMonitor.CommandFailed(dotnet, args, exitCode);
+                return false;
+            }
+            return true;
+        }
+
+        private static string GetRestoreArgs(string projectOrSolutionFile, string packageDirectory) =>
+            $"restore --no-dependencies \"{projectOrSolutionFile}\" --packages \"{packageDirectory}\" /p:DisableImplicitNuGetFallbackFolder=true";
+
+        public bool RestoreProjectToDirectory(string projectFile, string packageDirectory, string? pathToNugetConfig = null)
+        {
+            var args = GetRestoreArgs(projectFile, packageDirectory);
             if (pathToNugetConfig != null)
+            {
                 args += $" --configfile \"{pathToNugetConfig}\"";
+            }
             return RunCommand(args);
         }
 
+        public bool RestoreSolutionToDirectory(string solutionFile, string packageDirectory, out IList<string> projects)
+        {
+            var args = GetRestoreArgs(solutionFile, packageDirectory);
+            args += " --verbosity normal";
+            if (RunCommand(args, out var output))
+            {
+                var regex = RestoreProjectRegex();
+                projects = output
+                    .Select(line => regex.Match(line))
+                    .Where(match => match.Success)
+                    .Select(match => match.Groups[1].Value)
+                    .ToList();
+                return true;
+            }
+
+            projects = new List<string>();
+            return false;
+        }
+
         public bool New(string folder)
         {
             var args = $"new console --no-restore --output \"{folder}\"";
@@ -78,22 +117,21 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 
         private IList<string> GetListed(string args, string artifact)
         {
-            progressMonitor.RunningProcess($"{dotnet} {args}");
+            if (RunCommand(args, out var artifacts))
-            var pi = this.MakeDotnetStartInfo(args, redirectStandardOutput: true);
-            var exitCode = pi.ReadOutput(out var artifacts);
-            if (exitCode != 0)
             {
-                progressMonitor.CommandFailed(dotnet, args, exitCode);
-                return new List<string>();
-            }
                 progressMonitor.LogInfo($"Found {artifact}s: {string.Join("\n", artifacts)}");
                 return artifacts;
             }
+            return new List<string>();
+        }
 
         public bool Exec(string execArgs)
         {
             var args = $"exec {execArgs}";
             return RunCommand(args);
         }
+
+        [GeneratedRegex("Restored\\s+(.+\\.csproj)", RegexOptions.Compiled)]
+        private static partial Regex RestoreProjectRegex();
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs

@@ -4,7 +4,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 {
     internal interface IDotNet
     {
-        bool RestoreToDirectory(string project, string directory, string? pathToNugetConfig = null);
+        bool RestoreProjectToDirectory(string project, string directory, string? pathToNugetConfig = null);
+        bool RestoreSolutionToDirectory(string solution, string directory, out IList<string> projects);
         bool New(string folder);
         bool AddPackage(string folder, string package);
         IList<string> GetListedRuntimes();

csharp/extractor/Semmle.Extraction.Tests/FileContent.cs

@@ -1,6 +1,5 @@
 using Xunit;
 using System.Collections.Generic;
-using System.Linq;
 using Semmle.Util.Logging;
 using Semmle.Extraction.CSharp.DependencyFetching;
 

csharp/extractor/Semmle.Extraction.Tests/Runtime.cs

@@ -18,7 +18,13 @@ namespace Semmle.Extraction.Tests
 
         public bool New(string folder) => true;
 
-        public bool RestoreToDirectory(string project, string directory, string? pathToNugetConfig = null) => true;
+        public bool RestoreProjectToDirectory(string project, string directory, string? pathToNugetConfig = null) => true;
+
+        public bool RestoreSolutionToDirectory(string solution, string directory, out IList<string> projects)
+        {
+            projects = new List<string>();
+            return true;
+        }
 
         public IList<string> GetListedRuntimes() => runtimes;