Merge pull request #13658 from tamasvajk/cs/standalone/restore-impr

C#: Improve dotnet restore success rate in standalone extraction

csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyCache.cs

@@ -16,12 +16,12 @@ namespace Semmle.BuildAnalyser
         /// Locate all reference files and index them.
         /// </summary>
         /// <param name="dirs">Directories to search.</param>
-        /// <param name="progress">Callback for progress.</param>
+        /// <param name="progressMonitor">Callback for progress.</param>
-        public AssemblyCache(IEnumerable<string> dirs, IProgressMonitor progress)
+        public AssemblyCache(IEnumerable<string> dirs, ProgressMonitor progressMonitor)
         {
             foreach (var dir in dirs)
             {
-                progress.FindingFiles(dir);
+                progressMonitor.FindingFiles(dir);
                 AddReferenceDirectory(dir);
             }
             IndexReferences();

csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs

@@ -8,90 +8,61 @@ using System.Threading.Tasks;
 using System.Collections.Concurrent;
 using System.Text;
 using System.Security.Cryptography;
+using System.Text.RegularExpressions;
 
 namespace Semmle.BuildAnalyser
 {
-    /// <summary>
-    /// The output of a build analysis.
-    /// </summary>
-    internal interface IBuildAnalysis
-    {
-        /// <summary>
-        /// Full filepaths of external references.
-        /// </summary>
-        IEnumerable<string> ReferenceFiles { get; }
-
-        /// <summary>
-        /// Full filepaths of C# source files from project files.
-        /// </summary>
-        IEnumerable<string> ProjectSourceFiles { get; }
-
-        /// <summary>
-        /// Full filepaths of C# source files in the filesystem.
-        /// </summary>
-        IEnumerable<string> AllSourceFiles { get; }
-
-        /// <summary>
-        /// The assembly IDs which could not be resolved.
-        /// </summary>
-        IEnumerable<string> UnresolvedReferences { get; }
-
-        /// <summary>
-        /// List of source files referenced by projects but
-        /// which were not found in the filesystem.
-        /// </summary>
-        IEnumerable<string> MissingSourceFiles { get; }
-    }
-
     /// <summary>
     /// Main implementation of the build analysis.
     /// </summary>
-    internal sealed class BuildAnalysis : IBuildAnalysis, IDisposable
+    internal sealed partial class BuildAnalysis : IDisposable
     {
         private readonly AssemblyCache assemblyCache;
-        private readonly IProgressMonitor progressMonitor;
+        private readonly ProgressMonitor progressMonitor;
         private readonly IDictionary<string, bool> usedReferences = new ConcurrentDictionary<string, bool>();
         private readonly IDictionary<string, bool> sources = new ConcurrentDictionary<string, bool>();
         private readonly IDictionary<string, string> unresolvedReferences = new ConcurrentDictionary<string, string>();
-        private int failedProjects, succeededProjects;
+        private int failedProjects;
+        private int succeededProjects;
         private readonly string[] allSources;
         private int conflictedReferences = 0;
+        private readonly Options options;
+        private readonly DirectoryInfo sourceDir;
+        private readonly DotNet dotnet;
 
         /// <summary>
         /// Performs a C# build analysis.
         /// </summary>
         /// <param name="options">Analysis options from the command line.</param>
-        /// <param name="progress">Display of analysis progress.</param>
+        /// <param name="progressMonitor">Display of analysis progress.</param>
-        public BuildAnalysis(Options options, IProgressMonitor progress)
+        public BuildAnalysis(Options options, ProgressMonitor progressMonitor)
         {
             var startTime = DateTime.Now;
 
-            progressMonitor = progress;
+            this.options = options;
-            var sourceDir = new DirectoryInfo(options.SrcDir);
+            this.progressMonitor = progressMonitor;
+            this.sourceDir = new DirectoryInfo(options.SrcDir);
 
-            progressMonitor.FindingFiles(options.SrcDir);
-
-            allSources = sourceDir.GetFiles("*.cs", SearchOption.AllDirectories)
-                .Select(d => d.FullName)
-                .Where(d => !options.ExcludesFile(d))
-                .ToArray();
-
-            var dllDirNames = options.DllDirs.Select(Path.GetFullPath).ToList();
-            packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName));
-
-            if (options.UseNuGet)
-            {
             try
             {
-                    var nuget = new NugetPackages(sourceDir.FullName, packageDirectory);
+                this.dotnet = new DotNet(progressMonitor);
-                    nuget.InstallPackages(progressMonitor);
             }
-                catch (FileNotFoundException)
+            catch
             {
-                    progressMonitor.MissingNuGet();
+                progressMonitor.MissingDotNet();
-                }
+                throw;
             }
 
+            this.progressMonitor.FindingFiles(options.SrcDir);
+
+            this.allSources = GetFiles("*.cs").ToArray();
+            var allProjects = GetFiles("*.csproj");
+            var solutions = options.SolutionFile is not null
+                ? new[] { options.SolutionFile }
+                : GetFiles("*.sln");
+
+            var dllDirNames = options.DllDirs.Select(Path.GetFullPath).ToList();
+
             // Find DLLs in the .Net Framework
             if (options.ScanNetFrameworkDlls)
             {
@@ -100,30 +71,43 @@ namespace Semmle.BuildAnalyser
                 dllDirNames.Add(runtimeLocation);
             }
 
-            // TODO: remove the below when the required SDK is installed
-            using (new FileRenamer(sourceDir.GetFiles("global.json", SearchOption.AllDirectories)))
-            {
-                var solutions = options.SolutionFile is not null ?
-                        new[] { options.SolutionFile } :
-                        sourceDir.GetFiles("*.sln", SearchOption.AllDirectories).Select(d => d.FullName);
-
-                if (options.UseNuGet)
-                {
-                    RestoreSolutions(solutions);
-                }
-                dllDirNames.Add(packageDirectory.DirInfo.FullName);
-                assemblyCache = new BuildAnalyser.AssemblyCache(dllDirNames, progress);
-                AnalyseSolutions(solutions);
-
-                foreach (var filename in assemblyCache.AllAssemblies.Select(a => a.Filename))
-                    UseReference(filename);
-            }
-
             if (options.UseMscorlib)
             {
                 UseReference(typeof(object).Assembly.Location);
             }
 
+            packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName));
+
+            if (options.UseNuGet)
+            {
+                dllDirNames.Add(packageDirectory.DirInfo.FullName);
+                try
+                {
+                    var nuget = new NugetPackages(sourceDir.FullName, packageDirectory, progressMonitor);
+                    nuget.InstallPackages();
+                }
+                catch (FileNotFoundException)
+                {
+                    progressMonitor.MissingNuGet();
+                }
+
+                // TODO: remove the below when the required SDK is installed
+                using (new FileRenamer(sourceDir.GetFiles("global.json", SearchOption.AllDirectories)))
+                {
+                    Restore(solutions);
+                    Restore(allProjects);
+                    DownloadMissingPackages(allProjects);
+                }
+            }
+
+            assemblyCache = new AssemblyCache(dllDirNames, progressMonitor);
+            AnalyseSolutions(solutions);
+
+            foreach (var filename in assemblyCache.AllAssemblies.Select(a => a.Filename))
+            {
+                UseReference(filename);
+            }
+
             ResolveConflicts();
 
             // Output the findings
@@ -149,6 +133,13 @@ namespace Semmle.BuildAnalyser
                 DateTime.Now - startTime);
         }
 
+        private IEnumerable<string> GetFiles(string pattern)
+        {
+            return sourceDir.GetFiles(pattern, SearchOption.AllDirectories)
+                .Select(d => d.FullName)
+                .Where(d => !options.ExcludesFile(d));
+        }
+
         /// <summary>
         /// Computes a unique temp directory for the packages associated
         /// with this source tree. Use a SHA1 of the directory name.
@@ -158,9 +149,7 @@ namespace Semmle.BuildAnalyser
         private static string ComputeTempDirectory(string srcDir)
         {
             var bytes = Encoding.Unicode.GetBytes(srcDir);
-
+            var sha = SHA1.HashData(bytes);
-            using var sha1 = SHA1.Create();
-            var sha = sha1.ComputeHash(bytes);
             var sb = new StringBuilder();
             foreach (var b in sha.Take(8))
                 sb.AppendFormat("{0:x2}", b);
@@ -195,12 +184,15 @@ namespace Semmle.BuildAnalyser
 
             // Pick the highest version for each assembly name
             foreach (var r in sortedReferences)
+            {
                 finalAssemblyList[r.Name] = r;
-
+            }
             // Update the used references list
             usedReferences.Clear();
             foreach (var r in finalAssemblyList.Select(r => r.Value.Filename))
+            {
                 UseReference(r);
+            }
 
             // Report the results
             foreach (var r in sortedReferences)
@@ -278,8 +270,10 @@ namespace Semmle.BuildAnalyser
         private void AnalyseProjectFiles(IEnumerable<FileInfo> projectFiles)
         {
             foreach (var proj in projectFiles)
+            {
                 AnalyseProject(proj);
             }
+        }
 
         private void AnalyseProject(FileInfo project)
         {
@@ -324,36 +318,90 @@ namespace Semmle.BuildAnalyser
 
         }
 
-        private void Restore(string projectOrSolution)
+        private bool Restore(string target)
+        {
+            return dotnet.RestoreToDirectory(target, packageDirectory.DirInfo.FullName);
+        }
+
+        private void Restore(IEnumerable<string> targets)
+        {
+            foreach (var target in targets)
+            {
+                Restore(target);
+            }
+        }
+
+        private void DownloadMissingPackages(IEnumerable<string> restoreTargets)
+        {
+            var alreadyDownloadedPackages = Directory.GetDirectories(packageDirectory.DirInfo.FullName).Select(d => Path.GetFileName(d).ToLowerInvariant()).ToHashSet();
+            var notYetDownloadedPackages = new HashSet<string>();
+
+            var allFiles = GetFiles("*.*").ToArray();
+            foreach (var file in allFiles)
             {
-            int exit;
                 try
                 {
-                exit = DotNet.RestoreToDirectory(projectOrSolution, packageDirectory.DirInfo.FullName);
+                    using var sr = new StreamReader(file);
-            }
+                    ReadOnlySpan<char> line;
-            catch (FileNotFoundException)
+                    while ((line = sr.ReadLine()) != null)
                     {
-                exit = 2;
+                        foreach (var valueMatch in PackageReference().EnumerateMatches(line))
+                        {
+                            // We can't get the group from the ValueMatch, so doing it manually:
+                            var match = line.Slice(valueMatch.Index, valueMatch.Length);
+                            var includeIndex = match.IndexOf("Include", StringComparison.InvariantCultureIgnoreCase);
+                            if (includeIndex == -1)
+                            {
+                                continue;
                             }
 
-            switch (exit)
+                            match = match.Slice(includeIndex + "Include".Length + 1);
+
+                            var quoteIndex1 = match.IndexOf("\"");
+                            var quoteIndex2 = match.Slice(quoteIndex1 + 1).IndexOf("\"");
+
+                            var packageName = match.Slice(quoteIndex1 + 1, quoteIndex2).ToString().ToLowerInvariant();
+                            if (!alreadyDownloadedPackages.Contains(packageName))
                             {
-                case 0:
+                                notYetDownloadedPackages.Add(packageName);
-                case 1:
+                            }
-                    // No errors
+                        }
-                    break;
+                    }
-                default:
+                }
-                    progressMonitor.CommandFailed("dotnet", $"restore \"{projectOrSolution}\"", exit);
+                catch (Exception ex)
-                    break;
+                {
+                    progressMonitor.FailedToReadFile(file, ex);
+                    continue;
                 }
             }
 
-        public void RestoreSolutions(IEnumerable<string> solutions)
+            foreach (var package in notYetDownloadedPackages)
             {
-            Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = 4 }, Restore);
+                progressMonitor.NugetInstall(package);
+                using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package));
+                var success = dotnet.New(tempDir.DirInfo.FullName);
+                if (!success)
+                {
+                    continue;
+                }
+                success = dotnet.AddPackage(tempDir.DirInfo.FullName, package);
+                if (!success)
+                {
+                    continue;
                 }
 
-        public void AnalyseSolutions(IEnumerable<string> solutions)
+                success = Restore(tempDir.DirInfo.FullName);
+
+                // TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package.
+
+                if (!success)
+                {
+                    progressMonitor.FailedToRestoreNugetPackage(package);
+                }
+            }
+        }
+
+        private void AnalyseSolutions(IEnumerable<string> solutions)
         {
             Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = 4 }, solutionFile =>
             {
@@ -374,5 +422,8 @@ namespace Semmle.BuildAnalyser
         {
             packageDirectory?.Dispose();
         }
+
+        [GeneratedRegex("<PackageReference .*Include=\"(.*?)\".*/>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
+        private static partial Regex PackageReference();
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNet.cs

@@ -1,17 +1,66 @@
-using System.Diagnostics;
+using System;
+using System.Diagnostics;
 
 namespace Semmle.BuildAnalyser
 {
     /// <summary>
     /// Utilities to run the "dotnet" command.
     /// </summary>
-    internal static class DotNet
+    internal class DotNet
     {
-        public static int RestoreToDirectory(string projectOrSolutionFile, string packageDirectory)
+        private readonly ProgressMonitor progressMonitor;
+
+        public DotNet(ProgressMonitor progressMonitor)
         {
-            using var proc = Process.Start("dotnet", $"restore --no-dependencies \"{projectOrSolutionFile}\" --packages \"{packageDirectory}\" /p:DisableImplicitNuGetFallbackFolder=true");
+            this.progressMonitor = progressMonitor;
+            Info();
+        }
+
+        private void Info()
+        {
+            // TODO: make sure the below `dotnet` version is matching the one specified in global.json
+            progressMonitor.RunningProcess("dotnet --info");
+            using var proc = Process.Start("dotnet", "--info");
             proc.WaitForExit();
-            return proc.ExitCode;
+            var ret = proc.ExitCode;
+
+            if (ret != 0)
+            {
+                progressMonitor.CommandFailed("dotnet", "--info", ret);
+                throw new Exception($"dotnet --info failed with exit code {ret}.");
+            }
+        }
+
+        private bool RunCommand(string args)
+        {
+            progressMonitor.RunningProcess($"dotnet {args}");
+            using var proc = Process.Start("dotnet", args);
+            proc.WaitForExit();
+            if (proc.ExitCode != 0)
+            {
+                progressMonitor.CommandFailed("dotnet", args, proc.ExitCode);
+                return false;
+            }
+
+            return true;
+        }
+
+        public bool RestoreToDirectory(string projectOrSolutionFile, string packageDirectory)
+        {
+            var args = $"restore --no-dependencies \"{projectOrSolutionFile}\" --packages \"{packageDirectory}\" /p:DisableImplicitNuGetFallbackFolder=true";
+            return RunCommand(args);
+        }
+
+        public bool New(string folder)
+        {
+            var args = $"new console --no-restore --output \"{folder}\"";
+            return RunCommand(args);
+        }
+
+        public bool AddPackage(string folder, string package)
+        {
+            var args = $"add \"{folder}\" package \"{package}\" --no-restore";
+            return RunCommand(args);
         }
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs

@@ -17,26 +17,24 @@ namespace Semmle.BuildAnalyser
         /// <summary>
         /// Create the package manager for a specified source tree.
         /// </summary>
-        /// <param name="sourceDir">The source directory.</param>
+        public NugetPackages(string sourceDir, TemporaryDirectory packageDirectory, ProgressMonitor progressMonitor)
-        public NugetPackages(string sourceDir, TemporaryDirectory packageDirectory)
         {
             SourceDirectory = sourceDir;
             PackageDirectory = packageDirectory;
+            this.progressMonitor = progressMonitor;
 
             // Expect nuget.exe to be in a `nuget` directory under the directory containing this exe.
             var currentAssembly = System.Reflection.Assembly.GetExecutingAssembly().Location;
-            var directory = Path.GetDirectoryName(currentAssembly);
+            var directory = Path.GetDirectoryName(currentAssembly)
-            if (directory is null)
+                ?? throw new FileNotFoundException($"Directory path '{currentAssembly}' of current assembly is null");
-                throw new FileNotFoundException($"Directory path '{currentAssembly}' of current assembly is null");
-
             nugetExe = Path.Combine(directory, "nuget", "nuget.exe");
 
             if (!File.Exists(nugetExe))
                 throw new FileNotFoundException(string.Format("NuGet could not be found at {0}", nugetExe));
 
-            packages = new DirectoryInfo(SourceDirectory).
+            packages = new DirectoryInfo(SourceDirectory)
-                EnumerateFiles("packages.config", SearchOption.AllDirectories).
+                .EnumerateFiles("packages.config", SearchOption.AllDirectories)
-                ToArray();
+                .ToArray();
         }
 
         // List of package files to download.
@@ -51,11 +49,11 @@ namespace Semmle.BuildAnalyser
         /// Download the packages to the temp folder.
         /// </summary>
         /// <param name="pm">The progress monitor used for reporting errors etc.</param>
-        public void InstallPackages(IProgressMonitor pm)
+        public void InstallPackages()
         {
             foreach (var package in packages)
             {
-                RestoreNugetPackage(package.FullName, pm);
+                RestoreNugetPackage(package.FullName);
             }
         }
 
@@ -80,9 +78,9 @@ namespace Semmle.BuildAnalyser
         /// </summary>
         /// <param name="package">The package file.</param>
         /// <param name="pm">Where to log progress/errors.</param>
-        private void RestoreNugetPackage(string package, IProgressMonitor pm)
+        private void RestoreNugetPackage(string package)
         {
-            pm.NugetInstall(package);
+            progressMonitor.NugetInstall(package);
 
             /* Use nuget.exe to install a package.
              * Note that there is a clutch of NuGet assemblies which could be used to
@@ -115,7 +113,7 @@ namespace Semmle.BuildAnalyser
 
                 if (p is null)
                 {
-                    pm.FailedNugetCommand(pi.FileName, pi.Arguments, "Couldn't start process.");
+                    progressMonitor.FailedNugetCommand(pi.FileName, pi.Arguments, "Couldn't start process.");
                     return;
                 }
 
@@ -125,16 +123,17 @@ namespace Semmle.BuildAnalyser
                 p.WaitForExit();
                 if (p.ExitCode != 0)
                 {
-                    pm.FailedNugetCommand(pi.FileName, pi.Arguments, output + error);
+                    progressMonitor.FailedNugetCommand(pi.FileName, pi.Arguments, output + error);
                 }
             }
             catch (Exception ex)
                 when (ex is System.ComponentModel.Win32Exception || ex is FileNotFoundException)
             {
-                pm.FailedNugetCommand(pi.FileName, pi.Arguments, ex.Message);
+                progressMonitor.FailedNugetCommand(pi.FileName, pi.Arguments, ex.Message);
             }
         }
 
         private readonly string nugetExe;
+        private readonly ProgressMonitor progressMonitor;
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.Standalone/ProgressMonitor.cs

@@ -3,27 +3,7 @@ using System;
 
 namespace Semmle.BuildAnalyser
 {
-    /// <summary>
+    internal class ProgressMonitor
-    /// Callback for various events that may happen during the build analysis.
-    /// </summary>
-    internal interface IProgressMonitor
-    {
-        void FindingFiles(string dir);
-        void UnresolvedReference(string id, string project);
-        void AnalysingSolution(string filename);
-        void FailedProjectFile(string filename, string reason);
-        void FailedNugetCommand(string exe, string args, string message);
-        void NugetInstall(string package);
-        void ResolvedReference(string filename);
-        void Summary(int existingSources, int usedSources, int missingSources, int references, int unresolvedReferences, int resolvedConflicts, int totalProjects, int failedProjects, TimeSpan analysisTime);
-        void Log(Severity severity, string message);
-        void ResolvedConflict(string asm1, string asm2);
-        void MissingProject(string projectFile);
-        void CommandFailed(string exe, string arguments, int exitCode);
-        void MissingNuGet();
-    }
-
-    internal class ProgressMonitor : IProgressMonitor
     {
         private readonly ILogger logger;
 
@@ -117,5 +97,26 @@ namespace Semmle.BuildAnalyser
         {
             logger.Log(Severity.Error, "Missing nuget.exe");
         }
+
+        public void MissingDotNet()
+        {
+            logger.Log(Severity.Error, "Missing dotnet CLI");
+        }
+
+        public void RunningProcess(string command)
+        {
+            logger.Log(Severity.Info, $"Running {command}");
+        }
+
+        public void FailedToRestoreNugetPackage(string package)
+        {
+            logger.Log(Severity.Info, $"Failed to restore nuget package {package}");
+        }
+
+        public void FailedToReadFile(string file, Exception ex)
+        {
+            logger.Log(Severity.Info, $"Failed to read file {file}");
+            logger.Log(Severity.Debug, $"Failed to read file {file}, exception: {ex}");
+        }
     }
 }