diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyCache.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyCache.cs index 3a9e2485498..b92a708878a 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyCache.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyCache.cs @@ -16,12 +16,12 @@ namespace Semmle.BuildAnalyser /// Locate all reference files and index them. /// /// Directories to search. - /// Callback for progress. - public AssemblyCache(IEnumerable dirs, IProgressMonitor progress) + /// Callback for progress. + public AssemblyCache(IEnumerable dirs, ProgressMonitor progressMonitor) { foreach (var dir in dirs) { - progress.FindingFiles(dir); + progressMonitor.FindingFiles(dir); AddReferenceDirectory(dir); } IndexReferences(); diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs index c9d9694ad23..ad501c9e758 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs @@ -8,89 +8,60 @@ using System.Threading.Tasks; using System.Collections.Concurrent; using System.Text; using System.Security.Cryptography; +using System.Text.RegularExpressions; namespace Semmle.BuildAnalyser { - /// - /// The output of a build analysis. - /// - internal interface IBuildAnalysis - { - /// - /// Full filepaths of external references. - /// - IEnumerable ReferenceFiles { get; } - - /// - /// Full filepaths of C# source files from project files. - /// - IEnumerable ProjectSourceFiles { get; } - - /// - /// Full filepaths of C# source files in the filesystem. - /// - IEnumerable AllSourceFiles { get; } - - /// - /// The assembly IDs which could not be resolved. - /// - IEnumerable UnresolvedReferences { get; } - - /// - /// List of source files referenced by projects but - /// which were not found in the filesystem. - /// - IEnumerable MissingSourceFiles { get; } - } - /// /// Main implementation of the build analysis. /// - internal sealed class BuildAnalysis : IBuildAnalysis, IDisposable + internal sealed partial class BuildAnalysis : IDisposable { private readonly AssemblyCache assemblyCache; - private readonly IProgressMonitor progressMonitor; + private readonly ProgressMonitor progressMonitor; private readonly IDictionary usedReferences = new ConcurrentDictionary(); private readonly IDictionary sources = new ConcurrentDictionary(); private readonly IDictionary unresolvedReferences = new ConcurrentDictionary(); - private int failedProjects, succeededProjects; + private int failedProjects; + private int succeededProjects; private readonly string[] allSources; private int conflictedReferences = 0; + private readonly Options options; + private readonly DirectoryInfo sourceDir; + private readonly DotNet dotnet; /// /// Performs a C# build analysis. /// /// Analysis options from the command line. - /// Display of analysis progress. - public BuildAnalysis(Options options, IProgressMonitor progress) + /// Display of analysis progress. + public BuildAnalysis(Options options, ProgressMonitor progressMonitor) { var startTime = DateTime.Now; - progressMonitor = progress; - var sourceDir = new DirectoryInfo(options.SrcDir); + this.options = options; + this.progressMonitor = progressMonitor; + this.sourceDir = new DirectoryInfo(options.SrcDir); - progressMonitor.FindingFiles(options.SrcDir); + try + { + this.dotnet = new DotNet(progressMonitor); + } + catch + { + progressMonitor.MissingDotNet(); + throw; + } - allSources = sourceDir.GetFiles("*.cs", SearchOption.AllDirectories) - .Select(d => d.FullName) - .Where(d => !options.ExcludesFile(d)) - .ToArray(); + this.progressMonitor.FindingFiles(options.SrcDir); + + this.allSources = GetFiles("*.cs").ToArray(); + var allProjects = GetFiles("*.csproj"); + var solutions = options.SolutionFile is not null + ? new[] { options.SolutionFile } + : GetFiles("*.sln"); var dllDirNames = options.DllDirs.Select(Path.GetFullPath).ToList(); - packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName)); - - if (options.UseNuGet) - { - try - { - var nuget = new NugetPackages(sourceDir.FullName, packageDirectory); - nuget.InstallPackages(progressMonitor); - } - catch (FileNotFoundException) - { - progressMonitor.MissingNuGet(); - } - } // Find DLLs in the .Net Framework if (options.ScanNetFrameworkDlls) @@ -100,30 +71,43 @@ namespace Semmle.BuildAnalyser dllDirNames.Add(runtimeLocation); } - // TODO: remove the below when the required SDK is installed - using (new FileRenamer(sourceDir.GetFiles("global.json", SearchOption.AllDirectories))) - { - var solutions = options.SolutionFile is not null ? - new[] { options.SolutionFile } : - sourceDir.GetFiles("*.sln", SearchOption.AllDirectories).Select(d => d.FullName); - - if (options.UseNuGet) - { - RestoreSolutions(solutions); - } - dllDirNames.Add(packageDirectory.DirInfo.FullName); - assemblyCache = new BuildAnalyser.AssemblyCache(dllDirNames, progress); - AnalyseSolutions(solutions); - - foreach (var filename in assemblyCache.AllAssemblies.Select(a => a.Filename)) - UseReference(filename); - } - if (options.UseMscorlib) { UseReference(typeof(object).Assembly.Location); } + packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName)); + + if (options.UseNuGet) + { + dllDirNames.Add(packageDirectory.DirInfo.FullName); + try + { + var nuget = new NugetPackages(sourceDir.FullName, packageDirectory, progressMonitor); + nuget.InstallPackages(); + } + catch (FileNotFoundException) + { + progressMonitor.MissingNuGet(); + } + + // TODO: remove the below when the required SDK is installed + using (new FileRenamer(sourceDir.GetFiles("global.json", SearchOption.AllDirectories))) + { + Restore(solutions); + Restore(allProjects); + DownloadMissingPackages(allProjects); + } + } + + assemblyCache = new AssemblyCache(dllDirNames, progressMonitor); + AnalyseSolutions(solutions); + + foreach (var filename in assemblyCache.AllAssemblies.Select(a => a.Filename)) + { + UseReference(filename); + } + ResolveConflicts(); // Output the findings @@ -149,6 +133,13 @@ namespace Semmle.BuildAnalyser DateTime.Now - startTime); } + private IEnumerable GetFiles(string pattern) + { + return sourceDir.GetFiles(pattern, SearchOption.AllDirectories) + .Select(d => d.FullName) + .Where(d => !options.ExcludesFile(d)); + } + /// /// Computes a unique temp directory for the packages associated /// with this source tree. Use a SHA1 of the directory name. @@ -158,9 +149,7 @@ namespace Semmle.BuildAnalyser private static string ComputeTempDirectory(string srcDir) { var bytes = Encoding.Unicode.GetBytes(srcDir); - - using var sha1 = SHA1.Create(); - var sha = sha1.ComputeHash(bytes); + var sha = SHA1.HashData(bytes); var sb = new StringBuilder(); foreach (var b in sha.Take(8)) sb.AppendFormat("{0:x2}", b); @@ -195,12 +184,15 @@ namespace Semmle.BuildAnalyser // Pick the highest version for each assembly name foreach (var r in sortedReferences) + { finalAssemblyList[r.Name] = r; - + } // Update the used references list usedReferences.Clear(); foreach (var r in finalAssemblyList.Select(r => r.Value.Filename)) + { UseReference(r); + } // Report the results foreach (var r in sortedReferences) @@ -278,7 +270,9 @@ namespace Semmle.BuildAnalyser private void AnalyseProjectFiles(IEnumerable projectFiles) { foreach (var proj in projectFiles) + { AnalyseProject(proj); + } } private void AnalyseProject(FileInfo project) @@ -324,36 +318,90 @@ namespace Semmle.BuildAnalyser } - private void Restore(string projectOrSolution) + private bool Restore(string target) { - int exit; - try - { - exit = DotNet.RestoreToDirectory(projectOrSolution, packageDirectory.DirInfo.FullName); - } - catch (FileNotFoundException) - { - exit = 2; - } + return dotnet.RestoreToDirectory(target, packageDirectory.DirInfo.FullName); + } - switch (exit) + private void Restore(IEnumerable targets) + { + foreach (var target in targets) { - case 0: - case 1: - // No errors - break; - default: - progressMonitor.CommandFailed("dotnet", $"restore \"{projectOrSolution}\"", exit); - break; + Restore(target); } } - public void RestoreSolutions(IEnumerable solutions) + private void DownloadMissingPackages(IEnumerable restoreTargets) { - Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = 4 }, Restore); + var alreadyDownloadedPackages = Directory.GetDirectories(packageDirectory.DirInfo.FullName).Select(d => Path.GetFileName(d).ToLowerInvariant()).ToHashSet(); + var notYetDownloadedPackages = new HashSet(); + + var allFiles = GetFiles("*.*").ToArray(); + foreach (var file in allFiles) + { + try + { + using var sr = new StreamReader(file); + ReadOnlySpan line; + while ((line = sr.ReadLine()) != null) + { + foreach (var valueMatch in PackageReference().EnumerateMatches(line)) + { + // We can't get the group from the ValueMatch, so doing it manually: + var match = line.Slice(valueMatch.Index, valueMatch.Length); + var includeIndex = match.IndexOf("Include", StringComparison.InvariantCultureIgnoreCase); + if (includeIndex == -1) + { + continue; + } + + match = match.Slice(includeIndex + "Include".Length + 1); + + var quoteIndex1 = match.IndexOf("\""); + var quoteIndex2 = match.Slice(quoteIndex1 + 1).IndexOf("\""); + + var packageName = match.Slice(quoteIndex1 + 1, quoteIndex2).ToString().ToLowerInvariant(); + if (!alreadyDownloadedPackages.Contains(packageName)) + { + notYetDownloadedPackages.Add(packageName); + } + } + } + } + catch (Exception ex) + { + progressMonitor.FailedToReadFile(file, ex); + continue; + } + } + + foreach (var package in notYetDownloadedPackages) + { + progressMonitor.NugetInstall(package); + using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package)); + var success = dotnet.New(tempDir.DirInfo.FullName); + if (!success) + { + continue; + } + success = dotnet.AddPackage(tempDir.DirInfo.FullName, package); + if (!success) + { + continue; + } + + success = Restore(tempDir.DirInfo.FullName); + + // TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package. + + if (!success) + { + progressMonitor.FailedToRestoreNugetPackage(package); + } + } } - public void AnalyseSolutions(IEnumerable solutions) + private void AnalyseSolutions(IEnumerable solutions) { Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = 4 }, solutionFile => { @@ -374,5 +422,8 @@ namespace Semmle.BuildAnalyser { packageDirectory?.Dispose(); } + + [GeneratedRegex("", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)] + private static partial Regex PackageReference(); } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNet.cs index 4045519d3e0..dbe3b2c4a1e 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNet.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNet.cs @@ -1,17 +1,66 @@ -using System.Diagnostics; +using System; +using System.Diagnostics; namespace Semmle.BuildAnalyser { /// /// Utilities to run the "dotnet" command. /// - internal static class DotNet + internal class DotNet { - public static int RestoreToDirectory(string projectOrSolutionFile, string packageDirectory) + private readonly ProgressMonitor progressMonitor; + + public DotNet(ProgressMonitor progressMonitor) { - using var proc = Process.Start("dotnet", $"restore --no-dependencies \"{projectOrSolutionFile}\" --packages \"{packageDirectory}\" /p:DisableImplicitNuGetFallbackFolder=true"); + this.progressMonitor = progressMonitor; + Info(); + } + + private void Info() + { + // TODO: make sure the below `dotnet` version is matching the one specified in global.json + progressMonitor.RunningProcess("dotnet --info"); + using var proc = Process.Start("dotnet", "--info"); proc.WaitForExit(); - return proc.ExitCode; + var ret = proc.ExitCode; + + if (ret != 0) + { + progressMonitor.CommandFailed("dotnet", "--info", ret); + throw new Exception($"dotnet --info failed with exit code {ret}."); + } + } + + private bool RunCommand(string args) + { + progressMonitor.RunningProcess($"dotnet {args}"); + using var proc = Process.Start("dotnet", args); + proc.WaitForExit(); + if (proc.ExitCode != 0) + { + progressMonitor.CommandFailed("dotnet", args, proc.ExitCode); + return false; + } + + return true; + } + + public bool RestoreToDirectory(string projectOrSolutionFile, string packageDirectory) + { + var args = $"restore --no-dependencies \"{projectOrSolutionFile}\" --packages \"{packageDirectory}\" /p:DisableImplicitNuGetFallbackFolder=true"; + return RunCommand(args); + } + + public bool New(string folder) + { + var args = $"new console --no-restore --output \"{folder}\""; + return RunCommand(args); + } + + public bool AddPackage(string folder, string package) + { + var args = $"add \"{folder}\" package \"{package}\" --no-restore"; + return RunCommand(args); } } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs index 94e65d61462..ab5a71dd2c5 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs @@ -17,26 +17,24 @@ namespace Semmle.BuildAnalyser /// /// Create the package manager for a specified source tree. /// - /// The source directory. - public NugetPackages(string sourceDir, TemporaryDirectory packageDirectory) + public NugetPackages(string sourceDir, TemporaryDirectory packageDirectory, ProgressMonitor progressMonitor) { SourceDirectory = sourceDir; PackageDirectory = packageDirectory; + this.progressMonitor = progressMonitor; // Expect nuget.exe to be in a `nuget` directory under the directory containing this exe. var currentAssembly = System.Reflection.Assembly.GetExecutingAssembly().Location; - var directory = Path.GetDirectoryName(currentAssembly); - if (directory is null) - throw new FileNotFoundException($"Directory path '{currentAssembly}' of current assembly is null"); - + var directory = Path.GetDirectoryName(currentAssembly) + ?? throw new FileNotFoundException($"Directory path '{currentAssembly}' of current assembly is null"); nugetExe = Path.Combine(directory, "nuget", "nuget.exe"); if (!File.Exists(nugetExe)) throw new FileNotFoundException(string.Format("NuGet could not be found at {0}", nugetExe)); - packages = new DirectoryInfo(SourceDirectory). - EnumerateFiles("packages.config", SearchOption.AllDirectories). - ToArray(); + packages = new DirectoryInfo(SourceDirectory) + .EnumerateFiles("packages.config", SearchOption.AllDirectories) + .ToArray(); } // List of package files to download. @@ -51,11 +49,11 @@ namespace Semmle.BuildAnalyser /// Download the packages to the temp folder. /// /// The progress monitor used for reporting errors etc. - public void InstallPackages(IProgressMonitor pm) + public void InstallPackages() { foreach (var package in packages) { - RestoreNugetPackage(package.FullName, pm); + RestoreNugetPackage(package.FullName); } } @@ -80,9 +78,9 @@ namespace Semmle.BuildAnalyser /// /// The package file. /// Where to log progress/errors. - private void RestoreNugetPackage(string package, IProgressMonitor pm) + private void RestoreNugetPackage(string package) { - pm.NugetInstall(package); + progressMonitor.NugetInstall(package); /* Use nuget.exe to install a package. * Note that there is a clutch of NuGet assemblies which could be used to @@ -115,7 +113,7 @@ namespace Semmle.BuildAnalyser if (p is null) { - pm.FailedNugetCommand(pi.FileName, pi.Arguments, "Couldn't start process."); + progressMonitor.FailedNugetCommand(pi.FileName, pi.Arguments, "Couldn't start process."); return; } @@ -125,16 +123,17 @@ namespace Semmle.BuildAnalyser p.WaitForExit(); if (p.ExitCode != 0) { - pm.FailedNugetCommand(pi.FileName, pi.Arguments, output + error); + progressMonitor.FailedNugetCommand(pi.FileName, pi.Arguments, output + error); } } catch (Exception ex) when (ex is System.ComponentModel.Win32Exception || ex is FileNotFoundException) { - pm.FailedNugetCommand(pi.FileName, pi.Arguments, ex.Message); + progressMonitor.FailedNugetCommand(pi.FileName, pi.Arguments, ex.Message); } } private readonly string nugetExe; + private readonly ProgressMonitor progressMonitor; } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/ProgressMonitor.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/ProgressMonitor.cs index 5b1da929251..de1c5274c37 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/ProgressMonitor.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/ProgressMonitor.cs @@ -3,27 +3,7 @@ using System; namespace Semmle.BuildAnalyser { - /// - /// Callback for various events that may happen during the build analysis. - /// - internal interface IProgressMonitor - { - void FindingFiles(string dir); - void UnresolvedReference(string id, string project); - void AnalysingSolution(string filename); - void FailedProjectFile(string filename, string reason); - void FailedNugetCommand(string exe, string args, string message); - void NugetInstall(string package); - void ResolvedReference(string filename); - void Summary(int existingSources, int usedSources, int missingSources, int references, int unresolvedReferences, int resolvedConflicts, int totalProjects, int failedProjects, TimeSpan analysisTime); - void Log(Severity severity, string message); - void ResolvedConflict(string asm1, string asm2); - void MissingProject(string projectFile); - void CommandFailed(string exe, string arguments, int exitCode); - void MissingNuGet(); - } - - internal class ProgressMonitor : IProgressMonitor + internal class ProgressMonitor { private readonly ILogger logger; @@ -117,5 +97,26 @@ namespace Semmle.BuildAnalyser { logger.Log(Severity.Error, "Missing nuget.exe"); } + + public void MissingDotNet() + { + logger.Log(Severity.Error, "Missing dotnet CLI"); + } + + public void RunningProcess(string command) + { + logger.Log(Severity.Info, $"Running {command}"); + } + + public void FailedToRestoreNugetPackage(string package) + { + logger.Log(Severity.Info, $"Failed to restore nuget package {package}"); + } + + public void FailedToReadFile(string file, Exception ex) + { + logger.Log(Severity.Info, $"Failed to read file {file}"); + logger.Log(Severity.Debug, $"Failed to read file {file}, exception: {ex}"); + } } }