diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyCache.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyCache.cs
index 3a9e2485498..b92a708878a 100644
--- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyCache.cs
+++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyCache.cs
@@ -16,12 +16,12 @@ namespace Semmle.BuildAnalyser
/// Locate all reference files and index them.
///
/// Directories to search.
- /// Callback for progress.
- public AssemblyCache(IEnumerable dirs, IProgressMonitor progress)
+ /// Callback for progress.
+ public AssemblyCache(IEnumerable dirs, ProgressMonitor progressMonitor)
{
foreach (var dir in dirs)
{
- progress.FindingFiles(dir);
+ progressMonitor.FindingFiles(dir);
AddReferenceDirectory(dir);
}
IndexReferences();
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs
index c9d9694ad23..ad501c9e758 100644
--- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs
+++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs
@@ -8,89 +8,60 @@ using System.Threading.Tasks;
using System.Collections.Concurrent;
using System.Text;
using System.Security.Cryptography;
+using System.Text.RegularExpressions;
namespace Semmle.BuildAnalyser
{
- ///
- /// The output of a build analysis.
- ///
- internal interface IBuildAnalysis
- {
- ///
- /// Full filepaths of external references.
- ///
- IEnumerable ReferenceFiles { get; }
-
- ///
- /// Full filepaths of C# source files from project files.
- ///
- IEnumerable ProjectSourceFiles { get; }
-
- ///
- /// Full filepaths of C# source files in the filesystem.
- ///
- IEnumerable AllSourceFiles { get; }
-
- ///
- /// The assembly IDs which could not be resolved.
- ///
- IEnumerable UnresolvedReferences { get; }
-
- ///
- /// List of source files referenced by projects but
- /// which were not found in the filesystem.
- ///
- IEnumerable MissingSourceFiles { get; }
- }
-
///
/// Main implementation of the build analysis.
///
- internal sealed class BuildAnalysis : IBuildAnalysis, IDisposable
+ internal sealed partial class BuildAnalysis : IDisposable
{
private readonly AssemblyCache assemblyCache;
- private readonly IProgressMonitor progressMonitor;
+ private readonly ProgressMonitor progressMonitor;
private readonly IDictionary usedReferences = new ConcurrentDictionary();
private readonly IDictionary sources = new ConcurrentDictionary();
private readonly IDictionary unresolvedReferences = new ConcurrentDictionary();
- private int failedProjects, succeededProjects;
+ private int failedProjects;
+ private int succeededProjects;
private readonly string[] allSources;
private int conflictedReferences = 0;
+ private readonly Options options;
+ private readonly DirectoryInfo sourceDir;
+ private readonly DotNet dotnet;
///
/// Performs a C# build analysis.
///
/// Analysis options from the command line.
- /// Display of analysis progress.
- public BuildAnalysis(Options options, IProgressMonitor progress)
+ /// Display of analysis progress.
+ public BuildAnalysis(Options options, ProgressMonitor progressMonitor)
{
var startTime = DateTime.Now;
- progressMonitor = progress;
- var sourceDir = new DirectoryInfo(options.SrcDir);
+ this.options = options;
+ this.progressMonitor = progressMonitor;
+ this.sourceDir = new DirectoryInfo(options.SrcDir);
- progressMonitor.FindingFiles(options.SrcDir);
+ try
+ {
+ this.dotnet = new DotNet(progressMonitor);
+ }
+ catch
+ {
+ progressMonitor.MissingDotNet();
+ throw;
+ }
- allSources = sourceDir.GetFiles("*.cs", SearchOption.AllDirectories)
- .Select(d => d.FullName)
- .Where(d => !options.ExcludesFile(d))
- .ToArray();
+ this.progressMonitor.FindingFiles(options.SrcDir);
+
+ this.allSources = GetFiles("*.cs").ToArray();
+ var allProjects = GetFiles("*.csproj");
+ var solutions = options.SolutionFile is not null
+ ? new[] { options.SolutionFile }
+ : GetFiles("*.sln");
var dllDirNames = options.DllDirs.Select(Path.GetFullPath).ToList();
- packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName));
-
- if (options.UseNuGet)
- {
- try
- {
- var nuget = new NugetPackages(sourceDir.FullName, packageDirectory);
- nuget.InstallPackages(progressMonitor);
- }
- catch (FileNotFoundException)
- {
- progressMonitor.MissingNuGet();
- }
- }
// Find DLLs in the .Net Framework
if (options.ScanNetFrameworkDlls)
@@ -100,30 +71,43 @@ namespace Semmle.BuildAnalyser
dllDirNames.Add(runtimeLocation);
}
- // TODO: remove the below when the required SDK is installed
- using (new FileRenamer(sourceDir.GetFiles("global.json", SearchOption.AllDirectories)))
- {
- var solutions = options.SolutionFile is not null ?
- new[] { options.SolutionFile } :
- sourceDir.GetFiles("*.sln", SearchOption.AllDirectories).Select(d => d.FullName);
-
- if (options.UseNuGet)
- {
- RestoreSolutions(solutions);
- }
- dllDirNames.Add(packageDirectory.DirInfo.FullName);
- assemblyCache = new BuildAnalyser.AssemblyCache(dllDirNames, progress);
- AnalyseSolutions(solutions);
-
- foreach (var filename in assemblyCache.AllAssemblies.Select(a => a.Filename))
- UseReference(filename);
- }
-
if (options.UseMscorlib)
{
UseReference(typeof(object).Assembly.Location);
}
+ packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName));
+
+ if (options.UseNuGet)
+ {
+ dllDirNames.Add(packageDirectory.DirInfo.FullName);
+ try
+ {
+ var nuget = new NugetPackages(sourceDir.FullName, packageDirectory, progressMonitor);
+ nuget.InstallPackages();
+ }
+ catch (FileNotFoundException)
+ {
+ progressMonitor.MissingNuGet();
+ }
+
+ // TODO: remove the below when the required SDK is installed
+ using (new FileRenamer(sourceDir.GetFiles("global.json", SearchOption.AllDirectories)))
+ {
+ Restore(solutions);
+ Restore(allProjects);
+ DownloadMissingPackages(allProjects);
+ }
+ }
+
+ assemblyCache = new AssemblyCache(dllDirNames, progressMonitor);
+ AnalyseSolutions(solutions);
+
+ foreach (var filename in assemblyCache.AllAssemblies.Select(a => a.Filename))
+ {
+ UseReference(filename);
+ }
+
ResolveConflicts();
// Output the findings
@@ -149,6 +133,13 @@ namespace Semmle.BuildAnalyser
DateTime.Now - startTime);
}
+ private IEnumerable GetFiles(string pattern)
+ {
+ return sourceDir.GetFiles(pattern, SearchOption.AllDirectories)
+ .Select(d => d.FullName)
+ .Where(d => !options.ExcludesFile(d));
+ }
+
///
/// Computes a unique temp directory for the packages associated
/// with this source tree. Use a SHA1 of the directory name.
@@ -158,9 +149,7 @@ namespace Semmle.BuildAnalyser
private static string ComputeTempDirectory(string srcDir)
{
var bytes = Encoding.Unicode.GetBytes(srcDir);
-
- using var sha1 = SHA1.Create();
- var sha = sha1.ComputeHash(bytes);
+ var sha = SHA1.HashData(bytes);
var sb = new StringBuilder();
foreach (var b in sha.Take(8))
sb.AppendFormat("{0:x2}", b);
@@ -195,12 +184,15 @@ namespace Semmle.BuildAnalyser
// Pick the highest version for each assembly name
foreach (var r in sortedReferences)
+ {
finalAssemblyList[r.Name] = r;
-
+ }
// Update the used references list
usedReferences.Clear();
foreach (var r in finalAssemblyList.Select(r => r.Value.Filename))
+ {
UseReference(r);
+ }
// Report the results
foreach (var r in sortedReferences)
@@ -278,7 +270,9 @@ namespace Semmle.BuildAnalyser
private void AnalyseProjectFiles(IEnumerable projectFiles)
{
foreach (var proj in projectFiles)
+ {
AnalyseProject(proj);
+ }
}
private void AnalyseProject(FileInfo project)
@@ -324,36 +318,90 @@ namespace Semmle.BuildAnalyser
}
- private void Restore(string projectOrSolution)
+ private bool Restore(string target)
{
- int exit;
- try
- {
- exit = DotNet.RestoreToDirectory(projectOrSolution, packageDirectory.DirInfo.FullName);
- }
- catch (FileNotFoundException)
- {
- exit = 2;
- }
+ return dotnet.RestoreToDirectory(target, packageDirectory.DirInfo.FullName);
+ }
- switch (exit)
+ private void Restore(IEnumerable targets)
+ {
+ foreach (var target in targets)
{
- case 0:
- case 1:
- // No errors
- break;
- default:
- progressMonitor.CommandFailed("dotnet", $"restore \"{projectOrSolution}\"", exit);
- break;
+ Restore(target);
}
}
- public void RestoreSolutions(IEnumerable solutions)
+ private void DownloadMissingPackages(IEnumerable restoreTargets)
{
- Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = 4 }, Restore);
+ var alreadyDownloadedPackages = Directory.GetDirectories(packageDirectory.DirInfo.FullName).Select(d => Path.GetFileName(d).ToLowerInvariant()).ToHashSet();
+ var notYetDownloadedPackages = new HashSet();
+
+ var allFiles = GetFiles("*.*").ToArray();
+ foreach (var file in allFiles)
+ {
+ try
+ {
+ using var sr = new StreamReader(file);
+ ReadOnlySpan line;
+ while ((line = sr.ReadLine()) != null)
+ {
+ foreach (var valueMatch in PackageReference().EnumerateMatches(line))
+ {
+ // We can't get the group from the ValueMatch, so doing it manually:
+ var match = line.Slice(valueMatch.Index, valueMatch.Length);
+ var includeIndex = match.IndexOf("Include", StringComparison.InvariantCultureIgnoreCase);
+ if (includeIndex == -1)
+ {
+ continue;
+ }
+
+ match = match.Slice(includeIndex + "Include".Length + 1);
+
+ var quoteIndex1 = match.IndexOf("\"");
+ var quoteIndex2 = match.Slice(quoteIndex1 + 1).IndexOf("\"");
+
+ var packageName = match.Slice(quoteIndex1 + 1, quoteIndex2).ToString().ToLowerInvariant();
+ if (!alreadyDownloadedPackages.Contains(packageName))
+ {
+ notYetDownloadedPackages.Add(packageName);
+ }
+ }
+ }
+ }
+ catch (Exception ex)
+ {
+ progressMonitor.FailedToReadFile(file, ex);
+ continue;
+ }
+ }
+
+ foreach (var package in notYetDownloadedPackages)
+ {
+ progressMonitor.NugetInstall(package);
+ using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package));
+ var success = dotnet.New(tempDir.DirInfo.FullName);
+ if (!success)
+ {
+ continue;
+ }
+ success = dotnet.AddPackage(tempDir.DirInfo.FullName, package);
+ if (!success)
+ {
+ continue;
+ }
+
+ success = Restore(tempDir.DirInfo.FullName);
+
+ // TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package.
+
+ if (!success)
+ {
+ progressMonitor.FailedToRestoreNugetPackage(package);
+ }
+ }
}
- public void AnalyseSolutions(IEnumerable solutions)
+ private void AnalyseSolutions(IEnumerable solutions)
{
Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = 4 }, solutionFile =>
{
@@ -374,5 +422,8 @@ namespace Semmle.BuildAnalyser
{
packageDirectory?.Dispose();
}
+
+ [GeneratedRegex("", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
+ private static partial Regex PackageReference();
}
}
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNet.cs
index 4045519d3e0..dbe3b2c4a1e 100644
--- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNet.cs
+++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNet.cs
@@ -1,17 +1,66 @@
-using System.Diagnostics;
+using System;
+using System.Diagnostics;
namespace Semmle.BuildAnalyser
{
///
/// Utilities to run the "dotnet" command.
///
- internal static class DotNet
+ internal class DotNet
{
- public static int RestoreToDirectory(string projectOrSolutionFile, string packageDirectory)
+ private readonly ProgressMonitor progressMonitor;
+
+ public DotNet(ProgressMonitor progressMonitor)
{
- using var proc = Process.Start("dotnet", $"restore --no-dependencies \"{projectOrSolutionFile}\" --packages \"{packageDirectory}\" /p:DisableImplicitNuGetFallbackFolder=true");
+ this.progressMonitor = progressMonitor;
+ Info();
+ }
+
+ private void Info()
+ {
+ // TODO: make sure the below `dotnet` version is matching the one specified in global.json
+ progressMonitor.RunningProcess("dotnet --info");
+ using var proc = Process.Start("dotnet", "--info");
proc.WaitForExit();
- return proc.ExitCode;
+ var ret = proc.ExitCode;
+
+ if (ret != 0)
+ {
+ progressMonitor.CommandFailed("dotnet", "--info", ret);
+ throw new Exception($"dotnet --info failed with exit code {ret}.");
+ }
+ }
+
+ private bool RunCommand(string args)
+ {
+ progressMonitor.RunningProcess($"dotnet {args}");
+ using var proc = Process.Start("dotnet", args);
+ proc.WaitForExit();
+ if (proc.ExitCode != 0)
+ {
+ progressMonitor.CommandFailed("dotnet", args, proc.ExitCode);
+ return false;
+ }
+
+ return true;
+ }
+
+ public bool RestoreToDirectory(string projectOrSolutionFile, string packageDirectory)
+ {
+ var args = $"restore --no-dependencies \"{projectOrSolutionFile}\" --packages \"{packageDirectory}\" /p:DisableImplicitNuGetFallbackFolder=true";
+ return RunCommand(args);
+ }
+
+ public bool New(string folder)
+ {
+ var args = $"new console --no-restore --output \"{folder}\"";
+ return RunCommand(args);
+ }
+
+ public bool AddPackage(string folder, string package)
+ {
+ var args = $"add \"{folder}\" package \"{package}\" --no-restore";
+ return RunCommand(args);
}
}
}
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs
index 94e65d61462..ab5a71dd2c5 100644
--- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs
+++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs
@@ -17,26 +17,24 @@ namespace Semmle.BuildAnalyser
///
/// Create the package manager for a specified source tree.
///
- /// The source directory.
- public NugetPackages(string sourceDir, TemporaryDirectory packageDirectory)
+ public NugetPackages(string sourceDir, TemporaryDirectory packageDirectory, ProgressMonitor progressMonitor)
{
SourceDirectory = sourceDir;
PackageDirectory = packageDirectory;
+ this.progressMonitor = progressMonitor;
// Expect nuget.exe to be in a `nuget` directory under the directory containing this exe.
var currentAssembly = System.Reflection.Assembly.GetExecutingAssembly().Location;
- var directory = Path.GetDirectoryName(currentAssembly);
- if (directory is null)
- throw new FileNotFoundException($"Directory path '{currentAssembly}' of current assembly is null");
-
+ var directory = Path.GetDirectoryName(currentAssembly)
+ ?? throw new FileNotFoundException($"Directory path '{currentAssembly}' of current assembly is null");
nugetExe = Path.Combine(directory, "nuget", "nuget.exe");
if (!File.Exists(nugetExe))
throw new FileNotFoundException(string.Format("NuGet could not be found at {0}", nugetExe));
- packages = new DirectoryInfo(SourceDirectory).
- EnumerateFiles("packages.config", SearchOption.AllDirectories).
- ToArray();
+ packages = new DirectoryInfo(SourceDirectory)
+ .EnumerateFiles("packages.config", SearchOption.AllDirectories)
+ .ToArray();
}
// List of package files to download.
@@ -51,11 +49,11 @@ namespace Semmle.BuildAnalyser
/// Download the packages to the temp folder.
///
/// The progress monitor used for reporting errors etc.
- public void InstallPackages(IProgressMonitor pm)
+ public void InstallPackages()
{
foreach (var package in packages)
{
- RestoreNugetPackage(package.FullName, pm);
+ RestoreNugetPackage(package.FullName);
}
}
@@ -80,9 +78,9 @@ namespace Semmle.BuildAnalyser
///
/// The package file.
/// Where to log progress/errors.
- private void RestoreNugetPackage(string package, IProgressMonitor pm)
+ private void RestoreNugetPackage(string package)
{
- pm.NugetInstall(package);
+ progressMonitor.NugetInstall(package);
/* Use nuget.exe to install a package.
* Note that there is a clutch of NuGet assemblies which could be used to
@@ -115,7 +113,7 @@ namespace Semmle.BuildAnalyser
if (p is null)
{
- pm.FailedNugetCommand(pi.FileName, pi.Arguments, "Couldn't start process.");
+ progressMonitor.FailedNugetCommand(pi.FileName, pi.Arguments, "Couldn't start process.");
return;
}
@@ -125,16 +123,17 @@ namespace Semmle.BuildAnalyser
p.WaitForExit();
if (p.ExitCode != 0)
{
- pm.FailedNugetCommand(pi.FileName, pi.Arguments, output + error);
+ progressMonitor.FailedNugetCommand(pi.FileName, pi.Arguments, output + error);
}
}
catch (Exception ex)
when (ex is System.ComponentModel.Win32Exception || ex is FileNotFoundException)
{
- pm.FailedNugetCommand(pi.FileName, pi.Arguments, ex.Message);
+ progressMonitor.FailedNugetCommand(pi.FileName, pi.Arguments, ex.Message);
}
}
private readonly string nugetExe;
+ private readonly ProgressMonitor progressMonitor;
}
}
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/ProgressMonitor.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/ProgressMonitor.cs
index 5b1da929251..de1c5274c37 100644
--- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/ProgressMonitor.cs
+++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/ProgressMonitor.cs
@@ -3,27 +3,7 @@ using System;
namespace Semmle.BuildAnalyser
{
- ///
- /// Callback for various events that may happen during the build analysis.
- ///
- internal interface IProgressMonitor
- {
- void FindingFiles(string dir);
- void UnresolvedReference(string id, string project);
- void AnalysingSolution(string filename);
- void FailedProjectFile(string filename, string reason);
- void FailedNugetCommand(string exe, string args, string message);
- void NugetInstall(string package);
- void ResolvedReference(string filename);
- void Summary(int existingSources, int usedSources, int missingSources, int references, int unresolvedReferences, int resolvedConflicts, int totalProjects, int failedProjects, TimeSpan analysisTime);
- void Log(Severity severity, string message);
- void ResolvedConflict(string asm1, string asm2);
- void MissingProject(string projectFile);
- void CommandFailed(string exe, string arguments, int exitCode);
- void MissingNuGet();
- }
-
- internal class ProgressMonitor : IProgressMonitor
+ internal class ProgressMonitor
{
private readonly ILogger logger;
@@ -117,5 +97,26 @@ namespace Semmle.BuildAnalyser
{
logger.Log(Severity.Error, "Missing nuget.exe");
}
+
+ public void MissingDotNet()
+ {
+ logger.Log(Severity.Error, "Missing dotnet CLI");
+ }
+
+ public void RunningProcess(string command)
+ {
+ logger.Log(Severity.Info, $"Running {command}");
+ }
+
+ public void FailedToRestoreNugetPackage(string package)
+ {
+ logger.Log(Severity.Info, $"Failed to restore nuget package {package}");
+ }
+
+ public void FailedToReadFile(string file, Exception ex)
+ {
+ logger.Log(Severity.Info, $"Failed to read file {file}");
+ logger.Log(Severity.Debug, $"Failed to read file {file}, exception: {ex}");
+ }
}
}