hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-22 03:36:30 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #13658 from tamasvajk/cs/standalone/restore-impr

Browse Source 
C#: Improve dotnet restore success rate in standalone extraction
This commit is contained in:
Tamás Vajk
2023-07-06 10:10:05 +02:00
committed by GitHub
parent fab231c284 d0b8b683af
commit 14caaf119c
5 changed files with 247 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
csharp/extractor/Semmle.Extraction.CSharp.Standalone/AssemblyCache.cs
View File

@@ -16,12 +16,12 @@ namespace Semmle.BuildAnalyser
/// Locate all reference files and index them.
/// </summary>
/// <param name="dirs">Directories to search.</param>
/// <param name="progress">Callback for progress.</param>
public AssemblyCache(IEnumerable<string> dirs, IProgressMonitor progress)
/// <param name="progressMonitor">Callback for progress.</param>
public AssemblyCache(IEnumerable<string> dirs, ProgressMonitor progressMonitor)
{
foreach (var dir in dirs)
{
progress.FindingFiles(dir);
progressMonitor.FindingFiles(dir);
AddReferenceDirectory(dir);
}
IndexReferences();

255
csharp/extractor/Semmle.Extraction.CSharp.Standalone/BuildAnalysis.cs
View File

@@ -8,89 +8,60 @@ using System.Threading.Tasks;
using System.Collections.Concurrent;
using System.Text;
using System.Security.Cryptography;
using System.Text.RegularExpressions;
namespace Semmle.BuildAnalyser
{
/// <summary>
/// The output of a build analysis.
/// </summary>
internal interface IBuildAnalysis
{
/// <summary>
/// Full filepaths of external references.
/// </summary>
IEnumerable<string> ReferenceFiles { get; }
/// <summary>
/// Full filepaths of C# source files from project files.
/// </summary>
IEnumerable<string> ProjectSourceFiles { get; }
/// <summary>
/// Full filepaths of C# source files in the filesystem.
/// </summary>
IEnumerable<string> AllSourceFiles { get; }
/// <summary>
/// The assembly IDs which could not be resolved.
/// </summary>
IEnumerable<string> UnresolvedReferences { get; }
/// <summary>
/// List of source files referenced by projects but
/// which were not found in the filesystem.
/// </summary>
IEnumerable<string> MissingSourceFiles { get; }
}
/// <summary>
/// Main implementation of the build analysis.
/// </summary>
internal sealed class BuildAnalysis : IBuildAnalysis, IDisposable
internal sealed partial class BuildAnalysis : IDisposable
{
private readonly AssemblyCache assemblyCache;
private readonly IProgressMonitor progressMonitor;
private readonly ProgressMonitor progressMonitor;
private readonly IDictionary<string, bool> usedReferences = new ConcurrentDictionary<string, bool>();
private readonly IDictionary<string, bool> sources = new ConcurrentDictionary<string, bool>();
private readonly IDictionary<string, string> unresolvedReferences = new ConcurrentDictionary<string, string>();
private int failedProjects, succeededProjects;
private int failedProjects;
private int succeededProjects;
private readonly string[] allSources;
private int conflictedReferences = 0;
private readonly Options options;
private readonly DirectoryInfo sourceDir;
private readonly DotNet dotnet;
/// <summary>
/// Performs a C# build analysis.
/// </summary>
/// <param name="options">Analysis options from the command line.</param>
/// <param name="progress">Display of analysis progress.</param>
public BuildAnalysis(Options options, IProgressMonitor progress)
/// <param name="progressMonitor">Display of analysis progress.</param>
public BuildAnalysis(Options options, ProgressMonitor progressMonitor)
{
var startTime = DateTime.Now;
progressMonitor = progress;
var sourceDir = new DirectoryInfo(options.SrcDir);
this.options = options;
this.progressMonitor = progressMonitor;
this.sourceDir = new DirectoryInfo(options.SrcDir);
progressMonitor.FindingFiles(options.SrcDir);
try
{
this.dotnet = new DotNet(progressMonitor);
}
catch
{
progressMonitor.MissingDotNet();
throw;
}
allSources = sourceDir.GetFiles("*.cs", SearchOption.AllDirectories)
.Select(d => d.FullName)
.Where(d => !options.ExcludesFile(d))
.ToArray();
this.progressMonitor.FindingFiles(options.SrcDir);
this.allSources = GetFiles("*.cs").ToArray();
var allProjects = GetFiles("*.csproj");
var solutions = options.SolutionFile is not null
? new[] { options.SolutionFile }
: GetFiles("*.sln");
var dllDirNames = options.DllDirs.Select(Path.GetFullPath).ToList();
packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName));
if (options.UseNuGet)
{
try
{
var nuget = new NugetPackages(sourceDir.FullName, packageDirectory);
nuget.InstallPackages(progressMonitor);
}
catch (FileNotFoundException)
{
progressMonitor.MissingNuGet();
}
}
// Find DLLs in the .Net Framework
if (options.ScanNetFrameworkDlls)
@@ -100,30 +71,43 @@ namespace Semmle.BuildAnalyser
dllDirNames.Add(runtimeLocation);
}
// TODO: remove the below when the required SDK is installed
using (new FileRenamer(sourceDir.GetFiles("global.json", SearchOption.AllDirectories)))
{
var solutions = options.SolutionFile is not null ?
new[] { options.SolutionFile } :
sourceDir.GetFiles("*.sln", SearchOption.AllDirectories).Select(d => d.FullName);
if (options.UseNuGet)
{
RestoreSolutions(solutions);
}
dllDirNames.Add(packageDirectory.DirInfo.FullName);
assemblyCache = new BuildAnalyser.AssemblyCache(dllDirNames, progress);
AnalyseSolutions(solutions);
foreach (var filename in assemblyCache.AllAssemblies.Select(a => a.Filename))
UseReference(filename);
}
if (options.UseMscorlib)
{
UseReference(typeof(object).Assembly.Location);
}
packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName));
if (options.UseNuGet)
{
dllDirNames.Add(packageDirectory.DirInfo.FullName);
try
{
var nuget = new NugetPackages(sourceDir.FullName, packageDirectory, progressMonitor);
nuget.InstallPackages();
}
catch (FileNotFoundException)
{
progressMonitor.MissingNuGet();
}
// TODO: remove the below when the required SDK is installed
using (new FileRenamer(sourceDir.GetFiles("global.json", SearchOption.AllDirectories)))
{
Restore(solutions);
Restore(allProjects);
DownloadMissingPackages(allProjects);
}
}
assemblyCache = new AssemblyCache(dllDirNames, progressMonitor);
AnalyseSolutions(solutions);
foreach (var filename in assemblyCache.AllAssemblies.Select(a => a.Filename))
{
UseReference(filename);
}
ResolveConflicts();
// Output the findings
@@ -149,6 +133,13 @@ namespace Semmle.BuildAnalyser
DateTime.Now - startTime);
}
private IEnumerable<string> GetFiles(string pattern)
{
return sourceDir.GetFiles(pattern, SearchOption.AllDirectories)
.Select(d => d.FullName)
.Where(d => !options.ExcludesFile(d));
}
/// <summary>
/// Computes a unique temp directory for the packages associated
/// with this source tree. Use a SHA1 of the directory name.
@@ -158,9 +149,7 @@ namespace Semmle.BuildAnalyser
private static string ComputeTempDirectory(string srcDir)
{
var bytes = Encoding.Unicode.GetBytes(srcDir);
using var sha1 = SHA1.Create();
var sha = sha1.ComputeHash(bytes);
var sha = SHA1.HashData(bytes);
var sb = new StringBuilder();
foreach (var b in sha.Take(8))
sb.AppendFormat("{0:x2}", b);
@@ -195,12 +184,15 @@ namespace Semmle.BuildAnalyser
// Pick the highest version for each assembly name
foreach (var r in sortedReferences)
{
finalAssemblyList[r.Name] = r;
}
// Update the used references list
usedReferences.Clear();
foreach (var r in finalAssemblyList.Select(r => r.Value.Filename))
{
UseReference(r);
}
// Report the results
foreach (var r in sortedReferences)
@@ -278,7 +270,9 @@ namespace Semmle.BuildAnalyser
private void AnalyseProjectFiles(IEnumerable<FileInfo> projectFiles)
{
foreach (var proj in projectFiles)
{
AnalyseProject(proj);
}
}
private void AnalyseProject(FileInfo project)
@@ -324,36 +318,90 @@ namespace Semmle.BuildAnalyser
}
private void Restore(string projectOrSolution)
private bool Restore(string target)
{
int exit;
try
{
exit = DotNet.RestoreToDirectory(projectOrSolution, packageDirectory.DirInfo.FullName);
}
catch (FileNotFoundException)
{
exit = 2;
}
return dotnet.RestoreToDirectory(target, packageDirectory.DirInfo.FullName);
}
switch (exit)
private void Restore(IEnumerable<string> targets)
{
foreach (var target in targets)
{
case 0:
case 1:
// No errors
break;
default:
progressMonitor.CommandFailed("dotnet", $"restore \"{projectOrSolution}\"", exit);
break;
Restore(target);
}
}
public void RestoreSolutions(IEnumerable<string> solutions)
private void DownloadMissingPackages(IEnumerable<string> restoreTargets)
{
Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = 4 }, Restore);
var alreadyDownloadedPackages = Directory.GetDirectories(packageDirectory.DirInfo.FullName).Select(d => Path.GetFileName(d).ToLowerInvariant()).ToHashSet();
var notYetDownloadedPackages = new HashSet<string>();
var allFiles = GetFiles("*.*").ToArray();
foreach (var file in allFiles)
{
try
{
using var sr = new StreamReader(file);
ReadOnlySpan<char> line;
while ((line = sr.ReadLine()) != null)
{
foreach (var valueMatch in PackageReference().EnumerateMatches(line))
{
// We can't get the group from the ValueMatch, so doing it manually:
var match = line.Slice(valueMatch.Index, valueMatch.Length);
var includeIndex = match.IndexOf("Include", StringComparison.InvariantCultureIgnoreCase);
if (includeIndex == -1)
{
continue;
}
match = match.Slice(includeIndex + "Include".Length + 1);
var quoteIndex1 = match.IndexOf("\"");
var quoteIndex2 = match.Slice(quoteIndex1 + 1).IndexOf("\"");
var packageName = match.Slice(quoteIndex1 + 1, quoteIndex2).ToString().ToLowerInvariant();
if (!alreadyDownloadedPackages.Contains(packageName))
{
notYetDownloadedPackages.Add(packageName);
}
}
}
}
catch (Exception ex)
{
progressMonitor.FailedToReadFile(file, ex);
continue;
}
}
foreach (var package in notYetDownloadedPackages)
{
progressMonitor.NugetInstall(package);
using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package));
var success = dotnet.New(tempDir.DirInfo.FullName);
if (!success)
{
continue;
}
success = dotnet.AddPackage(tempDir.DirInfo.FullName, package);
if (!success)
{
continue;
}
success = Restore(tempDir.DirInfo.FullName);
// TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package.
if (!success)
{
progressMonitor.FailedToRestoreNugetPackage(package);
}
}
}
public void AnalyseSolutions(IEnumerable<string> solutions)
private void AnalyseSolutions(IEnumerable<string> solutions)
{
Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = 4 }, solutionFile =>
{
@@ -374,5 +422,8 @@ namespace Semmle.BuildAnalyser
{
packageDirectory?.Dispose();
}
[GeneratedRegex("<PackageReference .*Include=\"(.*?)\".*/>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
private static partial Regex PackageReference();
}
}

59
csharp/extractor/Semmle.Extraction.CSharp.Standalone/DotNet.cs
View File

@@ -1,17 +1,66 @@
using System.Diagnostics;
using System;
using System.Diagnostics;
namespace Semmle.BuildAnalyser
{
/// <summary>
/// Utilities to run the "dotnet" command.
/// </summary>
internal static class DotNet
internal class DotNet
{
public static int RestoreToDirectory(string projectOrSolutionFile, string packageDirectory)
private readonly ProgressMonitor progressMonitor;
public DotNet(ProgressMonitor progressMonitor)
{
using var proc = Process.Start("dotnet", $"restore --no-dependencies \"{projectOrSolutionFile}\" --packages \"{packageDirectory}\" /p:DisableImplicitNuGetFallbackFolder=true");
this.progressMonitor = progressMonitor;
Info();
}
private void Info()
{
// TODO: make sure the below `dotnet` version is matching the one specified in global.json
progressMonitor.RunningProcess("dotnet --info");
using var proc = Process.Start("dotnet", "--info");
proc.WaitForExit();
return proc.ExitCode;
var ret = proc.ExitCode;
if (ret != 0)
{
progressMonitor.CommandFailed("dotnet", "--info", ret);
throw new Exception($"dotnet --info failed with exit code {ret}.");
}
}
private bool RunCommand(string args)
{
progressMonitor.RunningProcess($"dotnet {args}");
using var proc = Process.Start("dotnet", args);
proc.WaitForExit();
if (proc.ExitCode != 0)
{
progressMonitor.CommandFailed("dotnet", args, proc.ExitCode);
return false;
}
return true;
}
public bool RestoreToDirectory(string projectOrSolutionFile, string packageDirectory)
{
var args = $"restore --no-dependencies \"{projectOrSolutionFile}\" --packages \"{packageDirectory}\" /p:DisableImplicitNuGetFallbackFolder=true";
return RunCommand(args);
}
public bool New(string folder)
{
var args = $"new console --no-restore --output \"{folder}\"";
return RunCommand(args);
}
public bool AddPackage(string folder, string package)
{
var args = $"add \"{folder}\" package \"{package}\" --no-restore";
return RunCommand(args);
}
}
}

31
csharp/extractor/Semmle.Extraction.CSharp.Standalone/NugetPackages.cs
View File

@@ -17,26 +17,24 @@ namespace Semmle.BuildAnalyser
/// <summary>
/// Create the package manager for a specified source tree.
/// </summary>
/// <param name="sourceDir">The source directory.</param>
public NugetPackages(string sourceDir, TemporaryDirectory packageDirectory)
public NugetPackages(string sourceDir, TemporaryDirectory packageDirectory, ProgressMonitor progressMonitor)
{
SourceDirectory = sourceDir;
PackageDirectory = packageDirectory;
this.progressMonitor = progressMonitor;
// Expect nuget.exe to be in a `nuget` directory under the directory containing this exe.
var currentAssembly = System.Reflection.Assembly.GetExecutingAssembly().Location;
var directory = Path.GetDirectoryName(currentAssembly);
if (directory is null)
throw new FileNotFoundException($"Directory path '{currentAssembly}' of current assembly is null");
var directory = Path.GetDirectoryName(currentAssembly)
?? throw new FileNotFoundException($"Directory path '{currentAssembly}' of current assembly is null");
nugetExe = Path.Combine(directory, "nuget", "nuget.exe");
if (!File.Exists(nugetExe))
throw new FileNotFoundException(string.Format("NuGet could not be found at {0}", nugetExe));
packages = new DirectoryInfo(SourceDirectory).
EnumerateFiles("packages.config", SearchOption.AllDirectories).
ToArray();
packages = new DirectoryInfo(SourceDirectory)
.EnumerateFiles("packages.config", SearchOption.AllDirectories)
.ToArray();
}
// List of package files to download.
@@ -51,11 +49,11 @@ namespace Semmle.BuildAnalyser
/// Download the packages to the temp folder.
/// </summary>
/// <param name="pm">The progress monitor used for reporting errors etc.</param>
public void InstallPackages(IProgressMonitor pm)
public void InstallPackages()
{
foreach (var package in packages)
{
RestoreNugetPackage(package.FullName, pm);
RestoreNugetPackage(package.FullName);
}
}
@@ -80,9 +78,9 @@ namespace Semmle.BuildAnalyser
/// </summary>
/// <param name="package">The package file.</param>
/// <param name="pm">Where to log progress/errors.</param>
private void RestoreNugetPackage(string package, IProgressMonitor pm)
private void RestoreNugetPackage(string package)
{
pm.NugetInstall(package);
progressMonitor.NugetInstall(package);
/* Use nuget.exe to install a package.
* Note that there is a clutch of NuGet assemblies which could be used to
@@ -115,7 +113,7 @@ namespace Semmle.BuildAnalyser
if (p is null)
{
pm.FailedNugetCommand(pi.FileName, pi.Arguments, "Couldn't start process.");
progressMonitor.FailedNugetCommand(pi.FileName, pi.Arguments, "Couldn't start process.");
return;
}
@@ -125,16 +123,17 @@ namespace Semmle.BuildAnalyser
p.WaitForExit();
if (p.ExitCode != 0)
{
pm.FailedNugetCommand(pi.FileName, pi.Arguments, output + error);
progressMonitor.FailedNugetCommand(pi.FileName, pi.Arguments, output + error);
}
}
catch (Exception ex)
when (ex is System.ComponentModel.Win32Exception || ex is FileNotFoundException)
{
pm.FailedNugetCommand(pi.FileName, pi.Arguments, ex.Message);
progressMonitor.FailedNugetCommand(pi.FileName, pi.Arguments, ex.Message);
}
}
private readonly string nugetExe;
private readonly ProgressMonitor progressMonitor;
}
}

43
csharp/extractor/Semmle.Extraction.CSharp.Standalone/ProgressMonitor.cs
View File

@@ -3,27 +3,7 @@ using System;
namespace Semmle.BuildAnalyser
{
/// <summary>
/// Callback for various events that may happen during the build analysis.
/// </summary>
internal interface IProgressMonitor
{
void FindingFiles(string dir);
void UnresolvedReference(string id, string project);
void AnalysingSolution(string filename);
void FailedProjectFile(string filename, string reason);
void FailedNugetCommand(string exe, string args, string message);
void NugetInstall(string package);
void ResolvedReference(string filename);
void Summary(int existingSources, int usedSources, int missingSources, int references, int unresolvedReferences, int resolvedConflicts, int totalProjects, int failedProjects, TimeSpan analysisTime);
void Log(Severity severity, string message);
void ResolvedConflict(string asm1, string asm2);
void MissingProject(string projectFile);
void CommandFailed(string exe, string arguments, int exitCode);
void MissingNuGet();
}
internal class ProgressMonitor : IProgressMonitor
internal class ProgressMonitor
{
private readonly ILogger logger;
@@ -117,5 +97,26 @@ namespace Semmle.BuildAnalyser
{
logger.Log(Severity.Error, "Missing nuget.exe");
}
public void MissingDotNet()
{
logger.Log(Severity.Error, "Missing dotnet CLI");
}
public void RunningProcess(string command)
{
logger.Log(Severity.Info, $"Running {command}");
}
public void FailedToRestoreNugetPackage(string package)
{
logger.Log(Severity.Info, $"Failed to restore nuget package {package}");
}
public void FailedToReadFile(string file, Exception ex)
{
logger.Log(Severity.Info, $"Failed to read file {file}");
logger.Log(Severity.Debug, $"Failed to read file {file}, exception: {ex}");
}
}
}