hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-04-20 06:24:03 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #14655 from michaelnebel/csharp/projectassetspackages

Browse Source 
C#: Use `project.assets.json` for package dependencies.
This commit is contained in:
Michael Nebel
2023-11-06 16:26:38 +01:00
committed by GitHub
parent 49428c4585 4bcf9e50a0
commit 3f0be47c7b
12 changed files with 634 additions and 197 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs
View File

@@ -27,12 +27,18 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (File.Exists(path))
{
pendingDllsToIndex.Enqueue(path);
continue;
}
else
if (Directory.Exists(path))
{
progressMonitor.FindingFiles(path);
AddReferenceDirectory(path);
}
else
{
progressMonitor.LogInfo("AssemblyCache: Path not found: " + path);
}
}
IndexReferences();
}

164
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Assets.cs Normal file
View File

@@ -0,0 +1,164 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Newtonsoft.Json.Linq;
using Semmle.Util;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
/// <summary>
/// Class for parsing project.assets.json files.
/// </summary>
internal class Assets
{
private readonly ProgressMonitor progressMonitor;
private static readonly string[] netFrameworks = new[] {
"microsoft.aspnetcore.app.ref",
"microsoft.netcore.app.ref",
"microsoft.netframework.referenceassemblies",
"microsoft.windowsdesktop.app.ref",
"netstandard.library.ref"
};
internal Assets(ProgressMonitor progressMonitor)
{
this.progressMonitor = progressMonitor;
}
/// <summary>
/// Class needed for deserializing parts of an assets file.
/// It holds information about a reference.
///
/// Type carries the type of the reference.
/// We are only interested in package references.
///
/// Compile holds information about the files needed for compilation.
/// However, if it is a .NET framework reference we assume that all files in the
/// package are needed for compilation.
/// </summary>
private record class ReferenceInfo(string? Type, Dictionary<string, object>? Compile);
/// <summary>
/// Add the package dependencies from the assets file to dependencies.
///
/// Parse a part of the JSon assets file and add the paths
/// to the dependencies required for compilation (and collect
/// information about used packages).
///
/// Example:
/// {
/// "Castle.Core/4.4.1": {
/// "type": "package",
/// "compile": {
/// "lib/netstandard1.5/Castle.Core.dll": {
/// "related": ".xml"
/// }
/// }
/// },
/// "Json.Net/1.0.33": {
/// "type": "package",
/// "compile": {
/// "lib/netstandard2.0/Json.Net.dll": {}
/// },
/// "runtime": {
/// "lib/netstandard2.0/Json.Net.dll": {}
/// }
/// }
/// }
///
/// Returns dependencies
/// RequiredPaths = {
/// "castle.core/4.4.1/lib/netstandard1.5/Castle.Core.dll",
/// "json.net/1.0.33/lib/netstandard2.0/Json.Net.dll"
/// }
/// UsedPackages = {
/// "castle.core",
/// "json.net"
/// }
/// </summary>
private DependencyContainer AddPackageDependencies(JObject json, DependencyContainer dependencies)
{
// If there are more than one framework we need to pick just one.
// To ensure stability we pick one based on the lexicographic order of
// the framework names.
var references = json
.GetProperty("targets")?
.Properties()?
.MaxBy(p => p.Name)?
.Value
.ToObject<Dictionary<string, ReferenceInfo>>();
if (references is null)
{
progressMonitor.LogDebug("No references found in the targets section in the assets file.");
return dependencies;
}
// Find all the compile dependencies for each reference and
// create the relative path to the dependency.
references
.ForEach(r =>
{
var info = r.Value;
var name = r.Key.ToLowerInvariant();
if (info.Type != "package")
{
return;
}
// If this is a .NET framework reference then include everything.
if (netFrameworks.Any(framework => name.StartsWith(framework)))
{
dependencies.Add(name);
}
else
{
info.Compile?
.ForEach(r => dependencies.Add(name, r.Key));
}
});
return dependencies;
}
/// <summary>
/// Parse `json` as project.assets.json content and add relative paths to the dependencies
/// (together with used package information) required for compilation.
/// </summary>
/// <returns>True if parsing succeeds, otherwise false.</returns>
public bool TryParse(string json, DependencyContainer dependencies)
{
try
{
var obj = JObject.Parse(json);
AddPackageDependencies(obj, dependencies);
return true;
}
catch (Exception e)
{
progressMonitor.LogDebug($"Failed to parse assets file (unexpected error): {e.Message}");
return false;
}
}
public static DependencyContainer GetCompilationDependencies(ProgressMonitor progressMonitor, IEnumerable<string> assets)
{
var parser = new Assets(progressMonitor);
var dependencies = new DependencyContainer();
assets.ForEach(asset =>
{
var json = File.ReadAllText(asset);
parser.TryParse(json, dependencies);
});
return dependencies;
}
}
internal static class JsonExtensions
{
internal static JObject? GetProperty(this JObject json, string property) =>
json[property] as JObject;
}
}

69
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyContainer.cs Normal file
View File

@@ -0,0 +1,69 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
/// <summary>
/// Container class for dependencies found in the assets file.
/// </summary>
internal class DependencyContainer
{
private readonly List<string> requiredPaths = new();
private readonly HashSet<string> usedPackages = new();
/// <summary>
/// In most cases paths in asset files point to dll's or the empty _._ file, which
/// is sometimes there to avoid the directory being empty.
/// That is, if the path specifically adds a .dll we use that, otherwise we as a fallback
/// add the entire directory (which should be fine in case of _._ as well).
/// </summary>
private static string ParseFilePath(string path)
{
if (path.EndsWith(".dll"))
{
return path;
}
return Path.GetDirectoryName(path) ?? path;
}
private static string GetPackageName(string package) =>
package
.Split(Path.DirectorySeparatorChar)
.First();
/// <summary>
/// Paths to dependencies required for compilation.
/// </summary>
public IEnumerable<string> RequiredPaths => requiredPaths;
/// <summary>
/// Packages that are used as a part of the required dependencies.
/// </summary>
public HashSet<string> UsedPackages => usedPackages;
/// <summary>
/// Add a dependency inside a package.
/// </summary>
public void Add(string package, string dependency)
{
var p = package.Replace('/', Path.DirectorySeparatorChar);
var d = dependency.Replace('/', Path.DirectorySeparatorChar);
var path = Path.Combine(p, ParseFilePath(d));
requiredPaths.Add(path);
usedPackages.Add(GetPackageName(p));
}
/// <summary>
/// Add a dependency to an entire package
/// </summary>
public void Add(string package)
{
var p = package.Replace('/', Path.DirectorySeparatorChar);
requiredPaths.Add(p);
usedPackages.Add(GetPackageName(p));
}
}
}

286
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
View File

@@ -31,9 +31,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private readonly IDotNet dotnet;
private readonly FileContent fileContent;
private readonly TemporaryDirectory packageDirectory;
private readonly TemporaryDirectory missingPackageDirectory;
private readonly TemporaryDirectory tempWorkingDirectory;
private readonly bool cleanupTempWorkingDirectory;
private readonly Lazy<Runtime> runtimeLazy;
private Runtime Runtime => runtimeLazy.Value;
/// <summary>
/// Performs C# dependency fetching.
/// </summary>
@@ -48,11 +52,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
this.sourceDir = new DirectoryInfo(srcDir);
packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName));
missingPackageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName, "missingpackages"));
tempWorkingDirectory = new TemporaryDirectory(FileUtils.GetTemporaryWorkingDirectory(out cleanupTempWorkingDirectory));
try
{
this.dotnet = DotNet.Make(options, progressMonitor, tempWorkingDirectory);
runtimeLazy = new Lazy<Runtime>(() => new Runtime(dotnet));
}
catch
{
@@ -74,13 +81,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var solutions = options.SolutionFile is not null
? new[] { options.SolutionFile }
: allNonBinaryFiles.SelectFileNamesByExtension(".sln");
var dllDirNames = options.DllDirs.Count == 0
var dllPaths = options.DllDirs.Count == 0
? allFiles.SelectFileNamesByExtension(".dll").ToList()
: options.DllDirs.Select(Path.GetFullPath).ToList();
if (options.UseNuGet)
{
dllDirNames.Add(packageDirectory.DirInfo.FullName);
try
{
var nuget = new NugetPackages(sourceDir.FullName, packageDirectory, progressMonitor);
@@ -91,40 +97,32 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
progressMonitor.MissingNuGet();
}
var restoredProjects = RestoreSolutions(solutions);
var restoredProjects = RestoreSolutions(solutions, out var assets1);
var projects = allProjects.Except(restoredProjects);
RestoreProjects(projects);
DownloadMissingPackages(allNonBinaryFiles);
}
RestoreProjects(projects, out var assets2);
var existsNetCoreRefNugetPackage = false;
var existsNetFrameworkRefNugetPackage = false;
var existsNetstandardLibRefNugetPackage = false;
var existsNetstandardLibNugetPackage = false;
var dependencies = Assets.GetCompilationDependencies(progressMonitor, assets1.Union(assets2));
var paths = dependencies
.RequiredPaths
.Select(d => Path.Combine(packageDirectory.DirInfo.FullName, d))
.ToList();
dllPaths.AddRange(paths);
LogAllUnusedPackages(dependencies);
DownloadMissingPackages(allNonBinaryFiles, dllPaths);
}
// Find DLLs in the .Net / Asp.Net Framework
// This block needs to come after the nuget restore, because the nuget restore might fetch the .NET Core/Framework reference assemblies.
if (options.ScanNetFrameworkDlls)
{
existsNetCoreRefNugetPackage = IsNugetPackageAvailable("microsoft.netcore.app.ref");
existsNetFrameworkRefNugetPackage = IsNugetPackageAvailable("microsoft.netframework.referenceassemblies");
existsNetstandardLibRefNugetPackage = IsNugetPackageAvailable("netstandard.library.ref");
existsNetstandardLibNugetPackage = IsNugetPackageAvailable("netstandard.library");
if (existsNetCoreRefNugetPackage
|| existsNetFrameworkRefNugetPackage
|| existsNetstandardLibRefNugetPackage
|| existsNetstandardLibNugetPackage)
{
progressMonitor.LogInfo("Found .NET Core/Framework DLLs in NuGet packages. Not adding installation directory.");
}
else
{
AddNetFrameworkDlls(dllDirNames);
}
AddNetFrameworkDlls(dllPaths);
AddAspNetCoreFrameworkDlls(dllPaths);
AddMicrosoftWindowsDesktopDlls(dllPaths);
}
assemblyCache = new AssemblyCache(dllDirNames, progressMonitor);
assemblyCache = new AssemblyCache(dllPaths, progressMonitor);
AnalyseSolutions(solutions);
foreach (var filename in assemblyCache.AllAssemblies.Select(a => a.Filename))
@@ -132,7 +130,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
UseReference(filename);
}
RemoveUnnecessaryNugetPackages(existsNetCoreRefNugetPackage, existsNetFrameworkRefNugetPackage, existsNetstandardLibRefNugetPackage, existsNetstandardLibNugetPackage);
RemoveNugetAnalyzerReferences();
ResolveConflicts();
// Output the findings
@@ -167,58 +165,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
DateTime.Now - startTime);
}
private void RemoveUnnecessaryNugetPackages(bool existsNetCoreRefNugetPackage, bool existsNetFrameworkRefNugetPackage,
bool existsNetstandardLibRefNugetPackage, bool existsNetstandardLibNugetPackage)
{
RemoveNugetAnalyzerReferences();
RemoveRuntimeNugetPackageReferences();
if (fileContent.IsNewProjectStructureUsed
&& !fileContent.UseAspNetCoreDlls)
{
// This might have been restored by the CLI even though the project isn't an asp.net core one.
RemoveNugetPackageReference("microsoft.aspnetcore.app.ref");
}
// Multiple dotnet framework packages could be present. We keep only one.
// The order of the packages is important, we're keeping the first one that is present in the nuget cache.
var packagesInPrioOrder = new (bool isPresent, string prefix)[]
{
// net7.0, ... net5.0, netcoreapp3.1, netcoreapp3.0
(existsNetCoreRefNugetPackage, "microsoft.netcore.app.ref"),
// net48, ..., net20
(existsNetFrameworkRefNugetPackage, "microsoft.netframework.referenceassemblies."),
// netstandard2.1
(existsNetstandardLibRefNugetPackage, "netstandard.library.ref"),
// netstandard2.0
(existsNetstandardLibNugetPackage, "netstandard.library")
};
for (var i = 0; i < packagesInPrioOrder.Length; i++)
{
var (isPresent, _) = packagesInPrioOrder[i];
if (!isPresent)
{
continue;
}
// Package is present, remove all the lower priority packages:
for (var j = i + 1; j < packagesInPrioOrder.Length; j++)
{
var (otherIsPresent, otherPrefix) = packagesInPrioOrder[j];
if (otherIsPresent)
{
RemoveNugetPackageReference(otherPrefix);
}
}
break;
}
// TODO: There could be multiple `microsoft.netframework.referenceassemblies` packages,
// we could keep the newest one, but this is covered by the conflict resolution logic
// (if the file names match)
}
private void RemoveNugetAnalyzerReferences()
{
if (!options.UseNuGet)
@@ -258,96 +204,110 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
}
}
private void AddNetFrameworkDlls(List<string> dllDirNames)
private void AddNetFrameworkDlls(List<string> dllPaths)
{
var runtime = new Runtime(dotnet);
// Multiple dotnet framework packages could be present.
// The order of the packages is important, we're adding the first one that is present in the nuget cache.
var packagesInPrioOrder = new string[]
{
"microsoft.netcore.app.ref", // net7.0, ... net5.0, netcoreapp3.1, netcoreapp3.0
"microsoft.netframework.referenceassemblies.", // net48, ..., net20
"netstandard.library.ref", // netstandard2.1
"netstandard.library" // netstandard2.0
};
var frameworkPath = packagesInPrioOrder
.Select(GetPackageDirectory)
.FirstOrDefault(dir => dir is not null);
if (frameworkPath is not null)
{
dllPaths.Add(frameworkPath);
progressMonitor.LogInfo("Found .NET Core/Framework DLLs in NuGet packages. Not adding installation directory.");
return;
}
string? runtimeLocation = null;
if (options.UseSelfContainedDotnet)
{
runtimeLocation = runtime.ExecutingRuntime;
runtimeLocation = Runtime.ExecutingRuntime;
}
else if (fileContent.IsNewProjectStructureUsed)
{
runtimeLocation = runtime.NetCoreRuntime;
runtimeLocation = Runtime.NetCoreRuntime;
}
else if (fileContent.IsLegacyProjectStructureUsed)
{
runtimeLocation = runtime.DesktopRuntime;
runtimeLocation = Runtime.DesktopRuntime;
}
runtimeLocation ??= runtime.ExecutingRuntime;
runtimeLocation ??= Runtime.ExecutingRuntime;
progressMonitor.LogInfo($".NET runtime location selected: {runtimeLocation}");
dllDirNames.Add(runtimeLocation);
if (fileContent.IsNewProjectStructureUsed
&& fileContent.UseAspNetCoreDlls
&& runtime.AspNetCoreRuntime is string aspRuntime)
{
progressMonitor.LogInfo($"ASP.NET runtime location selected: {aspRuntime}");
dllDirNames.Add(aspRuntime);
}
dllPaths.Add(runtimeLocation);
}
private void RemoveRuntimeNugetPackageReferences()
private void AddAspNetCoreFrameworkDlls(List<string> dllPaths)
{
var runtimePackagePrefixes = new[]
{
"microsoft.netcore.app.runtime",
"microsoft.aspnetcore.app.runtime",
"microsoft.windowsdesktop.app.runtime",
// legacy runtime packages:
"runtime.linux-x64.microsoft.netcore.app",
"runtime.osx-x64.microsoft.netcore.app",
"runtime.win-x64.microsoft.netcore.app",
// Internal implementation packages not meant for direct consumption:
"runtime."
};
RemoveNugetPackageReference(runtimePackagePrefixes);
}
private void RemoveNugetPackageReference(params string[] packagePrefixes)
{
if (!options.UseNuGet)
if (!fileContent.IsNewProjectStructureUsed || !fileContent.UseAspNetCoreDlls)
{
return;
}
var packageFolder = packageDirectory.DirInfo.FullName.ToLowerInvariant();
if (packageFolder == null)
// First try to find ASP.NET Core assemblies in the NuGet packages
if (GetPackageDirectory("microsoft.aspnetcore.app.ref") is string aspNetCorePackage)
{
return;
progressMonitor.LogInfo($"Found ASP.NET Core in NuGet packages. Not adding installation directory.");
dllPaths.Add(aspNetCorePackage);
}
var packagePathPrefixes = packagePrefixes.Select(p => Path.Combine(packageFolder, p.ToLowerInvariant()));
foreach (var filename in usedReferences.Keys)
else if (Runtime.AspNetCoreRuntime is string aspNetCoreRuntime)
{
var lowerFilename = filename.ToLowerInvariant();
if (packagePathPrefixes.Any(prefix => lowerFilename.StartsWith(prefix)))
{
usedReferences.Remove(filename);
progressMonitor.RemovedReference(filename);
}
progressMonitor.LogInfo($"ASP.NET runtime location selected: {aspNetCoreRuntime}");
dllPaths.Add(aspNetCoreRuntime);
}
}
private bool IsNugetPackageAvailable(string packagePrefix)
private void AddMicrosoftWindowsDesktopDlls(List<string> dllPaths)
{
if (GetPackageDirectory("microsoft.windowsdesktop.app.ref") is string windowsDesktopApp)
{
progressMonitor.LogInfo($"Found Windows Desktop App in NuGet packages.");
dllPaths.Add(windowsDesktopApp);
}
}
private string? GetPackageDirectory(string packagePrefix)
{
if (!options.UseNuGet)
{
return false;
return null;
}
return new DirectoryInfo(packageDirectory.DirInfo.FullName)
.EnumerateDirectories(packagePrefix + "*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
.Any();
.FirstOrDefault()?
.FullName;
}
private IEnumerable<string> GetAllPackageDirectories()
{
if (!options.UseNuGet)
{
return Enumerable.Empty<string>();
}
return new DirectoryInfo(packageDirectory.DirInfo.FullName)
.EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
.Select(d => d.FullName);
}
private void LogAllUnusedPackages(DependencyContainer dependencies) =>
GetAllPackageDirectories()
.Where(package => !dependencies.UsedPackages.Contains(package))
.ForEach(package => progressMonitor.LogInfo($"Unused package: {package}"));
private void GenerateSourceFileFromImplicitUsings()
{
var usings = new HashSet<string>();
@@ -437,7 +397,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// with this source tree. Use a SHA1 of the directory name.
/// </summary>
/// <returns>The full path of the temp directory.</returns>
private static string ComputeTempDirectory(string srcDir)
private static string ComputeTempDirectory(string srcDir, string packages = "packages")
{
var bytes = Encoding.Unicode.GetBytes(srcDir);
var sha = SHA1.HashData(bytes);
@@ -445,7 +405,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
foreach (var b in sha.Take(8))
sb.AppendFormat("{0:x2}", b);
return Path.Combine(Path.GetTempPath(), "GitHub", "packages", sb.ToString());
return Path.Combine(Path.GetTempPath(), "GitHub", packages, sb.ToString());
}
/// <summary>
@@ -623,41 +583,52 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
private bool RestoreProject(string project, bool forceDotnetRefAssemblyFetching, string? pathToNugetConfig = null) =>
dotnet.RestoreProjectToDirectory(project, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching, pathToNugetConfig);
private bool RestoreProject(string project, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> assets, string? pathToNugetConfig = null) =>
dotnet.RestoreProjectToDirectory(project, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching, out assets, pathToNugetConfig);
private bool RestoreSolution(string solution, out IEnumerable<string> projects) =>
dotnet.RestoreSolutionToDirectory(solution, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: true, out projects);
private bool RestoreSolution(string solution, out IEnumerable<string> projects, out IEnumerable<string> assets) =>
dotnet.RestoreSolutionToDirectory(solution, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: true, out projects, out assets);
/// <summary>
/// Executes `dotnet restore` on all solution files in solutions.
/// As opposed to RestoreProjects this is not run in parallel using PLINQ
/// as `dotnet restore` on a solution already uses multiple threads for restoring
/// the projects (this can be disabled with the `--disable-parallel` flag).
/// Populates assets with the relative paths to the assets files generated by the restore.
/// Returns a list of projects that are up to date with respect to restore.
/// </summary>
/// <param name="solutions">A list of paths to solution files.</param>
private IEnumerable<string> RestoreSolutions(IEnumerable<string> solutions) =>
solutions.SelectMany(solution =>
private IEnumerable<string> RestoreSolutions(IEnumerable<string> solutions, out IEnumerable<string> assets)
{
var assetFiles = new List<string>();
var projects = solutions.SelectMany(solution =>
{
RestoreSolution(solution, out var restoredProjects);
RestoreSolution(solution, out var restoredProjects, out var a);
assetFiles.AddRange(a);
return restoredProjects;
});
assets = assetFiles;
return projects;
}
/// <summary>
/// Executes `dotnet restore` on all projects in projects.
/// This is done in parallel for performance reasons.
/// Populates assets with the relative paths to the assets files generated by the restore.
/// </summary>
/// <param name="projects">A list of paths to project files.</param>
private void RestoreProjects(IEnumerable<string> projects)
private void RestoreProjects(IEnumerable<string> projects, out IEnumerable<string> assets)
{
var assetFiles = new List<string>();
Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, project =>
{
RestoreProject(project, forceDotnetRefAssemblyFetching: true);
RestoreProject(project, forceDotnetRefAssemblyFetching: true, out var a);
assetFiles.AddRange(a);
});
assets = assetFiles;
}
private void DownloadMissingPackages(List<FileInfo> allFiles)
private void DownloadMissingPackages(List<FileInfo> allFiles, List<string> dllPaths)
{
var nugetConfigs = allFiles.SelectFileNamesByName("nuget.config").ToArray();
string? nugetConfig = null;
@@ -698,13 +669,15 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
return;
}
success = RestoreProject(tempDir.DirInfo.FullName, forceDotnetRefAssemblyFetching: false, pathToNugetConfig: nugetConfig);
dotnet.RestoreProjectToDirectory(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: false, out var _, pathToNugetConfig: nugetConfig);
// TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package.
if (!success)
{
progressMonitor.FailedToRestoreNugetPackage(package);
}
});
dllPaths.Add(missingPackageDirectory.DirInfo.FullName);
}
private void AnalyseSolutions(IEnumerable<string> solutions)
@@ -724,26 +697,25 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
});
}
public void Dispose()
public void Dispose(TemporaryDirectory? dir, string name)
{
try
{
packageDirectory?.Dispose();
dir?.Dispose();
}
catch (Exception exc)
{
progressMonitor.LogInfo("Couldn't delete package directory: " + exc.Message);
progressMonitor.LogInfo($"Couldn't delete {name} directory {exc.Message}");
}
}
public void Dispose()
{
Dispose(packageDirectory, "package");
Dispose(missingPackageDirectory, "missing package");
if (cleanupTempWorkingDirectory)
{
try
{
tempWorkingDirectory?.Dispose();
}
catch (Exception exc)
{
progressMonitor.LogInfo("Couldn't delete temporary working directory: " + exc.Message);
}
Dispose(tempWorkingDirectory, "temporary working");
}
}
}

44
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs
View File

@@ -42,7 +42,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private string GetRestoreArgs(string projectOrSolutionFile, string packageDirectory, bool forceDotnetRefAssemblyFetching)
{
var args = $"restore --no-dependencies \"{projectOrSolutionFile}\" --packages \"{packageDirectory}\" /p:DisableImplicitNuGetFallbackFolder=true";
var args = $"restore --no-dependencies \"{projectOrSolutionFile}\" --packages \"{packageDirectory}\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal";
if (forceDotnetRefAssemblyFetching)
{
@@ -60,7 +60,19 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
return args;
}
public bool RestoreProjectToDirectory(string projectFile, string packageDirectory, bool forceDotnetRefAssemblyFetching, string? pathToNugetConfig = null)
private static IEnumerable<string> GetFirstGroupOnMatch(Regex regex, IEnumerable<string> lines) =>
lines
.Select(line => regex.Match(line))
.Where(match => match.Success)
.Select(match => match.Groups[1].Value);
private static IEnumerable<string> GetAssetsFilePaths(IEnumerable<string> lines) =>
GetFirstGroupOnMatch(AssetsFileRegex(), lines);
private static IEnumerable<string> GetRestoredProjects(IEnumerable<string> lines) =>
GetFirstGroupOnMatch(RestoredProjectRegex(), lines);
public bool RestoreProjectToDirectory(string projectFile, string packageDirectory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> assets, string? pathToNugetConfig = null)
{
var args = GetRestoreArgs(projectFile, packageDirectory, forceDotnetRefAssemblyFetching);
if (pathToNugetConfig != null)
@@ -68,25 +80,18 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
args += $" --configfile \"{pathToNugetConfig}\"";
}
return dotnetCliInvoker.RunCommand(args);
var success = dotnetCliInvoker.RunCommand(args, out var output);
assets = success ? GetAssetsFilePaths(output) : Array.Empty<string>();
return success;
}
public bool RestoreSolutionToDirectory(string solutionFile, string packageDirectory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> projects)
public bool RestoreSolutionToDirectory(string solutionFile, string packageDirectory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> projects, out IEnumerable<string> assets)
{
var args = GetRestoreArgs(solutionFile, packageDirectory, forceDotnetRefAssemblyFetching);
args += " --verbosity normal";
if (dotnetCliInvoker.RunCommand(args, out var output))
{
var regex = RestoreProjectRegex();
projects = output
.Select(line => regex.Match(line))
.Where(match => match.Success)
.Select(match => match.Groups[1].Value);
return true;
}
projects = Array.Empty<string>();
return false;
var success = dotnetCliInvoker.RunCommand(args, out var output);
projects = success ? GetRestoredProjects(output) : Array.Empty<string>();
assets = success ? GetAssetsFilePaths(output) : Array.Empty<string>();
return success;
}
public bool New(string folder)
@@ -121,6 +126,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
[GeneratedRegex("Restored\\s+(.+\\.csproj)", RegexOptions.Compiled)]
private static partial Regex RestoreProjectRegex();
private static partial Regex RestoredProjectRegex();
[GeneratedRegex("[Assets\\sfile\\shas\\snot\\schanged.\\sSkipping\\sassets\\sfile\\swriting.|Writing\\sassets\\sfile\\sto\\sdisk.]\\sPath:\\s(.*)", RegexOptions.Compiled)]
private static partial Regex AssetsFileRegex();
}
}

4
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs
View File

@@ -4,8 +4,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal interface IDotNet
{
bool RestoreProjectToDirectory(string project, string directory, bool forceDotnetRefAssemblyFetching, string? pathToNugetConfig = null);
bool RestoreSolutionToDirectory(string solutionFile, string packageDirectory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> projects);
bool RestoreProjectToDirectory(string project, string directory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> assets, string? pathToNugetConfig = null);
bool RestoreSolutionToDirectory(string solutionFile, string packageDirectory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> projects, out IEnumerable<string> assets);
bool New(string folder);
bool AddPackage(string folder, string package);
IList<string> GetListedRuntimes();

206
csharp/extractor/Semmle.Extraction.Tests/Assets.cs Normal file
View File

@@ -0,0 +1,206 @@
using Xunit;
using System.Linq;
using Semmle.Extraction.CSharp.DependencyFetching;
namespace Semmle.Extraction.Tests
{
public class AssetsTests
{
private static string FixExpectedPathOnWindows(string path) => path.Replace('\\', '/');
[Fact]
public void TestAssets1()
{
// Setup
var assets = new Assets(new ProgressMonitor(new LoggerStub()));
var json = assetsJson1;
var dependencies = new DependencyContainer();
// Execute
var success = assets.TryParse(json, dependencies);
// Verify
Assert.True(success);
Assert.Equal(5, dependencies.RequiredPaths.Count());
Assert.Equal(4, dependencies.UsedPackages.Count());
var normalizedPaths = dependencies.RequiredPaths.Select(FixExpectedPathOnWindows);
// Required references
Assert.Contains("castle.core/4.4.1/lib/netstandard1.5/Castle.Core.dll", normalizedPaths);
Assert.Contains("castle.core/4.4.1/lib/netstandard1.5/Castle.Core2.dll", normalizedPaths);
Assert.Contains("json.net/1.0.33/lib/netstandard2.0/Json.Net.dll", normalizedPaths);
Assert.Contains("microsoft.aspnetcore.cryptography.internal/6.0.8/lib/net6.0/Microsoft.AspNetCore.Cryptography.Internal.dll", normalizedPaths);
Assert.Contains("humanizer.core/2.8.26/lib/netstandard2.0", normalizedPaths);
// Used packages
Assert.Contains("castle.core", dependencies.UsedPackages);
Assert.Contains("json.net", dependencies.UsedPackages);
Assert.Contains("microsoft.aspnetcore.cryptography.internal", dependencies.UsedPackages);
Assert.Contains("humanizer.core", dependencies.UsedPackages);
}
[Fact]
public void TestAssets2()
{
// Setup
var assets = new Assets(new ProgressMonitor(new LoggerStub()));
var json = assetsJson2;
var dependencies = new DependencyContainer();
// Execute
var success = assets.TryParse(json, dependencies);
// Verify
Assert.True(success);
Assert.Equal(2, dependencies.RequiredPaths.Count());
var normalizedPaths = dependencies.RequiredPaths.Select(FixExpectedPathOnWindows);
// Required references
Assert.Contains("microsoft.netframework.referenceassemblies/1.0.3", normalizedPaths);
Assert.Contains("microsoft.netframework.referenceassemblies.net48/1.0.3", normalizedPaths);
// Used packages
Assert.Contains("microsoft.netframework.referenceassemblies", dependencies.UsedPackages);
Assert.Contains("microsoft.netframework.referenceassemblies.net48", dependencies.UsedPackages);
}
[Fact]
public void TestAssets3()
{
// Setup
var assets = new Assets(new ProgressMonitor(new LoggerStub()));
var json = "garbage data";
var dependencies = new DependencyContainer();
// Execute
var success = assets.TryParse(json, dependencies);
// Verify
Assert.False(success);
Assert.Empty(dependencies.RequiredPaths);
}
private readonly string assetsJson1 = """
{
"version": 3,
"targets": {
"net7.0": {
"Castle.Core/4.4.1": {
"type": "package",
"dependencies": {
"NETStandard.Library": "1.6.1",
"System.Collections.Specialized": "4.3.0",
},
"compile": {
"lib/netstandard1.5/Castle.Core.dll": {
"related": ".xml"
},
"lib/netstandard1.5/Castle.Core2.dll": {
"related": ".xml"
}
},
"runtime": {
"lib/netstandard1.5/Castle.Core.dll": {
"related": ".xml"
}
}
},
"Json.Net/1.0.33": {
"type": "package",
"compile": {
"lib/netstandard2.0/Json.Net.dll": {}
},
"runtime": {
"lib/netstandard2.0/Json.Net.dll": {}
}
},
"MessagePackAnalyzer/2.1.152": {
"type": "package"
},
"Microsoft.AspNetCore.Cryptography.Internal/6.0.8": {
"type": "package",
"compile": {
"lib/net6.0/Microsoft.AspNetCore.Cryptography.Internal.dll": {
"related": ".xml"
}
},
"runtime": {
"lib/net6.0/Microsoft.AspNetCore.Cryptography.Internal.dll": {
"related": ".xml"
}
}
},
"Humanizer.Core/2.8.26": {
"type": "package",
"compile": {
"lib/netstandard2.0/_._": {
"related": ".xml"
}
},
"runtime": {
"lib/netstandard2.0/Humanizer.dll": {
"related": ".xml"
}
}
},
"Nop.Core/4.5.0": {
"type": "project",
"compile": {
"bin/placeholder/Nop.Core.dll": {}
},
"runtime": {
"bin/placeholder/Nop.Core.dll": {}
}
},
}
},
"project": {
"version": "1.0.0",
"frameworks": {
"net7.0": {
"targetAlias": "net7.0",
"downloadDependencies": [
{
"name": "Microsoft.AspNetCore.App.Ref",
"version": "[7.0.2, 7.0.2]"
},
{
"name": "Microsoft.NETCore.App.Ref",
"version": "[7.0.2, 7.0.2]"
}
],
"frameworkReferences": {
"Microsoft.AspNetCore.App": {
"privateAssets": "none"
},
"Microsoft.NETCore.App": {
"privateAssets": "all"
}
}
}
}
}
}
""";
private readonly string assetsJson2 = """
{
"version": 3,
"targets": {
".NETFramework,Version=v4.8": {
"Microsoft.NETFramework.ReferenceAssemblies/1.0.3": {
"type": "package",
"dependencies": {
"Microsoft.NETFramework.ReferenceAssemblies.net48": "1.0.3"
}
},
"Microsoft.NETFramework.ReferenceAssemblies.net48/1.0.3": {
"type": "package",
"build": {
"build/Microsoft.NETFramework.ReferenceAssemblies.net48.targets": {}
}
}
}
}
}
""";
}
}

23
csharp/extractor/Semmle.Extraction.Tests/DotNet.cs
View File

@@ -43,9 +43,11 @@ namespace Semmle.Extraction.Tests
private static IList<string> MakeDotnetRestoreOutput() =>
new List<string> {
" Determining projects to restore...",
" Writing assets file to disk. Path: /path/to/project.assets.json",
" Restored /path/to/project.csproj (in 1.23 sec).",
" Other output...",
" More output...",
" Assets file has not changed. Skipping assets file writing. Path: /path/to/project2.assets.json",
" Restored /path/to/project2.csproj (in 4.56 sec).",
" Other output...",
};
@@ -99,26 +101,29 @@ namespace Semmle.Extraction.Tests
var dotnet = MakeDotnet(dotnetCliInvoker);
// Execute
dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages", false);
dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages", false, out var assets);
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();
Assert.Equal("restore --no-dependencies \"myproject.csproj\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true", lastArgs);
Assert.Equal("restore --no-dependencies \"myproject.csproj\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal", lastArgs);
}
[Fact]
public void TestDotnetRestoreProjectToDirectory2()
{
// Setup
var dotnetCliInvoker = new DotNetCliInvokerStub(new List<string>());
var dotnetCliInvoker = new DotNetCliInvokerStub(MakeDotnetRestoreOutput());
var dotnet = MakeDotnet(dotnetCliInvoker);
// Execute
dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages", false, "myconfig.config");
dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages", false, out var assets, "myconfig.config");
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();
Assert.Equal("restore --no-dependencies \"myproject.csproj\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true --configfile \"myconfig.config\"", lastArgs);
Assert.Equal("restore --no-dependencies \"myproject.csproj\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal --configfile \"myconfig.config\"", lastArgs);
Assert.Equal(2, assets.Count());
Assert.Contains("/path/to/project.assets.json", assets);
Assert.Contains("/path/to/project2.assets.json", assets);
}
[Fact]
@@ -129,7 +134,7 @@ namespace Semmle.Extraction.Tests
var dotnet = MakeDotnet(dotnetCliInvoker);
// Execute
dotnet.RestoreSolutionToDirectory("mysolution.sln", "mypackages", false, out var projects);
dotnet.RestoreSolutionToDirectory("mysolution.sln", "mypackages", false, out var projects, out var assets);
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();
@@ -137,6 +142,9 @@ namespace Semmle.Extraction.Tests
Assert.Equal(2, projects.Count());
Assert.Contains("/path/to/project.csproj", projects);
Assert.Contains("/path/to/project2.csproj", projects);
Assert.Equal(2, assets.Count());
Assert.Contains("/path/to/project.assets.json", assets);
Assert.Contains("/path/to/project2.assets.json", assets);
}
[Fact]
@@ -148,12 +156,13 @@ namespace Semmle.Extraction.Tests
dotnetCliInvoker.Success = false;
// Execute
dotnet.RestoreSolutionToDirectory("mysolution.sln", "mypackages", false, out var projects);
dotnet.RestoreSolutionToDirectory("mysolution.sln", "mypackages", false, out var projects, out var assets);
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();
Assert.Equal("restore --no-dependencies \"mysolution.sln\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal", lastArgs);
Assert.Empty(projects);
Assert.Empty(assets);
}
[Fact]

9
csharp/extractor/Semmle.Extraction.Tests/Runtime.cs
View File

@@ -19,11 +19,16 @@ namespace Semmle.Extraction.Tests
public bool New(string folder) => true;
public bool RestoreProjectToDirectory(string project, string directory, bool forceDotnetRefAssemblyFetching, string? pathToNugetConfig = null) => true;
public bool RestoreProjectToDirectory(string project, string directory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> assets, string? pathToNugetConfig = null)
{
assets = Array.Empty<string>();
return true;
}
public bool RestoreSolutionToDirectory(string solution, string directory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> projects)
public bool RestoreSolutionToDirectory(string solution, string directory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> projects, out IEnumerable<string> assets)
{
projects = Array.Empty<string>();
assets = Array.Empty<string>();
return true;
}

6
csharp/extractor/Semmle.Util/IEnumerableExtensions.cs
View File

@@ -113,5 +113,11 @@ namespace Semmle.Util
h = h * 7 + i.GetHashCode();
return h;
}
/// <summary>
/// Returns the sequence with nulls removed.
/// </summary>
public static IEnumerable<T> WhereNotNull<T>(this IEnumerable<T?> items) where T : class =>
items.Where(i => i is not null)!;
}
}

6
csharp/ql/integration-tests/posix-only/standalone_dependencies/Assemblies.expected
View File

@@ -1,7 +1,3 @@
| /avalara.avatax/21.10.0/lib/net20/Avalara.AvaTax.RestClient.net20.dll |
| /avalara.avatax/21.10.0/lib/net45/Avalara.AvaTax.RestClient.net45.dll |
| /avalara.avatax/21.10.0/lib/net461/Avalara.AvaTax.RestClient.net461.dll |
| /avalara.avatax/21.10.0/lib/netstandard16/Avalara.AvaTax.netstandard11.dll |
| /avalara.avatax/21.10.0/lib/netstandard20/Avalara.AvaTax.netstandard20.dll |
| /microsoft.bcl.asyncinterfaces/6.0.0/lib/netstandard2.1/Microsoft.Bcl.AsyncInterfaces.dll |
| /microsoft.netcore.app.ref/3.1.0/ref/netcoreapp3.1/System.Runtime.InteropServices.WindowsRuntime.dll |
@@ -168,4 +164,4 @@
| /microsoft.netcore.app.ref/7.0.2/ref/net7.0/WindowsBase.dll |
| /microsoft.netcore.app.ref/7.0.2/ref/net7.0/mscorlib.dll |
| /microsoft.netcore.app.ref/7.0.2/ref/net7.0/netstandard.dll |
| /newtonsoft.json/12.0.1/lib/portable-net45+win8+wp8+wpa81/Newtonsoft.Json.dll |
| /newtonsoft.json/12.0.1/lib/netstandard2.0/Newtonsoft.Json.dll |

6
csharp/ql/integration-tests/windows-only/standalone_dependencies/Assemblies.expected
View File

@@ -1,7 +1,3 @@
| /avalara.avatax/21.10.0/lib/net20/Avalara.AvaTax.RestClient.net20.dll |
| /avalara.avatax/21.10.0/lib/net45/Avalara.AvaTax.RestClient.net45.dll |
| /avalara.avatax/21.10.0/lib/net461/Avalara.AvaTax.RestClient.net461.dll |
| /avalara.avatax/21.10.0/lib/netstandard16/Avalara.AvaTax.netstandard11.dll |
| /avalara.avatax/21.10.0/lib/netstandard20/Avalara.AvaTax.netstandard20.dll |
| /microsoft.bcl.asyncinterfaces/6.0.0/lib/netstandard2.1/Microsoft.Bcl.AsyncInterfaces.dll |
| /microsoft.netcore.app.ref/3.1.0/ref/netcoreapp3.1/System.Runtime.InteropServices.WindowsRuntime.dll |
@@ -212,4 +208,4 @@
| /microsoft.windowsdesktop.app.ref/7.0.2/ref/net7.0/UIAutomationTypes.dll |
| /microsoft.windowsdesktop.app.ref/7.0.2/ref/net7.0/WindowsBase.dll |
| /microsoft.windowsdesktop.app.ref/7.0.2/ref/net7.0/WindowsFormsIntegration.dll |
| /newtonsoft.json/12.0.1/lib/portable-net45+win8+wp8+wpa81/Newtonsoft.Json.dll |
| /newtonsoft.json/12.0.1/lib/netstandard2.0/Newtonsoft.Json.dll |