hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-04-25 16:55:19 +02:00
Code Issues Packages Projects Releases Wiki Activity

C#: Inline dependency collection from asset files per group.

Browse Source
This commit is contained in:
Michael Nebel
2024-04-25 14:12:56 +02:00
parent 0124b0749f
commit 131d0b911f
5 changed files with 118 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Assets.cs
View File

@@ -16,6 +16,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
private readonly ILogger logger;
/// <summary>
/// Contains the dependencies found in the parsed asset files.
/// </summary>
public DependencyContainer Dependencies { get; } = new();
internal Assets(ILogger logger)
{
this.logger = logger;
@@ -72,7 +77,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// "json.net"
/// }
/// </summary>
private void AddPackageDependencies(JObject json, DependencyContainer dependencies)
private void AddPackageDependencies(JObject json)
{
// If there is more than one framework we need to pick just one.
// To ensure stability we pick one based on the lexicographic order of
@@ -107,13 +112,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
// If this is a framework reference then include everything.
if (FrameworkPackageNames.AllFrameworks.Any(framework => name.StartsWith(framework)))
{
dependencies.AddFramework(name);
Dependencies.AddFramework(name);
}
return;
}
info.Compile
.ForEach(r => dependencies.Add(name, r.Key));
.ForEach(r => Dependencies.Add(name, r.Key));
});
return;
@@ -149,7 +154,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// "microsoft.netcore.app.ref"
/// }
/// </summary>
private void AddFrameworkDependencies(JObject json, DependencyContainer dependencies)
private void AddFrameworkDependencies(JObject json)
{
var frameworks = json
@@ -178,7 +183,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
references
.Properties()
.ForEach(f => dependencies.AddFramework($"{f.Name}.Ref".ToLowerInvariant()));
.ForEach(f => Dependencies.AddFramework($"{f.Name}.Ref".ToLowerInvariant()));
}
/// <summary>
@@ -186,13 +191,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// (together with used package information) required for compilation.
/// </summary>
/// <returns>True if parsing succeeds, otherwise false.</returns>
public bool TryParse(string json, DependencyContainer dependencies)
public bool TryParse(string json)
{
try
{
var obj = JObject.Parse(json);
AddPackageDependencies(obj, dependencies);
AddFrameworkDependencies(obj, dependencies);
AddPackageDependencies(obj);
AddFrameworkDependencies(obj);
return true;
}
catch (Exception e)
@@ -217,19 +222,24 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
}
public static DependencyContainer GetCompilationDependencies(ILogger logger, IEnumerable<string> assets)
/// <summary>
/// Add the dependencies from the assets file to the dependencies.
/// </summary>
/// <param name="asset">Path to an asset file.</param>
public void AddDependencies(string asset)
{
var parser = new Assets(logger);
var dependencies = new DependencyContainer();
assets.ForEach(asset =>
if (TryReadAllText(asset, logger, out var json))
{
if (TryReadAllText(asset, logger, out var json))
{
parser.TryParse(json, dependencies);
}
});
return dependencies;
TryParse(json);
}
}
/// <summary>
/// Add the dependencies from the assets files to the dependencies.
/// </summary>
/// <param name="assets">Collection of paths to asset files.</param>
public void AddDependenciesRange(IEnumerable<string> assets) =>
assets.ForEach(AddDependencies);
}
internal static class JsonExtensions

18
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyContainer.cs
View File

@@ -12,7 +12,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// <summary>
/// Paths to dependencies required for compilation.
/// </summary>
public List<string> Paths { get; } = new();
public HashSet<string> Paths { get; } = new();
/// <summary>
/// Packages that are used as a part of the required dependencies.
@@ -68,4 +68,18 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
Packages.Add(GetPackageName(p));
}
}
}
internal static class DependencyContainerExtensions
{
/// <summary>
/// Flatten a list of containers into a single container.
/// </summary>
public static DependencyContainer Flatten(this IEnumerable<DependencyContainer> container) =>
container.Aggregate(new DependencyContainer(), (acc, c) =>
{
acc.Paths.UnionWith(c.Paths);
acc.Packages.UnionWith(c.Packages);
return acc;
});
}
}

2
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
View File

@@ -3,8 +3,6 @@ using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks;
using Semmle.Util;

32
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs
View File

@@ -144,11 +144,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
logger.LogError($"Failed to restore Nuget packages with nuget.exe: {exc.Message}");
}
var restoredProjects = RestoreSolutions(out var assets1);
var restoredProjects = RestoreSolutions(out var container);
var projects = fileProvider.Projects.Except(restoredProjects);
RestoreProjects(projects, out var assets2);
RestoreProjects(projects, out var containers);
var dependencies = Assets.GetCompilationDependencies(logger, assets1.Union(assets2));
containers.Add(container);
var dependencies = containers.Flatten();
var paths = dependencies
.Paths
@@ -198,14 +199,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// As opposed to RestoreProjects this is not run in parallel using PLINQ
/// as `dotnet restore` on a solution already uses multiple threads for restoring
/// the projects (this can be disabled with the `--disable-parallel` flag).
/// Populates assets with the relative paths to the assets files generated by the restore.
/// Populates dependencies with the relevant dependencies from the assets files generated by the restore.
/// Returns a list of projects that are up to date with respect to restore.
/// </summary>
private IEnumerable<string> RestoreSolutions(out IEnumerable<string> assets)
private IEnumerable<string> RestoreSolutions(out DependencyContainer dependencies)
{
var successCount = 0;
var nugetSourceFailures = 0;
var assetFiles = new List<string>();
var assets = new Assets(logger);
var projects = fileProvider.Solutions.SelectMany(solution =>
{
logger.LogInfo($"Restoring solution {solution}...");
@@ -218,10 +219,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
nugetSourceFailures++;
}
assetFiles.AddRange(res.AssetsFilePaths);
assets.AddDependenciesRange(res.AssetsFilePaths);
return res.RestoredProjects;
}).ToList();
assets = assetFiles;
dependencies = assets.Dependencies;
compilationInfoContainer.CompilationInfos.Add(("Successfully restored solution files", successCount.ToString()));
compilationInfoContainer.CompilationInfos.Add(("Failed solution restore with package source error", nugetSourceFailures.ToString()));
compilationInfoContainer.CompilationInfos.Add(("Restored projects through solution files", projects.Count.ToString()));
@@ -231,22 +232,24 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// <summary>
/// Executes `dotnet restore` on all projects in projects.
/// This is done in parallel for performance reasons.
/// Populates assets with the relative paths to the assets files generated by the restore.
/// Populates dependencies with the relative paths to the assets files generated by the restore.
/// </summary>
/// <param name="projects">A list of paths to project files.</param>
private void RestoreProjects(IEnumerable<string> projects, out IEnumerable<string> assets)
private void RestoreProjects(IEnumerable<string> projects, out List<DependencyContainer> dependencies)
{
var successCount = 0;
var nugetSourceFailures = 0;
var assetFiles = new List<string>();
List<DependencyContainer> collectedDependencies = [];
var sync = new object();
var projectGroups = projects.GroupBy(Path.GetDirectoryName);
Parallel.ForEach(projectGroups, new ParallelOptions { MaxDegreeOfParallelism = DependencyManager.Threads }, projectGroup =>
{
var assets = new Assets(logger);
foreach (var project in projectGroup)
{
logger.LogInfo($"Restoring project {project}...");
var res = dotnet.Restore(new(project, PackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
assets.AddDependenciesRange(res.AssetsFilePaths);
lock (sync)
{
if (res.Success)
@@ -257,11 +260,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
nugetSourceFailures++;
}
assetFiles.AddRange(res.AssetsFilePaths);
}
}
lock (sync)
{
collectedDependencies.Add(assets.Dependencies);
}
});
assets = assetFiles;
dependencies = collectedDependencies;
compilationInfoContainer.CompilationInfos.Add(("Successfully restored project files", successCount.ToString()));
compilationInfoContainer.CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString()));
}

118
csharp/extractor/Semmle.Extraction.Tests/Assets.cs
View File

@@ -14,29 +14,28 @@ namespace Semmle.Extraction.Tests
// Setup
var assets = new Assets(new LoggerStub());
var json = assetsJson1;
var dependencies = new DependencyContainer();
// Execute
var success = assets.TryParse(json, dependencies);
var success = assets.TryParse(json);
// Verify
Assert.True(success);
Assert.Equal(6, dependencies.Paths.Count());
Assert.Equal(5, dependencies.Packages.Count());
Assert.Equal(6, assets.Dependencies.Paths.Count);
Assert.Equal(5, assets.Dependencies.Packages.Count);
var normalizedPaths = dependencies.Paths.Select(FixExpectedPathOnWindows);
var normalizedPaths = assets.Dependencies.Paths.Select(FixExpectedPathOnWindows);
// Used references
Assert.Contains("castle.core/4.4.1/lib/netstandard1.5/Castle.Core.dll", normalizedPaths);
Assert.Contains("castle.core/4.4.1/lib/netstandard1.5/Castle.Core2.dll", normalizedPaths);
Assert.Contains("json.net/1.0.33/lib/netstandard2.0/Json.Net.dll", normalizedPaths);
Assert.Contains("microsoft.aspnetcore.cryptography.internal/6.0.8/lib/net6.0/Microsoft.AspNetCore.Cryptography.Internal.dll", normalizedPaths);
// Used packages
Assert.Contains("castle.core", dependencies.Packages);
Assert.Contains("json.net", dependencies.Packages);
Assert.Contains("microsoft.aspnetcore.cryptography.internal", dependencies.Packages);
Assert.Contains("castle.core", assets.Dependencies.Packages);
Assert.Contains("json.net", assets.Dependencies.Packages);
Assert.Contains("microsoft.aspnetcore.cryptography.internal", assets.Dependencies.Packages);
// Used frameworks
Assert.Contains("microsoft.netcore.app.ref", dependencies.Packages);
Assert.Contains("microsoft.aspnetcore.app.ref", dependencies.Packages);
Assert.Contains("microsoft.netcore.app.ref", assets.Dependencies.Packages);
Assert.Contains("microsoft.aspnetcore.app.ref", assets.Dependencies.Packages);
}
[Fact]
@@ -45,14 +44,13 @@ namespace Semmle.Extraction.Tests
// Setup
var assets = new Assets(new LoggerStub());
var json = "garbage data";
var dependencies = new DependencyContainer();
// Execute
var success = assets.TryParse(json, dependencies);
var success = assets.TryParse(json);
// Verify
Assert.False(success);
Assert.Empty(dependencies.Paths);
Assert.Empty(assets.Dependencies.Paths);
}
[Fact]
@@ -61,28 +59,27 @@ namespace Semmle.Extraction.Tests
// Setup
var assets = new Assets(new LoggerStub());
var json = assetsNet70;
var dependencies = new DependencyContainer();
// Execute
var success = assets.TryParse(json, dependencies);
var success = assets.TryParse(json);
// Verify
Assert.True(success);
Assert.Equal(4, dependencies.Paths.Count);
Assert.Equal(4, dependencies.Packages.Count);
Assert.Equal(4, assets.Dependencies.Paths.Count);
Assert.Equal(4, assets.Dependencies.Packages.Count);
var normalizedPaths = dependencies.Paths.Select(FixExpectedPathOnWindows);
var normalizedPaths = assets.Dependencies.Paths.Select(FixExpectedPathOnWindows);
// Used paths
Assert.Contains("microsoft.netcore.app.ref", normalizedPaths);
Assert.Contains("microsoft.aspnetcore.app.ref", normalizedPaths);
Assert.Contains("newtonsoft.json/12.0.1/lib/netstandard2.0/Newtonsoft.Json.dll", normalizedPaths);
Assert.Contains("newtonsoft.json.bson/1.0.2/lib/netstandard2.0/Newtonsoft.Json.Bson.dll", normalizedPaths);
// Used packages
Assert.Contains("microsoft.netcore.app.ref", dependencies.Packages);
Assert.Contains("microsoft.aspnetcore.app.ref", dependencies.Packages);
Assert.Contains("newtonsoft.json", dependencies.Packages);
Assert.Contains("newtonsoft.json.bson", dependencies.Packages);
Assert.Contains("microsoft.netcore.app.ref", assets.Dependencies.Packages);
Assert.Contains("microsoft.aspnetcore.app.ref", assets.Dependencies.Packages);
Assert.Contains("newtonsoft.json", assets.Dependencies.Packages);
Assert.Contains("newtonsoft.json.bson", assets.Dependencies.Packages);
}
@@ -92,25 +89,24 @@ namespace Semmle.Extraction.Tests
// Setup
var assets = new Assets(new LoggerStub());
var json = assetsNet48;
var dependencies = new DependencyContainer();
// Execute
var success = assets.TryParse(json, dependencies);
var success = assets.TryParse(json);
// Verify
Assert.True(success);
Assert.Equal(3, dependencies.Paths.Count);
Assert.Equal(3, dependencies.Packages.Count);
Assert.Equal(3, assets.Dependencies.Paths.Count);
Assert.Equal(3, assets.Dependencies.Packages.Count);
var normalizedPaths = dependencies.Paths.Select(FixExpectedPathOnWindows);
var normalizedPaths = assets.Dependencies.Paths.Select(FixExpectedPathOnWindows);
// Used references
Assert.Contains("microsoft.netframework.referenceassemblies.net48/1.0.2", normalizedPaths);
Assert.Contains("newtonsoft.json/12.0.1/lib/net45/Newtonsoft.Json.dll", normalizedPaths);
Assert.Contains("newtonsoft.json.bson/1.0.2/lib/net45/Newtonsoft.Json.Bson.dll", normalizedPaths);
// Used packages
Assert.Contains("microsoft.netframework.referenceassemblies.net48", dependencies.Packages);
Assert.Contains("newtonsoft.json", dependencies.Packages);
Assert.Contains("newtonsoft.json.bson", dependencies.Packages);
Assert.Contains("microsoft.netframework.referenceassemblies.net48", assets.Dependencies.Packages);
Assert.Contains("newtonsoft.json", assets.Dependencies.Packages);
Assert.Contains("newtonsoft.json.bson", assets.Dependencies.Packages);
}
[Fact]
@@ -119,26 +115,25 @@ namespace Semmle.Extraction.Tests
// Setup
var assets = new Assets(new LoggerStub());
var json = assetsNetstandard21;
var dependencies = new DependencyContainer();
// Execute
var success = assets.TryParse(json, dependencies);
var success = assets.TryParse(json);
// Verify
Assert.True(success);
Assert.Equal(3, dependencies.Paths.Count);
Assert.Equal(3, dependencies.Packages.Count);
Assert.Equal(3, assets.Dependencies.Paths.Count);
Assert.Equal(3, assets.Dependencies.Packages.Count);
var normalizedPaths = dependencies.Paths.Select(FixExpectedPathOnWindows);
var normalizedPaths = assets.Dependencies.Paths.Select(FixExpectedPathOnWindows);
// Used references
Assert.Contains("netstandard.library.ref", normalizedPaths);
Assert.Contains("newtonsoft.json/12.0.1/lib/netstandard2.0/Newtonsoft.Json.dll", normalizedPaths);
Assert.Contains("newtonsoft.json.bson/1.0.2/lib/netstandard2.0/Newtonsoft.Json.Bson.dll", normalizedPaths);
// Used packages
Assert.Contains("netstandard.library.ref", dependencies.Packages);
Assert.Contains("newtonsoft.json", dependencies.Packages);
Assert.Contains("newtonsoft.json.bson", dependencies.Packages);
Assert.Contains("netstandard.library.ref", assets.Dependencies.Packages);
Assert.Contains("newtonsoft.json", assets.Dependencies.Packages);
Assert.Contains("newtonsoft.json.bson", assets.Dependencies.Packages);
}
[Fact]
@@ -147,17 +142,16 @@ namespace Semmle.Extraction.Tests
// Setup
var assets = new Assets(new LoggerStub());
var json = assetsNetstandard16;
var dependencies = new DependencyContainer();
// Execute
var success = assets.TryParse(json, dependencies);
var success = assets.TryParse(json);
// Verify
Assert.True(success);
Assert.Equal(5, dependencies.Paths.Count);
Assert.Equal(5, dependencies.Packages.Count);
Assert.Equal(5, assets.Dependencies.Paths.Count);
Assert.Equal(5, assets.Dependencies.Packages.Count);
var normalizedPaths = dependencies.Paths.Select(FixExpectedPathOnWindows);
var normalizedPaths = assets.Dependencies.Paths.Select(FixExpectedPathOnWindows);
// Used references
Assert.Contains("netstandard.library/1.6.1", normalizedPaths);
@@ -166,11 +160,11 @@ namespace Semmle.Extraction.Tests
Assert.Contains("newtonsoft.json/12.0.1/lib/netstandard1.3/Newtonsoft.Json.dll", normalizedPaths);
Assert.Contains("newtonsoft.json.bson/1.0.2/lib/netstandard1.3/Newtonsoft.Json.Bson.dll", normalizedPaths);
// Used packages
Assert.Contains("netstandard.library", dependencies.Packages);
Assert.Contains("microsoft.csharp", dependencies.Packages);
Assert.Contains("microsoft.win32.primitives", dependencies.Packages);
Assert.Contains("newtonsoft.json", dependencies.Packages);
Assert.Contains("newtonsoft.json.bson", dependencies.Packages);
Assert.Contains("netstandard.library", assets.Dependencies.Packages);
Assert.Contains("microsoft.csharp", assets.Dependencies.Packages);
Assert.Contains("microsoft.win32.primitives", assets.Dependencies.Packages);
Assert.Contains("newtonsoft.json", assets.Dependencies.Packages);
Assert.Contains("newtonsoft.json.bson", assets.Dependencies.Packages);
}
[Fact]
@@ -179,26 +173,25 @@ namespace Semmle.Extraction.Tests
// Setup
var assets = new Assets(new LoggerStub());
var json = assetsNetcoreapp20;
var dependencies = new DependencyContainer();
// Execute
var success = assets.TryParse(json, dependencies);
var success = assets.TryParse(json);
// Verify
Assert.True(success);
Assert.Equal(144, dependencies.Paths.Count);
Assert.Equal(3, dependencies.Packages.Count);
Assert.Equal(144, assets.Dependencies.Paths.Count);
Assert.Equal(3, assets.Dependencies.Packages.Count);
var normalizedPaths = dependencies.Paths.Select(FixExpectedPathOnWindows);
var normalizedPaths = assets.Dependencies.Paths.Select(FixExpectedPathOnWindows);
// Used references (only some of them)
Assert.Contains("microsoft.netcore.app/2.0.0/ref/netcoreapp2.0/Microsoft.CSharp.dll", normalizedPaths);
Assert.Contains("newtonsoft.json/12.0.1/lib/netstandard2.0/Newtonsoft.Json.dll", normalizedPaths);
Assert.Contains("newtonsoft.json.bson/1.0.2/lib/netstandard2.0/Newtonsoft.Json.Bson.dll", normalizedPaths);
// Used packages
Assert.Contains("microsoft.netcore.app", dependencies.Packages);
Assert.Contains("newtonsoft.json", dependencies.Packages);
Assert.Contains("newtonsoft.json.bson", dependencies.Packages);
Assert.Contains("microsoft.netcore.app", assets.Dependencies.Packages);
Assert.Contains("newtonsoft.json", assets.Dependencies.Packages);
Assert.Contains("newtonsoft.json.bson", assets.Dependencies.Packages);
}
[Fact]
@@ -207,15 +200,14 @@ namespace Semmle.Extraction.Tests
// Setup
var assets = new Assets(new LoggerStub());
var json = assetsNetcoreapp31;
var dependencies = new DependencyContainer();
// Execute
var success = assets.TryParse(json, dependencies);
var success = assets.TryParse(json);
// Verify
Assert.True(success);
var normalizedPaths = dependencies.Paths.Select(FixExpectedPathOnWindows);
var normalizedPaths = assets.Dependencies.Paths.Select(FixExpectedPathOnWindows);
// Used paths
Assert.Contains("microsoft.netcore.app.ref", normalizedPaths);
@@ -223,10 +215,10 @@ namespace Semmle.Extraction.Tests
Assert.Contains("newtonsoft.json/12.0.1/lib/netstandard2.0/Newtonsoft.Json.dll", normalizedPaths);
Assert.Contains("newtonsoft.json.bson/1.0.2/lib/netstandard2.0/Newtonsoft.Json.Bson.dll", normalizedPaths);
// Used packages
Assert.Contains("microsoft.netcore.app.ref", dependencies.Packages);
Assert.Contains("microsoft.aspnetcore.app.ref", dependencies.Packages);
Assert.Contains("newtonsoft.json", dependencies.Packages);
Assert.Contains("newtonsoft.json.bson", dependencies.Packages);
Assert.Contains("microsoft.netcore.app.ref", assets.Dependencies.Packages);
Assert.Contains("microsoft.aspnetcore.app.ref", assets.Dependencies.Packages);
Assert.Contains("newtonsoft.json", assets.Dependencies.Packages);
Assert.Contains("newtonsoft.json.bson", assets.Dependencies.Packages);
}
/// <summary>