hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-04-30 03:05:15 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #16248 from michaelnebel/csharp/groupsprojectbeforerestore

Browse Source 
C#: Restore projects and collect dependencies for projects in the same folder sequentially.
This commit is contained in:
Michael Nebel
2024-04-29 14:05:40 +02:00
committed by GitHub
parent 1c611fecde d62e888b86
commit a304e2d8d6
12 changed files with 359 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Assets.cs
View File

@@ -16,6 +16,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
private readonly ILogger logger;
/// <summary>
/// Contains the dependencies found in the parsed assets files.
/// </summary>
public DependencyContainer Dependencies { get; } = new();
internal Assets(ILogger logger)
{
this.logger = logger;
@@ -72,7 +77,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// "json.net"
/// }
/// </summary>
private void AddPackageDependencies(JObject json, DependencyContainer dependencies)
private void AddPackageDependencies(JObject json)
{
// If there is more than one framework we need to pick just one.
// To ensure stability we pick one based on the lexicographic order of
@@ -107,13 +112,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
// If this is a framework reference then include everything.
if (FrameworkPackageNames.AllFrameworks.Any(framework => name.StartsWith(framework)))
{
dependencies.AddFramework(name);
Dependencies.AddFramework(name);
}
return;
}
info.Compile
.ForEach(r => dependencies.Add(name, r.Key));
.ForEach(r => Dependencies.Add(name, r.Key));
});
return;
@@ -149,7 +154,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// "microsoft.netcore.app.ref"
/// }
/// </summary>
private void AddFrameworkDependencies(JObject json, DependencyContainer dependencies)
private void AddFrameworkDependencies(JObject json)
{
var frameworks = json
@@ -178,7 +183,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
references
.Properties()
.ForEach(f => dependencies.AddFramework($"{f.Name}.Ref".ToLowerInvariant()));
.ForEach(f => Dependencies.AddFramework($"{f.Name}.Ref".ToLowerInvariant()));
}
/// <summary>
@@ -186,13 +191,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// (together with used package information) required for compilation.
/// </summary>
/// <returns>True if parsing succeeds, otherwise false.</returns>
public bool TryParse(string json, DependencyContainer dependencies)
public bool TryParse(string json)
{
try
{
var obj = JObject.Parse(json);
AddPackageDependencies(obj, dependencies);
AddFrameworkDependencies(obj, dependencies);
AddPackageDependencies(obj);
AddFrameworkDependencies(obj);
return true;
}
catch (Exception e)
@@ -217,19 +222,24 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
}
public static DependencyContainer GetCompilationDependencies(ILogger logger, IEnumerable<string> assets)
/// <summary>
/// Add the dependencies from the assets file to the dependencies.
/// </summary>
/// <param name="asset">Path to an assets file.</param>
public void AddDependencies(string asset)
{
var parser = new Assets(logger);
var dependencies = new DependencyContainer();
assets.ForEach(asset =>
if (TryReadAllText(asset, logger, out var json))
{
if (TryReadAllText(asset, logger, out var json))
{
parser.TryParse(json, dependencies);
}
});
return dependencies;
TryParse(json);
}
}
/// <summary>
/// Add the dependencies from the assets files to the dependencies.
/// </summary>
/// <param name="assets">Collection of paths to assets files.</param>
public void AddDependenciesRange(IEnumerable<string> assets) =>
assets.ForEach(AddDependencies);
}
internal static class JsonExtensions

20
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyContainer.cs
View File

@@ -12,7 +12,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// <summary>
/// Paths to dependencies required for compilation.
/// </summary>
public List<string> Paths { get; } = new();
public HashSet<string> Paths { get; } = new();
/// <summary>
/// Packages that are used as a part of the required dependencies.
@@ -45,7 +45,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var p = package.Replace('/', Path.DirectorySeparatorChar);
var d = dependency.Replace('/', Path.DirectorySeparatorChar);
// In most cases paths in asset files point to dll's or the empty _._ file.
// In most cases paths in assets files point to dll's or the empty _._ file.
// That is, for _._ we don't need to add anything.
if (Path.GetFileName(d) == "_._")
{
@@ -68,4 +68,18 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
Packages.Add(GetPackageName(p));
}
}
}
internal static class DependencyContainerExtensions
{
/// <summary>
/// Flatten a list of containers into a single container.
/// </summary>
public static DependencyContainer Flatten(this IEnumerable<DependencyContainer> containers, DependencyContainer init) =>
containers.Aggregate(init, (acc, container) =>
{
acc.Paths.UnionWith(container.Paths);
acc.Packages.UnionWith(container.Packages);
return acc;
});
}
}

2
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
View File

@@ -3,8 +3,6 @@ using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks;
using Semmle.Util;

53
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs
View File

@@ -1,4 +1,5 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
@@ -144,11 +145,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
logger.LogError($"Failed to restore Nuget packages with nuget.exe: {exc.Message}");
}
var restoredProjects = RestoreSolutions(out var assets1);
var restoredProjects = RestoreSolutions(out var container);
var projects = fileProvider.Projects.Except(restoredProjects);
RestoreProjects(projects, out var assets2);
RestoreProjects(projects, out var containers);
var dependencies = Assets.GetCompilationDependencies(logger, assets1.Union(assets2));
var dependencies = containers.Flatten(container);
var paths = dependencies
.Paths
@@ -198,14 +199,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// As opposed to RestoreProjects this is not run in parallel using PLINQ
/// as `dotnet restore` on a solution already uses multiple threads for restoring
/// the projects (this can be disabled with the `--disable-parallel` flag).
/// Populates assets with the relative paths to the assets files generated by the restore.
/// Populates dependencies with the relevant dependencies from the assets files generated by the restore.
/// Returns a list of projects that are up to date with respect to restore.
/// </summary>
private IEnumerable<string> RestoreSolutions(out IEnumerable<string> assets)
private IEnumerable<string> RestoreSolutions(out DependencyContainer dependencies)
{
var successCount = 0;
var nugetSourceFailures = 0;
var assetFiles = new List<string>();
var assets = new Assets(logger);
var projects = fileProvider.Solutions.SelectMany(solution =>
{
logger.LogInfo($"Restoring solution {solution}...");
@@ -218,10 +219,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
nugetSourceFailures++;
}
assetFiles.AddRange(res.AssetsFilePaths);
assets.AddDependenciesRange(res.AssetsFilePaths);
return res.RestoredProjects;
}).ToList();
assets = assetFiles;
dependencies = assets.Dependencies;
compilationInfoContainer.CompilationInfos.Add(("Successfully restored solution files", successCount.ToString()));
compilationInfoContainer.CompilationInfos.Add(("Failed solution restore with package source error", nugetSourceFailures.ToString()));
compilationInfoContainer.CompilationInfos.Add(("Restored projects through solution files", projects.Count.ToString()));
@@ -231,33 +232,39 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// <summary>
/// Executes `dotnet restore` on all projects in projects.
/// This is done in parallel for performance reasons.
/// Populates assets with the relative paths to the assets files generated by the restore.
/// Populates dependencies with the relative paths to the assets files generated by the restore.
/// </summary>
/// <param name="projects">A list of paths to project files.</param>
private void RestoreProjects(IEnumerable<string> projects, out IEnumerable<string> assets)
private void RestoreProjects(IEnumerable<string> projects, out ConcurrentBag<DependencyContainer> dependencies)
{
var successCount = 0;
var nugetSourceFailures = 0;
var assetFiles = new List<string>();
ConcurrentBag<DependencyContainer> collectedDependencies = [];
var sync = new object();
Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = DependencyManager.Threads }, project =>
var projectGroups = projects.GroupBy(Path.GetDirectoryName);
Parallel.ForEach(projectGroups, new ParallelOptions { MaxDegreeOfParallelism = DependencyManager.Threads }, projectGroup =>
{
logger.LogInfo($"Restoring project {project}...");
var res = dotnet.Restore(new(project, PackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
lock (sync)
var assets = new Assets(logger);
foreach (var project in projectGroup)
{
if (res.Success)
logger.LogInfo($"Restoring project {project}...");
var res = dotnet.Restore(new(project, PackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
assets.AddDependenciesRange(res.AssetsFilePaths);
lock (sync)
{
successCount++;
if (res.Success)
{
successCount++;
}
if (res.HasNugetPackageSourceError)
{
nugetSourceFailures++;
}
}
if (res.HasNugetPackageSourceError)
{
nugetSourceFailures++;
}
assetFiles.AddRange(res.AssetsFilePaths);
}
collectedDependencies.Add(assets.Dependencies);
});
assets = assetFiles;
dependencies = collectedDependencies;
compilationInfoContainer.CompilationInfos.Add(("Successfully restored project files", successCount.ToString()));
compilationInfoContainer.CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString()));
}