hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-05-02 04:05:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

C#: Remove unneeded options from standalone extractor

Browse Source
This commit is contained in:
Tamas Vajk
2023-12-12 10:24:21 +01:00
parent 16e86134f3
commit 21229b93bf
8 changed files with 82 additions and 228 deletions
Download Patch File Download Diff File Expand all files Collapse all files

144
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
View File

@@ -80,67 +80,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
this.nonGeneratedSources = allNonBinaryFiles.SelectFileNamesByExtension(".cs").ToList();
this.generatedSources = new();
var allProjects = allNonBinaryFiles.SelectFileNamesByExtension(".csproj");
var solutions = options.SolutionFile is not null
? new[] { options.SolutionFile }
: allNonBinaryFiles.SelectFileNamesByExtension(".sln");
var dllPaths = options.DllDirs.Count == 0
? allFiles.SelectFileNamesByExtension(".dll").ToHashSet()
: options.DllDirs.Select(Path.GetFullPath).ToHashSet();
if (options.UseNuGet)
{
try
{
var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, progressMonitor);
nuget.InstallPackages();
var nugetPackageDlls = legacyPackageDirectory.DirInfo.GetFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = true });
var nugetPackageDllPaths = nugetPackageDlls.Select(f => f.FullName).ToHashSet();
var excludedPaths = nugetPackageDllPaths
.Where(path => IsPathInSubfolder(path, legacyPackageDirectory.DirInfo.FullName, "tools"));
foreach (var excludedPath in excludedPaths)
{
progressMonitor.LogInfo($"Excluded Nuget DLL: {excludedPath}");
}
nugetPackageDllPaths.ExceptWith(excludedPaths);
dllPaths.UnionWith(nugetPackageDllPaths);
}
catch (FileNotFoundException)
{
progressMonitor.MissingNuGet();
}
var restoredProjects = RestoreSolutions(solutions, out var assets1);
var projects = allProjects.Except(restoredProjects);
RestoreProjects(projects, out var assets2);
var dependencies = Assets.GetCompilationDependencies(progressMonitor, assets1.Union(assets2));
var paths = dependencies
.Paths
.Select(d => Path.Combine(packageDirectory.DirInfo.FullName, d))
.ToList();
dllPaths.UnionWith(paths);
LogAllUnusedPackages(dependencies);
DownloadMissingPackages(allNonBinaryFiles, dllPaths);
}
var frameworkLocations = new HashSet<string>();
var allSolutions = allNonBinaryFiles.SelectFileNamesByExtension(".sln");
var dllPaths = allFiles.SelectFileNamesByExtension(".dll").ToHashSet();
RestoreNugetPackages(allNonBinaryFiles, allProjects, allSolutions, dllPaths);
// Find DLLs in the .Net / Asp.Net Framework
// This block needs to come after the nuget restore, because the nuget restore might fetch the .NET Core/Framework reference assemblies.
if (options.ScanNetFrameworkDlls)
{
AddNetFrameworkDlls(dllPaths, frameworkLocations);
AddAspNetCoreFrameworkDlls(dllPaths, frameworkLocations);
AddMicrosoftWindowsDesktopDlls(dllPaths, frameworkLocations);
}
// This needs to come after the nuget restore, because the nuget restore might fetch the .NET Core/Framework reference assemblies.
var frameworkLocations = AddFrameworkDlls(dllPaths);
assemblyCache = new AssemblyCache(dllPaths, frameworkLocations, progressMonitor);
AnalyseSolutions(solutions);
AnalyseSolutions(allSolutions);
foreach (var filename in assemblyCache.AllAssemblies.Select(a => a.Filename))
{
@@ -182,6 +131,58 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
DateTime.Now - startTime);
}
private HashSet<string> AddFrameworkDlls(HashSet<string> dllPaths)
{
var frameworkLocations = new HashSet<string>();
AddNetFrameworkDlls(dllPaths, frameworkLocations);
AddAspNetCoreFrameworkDlls(dllPaths, frameworkLocations);
AddMicrosoftWindowsDesktopDlls(dllPaths, frameworkLocations);
return frameworkLocations;
}
private void RestoreNugetPackages(List<FileInfo> allNonBinaryFiles, IEnumerable<string> allProjects, IEnumerable<string> allSolutions, HashSet<string> dllPaths)
{
try
{
var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, progressMonitor);
nuget.InstallPackages();
var nugetPackageDlls = legacyPackageDirectory.DirInfo.GetFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = true });
var nugetPackageDllPaths = nugetPackageDlls.Select(f => f.FullName).ToHashSet();
var excludedPaths = nugetPackageDllPaths
.Where(path => IsPathInSubfolder(path, legacyPackageDirectory.DirInfo.FullName, "tools"));
foreach (var excludedPath in excludedPaths)
{
progressMonitor.LogInfo($"Excluded Nuget DLL: {excludedPath}");
}
nugetPackageDllPaths.ExceptWith(excludedPaths);
dllPaths.UnionWith(nugetPackageDllPaths);
}
catch (FileNotFoundException)
{
progressMonitor.MissingNuGet();
}
var restoredProjects = RestoreSolutions(allSolutions, out var assets1);
var projects = allProjects.Except(restoredProjects);
RestoreProjects(projects, out var assets2);
var dependencies = Assets.GetCompilationDependencies(progressMonitor, assets1.Union(assets2));
var paths = dependencies
.Paths
.Select(d => Path.Combine(packageDirectory.DirInfo.FullName, d))
.ToList();
dllPaths.UnionWith(paths);
LogAllUnusedPackages(dependencies);
DownloadMissingPackages(allNonBinaryFiles, dllPaths);
}
private static bool IsPathInSubfolder(string path, string rootFolder, string subFolder)
{
return path.IndexOf(
@@ -192,11 +193,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private void RemoveNugetAnalyzerReferences()
{
if (!options.UseNuGet)
{
return;
}
var packageFolder = packageDirectory.DirInfo.FullName.ToLowerInvariant();
if (packageFolder == null)
{
@@ -279,11 +275,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
string? runtimeLocation = null;
if (options.UseSelfContainedDotnet)
{
runtimeLocation = Runtime.ExecutingRuntime;
}
else if (fileContent.IsNewProjectStructureUsed)
if (fileContent.IsNewProjectStructureUsed)
{
runtimeLocation = Runtime.NetCoreRuntime;
}
@@ -301,11 +293,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private void RemoveNugetPackageReference(string packagePrefix, ISet<string> dllPaths)
{
if (!options.UseNuGet)
{
return;
}
var packageFolder = packageDirectory.DirInfo.FullName.ToLowerInvariant();
if (packageFolder == null)
{
@@ -353,11 +340,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private string? GetPackageDirectory(string packagePrefix)
{
if (!options.UseNuGet)
{
return null;
}
return new DirectoryInfo(packageDirectory.DirInfo.FullName)
.EnumerateDirectories(packagePrefix + "*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
.FirstOrDefault()?
@@ -366,11 +348,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private IEnumerable<string> GetAllPackageDirectories()
{
if (!options.UseNuGet)
{
return Enumerable.Empty<string>();
}
return new DirectoryInfo(packageDirectory.DirInfo.FullName)
.EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
.Select(d => d.Name);
@@ -455,8 +432,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private IEnumerable<FileInfo> GetAllFiles()
{
var files = sourceDir.GetFiles("*.*", new EnumerationOptions { RecurseSubdirectories = true })
.Where(d => !options.ExcludesFile(d.FullName));
IEnumerable<FileInfo> files = sourceDir.GetFiles("*.*", new EnumerationOptions { RecurseSubdirectories = true });
if (options.DotNetPath != null)
{

52
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyOptions.cs
View File

@@ -10,43 +10,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// </summary>
public interface IDependencyOptions
{
/// <summary>
/// Directories to search DLLs in.
/// </summary>
IList<string> DllDirs { get; }
/// <summary>
/// Files/patterns to exclude.
/// </summary>
IList<string> Excludes { get; }
/// <summary>
/// Whether to analyse NuGet packages.
/// </summary>
bool UseNuGet { get; }
/// <summary>
/// The solution file to analyse, or null if not specified.
/// </summary>
string? SolutionFile { get; }
/// <summary>
/// Whether to use the packaged dotnet runtime.
/// </summary>
bool UseSelfContainedDotnet { get; }
/// <summary>
/// Whether to search the .Net framework directory.
/// </summary>
bool ScanNetFrameworkDlls { get; }
/// <summary>
/// Determine whether the given path should be excluded.
/// </summary>
/// <param name="path">The path to query.</param>
/// <returns>True iff the path matches an exclusion.</returns>
bool ExcludesFile(string path);
/// <summary>
/// The number of threads to use.
/// </summary>
@@ -62,21 +25,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
public static IDependencyOptions Default => new DependencyOptions();
public IList<string> DllDirs { get; set; } = new List<string>();
public IList<string> Excludes { get; set; } = new List<string>();
public bool UseNuGet { get; set; } = true;
public string? SolutionFile { get; set; }
public bool UseSelfContainedDotnet { get; set; } = false;
public bool ScanNetFrameworkDlls { get; set; } = true;
public bool ExcludesFile(string path) =>
Excludes.Any(path.Contains);
public int Threads { get; set; } = EnvironmentVariables.GetDefaultNumberOfThreads();
public string? DotNetPath { get; set; } = null;