hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-04-28 18:25:24 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #15600 from tamasvajk/buildless/no-mono-dlls

Browse Source 
C# Change desktop dotnet assembly lookup to fall back to nuget reference assemblies
This commit is contained in:
Tamás Vajk
2024-02-21 08:36:41 +01:00
committed by GitHub
parent f2e04c0cb2 216d6c0827
commit 70a2d16b1a
21 changed files with 228 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

166
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
View File

@@ -62,7 +62,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
try
{
this.dotnet = DotNet.Make(options, logger, tempWorkingDirectory);
runtimeLazy = new Lazy<Runtime>(() => new Runtime(dotnet));
runtimeLazy = new Lazy<Runtime>(() => new Runtime(dotnet, logger));
}
catch
{
@@ -112,7 +112,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
logger.LogInfo($"Unresolved reference {r.Key} in project {r.Value}");
}
var webViewExtractionOption = Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_STANDALONE_EXTRACT_WEB_VIEWS");
var webViewExtractionOption = Environment.GetEnvironmentVariable(EnvironmentVariableNames.WebViewGeneration);
if (webViewExtractionOption == null ||
bool.TryParse(webViewExtractionOption, out var shouldExtractWebViews) &&
shouldExtractWebViews)
@@ -159,6 +159,53 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
var frameworkLocations = new HashSet<string>();
var frameworkReferences = Environment.GetEnvironmentVariable(EnvironmentVariableNames.DotnetFrameworkReferences);
var frameworkReferencesUseSubfolders = Environment.GetEnvironmentVariable(EnvironmentVariableNames.DotnetFrameworkReferencesUseSubfolders);
_ = bool.TryParse(frameworkReferencesUseSubfolders, out var useSubfolders);
if (!string.IsNullOrWhiteSpace(frameworkReferences))
{
RemoveFrameworkNugetPackages(dllPaths);
RemoveNugetPackageReference(FrameworkPackageNames.AspNetCoreFramework, dllPaths);
RemoveNugetPackageReference(FrameworkPackageNames.WindowsDesktopFramework, dllPaths);
var frameworkPaths = frameworkReferences.Split(Path.PathSeparator, StringSplitOptions.RemoveEmptyEntries);
foreach (var path in frameworkPaths)
{
if (!Directory.Exists(path))
{
logger.LogError($"Specified framework reference path '{path}' does not exist.");
continue;
}
if (useSubfolders)
{
dllPaths.Add(path);
frameworkLocations.Add(path);
continue;
}
try
{
var dlls = Directory.GetFiles(path, "*.dll", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive });
if (dlls.Length == 0)
{
logger.LogError($"No DLLs found in specified framework reference path '{path}'.");
continue;
}
dllPaths.UnionWith(dlls);
frameworkLocations.UnionWith(dlls);
}
catch (Exception e)
{
logger.LogError($"Error while searching for DLLs in '{path}': {e.Message}");
}
}
return frameworkLocations;
}
AddNetFrameworkDlls(dllPaths, frameworkLocations);
AddAspNetCoreFrameworkDlls(dllPaths, frameworkLocations);
AddMicrosoftWindowsDesktopDlls(dllPaths, frameworkLocations);
@@ -204,9 +251,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
nugetPackageDllPaths.ExceptWith(excludedPaths);
dllPaths.UnionWith(nugetPackageDllPaths);
}
catch (Exception)
catch (Exception exc)
{
logger.LogError("Failed to restore Nuget packages with nuget.exe");
logger.LogError($"Failed to restore Nuget packages with nuget.exe: {exc.Message}");
}
var restoredProjects = RestoreSolutions(allSolutions, out var assets1);
@@ -297,6 +344,15 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
.ToArray();
}
private void RemoveFrameworkNugetPackages(ISet<string> dllPaths, int fromIndex = 0)
{
var packagesInPrioOrder = FrameworkPackageNames.NetFrameworks;
for (var i = fromIndex; i < packagesInPrioOrder.Length; i++)
{
RemoveNugetPackageReference(packagesInPrioOrder[i], dllPaths);
}
}
private void AddNetFrameworkDlls(ISet<string> dllPaths, ISet<string> frameworkLocations)
{
// Multiple dotnet framework packages could be present.
@@ -304,7 +360,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var packagesInPrioOrder = FrameworkPackageNames.NetFrameworks;
var frameworkPaths = packagesInPrioOrder
.Select((s, index) => (Index: index, Path: GetPackageDirectory(s)))
.Select((s, index) => (Index: index, Path: GetPackageDirectory(s, packageDirectory)))
.Where(pair => pair.Path is not null)
.ToArray();
@@ -318,12 +374,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
SelectNewestFrameworkPath(frameworkPath.Path, ".NET Framework", dllPaths, frameworkLocations);
for (var i = frameworkPath.Index + 1; i < packagesInPrioOrder.Length; i++)
{
RemoveNugetPackageReference(packagesInPrioOrder[i], dllPaths);
}
RemoveFrameworkNugetPackages(dllPaths, frameworkPath.Index + 1);
return;
}
@@ -336,6 +387,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
else if (fileContent.IsLegacyProjectStructureUsed)
{
runtimeLocation = Runtime.DesktopRuntime;
if (runtimeLocation is null)
{
logger.LogInfo("No .NET Desktop Runtime location found. Attempting to restore the .NET Framework reference assemblies manually.");
if (TryRestorePackageManually(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies, null))
{
runtimeLocation = GetPackageDirectory(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies, missingPackageDirectory);
}
}
}
runtimeLocation ??= Runtime.ExecutingRuntime;
@@ -375,7 +436,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
// First try to find ASP.NET Core assemblies in the NuGet packages
if (GetPackageDirectory(FrameworkPackageNames.AspNetCoreFramework) is string aspNetCorePackage)
if (GetPackageDirectory(FrameworkPackageNames.AspNetCoreFramework, packageDirectory) is string aspNetCorePackage)
{
SelectNewestFrameworkPath(aspNetCorePackage, "ASP.NET Core", dllPaths, frameworkLocations);
return;
@@ -391,15 +452,15 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private void AddMicrosoftWindowsDesktopDlls(ISet<string> dllPaths, ISet<string> frameworkLocations)
{
if (GetPackageDirectory(FrameworkPackageNames.WindowsDesktopFramework) is string windowsDesktopApp)
if (GetPackageDirectory(FrameworkPackageNames.WindowsDesktopFramework, packageDirectory) is string windowsDesktopApp)
{
SelectNewestFrameworkPath(windowsDesktopApp, "Windows Desktop App", dllPaths, frameworkLocations);
}
}
private string? GetPackageDirectory(string packagePrefix)
private string? GetPackageDirectory(string packagePrefix, TemporaryDirectory root)
{
return new DirectoryInfo(packageDirectory.DirInfo.FullName)
return new DirectoryInfo(root.DirInfo.FullName)
.EnumerateDirectories(packagePrefix + "*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
.FirstOrDefault()?
.FullName;
@@ -435,19 +496,19 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
// Hardcoded values from https://learn.microsoft.com/en-us/dotnet/core/project-sdk/overview#implicit-using-directives
usings.UnionWith(new[] { "System", "System.Collections.Generic", "System.IO", "System.Linq", "System.Net.Http", "System.Threading",
"System.Threading.Tasks" });
usings.UnionWith([ "System", "System.Collections.Generic", "System.IO", "System.Linq", "System.Net.Http", "System.Threading",
"System.Threading.Tasks" ]);
if (fileContent.UseAspNetCoreDlls)
{
usings.UnionWith(new[] { "System.Net.Http.Json", "Microsoft.AspNetCore.Builder", "Microsoft.AspNetCore.Hosting",
usings.UnionWith([ "System.Net.Http.Json", "Microsoft.AspNetCore.Builder", "Microsoft.AspNetCore.Hosting",
"Microsoft.AspNetCore.Http", "Microsoft.AspNetCore.Routing", "Microsoft.Extensions.Configuration",
"Microsoft.Extensions.DependencyInjection", "Microsoft.Extensions.Hosting", "Microsoft.Extensions.Logging" });
"Microsoft.Extensions.DependencyInjection", "Microsoft.Extensions.Hosting", "Microsoft.Extensions.Logging" ]);
}
if (fileContent.UseWindowsForms)
{
usings.UnionWith(new[] { "System.Drawing", "System.Windows.Forms" });
usings.UnionWith(["System.Drawing", "System.Windows.Forms"]);
}
usings.UnionWith(fileContent.CustomImplicitUsings);
@@ -869,38 +930,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, package =>
{
logger.LogInfo($"Restoring package {package}...");
using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package, "missingpackages_workingdir"));
var success = dotnet.New(tempDir.DirInfo.FullName);
var success = TryRestorePackageManually(package, nugetConfig);
if (!success)
{
return;
}
success = dotnet.AddPackage(tempDir.DirInfo.FullName, package);
if (!success)
{
return;
}
var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig));
if (!res.Success)
{
if (res.HasNugetPackageSourceError)
{
// Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true));
}
// TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package.
if (!res.Success)
{
logger.LogInfo($"Failed to restore nuget package {package}");
return;
}
}
lock (sync)
{
successCount++;
@@ -912,6 +947,43 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
dllPaths.Add(missingPackageDirectory.DirInfo.FullName);
}
private bool TryRestorePackageManually(string package, string? nugetConfig)
{
logger.LogInfo($"Restoring package {package}...");
using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package, "missingpackages_workingdir"));
var success = dotnet.New(tempDir.DirInfo.FullName);
if (!success)
{
return false;
}
success = dotnet.AddPackage(tempDir.DirInfo.FullName, package);
if (!success)
{
return false;
}
var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig));
if (!res.Success)
{
if (res.HasNugetPackageSourceError && nugetConfig is not null)
{
// Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true));
}
// TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package.
if (!res.Success)
{
logger.LogInfo($"Failed to restore nuget package {package}");
return false;
}
}
return true;
}
public void Dispose(TemporaryDirectory? dir, string name)
{
try

20
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs Normal file
View File

@@ -0,0 +1,20 @@
namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal class EnvironmentVariableNames
{
/// <summary>
/// Controls whether to generate source files from Asp.Net Core views (`.cshtml`, `.razor`).
/// </summary>
public const string WebViewGeneration = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_EXTRACT_WEB_VIEWS";
/// <summary>
/// Specifies the location of .Net framework references added to the compilation.
/// </summary>
public const string DotnetFrameworkReferences = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_DOTNET_FRAMEWORK_REFERENCES";
/// <summary>
/// Controls whether to use framework dependencies from subfolders.
/// </summary>
public const string DotnetFrameworkReferencesUseSubfolders = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_DOTNET_FRAMEWORK_REFERENCES_USE_SUBFOLDERS";
}
}

2
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FilePathFilter.cs
View File

@@ -31,7 +31,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
public IEnumerable<FileInfo> Filter(IEnumerable<FileInfo> files)
{
var filters = (Environment.GetEnvironmentVariable("LGTM_INDEX_FILTERS") ?? string.Empty).Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
var filters = (Environment.GetEnvironmentVariable("LGTM_INDEX_FILTERS") ?? string.Empty).Split(FileUtils.NewLineCharacters, StringSplitOptions.RemoveEmptyEntries);
if (filters.Length == 0)
{
return files;

12
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FrameworkPackageNames.cs
View File

@@ -1,25 +1,25 @@
using System.Collections.Generic;
using System.Linq;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal static class FrameworkPackageNames
{
public static string LatestNetFrameworkReferenceAssemblies { get; } = "microsoft.netframework.referenceassemblies.net481";
public static string AspNetCoreFramework { get; } = "microsoft.aspnetcore.app.ref";
public static string WindowsDesktopFramework { get; } = "microsoft.windowsdesktop.app.ref";
// The order of the packages is important.
public static string[] NetFrameworks { get; } = new string[]
{
public static string[] NetFrameworks { get; } =
[
"microsoft.netcore.app.ref", // net7.0, ... net5.0, netcoreapp3.1, netcoreapp3.0
"microsoft.netframework.referenceassemblies.", // net48, ..., net20
"netstandard.library.ref", // netstandard2.1
"netstandard.library" // netstandard2.0
};
];
public static IEnumerable<string> AllFrameworks { get; } =
NetFrameworks
.Union(new string[] { AspNetCoreFramework, WindowsDesktopFramework });
[.. NetFrameworks, AspNetCoreFramework, WindowsDesktopFramework];
}
}

2
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Razor.cs
View File

@@ -71,7 +71,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var args = new StringBuilder();
args.Append($"/target:exe /generatedfilesout:\"{outputFolder}\" /out:\"{dllPath}\" /analyzerconfig:\"{analyzerConfig}\" ");
foreach (var f in Directory.GetFiles(sourceGeneratorFolder, "*.dll"))
foreach (var f in Directory.GetFiles(sourceGeneratorFolder, "*.dll", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive }))
{
args.Append($"/analyzer:\"{f}\" ");
}

24
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Runtime.cs
View File

@@ -5,6 +5,7 @@ using System.Linq;
using System.Runtime.InteropServices;
using System.Text.RegularExpressions;
using Semmle.Util;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
@@ -17,12 +18,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private const string aspNetCoreApp = "Microsoft.AspNetCore.App";
private readonly IDotNet dotNet;
private readonly ILogger logger;
private readonly Lazy<Dictionary<string, DotNetVersion>> newestRuntimes;
private Dictionary<string, DotNetVersion> NewestRuntimes => newestRuntimes.Value;
public Runtime(IDotNet dotNet)
public Runtime(IDotNet dotNet, ILogger logger)
{
this.dotNet = dotNet;
this.logger = logger;
this.newestRuntimes = new(GetNewestRuntimes);
}
@@ -65,7 +68,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// Locates .NET Desktop Runtimes.
/// This includes Mono and Microsoft.NET.
/// </summary>
private static IEnumerable<string> DesktopRuntimes
private IEnumerable<string> DesktopRuntimes
{
get
{
@@ -76,20 +79,19 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
var monoPath = FileUtils.FindProgramOnPath(Win32.IsWindows() ? "mono.exe" : "mono");
var monoDirs = monoPath is not null
? new[] { Path.GetFullPath(Path.Combine(monoPath, "..", "lib", "mono")), monoPath }
: new[] { "/usr/lib/mono", "/usr/local/mono", "/usr/local/bin/mono", @"C:\Program Files\Mono\lib\mono" };
string[] monoDirs = monoPath is not null
? [Path.GetFullPath(Path.Combine(monoPath, "..", "lib", "mono")), monoPath]
: ["/usr/lib/mono", "/usr/local/mono", "/usr/local/bin/mono", @"C:\Program Files\Mono\lib\mono"];
var dir = monoDirs.FirstOrDefault(Directory.Exists);
if (dir is not null)
var monoDir = monoDirs.FirstOrDefault(Directory.Exists);
if (monoDir is not null)
{
return Directory.EnumerateDirectories(dir)
.Where(d => Char.IsDigit(Path.GetFileName(d)[0]))
return Directory.EnumerateDirectories(monoDir)
.Where(d => char.IsDigit(Path.GetFileName(d)[0]))
.OrderByDescending(Path.GetFileName);
}
return Enumerable.Empty<string>();
return [];
}
}

2
csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs
View File

@@ -386,7 +386,7 @@ namespace Semmle.Extraction.CSharp
if (compilerArguments.GeneratedFilesOutputDirectory is not null)
{
paths.AddRange(Directory.GetFiles(compilerArguments.GeneratedFilesOutputDirectory, "*.cs", SearchOption.AllDirectories));
paths.AddRange(Directory.GetFiles(compilerArguments.GeneratedFilesOutputDirectory, "*.cs", new EnumerationOptions { RecurseSubdirectories = true, MatchCasing = MatchCasing.CaseInsensitive }));
}
return ReadSyntaxTrees(

8
csharp/extractor/Semmle.Extraction.Tests/Runtime.cs
View File

@@ -47,7 +47,7 @@ namespace Semmle.Extraction.Tests
"Microsoft.NETCore.App 7.0.2 [/path/dotnet/shared/Microsoft.NETCore.App]"
};
var dotnet = new DotNetStub(listedRuntimes, null!);
var runtime = new Runtime(dotnet);
var runtime = new Runtime(dotnet, new LoggerStub());
// Execute
var runtimes = runtime.GetNewestRuntimes();
@@ -73,7 +73,7 @@ namespace Semmle.Extraction.Tests
"Microsoft.NETCore.App 8.0.0-preview.5.23280.8 [/path/dotnet/shared/Microsoft.NETCore.App]"
};
var dotnet = new DotNetStub(listedRuntimes, null!);
var runtime = new Runtime(dotnet);
var runtime = new Runtime(dotnet, new LoggerStub());
// Execute
var runtimes = runtime.GetNewestRuntimes();
@@ -96,7 +96,7 @@ namespace Semmle.Extraction.Tests
"Microsoft.NETCore.App 8.0.0-preview.5.23280.8 [/path/dotnet/shared/Microsoft.NETCore.App]"
};
var dotnet = new DotNetStub(listedRuntimes, null!);
var runtime = new Runtime(dotnet);
var runtime = new Runtime(dotnet, new LoggerStub());
// Execute
var runtimes = runtime.GetNewestRuntimes();
@@ -125,7 +125,7 @@ namespace Semmle.Extraction.Tests
@"Microsoft.WindowsDesktop.App 7.0.4 [C:\Program Files\dotnet\shared\Microsoft.WindowsDesktop.App]"
};
var dotnet = new DotNetStub(listedRuntimes, null!);
var runtime = new Runtime(dotnet);
var runtime = new Runtime(dotnet, new LoggerStub());
// Execute
var runtimes = runtime.GetNewestRuntimes();

2
csharp/extractor/Semmle.Extraction/CsProjFile.cs
View File

@@ -112,7 +112,7 @@ namespace Semmle.Extraction
.Where(s => s is not null)
?? Enumerable.Empty<string>();
var additionalCsFiles = System.IO.Directory.GetFiles(directoryName, "*.cs", SearchOption.AllDirectories);
var additionalCsFiles = System.IO.Directory.GetFiles(directoryName, "*.cs", new EnumerationOptions { RecurseSubdirectories = true, MatchCasing = MatchCasing.CaseInsensitive });
var projectReferences = root
.SelectNodes("/Project/ItemGroup/ProjectReference/@Include", mgr)

2
csharp/extractor/Semmle.Util/FileUtils.cs
View File

@@ -13,6 +13,8 @@ namespace Semmle.Util
{
public const string NugetExeUrl = "https://dist.nuget.org/win-x86-commandline/latest/nuget.exe";
public static readonly char[] NewLineCharacters = ['\r', '\n'];
public static string ConvertToWindows(string path)
{
return path.Replace('/', '\\');