hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-04-28 10:15:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

Change desktop dotnet assembly lookup to fall back to nuget reference assemblies

Browse Source
This commit is contained in:
Tamas Vajk
2024-02-13 14:19:57 +01:00
parent 04f0fb0483
commit 8f0f6963bb
6 changed files with 338 additions and 301 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
View File

@@ -61,7 +61,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
try
{
this.dotnet = DotNet.Make(options, logger, tempWorkingDirectory);
runtimeLazy = new Lazy<Runtime>(() => new Runtime(dotnet));
runtimeLazy = new Lazy<Runtime>(() => new Runtime(dotnet, logger));
}
catch
{
@@ -303,7 +303,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var packagesInPrioOrder = FrameworkPackageNames.NetFrameworks;
var frameworkPaths = packagesInPrioOrder
.Select((s, index) => (Index: index, Path: GetPackageDirectory(s)))
.Select((s, index) => (Index: index, Path: GetPackageDirectory(s, packageDirectory)))
.Where(pair => pair.Path is not null)
.ToArray();
@@ -335,6 +335,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
else if (fileContent.IsLegacyProjectStructureUsed)
{
runtimeLocation = Runtime.DesktopRuntime;
if (runtimeLocation is null)
{
logger.LogInfo("No .NET Desktop Runtime location found. Attempting to restore the .NET Framework reference assemblies manually.");
if (TryRestorePackageManually(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies, null))
{
runtimeLocation = GetPackageDirectory(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies, missingPackageDirectory);
}
}
}
runtimeLocation ??= Runtime.ExecutingRuntime;
@@ -374,7 +384,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
// First try to find ASP.NET Core assemblies in the NuGet packages
if (GetPackageDirectory(FrameworkPackageNames.AspNetCoreFramework) is string aspNetCorePackage)
if (GetPackageDirectory(FrameworkPackageNames.AspNetCoreFramework, packageDirectory) is string aspNetCorePackage)
{
SelectNewestFrameworkPath(aspNetCorePackage, "ASP.NET Core", dllPaths, frameworkLocations);
return;
@@ -390,15 +400,15 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private void AddMicrosoftWindowsDesktopDlls(ISet<string> dllPaths, ISet<string> frameworkLocations)
{
if (GetPackageDirectory(FrameworkPackageNames.WindowsDesktopFramework) is string windowsDesktopApp)
if (GetPackageDirectory(FrameworkPackageNames.WindowsDesktopFramework, packageDirectory) is string windowsDesktopApp)
{
SelectNewestFrameworkPath(windowsDesktopApp, "Windows Desktop App", dllPaths, frameworkLocations);
}
}
private string? GetPackageDirectory(string packagePrefix)
private string? GetPackageDirectory(string packagePrefix, TemporaryDirectory root)
{
return new DirectoryInfo(packageDirectory.DirInfo.FullName)
return new DirectoryInfo(root.DirInfo.FullName)
.EnumerateDirectories(packagePrefix + "*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
.FirstOrDefault()?
.FullName;
@@ -434,19 +444,19 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
// Hardcoded values from https://learn.microsoft.com/en-us/dotnet/core/project-sdk/overview#implicit-using-directives
usings.UnionWith(new[] { "System", "System.Collections.Generic", "System.IO", "System.Linq", "System.Net.Http", "System.Threading",
"System.Threading.Tasks" });
usings.UnionWith([ "System", "System.Collections.Generic", "System.IO", "System.Linq", "System.Net.Http", "System.Threading",
"System.Threading.Tasks" ]);
if (fileContent.UseAspNetCoreDlls)
{
usings.UnionWith(new[] { "System.Net.Http.Json", "Microsoft.AspNetCore.Builder", "Microsoft.AspNetCore.Hosting",
usings.UnionWith([ "System.Net.Http.Json", "Microsoft.AspNetCore.Builder", "Microsoft.AspNetCore.Hosting",
"Microsoft.AspNetCore.Http", "Microsoft.AspNetCore.Routing", "Microsoft.Extensions.Configuration",
"Microsoft.Extensions.DependencyInjection", "Microsoft.Extensions.Hosting", "Microsoft.Extensions.Logging" });
"Microsoft.Extensions.DependencyInjection", "Microsoft.Extensions.Hosting", "Microsoft.Extensions.Logging" ]);
}
if (fileContent.UseWindowsForms)
{
usings.UnionWith(new[] { "System.Drawing", "System.Windows.Forms" });
usings.UnionWith(["System.Drawing", "System.Windows.Forms"]);
}
usings.UnionWith(fileContent.CustomImplicitUsings);
@@ -828,38 +838,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, package =>
{
logger.LogInfo($"Restoring package {package}...");
using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package, "missingpackages_workingdir"));
var success = dotnet.New(tempDir.DirInfo.FullName);
var success = TryRestorePackageManually(package, nugetConfig);
if (!success)
{
return;
}
success = dotnet.AddPackage(tempDir.DirInfo.FullName, package);
if (!success)
{
return;
}
var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig));
if (!res.Success)
{
if (res.HasNugetPackageSourceError)
{
// Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true));
}
// TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package.
if (!res.Success)
{
logger.LogInfo($"Failed to restore nuget package {package}");
return;
}
}
lock (sync)
{
successCount++;
@@ -871,6 +855,43 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
dllPaths.Add(missingPackageDirectory.DirInfo.FullName);
}
private bool TryRestorePackageManually(string package, string? nugetConfig)
{
logger.LogInfo($"Restoring package {package}...");
using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package, "missingpackages_workingdir"));
var success = dotnet.New(tempDir.DirInfo.FullName);
if (!success)
{
return false;
}
success = dotnet.AddPackage(tempDir.DirInfo.FullName, package);
if (!success)
{
return false;
}
var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig));
if (!res.Success)
{
if (res.HasNugetPackageSourceError && nugetConfig is not null)
{
// Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true));
}
// TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package.
if (!res.Success)
{
logger.LogInfo($"Failed to restore nuget package {package}");
return false;
}
}
return true;
}
public void Dispose(TemporaryDirectory? dir, string name)
{
try

12
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FrameworkPackageNames.cs
View File

@@ -1,25 +1,25 @@
using System.Collections.Generic;
using System.Linq;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal static class FrameworkPackageNames
{
public static string LatestNetFrameworkReferenceAssemblies { get; } = "microsoft.netframework.referenceassemblies.net481";
public static string AspNetCoreFramework { get; } = "microsoft.aspnetcore.app.ref";
public static string WindowsDesktopFramework { get; } = "microsoft.windowsdesktop.app.ref";
// The order of the packages is important.
public static string[] NetFrameworks { get; } = new string[]
{
public static string[] NetFrameworks { get; } =
[
"microsoft.netcore.app.ref", // net7.0, ... net5.0, netcoreapp3.1, netcoreapp3.0
"microsoft.netframework.referenceassemblies.", // net48, ..., net20
"netstandard.library.ref", // netstandard2.1
"netstandard.library" // netstandard2.0
};
];
public static IEnumerable<string> AllFrameworks { get; } =
NetFrameworks
.Union(new string[] { AspNetCoreFramework, WindowsDesktopFramework });
[.. NetFrameworks, AspNetCoreFramework, WindowsDesktopFramework];
}
}

44
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Runtime.cs
View File

@@ -5,6 +5,7 @@ using System.Linq;
using System.Runtime.InteropServices;
using System.Text.RegularExpressions;
using Semmle.Util;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
@@ -17,12 +18,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private const string aspNetCoreApp = "Microsoft.AspNetCore.App";
private readonly IDotNet dotNet;
private readonly ILogger logger;
private readonly Lazy<Dictionary<string, DotNetVersion>> newestRuntimes;
private Dictionary<string, DotNetVersion> NewestRuntimes => newestRuntimes.Value;
public Runtime(IDotNet dotNet)
public Runtime(IDotNet dotNet, ILogger logger)
{
this.dotNet = dotNet;
this.logger = logger;
this.newestRuntimes = new(GetNewestRuntimes);
}
@@ -65,7 +68,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// Locates .NET Desktop Runtimes.
/// This includes Mono and Microsoft.NET.
/// </summary>
private static IEnumerable<string> DesktopRuntimes
private IEnumerable<string> DesktopRuntimes
{
get
{
@@ -75,21 +78,38 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
.OrderByDescending(Path.GetFileName);
}
var monoPath = FileUtils.FindProgramOnPath(Win32.IsWindows() ? "mono.exe" : "mono");
var monoDirs = monoPath is not null
? new[] { Path.GetFullPath(Path.Combine(monoPath, "..", "lib", "mono")), monoPath }
: new[] { "/usr/lib/mono", "/usr/local/mono", "/usr/local/bin/mono", @"C:\Program Files\Mono\lib\mono" };
string? monoDir = null;
var dir = monoDirs.FirstOrDefault(Directory.Exists);
if (dir is not null)
var monoPathEnv = Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_BUILDLESS_MONO_PATH");
if (monoPathEnv is not null)
{
return Directory.EnumerateDirectories(dir)
.Where(d => Char.IsDigit(Path.GetFileName(d)[0]))
if (Directory.Exists(monoPathEnv))
{
monoDir = monoPathEnv;
}
else
{
logger.LogError($"The directory specified in CODEQL_EXTRACTOR_CSHARP_BUILDLESS_MONO_PATH does not exist: {monoPathEnv}");
}
}
else
{
var monoPath = FileUtils.FindProgramOnPath(Win32.IsWindows() ? "mono.exe" : "mono");
string[] monoDirs = monoPath is not null
? [Path.GetFullPath(Path.Combine(monoPath, "..", "lib", "mono")), monoPath]
: ["/usr/lib/mono", "/usr/local/mono", "/usr/local/bin/mono", @"C:\Program Files\Mono\lib\mono"];
monoDir = monoDirs.FirstOrDefault(Directory.Exists);
}
if (monoDir is not null)
{
return Directory.EnumerateDirectories(monoDir)
.Where(d => char.IsDigit(Path.GetFileName(d)[0]))
.OrderByDescending(Path.GetFileName);
}
return Enumerable.Empty<string>();
return [];
}
}

8
csharp/extractor/Semmle.Extraction.Tests/Runtime.cs
View File

@@ -47,7 +47,7 @@ namespace Semmle.Extraction.Tests
"Microsoft.NETCore.App 7.0.2 [/path/dotnet/shared/Microsoft.NETCore.App]"
};
var dotnet = new DotNetStub(listedRuntimes, null!);
var runtime = new Runtime(dotnet);
var runtime = new Runtime(dotnet, new LoggerStub());
// Execute
var runtimes = runtime.GetNewestRuntimes();
@@ -73,7 +73,7 @@ namespace Semmle.Extraction.Tests
"Microsoft.NETCore.App 8.0.0-preview.5.23280.8 [/path/dotnet/shared/Microsoft.NETCore.App]"
};
var dotnet = new DotNetStub(listedRuntimes, null!);
var runtime = new Runtime(dotnet);
var runtime = new Runtime(dotnet, new LoggerStub());
// Execute
var runtimes = runtime.GetNewestRuntimes();
@@ -96,7 +96,7 @@ namespace Semmle.Extraction.Tests
"Microsoft.NETCore.App 8.0.0-preview.5.23280.8 [/path/dotnet/shared/Microsoft.NETCore.App]"
};
var dotnet = new DotNetStub(listedRuntimes, null!);
var runtime = new Runtime(dotnet);
var runtime = new Runtime(dotnet, new LoggerStub());
// Execute
var runtimes = runtime.GetNewestRuntimes();
@@ -125,7 +125,7 @@ namespace Semmle.Extraction.Tests
@"Microsoft.WindowsDesktop.App 7.0.4 [C:\Program Files\dotnet\shared\Microsoft.WindowsDesktop.App]"
};
var dotnet = new DotNetStub(listedRuntimes, null!);
var runtime = new Runtime(dotnet);
var runtime = new Runtime(dotnet, new LoggerStub());
// Execute
var runtimes = runtime.GetNewestRuntimes();