From f8b29ad70e05fbd731415e8153a4a149b2e53db2 Mon Sep 17 00:00:00 2001 From: Tamas Vajk Date: Fri, 16 Feb 2024 11:54:19 +0100 Subject: [PATCH] Introduce environment variable to specify framework assembly locations --- .../AutobuildOptions.cs | 2 +- .../DependencyManager.cs | 43 +++++++++++++++++++ .../EnvironmentVariableNames.cs | 14 +++++- .../FilePathFilter.cs | 2 +- .../Razor.cs | 2 +- .../Runtime.cs | 28 +++--------- .../Extractor/Extractor.cs | 2 +- .../extractor/Semmle.Extraction/CsProjFile.cs | 2 +- csharp/extractor/Semmle.Util/FileUtils.cs | 2 + .../test.py | 2 +- 10 files changed, 69 insertions(+), 30 deletions(-) diff --git a/csharp/autobuilder/Semmle.Autobuild.Shared/AutobuildOptions.cs b/csharp/autobuilder/Semmle.Autobuild.Shared/AutobuildOptions.cs index 8a02370bcf6..b7e3293a0f2 100644 --- a/csharp/autobuilder/Semmle.Autobuild.Shared/AutobuildOptions.cs +++ b/csharp/autobuilder/Semmle.Autobuild.Shared/AutobuildOptions.cs @@ -75,7 +75,7 @@ namespace Semmle.Autobuild.Shared return defaultValue; return value. - Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries). + Split(FileUtils.NewLineCharacters, StringSplitOptions.RemoveEmptyEntries). Select(s => AsStringWithExpandedEnvVars(s, actions)).ToArray(); } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs index 981f6aaa2ae..c02018f787a 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs @@ -158,6 +158,49 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { var frameworkLocations = new HashSet(); + var frameworkReferences = Environment.GetEnvironmentVariable(EnvironmentVariableNames.DotnetFrameworkReferences); + var frameworkReferencesUseSubfolders = Environment.GetEnvironmentVariable(EnvironmentVariableNames.DotnetFrameworkReferencesUseSubfolders); + _ = bool.TryParse(frameworkReferencesUseSubfolders, out var useSubfolders); + if (!string.IsNullOrWhiteSpace(frameworkReferences)) + { + var frameworkPaths = frameworkReferences.Split(FileUtils.NewLineCharacters, StringSplitOptions.RemoveEmptyEntries); + + foreach (var path in frameworkPaths) + { + if (!Directory.Exists(path)) + { + logger.LogError($"Specified framework reference path '{path}' does not exist."); + continue; + } + + if (useSubfolders) + { + dllPaths.Add(path); + frameworkLocations.Add(path); + continue; + } + + try + { + var dlls = Directory.GetFiles(path, "*.dll", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive }); + if (dlls.Length == 0) + { + logger.LogError($"No DLLs found in specified framework reference path '{path}'."); + continue; + } + + dllPaths.UnionWith(dlls); + frameworkLocations.UnionWith(dlls); + } + catch (Exception e) + { + logger.LogError($"Error while searching for DLLs in '{path}': {e.Message}"); + } + } + + return frameworkLocations; + } + AddNetFrameworkDlls(dllPaths, frameworkLocations); AddAspNetCoreFrameworkDlls(dllPaths, frameworkLocations); AddMicrosoftWindowsDesktopDlls(dllPaths, frameworkLocations); diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs index b9e8fdabea4..65a4664e83e 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs @@ -2,7 +2,19 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { internal class EnvironmentVariableNames { + /// + /// Controls whether to generate source files from Asp.Net Core views (`.cshtml`, `.razor`). + /// public const string WebViewGeneration = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_EXTRACT_WEB_VIEWS"; - public const string MonoPath = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_MONO_PATH"; + + /// + /// Specifies the location of .Net framework references added to the compilation. + /// + public const string DotnetFrameworkReferences = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_DOTNET_FRAMEWORK_REFERENCES"; + + /// + /// Controls whether to use framework dependencies from subfolders. + /// + public const string DotnetFrameworkReferencesUseSubfolders = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_DOTNET_FRAMEWORK_REFERENCES_USE_SUBFOLDERS"; } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FilePathFilter.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FilePathFilter.cs index 24d92e0f068..d1a3ed011d4 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FilePathFilter.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FilePathFilter.cs @@ -31,7 +31,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching public IEnumerable Filter(IEnumerable files) { - var filters = (Environment.GetEnvironmentVariable("LGTM_INDEX_FILTERS") ?? string.Empty).Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); + var filters = (Environment.GetEnvironmentVariable("LGTM_INDEX_FILTERS") ?? string.Empty).Split(FileUtils.NewLineCharacters, StringSplitOptions.RemoveEmptyEntries); if (filters.Length == 0) { return files; diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Razor.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Razor.cs index 598e3e7e1f7..e46d67685db 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Razor.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Razor.cs @@ -71,7 +71,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching var args = new StringBuilder(); args.Append($"/target:exe /generatedfilesout:\"{outputFolder}\" /out:\"{dllPath}\" /analyzerconfig:\"{analyzerConfig}\" "); - foreach (var f in Directory.GetFiles(sourceGeneratorFolder, "*.dll")) + foreach (var f in Directory.GetFiles(sourceGeneratorFolder, "*.dll", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive })) { args.Append($"/analyzer:\"{f}\" "); } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Runtime.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Runtime.cs index a3838c62590..90c4af2c4d3 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Runtime.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Runtime.cs @@ -78,30 +78,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching .OrderByDescending(Path.GetFileName); } - string? monoDir = null; - - var monoPathEnv = Environment.GetEnvironmentVariable(EnvironmentVariableNames.MonoPath); - if (monoPathEnv is not null) - { - if (Directory.Exists(monoPathEnv)) - { - monoDir = monoPathEnv; - } - else - { - logger.LogError($"The directory specified in {EnvironmentVariableNames.MonoPath} does not exist: {monoPathEnv}"); - } - } - else - { - var monoPath = FileUtils.FindProgramOnPath(Win32.IsWindows() ? "mono.exe" : "mono"); - string[] monoDirs = monoPath is not null - ? [Path.GetFullPath(Path.Combine(monoPath, "..", "lib", "mono")), monoPath] - : ["/usr/lib/mono", "/usr/local/mono", "/usr/local/bin/mono", @"C:\Program Files\Mono\lib\mono"]; - - monoDir = monoDirs.FirstOrDefault(Directory.Exists); - } + var monoPath = FileUtils.FindProgramOnPath(Win32.IsWindows() ? "mono.exe" : "mono"); + string[] monoDirs = monoPath is not null + ? [Path.GetFullPath(Path.Combine(monoPath, "..", "lib", "mono")), monoPath] + : ["/usr/lib/mono", "/usr/local/mono", "/usr/local/bin/mono", @"C:\Program Files\Mono\lib\mono"]; + var monoDir = monoDirs.FirstOrDefault(Directory.Exists); if (monoDir is not null) { return Directory.EnumerateDirectories(monoDir) diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs index 5d5bc5860f4..86677f68620 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs @@ -386,7 +386,7 @@ namespace Semmle.Extraction.CSharp if (compilerArguments.GeneratedFilesOutputDirectory is not null) { - paths.AddRange(Directory.GetFiles(compilerArguments.GeneratedFilesOutputDirectory, "*.cs", SearchOption.AllDirectories)); + paths.AddRange(Directory.GetFiles(compilerArguments.GeneratedFilesOutputDirectory, "*.cs", new EnumerationOptions { RecurseSubdirectories = true, MatchCasing = MatchCasing.CaseInsensitive })); } return ReadSyntaxTrees( diff --git a/csharp/extractor/Semmle.Extraction/CsProjFile.cs b/csharp/extractor/Semmle.Extraction/CsProjFile.cs index 10deb2883a3..bed9d746996 100644 --- a/csharp/extractor/Semmle.Extraction/CsProjFile.cs +++ b/csharp/extractor/Semmle.Extraction/CsProjFile.cs @@ -112,7 +112,7 @@ namespace Semmle.Extraction .Where(s => s is not null) ?? Enumerable.Empty(); - var additionalCsFiles = System.IO.Directory.GetFiles(directoryName, "*.cs", SearchOption.AllDirectories); + var additionalCsFiles = System.IO.Directory.GetFiles(directoryName, "*.cs", new EnumerationOptions { RecurseSubdirectories = true, MatchCasing = MatchCasing.CaseInsensitive }); var projectReferences = root .SelectNodes("/Project/ItemGroup/ProjectReference/@Include", mgr) diff --git a/csharp/extractor/Semmle.Util/FileUtils.cs b/csharp/extractor/Semmle.Util/FileUtils.cs index 3315c3e705e..09269e37e8b 100644 --- a/csharp/extractor/Semmle.Util/FileUtils.cs +++ b/csharp/extractor/Semmle.Util/FileUtils.cs @@ -13,6 +13,8 @@ namespace Semmle.Util { public const string NugetExeUrl = "https://dist.nuget.org/win-x86-commandline/latest/nuget.exe"; + public static readonly char[] NewLineCharacters = ['\r', '\n']; + public static string ConvertToWindows(string path) { return path.Replace('/', '\\'); diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_no_mono/test.py b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_no_mono/test.py index 1c8150e5d7b..ddd46a7f1e2 100644 --- a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_no_mono/test.py +++ b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_no_mono/test.py @@ -1,5 +1,5 @@ from create_database_utils import * import os -os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_MONO_PATH"] = "/non-existent-path" +os.environ["CODEQL_EXTRACTOR_CSHARP_BUILDLESS_DOTNET_FRAMEWORK_REFERENCES"] = "/non-existent-path" run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=buildless=true"])