diff --git a/cpp/autobuilder/Semmle.Autobuild.Cpp/Program.cs b/cpp/autobuilder/Semmle.Autobuild.Cpp/Program.cs index a7556197bcd..5fca6f556fb 100644 --- a/cpp/autobuilder/Semmle.Autobuild.Cpp/Program.cs +++ b/cpp/autobuilder/Semmle.Autobuild.Cpp/Program.cs @@ -1,5 +1,7 @@ using System; + using Semmle.Autobuild.Shared; +using Semmle.Util; namespace Semmle.Autobuild.Cpp { diff --git a/csharp/CSharp.sln b/csharp/CSharp.sln index 3c9f3ef4e4f..0ba39b813bb 100644 --- a/csharp/CSharp.sln +++ b/csharp/CSharp.sln @@ -34,6 +34,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Semmle.Autobuild.CSharp.Tes EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Semmle.Extraction.CSharp.DependencyStubGenerator", "extractor\Semmle.Extraction.CSharp.DependencyStubGenerator\Semmle.Extraction.CSharp.DependencyStubGenerator.csproj", "{0EDA21A3-ADD8-4C10-B494-58B12B526B76}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Semmle.Autobuild.Cpp", "..\cpp\autobuilder\Semmle.Autobuild.Cpp\Semmle.Autobuild.Cpp.csproj", "{125C4FB7-34DA-442A-9095-3EA1514270CD}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Semmle.Autobuild.Cpp.Tests", "..\cpp\autobuilder\Semmle.Autobuild.Cpp.Tests\Semmle.Autobuild.Cpp.Tests.csproj", "{72F369B7-0707-401A-802F-D526F272F9EE}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -102,6 +106,14 @@ Global {0EDA21A3-ADD8-4C10-B494-58B12B526B76}.Debug|Any CPU.Build.0 = Debug|Any CPU {0EDA21A3-ADD8-4C10-B494-58B12B526B76}.Release|Any CPU.ActiveCfg = Release|Any CPU {0EDA21A3-ADD8-4C10-B494-58B12B526B76}.Release|Any CPU.Build.0 = Release|Any CPU + {125C4FB7-34DA-442A-9095-3EA1514270CD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {125C4FB7-34DA-442A-9095-3EA1514270CD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {125C4FB7-34DA-442A-9095-3EA1514270CD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {125C4FB7-34DA-442A-9095-3EA1514270CD}.Release|Any CPU.Build.0 = Release|Any CPU + {72F369B7-0707-401A-802F-D526F272F9EE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {72F369B7-0707-401A-802F-D526F272F9EE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {72F369B7-0707-401A-802F-D526F272F9EE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {72F369B7-0707-401A-802F-D526F272F9EE}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs b/csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs index 9f5a4b74c2e..95fcb51072e 100644 --- a/csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs +++ b/csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs @@ -558,8 +558,6 @@ namespace Semmle.Autobuild.CSharp.Tests [Fact] public void TestLinuxBuildlessExtractionSuccess() { - actions.RunProcess["dotnet --list-sdks"] = 0; - actions.RunProcessOut["dotnet --list-sdks"] = "any version"; actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 0; actions.FileExists["csharp.log"] = true; actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = ""; @@ -569,14 +567,12 @@ namespace Semmle.Autobuild.CSharp.Tests actions.EnumerateDirectories[@"C:\Project"] = ""; var autobuilder = CreateAutoBuilder(false, buildless: "true"); - TestAutobuilderScript(autobuilder, 0, 2); + TestAutobuilderScript(autobuilder, 0, 1); } [Fact] public void TestLinuxBuildlessExtractionFailed() { - actions.RunProcess["dotnet --list-sdks"] = 0; - actions.RunProcessOut["dotnet --list-sdks"] = "any version"; actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 10; actions.FileExists["csharp.log"] = true; actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = ""; @@ -586,14 +582,12 @@ namespace Semmle.Autobuild.CSharp.Tests actions.EnumerateDirectories[@"C:\Project"] = ""; var autobuilder = CreateAutoBuilder(false, buildless: "true"); - TestAutobuilderScript(autobuilder, 10, 2); + TestAutobuilderScript(autobuilder, 10, 1); } [Fact] public void TestLinuxBuildlessExtractionSolution() { - actions.RunProcess["dotnet --list-sdks"] = 0; - actions.RunProcessOut["dotnet --list-sdks"] = "any version"; actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 0; actions.FileExists["csharp.log"] = true; actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = ""; @@ -603,28 +597,7 @@ namespace Semmle.Autobuild.CSharp.Tests actions.EnumerateDirectories[@"C:\Project"] = ""; var autobuilder = CreateAutoBuilder(false, buildless: "true"); - TestAutobuilderScript(autobuilder, 0, 2); - } - - [Fact] - public void TestLinuxBuildlessExtractionNoDotnet() - { - actions.RunProcess["dotnet --list-sdks"] = 1; - actions.RunProcessOut["dotnet --list-sdks"] = ""; - actions.RunProcess[@"chmod u+x scratch/.dotnet/dotnet-install.sh"] = 0; - actions.RunProcess[@"scratch/.dotnet/dotnet-install.sh --channel release --version 8.0.101 --install-dir scratch/.dotnet"] = 0; - actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone --dotnet scratch/.dotnet"] = 0; - actions.FileExists["csharp.log"] = true; - actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = ""; - actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = ""; - actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch"; - actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln"; - actions.EnumerateDirectories[@"C:\Project"] = ""; - actions.DownloadFiles.Add(("https://dot.net/v1/dotnet-install.sh", "scratch/.dotnet/dotnet-install.sh")); - actions.CreateDirectories.Add(@"scratch/.dotnet"); - - var autobuilder = CreateAutoBuilder(false, buildless: "true"); - TestAutobuilderScript(autobuilder, 0, 4); + TestAutobuilderScript(autobuilder, 0, 1); } private void SkipVsWhere() @@ -915,8 +888,6 @@ namespace Semmle.Autobuild.CSharp.Tests [Fact] public void TestSkipNugetBuildless() { - actions.RunProcess["dotnet --list-sdks"] = 0; - actions.RunProcessOut["dotnet --list-sdks"] = "any version"; actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 0; actions.FileExists["csharp.log"] = true; actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = ""; @@ -926,7 +897,7 @@ namespace Semmle.Autobuild.CSharp.Tests actions.EnumerateDirectories[@"C:\Project"] = ""; var autobuilder = CreateAutoBuilder(false, buildless: "true"); - TestAutobuilderScript(autobuilder, 0, 2); + TestAutobuilderScript(autobuilder, 0, 1); } diff --git a/csharp/autobuilder/Semmle.Autobuild.CSharp/CSharpAutobuilder.cs b/csharp/autobuilder/Semmle.Autobuild.CSharp/CSharpAutobuilder.cs index 859029eee32..aad91541064 100644 --- a/csharp/autobuilder/Semmle.Autobuild.CSharp/CSharpAutobuilder.cs +++ b/csharp/autobuilder/Semmle.Autobuild.CSharp/CSharpAutobuilder.cs @@ -50,11 +50,8 @@ namespace Semmle.Autobuild.CSharp attempt = new BuildCommandRule(DotNetRule.WithDotNet).Analyse(this, false) & CheckExtractorRun(true); break; case CSharpBuildStrategy.Buildless: - attempt = DotNetRule.WithDotNet(this, ensureDotNetAvailable: true, (dotNetPath, env) => - { - // No need to check that the extractor has been executed in buildless mode - return new StandaloneBuildRule(dotNetPath).Analyse(this, false); - }); + // No need to check that the extractor has been executed in buildless mode + attempt = new StandaloneBuildRule().Analyse(this, false); break; case CSharpBuildStrategy.MSBuild: attempt = new MsBuildRule().Analyse(this, false) & CheckExtractorRun(true); @@ -84,7 +81,7 @@ namespace Semmle.Autobuild.CSharp return 0; if (warnOnFailure) - Log(Severity.Error, "No C# code detected during build."); + Logger.LogError("No C# code detected during build."); return 1; }); diff --git a/csharp/autobuilder/Semmle.Autobuild.CSharp/Constants.cs b/csharp/autobuilder/Semmle.Autobuild.CSharp/Constants.cs deleted file mode 100644 index 6e0f0ded7a4..00000000000 --- a/csharp/autobuilder/Semmle.Autobuild.CSharp/Constants.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Semmle.Autobuild.CSharp -{ - internal static class Constants - { - // The version number should be kept in sync with the version .NET version used for building the application. - public const string LatestDotNetSdkVersion = "8.0.101"; - } -} diff --git a/csharp/autobuilder/Semmle.Autobuild.CSharp/DotNetRule.cs b/csharp/autobuilder/Semmle.Autobuild.CSharp/DotNetRule.cs index f5e519b1f90..cd25055da1a 100644 --- a/csharp/autobuilder/Semmle.Autobuild.CSharp/DotNetRule.cs +++ b/csharp/autobuilder/Semmle.Autobuild.CSharp/DotNetRule.cs @@ -5,7 +5,7 @@ using System.Linq; using Semmle.Util; using Semmle.Util.Logging; using Semmle.Autobuild.Shared; -using Newtonsoft.Json.Linq; +using Semmle.Extraction.CSharp.DependencyFetching; namespace Semmle.Autobuild.CSharp { @@ -39,11 +39,11 @@ namespace Semmle.Autobuild.CSharp if (notDotNetProject is not null) { - builder.Log(Severity.Info, "Not using .NET Core because of incompatible project {0}", notDotNetProject); + builder.Logger.Log(Severity.Info, "Not using .NET Core because of incompatible project {0}", notDotNetProject); return BuildScript.Failure; } - builder.Log(Severity.Info, "Attempting to build using .NET Core"); + builder.Logger.LogInfo("Attempting to build using .NET Core"); } return WithDotNet(builder, ensureDotNetAvailable: false, (dotNetPath, environment) => @@ -81,29 +81,22 @@ namespace Semmle.Autobuild.CSharp /// public static BuildScript WithDotNet(IAutobuilder builder, bool ensureDotNetAvailable, Func?, BuildScript> f) { - var installDir = builder.Actions.PathCombine(FileUtils.GetTemporaryWorkingDirectory(builder.Actions.GetEnvironmentVariable, builder.Options.Language.UpperCaseName, out var _), ".dotnet"); - var installScript = DownloadDotNet(builder, installDir, ensureDotNetAvailable); - return BuildScript.Bind(installScript, installed => + var temp = FileUtils.GetTemporaryWorkingDirectory(builder.Actions.GetEnvironmentVariable, builder.Options.Language.UpperCaseName, out var shouldCleanUp); + return DotNet.WithDotNet(builder.Actions, builder.Logger, builder.Paths.Select(x => x.Item1), temp, shouldCleanUp, ensureDotNetAvailable, builder.Options.DotNetVersion, installDir => { var env = new Dictionary { { "DOTNET_SKIP_FIRST_TIME_EXPERIENCE", "true" }, { "MSBUILDDISABLENODEREUSE", "1" } }; - if (installed == 0) + if (installDir is not null) { - // The installation succeeded, so use the newly installed .NET Core + // The installation succeeded, so use the newly installed .NET var path = builder.Actions.GetEnvironmentVariable("PATH"); var delim = builder.Actions.IsWindows() ? ";" : ":"; - env.Add("DOTNET_MULTILEVEL_LOOKUP", "false"); // prevent look up of other .NET Core SDKs + env.Add("DOTNET_MULTILEVEL_LOOKUP", "false"); // prevent look up of other .NET SDKs env.Add("PATH", installDir + delim + path); } - else - { - // The .NET SDK was not installed, either because the installation failed or because it was already installed. - installDir = null; - } - return f(installDir, env); }); } @@ -119,146 +112,6 @@ namespace Semmle.Autobuild.CSharp public static BuildScript WithDotNet(IAutobuilder builder, Func?, BuildScript> f) => WithDotNet(builder, ensureDotNetAvailable: false, (_, env) => f(env)); - /// - /// Returns a script for downloading relevant versions of the - /// .NET Core SDK. The SDK(s) will be installed at installDir - /// (provided that the script succeeds). - /// - private static BuildScript DownloadDotNet(IAutobuilder builder, string installDir, bool ensureDotNetAvailable) - { - if (!string.IsNullOrEmpty(builder.Options.DotNetVersion)) - // Specific version supplied in configuration: always use that - return DownloadDotNetVersion(builder, installDir, builder.Options.DotNetVersion); - - // Download versions mentioned in `global.json` files - // See https://docs.microsoft.com/en-us/dotnet/core/tools/global-json - var installScript = BuildScript.Success; - var validGlobalJson = false; - foreach (var path in builder.Paths.Select(p => p.Item1).Where(p => p.EndsWith("global.json", StringComparison.Ordinal))) - { - string version; - try - { - var o = JObject.Parse(File.ReadAllText(path)); - version = (string)(o?["sdk"]?["version"]!); - } - catch // lgtm[cs/catch-of-all-exceptions] - { - // not a valid global.json file - continue; - } - - installScript &= DownloadDotNetVersion(builder, installDir, version); - validGlobalJson = true; - } - - if (validGlobalJson) - { - return installScript; - } - - if (ensureDotNetAvailable) - { - return DownloadDotNetVersion(builder, installDir, Constants.LatestDotNetSdkVersion, needExactVersion: false); - } - - return BuildScript.Failure; - } - - /// - /// Returns a script for downloading a specific .NET Core SDK version, if the - /// version is not already installed. - /// - /// See https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet-install-script. - /// - private static BuildScript DownloadDotNetVersion(IAutobuilder builder, string path, string version, bool needExactVersion = true) - { - return BuildScript.Bind(GetInstalledSdksScript(builder.Actions), (sdks, sdksRet) => - { - if (needExactVersion && sdksRet == 0 && sdks.Count == 1 && sdks[0].StartsWith(version + " ", StringComparison.Ordinal)) - { - // The requested SDK is already installed (and no other SDKs are installed), so - // no need to reinstall - return BuildScript.Failure; - } - else if (!needExactVersion && sdksRet == 0 && sdks.Count > 0) - { - // there's at least one SDK installed, so no need to reinstall - return BuildScript.Failure; - } - else if (!needExactVersion && sdksRet != 0) - { - builder.Log(Severity.Info, "No .NET Core SDK found."); - } - - builder.Log(Severity.Info, "Attempting to download .NET Core {0}", version); - - if (builder.Actions.IsWindows()) - { - - var psCommand = $"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; &([scriptblock]::Create((Invoke-WebRequest -UseBasicParsing 'https://dot.net/v1/dotnet-install.ps1'))) -Version {version} -InstallDir {path}"; - - BuildScript GetInstall(string pwsh) => - new CommandBuilder(builder.Actions). - RunCommand(pwsh). - Argument("-NoProfile"). - Argument("-ExecutionPolicy"). - Argument("unrestricted"). - Argument("-Command"). - Argument("\"" + psCommand + "\""). - Script; - - return GetInstall("pwsh") | GetInstall("powershell"); - } - else - { - var dotnetInstallPath = builder.Actions.PathCombine(FileUtils.GetTemporaryWorkingDirectory( - builder.Actions.GetEnvironmentVariable, - builder.Options.Language.UpperCaseName, - out var shouldCleanUp), ".dotnet", "dotnet-install.sh"); - - var downloadDotNetInstallSh = BuildScript.DownloadFile( - "https://dot.net/v1/dotnet-install.sh", - dotnetInstallPath, - e => builder.Log(Severity.Warning, $"Failed to download 'dotnet-install.sh': {e.Message}")); - - var chmod = new CommandBuilder(builder.Actions). - RunCommand("chmod"). - Argument("u+x"). - Argument(dotnetInstallPath); - - var install = new CommandBuilder(builder.Actions). - RunCommand(dotnetInstallPath). - Argument("--channel"). - Argument("release"). - Argument("--version"). - Argument(version). - Argument("--install-dir"). - Argument(path); - - var buildScript = downloadDotNetInstallSh & chmod.Script & install.Script; - - if (shouldCleanUp) - { - var removeScript = new CommandBuilder(builder.Actions). - RunCommand("rm"). - Argument(dotnetInstallPath); - buildScript &= removeScript.Script; - } - - return buildScript; - } - }); - } - - private static BuildScript GetInstalledSdksScript(IBuildActions actions) - { - var listSdks = new CommandBuilder(actions, silent: true). - RunCommand("dotnet"). - Argument("--list-sdks"); - return listSdks.Script; - } - private static string DotNetCommand(IBuildActions actions, string? dotNetPath) => dotNetPath is not null ? actions.PathCombine(dotNetPath, "dotnet") : "dotnet"; diff --git a/csharp/autobuilder/Semmle.Autobuild.CSharp/Semmle.Autobuild.CSharp.csproj b/csharp/autobuilder/Semmle.Autobuild.CSharp/Semmle.Autobuild.CSharp.csproj index dbbe8c80471..911e71bfdc9 100644 --- a/csharp/autobuilder/Semmle.Autobuild.CSharp/Semmle.Autobuild.CSharp.csproj +++ b/csharp/autobuilder/Semmle.Autobuild.CSharp/Semmle.Autobuild.CSharp.csproj @@ -20,6 +20,7 @@ + diff --git a/csharp/autobuilder/Semmle.Autobuild.CSharp/StandaloneBuildRule.cs b/csharp/autobuilder/Semmle.Autobuild.CSharp/StandaloneBuildRule.cs index fc1bd47b721..c202a029674 100644 --- a/csharp/autobuilder/Semmle.Autobuild.CSharp/StandaloneBuildRule.cs +++ b/csharp/autobuilder/Semmle.Autobuild.CSharp/StandaloneBuildRule.cs @@ -8,13 +8,6 @@ namespace Semmle.Autobuild.CSharp /// internal class StandaloneBuildRule : IBuildRule { - private readonly string? dotNetPath; - - internal StandaloneBuildRule(string? dotNetPath) - { - this.dotNetPath = dotNetPath; - } - public BuildScript Analyse(IAutobuilder builder, bool auto) { if (builder.CodeQLExtractorLangRoot is null @@ -27,12 +20,6 @@ namespace Semmle.Autobuild.CSharp var cmd = new CommandBuilder(builder.Actions); cmd.RunCommand(standalone); - if (!string.IsNullOrEmpty(this.dotNetPath)) - { - cmd.Argument("--dotnet"); - cmd.QuoteArgument(this.dotNetPath); - } - return cmd.Script; } } diff --git a/csharp/autobuilder/Semmle.Autobuild.Shared/Autobuilder.cs b/csharp/autobuilder/Semmle.Autobuild.Shared/Autobuilder.cs index d6757418875..a23d29d2979 100644 --- a/csharp/autobuilder/Semmle.Autobuild.Shared/Autobuilder.cs +++ b/csharp/autobuilder/Semmle.Autobuild.Shared/Autobuilder.cs @@ -70,11 +70,9 @@ namespace Semmle.Autobuild.Shared IBuildActions Actions { get; } /// - /// Log a given build event to the console. + /// A logger. /// - /// The format string. - /// Inserts to the format string. - void Log(Severity severity, string format, params object[] args); + ILogger Logger { get; } /// /// Value of CODEQL_EXTRACTOR__ROOT environment variable. @@ -135,21 +133,7 @@ namespace Semmle.Autobuild.Shared /// True iff the path was found. public bool HasPath(string path) => Paths.Any(p => path == p.Item1); - private void FindFiles(string dir, int depth, int maxDepth, IList<(string, int)> results) - { - foreach (var f in Actions.EnumerateFiles(dir)) - { - results.Add((f, depth)); - } - if (depth < maxDepth) - { - foreach (var d in Actions.EnumerateDirectories(dir)) - { - FindFiles(d, depth + 1, maxDepth, results); - } - } - } /// /// The root of the source directory. @@ -196,12 +180,7 @@ namespace Semmle.Autobuild.Shared Options = options; DiagnosticClassifier = diagnosticClassifier; - pathsLazy = new Lazy>(() => - { - var files = new List<(string, int)>(); - FindFiles(options.RootDirectory, 0, options.SearchDepth, files); - return files.OrderBy(f => f.Item2).ToArray(); - }); + pathsLazy = new Lazy>(() => Actions.FindFiles(options.RootDirectory, options.SearchDepth)); projectsOrSolutionsToBuildLazy = new Lazy>(() => { @@ -214,7 +193,7 @@ namespace Semmle.Autobuild.Shared if (actions.FileExists(solution)) ret.Add(new Solution(this, solution, true)); else - Log(Severity.Error, $"The specified project or solution file {solution} was not found"); + logger.LogError($"The specified project or solution file {solution} was not found"); } return ret; } @@ -273,6 +252,8 @@ namespace Semmle.Autobuild.Shared logThreadId: false) ?? Verbosity.Info, logThreadId: false); + public ILogger Logger => logger; + private readonly IDiagnosticsWriter diagnostics; /// @@ -285,16 +266,6 @@ namespace Semmle.Autobuild.Shared return Path.GetRelativePath(this.RootDirectory, path); } - /// - /// Log a given build event to the console. - /// - /// The format string. - /// Inserts to the format string. - public void Log(Severity severity, string format, params object[] args) - { - logger.Log(severity, format, args); - } - /// /// Write to the diagnostics file. /// @@ -310,7 +281,7 @@ namespace Semmle.Autobuild.Shared /// The exit code, 0 for success and non-zero for failures. public int AttemptBuild() { - Log(Severity.Info, $"Working directory: {Options.RootDirectory}"); + logger.LogInfo($"Working directory: {Options.RootDirectory}"); var script = GetBuildScript(); @@ -319,12 +290,12 @@ namespace Semmle.Autobuild.Shared void startCallback(string s, bool silent) { - Log(silent ? Severity.Debug : Severity.Info, $"\nRunning {s}"); + logger.Log(silent ? Severity.Debug : Severity.Info, $"\nRunning {s}"); } void exitCallback(int ret, string msg, bool silent) { - Log(silent ? Severity.Debug : Severity.Info, $"Exit code {ret}{(string.IsNullOrEmpty(msg) ? "" : $": {msg}")}"); + logger.Log(silent ? Severity.Debug : Severity.Info, $"Exit code {ret}{(string.IsNullOrEmpty(msg) ? "" : $": {msg}")}"); } var onOutput = BuildOutputHandler(Console.Out); @@ -374,7 +345,7 @@ namespace Semmle.Autobuild.Shared protected BuildScript AutobuildFailure() => BuildScript.Create(actions => { - Log(Severity.Error, "Could not auto-detect a suitable build method"); + logger.LogError("Could not auto-detect a suitable build method"); AutobuildFailureDiagnostic(); diff --git a/csharp/autobuilder/Semmle.Autobuild.Shared/BuildCommandAutoRule.cs b/csharp/autobuilder/Semmle.Autobuild.Shared/BuildCommandAutoRule.cs index 3f099c7834a..d754b3c3134 100644 --- a/csharp/autobuilder/Semmle.Autobuild.Shared/BuildCommandAutoRule.cs +++ b/csharp/autobuilder/Semmle.Autobuild.Shared/BuildCommandAutoRule.cs @@ -55,7 +55,7 @@ namespace Semmle.Autobuild.Shared public BuildScript Analyse(IAutobuilder builder, bool auto) { - builder.Log(Severity.Info, "Attempting to locate build script"); + builder.Logger.LogInfo("Attempting to locate build script"); // a list of extensions for files that we consider to be scripts on the current platform var extensions = builder.Actions.IsWindows() ? winExtensions : linuxExtensions; diff --git a/csharp/autobuilder/Semmle.Autobuild.Shared/MsBuildRule.cs b/csharp/autobuilder/Semmle.Autobuild.Shared/MsBuildRule.cs index 3099449a1fb..bff10532abf 100644 --- a/csharp/autobuilder/Semmle.Autobuild.Shared/MsBuildRule.cs +++ b/csharp/autobuilder/Semmle.Autobuild.Shared/MsBuildRule.cs @@ -40,7 +40,7 @@ namespace Semmle.Autobuild.Shared return BuildScript.Failure; if (auto) - builder.Log(Severity.Info, "Attempting to build using MSBuild"); + builder.Logger.LogInfo("Attempting to build using MSBuild"); var vsTools = GetVcVarsBatFile(builder); @@ -54,7 +54,7 @@ namespace Semmle.Autobuild.Shared if (vsTools is null && builder.Actions.IsWindows()) { - builder.Log(Severity.Warning, "Could not find a suitable version of VsDevCmd.bat/vcvarsall.bat"); + builder.Logger.LogWarning("Could not find a suitable version of VsDevCmd.bat/vcvarsall.bat"); } // Use `nuget.exe` from source code repo, if present, otherwise first attempt with global @@ -165,18 +165,18 @@ namespace Semmle.Autobuild.Shared { foreach (var b in BuildTools.VcVarsAllBatFiles(builder.Actions)) { - builder.Log(Severity.Info, "Found {0} version {1}", b.Path, b.ToolsVersion); + builder.Logger.Log(Severity.Info, "Found {0} version {1}", b.Path, b.ToolsVersion); } vsTools = BuildTools.FindCompatibleVcVars(builder.Actions, msToolsVersion); if (vsTools is null) - builder.Log(Severity.Warning, "Could not find build tools matching version {0}", msToolsVersion); + builder.Logger.LogWarning("Could not find build tools matching version {0}", msToolsVersion); else - builder.Log(Severity.Info, "Setting Visual Studio tools to {0}", vsTools.Path); + builder.Logger.Log(Severity.Info, "Setting Visual Studio tools to {0}", vsTools.Path); } else { - builder.Log(Severity.Error, "The format of vstools_version is incorrect. Please specify an integer."); + builder.Logger.LogError("The format of vstools_version is incorrect. Please specify an integer."); } } @@ -189,18 +189,18 @@ namespace Semmle.Autobuild.Shared private static BuildScript DownloadNugetExe(IAutobuilder builder, string path) where TAutobuildOptions : AutobuildOptionsShared => BuildScript.Create(_ => { - builder.Log(Severity.Info, "Attempting to download nuget.exe"); + builder.Logger.LogInfo("Attempting to download nuget.exe"); return 0; }) & BuildScript.DownloadFile( FileUtils.NugetExeUrl, path, - e => builder.Log(Severity.Warning, $"Failed to download 'nuget.exe': {e.Message}")) + e => builder.Logger.LogWarning($"Failed to download 'nuget.exe': {e.Message}")) & BuildScript.Create(_ => { - builder.Log(Severity.Info, $"Successfully downloaded {path}"); + builder.Logger.LogInfo($"Successfully downloaded {path}"); return 0; }); } diff --git a/csharp/autobuilder/Semmle.Autobuild.Shared/Project.cs b/csharp/autobuilder/Semmle.Autobuild.Shared/Project.cs index 71522859871..5979b11e8fc 100644 --- a/csharp/autobuilder/Semmle.Autobuild.Shared/Project.cs +++ b/csharp/autobuilder/Semmle.Autobuild.Shared/Project.cs @@ -41,7 +41,7 @@ namespace Semmle.Autobuild.Shared } catch (Exception ex) when (ex is XmlException || ex is FileNotFoundException) { - builder.Log(Severity.Info, $"Unable to read project file {path}."); + builder.Logger.LogInfo($"Unable to read project file {path}."); return; } @@ -66,7 +66,7 @@ namespace Semmle.Autobuild.Shared catch // lgtm[cs/catch-of-all-exceptions] // Generic catch clause - Version constructor throws about 5 different exceptions. { - builder.Log(Severity.Warning, "Project {0} has invalid tools version {1}", path, toolsVersion); + builder.Logger.Log(Severity.Warning, "Project {0} has invalid tools version {1}", path, toolsVersion); } } diff --git a/csharp/autobuilder/Semmle.Autobuild.Shared/Solution.cs b/csharp/autobuilder/Semmle.Autobuild.Shared/Solution.cs index d48cfa15913..7a98a01de25 100644 --- a/csharp/autobuilder/Semmle.Autobuild.Shared/Solution.cs +++ b/csharp/autobuilder/Semmle.Autobuild.Shared/Solution.cs @@ -73,7 +73,7 @@ namespace Semmle.Autobuild.Shared return; } - builder.Log(Severity.Info, $"Unable to read solution file {path}."); + builder.Logger.LogInfo($"Unable to read solution file {path}."); includedProjects = Array.Empty>(); return; } diff --git a/csharp/extractor/Semmle.Extraction.CIL.Driver/Program.cs b/csharp/extractor/Semmle.Extraction.CIL.Driver/Program.cs index 8fcc6fefcb3..2009fe1c94f 100644 --- a/csharp/extractor/Semmle.Extraction.CIL.Driver/Program.cs +++ b/csharp/extractor/Semmle.Extraction.CIL.Driver/Program.cs @@ -46,7 +46,7 @@ namespace Semmle.Extraction.CIL.Driver .ToArray(); foreach (var missingRef in options.MissingReferences) - logger.Log(Severity.Info, " Missing assembly " + missingRef); + logger.LogInfo(" Missing assembly " + missingRef); var sw = new Stopwatch(); sw.Start(); diff --git a/csharp/extractor/Semmle.Extraction.CIL/Analyser.cs b/csharp/extractor/Semmle.Extraction.CIL/Analyser.cs index 4d7fa77c548..50288aadd94 100644 --- a/csharp/extractor/Semmle.Extraction.CIL/Analyser.cs +++ b/csharp/extractor/Semmle.Extraction.CIL/Analyser.cs @@ -45,7 +45,7 @@ namespace Semmle.Extraction.CIL } catch (Exception ex) // lgtm[cs/catch-of-all-exceptions] { - logger.Log(Severity.Error, string.Format("Exception extracting {0}: {1}", assemblyPath, ex)); + logger.LogError(string.Format("Exception extracting {0}: {1}", assemblyPath, ex)); } } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs index e3c64d468c1..21b7771ac79 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs @@ -7,6 +7,7 @@ using System.Security.Cryptography; using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; + using Semmle.Util; using Semmle.Util.Logging; @@ -27,8 +28,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching private readonly List generatedSources; private int dotnetFrameworkVersionVariantCount = 0; private int conflictedReferences = 0; - private readonly IDependencyOptions options; private readonly DirectoryInfo sourceDir; + private string? dotnetPath; private readonly IDotNet dotnet; private readonly FileContent fileContent; private readonly TemporaryDirectory packageDirectory; @@ -45,11 +46,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// /// Dependency fetching options /// Logger for dependency fetching progress. - public DependencyManager(string srcDir, IDependencyOptions options, ILogger logger) + public DependencyManager(string srcDir, ILogger logger) { var startTime = DateTime.Now; - this.options = options; this.logger = logger; this.sourceDir = new DirectoryInfo(srcDir); @@ -59,17 +59,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching tempWorkingDirectory = new TemporaryDirectory(FileUtils.GetTemporaryWorkingDirectory(out cleanupTempWorkingDirectory)); - try - { - this.dotnet = DotNet.Make(options, logger, tempWorkingDirectory); - runtimeLazy = new Lazy(() => new Runtime(dotnet, logger)); - } - catch - { - logger.LogError("Missing dotnet CLI"); - throw; - } - logger.LogInfo($"Finding files in {srcDir}..."); var allFiles = GetAllFiles().ToList(); @@ -85,6 +74,33 @@ namespace Semmle.Extraction.CSharp.DependencyFetching logger.LogInfo($"Found {allFiles.Count} files, {nonGeneratedSources.Count} source files, {allProjects.Count} project files, {allSolutions.Count} solution files, {dllPaths.Count} DLLs."); + void startCallback(string s, bool silent) + { + logger.Log(silent ? Severity.Debug : Severity.Info, $"\nRunning {s}"); + } + + void exitCallback(int ret, string msg, bool silent) + { + logger.Log(silent ? Severity.Debug : Severity.Info, $"Exit code {ret}{(string.IsNullOrEmpty(msg) ? "" : $": {msg}")}"); + } + + DotNet.WithDotNet(SystemBuildActions.Instance, logger, smallNonBinaryFiles, tempWorkingDirectory.ToString(), shouldCleanUp: false, ensureDotNetAvailable: true, version: null, installDir => + { + this.dotnetPath = installDir; + return BuildScript.Success; + }).Run(SystemBuildActions.Instance, startCallback, exitCallback); + + try + { + this.dotnet = DotNet.Make(logger, dotnetPath, tempWorkingDirectory); + runtimeLazy = new Lazy(() => new Runtime(dotnet)); + } + catch + { + logger.LogError("Missing dotnet CLI"); + throw; + } + RestoreNugetPackages(allNonBinaryFiles, allProjects, allSolutions, dllPaths); // Find DLLs in the .Net / Asp.Net Framework // This needs to come after the nuget restore, because the nuget restore might fetch the .NET Core/Framework reference assemblies. @@ -570,15 +586,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } } - public DependencyManager(string srcDir) : this(srcDir, DependencyOptions.Default, new ConsoleLogger(Verbosity.Info, logThreadId: true)) { } - private IEnumerable GetAllFiles() { IEnumerable files = sourceDir.GetFiles("*.*", new EnumerationOptions { RecurseSubdirectories = true }); - if (options.DotNetPath != null) + if (dotnetPath != null) { - files = files.Where(f => !f.FullName.StartsWith(options.DotNetPath, StringComparison.OrdinalIgnoreCase)); + files = files.Where(f => !f.FullName.StartsWith(dotnetPath, StringComparison.OrdinalIgnoreCase)); } files = files.Where(f => @@ -590,12 +604,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching return true; } - logger.Log(Severity.Warning, $"File {f.FullName} could not be processed."); + logger.LogWarning($"File {f.FullName} could not be processed."); return false; } catch (Exception ex) { - logger.Log(Severity.Warning, $"File {f.FullName} could not be processed: {ex.Message}"); + logger.LogWarning($"File {f.FullName} could not be processed: {ex.Message}"); return false; } }); @@ -651,7 +665,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } catch (AssemblyLoadException) { - logger.Log(Severity.Warning, $"Could not load assembly information from {usedReference.Key}"); + logger.LogWarning($"Could not load assembly information from {usedReference.Key}"); } } @@ -738,7 +752,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching private void AnalyseSolutions(IEnumerable solutions) { - Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, solutionFile => + Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = EnvironmentVariables.GetDefaultNumberOfThreads() }, solutionFile => { try { @@ -828,7 +842,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching var successCount = 0; var assetFiles = new List(); var sync = new object(); - Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, project => + Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = EnvironmentVariables.GetDefaultNumberOfThreads() }, project => { logger.LogInfo($"Restoring project {project}..."); var res = dotnet.Restore(new(project, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true)); @@ -928,7 +942,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching var successCount = 0; var sync = new object(); - Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, package => + Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = EnvironmentVariables.GetDefaultNumberOfThreads() }, package => { var success = TryRestorePackageManually(package.Name, nugetConfig, package.PackageReferenceSource); if (!success) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyOptions.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyOptions.cs deleted file mode 100644 index 59e5c54097c..00000000000 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyOptions.cs +++ /dev/null @@ -1,32 +0,0 @@ -using System; -using System.Linq; -using System.Collections.Generic; -using Semmle.Util; - -namespace Semmle.Extraction.CSharp.DependencyFetching -{ - /// - /// Dependency fetching related options. - /// - public interface IDependencyOptions - { - /// - /// The number of threads to use. - /// - int Threads { get; } - - /// - /// The path to the local ".dotnet" directory, if any. - /// - string? DotNetPath { get; } - } - - public class DependencyOptions : IDependencyOptions - { - public static IDependencyOptions Default => new DependencyOptions(); - - public int Threads { get; set; } = EnvironmentVariables.GetDefaultNumberOfThreads(); - - public string? DotNetPath { get; set; } = null; - } -} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs index 37c028920a8..84ec938edb0 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs @@ -2,7 +2,9 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Text.RegularExpressions; + +using Newtonsoft.Json.Linq; + using Semmle.Util; using Semmle.Util.Logging; @@ -11,29 +13,26 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// /// Utilities to run the "dotnet" command. /// - internal partial class DotNet : IDotNet + public partial class DotNet : IDotNet { private readonly IDotNetCliInvoker dotnetCliInvoker; - private readonly ILogger logger; private readonly TemporaryDirectory? tempWorkingDirectory; private DotNet(IDotNetCliInvoker dotnetCliInvoker, ILogger logger, TemporaryDirectory? tempWorkingDirectory = null) { - this.logger = logger; this.tempWorkingDirectory = tempWorkingDirectory; this.dotnetCliInvoker = dotnetCliInvoker; Info(); } - private DotNet(IDependencyOptions options, ILogger logger, TemporaryDirectory tempWorkingDirectory) : this(new DotNetCliInvoker(logger, Path.Combine(options.DotNetPath ?? string.Empty, "dotnet")), logger, tempWorkingDirectory) { } + private DotNet(ILogger logger, string? dotNetPath, TemporaryDirectory tempWorkingDirectory) : this(new DotNetCliInvoker(logger, Path.Combine(dotNetPath ?? string.Empty, "dotnet")), logger, tempWorkingDirectory) { } internal static IDotNet Make(IDotNetCliInvoker dotnetCliInvoker, ILogger logger) => new DotNet(dotnetCliInvoker, logger); - public static IDotNet Make(IDependencyOptions options, ILogger logger, TemporaryDirectory tempWorkingDirectory) => new DotNet(options, logger, tempWorkingDirectory); + public static IDotNet Make(ILogger logger, string? dotNetPath, TemporaryDirectory tempWorkingDirectory) => new DotNet(logger, dotNetPath, tempWorkingDirectory); private void Info() { - // TODO: make sure the below `dotnet` version is matching the one specified in global.json var res = dotnetCliInvoker.RunCommand("--info"); if (!res) { @@ -108,5 +107,165 @@ namespace Semmle.Extraction.CSharp.DependencyFetching var args = $"exec {execArgs}"; return dotnetCliInvoker.RunCommand(args); } + + // The version number should be kept in sync with the version .NET version used for building the application. + public const string LatestDotNetSdkVersion = "8.0.101"; + + /// + /// Returns a script for downloading relevant versions of the + /// .NET SDK. The SDK(s) will be installed at installDir + /// (provided that the script succeeds). + /// + private static BuildScript DownloadDotNet(IBuildActions actions, ILogger logger, IEnumerable files, string tempWorkingDirectory, bool shouldCleanUp, string installDir, string? version, bool ensureDotNetAvailable) + { + if (!string.IsNullOrEmpty(version)) + // Specific version requested + return DownloadDotNetVersion(actions, logger, tempWorkingDirectory, shouldCleanUp, installDir, version); + + // Download versions mentioned in `global.json` files + // See https://docs.microsoft.com/en-us/dotnet/core/tools/global-json + var installScript = BuildScript.Success; + var validGlobalJson = false; + + foreach (var path in files.Where(p => p.EndsWith("global.json", StringComparison.Ordinal))) + { + try + { + var o = JObject.Parse(File.ReadAllText(path)); + version = (string)(o?["sdk"]?["version"]!); + } + catch + { + // not a valid global.json file + continue; + } + + installScript &= DownloadDotNetVersion(actions, logger, tempWorkingDirectory, shouldCleanUp, installDir, version); + validGlobalJson = true; + } + + if (validGlobalJson) + { + return installScript; + } + + if (ensureDotNetAvailable) + { + return DownloadDotNetVersion(actions, logger, tempWorkingDirectory, shouldCleanUp, installDir, LatestDotNetSdkVersion, needExactVersion: false); + } + + return BuildScript.Failure; + } + + /// + /// Returns a script for downloading a specific .NET SDK version, if the + /// version is not already installed. + /// + /// See https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet-install-script. + /// + private static BuildScript DownloadDotNetVersion(IBuildActions actions, ILogger logger, string tempWorkingDirectory, bool shouldCleanUp, string path, string version, bool needExactVersion = true) + { + return BuildScript.Bind(GetInstalledSdksScript(actions), (sdks, sdksRet) => + { + if (needExactVersion && sdksRet == 0 && sdks.Count == 1 && sdks[0].StartsWith(version + " ", StringComparison.Ordinal)) + { + // The requested SDK is already installed (and no other SDKs are installed), so + // no need to reinstall + return BuildScript.Failure; + } + else if (!needExactVersion && sdksRet == 0 && sdks.Count > 0) + { + // there's at least one SDK installed, so no need to reinstall + return BuildScript.Failure; + } + else if (!needExactVersion && sdksRet != 0) + { + logger.LogInfo("No .NET SDK found."); + } + + logger.LogInfo($"Attempting to download .NET {version}"); + + if (actions.IsWindows()) + { + + var psCommand = $"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; &([scriptblock]::Create((Invoke-WebRequest -UseBasicParsing 'https://dot.net/v1/dotnet-install.ps1'))) -Version {version} -InstallDir {path}"; + + BuildScript GetInstall(string pwsh) => + new CommandBuilder(actions). + RunCommand(pwsh). + Argument("-NoProfile"). + Argument("-ExecutionPolicy"). + Argument("unrestricted"). + Argument("-Command"). + Argument("\"" + psCommand + "\""). + Script; + + return GetInstall("pwsh") | GetInstall("powershell"); + } + else + { + var dotnetInstallPath = actions.PathCombine(tempWorkingDirectory, ".dotnet", "dotnet-install.sh"); + + var downloadDotNetInstallSh = BuildScript.DownloadFile( + "https://dot.net/v1/dotnet-install.sh", + dotnetInstallPath, + e => logger.LogWarning($"Failed to download 'dotnet-install.sh': {e.Message}")); + + var chmod = new CommandBuilder(actions). + RunCommand("chmod"). + Argument("u+x"). + Argument(dotnetInstallPath); + + var install = new CommandBuilder(actions). + RunCommand(dotnetInstallPath). + Argument("--channel"). + Argument("release"). + Argument("--version"). + Argument(version). + Argument("--install-dir"). + Argument(path); + + var buildScript = downloadDotNetInstallSh & chmod.Script & install.Script; + + if (shouldCleanUp) + { + buildScript &= BuildScript.DeleteFile(dotnetInstallPath); + } + + return buildScript; + } + }); + } + + private static BuildScript GetInstalledSdksScript(IBuildActions actions) + { + var listSdks = new CommandBuilder(actions, silent: true). + RunCommand("dotnet"). + Argument("--list-sdks"); + return listSdks.Script; + } + + /// + /// Returns a script that attempts to download relevant version(s) of the + /// .NET SDK, followed by running the script generated by . + /// + /// The argument to is the path to the directory in which the + /// .NET SDK(s) were installed. + /// + public static BuildScript WithDotNet(IBuildActions actions, ILogger logger, IEnumerable files, string tempWorkingDirectory, bool shouldCleanUp, bool ensureDotNetAvailable, string? version, Func f) + { + var installDir = actions.PathCombine(tempWorkingDirectory, ".dotnet"); + var installScript = DownloadDotNet(actions, logger, files, tempWorkingDirectory, shouldCleanUp, installDir, version, ensureDotNetAvailable); + return BuildScript.Bind(installScript, installed => + { + if (installed != 0) + { + // The .NET SDK was not installed, either because the installation failed or because it was already installed. + installDir = null; + } + + return f(installDir); + }); + } } -} +} \ No newline at end of file diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FilePathFilter.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FilePathFilter.cs index d1a3ed011d4..1a1e9934e8c 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FilePathFilter.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FilePathFilter.cs @@ -55,12 +55,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } else { - logger.Log(Severity.Info, $"Invalid filter: {filter}"); + logger.LogInfo($"Invalid filter: {filter}"); continue; } var regex = new FilePattern(filterText).RegexPattern; - logger.Log(Severity.Info, $"Filtering {(include ? "in" : "out")} files matching '{regex}'. Original glob filter: '{filter}'"); + logger.LogInfo($"Filtering {(include ? "in" : "out")} files matching '{regex}'. Original glob filter: '{filter}'"); pathFilters.Add(new PathFilter(new Regex(regex, RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline), include)); } @@ -91,7 +91,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching if (!include) { - logger.Log(Severity.Info, $"Excluding '{f.FileInfo.FullName}'"); + logger.LogInfo($"Excluding '{f.FileInfo.FullName}'"); } return include; diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs index 5904248b34f..d66135c1644 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs @@ -5,7 +5,7 @@ using System.Text.RegularExpressions; namespace Semmle.Extraction.CSharp.DependencyFetching { - internal interface IDotNet + public interface IDotNet { RestoreResult Restore(RestoreSettings restoreSettings); bool New(string folder); @@ -15,9 +15,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching bool Exec(string execArgs); } - internal record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, string? PathToNugetConfig = null, bool ForceReevaluation = false); + public record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, string? PathToNugetConfig = null, bool ForceReevaluation = false); - internal partial record class RestoreResult(bool Success, IList Output) + public partial record class RestoreResult(bool Success, IList Output) { private readonly Lazy> assetsFilePaths = new(() => GetFirstGroupOnMatch(AssetsFileRegex(), Output)); public IEnumerable AssetsFilePaths => Success ? assetsFilePaths.Value : Array.Empty(); diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Runtime.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Runtime.cs index 90c4af2c4d3..a489ec504c4 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Runtime.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Runtime.cs @@ -18,14 +18,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching private const string aspNetCoreApp = "Microsoft.AspNetCore.App"; private readonly IDotNet dotNet; - private readonly ILogger logger; private readonly Lazy> newestRuntimes; private Dictionary NewestRuntimes => newestRuntimes.Value; - public Runtime(IDotNet dotNet, ILogger logger) + public Runtime(IDotNet dotNet) { this.dotNet = dotNet; - this.logger = logger; this.newestRuntimes = new(GetNewestRuntimes); } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyStubGenerator/Program.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyStubGenerator/Program.cs index 8160bfdd3ae..12f06a58fd4 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyStubGenerator/Program.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyStubGenerator/Program.cs @@ -3,7 +3,7 @@ using Semmle.Extraction.CSharp.StubGenerator; using Semmle.Util.Logging; var logger = new ConsoleLogger(Verbosity.Info, logThreadId: false); -using var dependencyManager = new DependencyManager(".", DependencyOptions.Default, logger); +using var dependencyManager = new DependencyManager(".", logger); StubGenerator.GenerateStubs(logger, dependencyManager.ReferenceFiles, "codeql_csharp_stubs"); return 0; diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs index 115a8d418b6..fb314e525c1 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs @@ -138,7 +138,7 @@ namespace Semmle.Extraction.CSharp.Standalone using var logger = new ConsoleLogger(options.Verbosity, logThreadId: true); logger.Log(Severity.Info, "Extracting C# in buildless mode"); - using var dependencyManager = new DependencyManager(options.SrcDir, options.Dependencies, logger); + using var dependencyManager = new DependencyManager(options.SrcDir, logger); if (!dependencyManager.AllSourceFiles.Any()) { diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Options.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Options.cs index f6e21b32bbd..2efe7704775 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Options.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Options.cs @@ -23,18 +23,6 @@ namespace Semmle.Extraction.CSharp.Standalone } } - public override bool HandleOption(string key, string value) - { - switch (key) - { - case "dotnet": - dependencies.DotNetPath = value; - return true; - default: - return base.HandleOption(key, value); - } - } - public override bool HandleArgument(string arg) { return true; @@ -51,12 +39,6 @@ namespace Semmle.Extraction.CSharp.Standalone /// public string SrcDir { get; } = Directory.GetCurrentDirectory(); - private readonly DependencyOptions dependencies = new DependencyOptions(); - /// - /// Dependency fetching related options. - /// - public IDependencyOptions Dependencies => dependencies; - /// /// Whether errors were encountered parsing the arguments. /// diff --git a/csharp/extractor/Semmle.Extraction.CSharp.StubGenerator/StubGenerator.cs b/csharp/extractor/Semmle.Extraction.CSharp.StubGenerator/StubGenerator.cs index 32ae6d25161..0f8347eb0bc 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.StubGenerator/StubGenerator.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.StubGenerator/StubGenerator.cs @@ -36,7 +36,7 @@ public static class StubGenerator references.Add((reference, path)); }); - logger.Log(Severity.Info, $"Generating stubs for {references.Count} assemblies."); + logger.LogInfo($"Generating stubs for {references.Count} assemblies."); var compilation = CSharpCompilation.Create( "stubgenerator.dll", @@ -50,7 +50,7 @@ public static class StubGenerator }); stopWatch.Stop(); - logger.Log(Severity.Info, $"Stub generation took {stopWatch.Elapsed}."); + logger.LogInfo($"Stub generation took {stopWatch.Elapsed}."); return stubPaths.ToArray(); } diff --git a/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs b/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs index 0d4ed6c4b92..17bc477bde8 100644 --- a/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs +++ b/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs @@ -47,7 +47,7 @@ namespace Semmle.Extraction.Tests "Microsoft.NETCore.App 7.0.2 [/path/dotnet/shared/Microsoft.NETCore.App]" }; var dotnet = new DotNetStub(listedRuntimes, null!); - var runtime = new Runtime(dotnet, new LoggerStub()); + var runtime = new Runtime(dotnet); // Execute var runtimes = runtime.GetNewestRuntimes(); @@ -73,7 +73,7 @@ namespace Semmle.Extraction.Tests "Microsoft.NETCore.App 8.0.0-preview.5.23280.8 [/path/dotnet/shared/Microsoft.NETCore.App]" }; var dotnet = new DotNetStub(listedRuntimes, null!); - var runtime = new Runtime(dotnet, new LoggerStub()); + var runtime = new Runtime(dotnet); // Execute var runtimes = runtime.GetNewestRuntimes(); @@ -96,7 +96,7 @@ namespace Semmle.Extraction.Tests "Microsoft.NETCore.App 8.0.0-preview.5.23280.8 [/path/dotnet/shared/Microsoft.NETCore.App]" }; var dotnet = new DotNetStub(listedRuntimes, null!); - var runtime = new Runtime(dotnet, new LoggerStub()); + var runtime = new Runtime(dotnet); // Execute var runtimes = runtime.GetNewestRuntimes(); @@ -125,7 +125,7 @@ namespace Semmle.Extraction.Tests @"Microsoft.WindowsDesktop.App 7.0.4 [C:\Program Files\dotnet\shared\Microsoft.WindowsDesktop.App]" }; var dotnet = new DotNetStub(listedRuntimes, null!); - var runtime = new Runtime(dotnet, new LoggerStub()); + var runtime = new Runtime(dotnet); // Execute var runtimes = runtime.GetNewestRuntimes(); diff --git a/csharp/extractor/Semmle.Extraction/Extractor/Extractor.cs b/csharp/extractor/Semmle.Extraction/Extractor/Extractor.cs index 362b7910c56..db0b30fb2b8 100644 --- a/csharp/extractor/Semmle.Extraction/Extractor/Extractor.cs +++ b/csharp/extractor/Semmle.Extraction/Extractor/Extractor.cs @@ -43,7 +43,7 @@ namespace Semmle.Extraction ++Errors; if (Errors == maxErrors) { - Logger.Log(Severity.Info, " Stopping logging after {0} errors", Errors); + Logger.LogInfo(" Stopping logging after {0} errors", Errors); } } diff --git a/csharp/extractor/Semmle.Extraction/TrapWriter.cs b/csharp/extractor/Semmle.Extraction/TrapWriter.cs index 7f2c38466c0..aff9983dd7c 100644 --- a/csharp/extractor/Semmle.Extraction/TrapWriter.cs +++ b/csharp/extractor/Semmle.Extraction/TrapWriter.cs @@ -227,7 +227,7 @@ namespace Semmle.Extraction { // If this happened, it was probably because the same file was compiled multiple times. // In any case, this is not a fatal error. - logger.Log(Severity.Warning, "Problem archiving " + dest + ": " + ex); + logger.LogWarning("Problem archiving " + dest + ": " + ex); } } diff --git a/csharp/extractor/Semmle.Util/BuildActions.cs b/csharp/extractor/Semmle.Util/BuildActions.cs index 42a6a4d9d12..e3a4e7ecafe 100644 --- a/csharp/extractor/Semmle.Util/BuildActions.cs +++ b/csharp/extractor/Semmle.Util/BuildActions.cs @@ -287,4 +287,30 @@ namespace Semmle.Util public static IBuildActions Instance { get; } = new SystemBuildActions(); } + + public static class BuildActionExtensions + { + private static void FindFiles(this IBuildActions actions, string dir, int depth, int? maxDepth, IList<(string, int)> results) + { + foreach (var f in actions.EnumerateFiles(dir)) + { + results.Add((f, depth)); + } + + if (depth < maxDepth) + { + foreach (var d in actions.EnumerateDirectories(dir)) + { + actions.FindFiles(d, depth + 1, maxDepth, results); + } + } + } + + public static (string path, int depth)[] FindFiles(this IBuildActions actions, string dir, int? maxDepth) + { + var results = new List<(string, int)>(); + actions.FindFiles(dir, 0, maxDepth, results); + return results.OrderBy(f => f.Item2).ToArray(); + } + } } diff --git a/csharp/extractor/Semmle.Util/CanonicalPathCache.cs b/csharp/extractor/Semmle.Util/CanonicalPathCache.cs index 87fcb3717bd..e7e4c07c36e 100644 --- a/csharp/extractor/Semmle.Util/CanonicalPathCache.cs +++ b/csharp/extractor/Semmle.Util/CanonicalPathCache.cs @@ -268,7 +268,7 @@ namespace Semmle.Util catch // lgtm[cs/catch-of-all-exceptions] { // Failed to late-bind a suitable library. - logger.Log(Severity.Warning, "Preserving symlinks in canonical paths"); + logger.LogWarning("Preserving symlinks in canonical paths"); pathStrategy = new QueryDirectoryStrategy(); } break; diff --git a/csharp/extractor/Semmle.Util/FileUtils.cs b/csharp/extractor/Semmle.Util/FileUtils.cs index 09269e37e8b..094c4da3338 100644 --- a/csharp/extractor/Semmle.Util/FileUtils.cs +++ b/csharp/extractor/Semmle.Util/FileUtils.cs @@ -133,14 +133,14 @@ namespace Semmle.Util var directoryName = Path.GetDirectoryName(nested); if (directoryName is null) { - logger.Log(Severity.Warning, "Failed to get directory name from path '" + nested + "'."); + logger.LogWarning("Failed to get directory name from path '" + nested + "'."); throw new InvalidOperationException(); } Directory.CreateDirectory(directoryName); } catch (PathTooLongException) { - logger.Log(Severity.Warning, "Failed to create parent directory of '" + nested + "': Path too long."); + logger.LogWarning("Failed to create parent directory of '" + nested + "': Path too long."); throw; } return nested;