diff --git a/csharp/autobuilder/Semmle.Autobuild.Tests/BuildScripts.cs b/csharp/autobuilder/Semmle.Autobuild.Tests/BuildScripts.cs index 20d19a3525c..a2e59cf7ac3 100644 --- a/csharp/autobuilder/Semmle.Autobuild.Tests/BuildScripts.cs +++ b/csharp/autobuilder/Semmle.Autobuild.Tests/BuildScripts.cs @@ -185,7 +185,7 @@ namespace Semmle.Extraction.Tests // Records the arguments passed to StartCallback. IList StartCallbackIn = new List(); - void StartCallback(string s) + void StartCallback(string s, bool silent) { StartCallbackIn.Add(s); } @@ -194,7 +194,7 @@ namespace Semmle.Extraction.Tests IList EndCallbackIn = new List(); IList EndCallbackReturn = new List(); - void EndCallback(int ret, string s) + void EndCallback(int ret, string s, bool silent) { EndCallbackReturn.Add(ret); EndCallbackIn.Add(s); @@ -203,7 +203,7 @@ namespace Semmle.Extraction.Tests [Fact] public void TestBuildCommand() { - var cmd = BuildScript.Create("abc", "def ghi", null, null); + var cmd = BuildScript.Create("abc", "def ghi", false, null, null); Actions.RunProcess["abc def ghi"] = 1; cmd.Run(Actions, StartCallback, EndCallback); @@ -216,7 +216,7 @@ namespace Semmle.Extraction.Tests [Fact] public void TestAnd1() { - var cmd = BuildScript.Create("abc", "def ghi", null, null) & BuildScript.Create("odasa", null, null, null); + var cmd = BuildScript.Create("abc", "def ghi", false, null, null) & BuildScript.Create("odasa", null, false, null, null); Actions.RunProcess["abc def ghi"] = 1; cmd.Run(Actions, StartCallback, EndCallback); @@ -230,7 +230,7 @@ namespace Semmle.Extraction.Tests [Fact] public void TestAnd2() { - var cmd = BuildScript.Create("odasa", null, null, null) & BuildScript.Create("abc", "def ghi", null, null); + var cmd = BuildScript.Create("odasa", null, false, null, null) & BuildScript.Create("abc", "def ghi", false, null, null); Actions.RunProcess["abc def ghi"] = 1; Actions.RunProcess["odasa "] = 0; @@ -250,7 +250,7 @@ namespace Semmle.Extraction.Tests [Fact] public void TestOr1() { - var cmd = BuildScript.Create("odasa", null, null, null) | BuildScript.Create("abc", "def ghi", null, null); + var cmd = BuildScript.Create("odasa", null, false, null, null) | BuildScript.Create("abc", "def ghi", false, null, null); Actions.RunProcess["abc def ghi"] = 1; Actions.RunProcess["odasa "] = 0; @@ -266,7 +266,7 @@ namespace Semmle.Extraction.Tests [Fact] public void TestOr2() { - var cmd = BuildScript.Create("abc", "def ghi", null, null) | BuildScript.Create("odasa", null, null, null); + var cmd = BuildScript.Create("abc", "def ghi", false, null, null) | BuildScript.Create("odasa", null, false, null, null); Actions.RunProcess["abc def ghi"] = 1; Actions.RunProcess["odasa "] = 0; @@ -375,7 +375,7 @@ namespace Semmle.Extraction.Tests Actions.RunProcess["cmd.exe /C dotnet --info"] = 0; Actions.RunProcess["cmd.exe /C dotnet clean test.csproj"] = 0; Actions.RunProcess["cmd.exe /C dotnet restore test.csproj"] = 0; - Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --auto dotnet build --no-incremental /p:UseSharedCompilation=false test.csproj"] = 0; + Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --auto dotnet build --no-incremental test.csproj"] = 0; Actions.RunProcess[@"cmd.exe /C C:\codeql\tools\java\bin\java -jar C:\codeql\csharp\tools\extractor-asp.jar ."] = 0; Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --xml --extensions config csproj props xml"] = 0; Actions.FileExists["csharp.log"] = true; @@ -401,6 +401,9 @@ namespace Semmle.Extraction.Tests [Fact] public void TestLinuxCSharpAutoBuilder() { + Actions.RunProcess["dotnet --list-runtimes"] = 0; + Actions.RunProcessOut["dotnet --list-runtimes"] = @"Microsoft.AspNetCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.AspNetCore.App] +Microsoft.NETCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.NETCore.App]"; Actions.RunProcess["dotnet --info"] = 0; Actions.RunProcess["dotnet clean test.csproj"] = 0; Actions.RunProcess["dotnet restore test.csproj"] = 0; @@ -424,7 +427,7 @@ namespace Semmle.Extraction.Tests Actions.LoadXml["test.csproj"] = xml; var autobuilder = CreateAutoBuilder("csharp", false); - TestAutobuilderScript(autobuilder, 0, 6); + TestAutobuilderScript(autobuilder, 0, 7); } [Fact] @@ -598,6 +601,8 @@ namespace Semmle.Extraction.Tests [Fact] public void TestLinuxBuildCommand() { + Actions.RunProcess["dotnet --list-runtimes"] = 1; + Actions.RunProcessOut["dotnet --list-runtimes"] = ""; Actions.RunProcess[@"C:\odasa/tools/odasa index --auto ""./build.sh --skip-tests"""] = 0; Actions.RunProcess[@"C:\codeql\tools\java/bin/java -jar C:\codeql\csharp/tools/extractor-asp.jar ."] = 0; Actions.RunProcess[@"C:\odasa/tools/odasa index --xml --extensions config csproj props xml"] = 0; @@ -610,7 +615,7 @@ namespace Semmle.Extraction.Tests SkipVsWhere(); var autobuilder = CreateAutoBuilder("csharp", false, buildCommand: "./build.sh --skip-tests"); - TestAutobuilderScript(autobuilder, 0, 3); + TestAutobuilderScript(autobuilder, 0, 4); } [Fact] @@ -621,6 +626,8 @@ namespace Semmle.Extraction.Tests Actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = ""; Actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = ""; Actions.RunProcess["/bin/chmod u+x build/build.sh"] = 0; + Actions.RunProcess["dotnet --list-runtimes"] = 1; + Actions.RunProcessOut["dotnet --list-runtimes"] = ""; Actions.RunProcess[@"C:\odasa/tools/odasa index --auto build/build.sh"] = 0; Actions.RunProcessWorkingDirectory[@"C:\odasa/tools/odasa index --auto build/build.sh"] = "build"; Actions.RunProcess[@"C:\codeql\tools\java/bin/java -jar C:\codeql\csharp/tools/extractor-asp.jar ."] = 0; @@ -628,7 +635,7 @@ namespace Semmle.Extraction.Tests Actions.FileExists["csharp.log"] = true; var autobuilder = CreateAutoBuilder("csharp", false); - TestAutobuilderScript(autobuilder, 0, 4); + TestAutobuilderScript(autobuilder, 0, 5); } [Fact] @@ -640,12 +647,14 @@ namespace Semmle.Extraction.Tests Actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = ""; Actions.RunProcess["/bin/chmod u+x build.sh"] = 0; + Actions.RunProcess["dotnet --list-runtimes"] = 1; + Actions.RunProcessOut["dotnet --list-runtimes"] = ""; Actions.RunProcess[@"C:\odasa/tools/odasa index --auto build.sh"] = 0; Actions.RunProcessWorkingDirectory[@"C:\odasa/tools/odasa index --auto build.sh"] = ""; Actions.FileExists["csharp.log"] = false; var autobuilder = CreateAutoBuilder("csharp", false); - TestAutobuilderScript(autobuilder, 1, 2); + TestAutobuilderScript(autobuilder, 1, 3); } [Fact] @@ -657,12 +666,14 @@ namespace Semmle.Extraction.Tests Actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = ""; Actions.RunProcess["/bin/chmod u+x build.sh"] = 0; + Actions.RunProcess["dotnet --list-runtimes"] = 1; + Actions.RunProcessOut["dotnet --list-runtimes"] = ""; Actions.RunProcess[@"C:\odasa/tools/odasa index --auto build.sh"] = 5; Actions.RunProcessWorkingDirectory[@"C:\odasa/tools/odasa index --auto build.sh"] = ""; Actions.FileExists["csharp.log"] = true; var autobuilder = CreateAutoBuilder("csharp", false); - TestAutobuilderScript(autobuilder, 1, 2); + TestAutobuilderScript(autobuilder, 1, 3); } [Fact] @@ -748,7 +759,7 @@ namespace Semmle.Extraction.Tests TestAutobuilderScript(autobuilder, 0, 6); } - [Fact] + [Fact] public void TestWindowCSharpMsBuildMultipleSolutions() { Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\csharp\nuget\nuget.exe restore test1.csproj"] = 0; @@ -871,6 +882,9 @@ namespace Semmle.Extraction.Tests [Fact] public void TestSkipNugetDotnet() { + Actions.RunProcess["dotnet --list-runtimes"] = 0; + Actions.RunProcessOut["dotnet --list-runtimes"] = @"Microsoft.AspNetCore.App 2.1.3 [/usr/local/share/dotnet/shared/Microsoft.AspNetCore.App] +Microsoft.NETCore.App 2.1.3 [/usr/local/share/dotnet/shared/Microsoft.NETCore.App]"; Actions.RunProcess["dotnet --info"] = 0; Actions.RunProcess["dotnet clean test.csproj"] = 0; Actions.RunProcess["dotnet restore test.csproj"] = 0; @@ -894,7 +908,7 @@ namespace Semmle.Extraction.Tests Actions.LoadXml["test.csproj"] = xml; var autobuilder = CreateAutoBuilder("csharp", false, dotnetArguments: "--no-restore"); // nugetRestore=false does not work for now. - TestAutobuilderScript(autobuilder, 0, 6); + TestAutobuilderScript(autobuilder, 0, 7); } [Fact] @@ -906,10 +920,13 @@ namespace Semmle.Extraction.Tests Actions.RunProcess[@"chmod u+x dotnet-install.sh"] = 0; Actions.RunProcess[@"./dotnet-install.sh --channel release --version 2.1.3 --install-dir C:\Project/.dotnet"] = 0; Actions.RunProcess[@"rm dotnet-install.sh"] = 0; + Actions.RunProcess[@"C:\Project/.dotnet/dotnet --list-runtimes"] = 0; + Actions.RunProcessOut[@"C:\Project/.dotnet/dotnet --list-runtimes"] = @"Microsoft.AspNetCore.App 3.0.0 [/usr/local/share/dotnet/shared/Microsoft.AspNetCore.App] +Microsoft.NETCore.App 3.0.0 [/usr/local/share/dotnet/shared/Microsoft.NETCore.App]"; Actions.RunProcess[@"C:\Project/.dotnet/dotnet --info"] = 0; Actions.RunProcess[@"C:\Project/.dotnet/dotnet clean test.csproj"] = 0; Actions.RunProcess[@"C:\Project/.dotnet/dotnet restore test.csproj"] = 0; - Actions.RunProcess[@"C:\odasa/tools/odasa index --auto C:\Project/.dotnet/dotnet build --no-incremental /p:UseSharedCompilation=false test.csproj"] = 0; + Actions.RunProcess[@"C:\odasa/tools/odasa index --auto C:\Project/.dotnet/dotnet build --no-incremental test.csproj"] = 0; Actions.RunProcess[@"C:\codeql\tools\java/bin/java -jar C:\codeql\csharp/tools/extractor-asp.jar ."] = 0; Actions.RunProcess[@"C:\odasa/tools/odasa index --xml --extensions config csproj props xml"] = 0; Actions.FileExists["csharp.log"] = true; @@ -930,18 +947,24 @@ namespace Semmle.Extraction.Tests Actions.LoadXml["test.csproj"] = xml; var autobuilder = CreateAutoBuilder("csharp", false, dotnetVersion: "2.1.3"); - TestAutobuilderScript(autobuilder, 0, 11); + TestAutobuilderScript(autobuilder, 0, 12); } [Fact] public void TestDotnetVersionAlreadyInstalled() { Actions.RunProcess["dotnet --list-sdks"] = 0; - Actions.RunProcessOut["dotnet --list-sdks"] = "2.1.3 [C:\\Program Files\\dotnet\\sdks]\n2.1.4 [C:\\Program Files\\dotnet\\sdks]"; + Actions.RunProcessOut["dotnet --list-sdks"] = @"2.1.3 [C:\Program Files\dotnet\sdks] +2.1.4 [C:\Program Files\dotnet\sdks]"; Actions.RunProcess[@"curl -L -sO https://dot.net/v1/dotnet-install.sh"] = 0; Actions.RunProcess[@"chmod u+x dotnet-install.sh"] = 0; Actions.RunProcess[@"./dotnet-install.sh --channel release --version 2.1.3 --install-dir C:\Project/.dotnet"] = 0; Actions.RunProcess[@"rm dotnet-install.sh"] = 0; + Actions.RunProcess[@"C:\Project/.dotnet/dotnet --list-runtimes"] = 0; + Actions.RunProcessOut[@"C:\Project/.dotnet/dotnet --list-runtimes"] = @"Microsoft.AspNetCore.App 2.1.3 [/usr/local/share/dotnet/shared/Microsoft.AspNetCore.App] +Microsoft.AspNetCore.App 2.1.4 [/usr/local/share/dotnet/shared/Microsoft.AspNetCore.App] +Microsoft.NETCore.App 2.1.3 [/usr/local/share/dotnet/shared/Microsoft.NETCore.App] +Microsoft.NETCore.App 2.1.4 [/usr/local/share/dotnet/shared/Microsoft.NETCore.App]"; Actions.RunProcess[@"C:\Project/.dotnet/dotnet --info"] = 0; Actions.RunProcess[@"C:\Project/.dotnet/dotnet clean test.csproj"] = 0; Actions.RunProcess[@"C:\Project/.dotnet/dotnet restore test.csproj"] = 0; @@ -966,7 +989,7 @@ namespace Semmle.Extraction.Tests Actions.LoadXml["test.csproj"] = xml; var autobuilder = CreateAutoBuilder("csharp", false, dotnetVersion: "2.1.3"); - TestAutobuilderScript(autobuilder, 0, 11); + TestAutobuilderScript(autobuilder, 0, 12); } [Fact] @@ -979,7 +1002,7 @@ namespace Semmle.Extraction.Tests Actions.RunProcess[@"cmd.exe /C C:\Project\.dotnet\dotnet --info"] = 0; Actions.RunProcess[@"cmd.exe /C C:\Project\.dotnet\dotnet clean test.csproj"] = 0; Actions.RunProcess[@"cmd.exe /C C:\Project\.dotnet\dotnet restore test.csproj"] = 0; - Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --auto C:\Project\.dotnet\dotnet build --no-incremental /p:UseSharedCompilation=false test.csproj"] = 0; + Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --auto C:\Project\.dotnet\dotnet build --no-incremental test.csproj"] = 0; Actions.RunProcess[@"cmd.exe /C C:\codeql\tools\java\bin\java -jar C:\codeql\csharp\tools\extractor-asp.jar ."] = 0; Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --xml --extensions config csproj props xml"] = 0; Actions.FileExists["csharp.log"] = true; diff --git a/csharp/autobuilder/Semmle.Autobuild/Autobuilder.cs b/csharp/autobuilder/Semmle.Autobuild/Autobuilder.cs index 001934f489d..73e6fcd181f 100644 --- a/csharp/autobuilder/Semmle.Autobuild/Autobuilder.cs +++ b/csharp/autobuilder/Semmle.Autobuild/Autobuilder.cs @@ -214,8 +214,16 @@ namespace Semmle.Autobuild if (Options.IgnoreErrors) script |= BuildScript.Success; - void startCallback(string s) => Log(Severity.Info, $"\nRunning {s}"); - void exitCallback(int ret, string msg) => Log(Severity.Info, $"Exit code {ret}{(string.IsNullOrEmpty(msg) ? "" : $": {msg}")}"); + void startCallback(string s, bool silent) + { + Log(silent ? Severity.Debug : Severity.Info, $"\nRunning {s}"); + } + + void exitCallback(int ret, string msg, bool silent) + { + Log(silent ? Severity.Debug : Severity.Info, $"Exit code {ret}{(string.IsNullOrEmpty(msg) ? "" : $": {msg}")}"); + } + return script.Run(Actions, startCallback, exitCallback); } diff --git a/csharp/autobuilder/Semmle.Autobuild/BuildActions.cs b/csharp/autobuilder/Semmle.Autobuild/BuildActions.cs index edf4fc752c1..837f6e3f69e 100644 --- a/csharp/autobuilder/Semmle.Autobuild/BuildActions.cs +++ b/csharp/autobuilder/Semmle.Autobuild/BuildActions.cs @@ -141,12 +141,8 @@ namespace Semmle.Autobuild pi.WorkingDirectory = workingDirectory; // Environment variables can only be used when not redirecting stdout - if (!redirectStandardOutput) - { - pi.Environment["UseSharedCompilation"] = "false"; - if (environment != null) - environment.ForEach(kvp => pi.Environment[kvp.Key] = kvp.Value); - } + if (!redirectStandardOutput && environment != null) + environment.ForEach(kvp => pi.Environment[kvp.Key] = kvp.Value); return pi; } diff --git a/csharp/autobuilder/Semmle.Autobuild/BuildCommandAutoRule.cs b/csharp/autobuilder/Semmle.Autobuild/BuildCommandAutoRule.cs index 09c41ccf74c..e7e2c9255dd 100644 --- a/csharp/autobuilder/Semmle.Autobuild/BuildCommandAutoRule.cs +++ b/csharp/autobuilder/Semmle.Autobuild/BuildCommandAutoRule.cs @@ -43,9 +43,9 @@ namespace Semmle.Autobuild var dir = Path.GetDirectoryName(scriptPath); // A specific .NET Core version may be required - return chmodScript & DotNetRule.WithDotNet(builder, dotNet => + return chmodScript & DotNetRule.WithDotNet(builder, environment => { - var command = new CommandBuilder(builder.Actions, dir, dotNet?.Environment); + var command = new CommandBuilder(builder.Actions, dir, environment); // A specific Visual Studio version may be required var vsTools = MsBuildRule.GetVcVarsBatFile(builder); diff --git a/csharp/autobuilder/Semmle.Autobuild/BuildCommandRule.cs b/csharp/autobuilder/Semmle.Autobuild/BuildCommandRule.cs index 2e582b707f2..fe91503ec8f 100644 --- a/csharp/autobuilder/Semmle.Autobuild/BuildCommandRule.cs +++ b/csharp/autobuilder/Semmle.Autobuild/BuildCommandRule.cs @@ -11,9 +11,9 @@ return BuildScript.Failure; // Custom build commands may require a specific .NET Core version - return DotNetRule.WithDotNet(builder, dotNet => + return DotNetRule.WithDotNet(builder, environment => { - var command = new CommandBuilder(builder.Actions, null, dotNet?.Environment); + var command = new CommandBuilder(builder.Actions, null, environment); // Custom build commands may require a specific Visual Studio version var vsTools = MsBuildRule.GetVcVarsBatFile(builder); diff --git a/csharp/autobuilder/Semmle.Autobuild/BuildScript.cs b/csharp/autobuilder/Semmle.Autobuild/BuildScript.cs index a41011e159f..93ea941d58b 100644 --- a/csharp/autobuilder/Semmle.Autobuild/BuildScript.cs +++ b/csharp/autobuilder/Semmle.Autobuild/BuildScript.cs @@ -25,7 +25,7 @@ namespace Semmle.Autobuild /// an exit message. /// /// The exit code from this build script. - public abstract int Run(IBuildActions actions, Action startCallback, Action exitCallBack); + public abstract int Run(IBuildActions actions, Action startCallback, Action exitCallBack); /// /// Run this build command. @@ -44,33 +44,36 @@ namespace Semmle.Autobuild /// /// Contents of standard out. /// The exit code from this build script. - public abstract int Run(IBuildActions actions, Action startCallback, Action exitCallBack, out IList stdout); + public abstract int Run(IBuildActions actions, Action startCallback, Action exitCallBack, out IList stdout); class BuildCommand : BuildScript { readonly string exe, arguments, workingDirectory; readonly IDictionary environment; + readonly bool silent; /// /// Create a simple build command. /// /// The executable to run. /// The arguments to the executable, or null. + /// Whether this command should run silently. /// The working directory (null for current directory). /// Additional environment variables. - public BuildCommand(string exe, string argumentsOpt, string workingDirectory = null, IDictionary environment = null) + public BuildCommand(string exe, string argumentsOpt, bool silent, string workingDirectory = null, IDictionary environment = null) { this.exe = exe; this.arguments = argumentsOpt ?? ""; + this.silent = silent; this.workingDirectory = workingDirectory; this.environment = environment; } public override string ToString() => exe + " " + arguments; - public override int Run(IBuildActions actions, Action startCallback, Action exitCallBack) + public override int Run(IBuildActions actions, Action startCallback, Action exitCallBack) { - startCallback(this.ToString()); + startCallback(this.ToString(), silent); var ret = 1; var retMessage = ""; try @@ -83,13 +86,13 @@ namespace Semmle.Autobuild retMessage = ex.Message; } - exitCallBack(ret, retMessage); + exitCallBack(ret, retMessage, silent); return ret; } - public override int Run(IBuildActions actions, Action startCallback, Action exitCallBack, out IList stdout) + public override int Run(IBuildActions actions, Action startCallback, Action exitCallBack, out IList stdout) { - startCallback(this.ToString()); + startCallback(this.ToString(), silent); var ret = 1; var retMessage = ""; try @@ -102,7 +105,7 @@ namespace Semmle.Autobuild retMessage = ex.Message; stdout = new string[0]; } - exitCallBack(ret, retMessage); + exitCallBack(ret, retMessage, silent); return ret; } @@ -116,9 +119,9 @@ namespace Semmle.Autobuild this.func = func; } - public override int Run(IBuildActions actions, Action startCallback, Action exitCallBack) => func(actions); + public override int Run(IBuildActions actions, Action startCallback, Action exitCallBack) => func(actions); - public override int Run(IBuildActions actions, Action startCallback, Action exitCallBack, out IList stdout) + public override int Run(IBuildActions actions, Action startCallback, Action exitCallBack, out IList stdout) { stdout = new string[0]; return func(actions); @@ -142,7 +145,7 @@ namespace Semmle.Autobuild this.s2b = s2; } - public override int Run(IBuildActions actions, Action startCallback, Action exitCallBack) + public override int Run(IBuildActions actions, Action startCallback, Action exitCallBack) { int ret1; if (s2a != null) @@ -155,7 +158,7 @@ namespace Semmle.Autobuild return s2b(ret1).Run(actions, startCallback, exitCallBack); } - public override int Run(IBuildActions actions, Action startCallback, Action exitCallBack, out IList stdout) + public override int Run(IBuildActions actions, Action startCallback, Action exitCallBack, out IList stdout) { var ret1 = s1.Run(actions, startCallback, exitCallBack, out var stdout1); var ret2 = (s2a != null ? s2a(stdout1, ret1) : s2b(ret1)).Run(actions, startCallback, exitCallBack, out var stdout2); @@ -171,10 +174,11 @@ namespace Semmle.Autobuild /// Creates a simple build script that runs the specified exe. /// /// The arguments to the executable, or null. + /// Whether the executable should run silently. /// The working directory (null for current directory). /// Additional environment variables. - public static BuildScript Create(string exe, string argumentsOpt, string workingDirectory, IDictionary environment) => - new BuildCommand(exe, argumentsOpt, workingDirectory, environment); + public static BuildScript Create(string exe, string argumentsOpt, bool silent, string workingDirectory, IDictionary environment) => + new BuildCommand(exe, argumentsOpt, silent, workingDirectory, environment); /// /// Creates a simple build script that runs the specified function. diff --git a/csharp/autobuilder/Semmle.Autobuild/CommandBuilder.cs b/csharp/autobuilder/Semmle.Autobuild/CommandBuilder.cs index 273cc52b036..7b95e495697 100644 --- a/csharp/autobuilder/Semmle.Autobuild/CommandBuilder.cs +++ b/csharp/autobuilder/Semmle.Autobuild/CommandBuilder.cs @@ -17,13 +17,15 @@ namespace Semmle.Autobuild readonly EscapeMode escapingMode; readonly string workingDirectory; readonly IDictionary environment; + readonly bool silent; /// /// Initializes a new instance of the class. /// /// The working directory (null for current directory). /// Additional environment variables. - public CommandBuilder(IBuildActions actions, string workingDirectory = null, IDictionary environment = null) + /// Whether this command should be run silently. + public CommandBuilder(IBuildActions actions, string workingDirectory = null, IDictionary environment = null, bool silent = false) { arguments = new StringBuilder(); if (actions.IsWindows()) @@ -40,6 +42,7 @@ namespace Semmle.Autobuild firstCommand = true; this.workingDirectory = workingDirectory; this.environment = environment; + this.silent = silent; } void OdasaIndex(string odasa) @@ -190,6 +193,6 @@ namespace Semmle.Autobuild /// /// Returns a build script that contains just this command. /// - public BuildScript Script => BuildScript.Create(executable, arguments.ToString(), workingDirectory, environment); + public BuildScript Script => BuildScript.Create(executable, arguments.ToString(), silent, workingDirectory, environment); } } diff --git a/csharp/autobuilder/Semmle.Autobuild/DotNetRule.cs b/csharp/autobuilder/Semmle.Autobuild/DotNetRule.cs index 789d68db30a..c9b33cef8d1 100644 --- a/csharp/autobuilder/Semmle.Autobuild/DotNetRule.cs +++ b/csharp/autobuilder/Semmle.Autobuild/DotNetRule.cs @@ -5,6 +5,7 @@ using Newtonsoft.Json.Linq; using System.Collections.Generic; using System.IO; using Semmle.Util; +using System.Text.RegularExpressions; namespace Semmle.Autobuild { @@ -34,22 +35,20 @@ namespace Semmle.Autobuild builder.Log(Severity.Info, "Attempting to build using .NET Core"); } - return WithDotNet(builder, dotNet => + return WithDotNet(builder, (dotNetPath, environment, compatibleClr) => { - var ret = GetInfoCommand(builder.Actions, dotNet); + var ret = GetInfoCommand(builder.Actions, dotNetPath, environment); foreach (var projectOrSolution in builder.ProjectsOrSolutionsToBuild) { - var cleanCommand = GetCleanCommand(builder.Actions, dotNet); + var cleanCommand = GetCleanCommand(builder.Actions, dotNetPath, environment); cleanCommand.QuoteArgument(projectOrSolution.FullPath); var clean = cleanCommand.Script; - var restoreCommand = GetRestoreCommand(builder.Actions, dotNet); + var restoreCommand = GetRestoreCommand(builder.Actions, dotNetPath, environment); restoreCommand.QuoteArgument(projectOrSolution.FullPath); var restore = restoreCommand.Script; - var buildCommand = GetBuildCommand(builder, dotNet); - buildCommand.QuoteArgument(projectOrSolution.FullPath); - var build = buildCommand.Script; + var build = GetBuildScript(builder, dotNetPath, environment, compatibleClr, projectOrSolution.FullPath); ret &= clean & BuildScript.Try(restore) & build; } @@ -57,38 +56,73 @@ namespace Semmle.Autobuild }); } - /// - /// Returns a script that attempts to download relevant version(s) of the - /// .NET Core SDK, followed by running the script generated by . - /// - /// The first element DotNetPath of the argument to - /// is the path where .NET Core was installed, and the second element Environment - /// is any additional required environment variables. The tuple argument is null - /// when the installation failed. - /// - public static BuildScript WithDotNet(Autobuilder builder, Func<(string DotNetPath, IDictionary Environment)?, BuildScript> f) + static BuildScript WithDotNet(Autobuilder builder, Func, bool, BuildScript> f) { var installDir = builder.Actions.PathCombine(builder.Options.RootDirectory, ".dotnet"); var installScript = DownloadDotNet(builder, installDir); return BuildScript.Bind(installScript, installed => + { + Dictionary env; + if (installed == 0) { - if (installed == 0) - { - // The installation succeeded, so use the newly installed .NET Core - var path = builder.Actions.GetEnvironmentVariable("PATH"); - var delim = builder.Actions.IsWindows() ? ";" : ":"; - var env = new Dictionary{ + // The installation succeeded, so use the newly installed .NET Core + var path = builder.Actions.GetEnvironmentVariable("PATH"); + var delim = builder.Actions.IsWindows() ? ";" : ":"; + env = new Dictionary{ { "DOTNET_MULTILEVEL_LOOKUP", "false" }, // prevent look up of other .NET Core SDKs { "DOTNET_SKIP_FIRST_TIME_EXPERIENCE", "true" }, { "PATH", installDir + delim + path } }; - return f((installDir, env)); + } + else + { + installDir = null; + env = null; + } + + // The CLR tracer is always compatible on Windows + if (builder.Actions.IsWindows()) + return f(installDir, env, true); + + // The CLR tracer is only compatible on .NET Core >= 3 on Linux and macOS (see + // https://github.com/dotnet/coreclr/issues/19622) + return BuildScript.Bind(GetInstalledRuntimesScript(builder.Actions, installDir, env), (runtimes, runtimesRet) => + { + var compatibleClr = false; + if (runtimesRet == 0) + { + var minimumVersion = new Version(3, 0); + var regex = new Regex(@"Microsoft\.NETCore\.App (\d\.\d\.\d)"); + compatibleClr = runtimes. + Select(runtime => regex.Match(runtime)). + Where(m => m.Success). + Select(m => m.Groups[1].Value). + Any(m => Version.TryParse(m, out var v) && v >= minimumVersion); } - return f(null); + if (!compatibleClr) + { + if (env == null) + env = new Dictionary(); + env.Add("UseSharedCompilation", "false"); + } + + return f(installDir, env, compatibleClr); }); + }); } + /// + /// Returns a script that attempts to download relevant version(s) of the + /// .NET Core SDK, followed by running the script generated by . + /// + /// The argument to is any additional required environment + /// variables needed by the installed .NET Core (null when no variables + /// are needed). + /// + public static BuildScript WithDotNet(Autobuilder builder, Func, BuildScript> f) + => WithDotNet(builder, (_1, env, _2) => f(env)); + /// /// Returns a script for downloading relevant versions of the /// .NET Core SDK. The SDK(s) will be installed at installDir @@ -225,7 +259,7 @@ Invoke-Command -ScriptBlock $ScriptBlock"; static BuildScript GetInstalledSdksScript(IBuildActions actions) { - var listSdks = new CommandBuilder(actions). + var listSdks = new CommandBuilder(actions, silent: true). RunCommand("dotnet"). Argument("--list-sdks"); return listSdks.Script; @@ -234,38 +268,62 @@ Invoke-Command -ScriptBlock $ScriptBlock"; static string DotNetCommand(IBuildActions actions, string dotNetPath) => dotNetPath != null ? actions.PathCombine(dotNetPath, "dotnet") : "dotnet"; - BuildScript GetInfoCommand(IBuildActions actions, (string DotNetPath, IDictionary Environment)? arg) + BuildScript GetInfoCommand(IBuildActions actions, string dotNetPath, IDictionary environment) { - var info = new CommandBuilder(actions, null, arg?.Environment). - RunCommand(DotNetCommand(actions, arg?.DotNetPath)). + var info = new CommandBuilder(actions, null, environment). + RunCommand(DotNetCommand(actions, dotNetPath)). Argument("--info"); return info.Script; } - CommandBuilder GetCleanCommand(IBuildActions actions, (string DotNetPath, IDictionary Environment)? arg) + CommandBuilder GetCleanCommand(IBuildActions actions, string dotNetPath, IDictionary environment) { - var clean = new CommandBuilder(actions, null, arg?.Environment). - RunCommand(DotNetCommand(actions, arg?.DotNetPath)). + var clean = new CommandBuilder(actions, null, environment). + RunCommand(DotNetCommand(actions, dotNetPath)). Argument("clean"); return clean; } - CommandBuilder GetRestoreCommand(IBuildActions actions, (string DotNetPath, IDictionary Environment)? arg) + CommandBuilder GetRestoreCommand(IBuildActions actions, string dotNetPath, IDictionary environment) { - var restore = new CommandBuilder(actions, null, arg?.Environment). - RunCommand(DotNetCommand(actions, arg?.DotNetPath)). + var restore = new CommandBuilder(actions, null, environment). + RunCommand(DotNetCommand(actions, dotNetPath)). Argument("restore"); return restore; } - CommandBuilder GetBuildCommand(Autobuilder builder, (string DotNetPath, IDictionary Environment)? arg) + static BuildScript GetInstalledRuntimesScript(IBuildActions actions, string dotNetPath, IDictionary environment) { - var build = new CommandBuilder(builder.Actions, null, arg?.Environment); - return builder.MaybeIndex(build, DotNetCommand(builder.Actions, arg?.DotNetPath)). + var listSdks = new CommandBuilder(actions, environment: environment, silent: true). + RunCommand(DotNetCommand(actions, dotNetPath)). + Argument("--list-runtimes"); + return listSdks.Script; + } + + /// + /// Gets the `dotnet build` script. + /// + /// The CLR tracer only works on .NET Core >= 3 on Linux and macOS (see + /// https://github.com/dotnet/coreclr/issues/19622), so in case we are + /// running on an older .NET Core, we disable shared compilation (and + /// hence the need for CLR tracing), by adding a + /// `/p:UseSharedCompilation=false` argument. + /// + BuildScript GetBuildScript(Autobuilder builder, string dotNetPath, IDictionary environment, bool compatibleClr, string projOrSln) + { + var build = new CommandBuilder(builder.Actions, null, environment); + var script = builder.MaybeIndex(build, DotNetCommand(builder.Actions, dotNetPath)). Argument("build"). - Argument("--no-incremental"). - Argument("/p:UseSharedCompilation=false"). - Argument(builder.Options.DotNetArguments); + Argument("--no-incremental"); + + return compatibleClr ? + script.Argument(builder.Options.DotNetArguments). + QuoteArgument(projOrSln). + Script : + script.Argument("/p:UseSharedCompilation=false"). + Argument(builder.Options.DotNetArguments). + QuoteArgument(projOrSln). + Script; } } }