Merge pull request #2278 from hvitved/csharp/autobuilder/shared-compilation

C#: Only set `UseSharedCompilation=false` in autobuilder when needed
This commit is contained in:
Calum Grant
2019-11-25 18:37:09 +00:00
committed by GitHub
8 changed files with 182 additions and 90 deletions

View File

@@ -185,7 +185,7 @@ namespace Semmle.Extraction.Tests
// Records the arguments passed to StartCallback.
IList<string> StartCallbackIn = new List<string>();
void StartCallback(string s)
void StartCallback(string s, bool silent)
{
StartCallbackIn.Add(s);
}
@@ -194,7 +194,7 @@ namespace Semmle.Extraction.Tests
IList<string> EndCallbackIn = new List<string>();
IList<int> EndCallbackReturn = new List<int>();
void EndCallback(int ret, string s)
void EndCallback(int ret, string s, bool silent)
{
EndCallbackReturn.Add(ret);
EndCallbackIn.Add(s);
@@ -203,7 +203,7 @@ namespace Semmle.Extraction.Tests
[Fact]
public void TestBuildCommand()
{
var cmd = BuildScript.Create("abc", "def ghi", null, null);
var cmd = BuildScript.Create("abc", "def ghi", false, null, null);
Actions.RunProcess["abc def ghi"] = 1;
cmd.Run(Actions, StartCallback, EndCallback);
@@ -216,7 +216,7 @@ namespace Semmle.Extraction.Tests
[Fact]
public void TestAnd1()
{
var cmd = BuildScript.Create("abc", "def ghi", null, null) & BuildScript.Create("odasa", null, null, null);
var cmd = BuildScript.Create("abc", "def ghi", false, null, null) & BuildScript.Create("odasa", null, false, null, null);
Actions.RunProcess["abc def ghi"] = 1;
cmd.Run(Actions, StartCallback, EndCallback);
@@ -230,7 +230,7 @@ namespace Semmle.Extraction.Tests
[Fact]
public void TestAnd2()
{
var cmd = BuildScript.Create("odasa", null, null, null) & BuildScript.Create("abc", "def ghi", null, null);
var cmd = BuildScript.Create("odasa", null, false, null, null) & BuildScript.Create("abc", "def ghi", false, null, null);
Actions.RunProcess["abc def ghi"] = 1;
Actions.RunProcess["odasa "] = 0;
@@ -250,7 +250,7 @@ namespace Semmle.Extraction.Tests
[Fact]
public void TestOr1()
{
var cmd = BuildScript.Create("odasa", null, null, null) | BuildScript.Create("abc", "def ghi", null, null);
var cmd = BuildScript.Create("odasa", null, false, null, null) | BuildScript.Create("abc", "def ghi", false, null, null);
Actions.RunProcess["abc def ghi"] = 1;
Actions.RunProcess["odasa "] = 0;
@@ -266,7 +266,7 @@ namespace Semmle.Extraction.Tests
[Fact]
public void TestOr2()
{
var cmd = BuildScript.Create("abc", "def ghi", null, null) | BuildScript.Create("odasa", null, null, null);
var cmd = BuildScript.Create("abc", "def ghi", false, null, null) | BuildScript.Create("odasa", null, false, null, null);
Actions.RunProcess["abc def ghi"] = 1;
Actions.RunProcess["odasa "] = 0;
@@ -375,7 +375,7 @@ namespace Semmle.Extraction.Tests
Actions.RunProcess["cmd.exe /C dotnet --info"] = 0;
Actions.RunProcess["cmd.exe /C dotnet clean test.csproj"] = 0;
Actions.RunProcess["cmd.exe /C dotnet restore test.csproj"] = 0;
Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --auto dotnet build --no-incremental /p:UseSharedCompilation=false test.csproj"] = 0;
Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --auto dotnet build --no-incremental test.csproj"] = 0;
Actions.RunProcess[@"cmd.exe /C C:\codeql\tools\java\bin\java -jar C:\codeql\csharp\tools\extractor-asp.jar ."] = 0;
Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --xml --extensions config csproj props xml"] = 0;
Actions.FileExists["csharp.log"] = true;
@@ -401,6 +401,9 @@ namespace Semmle.Extraction.Tests
[Fact]
public void TestLinuxCSharpAutoBuilder()
{
Actions.RunProcess["dotnet --list-runtimes"] = 0;
Actions.RunProcessOut["dotnet --list-runtimes"] = @"Microsoft.AspNetCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.AspNetCore.App]
Microsoft.NETCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.NETCore.App]";
Actions.RunProcess["dotnet --info"] = 0;
Actions.RunProcess["dotnet clean test.csproj"] = 0;
Actions.RunProcess["dotnet restore test.csproj"] = 0;
@@ -424,7 +427,7 @@ namespace Semmle.Extraction.Tests
Actions.LoadXml["test.csproj"] = xml;
var autobuilder = CreateAutoBuilder("csharp", false);
TestAutobuilderScript(autobuilder, 0, 6);
TestAutobuilderScript(autobuilder, 0, 7);
}
[Fact]
@@ -598,6 +601,8 @@ namespace Semmle.Extraction.Tests
[Fact]
public void TestLinuxBuildCommand()
{
Actions.RunProcess["dotnet --list-runtimes"] = 1;
Actions.RunProcessOut["dotnet --list-runtimes"] = "";
Actions.RunProcess[@"C:\odasa/tools/odasa index --auto ""./build.sh --skip-tests"""] = 0;
Actions.RunProcess[@"C:\codeql\tools\java/bin/java -jar C:\codeql\csharp/tools/extractor-asp.jar ."] = 0;
Actions.RunProcess[@"C:\odasa/tools/odasa index --xml --extensions config csproj props xml"] = 0;
@@ -610,7 +615,7 @@ namespace Semmle.Extraction.Tests
SkipVsWhere();
var autobuilder = CreateAutoBuilder("csharp", false, buildCommand: "./build.sh --skip-tests");
TestAutobuilderScript(autobuilder, 0, 3);
TestAutobuilderScript(autobuilder, 0, 4);
}
[Fact]
@@ -621,6 +626,8 @@ namespace Semmle.Extraction.Tests
Actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
Actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
Actions.RunProcess["/bin/chmod u+x build/build.sh"] = 0;
Actions.RunProcess["dotnet --list-runtimes"] = 1;
Actions.RunProcessOut["dotnet --list-runtimes"] = "";
Actions.RunProcess[@"C:\odasa/tools/odasa index --auto build/build.sh"] = 0;
Actions.RunProcessWorkingDirectory[@"C:\odasa/tools/odasa index --auto build/build.sh"] = "build";
Actions.RunProcess[@"C:\codeql\tools\java/bin/java -jar C:\codeql\csharp/tools/extractor-asp.jar ."] = 0;
@@ -628,7 +635,7 @@ namespace Semmle.Extraction.Tests
Actions.FileExists["csharp.log"] = true;
var autobuilder = CreateAutoBuilder("csharp", false);
TestAutobuilderScript(autobuilder, 0, 4);
TestAutobuilderScript(autobuilder, 0, 5);
}
[Fact]
@@ -640,12 +647,14 @@ namespace Semmle.Extraction.Tests
Actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
Actions.RunProcess["/bin/chmod u+x build.sh"] = 0;
Actions.RunProcess["dotnet --list-runtimes"] = 1;
Actions.RunProcessOut["dotnet --list-runtimes"] = "";
Actions.RunProcess[@"C:\odasa/tools/odasa index --auto build.sh"] = 0;
Actions.RunProcessWorkingDirectory[@"C:\odasa/tools/odasa index --auto build.sh"] = "";
Actions.FileExists["csharp.log"] = false;
var autobuilder = CreateAutoBuilder("csharp", false);
TestAutobuilderScript(autobuilder, 1, 2);
TestAutobuilderScript(autobuilder, 1, 3);
}
[Fact]
@@ -657,12 +666,14 @@ namespace Semmle.Extraction.Tests
Actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
Actions.RunProcess["/bin/chmod u+x build.sh"] = 0;
Actions.RunProcess["dotnet --list-runtimes"] = 1;
Actions.RunProcessOut["dotnet --list-runtimes"] = "";
Actions.RunProcess[@"C:\odasa/tools/odasa index --auto build.sh"] = 5;
Actions.RunProcessWorkingDirectory[@"C:\odasa/tools/odasa index --auto build.sh"] = "";
Actions.FileExists["csharp.log"] = true;
var autobuilder = CreateAutoBuilder("csharp", false);
TestAutobuilderScript(autobuilder, 1, 2);
TestAutobuilderScript(autobuilder, 1, 3);
}
[Fact]
@@ -748,7 +759,7 @@ namespace Semmle.Extraction.Tests
TestAutobuilderScript(autobuilder, 0, 6);
}
[Fact]
[Fact]
public void TestWindowCSharpMsBuildMultipleSolutions()
{
Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\csharp\nuget\nuget.exe restore test1.csproj"] = 0;
@@ -871,6 +882,9 @@ namespace Semmle.Extraction.Tests
[Fact]
public void TestSkipNugetDotnet()
{
Actions.RunProcess["dotnet --list-runtimes"] = 0;
Actions.RunProcessOut["dotnet --list-runtimes"] = @"Microsoft.AspNetCore.App 2.1.3 [/usr/local/share/dotnet/shared/Microsoft.AspNetCore.App]
Microsoft.NETCore.App 2.1.3 [/usr/local/share/dotnet/shared/Microsoft.NETCore.App]";
Actions.RunProcess["dotnet --info"] = 0;
Actions.RunProcess["dotnet clean test.csproj"] = 0;
Actions.RunProcess["dotnet restore test.csproj"] = 0;
@@ -894,7 +908,7 @@ namespace Semmle.Extraction.Tests
Actions.LoadXml["test.csproj"] = xml;
var autobuilder = CreateAutoBuilder("csharp", false, dotnetArguments: "--no-restore"); // nugetRestore=false does not work for now.
TestAutobuilderScript(autobuilder, 0, 6);
TestAutobuilderScript(autobuilder, 0, 7);
}
[Fact]
@@ -906,10 +920,13 @@ namespace Semmle.Extraction.Tests
Actions.RunProcess[@"chmod u+x dotnet-install.sh"] = 0;
Actions.RunProcess[@"./dotnet-install.sh --channel release --version 2.1.3 --install-dir C:\Project/.dotnet"] = 0;
Actions.RunProcess[@"rm dotnet-install.sh"] = 0;
Actions.RunProcess[@"C:\Project/.dotnet/dotnet --list-runtimes"] = 0;
Actions.RunProcessOut[@"C:\Project/.dotnet/dotnet --list-runtimes"] = @"Microsoft.AspNetCore.App 3.0.0 [/usr/local/share/dotnet/shared/Microsoft.AspNetCore.App]
Microsoft.NETCore.App 3.0.0 [/usr/local/share/dotnet/shared/Microsoft.NETCore.App]";
Actions.RunProcess[@"C:\Project/.dotnet/dotnet --info"] = 0;
Actions.RunProcess[@"C:\Project/.dotnet/dotnet clean test.csproj"] = 0;
Actions.RunProcess[@"C:\Project/.dotnet/dotnet restore test.csproj"] = 0;
Actions.RunProcess[@"C:\odasa/tools/odasa index --auto C:\Project/.dotnet/dotnet build --no-incremental /p:UseSharedCompilation=false test.csproj"] = 0;
Actions.RunProcess[@"C:\odasa/tools/odasa index --auto C:\Project/.dotnet/dotnet build --no-incremental test.csproj"] = 0;
Actions.RunProcess[@"C:\codeql\tools\java/bin/java -jar C:\codeql\csharp/tools/extractor-asp.jar ."] = 0;
Actions.RunProcess[@"C:\odasa/tools/odasa index --xml --extensions config csproj props xml"] = 0;
Actions.FileExists["csharp.log"] = true;
@@ -930,18 +947,24 @@ namespace Semmle.Extraction.Tests
Actions.LoadXml["test.csproj"] = xml;
var autobuilder = CreateAutoBuilder("csharp", false, dotnetVersion: "2.1.3");
TestAutobuilderScript(autobuilder, 0, 11);
TestAutobuilderScript(autobuilder, 0, 12);
}
[Fact]
public void TestDotnetVersionAlreadyInstalled()
{
Actions.RunProcess["dotnet --list-sdks"] = 0;
Actions.RunProcessOut["dotnet --list-sdks"] = "2.1.3 [C:\\Program Files\\dotnet\\sdks]\n2.1.4 [C:\\Program Files\\dotnet\\sdks]";
Actions.RunProcessOut["dotnet --list-sdks"] = @"2.1.3 [C:\Program Files\dotnet\sdks]
2.1.4 [C:\Program Files\dotnet\sdks]";
Actions.RunProcess[@"curl -L -sO https://dot.net/v1/dotnet-install.sh"] = 0;
Actions.RunProcess[@"chmod u+x dotnet-install.sh"] = 0;
Actions.RunProcess[@"./dotnet-install.sh --channel release --version 2.1.3 --install-dir C:\Project/.dotnet"] = 0;
Actions.RunProcess[@"rm dotnet-install.sh"] = 0;
Actions.RunProcess[@"C:\Project/.dotnet/dotnet --list-runtimes"] = 0;
Actions.RunProcessOut[@"C:\Project/.dotnet/dotnet --list-runtimes"] = @"Microsoft.AspNetCore.App 2.1.3 [/usr/local/share/dotnet/shared/Microsoft.AspNetCore.App]
Microsoft.AspNetCore.App 2.1.4 [/usr/local/share/dotnet/shared/Microsoft.AspNetCore.App]
Microsoft.NETCore.App 2.1.3 [/usr/local/share/dotnet/shared/Microsoft.NETCore.App]
Microsoft.NETCore.App 2.1.4 [/usr/local/share/dotnet/shared/Microsoft.NETCore.App]";
Actions.RunProcess[@"C:\Project/.dotnet/dotnet --info"] = 0;
Actions.RunProcess[@"C:\Project/.dotnet/dotnet clean test.csproj"] = 0;
Actions.RunProcess[@"C:\Project/.dotnet/dotnet restore test.csproj"] = 0;
@@ -966,7 +989,7 @@ namespace Semmle.Extraction.Tests
Actions.LoadXml["test.csproj"] = xml;
var autobuilder = CreateAutoBuilder("csharp", false, dotnetVersion: "2.1.3");
TestAutobuilderScript(autobuilder, 0, 11);
TestAutobuilderScript(autobuilder, 0, 12);
}
[Fact]
@@ -979,7 +1002,7 @@ namespace Semmle.Extraction.Tests
Actions.RunProcess[@"cmd.exe /C C:\Project\.dotnet\dotnet --info"] = 0;
Actions.RunProcess[@"cmd.exe /C C:\Project\.dotnet\dotnet clean test.csproj"] = 0;
Actions.RunProcess[@"cmd.exe /C C:\Project\.dotnet\dotnet restore test.csproj"] = 0;
Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --auto C:\Project\.dotnet\dotnet build --no-incremental /p:UseSharedCompilation=false test.csproj"] = 0;
Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --auto C:\Project\.dotnet\dotnet build --no-incremental test.csproj"] = 0;
Actions.RunProcess[@"cmd.exe /C C:\codeql\tools\java\bin\java -jar C:\codeql\csharp\tools\extractor-asp.jar ."] = 0;
Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --xml --extensions config csproj props xml"] = 0;
Actions.FileExists["csharp.log"] = true;

View File

@@ -214,8 +214,16 @@ namespace Semmle.Autobuild
if (Options.IgnoreErrors)
script |= BuildScript.Success;
void startCallback(string s) => Log(Severity.Info, $"\nRunning {s}");
void exitCallback(int ret, string msg) => Log(Severity.Info, $"Exit code {ret}{(string.IsNullOrEmpty(msg) ? "" : $": {msg}")}");
void startCallback(string s, bool silent)
{
Log(silent ? Severity.Debug : Severity.Info, $"\nRunning {s}");
}
void exitCallback(int ret, string msg, bool silent)
{
Log(silent ? Severity.Debug : Severity.Info, $"Exit code {ret}{(string.IsNullOrEmpty(msg) ? "" : $": {msg}")}");
}
return script.Run(Actions, startCallback, exitCallback);
}

View File

@@ -141,12 +141,8 @@ namespace Semmle.Autobuild
pi.WorkingDirectory = workingDirectory;
// Environment variables can only be used when not redirecting stdout
if (!redirectStandardOutput)
{
pi.Environment["UseSharedCompilation"] = "false";
if (environment != null)
environment.ForEach(kvp => pi.Environment[kvp.Key] = kvp.Value);
}
if (!redirectStandardOutput && environment != null)
environment.ForEach(kvp => pi.Environment[kvp.Key] = kvp.Value);
return pi;
}

View File

@@ -43,9 +43,9 @@ namespace Semmle.Autobuild
var dir = Path.GetDirectoryName(scriptPath);
// A specific .NET Core version may be required
return chmodScript & DotNetRule.WithDotNet(builder, dotNet =>
return chmodScript & DotNetRule.WithDotNet(builder, environment =>
{
var command = new CommandBuilder(builder.Actions, dir, dotNet?.Environment);
var command = new CommandBuilder(builder.Actions, dir, environment);
// A specific Visual Studio version may be required
var vsTools = MsBuildRule.GetVcVarsBatFile(builder);

View File

@@ -11,9 +11,9 @@
return BuildScript.Failure;
// Custom build commands may require a specific .NET Core version
return DotNetRule.WithDotNet(builder, dotNet =>
return DotNetRule.WithDotNet(builder, environment =>
{
var command = new CommandBuilder(builder.Actions, null, dotNet?.Environment);
var command = new CommandBuilder(builder.Actions, null, environment);
// Custom build commands may require a specific Visual Studio version
var vsTools = MsBuildRule.GetVcVarsBatFile(builder);

View File

@@ -25,7 +25,7 @@ namespace Semmle.Autobuild
/// an exit message.
/// </param>
/// <returns>The exit code from this build script.</returns>
public abstract int Run(IBuildActions actions, Action<string> startCallback, Action<int, string> exitCallBack);
public abstract int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack);
/// <summary>
/// Run this build command.
@@ -44,33 +44,36 @@ namespace Semmle.Autobuild
/// </param>
/// <param name="stdout">Contents of standard out.</param>
/// <returns>The exit code from this build script.</returns>
public abstract int Run(IBuildActions actions, Action<string> startCallback, Action<int, string> exitCallBack, out IList<string> stdout);
public abstract int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, out IList<string> stdout);
class BuildCommand : BuildScript
{
readonly string exe, arguments, workingDirectory;
readonly IDictionary<string, string> environment;
readonly bool silent;
/// <summary>
/// Create a simple build command.
/// </summary>
/// <param name="exe">The executable to run.</param>
/// <param name="argumentsOpt">The arguments to the executable, or null.</param>
/// <param name="silent">Whether this command should run silently.</param>
/// <param name="workingDirectory">The working directory (<code>null</code> for current directory).</param>
/// <param name="environment">Additional environment variables.</param>
public BuildCommand(string exe, string argumentsOpt, string workingDirectory = null, IDictionary<string, string> environment = null)
public BuildCommand(string exe, string argumentsOpt, bool silent, string workingDirectory = null, IDictionary<string, string> environment = null)
{
this.exe = exe;
this.arguments = argumentsOpt ?? "";
this.silent = silent;
this.workingDirectory = workingDirectory;
this.environment = environment;
}
public override string ToString() => exe + " " + arguments;
public override int Run(IBuildActions actions, Action<string> startCallback, Action<int, string> exitCallBack)
public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack)
{
startCallback(this.ToString());
startCallback(this.ToString(), silent);
var ret = 1;
var retMessage = "";
try
@@ -83,13 +86,13 @@ namespace Semmle.Autobuild
retMessage = ex.Message;
}
exitCallBack(ret, retMessage);
exitCallBack(ret, retMessage, silent);
return ret;
}
public override int Run(IBuildActions actions, Action<string> startCallback, Action<int, string> exitCallBack, out IList<string> stdout)
public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, out IList<string> stdout)
{
startCallback(this.ToString());
startCallback(this.ToString(), silent);
var ret = 1;
var retMessage = "";
try
@@ -102,7 +105,7 @@ namespace Semmle.Autobuild
retMessage = ex.Message;
stdout = new string[0];
}
exitCallBack(ret, retMessage);
exitCallBack(ret, retMessage, silent);
return ret;
}
@@ -116,9 +119,9 @@ namespace Semmle.Autobuild
this.func = func;
}
public override int Run(IBuildActions actions, Action<string> startCallback, Action<int, string> exitCallBack) => func(actions);
public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack) => func(actions);
public override int Run(IBuildActions actions, Action<string> startCallback, Action<int, string> exitCallBack, out IList<string> stdout)
public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, out IList<string> stdout)
{
stdout = new string[0];
return func(actions);
@@ -142,7 +145,7 @@ namespace Semmle.Autobuild
this.s2b = s2;
}
public override int Run(IBuildActions actions, Action<string> startCallback, Action<int, string> exitCallBack)
public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack)
{
int ret1;
if (s2a != null)
@@ -155,7 +158,7 @@ namespace Semmle.Autobuild
return s2b(ret1).Run(actions, startCallback, exitCallBack);
}
public override int Run(IBuildActions actions, Action<string> startCallback, Action<int, string> exitCallBack, out IList<string> stdout)
public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, out IList<string> stdout)
{
var ret1 = s1.Run(actions, startCallback, exitCallBack, out var stdout1);
var ret2 = (s2a != null ? s2a(stdout1, ret1) : s2b(ret1)).Run(actions, startCallback, exitCallBack, out var stdout2);
@@ -171,10 +174,11 @@ namespace Semmle.Autobuild
/// Creates a simple build script that runs the specified exe.
/// </summary>
/// <param name="argumentsOpt">The arguments to the executable, or null.</param>
/// <param name="silent">Whether the executable should run silently.</param>
/// <param name="workingDirectory">The working directory (<code>null</code> for current directory).</param>
/// <param name="environment">Additional environment variables.</param>
public static BuildScript Create(string exe, string argumentsOpt, string workingDirectory, IDictionary<string, string> environment) =>
new BuildCommand(exe, argumentsOpt, workingDirectory, environment);
public static BuildScript Create(string exe, string argumentsOpt, bool silent, string workingDirectory, IDictionary<string, string> environment) =>
new BuildCommand(exe, argumentsOpt, silent, workingDirectory, environment);
/// <summary>
/// Creates a simple build script that runs the specified function.

View File

@@ -17,13 +17,15 @@ namespace Semmle.Autobuild
readonly EscapeMode escapingMode;
readonly string workingDirectory;
readonly IDictionary<string, string> environment;
readonly bool silent;
/// <summary>
/// Initializes a new instance of the <see cref="T:Semmle.Autobuild.CommandBuilder"/> class.
/// </summary>
/// <param name="workingDirectory">The working directory (<code>null</code> for current directory).</param>
/// <param name="environment">Additional environment variables.</param>
public CommandBuilder(IBuildActions actions, string workingDirectory = null, IDictionary<string, string> environment = null)
/// <param name="silent">Whether this command should be run silently.</param>
public CommandBuilder(IBuildActions actions, string workingDirectory = null, IDictionary<string, string> environment = null, bool silent = false)
{
arguments = new StringBuilder();
if (actions.IsWindows())
@@ -40,6 +42,7 @@ namespace Semmle.Autobuild
firstCommand = true;
this.workingDirectory = workingDirectory;
this.environment = environment;
this.silent = silent;
}
void OdasaIndex(string odasa)
@@ -190,6 +193,6 @@ namespace Semmle.Autobuild
/// <summary>
/// Returns a build script that contains just this command.
/// </summary>
public BuildScript Script => BuildScript.Create(executable, arguments.ToString(), workingDirectory, environment);
public BuildScript Script => BuildScript.Create(executable, arguments.ToString(), silent, workingDirectory, environment);
}
}

View File

@@ -5,6 +5,7 @@ using Newtonsoft.Json.Linq;
using System.Collections.Generic;
using System.IO;
using Semmle.Util;
using System.Text.RegularExpressions;
namespace Semmle.Autobuild
{
@@ -34,22 +35,20 @@ namespace Semmle.Autobuild
builder.Log(Severity.Info, "Attempting to build using .NET Core");
}
return WithDotNet(builder, dotNet =>
return WithDotNet(builder, (dotNetPath, environment, compatibleClr) =>
{
var ret = GetInfoCommand(builder.Actions, dotNet);
var ret = GetInfoCommand(builder.Actions, dotNetPath, environment);
foreach (var projectOrSolution in builder.ProjectsOrSolutionsToBuild)
{
var cleanCommand = GetCleanCommand(builder.Actions, dotNet);
var cleanCommand = GetCleanCommand(builder.Actions, dotNetPath, environment);
cleanCommand.QuoteArgument(projectOrSolution.FullPath);
var clean = cleanCommand.Script;
var restoreCommand = GetRestoreCommand(builder.Actions, dotNet);
var restoreCommand = GetRestoreCommand(builder.Actions, dotNetPath, environment);
restoreCommand.QuoteArgument(projectOrSolution.FullPath);
var restore = restoreCommand.Script;
var buildCommand = GetBuildCommand(builder, dotNet);
buildCommand.QuoteArgument(projectOrSolution.FullPath);
var build = buildCommand.Script;
var build = GetBuildScript(builder, dotNetPath, environment, compatibleClr, projectOrSolution.FullPath);
ret &= clean & BuildScript.Try(restore) & build;
}
@@ -57,38 +56,73 @@ namespace Semmle.Autobuild
});
}
/// <summary>
/// Returns a script that attempts to download relevant version(s) of the
/// .NET Core SDK, followed by running the script generated by <paramref name="f"/>.
///
/// The first element <code>DotNetPath</code> of the argument to <paramref name="f"/>
/// is the path where .NET Core was installed, and the second element <code>Environment</code>
/// is any additional required environment variables. The tuple argument is <code>null</code>
/// when the installation failed.
/// </summary>
public static BuildScript WithDotNet(Autobuilder builder, Func<(string DotNetPath, IDictionary<string, string> Environment)?, BuildScript> f)
static BuildScript WithDotNet(Autobuilder builder, Func<string, IDictionary<string, string>, bool, BuildScript> f)
{
var installDir = builder.Actions.PathCombine(builder.Options.RootDirectory, ".dotnet");
var installScript = DownloadDotNet(builder, installDir);
return BuildScript.Bind(installScript, installed =>
{
Dictionary<string, string> env;
if (installed == 0)
{
if (installed == 0)
{
// The installation succeeded, so use the newly installed .NET Core
var path = builder.Actions.GetEnvironmentVariable("PATH");
var delim = builder.Actions.IsWindows() ? ";" : ":";
var env = new Dictionary<string, string>{
// The installation succeeded, so use the newly installed .NET Core
var path = builder.Actions.GetEnvironmentVariable("PATH");
var delim = builder.Actions.IsWindows() ? ";" : ":";
env = new Dictionary<string, string>{
{ "DOTNET_MULTILEVEL_LOOKUP", "false" }, // prevent look up of other .NET Core SDKs
{ "DOTNET_SKIP_FIRST_TIME_EXPERIENCE", "true" },
{ "PATH", installDir + delim + path }
};
return f((installDir, env));
}
else
{
installDir = null;
env = null;
}
// The CLR tracer is always compatible on Windows
if (builder.Actions.IsWindows())
return f(installDir, env, true);
// The CLR tracer is only compatible on .NET Core >= 3 on Linux and macOS (see
// https://github.com/dotnet/coreclr/issues/19622)
return BuildScript.Bind(GetInstalledRuntimesScript(builder.Actions, installDir, env), (runtimes, runtimesRet) =>
{
var compatibleClr = false;
if (runtimesRet == 0)
{
var minimumVersion = new Version(3, 0);
var regex = new Regex(@"Microsoft\.NETCore\.App (\d\.\d\.\d)");
compatibleClr = runtimes.
Select(runtime => regex.Match(runtime)).
Where(m => m.Success).
Select(m => m.Groups[1].Value).
Any(m => Version.TryParse(m, out var v) && v >= minimumVersion);
}
return f(null);
if (!compatibleClr)
{
if (env == null)
env = new Dictionary<string, string>();
env.Add("UseSharedCompilation", "false");
}
return f(installDir, env, compatibleClr);
});
});
}
/// <summary>
/// Returns a script that attempts to download relevant version(s) of the
/// .NET Core SDK, followed by running the script generated by <paramref name="f"/>.
///
/// The argument to <paramref name="f"/> is any additional required environment
/// variables needed by the installed .NET Core (<code>null</code> when no variables
/// are needed).
/// </summary>
public static BuildScript WithDotNet(Autobuilder builder, Func<IDictionary<string, string>, BuildScript> f)
=> WithDotNet(builder, (_1, env, _2) => f(env));
/// <summary>
/// Returns a script for downloading relevant versions of the
/// .NET Core SDK. The SDK(s) will be installed at <code>installDir</code>
@@ -225,7 +259,7 @@ Invoke-Command -ScriptBlock $ScriptBlock";
static BuildScript GetInstalledSdksScript(IBuildActions actions)
{
var listSdks = new CommandBuilder(actions).
var listSdks = new CommandBuilder(actions, silent: true).
RunCommand("dotnet").
Argument("--list-sdks");
return listSdks.Script;
@@ -234,38 +268,62 @@ Invoke-Command -ScriptBlock $ScriptBlock";
static string DotNetCommand(IBuildActions actions, string dotNetPath) =>
dotNetPath != null ? actions.PathCombine(dotNetPath, "dotnet") : "dotnet";
BuildScript GetInfoCommand(IBuildActions actions, (string DotNetPath, IDictionary<string, string> Environment)? arg)
BuildScript GetInfoCommand(IBuildActions actions, string dotNetPath, IDictionary<string, string> environment)
{
var info = new CommandBuilder(actions, null, arg?.Environment).
RunCommand(DotNetCommand(actions, arg?.DotNetPath)).
var info = new CommandBuilder(actions, null, environment).
RunCommand(DotNetCommand(actions, dotNetPath)).
Argument("--info");
return info.Script;
}
CommandBuilder GetCleanCommand(IBuildActions actions, (string DotNetPath, IDictionary<string, string> Environment)? arg)
CommandBuilder GetCleanCommand(IBuildActions actions, string dotNetPath, IDictionary<string, string> environment)
{
var clean = new CommandBuilder(actions, null, arg?.Environment).
RunCommand(DotNetCommand(actions, arg?.DotNetPath)).
var clean = new CommandBuilder(actions, null, environment).
RunCommand(DotNetCommand(actions, dotNetPath)).
Argument("clean");
return clean;
}
CommandBuilder GetRestoreCommand(IBuildActions actions, (string DotNetPath, IDictionary<string, string> Environment)? arg)
CommandBuilder GetRestoreCommand(IBuildActions actions, string dotNetPath, IDictionary<string, string> environment)
{
var restore = new CommandBuilder(actions, null, arg?.Environment).
RunCommand(DotNetCommand(actions, arg?.DotNetPath)).
var restore = new CommandBuilder(actions, null, environment).
RunCommand(DotNetCommand(actions, dotNetPath)).
Argument("restore");
return restore;
}
CommandBuilder GetBuildCommand(Autobuilder builder, (string DotNetPath, IDictionary<string, string> Environment)? arg)
static BuildScript GetInstalledRuntimesScript(IBuildActions actions, string dotNetPath, IDictionary<string, string> environment)
{
var build = new CommandBuilder(builder.Actions, null, arg?.Environment);
return builder.MaybeIndex(build, DotNetCommand(builder.Actions, arg?.DotNetPath)).
var listSdks = new CommandBuilder(actions, environment: environment, silent: true).
RunCommand(DotNetCommand(actions, dotNetPath)).
Argument("--list-runtimes");
return listSdks.Script;
}
/// <summary>
/// Gets the `dotnet build` script.
///
/// The CLR tracer only works on .NET Core >= 3 on Linux and macOS (see
/// https://github.com/dotnet/coreclr/issues/19622), so in case we are
/// running on an older .NET Core, we disable shared compilation (and
/// hence the need for CLR tracing), by adding a
/// `/p:UseSharedCompilation=false` argument.
/// </summary>
BuildScript GetBuildScript(Autobuilder builder, string dotNetPath, IDictionary<string, string> environment, bool compatibleClr, string projOrSln)
{
var build = new CommandBuilder(builder.Actions, null, environment);
var script = builder.MaybeIndex(build, DotNetCommand(builder.Actions, dotNetPath)).
Argument("build").
Argument("--no-incremental").
Argument("/p:UseSharedCompilation=false").
Argument(builder.Options.DotNetArguments);
Argument("--no-incremental");
return compatibleClr ?
script.Argument(builder.Options.DotNetArguments).
QuoteArgument(projOrSln).
Script :
script.Argument("/p:UseSharedCompilation=false").
Argument(builder.Options.DotNetArguments).
QuoteArgument(projOrSln).
Script;
}
}
}