hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-16 16:53:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

C#: Remove support for legacy LGTM options in autobuilder

Browse Source
This commit is contained in:
Tom Hvitved
2024-03-22 10:08:04 +01:00
parent eca6c00003
commit 9d124197e8
15 changed files with 58 additions and 503 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/BuildScripts.cs
View File

@@ -250,12 +250,7 @@ namespace Semmle.Autobuild.Cpp.Tests
EndCallbackIn.Add(s);
}
CppAutobuilder CreateAutoBuilder(bool isWindows,
string? buildless = null, string? solution = null, string? buildCommand = null, string? ignoreErrors = null,
string? msBuildArguments = null, string? msBuildPlatform = null, string? msBuildConfiguration = null, string? msBuildTarget = null,
string? dotnetArguments = null, string? dotnetVersion = null, string? vsToolsVersion = null,
string? nugetRestore = null, string? allSolutions = null,
string cwd = @"C:\Project")
CppAutobuilder CreateAutoBuilder(bool isWindows, string? dotnetVersion = null, string cwd = @"C:\Project")
{
string codeqlUpperLanguage = Language.Cpp.UpperCaseName;
Actions.GetEnvironmentVariable[$"CODEQL_AUTOBUILDER_{codeqlUpperLanguage}_NO_INDEXING"] = "false";
@@ -265,22 +260,7 @@ namespace Semmle.Autobuild.Cpp.Tests
Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_DIAGNOSTIC_DIR"] = "";
Actions.GetEnvironmentVariable["CODEQL_JAVA_HOME"] = @"C:\codeql\tools\java";
Actions.GetEnvironmentVariable["CODEQL_PLATFORM"] = "win64";
Actions.GetEnvironmentVariable["SEMMLE_DIST"] = @"C:\odasa";
Actions.GetEnvironmentVariable["SEMMLE_JAVA_HOME"] = @"C:\odasa\tools\java";
Actions.GetEnvironmentVariable["SEMMLE_PLATFORM_TOOLS"] = @"C:\odasa\tools";
Actions.GetEnvironmentVariable["LGTM_INDEX_VSTOOLS_VERSION"] = vsToolsVersion;
Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_ARGUMENTS"] = msBuildArguments;
Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_PLATFORM"] = msBuildPlatform;
Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_CONFIGURATION"] = msBuildConfiguration;
Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_TARGET"] = msBuildTarget;
Actions.GetEnvironmentVariable["LGTM_INDEX_DOTNET_ARGUMENTS"] = dotnetArguments;
Actions.GetEnvironmentVariable["LGTM_INDEX_DOTNET_VERSION"] = dotnetVersion;
Actions.GetEnvironmentVariable["LGTM_INDEX_BUILD_COMMAND"] = buildCommand;
Actions.GetEnvironmentVariable["LGTM_INDEX_SOLUTION"] = solution;
Actions.GetEnvironmentVariable["LGTM_INDEX_IGNORE_ERRORS"] = ignoreErrors;
Actions.GetEnvironmentVariable["LGTM_INDEX_BUILDLESS"] = buildless;
Actions.GetEnvironmentVariable["LGTM_INDEX_ALL_SOLUTIONS"] = allSolutions;
Actions.GetEnvironmentVariable["LGTM_INDEX_NUGET_RESTORE"] = nugetRestore;
Actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_OPTION_DOTNET_VERSION"] = dotnetVersion;
Actions.GetEnvironmentVariable["ProgramFiles(x86)"] = isWindows ? @"C:\Program Files (x86)" : null;
Actions.GetCurrentDirectory = cwd;
Actions.IsWindows = isWindows;

3
cpp/autobuilder/Semmle.Autobuild.Cpp/CppAutobuilder.cs
View File

@@ -26,9 +26,6 @@ namespace Semmle.Autobuild.Cpp
public override BuildScript GetBuildScript()
{
if (Options.BuildCommand != null)
return new BuildCommandRule((_, f) => f(null)).Analyse(this, false);
return
// First try MSBuild
new MsBuildRule().Analyse(this, true) |

252
csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs
View File

@@ -399,10 +399,8 @@ namespace Semmle.Autobuild.CSharp.Tests
}
private CSharpAutobuilder CreateAutoBuilder(bool isWindows,
string? buildless = null, string? solution = null, string? buildCommand = null, string? ignoreErrors = null,
string? msBuildArguments = null, string? msBuildPlatform = null, string? msBuildConfiguration = null, string? msBuildTarget = null,
string? dotnetArguments = null, string? dotnetVersion = null, string? vsToolsVersion = null,
string? nugetRestore = null, string? allSolutions = null,
string? buildless = null,
string? dotnetVersion = null,
string cwd = @"C:\Project")
{
var codeqlUpperLanguage = Language.CSharp.UpperCaseName;
@@ -412,20 +410,9 @@ namespace Semmle.Autobuild.CSharp.Tests
actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_DIAGNOSTIC_DIR"] = "";
actions.GetEnvironmentVariable["CODEQL_JAVA_HOME"] = @"C:\codeql\tools\java";
actions.GetEnvironmentVariable["CODEQL_PLATFORM"] = isWindows ? "win64" : "linux64";
actions.GetEnvironmentVariable["LGTM_INDEX_VSTOOLS_VERSION"] = vsToolsVersion;
actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_ARGUMENTS"] = msBuildArguments;
actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_PLATFORM"] = msBuildPlatform;
actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_CONFIGURATION"] = msBuildConfiguration;
actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_TARGET"] = msBuildTarget;
actions.GetEnvironmentVariable["LGTM_INDEX_DOTNET_ARGUMENTS"] = dotnetArguments;
actions.GetEnvironmentVariable["LGTM_INDEX_DOTNET_VERSION"] = dotnetVersion;
actions.GetEnvironmentVariable["LGTM_INDEX_BUILD_COMMAND"] = buildCommand;
actions.GetEnvironmentVariable["LGTM_INDEX_SOLUTION"] = solution;
actions.GetEnvironmentVariable["LGTM_INDEX_IGNORE_ERRORS"] = ignoreErrors;
actions.GetEnvironmentVariable["LGTM_INDEX_BUILDLESS"] = buildless;
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_OPTION_BUILDLESS"] = buildless;
actions.GetEnvironmentVariable["LGTM_INDEX_ALL_SOLUTIONS"] = allSolutions;
actions.GetEnvironmentVariable["LGTM_INDEX_NUGET_RESTORE"] = nugetRestore;
if (dotnetVersion is not null)
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_OPTION_DOTNET_VERSION"] = dotnetVersion;
actions.GetEnvironmentVariable["ProgramFiles(x86)"] = isWindows ? @"C:\Program Files (x86)" : null;
actions.GetCurrentDirectory = cwd;
actions.IsWindows = isWindows;
@@ -600,15 +587,6 @@ namespace Semmle.Autobuild.CSharp.Tests
TestAutobuilderScript(autobuilder, 0, 1);
}
private void SkipVsWhere()
{
actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = false;
actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = false;
actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat"] = false;
actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat"] = false;
actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat"] = false;
}
private void TestAutobuilderScript(CSharpAutobuilder autobuilder, int expectedOutput, int commandsRun)
{
Assert.Equal(expectedOutput, autobuilder.GetBuildScript().Run(actions, StartCallback, EndCallback));
@@ -628,23 +606,6 @@ namespace Semmle.Autobuild.CSharp.Tests
}
}
[Fact]
public void TestLinuxBuildCommand()
{
actions.RunProcess["./build.sh --skip-tests"] = 0;
actions.FileExists["csharp.log"] = true;
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
actions.EnumerateDirectories[@"C:\Project"] = "";
SkipVsWhere();
var autobuilder = CreateAutoBuilder(false, buildCommand: "./build.sh --skip-tests");
TestAutobuilderScript(autobuilder, 0, 1);
}
[Fact]
public void TestLinuxBuildSh()
{
@@ -714,177 +675,6 @@ namespace Semmle.Autobuild.CSharp.Tests
TestAutobuilderScript(autobuilder, 0, 1);
}
[Fact]
public void TestWindowsBuildBatIgnoreErrors()
{
actions.EnumerateFiles[@"C:\Project"] = "foo.cs\nbuild.bat";
actions.EnumerateDirectories[@"C:\Project"] = "";
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
actions.RunProcess[@"cmd.exe /C C:\Project\build.bat"] = 1;
actions.RunProcessWorkingDirectory[@"cmd.exe /C C:\Project\build.bat"] = @"C:\Project";
actions.RunProcess[@"cmd.exe /C C:\codeql\tools\java\bin\java -jar C:\codeql\csharp\tools\extractor-asp.jar ."] = 0;
actions.RunProcess[@"cmd.exe /C C:\codeql\tools\codeql index --xml --extensions config"] = 0;
actions.FileExists["csharp.log"] = true;
var autobuilder = CreateAutoBuilder(true, ignoreErrors: "true");
TestAutobuilderScript(autobuilder, 1, 1);
}
[Fact]
public void TestWindowsCmdIgnoreErrors()
{
actions.RunProcess["cmd.exe /C ^\"build.cmd^ --skip-tests^\""] = 3;
actions.RunProcess[@"cmd.exe /C C:\codeql\tools\java\bin\java -jar C:\codeql\csharp\tools\extractor-asp.jar ."] = 0;
actions.RunProcess[@"cmd.exe /C C:\codeql\tools\codeql index --xml --extensions config"] = 0;
actions.FileExists["csharp.log"] = true;
SkipVsWhere();
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
actions.EnumerateDirectories[@"C:\Project"] = "";
var autobuilder = CreateAutoBuilder(true, buildCommand: "build.cmd --skip-tests", ignoreErrors: "true");
TestAutobuilderScript(autobuilder, 3, 1);
}
[Fact]
public void TestWindowCSharpMsBuild()
{
actions.RunProcess[@"cmd.exe /C C:\Project\.nuget\nuget.exe restore C:\Project\test1.sln -DisableParallelProcessing"] = 0;
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program^ Files^ ^(x86^)\\Microsoft^ Visual^ Studio^ 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test1.sln /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /P:Fu=Bar"] = 0;
actions.RunProcess[@"cmd.exe /C C:\Project\.nuget\nuget.exe restore C:\Project\test2.sln -DisableParallelProcessing"] = 0;
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program^ Files^ ^(x86^)\\Microsoft^ Visual^ Studio^ 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test2.sln /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /P:Fu=Bar"] = 0;
actions.FileExists["csharp.log"] = true;
actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = false;
actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = false;
actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat"] = true;
actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat"] = false;
actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat"] = true;
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest1.cs\ntest2.cs";
actions.EnumerateFiles[@"C:\Project\.nuget"] = "nuget.exe";
actions.EnumerateDirectories[@"C:\Project"] = @".nuget";
actions.EnumerateDirectories[@"C:\Project\.nuget"] = "";
var autobuilder = CreateAutoBuilder(true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows", msBuildPlatform: "x86", msBuildConfiguration: "Debug",
vsToolsVersion: "12", allSolutions: "true");
var testSolution1 = new TestSolution(@"C:\Project\test1.sln");
var testSolution2 = new TestSolution(@"C:\Project\test2.sln");
autobuilder.ProjectsOrSolutionsToBuild.Add(testSolution1);
autobuilder.ProjectsOrSolutionsToBuild.Add(testSolution2);
TestAutobuilderScript(autobuilder, 0, 4);
}
[Fact]
public void TestWindowCSharpMsBuildMultipleSolutions()
{
actions.RunProcess[@"cmd.exe /C nuget restore C:\Project\test1.csproj -DisableParallelProcessing"] = 0;
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program^ Files^ ^(x86^)\\Microsoft^ Visual^ Studio^ 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test1.csproj /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /P:Fu=Bar"] = 0;
actions.RunProcess[@"cmd.exe /C nuget restore C:\Project\test2.csproj -DisableParallelProcessing"] = 0;
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program^ Files^ ^(x86^)\\Microsoft^ Visual^ Studio^ 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test2.csproj /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /P:Fu=Bar"] = 0;
actions.FileExists["csharp.log"] = true;
actions.FileExists[@"C:\Project\test1.csproj"] = true;
actions.FileExists[@"C:\Project\test2.csproj"] = true;
actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = false;
actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = false;
actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat"] = true;
actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat"] = false;
actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat"] = true;
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
actions.EnumerateFiles[@"C:\Project"] = "test1.csproj\ntest2.csproj\ntest1.cs\ntest2.cs";
actions.EnumerateDirectories[@"C:\Project"] = "";
var csproj1 = new XmlDocument();
csproj1.LoadXml(@"<?xml version=""1.0"" encoding=""utf - 8""?>
<Project ToolsVersion=""15.0"" xmlns=""http://schemas.microsoft.com/developer/msbuild/2003"">
<ItemGroup>
<Compile Include=""test1.cs"" />
</ItemGroup>
</Project>");
actions.LoadXml[@"C:\Project\test1.csproj"] = csproj1;
var csproj2 = new XmlDocument();
csproj2.LoadXml(@"<?xml version=""1.0"" encoding=""utf - 8""?>
<Project ToolsVersion=""15.0"" xmlns=""http://schemas.microsoft.com/developer/msbuild/2003"">
<ItemGroup>
<Compile Include=""test1.cs"" />
</ItemGroup>
</Project>");
actions.LoadXml[@"C:\Project\test2.csproj"] = csproj2;
var autobuilder = CreateAutoBuilder(true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows", msBuildPlatform: "x86", msBuildConfiguration: "Debug",
vsToolsVersion: "12");
TestAutobuilderScript(autobuilder, 0, 4);
}
[Fact]
public void TestWindowCSharpMsBuildFailed()
{
actions.RunProcess[@"cmd.exe /C nuget restore C:\Project\test1.sln -DisableParallelProcessing"] = 0;
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program^ Files^ ^(x86^)\\Microsoft^ Visual^ Studio^ 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test1.sln /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /P:Fu=Bar"] = 1;
actions.FileExists["csharp.log"] = true;
actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = false;
actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = false;
actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat"] = true;
actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat"] = false;
actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat"] = true;
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest1.cs\ntest2.cs";
actions.EnumerateDirectories[@"C:\Project"] = "";
var autobuilder = CreateAutoBuilder(true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows", msBuildPlatform: "x86", msBuildConfiguration: "Debug",
vsToolsVersion: "12", allSolutions: "true");
var testSolution1 = new TestSolution(@"C:\Project\test1.sln");
var testSolution2 = new TestSolution(@"C:\Project\test2.sln");
autobuilder.ProjectsOrSolutionsToBuild.Add(testSolution1);
autobuilder.ProjectsOrSolutionsToBuild.Add(testSolution2);
TestAutobuilderScript(autobuilder, 1, 2);
}
[Fact]
public void TestSkipNugetMsBuild()
{
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program^ Files^ ^(x86^)\\Microsoft^ Visual^ Studio^ 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test1.sln /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /P:Fu=Bar"] = 0;
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program^ Files^ ^(x86^)\\Microsoft^ Visual^ Studio^ 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test2.sln /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /P:Fu=Bar"] = 0;
actions.FileExists["csharp.log"] = true;
actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = false;
actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = false;
actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat"] = true;
actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat"] = false;
actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat"] = true;
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest1.cs\ntest2.cs";
actions.EnumerateDirectories[@"C:\Project"] = "";
var autobuilder = CreateAutoBuilder(true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows",
msBuildPlatform: "x86", msBuildConfiguration: "Debug", vsToolsVersion: "12",
allSolutions: "true", nugetRestore: "false");
var testSolution1 = new TestSolution(@"C:\Project\test1.sln");
var testSolution2 = new TestSolution(@"C:\Project\test2.sln");
autobuilder.ProjectsOrSolutionsToBuild.Add(testSolution1);
autobuilder.ProjectsOrSolutionsToBuild.Add(testSolution2);
TestAutobuilderScript(autobuilder, 0, 2);
}
[Fact]
public void TestSkipNugetBuildless()
{
@@ -900,35 +690,6 @@ namespace Semmle.Autobuild.CSharp.Tests
TestAutobuilderScript(autobuilder, 0, 1);
}
[Fact]
public void TestSkipNugetDotnet()
{
actions.RunProcess["dotnet --info"] = 0;
actions.RunProcess[@"dotnet clean C:\Project/test.csproj"] = 0;
actions.RunProcess[@"dotnet restore C:\Project/test.csproj"] = 0;
actions.RunProcess[@"dotnet build --no-incremental --no-restore C:\Project/test.csproj"] = 0;
actions.FileExists["csharp.log"] = true;
actions.FileExists[@"C:\Project/test.csproj"] = true;
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.cs\ntest.csproj";
actions.EnumerateDirectories[@"C:\Project"] = "";
var xml = new XmlDocument();
xml.LoadXml(@"<Project Sdk=""Microsoft.NET.Sdk"">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp2.1</TargetFramework>
</PropertyGroup>
</Project>");
actions.LoadXml[@"C:\Project/test.csproj"] = xml;
var autobuilder = CreateAutoBuilder(false, dotnetArguments: "--no-restore"); // nugetRestore=false does not work for now.
TestAutobuilderScript(autobuilder, 0, 4);
}
[Fact]
public void TestDotnetVersionNotInstalled()
{
@@ -1053,7 +814,7 @@ namespace Semmle.Autobuild.CSharp.Tests
{
actions.RunProcess[@"cmd.exe /C nuget restore C:\Project\dirs.proj -DisableParallelProcessing"] = 1;
actions.RunProcess[@"cmd.exe /C scratch\.nuget\nuget.exe restore C:\Project\dirs.proj -DisableParallelProcessing"] = 0;
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program^ Files^ ^(x86^)\\Microsoft^ Visual^ Studio^ 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\dirs.proj /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /P:Fu=Bar"] = 0;
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program^ Files^ ^(x86^)\\Microsoft^ Visual^ Studio^ 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\dirs.proj /t:rebuild"] = 0;
actions.FileExists["csharp.log"] = true;
actions.FileExists[@"C:\Project\a\test.csproj"] = true;
actions.FileExists[@"C:\Project\dirs.proj"] = true;
@@ -1088,8 +849,7 @@ namespace Semmle.Autobuild.CSharp.Tests
</Project>");
actions.LoadXml[@"C:\Project\dirs.proj"] = dirsproj;
var autobuilder = CreateAutoBuilder(true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows", msBuildPlatform: "x86", msBuildConfiguration: "Debug",
vsToolsVersion: "12", allSolutions: "true");
var autobuilder = CreateAutoBuilder(true);
TestAutobuilderScript(autobuilder, 0, 3);
}

30
csharp/autobuilder/Semmle.Autobuild.CSharp/CSharpAutobuilder.cs
View File

@@ -25,9 +25,11 @@ namespace Semmle.Autobuild.CSharp
/// </summary>
public CSharpAutobuildOptions(IBuildActions actions) : base(actions)
{
Buildless = actions.GetEnvironmentVariable(lgtmPrefix + "BUILDLESS").AsBool("buildless", false) ||
Buildless =
actions.GetEnvironmentVariable(extractorOptionPrefix + "BUILDLESS").AsBool("buildless", false) ||
actions.GetEnvironmentVariable(buildModeEnvironmentVariable)?.ToLower() == "none";
}
}
@@ -46,19 +48,10 @@ namespace Semmle.Autobuild.CSharp
var attempt = BuildScript.Failure;
switch (GetCSharpBuildStrategy())
{
case CSharpBuildStrategy.CustomBuildCommand:
attempt = new BuildCommandRule(DotNetRule.WithDotNet).Analyse(this, false) & CheckExtractorRun(true);
break;
case CSharpBuildStrategy.Buildless:
// No need to check that the extractor has been executed in buildless mode
attempt = new StandaloneBuildRule().Analyse(this, false);
break;
case CSharpBuildStrategy.MSBuild:
attempt = new MsBuildRule().Analyse(this, false) & CheckExtractorRun(true);
break;
case CSharpBuildStrategy.DotNet:
attempt = new DotNetRule().Analyse(this, false) & CheckExtractorRun(true);
break;
case CSharpBuildStrategy.Auto:
attempt =
// Attempt a few different build strategies to see if one works
@@ -198,32 +191,15 @@ namespace Semmle.Autobuild.CSharp
/// </summary>
private CSharpBuildStrategy GetCSharpBuildStrategy()
{
if (Options.BuildCommand is not null)
return CSharpBuildStrategy.CustomBuildCommand;
if (Options.Buildless)
return CSharpBuildStrategy.Buildless;
if (Options.MsBuildArguments is not null
|| Options.MsBuildConfiguration is not null
|| Options.MsBuildPlatform is not null
|| Options.MsBuildTarget is not null)
{
return CSharpBuildStrategy.MSBuild;
}
if (Options.DotNetArguments is not null || Options.DotNetVersion is not null)
return CSharpBuildStrategy.DotNet;
return CSharpBuildStrategy.Auto;
}
private enum CSharpBuildStrategy
{
CustomBuildCommand,
Buildless,
MSBuild,
DotNet,
Auto
}
}

3
csharp/autobuilder/Semmle.Autobuild.CSharp/DotNetRule.cs
View File

@@ -150,8 +150,7 @@ namespace Semmle.Autobuild.CSharp
Argument("--no-incremental");
return
script.Argument(builder.Options.DotNetArguments).
QuoteArgument(projOrSln).
script.QuoteArgument(projOrSln).
Script;
}
}

29
csharp/autobuilder/Semmle.Autobuild.Shared/AutobuildOptions.cs
View File

@@ -11,24 +11,9 @@ namespace Semmle.Autobuild.Shared
/// </summary>
public abstract class AutobuildOptionsShared
{
protected const string lgtmPrefix = "LGTM_INDEX_";
public int SearchDepth { get; } = 3;
public string RootDirectory { get; }
public string? VsToolsVersion { get; }
public string? MsBuildArguments { get; }
public string? MsBuildPlatform { get; }
public string? MsBuildConfiguration { get; }
public string? MsBuildTarget { get; }
public string? DotNetArguments { get; }
public string? DotNetVersion { get; }
public string? BuildCommand { get; }
public IEnumerable<string> Solution { get; }
public bool IgnoreErrors { get; }
public bool AllSolutions { get; }
public bool NugetRestore { get; }
public abstract Language Language { get; }
/// <summary>
@@ -38,19 +23,7 @@ namespace Semmle.Autobuild.Shared
public AutobuildOptionsShared(IBuildActions actions)
{
RootDirectory = actions.GetCurrentDirectory();
VsToolsVersion = actions.GetEnvironmentVariable(lgtmPrefix + "VSTOOLS_VERSION");
MsBuildArguments = actions.GetEnvironmentVariable(lgtmPrefix + "MSBUILD_ARGUMENTS")?.AsStringWithExpandedEnvVars(actions);
MsBuildPlatform = actions.GetEnvironmentVariable(lgtmPrefix + "MSBUILD_PLATFORM");
MsBuildConfiguration = actions.GetEnvironmentVariable(lgtmPrefix + "MSBUILD_CONFIGURATION");
MsBuildTarget = actions.GetEnvironmentVariable(lgtmPrefix + "MSBUILD_TARGET");
DotNetArguments = actions.GetEnvironmentVariable(lgtmPrefix + "DOTNET_ARGUMENTS")?.AsStringWithExpandedEnvVars(actions);
DotNetVersion = actions.GetEnvironmentVariable(lgtmPrefix + "DOTNET_VERSION");
BuildCommand = actions.GetEnvironmentVariable(lgtmPrefix + "BUILD_COMMAND");
Solution = actions.GetEnvironmentVariable(lgtmPrefix + "SOLUTION").AsListWithExpandedEnvVars(actions, Array.Empty<string>());
IgnoreErrors = actions.GetEnvironmentVariable(lgtmPrefix + "IGNORE_ERRORS").AsBool("ignore_errors", false);
AllSolutions = actions.GetEnvironmentVariable(lgtmPrefix + "ALL_SOLUTIONS").AsBool("all_solutions", false);
NugetRestore = actions.GetEnvironmentVariable(lgtmPrefix + "NUGET_RESTORE").AsBool("nuget_restore", true);
DotNetVersion = actions.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_OPTION_DOTNET_VERSION");
}
}

19
csharp/autobuilder/Semmle.Autobuild.Shared/Autobuilder.cs
View File

@@ -161,9 +161,6 @@ namespace Semmle.Autobuild.Shared
if (matchingFiles.Length == 0)
return null;
if (Options.AllSolutions)
return matchingFiles.Select(p => p.ProjectOrSolution);
return matchingFiles
.Where(f => f.DistanceFromRoot == matchingFiles[0].DistanceFromRoot)
.Select(f => f.ProjectOrSolution);
@@ -185,19 +182,6 @@ namespace Semmle.Autobuild.Shared
projectsOrSolutionsToBuildLazy = new Lazy<IList<IProjectOrSolution>>(() =>
{
List<IProjectOrSolution>? ret;
if (options.Solution.Any())
{
ret = new List<IProjectOrSolution>();
foreach (var solution in options.Solution)
{
if (actions.FileExists(solution))
ret.Add(new Solution<TAutobuildOptions>(this, solution, true));
else
logger.LogError($"The specified project or solution file {solution} was not found");
}
return ret;
}
// First look for `.proj` files
ret = FindFiles(".proj", f => new Project<TAutobuildOptions>(this, f))?.ToList();
if (ret is not null)
@@ -285,9 +269,6 @@ namespace Semmle.Autobuild.Shared
var script = GetBuildScript();
if (Options.IgnoreErrors)
script |= BuildScript.Success;
void startCallback(string s, bool silent)
{
logger.Log(silent ? Severity.Debug : Severity.Info, $"\nRunning {s}");

5
csharp/autobuilder/Semmle.Autobuild.Shared/BuildCommandAutoRule.cs
View File

@@ -82,11 +82,6 @@ namespace Semmle.Autobuild.Shared
{
var command = new CommandBuilder(builder.Actions, dir, environment);
// A specific Visual Studio version may be required
var vsTools = MsBuildRule.GetVcVarsBatFile(builder);
if (vsTools is not null)
command.CallBatFile(vsTools.Path);
command.RunCommand(this.ScriptPath);
return command.Script;
});

37
csharp/autobuilder/Semmle.Autobuild.Shared/BuildCommandRule.cs
View File

@@ -1,37 +0,0 @@
using Semmle.Util;
namespace Semmle.Autobuild.Shared
{
/// <summary>
/// Execute the build_command rule.
/// </summary>
public class BuildCommandRule : IBuildRule<AutobuildOptionsShared>
{
private readonly WithDotNet<AutobuildOptionsShared> withDotNet;
public BuildCommandRule(WithDotNet<AutobuildOptionsShared> withDotNet)
{
this.withDotNet = withDotNet;
}
public BuildScript Analyse(IAutobuilder<AutobuildOptionsShared> builder, bool auto)
{
if (builder.Options.BuildCommand is null)
return BuildScript.Failure;
// Custom build commands may require a specific .NET Core version
return withDotNet(builder, environment =>
{
var command = new CommandBuilder(builder.Actions, null, environment);
// Custom build commands may require a specific Visual Studio version
var vsTools = MsBuildRule.GetVcVarsBatFile(builder);
if (vsTools is not null)
command.CallBatFile(vsTools.Path);
command.RunCommand(builder.Options.BuildCommand);
return command.Script;
});
}
}
}

121
csharp/autobuilder/Semmle.Autobuild.Shared/MsBuildRule.cs
View File

@@ -42,9 +42,9 @@ namespace Semmle.Autobuild.Shared
if (auto)
builder.Logger.LogInfo("Attempting to build using MSBuild");
var vsTools = GetVcVarsBatFile(builder);
VcVarsBatFile? vsTools = null;
if (vsTools is null && builder.ProjectsOrSolutionsToBuild.Any())
if (builder.ProjectsOrSolutionsToBuild.Any())
{
var firstSolution = builder.ProjectsOrSolutionsToBuild.OfType<ISolution>().FirstOrDefault();
vsTools = firstSolution is not null
@@ -67,46 +67,44 @@ namespace Semmle.Autobuild.Shared
foreach (var projectOrSolution in builder.ProjectsOrSolutionsToBuild)
{
if (builder.Options.NugetRestore)
BuildScript GetNugetRestoreScript() =>
new CommandBuilder(builder.Actions).
RunCommand(nuget).
Argument("restore").
QuoteArgument(projectOrSolution.FullPath).
Argument("-DisableParallelProcessing").
Script;
var nugetRestore = GetNugetRestoreScript();
var msbuildRestoreCommand = new CommandBuilder(builder.Actions).
MsBuildCommand(builder).
Argument("/t:restore").
QuoteArgument(projectOrSolution.FullPath);
if (builder.Actions.IsRunningOnAppleSilicon())
{
BuildScript GetNugetRestoreScript() =>
new CommandBuilder(builder.Actions).
RunCommand(nuget).
Argument("restore").
QuoteArgument(projectOrSolution.FullPath).
Argument("-DisableParallelProcessing").
Script;
var nugetRestore = GetNugetRestoreScript();
var msbuildRestoreCommand = new CommandBuilder(builder.Actions).
MsBuildCommand(builder).
Argument("/t:restore").
QuoteArgument(projectOrSolution.FullPath);
// On Apple Silicon, only try package restore with `dotnet msbuild /t:restore`
ret &= BuildScript.Try(msbuildRestoreCommand.Script);
}
else if (nugetDownloaded)
{
ret &= BuildScript.Try(nugetRestore | msbuildRestoreCommand.Script);
}
else
{
// If `nuget restore` fails, and we have not already attempted to download `nuget.exe`,
// download it and reattempt `nuget restore`.
var nugetDownloadAndRestore =
BuildScript.Bind(DownloadNugetExe(builder, nugetDownloadPath), exitCode =>
{
nugetDownloaded = true;
if (exitCode != 0)
return BuildScript.Failure;
if (builder.Actions.IsRunningOnAppleSilicon())
{
// On Apple Silicon, only try package restore with `dotnet msbuild /t:restore`
ret &= BuildScript.Try(msbuildRestoreCommand.Script);
}
else if (nugetDownloaded)
{
ret &= BuildScript.Try(nugetRestore | msbuildRestoreCommand.Script);
}
else
{
// If `nuget restore` fails, and we have not already attempted to download `nuget.exe`,
// download it and reattempt `nuget restore`.
var nugetDownloadAndRestore =
BuildScript.Bind(DownloadNugetExe(builder, nugetDownloadPath), exitCode =>
{
nugetDownloaded = true;
if (exitCode != 0)
return BuildScript.Failure;
nuget = nugetDownloadPath;
return GetNugetRestoreScript();
});
ret &= BuildScript.Try(nugetRestore | nugetDownloadAndRestore | msbuildRestoreCommand.Script);
}
nuget = nugetDownloadPath;
return GetNugetRestoreScript();
});
ret &= BuildScript.Try(nugetRestore | nugetDownloadAndRestore | msbuildRestoreCommand.Script);
}
var command = new CommandBuilder(builder.Actions);
@@ -124,9 +122,9 @@ namespace Semmle.Autobuild.Shared
command.MsBuildCommand(builder);
command.QuoteArgument(projectOrSolution.FullPath);
var target = builder.Options.MsBuildTarget ?? "rebuild";
var platform = builder.Options.MsBuildPlatform ?? (projectOrSolution is ISolution s1 ? s1.DefaultPlatformName : null);
var configuration = builder.Options.MsBuildConfiguration ?? (projectOrSolution is ISolution s2 ? s2.DefaultConfigurationName : null);
var target = "rebuild";
var platform = projectOrSolution is ISolution s1 ? s1.DefaultPlatformName : null;
var configuration = projectOrSolution is ISolution s2 ? s2.DefaultConfigurationName : null;
command.Argument("/t:" + target);
if (platform is not null)
@@ -134,8 +132,6 @@ namespace Semmle.Autobuild.Shared
if (configuration is not null)
command.Argument(string.Format("/p:Configuration=\"{0}\"", configuration));
command.Argument(builder.Options.MsBuildArguments);
// append the build script which invokes msbuild to the overall build script `ret`;
// we insert a check that building the current project or solution was successful:
// if it was not successful, we add it to `FailedProjectsOrSolutions`
@@ -148,41 +144,6 @@ namespace Semmle.Autobuild.Shared
return ret;
}
/// <summary>
/// Gets the BAT file used to initialize the appropriate Visual Studio
/// version/platform, as specified by the `vstools_version` property in
/// lgtm.yml.
///
/// Returns <code>null</code> when no version is specified.
/// </summary>
public static VcVarsBatFile? GetVcVarsBatFile<TAutobuildOptions>(IAutobuilder<TAutobuildOptions> builder) where TAutobuildOptions : AutobuildOptionsShared
{
VcVarsBatFile? vsTools = null;
if (builder.Options.VsToolsVersion is not null)
{
if (int.TryParse(builder.Options.VsToolsVersion, out var msToolsVersion))
{
foreach (var b in BuildTools.VcVarsAllBatFiles(builder.Actions))
{
builder.Logger.Log(Severity.Info, "Found {0} version {1}", b.Path, b.ToolsVersion);
}
vsTools = BuildTools.FindCompatibleVcVars(builder.Actions, msToolsVersion);
if (vsTools is null)
builder.Logger.LogWarning("Could not find build tools matching version {0}", msToolsVersion);
else
builder.Logger.Log(Severity.Info, "Setting Visual Studio tools to {0}", vsTools.Path);
}
else
{
builder.Logger.LogError("The format of vstools_version is incorrect. Please specify an integer.");
}
}
return vsTools;
}
/// <summary>
/// Returns a script for downloading `nuget.exe` from nuget.org.
/// </summary>

6
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs
View File

@@ -89,11 +89,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
return dotnetCliInvoker.RunCommand(args);
}
public IList<string> GetListedRuntimes() => GetListed("--list-runtimes", "runtime");
public IList<string> GetListedRuntimes() => GetListed("--list-runtimes");
public IList<string> GetListedSdks() => GetListed("--list-sdks", "SDK");
public IList<string> GetListedSdks() => GetListed("--list-sdks");
private IList<string> GetListed(string args, string artifact)
private IList<string> GetListed(string args)
{
if (dotnetCliInvoker.RunCommand(args, out var artifacts))
{

2
csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs
View File

@@ -189,7 +189,7 @@ namespace Semmle.Extraction.CSharp
// compilation.Clone() is used to allow symbols to be garbage collected.
using var trapWriter = transformedSourcePath.CreateTrapWriter(Logger, options.TrapCompression, discardDuplicates: false);
upToDate = options.Fast && FileIsUpToDate(sourcePath, trapWriter.TrapFile);
upToDate = FileIsUpToDate(sourcePath, trapWriter.TrapFile);
var currentTaskId = IncrementTaskCount();
ReportProgressTaskStarted(currentTaskId, sourcePath);

8
csharp/extractor/Semmle.Extraction.CSharp/Extractor/Options.cs
View File

@@ -35,15 +35,7 @@ namespace Semmle.Extraction.CSharp
public static Options CreateWithEnvironment(string[] arguments)
{
var options = new Options();
var extractionOptions = Environment.GetEnvironmentVariable("LGTM_INDEX_EXTRACTOR");
var argsList = new List<string>(arguments);
if (!string.IsNullOrEmpty(extractionOptions))
{
argsList.AddRange(extractionOptions.Split(' '));
}
options.ParseArguments(argsList);
return options;
}

14
csharp/extractor/Semmle.Extraction.Tests/Options.cs
View File

@@ -12,11 +12,6 @@ namespace Semmle.Extraction.Tests
private CSharp.Options? options;
private CSharp.Standalone.Options? standaloneOptions;
public OptionsTests()
{
Environment.SetEnvironmentVariable("LGTM_INDEX_EXTRACTOR", "");
}
[Fact]
public void DefaultOptions()
{
@@ -28,7 +23,6 @@ namespace Semmle.Extraction.Tests
Assert.True(options.Threads >= 1);
Assert.Equal(Verbosity.Info, options.LegacyVerbosity);
Assert.False(options.Console);
Assert.False(options.Fast);
Assert.Equal(TrapWriter.CompressionMode.Brotli, options.TrapCompression);
}
@@ -165,14 +159,6 @@ namespace Semmle.Extraction.Tests
Assert.True(standaloneOptions.Help);
}
[Fact]
public void Fast()
{
Environment.SetEnvironmentVariable("LGTM_INDEX_EXTRACTOR", "--fast");
options = CSharp.Options.CreateWithEnvironment(Array.Empty<string>());
Assert.True(options.Fast);
}
[Fact]
public void ArchiveArguments()
{

8
csharp/extractor/Semmle.Extraction/Options.cs
View File

@@ -60,11 +60,6 @@ namespace Semmle.Extraction
/// </summary>
public bool Cache { get; private set; } = true;
/// <summary>
/// Whether "fast extraction mode" has been enabled.
/// </summary>
public bool Fast { get; private set; } = false;
/// <summary>
/// Whether extraction is done using `codeql test run`.
/// </summary>
@@ -115,9 +110,6 @@ namespace Semmle.Extraction
case "cache":
Cache = value;
return true;
case "fast":
Fast = value;
return true;
case "qltest":
QlTest = value;
return true;