C#: Factor C++ parts out of autobuilder

This commit is contained in:
Tom Hvitved
2020-06-29 16:01:16 +02:00
parent bb9c8881d6
commit 498ee9b5f5
32 changed files with 721 additions and 288 deletions

View File

@@ -0,0 +1,296 @@
using Xunit;
using Semmle.Autobuild.Shared;
using System.Collections.Generic;
using System;
using System.Linq;
using Microsoft.Build.Construction;
using System.Xml;
namespace Semmle.Autobuild.Cpp.Tests
{
/// <summary>
/// Test class to script Autobuilder scenarios.
/// For most methods, it uses two fields:
/// - an IList to capture the the arguments passed to it
/// - an IDictionary of possible return values.
/// </summary>
class TestActions : IBuildActions
{
/// <summary>
/// List of strings passed to FileDelete.
/// </summary>
public IList<string> FileDeleteIn = new List<string>();
void IBuildActions.FileDelete(string file)
{
FileDeleteIn.Add(file);
}
public IList<string> FileExistsIn = new List<string>();
public IDictionary<string, bool> FileExists = new Dictionary<string, bool>();
bool IBuildActions.FileExists(string file)
{
FileExistsIn.Add(file);
if (FileExists.TryGetValue(file, out var ret))
return ret;
if (FileExists.TryGetValue(System.IO.Path.GetFileName(file), out ret))
return ret;
throw new ArgumentException("Missing FileExists " + file);
}
public IList<string> RunProcessIn = new List<string>();
public IDictionary<string, int> RunProcess = new Dictionary<string, int>();
public IDictionary<string, string> RunProcessOut = new Dictionary<string, string>();
public IDictionary<string, string> RunProcessWorkingDirectory = new Dictionary<string, string>();
int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory, IDictionary<string, string>? env, out IList<string> stdOut)
{
var pattern = cmd + " " + args;
RunProcessIn.Add(pattern);
if (RunProcessOut.TryGetValue(pattern, out var str))
stdOut = str.Split("\n");
else
throw new ArgumentException("Missing RunProcessOut " + pattern);
RunProcessWorkingDirectory.TryGetValue(pattern, out var wd);
if (wd != workingDirectory)
throw new ArgumentException("Missing RunProcessWorkingDirectory " + pattern);
if (RunProcess.TryGetValue(pattern, out var ret))
return ret;
throw new ArgumentException("Missing RunProcess " + pattern);
}
int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory, IDictionary<string, string>? env)
{
var pattern = cmd + " " + args;
RunProcessIn.Add(pattern);
RunProcessWorkingDirectory.TryGetValue(pattern, out var wd);
if (wd != workingDirectory)
throw new ArgumentException("Missing RunProcessWorkingDirectory " + pattern);
if (RunProcess.TryGetValue(pattern, out var ret))
return ret;
throw new ArgumentException("Missing RunProcess " + pattern);
}
public IList<string> DirectoryDeleteIn = new List<string>();
void IBuildActions.DirectoryDelete(string dir, bool recursive)
{
DirectoryDeleteIn.Add(dir);
}
public IDictionary<string, bool> DirectoryExists = new Dictionary<string, bool>();
public IList<string> DirectoryExistsIn = new List<string>();
bool IBuildActions.DirectoryExists(string dir)
{
DirectoryExistsIn.Add(dir);
if (DirectoryExists.TryGetValue(dir, out var ret))
return ret;
throw new ArgumentException("Missing DirectoryExists " + dir);
}
public IDictionary<string, string?> GetEnvironmentVariable = new Dictionary<string, string?>();
string? IBuildActions.GetEnvironmentVariable(string name)
{
if (GetEnvironmentVariable.TryGetValue(name, out var ret))
return ret;
throw new ArgumentException("Missing GetEnvironmentVariable " + name);
}
public string GetCurrentDirectory = "";
string IBuildActions.GetCurrentDirectory()
{
return GetCurrentDirectory;
}
public IDictionary<string, string> EnumerateFiles = new Dictionary<string, string>();
IEnumerable<string> IBuildActions.EnumerateFiles(string dir)
{
if (EnumerateFiles.TryGetValue(dir, out var str))
return str.Split("\n");
throw new ArgumentException("Missing EnumerateFiles " + dir);
}
public IDictionary<string, string> EnumerateDirectories = new Dictionary<string, string>();
IEnumerable<string> IBuildActions.EnumerateDirectories(string dir)
{
if (EnumerateDirectories.TryGetValue(dir, out var str))
return string.IsNullOrEmpty(str) ? Enumerable.Empty<string>() : str.Split("\n");
throw new ArgumentException("Missing EnumerateDirectories " + dir);
}
public bool IsWindows;
bool IBuildActions.IsWindows() => IsWindows;
string IBuildActions.PathCombine(params string[] parts)
{
return string.Join(IsWindows ? '\\' : '/', parts.Where(p => !string.IsNullOrWhiteSpace(p)));
}
string IBuildActions.GetFullPath(string path) => path;
void IBuildActions.WriteAllText(string filename, string contents)
{
}
public IDictionary<string, XmlDocument> LoadXml = new Dictionary<string, XmlDocument>();
XmlDocument IBuildActions.LoadXml(string filename)
{
if (LoadXml.TryGetValue(filename, out var xml))
return xml;
throw new ArgumentException("Missing LoadXml " + filename);
}
public string EnvironmentExpandEnvironmentVariables(string s)
{
foreach (var kvp in GetEnvironmentVariable)
s = s.Replace($"%{kvp.Key}%", kvp.Value);
return s;
}
}
/// <summary>
/// A fake solution to build.
/// </summary>
class TestSolution : ISolution
{
public IEnumerable<SolutionConfigurationInSolution> Configurations => throw new NotImplementedException();
public string DefaultConfigurationName => "Release";
public string DefaultPlatformName => "x86";
public string FullPath { get; set; }
public Version ToolsVersion => new Version("14.0");
public IEnumerable<IProjectOrSolution> IncludedProjects => throw new NotImplementedException();
public TestSolution(string path)
{
FullPath = path;
}
}
public class BuildScriptTests
{
TestActions Actions = new TestActions();
// Records the arguments passed to StartCallback.
IList<string> StartCallbackIn = new List<string>();
void StartCallback(string s, bool silent)
{
StartCallbackIn.Add(s);
}
// Records the arguments passed to EndCallback
IList<string> EndCallbackIn = new List<string>();
IList<int> EndCallbackReturn = new List<int>();
void EndCallback(int ret, string s, bool silent)
{
EndCallbackReturn.Add(ret);
EndCallbackIn.Add(s);
}
CppAutobuilder CreateAutoBuilder(bool isWindows,
string? buildless = null, string? solution = null, string? buildCommand = null, string? ignoreErrors = null,
string? msBuildArguments = null, string? msBuildPlatform = null, string? msBuildConfiguration = null, string? msBuildTarget = null,
string? dotnetArguments = null, string? dotnetVersion = null, string? vsToolsVersion = null,
string? nugetRestore = null, string? allSolutions = null,
string cwd = @"C:\Project")
{
string codeqlUpperLanguage = Language.Cpp.UpperCaseName;
Actions.GetEnvironmentVariable[$"CODEQL_AUTOBUILDER_{codeqlUpperLanguage}_NO_INDEXING"] = "false";
Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_TRAP_DIR"] = "";
Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_SOURCE_ARCHIVE_DIR"] = "";
Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_ROOT"] = $@"C:\codeql\{codeqlUpperLanguage.ToLowerInvariant()}";
Actions.GetEnvironmentVariable["CODEQL_JAVA_HOME"] = @"C:\codeql\tools\java";
Actions.GetEnvironmentVariable["SEMMLE_DIST"] = @"C:\odasa";
Actions.GetEnvironmentVariable["SEMMLE_JAVA_HOME"] = @"C:\odasa\tools\java";
Actions.GetEnvironmentVariable["SEMMLE_PLATFORM_TOOLS"] = @"C:\odasa\tools";
Actions.GetEnvironmentVariable["LGTM_INDEX_VSTOOLS_VERSION"] = vsToolsVersion;
Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_ARGUMENTS"] = msBuildArguments;
Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_PLATFORM"] = msBuildPlatform;
Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_CONFIGURATION"] = msBuildConfiguration;
Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_TARGET"] = msBuildTarget;
Actions.GetEnvironmentVariable["LGTM_INDEX_DOTNET_ARGUMENTS"] = dotnetArguments;
Actions.GetEnvironmentVariable["LGTM_INDEX_DOTNET_VERSION"] = dotnetVersion;
Actions.GetEnvironmentVariable["LGTM_INDEX_BUILD_COMMAND"] = buildCommand;
Actions.GetEnvironmentVariable["LGTM_INDEX_SOLUTION"] = solution;
Actions.GetEnvironmentVariable["LGTM_INDEX_IGNORE_ERRORS"] = ignoreErrors;
Actions.GetEnvironmentVariable["LGTM_INDEX_BUILDLESS"] = buildless;
Actions.GetEnvironmentVariable["LGTM_INDEX_ALL_SOLUTIONS"] = allSolutions;
Actions.GetEnvironmentVariable["LGTM_INDEX_NUGET_RESTORE"] = nugetRestore;
Actions.GetEnvironmentVariable["ProgramFiles(x86)"] = isWindows ? @"C:\Program Files (x86)" : null;
Actions.GetCurrentDirectory = cwd;
Actions.IsWindows = isWindows;
var options = new AutobuildOptions(Actions, Language.Cpp);
return new CppAutobuilder(Actions, options);
}
void TestAutobuilderScript(Autobuilder autobuilder, int expectedOutput, int commandsRun)
{
Assert.Equal(expectedOutput, autobuilder.GetBuildScript().Run(Actions, StartCallback, EndCallback));
// Check expected commands actually ran
Assert.Equal(commandsRun, StartCallbackIn.Count);
Assert.Equal(commandsRun, EndCallbackIn.Count);
Assert.Equal(commandsRun, EndCallbackReturn.Count);
var action = Actions.RunProcess.GetEnumerator();
for (int cmd = 0; cmd < commandsRun; ++cmd)
{
Assert.True(action.MoveNext());
Assert.Equal(action.Current.Key, StartCallbackIn[cmd]);
Assert.Equal(action.Current.Value, EndCallbackReturn[cmd]);
}
}
[Fact]
public void TestDefaultCppAutobuilder()
{
Actions.EnumerateFiles[@"C:\Project"] = "";
Actions.EnumerateDirectories[@"C:\Project"] = "";
var autobuilder = CreateAutoBuilder(true);
var script = autobuilder.GetBuildScript();
// Fails due to no solutions present.
Assert.NotEqual(0, script.Run(Actions, StartCallback, EndCallback));
}
[Fact]
public void TestCppAutobuilderSuccess()
{
Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\csharp\nuget\nuget.exe restore C:\Project\test.sln"] = 1;
Actions.RunProcess[@"cmd.exe /C CALL ^""C:\Program Files ^(x86^)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat^"" && set Platform=&& type NUL && C:\odasa\tools\odasa index --auto msbuild C:\Project\test.sln /p:UseSharedCompilation=false /t:rebuild /p:Platform=""x86"" /p:Configuration=""Release"" /p:MvcBuildViews=true"] = 0;
Actions.RunProcessOut[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = "";
Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = 1;
Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationVersion"] = 0;
Actions.RunProcessOut[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationVersion"] = "";
Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = true;
Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat"] = true;
Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat"] = true;
Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat"] = true;
Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = true;
Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.slx";
Actions.EnumerateDirectories[@"C:\Project"] = "";
var autobuilder = CreateAutoBuilder(true);
var solution = new TestSolution(@"C:\Project\test.sln");
autobuilder.ProjectsOrSolutionsToBuild.Add(solution);
TestAutobuilderScript(autobuilder, 0, 2);
}
}
}

View File

@@ -0,0 +1,25 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp3.0</TargetFramework>
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
<RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="System.IO.FileSystem" Version="4.3.0" />
<PackageReference Include="System.IO.FileSystem.Primitives" Version="4.3.0" />
<PackageReference Include="xunit" Version="2.4.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.1">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
</PackageReference>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Semmle.Autobuild.Cpp\Semmle.Autobuild.Cpp.csproj" />
<ProjectReference Include="..\..\..\csharp\autobuilder\Semmle.Autobuild.Shared\Semmle.Autobuild.Shared.csproj" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,23 @@
using Semmle.Autobuild.Shared;
namespace Semmle.Autobuild.Cpp
{
public class CppAutobuilder : Autobuilder
{
public CppAutobuilder(IBuildActions actions, AutobuildOptions options) : base(actions, options) { }
public override BuildScript GetBuildScript()
{
if (Options.BuildCommand != null)
return new BuildCommandRule((_, f) => f(null)).Analyse(this, false);
return
// First try MSBuild
new MsBuildRule().Analyse(this, true) |
// Then look for a script that might be a build script
(() => new BuildCommandAutoRule((_, f) => f(null)).Analyse(this, true)) |
// All attempts failed: print message
AutobuildFailure();
}
}
}

View File

@@ -1,6 +1,7 @@
using System;
using Semmle.Autobuild.Shared;
namespace Semmle.Autobuild
namespace Semmle.Autobuild.Cpp
{
class Program
{
@@ -10,11 +11,11 @@ namespace Semmle.Autobuild
try
{
var actions = SystemBuildActions.Instance;
var options = new AutobuildOptions(actions);
var options = new AutobuildOptions(actions, Language.Cpp);
try
{
Console.WriteLine($"Semmle autobuilder for {options.Language}");
var builder = new Autobuilder(actions, options);
Console.WriteLine("CodeQL C++ autobuilder");
var builder = new CppAutobuilder(actions, options);
return builder.AttemptBuild();
}
catch(InvalidEnvironmentException ex)

View File

@@ -4,12 +4,12 @@ using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("Semmle.Autobuild.Tests")]
[assembly: AssemblyTitle("Semmle.Autobuild.Cpp")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("")]
[assembly: AssemblyProduct("Semmle.Extraction.Tests")]
[assembly: AssemblyCopyright("Copyright © 2018")]
[assembly: AssemblyCompany("GitHub")]
[assembly: AssemblyProduct("CodeQL autobuilder for C++")]
[assembly: AssemblyCopyright("Copyright © GitHub 2020")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]

View File

@@ -0,0 +1,28 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netcoreapp3.0</TargetFramework>
<AssemblyName>Semmle.Autobuild.Cpp</AssemblyName>
<RootNamespace>Semmle.Autobuild.Cpp</RootNamespace>
<ApplicationIcon />
<OutputType>Exe</OutputType>
<StartupObject />
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
<RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<Folder Include="Properties\" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Build" Version="16.0.461" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\..\csharp\extractor\Semmle.Util\Semmle.Util.csproj" />
<ProjectReference Include="..\..\..\csharp\autobuilder\Semmle.Autobuild.Shared\Semmle.Autobuild.Shared.csproj" />
</ItemGroup>
</Project>

View File

@@ -11,8 +11,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CSharp",
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CIL", "extractor\Semmle.Extraction.CIL\Semmle.Extraction.CIL.csproj", "{399A1579-68F0-40F4-9A23-F241BA697F9C}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Autobuild", "autobuilder\Semmle.Autobuild\Semmle.Autobuild.csproj", "{5131EF00-0BA9-4436-A3B0-C5CDAB4B194C}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CSharp.Standalone", "extractor\Semmle.Extraction.CSharp.Standalone\Semmle.Extraction.CSharp.Standalone.csproj", "{D00E7D25-0FA0-48EC-B048-CD60CE1B30D8}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CIL.Driver", "extractor\Semmle.Extraction.CIL.Driver\Semmle.Extraction.CIL.Driver.csproj", "{EFA400B3-C1CE-446F-A4E2-8B44E61EF47C}"
@@ -23,7 +21,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.Tests", "
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Util.Tests", "extractor\Semmle.Util.Tests\Semmle.Util.Tests.csproj", "{55A620F0-23F6-440D-A5BA-0567613B3C0F}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Autobuild.Tests", "autobuilder\Semmle.Autobuild.Tests\Semmle.Autobuild.Tests.csproj", "{CE267461-D762-4F53-A275-685A0A4EC48D}"
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Semmle.Autobuild.Shared", "autobuilder\Semmle.Autobuild.Shared\Semmle.Autobuild.Shared.csproj", "{133F2B5B-FD25-4BD9-B34C-062CC6BB4178}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Semmle.Autobuild.CSharp", "autobuilder\Semmle.Autobuild.CSharp\Semmle.Autobuild.CSharp.csproj", "{F3C07863-3759-4A0B-B777-8A0E0FDB1A41}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Semmle.Autobuild.CSharp.Tests", "autobuilder\Semmle.Autobuild.CSharp.Tests\Semmle.Autobuild.CSharp.Tests.csproj", "{34256E8F-866A-46C1-800E-3DF69FD1DCB7}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -47,10 +49,6 @@ Global
{399A1579-68F0-40F4-9A23-F241BA697F9C}.Debug|Any CPU.Build.0 = Debug|Any CPU
{399A1579-68F0-40F4-9A23-F241BA697F9C}.Release|Any CPU.ActiveCfg = Release|Any CPU
{399A1579-68F0-40F4-9A23-F241BA697F9C}.Release|Any CPU.Build.0 = Release|Any CPU
{5131EF00-0BA9-4436-A3B0-C5CDAB4B194C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{5131EF00-0BA9-4436-A3B0-C5CDAB4B194C}.Debug|Any CPU.Build.0 = Debug|Any CPU
{5131EF00-0BA9-4436-A3B0-C5CDAB4B194C}.Release|Any CPU.ActiveCfg = Release|Any CPU
{5131EF00-0BA9-4436-A3B0-C5CDAB4B194C}.Release|Any CPU.Build.0 = Release|Any CPU
{D00E7D25-0FA0-48EC-B048-CD60CE1B30D8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{D00E7D25-0FA0-48EC-B048-CD60CE1B30D8}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D00E7D25-0FA0-48EC-B048-CD60CE1B30D8}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -69,10 +67,18 @@ Global
{55A620F0-23F6-440D-A5BA-0567613B3C0F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{55A620F0-23F6-440D-A5BA-0567613B3C0F}.Debug|Any CPU.Build.0 = Debug|Any CPU
{55A620F0-23F6-440D-A5BA-0567613B3C0F}.Release|Any CPU.ActiveCfg = Release|Any CPU
{CE267461-D762-4F53-A275-685A0A4EC48D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{CE267461-D762-4F53-A275-685A0A4EC48D}.Debug|Any CPU.Build.0 = Debug|Any CPU
{CE267461-D762-4F53-A275-685A0A4EC48D}.Release|Any CPU.ActiveCfg = Release|Any CPU
{CE267461-D762-4F53-A275-685A0A4EC48D}.Release|Any CPU.Build.0 = Release|Any CPU
{133F2B5B-FD25-4BD9-B34C-062CC6BB4178}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{133F2B5B-FD25-4BD9-B34C-062CC6BB4178}.Debug|Any CPU.Build.0 = Debug|Any CPU
{133F2B5B-FD25-4BD9-B34C-062CC6BB4178}.Release|Any CPU.ActiveCfg = Release|Any CPU
{133F2B5B-FD25-4BD9-B34C-062CC6BB4178}.Release|Any CPU.Build.0 = Release|Any CPU
{F3C07863-3759-4A0B-B777-8A0E0FDB1A41}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{F3C07863-3759-4A0B-B777-8A0E0FDB1A41}.Debug|Any CPU.Build.0 = Debug|Any CPU
{F3C07863-3759-4A0B-B777-8A0E0FDB1A41}.Release|Any CPU.ActiveCfg = Release|Any CPU
{F3C07863-3759-4A0B-B777-8A0E0FDB1A41}.Release|Any CPU.Build.0 = Release|Any CPU
{34256E8F-866A-46C1-800E-3DF69FD1DCB7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{34256E8F-866A-46C1-800E-3DF69FD1DCB7}.Debug|Any CPU.Build.0 = Debug|Any CPU
{34256E8F-866A-46C1-800E-3DF69FD1DCB7}.Release|Any CPU.ActiveCfg = Release|Any CPU
{34256E8F-866A-46C1-800E-3DF69FD1DCB7}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE

View File

@@ -1,12 +1,12 @@
using Xunit;
using Semmle.Autobuild;
using Semmle.Autobuild.Shared;
using System.Collections.Generic;
using System;
using System.Linq;
using Microsoft.Build.Construction;
using System.Xml;
namespace Semmle.Extraction.Tests
namespace Semmle.Autobuild.CSharp.Tests
{
/// <summary>
/// Test class to script Autobuilder scenarios.
@@ -333,22 +333,21 @@ namespace Semmle.Extraction.Tests
Assert.Equal(0, BuildScript.Try(BuildScript.Failure).Run(Actions, StartCallback, EndCallback));
}
Autobuilder CreateAutoBuilder(string lgtmLanguage, bool isWindows,
CSharpAutobuilder CreateAutoBuilder(bool isWindows,
string? buildless = null, string? solution = null, string? buildCommand = null, string? ignoreErrors = null,
string? msBuildArguments = null, string? msBuildPlatform = null, string? msBuildConfiguration = null, string? msBuildTarget = null,
string? dotnetArguments = null, string? dotnetVersion = null, string? vsToolsVersion = null,
string? nugetRestore = null, string? allSolutions = null,
string cwd = @"C:\Project")
{
string codeqlUpperLanguage = lgtmLanguage.ToUpper();
string codeqlUpperLanguage = Language.CSharp.UpperCaseName;
Actions.GetEnvironmentVariable[$"CODEQL_AUTOBUILDER_{codeqlUpperLanguage}_NO_INDEXING"] = "false";
Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_TRAP_DIR"] = "";
Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_SOURCE_ARCHIVE_DIR"] = "";
Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_ROOT"] = $@"C:\codeql\{lgtmLanguage}";
Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_ROOT"] = $@"C:\codeql\{codeqlUpperLanguage.ToLowerInvariant()}";
Actions.GetEnvironmentVariable["CODEQL_JAVA_HOME"] = @"C:\codeql\tools\java";
Actions.GetEnvironmentVariable["SEMMLE_DIST"] = @"C:\odasa";
Actions.GetEnvironmentVariable["SEMMLE_JAVA_HOME"] = @"C:\odasa\tools\java";
Actions.GetEnvironmentVariable["LGTM_PROJECT_LANGUAGE"] = lgtmLanguage;
Actions.GetEnvironmentVariable["SEMMLE_PLATFORM_TOOLS"] = @"C:\odasa\tools";
Actions.GetEnvironmentVariable["LGTM_INDEX_VSTOOLS_VERSION"] = vsToolsVersion;
Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_ARGUMENTS"] = msBuildArguments;
@@ -367,8 +366,8 @@ namespace Semmle.Extraction.Tests
Actions.GetCurrentDirectory = cwd;
Actions.IsWindows = isWindows;
var options = new AutobuildOptions(Actions);
return new Autobuilder(Actions, options);
var options = new AutobuildOptions(Actions, Language.CSharp);
return new CSharpAutobuilder(Actions, options);
}
[Fact]
@@ -396,7 +395,7 @@ namespace Semmle.Extraction.Tests
</Project>");
Actions.LoadXml["test.csproj"] = xml;
var autobuilder = CreateAutoBuilder("csharp", true);
var autobuilder = CreateAutoBuilder(true);
TestAutobuilderScript(autobuilder, 0, 6);
}
@@ -428,7 +427,7 @@ Microsoft.NETCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
</Project>");
Actions.LoadXml["test.csproj"] = xml;
var autobuilder = CreateAutoBuilder("csharp", false);
var autobuilder = CreateAutoBuilder(false);
TestAutobuilderScript(autobuilder, 0, 7);
}
@@ -441,47 +440,10 @@ Microsoft.NETCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.cs";
Actions.EnumerateDirectories[@"C:\Project"] = "";
var autobuilder = CreateAutoBuilder("csharp", false);
var autobuilder = CreateAutoBuilder(false);
TestAutobuilderScript(autobuilder, 1, 0);
}
[Fact]
public void TestDefaultCppAutobuilder()
{
Actions.EnumerateFiles[@"C:\Project"] = "";
Actions.EnumerateDirectories[@"C:\Project"] = "";
var autobuilder = CreateAutoBuilder("cpp", true);
var script = autobuilder.GetBuildScript();
// Fails due to no solutions present.
Assert.NotEqual(0, script.Run(Actions, StartCallback, EndCallback));
}
[Fact]
public void TestCppAutobuilderSuccess()
{
Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\csharp\nuget\nuget.exe restore C:\Project\test.sln"] = 1;
Actions.RunProcess[@"cmd.exe /C CALL ^""C:\Program Files ^(x86^)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat^"" && set Platform=&& type NUL && C:\odasa\tools\odasa index --auto msbuild C:\Project\test.sln /p:UseSharedCompilation=false /t:rebuild /p:Platform=""x86"" /p:Configuration=""Release"" /p:MvcBuildViews=true"] = 0;
Actions.RunProcessOut[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = "";
Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = 1;
Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationVersion"] = 0;
Actions.RunProcessOut[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationVersion"] = "";
Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = true;
Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat"] = true;
Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat"] = true;
Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat"] = true;
Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = true;
Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.slx";
Actions.EnumerateDirectories[@"C:\Project"] = "";
var autobuilder = CreateAutoBuilder("cpp", true);
var solution = new TestSolution(@"C:\Project\test.sln");
autobuilder.ProjectsOrSolutionsToBuild.Add(solution);
TestAutobuilderScript(autobuilder, 0, 2);
}
[Fact]
public void TestVsWhereSucceeded()
{
@@ -538,7 +500,7 @@ Microsoft.NETCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
Actions.EnumerateDirectories[@"C:\Project"] = "";
var autobuilder = CreateAutoBuilder("csharp", false, buildless: "true");
var autobuilder = CreateAutoBuilder(false, buildless: "true");
TestAutobuilderScript(autobuilder, 0, 3);
}
@@ -552,7 +514,7 @@ Microsoft.NETCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
Actions.EnumerateDirectories[@"C:\Project"] = "";
var autobuilder = CreateAutoBuilder("csharp", false, buildless: "true");
var autobuilder = CreateAutoBuilder(false, buildless: "true");
TestAutobuilderScript(autobuilder, 10, 1);
}
@@ -568,7 +530,7 @@ Microsoft.NETCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
Actions.EnumerateDirectories[@"C:\Project"] = "";
var autobuilder = CreateAutoBuilder("csharp", false, buildless: "true", solution: "foo.sln");
var autobuilder = CreateAutoBuilder(false, buildless: "true", solution: "foo.sln");
TestAutobuilderScript(autobuilder, 0, 3);
}
@@ -616,7 +578,7 @@ Microsoft.NETCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
SkipVsWhere();
var autobuilder = CreateAutoBuilder("csharp", false, buildCommand: "./build.sh --skip-tests");
var autobuilder = CreateAutoBuilder(false, buildCommand: "./build.sh --skip-tests");
TestAutobuilderScript(autobuilder, 0, 4);
}
@@ -636,7 +598,7 @@ Microsoft.NETCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
Actions.RunProcess[@"C:\odasa/tools/odasa index --xml --extensions config csproj props xml"] = 0;
Actions.FileExists["csharp.log"] = true;
var autobuilder = CreateAutoBuilder("csharp", false);
var autobuilder = CreateAutoBuilder(false);
TestAutobuilderScript(autobuilder, 0, 5);
}
@@ -655,7 +617,7 @@ Microsoft.NETCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
Actions.RunProcessWorkingDirectory[@"C:\odasa/tools/odasa index --auto build.sh"] = "";
Actions.FileExists["csharp.log"] = false;
var autobuilder = CreateAutoBuilder("csharp", false);
var autobuilder = CreateAutoBuilder(false);
TestAutobuilderScript(autobuilder, 1, 3);
}
@@ -674,7 +636,7 @@ Microsoft.NETCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
Actions.RunProcessWorkingDirectory[@"C:\odasa/tools/odasa index --auto build.sh"] = "";
Actions.FileExists["csharp.log"] = true;
var autobuilder = CreateAutoBuilder("csharp", false);
var autobuilder = CreateAutoBuilder(false);
TestAutobuilderScript(autobuilder, 1, 3);
}
@@ -691,7 +653,7 @@ Microsoft.NETCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --xml --extensions config csproj props xml"] = 0;
Actions.FileExists["csharp.log"] = true;
var autobuilder = CreateAutoBuilder("csharp", true);
var autobuilder = CreateAutoBuilder(true);
TestAutobuilderScript(autobuilder, 0, 3);
}
@@ -708,7 +670,7 @@ Microsoft.NETCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --xml --extensions config"] = 0;
Actions.FileExists["csharp.log"] = true;
var autobuilder = CreateAutoBuilder("csharp", true, ignoreErrors: "true");
var autobuilder = CreateAutoBuilder(true, ignoreErrors: "true");
TestAutobuilderScript(autobuilder, 1, 1);
}
@@ -726,7 +688,7 @@ Microsoft.NETCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
Actions.EnumerateDirectories[@"C:\Project"] = "";
var autobuilder = CreateAutoBuilder("csharp", true, buildCommand: "build.cmd --skip-tests", ignoreErrors: "true");
var autobuilder = CreateAutoBuilder(true, buildCommand: "build.cmd --skip-tests", ignoreErrors: "true");
TestAutobuilderScript(autobuilder, 3, 1);
}
@@ -751,7 +713,7 @@ Microsoft.NETCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest1.cs\ntest2.cs";
Actions.EnumerateDirectories[@"C:\Project"] = "";
var autobuilder = CreateAutoBuilder("csharp", true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows", msBuildPlatform: "x86", msBuildConfiguration: "Debug",
var autobuilder = CreateAutoBuilder(true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows", msBuildPlatform: "x86", msBuildConfiguration: "Debug",
vsToolsVersion: "12", allSolutions: "true");
var testSolution1 = new TestSolution(@"C:\Project\test1.sln");
var testSolution2 = new TestSolution(@"C:\Project\test2.sln");
@@ -802,7 +764,7 @@ Microsoft.NETCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
</Project>");
Actions.LoadXml["test2.csproj"] = csproj2;
var autobuilder = CreateAutoBuilder("csharp", true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows", msBuildPlatform: "x86", msBuildConfiguration: "Debug",
var autobuilder = CreateAutoBuilder(true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows", msBuildPlatform: "x86", msBuildConfiguration: "Debug",
vsToolsVersion: "12");
TestAutobuilderScript(autobuilder, 0, 6);
@@ -824,7 +786,7 @@ Microsoft.NETCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest1.cs\ntest2.cs";
Actions.EnumerateDirectories[@"C:\Project"] = "";
var autobuilder = CreateAutoBuilder("csharp", true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows", msBuildPlatform: "x86", msBuildConfiguration: "Debug",
var autobuilder = CreateAutoBuilder(true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows", msBuildPlatform: "x86", msBuildConfiguration: "Debug",
vsToolsVersion: "12", allSolutions: "true");
var testSolution1 = new TestSolution(@"C:\Project\test1.sln");
var testSolution2 = new TestSolution(@"C:\Project\test2.sln");
@@ -853,7 +815,7 @@ Microsoft.NETCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest1.cs\ntest2.cs";
Actions.EnumerateDirectories[@"C:\Project"] = "";
var autobuilder = CreateAutoBuilder("csharp", true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows",
var autobuilder = CreateAutoBuilder(true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows",
msBuildPlatform: "x86", msBuildConfiguration: "Debug", vsToolsVersion: "12",
allSolutions: "true", nugetRestore: "false");
var testSolution1 = new TestSolution(@"C:\Project\test1.sln");
@@ -876,7 +838,7 @@ Microsoft.NETCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
Actions.EnumerateDirectories[@"C:\Project"] = "";
var autobuilder = CreateAutoBuilder("csharp", false, buildless: "true", solution: "foo.sln", nugetRestore: "false");
var autobuilder = CreateAutoBuilder(false, buildless: "true", solution: "foo.sln", nugetRestore: "false");
TestAutobuilderScript(autobuilder, 0, 3);
}
@@ -909,7 +871,7 @@ Microsoft.NETCore.App 2.1.3 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
</Project>");
Actions.LoadXml["test.csproj"] = xml;
var autobuilder = CreateAutoBuilder("csharp", false, dotnetArguments: "--no-restore"); // nugetRestore=false does not work for now.
var autobuilder = CreateAutoBuilder(false, dotnetArguments: "--no-restore"); // nugetRestore=false does not work for now.
TestAutobuilderScript(autobuilder, 0, 7);
}
@@ -948,7 +910,7 @@ Microsoft.NETCore.App 3.0.0 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
</Project>");
Actions.LoadXml["test.csproj"] = xml;
var autobuilder = CreateAutoBuilder("csharp", false, dotnetVersion: "2.1.3");
var autobuilder = CreateAutoBuilder(false, dotnetVersion: "2.1.3");
TestAutobuilderScript(autobuilder, 0, 12);
}
@@ -990,7 +952,7 @@ Microsoft.NETCore.App 2.1.4 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
</Project>");
Actions.LoadXml["test.csproj"] = xml;
var autobuilder = CreateAutoBuilder("csharp", false, dotnetVersion: "2.1.3");
var autobuilder = CreateAutoBuilder(false, dotnetVersion: "2.1.3");
TestAutobuilderScript(autobuilder, 0, 12);
}
@@ -1024,7 +986,7 @@ Microsoft.NETCore.App 2.1.4 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
</Project>");
Actions.LoadXml["test.csproj"] = xml;
var autobuilder = CreateAutoBuilder("csharp", true, dotnetVersion: "2.1.3");
var autobuilder = CreateAutoBuilder(true, dotnetVersion: "2.1.3");
TestAutobuilderScript(autobuilder, 0, 9);
}
@@ -1066,7 +1028,7 @@ Microsoft.NETCore.App 2.1.4 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
</Project>");
Actions.LoadXml["dirs.proj"] = dirsproj;
var autobuilder = CreateAutoBuilder("csharp", true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows", msBuildPlatform: "x86", msBuildConfiguration: "Debug",
var autobuilder = CreateAutoBuilder(true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows", msBuildPlatform: "x86", msBuildConfiguration: "Debug",
vsToolsVersion: "12", allSolutions: "true");
TestAutobuilderScript(autobuilder, 0, 4);
}
@@ -1103,7 +1065,7 @@ Microsoft.NETCore.App 2.1.4 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
</Project>");
Actions.LoadXml["dirs.proj"] = dirsproj;
var autobuilder = CreateAutoBuilder("csharp", false);
var autobuilder = CreateAutoBuilder(false);
TestAutobuilderScript(autobuilder, 0, 4);
}
@@ -1125,7 +1087,7 @@ Microsoft.NETCore.App 2.1.4 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
</Project>");
Actions.LoadXml["dirs.proj"] = dirsproj1;
var autobuilder = CreateAutoBuilder("csharp", false);
var autobuilder = CreateAutoBuilder(false);
TestAutobuilderScript(autobuilder, 1, 0);
}

View File

@@ -19,7 +19,7 @@
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Semmle.Autobuild\Semmle.Autobuild.csproj" />
<ProjectReference Include="..\Semmle.Autobuild.CSharp\Semmle.Autobuild.CSharp.csproj" />
<ProjectReference Include="..\Semmle.Autobuild.Shared\Semmle.Autobuild.Shared.csproj" />
</ItemGroup>
</Project>

View File

@@ -1,4 +1,6 @@
namespace Semmle.Autobuild
using Semmle.Autobuild.Shared;
namespace Semmle.Autobuild.CSharp
{
/// <summary>
/// ASP extraction.

View File

@@ -0,0 +1,127 @@
using Semmle.Extraction.CSharp;
using Semmle.Util.Logging;
using Semmle.Autobuild.Shared;
namespace Semmle.Autobuild.CSharp
{
public class CSharpAutobuilder : Autobuilder
{
public CSharpAutobuilder(IBuildActions actions, AutobuildOptions options) : base(actions, options) { }
public override BuildScript GetBuildScript()
{
/// <summary>
/// A script that checks that the C# extractor has been executed.
/// </summary>
BuildScript CheckExtractorRun(bool warnOnFailure) =>
BuildScript.Create(actions =>
{
if (actions.FileExists(Extractor.GetCSharpLogPath()))
return 0;
if (warnOnFailure)
Log(Severity.Error, "No C# code detected during build.");
return 1;
});
var attempt = BuildScript.Failure;
switch (GetCSharpBuildStrategy())
{
case CSharpBuildStrategy.CustomBuildCommand:
attempt = new BuildCommandRule(DotNetRule.WithDotNet).Analyse(this, false) & CheckExtractorRun(true);
break;
case CSharpBuildStrategy.Buildless:
// No need to check that the extractor has been executed in buildless mode
attempt = new StandaloneBuildRule().Analyse(this, false);
break;
case CSharpBuildStrategy.MSBuild:
attempt = new MsBuildRule().Analyse(this, false) & CheckExtractorRun(true);
break;
case CSharpBuildStrategy.DotNet:
attempt = new DotNetRule().Analyse(this, false) & CheckExtractorRun(true);
break;
case CSharpBuildStrategy.Auto:
var cleanTrapFolder =
BuildScript.DeleteDirectory(TrapDir);
var cleanSourceArchive =
BuildScript.DeleteDirectory(SourceArchiveDir);
var tryCleanExtractorArgsLogs =
BuildScript.Create(actions =>
{
foreach (var file in Extractor.GetCSharpArgsLogs())
try
{
actions.FileDelete(file);
}
catch // lgtm[cs/catch-of-all-exceptions] lgtm[cs/empty-catch-block]
{ }
return 0;
});
var attemptExtractorCleanup =
BuildScript.Try(cleanTrapFolder) &
BuildScript.Try(cleanSourceArchive) &
tryCleanExtractorArgsLogs &
BuildScript.DeleteFile(Extractor.GetCSharpLogPath());
/// <summary>
/// Execute script `s` and check that the C# extractor has been executed.
/// If either fails, attempt to cleanup any artifacts produced by the extractor,
/// and exit with code 1, in order to proceed to the next attempt.
/// </summary>
BuildScript IntermediateAttempt(BuildScript s) =>
(s & CheckExtractorRun(false)) |
(attemptExtractorCleanup & BuildScript.Failure);
attempt =
// First try .NET Core
IntermediateAttempt(new DotNetRule().Analyse(this, true)) |
// Then MSBuild
(() => IntermediateAttempt(new MsBuildRule().Analyse(this, true))) |
// And finally look for a script that might be a build script
(() => new BuildCommandAutoRule(DotNetRule.WithDotNet).Analyse(this, true) & CheckExtractorRun(true)) |
// All attempts failed: print message
AutobuildFailure();
break;
}
return
attempt &
(() => new AspBuildRule().Analyse(this, false)) &
(() => new XmlBuildRule().Analyse(this, false));
}
/// <summary>
/// Gets the build strategy that the autobuilder should apply, based on the
/// options in the `lgtm.yml` file.
/// </summary>
CSharpBuildStrategy GetCSharpBuildStrategy()
{
if (Options.BuildCommand != null)
return CSharpBuildStrategy.CustomBuildCommand;
if (Options.Buildless)
return CSharpBuildStrategy.Buildless;
if (Options.MsBuildArguments != null
|| Options.MsBuildConfiguration != null
|| Options.MsBuildPlatform != null
|| Options.MsBuildTarget != null)
return CSharpBuildStrategy.MSBuild;
if (Options.DotNetArguments != null || Options.DotNetVersion != null)
return CSharpBuildStrategy.DotNet;
return CSharpBuildStrategy.Auto;
}
enum CSharpBuildStrategy
{
CustomBuildCommand,
Buildless,
MSBuild,
DotNet,
Auto
}
}
}

View File

@@ -6,8 +6,9 @@ using System.Collections.Generic;
using System.IO;
using Semmle.Util;
using System.Text.RegularExpressions;
using Semmle.Autobuild.Shared;
namespace Semmle.Autobuild
namespace Semmle.Autobuild.CSharp
{
/// <summary>
/// A build rule where the build command is of the form "dotnet build".

View File

@@ -0,0 +1,33 @@
using System;
using Semmle.Autobuild.Shared;
namespace Semmle.Autobuild.CSharp
{
class Program
{
static int Main()
{
try
{
var actions = SystemBuildActions.Instance;
var options = new AutobuildOptions(actions, Language.CSharp);
try
{
Console.WriteLine("CodeQL C# autobuilder");
var builder = new CSharpAutobuilder(actions, options);
return builder.AttemptBuild();
}
catch (InvalidEnvironmentException ex)
{
Console.WriteLine("The environment is invalid: {0}", ex.Message);
}
}
catch (ArgumentOutOfRangeException ex)
{
Console.WriteLine("The value \"{0}\" for parameter \"{1}\" is invalid", ex.ActualValue, ex.ParamName);
}
return 1;
}
}
}

View File

@@ -0,0 +1,32 @@
using System.Reflection;
using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("Semmle.Autobuild.CSharp")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("GitHub")]
[assembly: AssemblyProduct("CodeQL autobuilder for C#")]
[assembly: AssemblyCopyright("Copyright © GitHub 2020")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]

View File

@@ -2,8 +2,8 @@
<PropertyGroup>
<TargetFramework>netcoreapp3.0</TargetFramework>
<AssemblyName>Semmle.Autobuild</AssemblyName>
<RootNamespace>Semmle.Autobuild</RootNamespace>
<AssemblyName>Semmle.Autobuild.CSharp</AssemblyName>
<RootNamespace>Semmle.Autobuild.CSharp</RootNamespace>
<ApplicationIcon />
<OutputType>Exe</OutputType>
<StartupObject />
@@ -24,6 +24,7 @@
<ItemGroup>
<ProjectReference Include="..\..\extractor\Semmle.Util\Semmle.Util.csproj" />
<ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp\Semmle.Extraction.CSharp.csproj" />
<ProjectReference Include="..\Semmle.Autobuild.Shared\Semmle.Autobuild.Shared.csproj" />
</ItemGroup>
</Project>

View File

@@ -1,4 +1,6 @@
namespace Semmle.Autobuild
using Semmle.Autobuild.Shared;
namespace Semmle.Autobuild.CSharp
{
/// <summary>
/// Build using standalone extraction.

View File

@@ -1,4 +1,6 @@
namespace Semmle.Autobuild
using Semmle.Autobuild.Shared;
namespace Semmle.Autobuild.CSharp
{
/// <summary>
/// XML extraction.

View File

@@ -2,7 +2,7 @@
using System.Linq;
using System.Text.RegularExpressions;
namespace Semmle.Autobuild
namespace Semmle.Autobuild.Shared
{
/// <summary>
/// Encapsulates build options.
@@ -35,7 +35,7 @@ namespace Semmle.Autobuild
/// Reads options from environment variables.
/// Throws ArgumentOutOfRangeException for invalid arguments.
/// </summary>
public AutobuildOptions(IBuildActions actions)
public AutobuildOptions(IBuildActions actions, Language language)
{
RootDirectory = actions.GetCurrentDirectory();
VsToolsVersion = actions.GetEnvironmentVariable(prefix + "VSTOOLS_VERSION");
@@ -53,7 +53,7 @@ namespace Semmle.Autobuild
AllSolutions = actions.GetEnvironmentVariable(prefix + "ALL_SOLUTIONS").AsBool("all_solutions", false);
NugetRestore = actions.GetEnvironmentVariable(prefix + "NUGET_RESTORE").AsBool("nuget_restore", true);
Language = actions.GetEnvironmentVariable("LGTM_PROJECT_LANGUAGE").AsLanguage();
Language = language;
Indexing = !actions.GetEnvironmentVariable($"CODEQL_AUTOBUILDER_{Language.UpperCaseName}_NO_INDEXING").AsBool("no_indexing", false);
}
@@ -81,21 +81,6 @@ namespace Semmle.Autobuild
}
}
public static Language AsLanguage(this string? key)
{
switch (key)
{
case null:
throw new ArgumentException("Environment variable required: LGTM_PROJECT_LANGUAGE");
case "csharp":
return Language.CSharp;
case "cpp":
return Language.Cpp;
default:
throw new ArgumentException("Language key not understood: '" + key + "'");
}
}
public static string[] AsListWithExpandedEnvVars(this string? value, IBuildActions actions, string[] defaultValue)
{
if (value == null)

View File

@@ -1,16 +1,15 @@
using Semmle.Extraction.CSharp;
using Semmle.Util.Logging;
using Semmle.Util.Logging;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
namespace Semmle.Autobuild
namespace Semmle.Autobuild.Shared
{
/// <summary>
/// A build rule analyses the files in "builder" and outputs a build script.
/// </summary>
interface IBuildRule
public interface IBuildRule
{
/// <summary>
/// Analyse the files and produce a build script.
@@ -23,7 +22,7 @@ namespace Semmle.Autobuild
/// <summary>
/// Exception indicating that environment variables are missing or invalid.
/// </summary>
class InvalidEnvironmentException : Exception
public class InvalidEnvironmentException : Exception
{
public InvalidEnvironmentException(string m) : base(m) { }
}
@@ -35,7 +34,7 @@ namespace Semmle.Autobuild
/// The overall design is intended to be extensible so that in theory,
/// it should be possible to add new build rules without touching this code.
/// </summary>
public class Autobuilder
public abstract class Autobuilder
{
/// <summary>
/// Full file paths of files found in the project directory, as well as
@@ -126,10 +125,10 @@ namespace Semmle.Autobuild
ToArray();
if (matchingFiles.Length == 0)
return null;
return null;
if (Options.AllSolutions)
return matchingFiles.Select(p => p.ProjectOrSolution);
return matchingFiles.Select(p => p.ProjectOrSolution);
return matchingFiles.
Where(f => f.DistanceFromRoot == matchingFiles[0].DistanceFromRoot).
@@ -188,9 +187,9 @@ namespace Semmle.Autobuild
SemmleDist = Actions.GetEnvironmentVariable("SEMMLE_DIST");
SemmlePlatformTools = Actions.GetEnvironmentVariable("SEMMLE_PLATFORM_TOOLS");
JavaHome =
JavaHome =
Actions.GetEnvironmentVariable("CODEQL_JAVA_HOME") ??
Actions.GetEnvironmentVariable("SEMMLE_JAVA_HOME") ??
Actions.GetEnvironmentVariable("SEMMLE_JAVA_HOME") ??
throw new InvalidEnvironmentException("The environment variable CODEQL_JAVA_HOME or SEMMLE_JAVA_HOME has not been set.");
Distribution =
@@ -209,9 +208,9 @@ namespace Semmle.Autobuild
throw new InvalidEnvironmentException($"The environment variable CODEQL_EXTRACTOR_{this.Options.Language.UpperCaseName}_SOURCE_ARCHIVE_DIR or SOURCE_ARCHIVE has not been set.");
}
private string TrapDir { get; }
protected string TrapDir { get; }
private string SourceArchiveDir { get; }
protected string SourceArchiveDir { get; }
readonly ILogger logger = new ConsoleLogger(Verbosity.Info);
@@ -234,6 +233,7 @@ namespace Semmle.Autobuild
Log(Severity.Info, $"Working directory: {Options.RootDirectory}");
var script = GetBuildScript();
if (Options.IgnoreErrors)
script |= BuildScript.Success;
@@ -253,143 +253,9 @@ namespace Semmle.Autobuild
/// <summary>
/// Returns the build script to use for this project.
/// </summary>
public BuildScript GetBuildScript()
{
var isCSharp = Options.Language == Language.CSharp;
return isCSharp ? GetCSharpBuildScript() : GetCppBuildScript();
}
public abstract BuildScript GetBuildScript();
BuildScript GetCSharpBuildScript()
{
/// <summary>
/// A script that checks that the C# extractor has been executed.
/// </summary>
BuildScript CheckExtractorRun(bool warnOnFailure) =>
BuildScript.Create(actions =>
{
if (actions.FileExists(Extractor.GetCSharpLogPath()))
return 0;
if (warnOnFailure)
Log(Severity.Error, "No C# code detected during build.");
return 1;
});
var attempt = BuildScript.Failure;
switch (GetCSharpBuildStrategy())
{
case CSharpBuildStrategy.CustomBuildCommand:
attempt = new BuildCommandRule().Analyse(this, false) & CheckExtractorRun(true);
break;
case CSharpBuildStrategy.Buildless:
// No need to check that the extractor has been executed in buildless mode
attempt = new StandaloneBuildRule().Analyse(this, false);
break;
case CSharpBuildStrategy.MSBuild:
attempt = new MsBuildRule().Analyse(this, false) & CheckExtractorRun(true);
break;
case CSharpBuildStrategy.DotNet:
attempt = new DotNetRule().Analyse(this, false) & CheckExtractorRun(true);
break;
case CSharpBuildStrategy.Auto:
var cleanTrapFolder =
BuildScript.DeleteDirectory(TrapDir);
var cleanSourceArchive =
BuildScript.DeleteDirectory(SourceArchiveDir);
var tryCleanExtractorArgsLogs =
BuildScript.Create(actions =>
{
foreach (var file in Extractor.GetCSharpArgsLogs())
try
{
actions.FileDelete(file);
}
catch // lgtm[cs/catch-of-all-exceptions] lgtm[cs/empty-catch-block]
{ }
return 0;
});
var attemptExtractorCleanup =
BuildScript.Try(cleanTrapFolder) &
BuildScript.Try(cleanSourceArchive) &
tryCleanExtractorArgsLogs &
BuildScript.DeleteFile(Extractor.GetCSharpLogPath());
/// <summary>
/// Execute script `s` and check that the C# extractor has been executed.
/// If either fails, attempt to cleanup any artifacts produced by the extractor,
/// and exit with code 1, in order to proceed to the next attempt.
/// </summary>
BuildScript IntermediateAttempt(BuildScript s) =>
(s & CheckExtractorRun(false)) |
(attemptExtractorCleanup & BuildScript.Failure);
attempt =
// First try .NET Core
IntermediateAttempt(new DotNetRule().Analyse(this, true)) |
// Then MSBuild
(() => IntermediateAttempt(new MsBuildRule().Analyse(this, true))) |
// And finally look for a script that might be a build script
(() => new BuildCommandAutoRule().Analyse(this, true) & CheckExtractorRun(true)) |
// All attempts failed: print message
AutobuildFailure();
break;
}
return
attempt &
(() => new AspBuildRule().Analyse(this, false)) &
(() => new XmlBuildRule().Analyse(this, false));
}
/// <summary>
/// Gets the build strategy that the autobuilder should apply, based on the
/// options in the `lgtm.yml` file.
/// </summary>
CSharpBuildStrategy GetCSharpBuildStrategy()
{
if (Options.BuildCommand != null)
return CSharpBuildStrategy.CustomBuildCommand;
if (Options.Buildless)
return CSharpBuildStrategy.Buildless;
if (Options.MsBuildArguments != null
|| Options.MsBuildConfiguration != null
|| Options.MsBuildPlatform != null
|| Options.MsBuildTarget != null)
return CSharpBuildStrategy.MSBuild;
if (Options.DotNetArguments != null || Options.DotNetVersion != null)
return CSharpBuildStrategy.DotNet;
return CSharpBuildStrategy.Auto;
}
enum CSharpBuildStrategy
{
CustomBuildCommand,
Buildless,
MSBuild,
DotNet,
Auto
}
BuildScript GetCppBuildScript()
{
if (Options.BuildCommand != null)
return new BuildCommandRule().Analyse(this, false);
return
// First try MSBuild
new MsBuildRule().Analyse(this, true) |
// Then look for a script that might be a build script
(() => new BuildCommandAutoRule().Analyse(this, true)) |
// All attempts failed: print message
AutobuildFailure();
}
BuildScript AutobuildFailure() =>
protected BuildScript AutobuildFailure() =>
BuildScript.Create(actions =>
{
Log(Severity.Error, "Could not auto-detect a suitable build method");
@@ -426,6 +292,6 @@ namespace Semmle.Autobuild
/// an <code>odasa --index</code>, unless indexing has been disabled, in which case
/// <paramref name="cmd"/> is run directly.
/// </summary>
internal CommandBuilder MaybeIndex(CommandBuilder builder, string cmd) => Options.Indexing && !(Odasa is null) ? builder.IndexCommand(Odasa, cmd) : builder.RunCommand(cmd);
public CommandBuilder MaybeIndex(CommandBuilder builder, string cmd) => Options.Indexing && !(Odasa is null) ? builder.IndexCommand(Odasa, cmd) : builder.RunCommand(cmd);
}
}

View File

@@ -5,7 +5,7 @@ using System.Diagnostics;
using System.IO;
using System.Xml;
namespace Semmle.Autobuild
namespace Semmle.Autobuild.Shared
{
/// <summary>
/// Wrapper around system calls so that the build scripts can be unit-tested.
@@ -124,7 +124,7 @@ namespace Semmle.Autobuild
/// <summary>
/// An implementation of IBuildActions that actually performs the requested operations.
/// </summary>
class SystemBuildActions : IBuildActions
public class SystemBuildActions : IBuildActions
{
void IBuildActions.FileDelete(string file) => File.Delete(file);

View File

@@ -1,15 +1,23 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Semmle.Util.Logging;
namespace Semmle.Autobuild
namespace Semmle.Autobuild.Shared
{
/// <summary>
/// Auto-detection of build scripts.
/// </summary>
class BuildCommandAutoRule : IBuildRule
public class BuildCommandAutoRule : IBuildRule
{
private readonly Func<Autobuilder, Func<IDictionary<string, string>?, BuildScript>, BuildScript> withDotNet;
public BuildCommandAutoRule(Func<Autobuilder, Func<IDictionary<string, string>?, BuildScript>, BuildScript> withDotNet)
{
this.withDotNet = withDotNet;
}
readonly IEnumerable<string> winExtensions = new List<string> {
".bat",
".cmd",
@@ -43,7 +51,7 @@ namespace Semmle.Autobuild
string? dir = Path.GetDirectoryName(scriptPath);
// A specific .NET Core version may be required
return chmodScript & DotNetRule.WithDotNet(builder, environment =>
return chmodScript & withDotNet(builder, environment =>
{
var command = new CommandBuilder(builder.Actions, dir, environment);

View File

@@ -1,17 +1,27 @@
namespace Semmle.Autobuild
using System;
using System.Collections.Generic;
namespace Semmle.Autobuild.Shared
{
/// <summary>
/// Execute the build_command rule.
/// </summary>
class BuildCommandRule : IBuildRule
public class BuildCommandRule : IBuildRule
{
private readonly Func<Autobuilder, Func<IDictionary<string, string>?, BuildScript>, BuildScript> withDotNet;
public BuildCommandRule(Func<Autobuilder, Func<IDictionary<string, string>?, BuildScript>, BuildScript> withDotNet)
{
this.withDotNet = withDotNet;
}
public BuildScript Analyse(Autobuilder builder, bool auto)
{
if (builder.Options.BuildCommand == null)
return BuildScript.Failure;
// Custom build commands may require a specific .NET Core version
return DotNetRule.WithDotNet(builder, environment =>
return withDotNet(builder, environment =>
{
var command = new CommandBuilder(builder.Actions, null, environment);

View File

@@ -2,7 +2,7 @@
using System.Collections.Generic;
using System.IO;
namespace Semmle.Autobuild
namespace Semmle.Autobuild.Shared
{
/// <summary>
/// A build script.

View File

@@ -1,7 +1,7 @@
using System.Collections.Generic;
using System.Linq;
namespace Semmle.Autobuild
namespace Semmle.Autobuild.Shared
{
/// <summary>
/// A BAT file used to initialise the appropriate

View File

@@ -2,12 +2,12 @@ using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace Semmle.Autobuild
namespace Semmle.Autobuild.Shared
{
/// <summary>
/// Utility to construct a build command.
/// </summary>
class CommandBuilder
public class CommandBuilder
{
enum EscapeMode { Process, Cmd };

View File

@@ -1,4 +1,4 @@
namespace Semmle.Autobuild
namespace Semmle.Autobuild.Shared
{
public sealed class Language
{

View File

@@ -1,12 +1,12 @@
using Semmle.Util.Logging;
using System.Linq;
namespace Semmle.Autobuild
namespace Semmle.Autobuild.Shared
{
/// <summary>
/// A build rule using msbuild.
/// </summary>
class MsBuildRule : IBuildRule
public class MsBuildRule : IBuildRule
{
/// <summary>
/// The name of the msbuild command.

View File

@@ -5,7 +5,7 @@ using System.Linq;
using System.Xml;
using Semmle.Util.Logging;
namespace Semmle.Autobuild
namespace Semmle.Autobuild.Shared
{
/// <summary>
/// Representation of a .proj file, a .csproj file (C#), or a .vcxproj file (C++).

View File

@@ -2,7 +2,7 @@
using System.IO;
using System.Linq;
namespace Semmle.Autobuild
namespace Semmle.Autobuild.Shared
{
/// <summary>
/// A file that can be the target in an invocation of `msbuild` or `dotnet build`.

View File

@@ -4,12 +4,12 @@ using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("Semmle.Autobuild")]
[assembly: AssemblyTitle("Semmle.Autobuild.Shared")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("Semmle")]
[assembly: AssemblyProduct("Semmle Visual Studio Autobuild")]
[assembly: AssemblyCopyright("Copyright © Semmle 2017")]
[assembly: AssemblyCompany("GitHub")]
[assembly: AssemblyProduct("CodeQL autobuilder")]
[assembly: AssemblyCopyright("Copyright © GitHub 2020")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]

View File

@@ -0,0 +1,24 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netcoreapp3.0</TargetFramework>
<AssemblyName>Semmle.Autobuild.Shared</AssemblyName>
<RootNamespace>Semmle.Autobuild.Shared</RootNamespace>
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
<RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<Folder Include="Properties\" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Build" Version="16.0.461" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\extractor\Semmle.Util\Semmle.Util.csproj" />
</ItemGroup>
</Project>

View File

@@ -3,11 +3,10 @@ using Microsoft.Build.Exceptions;
using System;
using System.Collections.Generic;
using System.Linq;
using Semmle.Util;
using System.IO;
using Semmle.Util.Logging;
namespace Semmle.Autobuild
namespace Semmle.Autobuild.Shared
{
/// <summary>
/// A solution file, extension .sln.