C++/C#: Move the Windows autobuilder into a subfolder in ql/csharp.

This is a necessary preparation for moving the C# dependency management to `paket`,
which in turn is a necessary preparation for moving the C# build to bazel.

As we discovered in https://github.com/github/codeql/pull/16376,
`paket` tries to restore all projects recursively from the root folder.
If we support building C# code under both `ql/csharp` and `ql/cpp`, we need
to have a single lockfile under `ql`, as both codebases share the same set of dependencies
(and utilities from `ql/csharp/extractor`).
Then, `paket` will also try to restore things that look like "C# projects" in other languages'
folders, which is not what we want.
Therefore, we address this by moving all C# code into a common root directory, `ql/csharp`.

This needs an internal PR to adjust the buildsystem to look for the autobuilder in the new location.
This commit is contained in:
Cornelius Riemenschneider
2024-05-14 13:35:12 +02:00
parent 4d54c09a6f
commit 869bf8afc6
11 changed files with 9 additions and 19 deletions

View File

@@ -1,13 +0,0 @@
obj/
TestResults/
*.manifest
*.pdb
*.suo
*.mdb
*.vsmdi
csharp.log
**/bin/Debug
**/bin/Release
*.tlog
.vs
*.user

View File

@@ -0,0 +1 @@
The Windows autobuilder that used to live in this directory moved to `csharp/autobuilder/Semmle.Autobuild.Cpp`.

View File

@@ -1,334 +0,0 @@
using Xunit;
using Semmle.Autobuild.Shared;
using Semmle.Util;
using System.Collections.Generic;
using System;
using System.Linq;
using Microsoft.Build.Construction;
using System.Xml;
using System.IO;
namespace Semmle.Autobuild.Cpp.Tests
{
/// <summary>
/// Test class to script Autobuilder scenarios.
/// For most methods, it uses two fields:
/// - an IList to capture the the arguments passed to it
/// - an IDictionary of possible return values.
/// </summary>
class TestActions : IBuildActions
{
/// <summary>
/// List of strings passed to FileDelete.
/// </summary>
public IList<string> FileDeleteIn = new List<string>();
void IBuildActions.FileDelete(string file)
{
FileDeleteIn.Add(file);
}
public IList<string> FileExistsIn = new List<string>();
public IDictionary<string, bool> FileExists = new Dictionary<string, bool>();
bool IBuildActions.FileExists(string file)
{
FileExistsIn.Add(file);
if (FileExists.TryGetValue(file, out var ret))
return ret;
if (FileExists.TryGetValue(System.IO.Path.GetFileName(file), out ret))
return ret;
throw new ArgumentException("Missing FileExists " + file);
}
public IList<string> RunProcessIn = new List<string>();
public IDictionary<string, int> RunProcess = new Dictionary<string, int>();
public IDictionary<string, string> RunProcessOut = new Dictionary<string, string>();
public IDictionary<string, string> RunProcessWorkingDirectory = new Dictionary<string, string>();
public HashSet<string> CreateDirectories { get; } = new HashSet<string>();
public HashSet<(string, string)> DownloadFiles { get; } = new HashSet<(string, string)>();
int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory, IDictionary<string, string>? env, out IList<string> stdOut)
{
var pattern = cmd + " " + args;
RunProcessIn.Add(pattern);
if (RunProcessOut.TryGetValue(pattern, out var str))
stdOut = str.Split("\n");
else
throw new ArgumentException("Missing RunProcessOut " + pattern);
RunProcessWorkingDirectory.TryGetValue(pattern, out var wd);
if (wd != workingDirectory)
throw new ArgumentException("Missing RunProcessWorkingDirectory " + pattern);
if (RunProcess.TryGetValue(pattern, out var ret))
return ret;
throw new ArgumentException("Missing RunProcess " + pattern);
}
int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory, IDictionary<string, string>? env)
{
var pattern = cmd + " " + args;
RunProcessIn.Add(pattern);
RunProcessWorkingDirectory.TryGetValue(pattern, out var wd);
if (wd != workingDirectory)
throw new ArgumentException("Missing RunProcessWorkingDirectory " + pattern);
if (RunProcess.TryGetValue(pattern, out var ret))
return ret;
throw new ArgumentException("Missing RunProcess " + pattern);
}
int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory, IDictionary<string, string>? env, BuildOutputHandler onOutput, BuildOutputHandler onError)
{
var ret = (this as IBuildActions).RunProcess(cmd, args, workingDirectory, env, out var stdout);
stdout.ForEach(line => onOutput(line));
return ret;
}
public IList<string> DirectoryDeleteIn = new List<string>();
void IBuildActions.DirectoryDelete(string dir, bool recursive)
{
DirectoryDeleteIn.Add(dir);
}
public IDictionary<string, bool> DirectoryExists = new Dictionary<string, bool>();
public IList<string> DirectoryExistsIn = new List<string>();
bool IBuildActions.DirectoryExists(string dir)
{
DirectoryExistsIn.Add(dir);
if (DirectoryExists.TryGetValue(dir, out var ret))
return ret;
throw new ArgumentException("Missing DirectoryExists " + dir);
}
public IDictionary<string, string?> GetEnvironmentVariable = new Dictionary<string, string?>();
string? IBuildActions.GetEnvironmentVariable(string name)
{
if (GetEnvironmentVariable.TryGetValue(name, out var ret))
return ret;
throw new ArgumentException("Missing GetEnvironmentVariable " + name);
}
public string GetCurrentDirectory = "";
string IBuildActions.GetCurrentDirectory()
{
return GetCurrentDirectory;
}
public IDictionary<string, string> EnumerateFiles = new Dictionary<string, string>();
IEnumerable<string> IBuildActions.EnumerateFiles(string dir)
{
if (EnumerateFiles.TryGetValue(dir, out var str))
return str.Split("\n");
throw new ArgumentException("Missing EnumerateFiles " + dir);
}
public IDictionary<string, string> EnumerateDirectories = new Dictionary<string, string>();
IEnumerable<string> IBuildActions.EnumerateDirectories(string dir)
{
if (EnumerateDirectories.TryGetValue(dir, out var str))
return string.IsNullOrEmpty(str) ? Enumerable.Empty<string>() : str.Split("\n");
throw new ArgumentException("Missing EnumerateDirectories " + dir);
}
public bool IsWindows;
bool IBuildActions.IsWindows() => IsWindows;
public bool IsMacOs { get; set; }
bool IBuildActions.IsMacOs() => IsMacOs;
public bool IsRunningOnAppleSilicon { get; set; }
bool IBuildActions.IsRunningOnAppleSilicon() => IsRunningOnAppleSilicon;
string IBuildActions.PathCombine(params string[] parts)
{
return string.Join(IsWindows ? '\\' : '/', parts.Where(p => !string.IsNullOrWhiteSpace(p)));
}
string IBuildActions.GetFullPath(string path) => path;
string? IBuildActions.GetFileName(string? path) => Path.GetFileName(path?.Replace('\\', '/'));
public string? GetDirectoryName(string? path)
{
var dir = Path.GetDirectoryName(path?.Replace('\\', '/'));
return dir is null ? path : path?.Substring(0, dir.Length);
}
void IBuildActions.WriteAllText(string filename, string contents)
{
}
public IDictionary<string, XmlDocument> LoadXml = new Dictionary<string, XmlDocument>();
XmlDocument IBuildActions.LoadXml(string filename)
{
if (LoadXml.TryGetValue(filename, out var xml))
return xml;
throw new ArgumentException("Missing LoadXml " + filename);
}
public string EnvironmentExpandEnvironmentVariables(string s)
{
foreach (var kvp in GetEnvironmentVariable)
s = s.Replace($"%{kvp.Key}%", kvp.Value);
return s;
}
public void CreateDirectory(string path)
{
if (!CreateDirectories.Contains(path))
throw new ArgumentException($"Missing CreateDirectory, {path}");
}
public void DownloadFile(string address, string fileName)
{
if (!DownloadFiles.Contains((address, fileName)))
throw new ArgumentException($"Missing DownloadFile, {address}, {fileName}");
}
public IDiagnosticsWriter CreateDiagnosticsWriter(string filename) => new TestDiagnosticWriter();
}
internal class TestDiagnosticWriter : IDiagnosticsWriter
{
public IList<DiagnosticMessage> Diagnostics { get; } = new List<DiagnosticMessage>();
public void AddEntry(DiagnosticMessage message) => this.Diagnostics.Add(message);
public void Dispose() { }
}
/// <summary>
/// A fake solution to build.
/// </summary>
class TestSolution : ISolution
{
public IEnumerable<SolutionConfigurationInSolution> Configurations => throw new NotImplementedException();
public string DefaultConfigurationName => "Release";
public string DefaultPlatformName => "x86";
public string FullPath { get; set; }
public Version ToolsVersion => new Version("14.0");
public IEnumerable<IProjectOrSolution> IncludedProjects => throw new NotImplementedException();
public TestSolution(string path)
{
FullPath = path;
}
}
public class BuildScriptTests
{
TestActions Actions = new TestActions();
// Records the arguments passed to StartCallback.
IList<string> StartCallbackIn = new List<string>();
void StartCallback(string s, bool silent)
{
StartCallbackIn.Add(s);
}
// Records the arguments passed to EndCallback
IList<string> EndCallbackIn = new List<string>();
IList<int> EndCallbackReturn = new List<int>();
void EndCallback(int ret, string s, bool silent)
{
EndCallbackReturn.Add(ret);
EndCallbackIn.Add(s);
}
CppAutobuilder CreateAutoBuilder(bool isWindows, string? dotnetVersion = null, string cwd = @"C:\Project")
{
string codeqlUpperLanguage = Language.Cpp.UpperCaseName;
Actions.GetEnvironmentVariable[$"CODEQL_AUTOBUILDER_{codeqlUpperLanguage}_NO_INDEXING"] = "false";
Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_TRAP_DIR"] = "";
Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_SOURCE_ARCHIVE_DIR"] = "";
Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_ROOT"] = $@"C:\codeql\{codeqlUpperLanguage.ToLowerInvariant()}";
Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_DIAGNOSTIC_DIR"] = "";
Actions.GetEnvironmentVariable["CODEQL_JAVA_HOME"] = @"C:\codeql\tools\java";
Actions.GetEnvironmentVariable["CODEQL_PLATFORM"] = "win64";
Actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_OPTION_DOTNET_VERSION"] = dotnetVersion;
Actions.GetEnvironmentVariable["ProgramFiles(x86)"] = isWindows ? @"C:\Program Files (x86)" : null;
Actions.GetCurrentDirectory = cwd;
Actions.IsWindows = isWindows;
var options = new CppAutobuildOptions(Actions);
return new CppAutobuilder(Actions, options);
}
void TestAutobuilderScript(CppAutobuilder autobuilder, int expectedOutput, int commandsRun)
{
Assert.Equal(expectedOutput, autobuilder.GetBuildScript().Run(Actions, StartCallback, EndCallback));
// Check expected commands actually ran
Assert.Equal(commandsRun, StartCallbackIn.Count);
Assert.Equal(commandsRun, EndCallbackIn.Count);
Assert.Equal(commandsRun, EndCallbackReturn.Count);
var action = Actions.RunProcess.GetEnumerator();
for (int cmd = 0; cmd < commandsRun; ++cmd)
{
Assert.True(action.MoveNext());
Assert.Equal(action.Current.Key, StartCallbackIn[cmd]);
Assert.Equal(action.Current.Value, EndCallbackReturn[cmd]);
}
}
[Fact]
public void TestDefaultCppAutobuilder()
{
Actions.EnumerateFiles[@"C:\Project"] = "";
Actions.EnumerateDirectories[@"C:\Project"] = "";
var autobuilder = CreateAutoBuilder(true);
var script = autobuilder.GetBuildScript();
// Fails due to no solutions present.
Assert.NotEqual(0, script.Run(Actions, StartCallback, EndCallback));
}
[Fact]
public void TestCppAutobuilderSuccess()
{
Actions.RunProcess[@"cmd.exe /C nuget restore C:\Project\test.sln -DisableParallelProcessing"] = 1;
Actions.RunProcess[@"cmd.exe /C scratch\.nuget\nuget.exe restore C:\Project\test.sln -DisableParallelProcessing"] = 0;
Actions.RunProcess[@"cmd.exe /C CALL ^""C:\Program^ Files^ ^(x86^)\Microsoft^ Visual^ Studio^ 14.0\VC\vcvarsall.bat^"" && set Platform=&& type NUL && msbuild C:\Project\test.sln /t:rebuild /p:Platform=""x86"" /p:Configuration=""Release"""] = 0;
Actions.RunProcessOut[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = "";
Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = 1;
Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationVersion"] = 0;
Actions.RunProcessOut[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationVersion"] = "";
Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = true;
Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat"] = true;
Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat"] = true;
Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat"] = true;
Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = true;
Actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CPP_SCRATCH_DIR"] = "scratch";
Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.slx";
Actions.EnumerateDirectories[@"C:\Project"] = "";
Actions.CreateDirectories.Add(@"scratch\.nuget");
Actions.DownloadFiles.Add(("https://dist.nuget.org/win-x86-commandline/latest/nuget.exe", @"scratch\.nuget\nuget.exe"));
var autobuilder = CreateAutoBuilder(true);
var solution = new TestSolution(@"C:\Project\test.sln");
autobuilder.ProjectsOrSolutionsToBuild.Add(solution);
TestAutobuilderScript(autobuilder, 0, 3);
}
}
}

View File

@@ -1,26 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net8.0</TargetFramework>
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
<RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="System.IO.FileSystem" Version="4.3.0" />
<PackageReference Include="System.IO.FileSystem.Primitives" Version="4.3.0" />
<PackageReference Include="xunit" Version="2.6.2" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.5.4">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Semmle.Autobuild.Cpp\Semmle.Autobuild.Cpp.csproj" />
<ProjectReference Include="..\..\..\csharp\autobuilder\Semmle.Autobuild.Shared\Semmle.Autobuild.Shared.csproj" />
</ItemGroup>
</Project>

View File

@@ -1,38 +0,0 @@
using Semmle.Autobuild.Shared;
using Semmle.Util;
namespace Semmle.Autobuild.Cpp
{
/// <summary>
/// Encapsulates C++ build options.
/// </summary>
public class CppAutobuildOptions : AutobuildOptionsShared
{
public override Language Language => Language.Cpp;
/// <summary>
/// Reads options from environment variables.
/// Throws ArgumentOutOfRangeException for invalid arguments.
/// </summary>
public CppAutobuildOptions(IBuildActions actions) : base(actions)
{
}
}
public class CppAutobuilder : Autobuilder<CppAutobuildOptions>
{
public CppAutobuilder(IBuildActions actions, CppAutobuildOptions options) : base(actions, options, new DiagnosticClassifier()) { }
public override BuildScript GetBuildScript()
{
return
// First try MSBuild
new MsBuildRule().Analyse(this, true) |
// Then look for a script that might be a build script
(() => new BuildCommandAutoRule((_, f) => f(null)).Analyse(this, true)) |
// All attempts failed: print message
AutobuildFailure();
}
}
}

View File

@@ -1,35 +0,0 @@
using System;
using Semmle.Autobuild.Shared;
using Semmle.Util;
namespace Semmle.Autobuild.Cpp
{
class Program
{
static int Main()
{
try
{
var actions = SystemBuildActions.Instance;
var options = new CppAutobuildOptions(actions);
try
{
Console.WriteLine("CodeQL C++ autobuilder");
using var builder = new CppAutobuilder(actions, options);
return builder.AttemptBuild();
}
catch (InvalidEnvironmentException ex)
{
Console.WriteLine("The environment is invalid: {0}", ex.Message);
}
}
catch (ArgumentOutOfRangeException ex)
{
Console.WriteLine("The value \"{0}\" for parameter \"{1}\" is invalid", ex.ActualValue, ex.ParamName);
}
return 1;
}
}
}

View File

@@ -1,32 +0,0 @@
using System.Reflection;
using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("Semmle.Autobuild.Cpp")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("GitHub")]
[assembly: AssemblyProduct("CodeQL autobuilder for C++")]
[assembly: AssemblyCopyright("Copyright © GitHub 2020")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]

View File

@@ -1,28 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<AssemblyName>Semmle.Autobuild.Cpp</AssemblyName>
<RootNamespace>Semmle.Autobuild.Cpp</RootNamespace>
<ApplicationIcon />
<OutputType>Exe</OutputType>
<StartupObject />
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
<RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<Folder Include="Properties\" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Build" Version="17.8.3" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\..\csharp\extractor\Semmle.Util\Semmle.Util.csproj" />
<ProjectReference Include="..\..\..\csharp\autobuilder\Semmle.Autobuild.Shared\Semmle.Autobuild.Shared.csproj" />
</ItemGroup>
</Project>