C#: Remove legacy odasa support

The following environment variables are no longer supported:

```
ODASA_BUILD_ERROR_DIR
ODASA_CSHARP_LAYOUT
ODASA_SNAPSHOT
SEMMLE_DIST
SEMMLE_EXTRACTOR_OPTIONS
SEMMLE_PLATFORM_TOOLS
SEMMLE_PRESERVE_SYMLINKS
SOURCE_ARCHIVE
TRAP_FOLDER
```
This commit is contained in:
Tom Hvitved
2022-03-03 15:53:39 +01:00
parent a70ed71c01
commit c463dc9d1a
26 changed files with 102 additions and 650 deletions

View File

@@ -51,7 +51,7 @@ namespace Semmle.Autobuild.CSharp.Tests
int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory, IDictionary<string, string>? env, out IList<string> stdOut)
{
var pattern = cmd + " " + args;
var pattern = string.IsNullOrEmpty(args) ? cmd : cmd + " " + args;
RunProcessIn.Add(pattern);
if (!RunProcessOut.TryGetValue(pattern, out var str))
@@ -62,7 +62,7 @@ namespace Semmle.Autobuild.CSharp.Tests
RunProcessWorkingDirectory.TryGetValue(pattern, out var wd);
if (wd != workingDirectory)
throw new ArgumentException("Missing RunProcessWorkingDirectory " + pattern);
throw new ArgumentException($"Unexpected RunProcessWorkingDirectory, got {wd ?? "null"} expected {workingDirectory ?? "null"} in {pattern}");
if (!RunProcess.TryGetValue(pattern, out var ret))
throw new ArgumentException("Missing RunProcess " + pattern);
@@ -72,12 +72,12 @@ namespace Semmle.Autobuild.CSharp.Tests
int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory, IDictionary<string, string>? env)
{
var pattern = cmd + " " + args;
var pattern = string.IsNullOrEmpty(args) ? cmd : cmd + " " + args;
RunProcessIn.Add(pattern);
RunProcessWorkingDirectory.TryGetValue(pattern, out var wd);
if (wd != workingDirectory)
throw new ArgumentException("Missing RunProcessWorkingDirectory " + pattern);
throw new ArgumentException($"Unexpected RunProcessWorkingDirectory, got {wd ?? "null"} expected {workingDirectory ?? "null"} in {pattern}");
if (!RunProcess.TryGetValue(pattern, out var ret))
throw new ArgumentException("Missing RunProcess " + pattern);
@@ -255,7 +255,7 @@ namespace Semmle.Autobuild.CSharp.Tests
[Fact]
public void TestAnd1()
{
var cmd = BuildScript.Create("abc", "def ghi", false, null, null) & BuildScript.Create("odasa", null, false, null, null);
var cmd = BuildScript.Create("abc", "def ghi", false, null, null) & BuildScript.Create("codeql", null, false, null, null);
actions.RunProcess["abc def ghi"] = 1;
cmd.Run(actions, StartCallback, EndCallback);
@@ -269,14 +269,14 @@ namespace Semmle.Autobuild.CSharp.Tests
[Fact]
public void TestAnd2()
{
var cmd = BuildScript.Create("odasa", null, false, null, null) & BuildScript.Create("abc", "def ghi", false, null, null);
var cmd = BuildScript.Create("codeql", null, false, null, null) & BuildScript.Create("abc", "def ghi", false, null, null);
actions.RunProcess["abc def ghi"] = 1;
actions.RunProcess["odasa "] = 0;
actions.RunProcess["codeql"] = 0;
cmd.Run(actions, StartCallback, EndCallback);
Assert.Equal("odasa ", actions.RunProcessIn[0]);
Assert.Equal("odasa ", startCallbackIn[0]);
Assert.Equal("codeql", actions.RunProcessIn[0]);
Assert.Equal("codeql", startCallbackIn[0]);
Assert.Equal("", endCallbackIn[0]);
Assert.Equal(0, endCallbackReturn[0]);
@@ -289,14 +289,14 @@ namespace Semmle.Autobuild.CSharp.Tests
[Fact]
public void TestOr1()
{
var cmd = BuildScript.Create("odasa", null, false, null, null) | BuildScript.Create("abc", "def ghi", false, null, null);
var cmd = BuildScript.Create("codeql", null, false, null, null) | BuildScript.Create("abc", "def ghi", false, null, null);
actions.RunProcess["abc def ghi"] = 1;
actions.RunProcess["odasa "] = 0;
actions.RunProcess["codeql"] = 0;
cmd.Run(actions, StartCallback, EndCallback);
Assert.Equal("odasa ", actions.RunProcessIn[0]);
Assert.Equal("odasa ", startCallbackIn[0]);
Assert.Equal("codeql", actions.RunProcessIn[0]);
Assert.Equal("codeql", startCallbackIn[0]);
Assert.Equal("", endCallbackIn[0]);
Assert.Equal(0, endCallbackReturn[0]);
Assert.Equal(1, endCallbackReturn.Count);
@@ -305,10 +305,10 @@ namespace Semmle.Autobuild.CSharp.Tests
[Fact]
public void TestOr2()
{
var cmd = BuildScript.Create("abc", "def ghi", false, null, null) | BuildScript.Create("odasa", null, false, null, null);
var cmd = BuildScript.Create("abc", "def ghi", false, null, null) | BuildScript.Create("codeql", null, false, null, null);
actions.RunProcess["abc def ghi"] = 1;
actions.RunProcess["odasa "] = 0;
actions.RunProcess["codeql"] = 0;
cmd.Run(actions, StartCallback, EndCallback);
Assert.Equal("abc def ghi", actions.RunProcessIn[0]);
@@ -316,8 +316,8 @@ namespace Semmle.Autobuild.CSharp.Tests
Assert.Equal("", endCallbackIn[0]);
Assert.Equal(1, endCallbackReturn[0]);
Assert.Equal("odasa ", actions.RunProcessIn[1]);
Assert.Equal("odasa ", startCallbackIn[1]);
Assert.Equal("codeql", actions.RunProcessIn[1]);
Assert.Equal("codeql", startCallbackIn[1]);
Assert.Equal("", endCallbackIn[1]);
Assert.Equal(0, endCallbackReturn[1]);
}
@@ -385,9 +385,6 @@ namespace Semmle.Autobuild.CSharp.Tests
actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_ROOT"] = $@"C:\codeql\{codeqlUpperLanguage.ToLowerInvariant()}";
actions.GetEnvironmentVariable["CODEQL_JAVA_HOME"] = @"C:\codeql\tools\java";
actions.GetEnvironmentVariable["CODEQL_PLATFORM"] = isWindows ? "win64" : "linux64";
actions.GetEnvironmentVariable["SEMMLE_DIST"] = @"C:\odasa";
actions.GetEnvironmentVariable["SEMMLE_JAVA_HOME"] = @"C:\odasa\tools\java";
actions.GetEnvironmentVariable["SEMMLE_PLATFORM_TOOLS"] = @"C:\odasa\tools";
actions.GetEnvironmentVariable["LGTM_INDEX_VSTOOLS_VERSION"] = vsToolsVersion;
actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_ARGUMENTS"] = msBuildArguments;
actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_PLATFORM"] = msBuildPlatform;
@@ -416,7 +413,7 @@ namespace Semmle.Autobuild.CSharp.Tests
actions.RunProcess["cmd.exe /C dotnet --info"] = 0;
actions.RunProcess[@"cmd.exe /C dotnet clean C:\Project\test.csproj"] = 0;
actions.RunProcess[@"cmd.exe /C dotnet restore C:\Project\test.csproj"] = 0;
actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --auto dotnet build --no-incremental /p:UseSharedCompilation=false C:\Project\test.csproj"] = 0;
actions.RunProcess[@"cmd.exe /C dotnet build --no-incremental /p:UseSharedCompilation=false C:\Project\test.csproj"] = 0;
actions.FileExists["csharp.log"] = true;
actions.FileExists[@"C:\Project\test.csproj"] = true;
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
@@ -443,7 +440,7 @@ namespace Semmle.Autobuild.CSharp.Tests
actions.RunProcess["dotnet --info"] = 0;
actions.RunProcess[@"dotnet clean C:\Project/test.csproj"] = 0;
actions.RunProcess[@"dotnet restore C:\Project/test.csproj"] = 0;
actions.RunProcess[@"C:\odasa/tools/odasa index --auto dotnet build --no-incremental /p:UseSharedCompilation=false C:\Project/test.csproj"] = 0;
actions.RunProcess[@"dotnet build --no-incremental /p:UseSharedCompilation=false C:\Project/test.csproj"] = 0;
actions.FileExists["csharp.log"] = true;
actions.FileExists[@"C:\Project/test.csproj"] = true;
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
@@ -601,7 +598,7 @@ namespace Semmle.Autobuild.CSharp.Tests
[Fact]
public void TestLinuxBuildCommand()
{
actions.RunProcess[@"C:\odasa/tools/odasa index --auto ""./build.sh --skip-tests"""] = 0;
actions.RunProcess["./build.sh --skip-tests"] = 0;
actions.FileExists["csharp.log"] = true;
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
@@ -622,8 +619,8 @@ namespace Semmle.Autobuild.CSharp.Tests
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
actions.RunProcess[@"/bin/chmod u+x C:\Project/build/build.sh"] = 0;
actions.RunProcess[@"C:\odasa/tools/odasa index --auto C:\Project/build/build.sh"] = 0;
actions.RunProcessWorkingDirectory[@"C:\odasa/tools/odasa index --auto C:\Project/build/build.sh"] = @"C:\Project/build";
actions.RunProcess[@"C:\Project/build/build.sh"] = 0;
actions.RunProcessWorkingDirectory[@"C:\Project/build/build.sh"] = @"C:\Project/build";
actions.FileExists["csharp.log"] = true;
var autobuilder = CreateAutoBuilder(false);
@@ -639,8 +636,8 @@ namespace Semmle.Autobuild.CSharp.Tests
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
actions.RunProcess[@"/bin/chmod u+x C:\Project/build.sh"] = 0;
actions.RunProcess[@"C:\odasa/tools/odasa index --auto C:\Project/build.sh"] = 0;
actions.RunProcessWorkingDirectory[@"C:\odasa/tools/odasa index --auto C:\Project/build.sh"] = @"C:\Project";
actions.RunProcess[@"C:\Project/build.sh"] = 0;
actions.RunProcessWorkingDirectory[@"C:\Project/build.sh"] = @"C:\Project";
actions.FileExists["csharp.log"] = false;
var autobuilder = CreateAutoBuilder(false);
@@ -656,8 +653,8 @@ namespace Semmle.Autobuild.CSharp.Tests
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
actions.RunProcess[@"/bin/chmod u+x C:\Project/build.sh"] = 0;
actions.RunProcess[@"C:\odasa/tools/odasa index --auto C:\Project/build.sh"] = 5;
actions.RunProcessWorkingDirectory[@"C:\odasa/tools/odasa index --auto C:\Project/build.sh"] = @"C:\Project";
actions.RunProcess[@"C:\Project/build.sh"] = 5;
actions.RunProcessWorkingDirectory[@"C:\Project/build.sh"] = @"C:\Project";
actions.FileExists["csharp.log"] = true;
var autobuilder = CreateAutoBuilder(false);
@@ -671,8 +668,8 @@ namespace Semmle.Autobuild.CSharp.Tests
actions.EnumerateDirectories[@"C:\Project"] = "";
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --auto C:\Project\build.bat"] = 0;
actions.RunProcessWorkingDirectory[@"cmd.exe /C C:\odasa\tools\odasa index --auto C:\Project\build.bat"] = @"C:\Project";
actions.RunProcess[@"cmd.exe /C C:\Project\build.bat"] = 0;
actions.RunProcessWorkingDirectory[@"cmd.exe /C C:\Project\build.bat"] = @"C:\Project";
actions.FileExists["csharp.log"] = true;
var autobuilder = CreateAutoBuilder(true);
@@ -686,10 +683,10 @@ namespace Semmle.Autobuild.CSharp.Tests
actions.EnumerateDirectories[@"C:\Project"] = "";
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --auto C:\Project\build.bat"] = 1;
actions.RunProcessWorkingDirectory[@"cmd.exe /C C:\odasa\tools\odasa index --auto C:\Project\build.bat"] = @"C:\Project";
actions.RunProcess[@"cmd.exe /C C:\Project\build.bat"] = 1;
actions.RunProcessWorkingDirectory[@"cmd.exe /C C:\Project\build.bat"] = @"C:\Project";
actions.RunProcess[@"cmd.exe /C C:\codeql\tools\java\bin\java -jar C:\codeql\csharp\tools\extractor-asp.jar ."] = 0;
actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --xml --extensions config"] = 0;
actions.RunProcess[@"cmd.exe /C C:\codeql\tools\codeql index --xml --extensions config"] = 0;
actions.FileExists["csharp.log"] = true;
var autobuilder = CreateAutoBuilder(true, ignoreErrors: "true");
@@ -699,9 +696,9 @@ namespace Semmle.Autobuild.CSharp.Tests
[Fact]
public void TestWindowsCmdIgnoreErrors()
{
actions.RunProcess["cmd.exe /C C:\\odasa\\tools\\odasa index --auto ^\"build.cmd --skip-tests^\""] = 3;
actions.RunProcess["cmd.exe /C ^\"build.cmd --skip-tests^\""] = 3;
actions.RunProcess[@"cmd.exe /C C:\codeql\tools\java\bin\java -jar C:\codeql\csharp\tools\extractor-asp.jar ."] = 0;
actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --xml --extensions config"] = 0;
actions.RunProcess[@"cmd.exe /C C:\codeql\tools\codeql index --xml --extensions config"] = 0;
actions.FileExists["csharp.log"] = true;
SkipVsWhere();
@@ -718,9 +715,9 @@ namespace Semmle.Autobuild.CSharp.Tests
public void TestWindowCSharpMsBuild()
{
actions.RunProcess[@"cmd.exe /C C:\Project\.nuget\nuget.exe restore C:\Project\test1.sln -DisableParallelProcessing"] = 0;
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program Files ^(x86^)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && C:\\odasa\\tools\\odasa index --auto msbuild C:\\Project\\test1.sln /p:UseSharedCompilation=false /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /p:MvcBuildViews=true /P:Fu=Bar"] = 0;
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program Files ^(x86^)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test1.sln /p:UseSharedCompilation=false /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /p:MvcBuildViews=true /P:Fu=Bar"] = 0;
actions.RunProcess[@"cmd.exe /C C:\Project\.nuget\nuget.exe restore C:\Project\test2.sln -DisableParallelProcessing"] = 0;
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program Files ^(x86^)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && C:\\odasa\\tools\\odasa index --auto msbuild C:\\Project\\test2.sln /p:UseSharedCompilation=false /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /p:MvcBuildViews=true /P:Fu=Bar"] = 0;
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program Files ^(x86^)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test2.sln /p:UseSharedCompilation=false /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /p:MvcBuildViews=true /P:Fu=Bar"] = 0;
actions.FileExists["csharp.log"] = true;
actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = false;
actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = false;
@@ -749,9 +746,9 @@ namespace Semmle.Autobuild.CSharp.Tests
public void TestWindowCSharpMsBuildMultipleSolutions()
{
actions.RunProcess[@"cmd.exe /C nuget restore C:\Project\test1.csproj -DisableParallelProcessing"] = 0;
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program Files ^(x86^)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && C:\\odasa\\tools\\odasa index --auto msbuild C:\\Project\\test1.csproj /p:UseSharedCompilation=false /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /p:MvcBuildViews=true /P:Fu=Bar"] = 0;
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program Files ^(x86^)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test1.csproj /p:UseSharedCompilation=false /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /p:MvcBuildViews=true /P:Fu=Bar"] = 0;
actions.RunProcess[@"cmd.exe /C nuget restore C:\Project\test2.csproj -DisableParallelProcessing"] = 0;
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program Files ^(x86^)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && C:\\odasa\\tools\\odasa index --auto msbuild C:\\Project\\test2.csproj /p:UseSharedCompilation=false /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /p:MvcBuildViews=true /P:Fu=Bar"] = 0;
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program Files ^(x86^)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test2.csproj /p:UseSharedCompilation=false /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /p:MvcBuildViews=true /P:Fu=Bar"] = 0;
actions.FileExists["csharp.log"] = true;
actions.FileExists[@"C:\Project\test1.csproj"] = true;
actions.FileExists[@"C:\Project\test2.csproj"] = true;
@@ -794,7 +791,7 @@ namespace Semmle.Autobuild.CSharp.Tests
public void TestWindowCSharpMsBuildFailed()
{
actions.RunProcess[@"cmd.exe /C nuget restore C:\Project\test1.sln -DisableParallelProcessing"] = 0;
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program Files ^(x86^)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && C:\\odasa\\tools\\odasa index --auto msbuild C:\\Project\\test1.sln /p:UseSharedCompilation=false /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /p:MvcBuildViews=true /P:Fu=Bar"] = 1;
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program Files ^(x86^)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test1.sln /p:UseSharedCompilation=false /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /p:MvcBuildViews=true /P:Fu=Bar"] = 1;
actions.FileExists["csharp.log"] = true;
actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = false;
actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = false;
@@ -820,8 +817,8 @@ namespace Semmle.Autobuild.CSharp.Tests
[Fact]
public void TestSkipNugetMsBuild()
{
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program Files ^(x86^)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && C:\\odasa\\tools\\odasa index --auto msbuild C:\\Project\\test1.sln /p:UseSharedCompilation=false /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /p:MvcBuildViews=true /P:Fu=Bar"] = 0;
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program Files ^(x86^)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && C:\\odasa\\tools\\odasa index --auto msbuild C:\\Project\\test2.sln /p:UseSharedCompilation=false /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /p:MvcBuildViews=true /P:Fu=Bar"] = 0;
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program Files ^(x86^)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test1.sln /p:UseSharedCompilation=false /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /p:MvcBuildViews=true /P:Fu=Bar"] = 0;
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program Files ^(x86^)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test2.sln /p:UseSharedCompilation=false /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /p:MvcBuildViews=true /P:Fu=Bar"] = 0;
actions.FileExists["csharp.log"] = true;
actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = false;
actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = false;
@@ -865,7 +862,7 @@ namespace Semmle.Autobuild.CSharp.Tests
actions.RunProcess["dotnet --info"] = 0;
actions.RunProcess[@"dotnet clean C:\Project/test.csproj"] = 0;
actions.RunProcess[@"dotnet restore C:\Project/test.csproj"] = 0;
actions.RunProcess[@"C:\odasa/tools/odasa index --auto dotnet build --no-incremental /p:UseSharedCompilation=false --no-restore C:\Project/test.csproj"] = 0;
actions.RunProcess[@"dotnet build --no-incremental /p:UseSharedCompilation=false --no-restore C:\Project/test.csproj"] = 0;
actions.FileExists["csharp.log"] = true;
actions.FileExists[@"C:\Project/test.csproj"] = true;
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
@@ -897,7 +894,7 @@ namespace Semmle.Autobuild.CSharp.Tests
actions.RunProcess[@"C:\Project/.dotnet/dotnet --info"] = 0;
actions.RunProcess[@"C:\Project/.dotnet/dotnet clean C:\Project/test.csproj"] = 0;
actions.RunProcess[@"C:\Project/.dotnet/dotnet restore C:\Project/test.csproj"] = 0;
actions.RunProcess[@"C:\odasa/tools/odasa index --auto C:\Project/.dotnet/dotnet build --no-incremental /p:UseSharedCompilation=false C:\Project/test.csproj"] = 0;
actions.RunProcess[@"C:\Project/.dotnet/dotnet build --no-incremental /p:UseSharedCompilation=false C:\Project/test.csproj"] = 0;
actions.FileExists["csharp.log"] = true;
actions.FileExists["test.csproj"] = true;
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
@@ -932,7 +929,7 @@ namespace Semmle.Autobuild.CSharp.Tests
actions.RunProcess[@"C:\Project/.dotnet/dotnet --info"] = 0;
actions.RunProcess[@"C:\Project/.dotnet/dotnet clean C:\Project/test.csproj"] = 0;
actions.RunProcess[@"C:\Project/.dotnet/dotnet restore C:\Project/test.csproj"] = 0;
actions.RunProcess[@"C:\odasa/tools/odasa index --auto C:\Project/.dotnet/dotnet build --no-incremental /p:UseSharedCompilation=false C:\Project/test.csproj"] = 0;
actions.RunProcess[@"C:\Project/.dotnet/dotnet build --no-incremental /p:UseSharedCompilation=false C:\Project/test.csproj"] = 0;
actions.FileExists["csharp.log"] = true;
actions.FileExists["test.csproj"] = true;
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
@@ -963,7 +960,7 @@ namespace Semmle.Autobuild.CSharp.Tests
actions.RunProcess[@"cmd.exe /C C:\Project\.dotnet\dotnet --info"] = 0;
actions.RunProcess[@"cmd.exe /C C:\Project\.dotnet\dotnet clean C:\Project\test.csproj"] = 0;
actions.RunProcess[@"cmd.exe /C C:\Project\.dotnet\dotnet restore C:\Project\test.csproj"] = 0;
actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --auto C:\Project\.dotnet\dotnet build --no-incremental /p:UseSharedCompilation=false C:\Project\test.csproj"] = 0;
actions.RunProcess[@"cmd.exe /C C:\Project\.dotnet\dotnet build --no-incremental /p:UseSharedCompilation=false C:\Project\test.csproj"] = 0;
actions.FileExists["csharp.log"] = true;
actions.FileExists[@"C:\Project\test.csproj"] = true;
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
@@ -1011,7 +1008,7 @@ namespace Semmle.Autobuild.CSharp.Tests
{
actions.RunProcess[@"cmd.exe /C nuget restore C:\Project\dirs.proj -DisableParallelProcessing"] = 1;
actions.RunProcess[@"cmd.exe /C C:\Project\.nuget\nuget.exe restore C:\Project\dirs.proj -DisableParallelProcessing"] = 0;
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program Files ^(x86^)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && C:\\odasa\\tools\\odasa index --auto msbuild C:\\Project\\dirs.proj /p:UseSharedCompilation=false /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /p:MvcBuildViews=true /P:Fu=Bar"] = 0;
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program Files ^(x86^)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\dirs.proj /p:UseSharedCompilation=false /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /p:MvcBuildViews=true /P:Fu=Bar"] = 0;
actions.FileExists["csharp.log"] = true;
actions.FileExists[@"C:\Project\a\test.csproj"] = true;
actions.FileExists[@"C:\Project\dirs.proj"] = true;
@@ -1055,7 +1052,7 @@ namespace Semmle.Autobuild.CSharp.Tests
{
actions.RunProcess[@"nuget restore C:\Project/dirs.proj -DisableParallelProcessing"] = 1;
actions.RunProcess[@"mono C:\Project/.nuget/nuget.exe restore C:\Project/dirs.proj -DisableParallelProcessing"] = 0;
actions.RunProcess[@"C:\odasa/tools/odasa index --auto msbuild C:\Project/dirs.proj /p:UseSharedCompilation=false /t:rebuild /p:MvcBuildViews=true"] = 0;
actions.RunProcess[@"msbuild C:\Project/dirs.proj /p:UseSharedCompilation=false /t:rebuild /p:MvcBuildViews=true"] = 0;
actions.FileExists["csharp.log"] = true;
actions.FileExists[@"C:\Project/a/test.csproj"] = true;
actions.FileExists[@"C:\Project/dirs.proj"] = true;

View File

@@ -240,7 +240,7 @@ namespace Semmle.Autobuild.CSharp
private static BuildScript GetBuildScript(Autobuilder builder, string? dotNetPath, IDictionary<string, string>? environment, string projOrSln)
{
var build = new CommandBuilder(builder.Actions, null, environment);
var script = builder.MaybeIndex(build, DotNetCommand(builder.Actions, dotNetPath)).
var script = build.RunCommand(DotNetCommand(builder.Actions, dotNetPath)).
Argument("build").
Argument("--no-incremental");

View File

@@ -17,10 +17,6 @@ namespace Semmle.Autobuild.CSharp
{
standalone = builder.Actions.PathCombine(builder.CodeQLExtractorLangRoot, "tools", builder.CodeQlPlatform, "Semmle.Extraction.CSharp.Standalone");
}
else if (builder.SemmlePlatformTools is not null)
{
standalone = builder.Actions.PathCombine(builder.SemmlePlatformTools, "csharp", "Semmle.Extraction.CSharp.Standalone");
}
else
{
return BuildScript.Failure;

View File

@@ -190,18 +190,14 @@ namespace Semmle.Autobuild.Shared
});
CodeQLExtractorLangRoot = Actions.GetEnvironmentVariable($"CODEQL_EXTRACTOR_{this.Options.Language.UpperCaseName}_ROOT");
SemmlePlatformTools = Actions.GetEnvironmentVariable("SEMMLE_PLATFORM_TOOLS");
CodeQlPlatform = Actions.GetEnvironmentVariable("CODEQL_PLATFORM");
TrapDir =
Actions.GetEnvironmentVariable($"CODEQL_EXTRACTOR_{this.Options.Language.UpperCaseName}_TRAP_DIR") ??
Actions.GetEnvironmentVariable("TRAP_FOLDER") ??
throw new InvalidEnvironmentException($"The environment variable CODEQL_EXTRACTOR_{this.Options.Language.UpperCaseName}_TRAP_DIR or TRAP_FOLDER has not been set.");
throw new InvalidEnvironmentException($"The environment variable CODEQL_EXTRACTOR_{this.Options.Language.UpperCaseName}_TRAP_DIR has not been set.");
SourceArchiveDir =
Actions.GetEnvironmentVariable($"CODEQL_EXTRACTOR_{this.Options.Language.UpperCaseName}_SOURCE_ARCHIVE_DIR") ??
Actions.GetEnvironmentVariable("SOURCE_ARCHIVE") ??
throw new InvalidEnvironmentException($"The environment variable CODEQL_EXTRACTOR_{this.Options.Language.UpperCaseName}_SOURCE_ARCHIVE_DIR or SOURCE_ARCHIVE has not been set.");
}
@@ -264,34 +260,9 @@ namespace Semmle.Autobuild.Shared
/// </summary>
public string? CodeQLExtractorLangRoot { get; }
/// <summary>
/// Value of SEMMLE_PLATFORM_TOOLS environment variable.
/// </summary>
public string? SemmlePlatformTools { get; }
/// <summary>
/// Value of CODEQL_PLATFORM environment variable.
/// </summary>
public string? CodeQlPlatform { get; }
/// <summary>
/// The absolute path of the odasa executable.
/// null if we are running in CodeQL.
/// </summary>
public string? Odasa
{
get
{
var semmleDist = Actions.GetEnvironmentVariable("SEMMLE_DIST");
return semmleDist is null ? null : Actions.PathCombine(semmleDist, "tools", "odasa");
}
}
/// <summary>
/// Construct a command that executed the given <paramref name="cmd"/> wrapped in
/// an <code>odasa --index</code>, unless indexing has been disabled, in which case
/// <paramref name="cmd"/> is run directly.
/// </summary>
public CommandBuilder MaybeIndex(CommandBuilder builder, string cmd) => Odasa is null ? builder.RunCommand(cmd) : builder.IndexCommand(Odasa, cmd);
}
}

View File

@@ -58,7 +58,7 @@ namespace Semmle.Autobuild.Shared
if (vsTools is not null)
command.CallBatFile(vsTools.Path);
builder.MaybeIndex(command, scriptPath);
command.RunCommand(scriptPath);
return command.Script;
});
}

View File

@@ -26,7 +26,7 @@
var vsTools = MsBuildRule.GetVcVarsBatFile(builder);
if (vsTools is not null)
command.CallBatFile(vsTools.Path);
builder.MaybeIndex(command, builder.Options.BuildCommand);
command.RunCommand(builder.Options.BuildCommand);
return command.Script;
});

View File

@@ -70,7 +70,7 @@ namespace Semmle.Autobuild.Shared
this.environment = environment;
}
public override string ToString() => exe + " " + arguments;
public override string ToString() => arguments.Length > 0 ? exe + " " + arguments : exe;
public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack)
{

View File

@@ -45,11 +45,6 @@ namespace Semmle.Autobuild.Shared
this.silent = silent;
}
private void OdasaIndex(string odasa)
{
RunCommand(odasa, "index --auto");
}
public CommandBuilder CallBatFile(string batFile, string? argumentsOpt = null)
{
NextCommand();
@@ -59,21 +54,6 @@ namespace Semmle.Autobuild.Shared
return this;
}
/// <summary>
/// Perform odasa index on a given command or BAT file.
/// </summary>
/// <param name="odasa">The odasa executable.</param>
/// <param name="command">The command to run.</param>
/// <param name="argumentsOpt">Additional arguments.</param>
/// <returns>this for chaining calls.</returns>
public CommandBuilder IndexCommand(string odasa, string command, string? argumentsOpt = null)
{
OdasaIndex(odasa);
QuoteArgument(command);
Argument(argumentsOpt);
return this;
}
private static readonly char[] specialChars = { ' ', '\t', '\n', '\v', '\"' };
private static readonly char[] cmdMetacharacter = { '(', ')', '%', '!', '^', '\"', '<', '>', '&', '|' };

View File

@@ -95,7 +95,7 @@ namespace Semmle.Autobuild.Shared
command.RunCommand("set Platform=&& type NUL", quoteExe: false);
}
builder.MaybeIndex(command, msBuild);
command.RunCommand(msBuild);
command.QuoteArgument(projectOrSolution.FullPath);
command.Argument("/p:UseSharedCompilation=false");

View File

@@ -1,14 +0,0 @@
**/mcs.exe:
**/csc.exe:
invoke ${env.SEMMLE_PLATFORM_TOOLS}/csharp/Semmle.Extraction.CSharp.Driver
prepend --compiler
prepend "${compiler}"
prepend --cil
**/mono*:
**/dotnet:
invoke ${odasa_tools}/extract-csharp.sh
**/msbuild:
**/xbuild:
replace yes
invoke ${compiler}
append /p:UseSharedCompilation=false

View File

@@ -1,16 +0,0 @@
#!/bin/bash
echo extract-csharp.sh: Called with arguments: "$@"
extractor=$SEMMLE_PLATFORM_TOOLS/csharp/Semmle.Extraction.CSharp.Driver
for i in "$@"
do
shift
if [[ `basename -- "$i"` =~ csc.exe|mcs.exe|csc.dll ]]
then
echo extract-csharp.sh: exec $extractor --cil $@
exec "$extractor" --compiler $i --cil $@
fi
done
echo extract-csharp.sh: Not a compiler invocation

View File

@@ -1,9 +0,0 @@
**\fakes*.exe:
**\moles*.exe:
order compiler
trace no
**\csc*.exe:
invoke ${env.SEMMLE_PLATFORM_TOOLS}\csharp\Semmle.Extraction.CSharp.Driver.exe
prepend --compiler
prepend "${compiler}"
prepend --cil

View File

@@ -20,11 +20,11 @@ namespace Semmle.Extraction.CIL.Driver
Console.WriteLine(" path A directory/dll/exe to analyze");
}
private static void ExtractAssembly(Layout layout, string assemblyPath, ILogger logger, CommonOptions options)
private static void ExtractAssembly(string assemblyPath, ILogger logger, CommonOptions options)
{
var sw = new Stopwatch();
sw.Start();
Analyser.ExtractCIL(layout, assemblyPath, logger, options, out _, out _);
Analyser.ExtractCIL(assemblyPath, logger, options, out _, out _);
sw.Stop();
logger.Log(Severity.Info, " {0} ({1})", assemblyPath, sw.Elapsed);
}
@@ -38,12 +38,11 @@ namespace Semmle.Extraction.CIL.Driver
}
var options = new ExtractorOptions(args);
var layout = new Layout();
using var logger = new ConsoleLogger(options.Verbosity);
var actions = options.AssembliesToExtract
.Select(asm => asm.Filename)
.Select<string, Action>(filename => () => ExtractAssembly(layout, filename, logger, options))
.Select<string, Action>(filename => () => ExtractAssembly(filename, logger, options))
.ToArray();
foreach (var missingRef in options.MissingReferences)

View File

@@ -25,7 +25,7 @@ namespace Semmle.Extraction.CIL
/// <param name="extractPdbs">Whether to extract PDBs.</param>
/// <param name="trapFile">The path of the trap file.</param>
/// <param name="extracted">Whether the file was extracted (false=cached).</param>
public static void ExtractCIL(Layout layout, string assemblyPath, ILogger logger, CommonOptions options, out string trapFile, out bool extracted)
public static void ExtractCIL(string assemblyPath, ILogger logger, CommonOptions options, out string trapFile, out bool extracted)
{
trapFile = "";
extracted = false;
@@ -35,8 +35,7 @@ namespace Semmle.Extraction.CIL
var pathTransformer = new PathTransformer(canonicalPathCache);
var extractor = new TracingExtractor(assemblyPath, logger, pathTransformer, options);
var transformedAssemblyPath = pathTransformer.Transform(assemblyPath);
var project = layout.LookupProjectOrDefault(transformedAssemblyPath);
using var trapWriter = project.CreateTrapWriter(logger, transformedAssemblyPath.WithSuffix(".cil"), options.TrapCompression, discardDuplicates: true);
using var trapWriter = transformedAssemblyPath.WithSuffix(".cil").CreateTrapWriter(logger, options.TrapCompression, discardDuplicates: true);
trapFile = trapWriter.TrapFile;
if (!options.Cache || !System.IO.File.Exists(trapFile))
{

View File

@@ -84,7 +84,7 @@ namespace Semmle.Extraction.CSharp.Standalone
foreach (var r in DesktopRuntimes)
yield return r;
// A bad choice if it's the self-contained runtime distributed in odasa dist.
// A bad choice if it's the self-contained runtime distributed in codeql dist.
yield return ExecutingRuntime;
}
}

View File

@@ -14,7 +14,6 @@ namespace Semmle.Extraction.CSharp
public void Initialize(CSharpCompilation compilationIn, CommonOptions options)
{
compilation = compilationIn;
layout = new Layout();
extractor = new StandaloneExtractor(Logger, PathTransformer, options);
this.options = options;
LogExtractorInfo(Extraction.Extractor.Version);

View File

@@ -18,7 +18,6 @@ namespace Semmle.Extraction.CSharp
{
protected Extraction.Extractor? extractor;
protected CSharpCompilation? compilation;
protected Layout? layout;
protected CommonOptions? options;
private readonly object progressMutex = new object();
@@ -125,8 +124,7 @@ namespace Semmle.Extraction.CSharp
var assemblyPath = r.FilePath!;
var transformedAssemblyPath = PathTransformer.Transform(assemblyPath);
var projectLayout = layout.LookupProjectOrDefault(transformedAssemblyPath);
using var trapWriter = projectLayout.CreateTrapWriter(Logger, transformedAssemblyPath, options.TrapCompression, discardDuplicates: true);
using var trapWriter = transformedAssemblyPath.CreateTrapWriter(Logger, options.TrapCompression, discardDuplicates: true);
var skipExtraction = options.Cache && File.Exists(trapWriter.TrapFile);
@@ -178,7 +176,7 @@ namespace Semmle.Extraction.CSharp
{
var stopwatch = new Stopwatch();
stopwatch.Start();
CIL.Analyser.ExtractCIL(layout, r.FilePath!, Logger, options, out var trapFile, out var extracted);
CIL.Analyser.ExtractCIL(r.FilePath!, Logger, options, out var trapFile, out var extracted);
stopwatch.Stop();
ReportProgress(r.FilePath, trapFile, stopwatch.Elapsed, extracted ? AnalysisAction.Extracted : AnalysisAction.UpToDate);
}
@@ -192,44 +190,35 @@ namespace Semmle.Extraction.CSharp
var sourcePath = tree.FilePath;
var transformedSourcePath = PathTransformer.Transform(sourcePath);
var projectLayout = layout.LookupProjectOrNull(transformedSourcePath);
var excluded = projectLayout is null;
var trapPath = excluded ? "" : projectLayout!.GetTrapPath(Logger, transformedSourcePath, options.TrapCompression);
var trapPath = transformedSourcePath.GetTrapPath(Logger, options.TrapCompression);
var upToDate = false;
if (!excluded)
// compilation.Clone() is used to allow symbols to be garbage collected.
using var trapWriter = transformedSourcePath.CreateTrapWriter(Logger, options.TrapCompression, discardDuplicates: false);
upToDate = options.Fast && FileIsUpToDate(sourcePath, trapWriter.TrapFile);
if (!upToDate)
{
// compilation.Clone() is used to allow symbols to be garbage collected.
using var trapWriter = projectLayout!.CreateTrapWriter(Logger, transformedSourcePath, options.TrapCompression, discardDuplicates: false);
var cx = new Context(extractor, compilation.Clone(), trapWriter, new SourceScope(tree), addAssemblyTrapPrefix);
// Ensure that the file itself is populated in case the source file is totally empty
var root = tree.GetRoot();
Entities.File.Create(cx, root.SyntaxTree.FilePath);
upToDate = options.Fast && FileIsUpToDate(sourcePath, trapWriter.TrapFile);
if (!upToDate)
var csNode = (CSharpSyntaxNode)root;
var directiveVisitor = new DirectiveVisitor(cx);
csNode.Accept(directiveVisitor);
foreach (var branch in directiveVisitor.BranchesTaken)
{
var cx = new Context(extractor, compilation.Clone(), trapWriter, new SourceScope(tree), addAssemblyTrapPrefix);
// Ensure that the file itself is populated in case the source file is totally empty
var root = tree.GetRoot();
Entities.File.Create(cx, root.SyntaxTree.FilePath);
var csNode = (CSharpSyntaxNode)root;
var directiveVisitor = new DirectiveVisitor(cx);
csNode.Accept(directiveVisitor);
foreach (var branch in directiveVisitor.BranchesTaken)
{
cx.TrapStackSuffix.Add(branch);
}
csNode.Accept(new CompilationUnitVisitor(cx));
cx.PopulateAll();
CommentPopulator.ExtractCommentBlocks(cx, cx.CommentGenerator);
cx.PopulateAll();
cx.TrapStackSuffix.Add(branch);
}
csNode.Accept(new CompilationUnitVisitor(cx));
cx.PopulateAll();
CommentPopulator.ExtractCommentBlocks(cx, cx.CommentGenerator);
cx.PopulateAll();
}
ReportProgress(sourcePath, trapPath, stopwatch.Elapsed, excluded
? AnalysisAction.Excluded
: upToDate
? AnalysisAction.UpToDate
: AnalysisAction.Extracted);
ReportProgress(sourcePath, trapPath, stopwatch.Elapsed, upToDate ? AnalysisAction.UpToDate : AnalysisAction.Extracted);
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{

View File

@@ -452,19 +452,6 @@ namespace Semmle.Extraction.CSharp
if (!string.IsNullOrEmpty(codeQlLogDir))
return codeQlLogDir;
var snapshot = Environment.GetEnvironmentVariable("ODASA_SNAPSHOT");
if (!string.IsNullOrEmpty(snapshot))
return Path.Combine(snapshot, "log");
var buildErrorDir = Environment.GetEnvironmentVariable("ODASA_BUILD_ERROR_DIR");
if (!string.IsNullOrEmpty(buildErrorDir))
// Used by `qltest`
return buildErrorDir;
var traps = Environment.GetEnvironmentVariable("TRAP_FOLDER");
if (!string.IsNullOrEmpty(traps))
return traps;
return Directory.GetCurrentDirectory();
}
}

View File

@@ -40,8 +40,7 @@ namespace Semmle.Extraction.CSharp
public static Options CreateWithEnvironment(string[] arguments)
{
var options = new Options();
var extractionOptions = Environment.GetEnvironmentVariable("SEMMLE_EXTRACTOR_OPTIONS") ??
Environment.GetEnvironmentVariable("LGTM_INDEX_EXTRACTOR");
var extractionOptions = Environment.GetEnvironmentVariable("LGTM_INDEX_EXTRACTOR");
var argsList = new List<string>(arguments);

View File

@@ -46,7 +46,6 @@ namespace Semmle.Extraction.CSharp
{
if (!init)
throw new InternalError("EndInitialize called without BeginInitialize returning true");
this.layout = new Layout();
this.options = options;
this.compilation = compilation;
this.extractor = new TracingExtractor(GetOutputName(compilation, commandLineArguments), Logger, PathTransformer, options);
@@ -202,8 +201,7 @@ namespace Semmle.Extraction.CSharp
var assemblyPath = ((TracingExtractor?)extractor).OutputPath;
var transformedAssemblyPath = PathTransformer.Transform(assemblyPath);
var assembly = compilation.Assembly;
var projectLayout = layout.LookupProjectOrDefault(transformedAssemblyPath);
var trapWriter = projectLayout.CreateTrapWriter(Logger, transformedAssemblyPath, options.TrapCompression, discardDuplicates: false);
var trapWriter = transformedAssemblyPath.CreateTrapWriter(Logger, options.TrapCompression, discardDuplicates: false);
compilationTrapFile = trapWriter; // Dispose later
var cx = new Context(extractor, compilation.Clone(), trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix);

View File

@@ -1,231 +0,0 @@
using System.IO;
using Xunit;
using Semmle.Util.Logging;
using System.Runtime.InteropServices;
namespace Semmle.Extraction.Tests
{
internal struct TransformedPathStub : PathTransformer.ITransformedPath
{
private readonly string value;
public TransformedPathStub(string value) => this.value = value;
public string Value => value;
public string Extension => throw new System.NotImplementedException();
public string NameWithoutExtension => throw new System.NotImplementedException();
public PathTransformer.ITransformedPath ParentDirectory => throw new System.NotImplementedException();
public string DatabaseId => throw new System.NotImplementedException();
public PathTransformer.ITransformedPath WithSuffix(string suffix)
{
throw new System.NotImplementedException();
}
}
public class Layout
{
private readonly ILogger logger = new LoggerMock();
[Fact]
public void TestDefaultLayout()
{
var layout = new Semmle.Extraction.Layout(null, null, null);
var project = layout.LookupProjectOrNull(new TransformedPathStub("foo.cs"));
Assert.NotNull(project);
// All files are mapped when there's no layout file.
Assert.True(layout.FileInLayout(new TransformedPathStub("foo.cs")));
// Test trap filename
var tmpDir = Path.GetTempPath();
Directory.SetCurrentDirectory(tmpDir);
if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
// `Directory.SetCurrentDirectory()` seems to slightly change the path on macOS,
// so adjusting it:
Assert.NotEqual(Directory.GetCurrentDirectory(), tmpDir);
tmpDir = "/private" + tmpDir;
// Remove trailing slash:
Assert.Equal('/', tmpDir[tmpDir.Length - 1]);
tmpDir = tmpDir.Substring(0, tmpDir.Length - 1);
Assert.Equal(Directory.GetCurrentDirectory(), tmpDir);
}
var f1 = project!.GetTrapPath(logger, new TransformedPathStub("foo.cs"), TrapWriter.CompressionMode.Gzip);
var g1 = TrapWriter.NestPaths(logger, tmpDir, "foo.cs.trap.gz");
Assert.Equal(f1, g1);
// Test trap file generation
var trapwriterFilename = project.GetTrapPath(logger, new TransformedPathStub("foo.cs"), TrapWriter.CompressionMode.Gzip);
using (var trapwriter = project.CreateTrapWriter(logger, new TransformedPathStub("foo.cs"), TrapWriter.CompressionMode.Gzip, discardDuplicates: false))
{
trapwriter.Emit("1=*");
Assert.False(File.Exists(trapwriterFilename));
}
Assert.True(File.Exists(trapwriterFilename));
File.Delete(trapwriterFilename);
}
[Fact]
public void TestLayoutFile()
{
File.WriteAllLines("layout.txt", new string[]
{
"# Section",
"TRAP_FOLDER=" + Path.GetFullPath("snapshot\\trap"),
"ODASA_DB=snapshot\\db-csharp",
"SOURCE_ARCHIVE=" + Path.GetFullPath("snapshot\\archive"),
"ODASA_BUILD_ERROR_DIR=snapshot\build-errors",
"-foo.cs",
"bar.cs",
"-excluded",
"excluded/foo.cs",
"included"
});
var layout = new Semmle.Extraction.Layout(null, null, "layout.txt");
// Test general pattern matching
Assert.True(layout.FileInLayout(new TransformedPathStub("bar.cs")));
Assert.False(layout.FileInLayout(new TransformedPathStub("foo.cs")));
Assert.False(layout.FileInLayout(new TransformedPathStub("goo.cs")));
Assert.False(layout.FileInLayout(new TransformedPathStub("excluded/bar.cs")));
Assert.True(layout.FileInLayout(new TransformedPathStub("excluded/foo.cs")));
Assert.True(layout.FileInLayout(new TransformedPathStub("included/foo.cs")));
// Test the trap file
var project = layout.LookupProjectOrNull(new TransformedPathStub("bar.cs"));
Assert.NotNull(project);
var trapwriterFilename = project!.GetTrapPath(logger, new TransformedPathStub("bar.cs"), TrapWriter.CompressionMode.Gzip);
Assert.Equal(TrapWriter.NestPaths(logger, Path.GetFullPath("snapshot\\trap"), "bar.cs.trap.gz"),
trapwriterFilename);
// Test the source archive
var trapWriter = project.CreateTrapWriter(logger, new TransformedPathStub("bar.cs"), TrapWriter.CompressionMode.Gzip, discardDuplicates: false);
trapWriter.Archive("layout.txt", new TransformedPathStub("layout.txt"), System.Text.Encoding.ASCII);
var writtenFile = TrapWriter.NestPaths(logger, Path.GetFullPath("snapshot\\archive"), "layout.txt");
Assert.True(File.Exists(writtenFile));
File.Delete("layout.txt");
}
[Fact]
public void TestTrapOverridesLayout()
{
// When you specify both a trap file and a layout, use the trap file.
var layout = new Semmle.Extraction.Layout(Path.GetFullPath("snapshot\\trap"), null, "something.txt");
Assert.True(layout.FileInLayout(new TransformedPathStub("bar.cs")));
var subProject = layout.LookupProjectOrNull(new TransformedPathStub("foo.cs"));
Assert.NotNull(subProject);
var f1 = subProject!.GetTrapPath(logger, new TransformedPathStub("foo.cs"), TrapWriter.CompressionMode.Gzip);
var g1 = TrapWriter.NestPaths(logger, Path.GetFullPath("snapshot\\trap"), "foo.cs.trap.gz");
Assert.Equal(f1, g1);
}
[Fact]
public void TestMultipleSections()
{
File.WriteAllLines("layout.txt", new string[]
{
"# Section 1",
"TRAP_FOLDER=" + Path.GetFullPath("snapshot\\trap1"),
"ODASA_DB=snapshot\\db-csharp",
"SOURCE_ARCHIVE=" + Path.GetFullPath("snapshot\\archive1"),
"ODASA_BUILD_ERROR_DIR=snapshot\build-errors",
"foo.cs",
"# Section 2",
"TRAP_FOLDER=" + Path.GetFullPath("snapshot\\trap2"),
"ODASA_DB=snapshot\\db-csharp",
"SOURCE_ARCHIVE=" + Path.GetFullPath("snapshot\\archive2"),
"ODASA_BUILD_ERROR_DIR=snapshot\build-errors",
"bar.cs",
});
var layout = new Semmle.Extraction.Layout(null, null, "layout.txt");
// Use Section 2
Assert.True(layout.FileInLayout(new TransformedPathStub("bar.cs")));
var subProject = layout.LookupProjectOrNull(new TransformedPathStub("bar.cs"));
Assert.NotNull(subProject);
var f1 = subProject!.GetTrapPath(logger, new TransformedPathStub("bar.cs"), TrapWriter.CompressionMode.Gzip);
var g1 = TrapWriter.NestPaths(logger, Path.GetFullPath("snapshot\\trap2"), "bar.cs.trap.gz");
Assert.Equal(f1, g1);
// Use Section 1
Assert.True(layout.FileInLayout(new TransformedPathStub("foo.cs")));
subProject = layout.LookupProjectOrNull(new TransformedPathStub("foo.cs"));
Assert.NotNull(subProject);
var f2 = subProject!.GetTrapPath(logger, new TransformedPathStub("foo.cs"), TrapWriter.CompressionMode.Gzip);
var g2 = TrapWriter.NestPaths(logger, Path.GetFullPath("snapshot\\trap1"), "foo.cs.trap.gz");
Assert.Equal(f2, g2);
// boo.dll is not in the layout, so use layout from first section.
Assert.False(layout.FileInLayout(new TransformedPathStub("boo.dll")));
var f3 = layout.LookupProjectOrDefault(new TransformedPathStub("boo.dll")).GetTrapPath(logger, new TransformedPathStub("boo.dll"), TrapWriter.CompressionMode.Gzip);
var g3 = TrapWriter.NestPaths(logger, Path.GetFullPath("snapshot\\trap1"), "boo.dll.trap.gz");
Assert.Equal(f3, g3);
// boo.cs is not in the layout, so return null
Assert.False(layout.FileInLayout(new TransformedPathStub("boo.cs")));
Assert.Null(layout.LookupProjectOrNull(new TransformedPathStub("boo.cs")));
}
[Fact]
public void MissingLayout()
{
Assert.Throws<Extraction.Layout.InvalidLayoutException>(() =>
new Semmle.Extraction.Layout(null, null, "nosuchfile.txt"));
}
[Fact]
public void EmptyLayout()
{
File.Create("layout.txt").Close();
Assert.Throws<Extraction.Layout.InvalidLayoutException>(() =>
new Semmle.Extraction.Layout(null, null, "layout.txt"));
}
[Fact]
public void InvalidLayout()
{
File.WriteAllLines("layout.txt", new string[]
{
"# Section 1"
});
Assert.Throws<Extraction.Layout.InvalidLayoutException>(() =>
new Semmle.Extraction.Layout(null, null, "layout.txt"));
}
private sealed class LoggerMock : ILogger
{
public void Dispose() { }
public void Log(Severity s, string text) { }
}
}
internal static class TrapWriterTestExtensions
{
public static void Emit(this TrapWriter trapFile, string s)
{
trapFile.Emit(new StringTrapEmitter(s));
}
private class StringTrapEmitter : ITrapEmitter
{
private readonly string content;
public StringTrapEmitter(string content)
{
this.content = content;
}
public void EmitTrap(TextWriter trapFile)
{
trapFile.Write(content);
}
}
}
}

View File

@@ -14,7 +14,6 @@ namespace Semmle.Extraction.Tests
public OptionsTests()
{
Environment.SetEnvironmentVariable("SEMMLE_EXTRACTOR_OPTIONS", "");
Environment.SetEnvironmentVariable("LGTM_INDEX_EXTRACTOR", "");
}
@@ -126,14 +125,14 @@ namespace Semmle.Extraction.Tests
[Fact]
public void EnvironmentVariables()
{
Environment.SetEnvironmentVariable("SEMMLE_EXTRACTOR_OPTIONS", "--cil c");
Environment.SetEnvironmentVariable("LGTM_INDEX_EXTRACTOR", "--cil c");
options = CSharp.Options.CreateWithEnvironment(new string[] { "a", "b" });
Assert.True(options.CIL);
Assert.Equal("a", options.CompilerArguments[0]);
Assert.Equal("b", options.CompilerArguments[1]);
Assert.Equal("c", options.CompilerArguments[2]);
Environment.SetEnvironmentVariable("SEMMLE_EXTRACTOR_OPTIONS", "");
Environment.SetEnvironmentVariable("LGTM_INDEX_EXTRACTOR", "");
Environment.SetEnvironmentVariable("LGTM_INDEX_EXTRACTOR", "--nocil");
options = CSharp.Options.CreateWithEnvironment(new string[] { "--cil" });
Assert.False(options.CIL);

View File

@@ -1,204 +0,0 @@
using Semmle.Util.Logging;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
namespace Semmle.Extraction
{
/// <summary>
/// An extractor layout file.
/// Represents the layout of projects into trap folders and source archives.
/// </summary>
public sealed class Layout
{
/// <summary>
/// Exception thrown when the layout file is invalid.
/// </summary>
public class InvalidLayoutException : Exception
{
public InvalidLayoutException(string file, string message) :
base("ODASA_CSHARP_LAYOUT " + file + " " + message)
{
}
}
/// <summary>
/// List of blocks in the layout file.
/// </summary>
private readonly List<LayoutBlock> blocks;
/// <summary>
/// A subproject in the layout file.
/// </summary>
public class SubProject
{
/// <summary>
/// The trap folder, or null for current directory.
/// </summary>
public string? TRAP_FOLDER { get; }
/// <summary>
/// The source archive, or null to skip.
/// </summary>
public string? SOURCE_ARCHIVE { get; }
public SubProject(string? traps, string? archive)
{
TRAP_FOLDER = traps;
SOURCE_ARCHIVE = archive;
}
/// <summary>
/// Gets the name of the trap file for a given source/assembly file.
/// </summary>
/// <param name="srcFile">The source file.</param>
/// <returns>The full filepath of the trap file.</returns>
public string GetTrapPath(ILogger logger, PathTransformer.ITransformedPath srcFile, TrapWriter.CompressionMode trapCompression) =>
TrapWriter.TrapPath(logger, TRAP_FOLDER, srcFile, trapCompression);
/// <summary>
/// Creates a trap writer for a given source/assembly file.
/// </summary>
/// <param name="srcFile">The source file.</param>
/// <returns>A newly created TrapWriter.</returns>
public TrapWriter CreateTrapWriter(ILogger logger, PathTransformer.ITransformedPath srcFile, TrapWriter.CompressionMode trapCompression, bool discardDuplicates) =>
new TrapWriter(logger, srcFile, TRAP_FOLDER, SOURCE_ARCHIVE, trapCompression, discardDuplicates);
}
private readonly SubProject defaultProject;
/// <summary>
/// Finds the suitable directories for a given source file.
/// Returns null if not included in the layout.
/// </summary>
/// <param name="sourceFile">The file to look up.</param>
/// <returns>The relevant subproject, or null if not found.</returns>
public SubProject? LookupProjectOrNull(PathTransformer.ITransformedPath sourceFile)
{
if (!useLayoutFile)
return defaultProject;
return blocks
.Where(block => block.Matches(sourceFile))
.Select(block => block.Directories)
.FirstOrDefault();
}
/// <summary>
/// Finds the suitable directories for a given source file.
/// Returns the default project if not included in the layout.
/// </summary>
/// <param name="sourceFile">The file to look up.</param>
/// <returns>The relevant subproject, or DefaultProject if not found.</returns>
public SubProject LookupProjectOrDefault(PathTransformer.ITransformedPath sourceFile)
{
return LookupProjectOrNull(sourceFile) ?? defaultProject;
}
private readonly bool useLayoutFile;
/// <summary>
/// Default constructor reads parameters from the environment.
/// </summary>
public Layout() : this(
Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_TRAP_DIR") ?? Environment.GetEnvironmentVariable("TRAP_FOLDER"),
Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR") ?? Environment.GetEnvironmentVariable("SOURCE_ARCHIVE"),
Environment.GetEnvironmentVariable("ODASA_CSHARP_LAYOUT"))
{
}
/// <summary>
/// Creates the project layout. Reads the layout file if specified.
/// </summary>
/// <param name="traps">Directory for trap files, or null to use layout/current directory.</param>
/// <param name="archive">Directory for source archive, or null for layout/no archive.</param>
/// <param name="layout">Path of layout file, or null for no layout.</param>
/// <exception cref="InvalidLayoutException">Failed to read layout file.</exception>
public Layout(string? traps, string? archive, string? layout)
{
useLayoutFile = string.IsNullOrEmpty(traps) && !string.IsNullOrEmpty(layout);
blocks = new List<LayoutBlock>();
if (useLayoutFile)
{
ReadLayoutFile(layout!);
defaultProject = blocks[0].Directories;
}
else
{
defaultProject = new SubProject(traps, archive);
}
}
/// <summary>
/// Is the source file included in the layout?
/// </summary>
/// <param name="path">The absolute path of the file to query.</param>
/// <returns>True iff there is no layout file or the layout file specifies the file.</returns>
public bool FileInLayout(PathTransformer.ITransformedPath path) => LookupProjectOrNull(path) is not null;
private void ReadLayoutFile(string layout)
{
try
{
var lines = File.ReadAllLines(layout);
var i = 0;
while (!lines[i].StartsWith("#"))
i++;
while (i < lines.Length)
{
var block = new LayoutBlock(lines, ref i);
blocks.Add(block);
}
if (blocks.Count == 0)
throw new InvalidLayoutException(layout, "contains no blocks");
}
catch (IOException ex)
{
throw new InvalidLayoutException(layout, ex.Message);
}
catch (IndexOutOfRangeException)
{
throw new InvalidLayoutException(layout, "is invalid");
}
}
}
internal sealed class LayoutBlock
{
private readonly List<FilePattern> filePatterns = new List<FilePattern>();
public Layout.SubProject Directories { get; }
private static string? ReadVariable(string name, string line)
{
var prefix = name + "=";
if (!line.StartsWith(prefix))
return null;
return line.Substring(prefix.Length).Trim();
}
public LayoutBlock(string[] lines, ref int i)
{
// first line: #name
i++;
var trapFolder = ReadVariable("TRAP_FOLDER", lines[i++]);
// Don't care about ODASA_DB.
ReadVariable("ODASA_DB", lines[i++]);
var sourceArchive = ReadVariable("SOURCE_ARCHIVE", lines[i++]);
Directories = new Layout.SubProject(trapFolder, sourceArchive);
// Don't care about ODASA_BUILD_ERROR_DIR.
ReadVariable("ODASA_BUILD_ERROR_DIR", lines[i++]);
while (i < lines.Length && !lines[i].StartsWith("#"))
{
filePatterns.Add(new FilePattern(lines[i++]));
}
}
public bool Matches(PathTransformer.ITransformedPath path) => FilePattern.Matches(filePatterns, path.Value, out var _);
}
}

View File

@@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.IO;
using System.Diagnostics.CodeAnalysis;
using Semmle.Util;
using Semmle.Util.Logging;
namespace Semmle.Extraction
{
@@ -35,6 +36,20 @@ namespace Semmle.Extraction
ITransformedPath WithSuffix(string suffix);
string DatabaseId { get; }
/// <summary>
/// Gets the name of the trap file for this file.
/// </summary>
/// <returns>The full filepath of the trap file.</returns>
public string GetTrapPath(ILogger logger, TrapWriter.CompressionMode trapCompression) =>
TrapWriter.TrapPath(logger, Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"), this, trapCompression);
/// <summary>
/// Creates a trap writer for this file.
/// </summary>
/// <returns>A newly created TrapWriter.</returns>
public TrapWriter CreateTrapWriter(ILogger logger, TrapWriter.CompressionMode trapCompression, bool discardDuplicates) =>
new(logger, this, Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"), Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"), trapCompression, discardDuplicates);
}
private struct TransformedPath : ITransformedPath

View File

@@ -234,9 +234,7 @@ namespace Semmle.Util
/// <returns>A new CanonicalPathCache.</returns>
public static CanonicalPathCache Create(ILogger logger, int maxCapacity)
{
var preserveSymlinks =
Environment.GetEnvironmentVariable("CODEQL_PRESERVE_SYMLINKS") == "true" ||
Environment.GetEnvironmentVariable("SEMMLE_PRESERVE_SYMLINKS") == "true";
var preserveSymlinks = Environment.GetEnvironmentVariable("CODEQL_PRESERVE_SYMLINKS") == "true";
return Create(logger, maxCapacity, preserveSymlinks ? CanonicalPathCache.Symlinks.Preserve : CanonicalPathCache.Symlinks.Follow);
}

View File

@@ -74,7 +74,7 @@ namespace Semmle.Util.Logging
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{
Console.Error.WriteLine("SEMMLE: Couldn't initialise C# extractor output: " + ex.Message + "\n" + ex.StackTrace);
Console.Error.WriteLine("CodeQL: Couldn't initialise C# extractor output: " + ex.Message + "\n" + ex.StackTrace);
Console.Error.Flush();
throw;
}