mirror of
https://github.com/github/codeql.git
synced 2025-12-16 16:53:25 +01:00
C#: Remove legacy odasa support
The following environment variables are no longer supported: ``` ODASA_BUILD_ERROR_DIR ODASA_CSHARP_LAYOUT ODASA_SNAPSHOT SEMMLE_DIST SEMMLE_EXTRACTOR_OPTIONS SEMMLE_PLATFORM_TOOLS SEMMLE_PRESERVE_SYMLINKS SOURCE_ARCHIVE TRAP_FOLDER ```
This commit is contained in:
@@ -51,7 +51,7 @@ namespace Semmle.Autobuild.CSharp.Tests
|
||||
|
||||
int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory, IDictionary<string, string>? env, out IList<string> stdOut)
|
||||
{
|
||||
var pattern = cmd + " " + args;
|
||||
var pattern = string.IsNullOrEmpty(args) ? cmd : cmd + " " + args;
|
||||
RunProcessIn.Add(pattern);
|
||||
|
||||
if (!RunProcessOut.TryGetValue(pattern, out var str))
|
||||
@@ -62,7 +62,7 @@ namespace Semmle.Autobuild.CSharp.Tests
|
||||
RunProcessWorkingDirectory.TryGetValue(pattern, out var wd);
|
||||
|
||||
if (wd != workingDirectory)
|
||||
throw new ArgumentException("Missing RunProcessWorkingDirectory " + pattern);
|
||||
throw new ArgumentException($"Unexpected RunProcessWorkingDirectory, got {wd ?? "null"} expected {workingDirectory ?? "null"} in {pattern}");
|
||||
|
||||
if (!RunProcess.TryGetValue(pattern, out var ret))
|
||||
throw new ArgumentException("Missing RunProcess " + pattern);
|
||||
@@ -72,12 +72,12 @@ namespace Semmle.Autobuild.CSharp.Tests
|
||||
|
||||
int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory, IDictionary<string, string>? env)
|
||||
{
|
||||
var pattern = cmd + " " + args;
|
||||
var pattern = string.IsNullOrEmpty(args) ? cmd : cmd + " " + args;
|
||||
RunProcessIn.Add(pattern);
|
||||
RunProcessWorkingDirectory.TryGetValue(pattern, out var wd);
|
||||
|
||||
if (wd != workingDirectory)
|
||||
throw new ArgumentException("Missing RunProcessWorkingDirectory " + pattern);
|
||||
throw new ArgumentException($"Unexpected RunProcessWorkingDirectory, got {wd ?? "null"} expected {workingDirectory ?? "null"} in {pattern}");
|
||||
|
||||
if (!RunProcess.TryGetValue(pattern, out var ret))
|
||||
throw new ArgumentException("Missing RunProcess " + pattern);
|
||||
@@ -255,7 +255,7 @@ namespace Semmle.Autobuild.CSharp.Tests
|
||||
[Fact]
|
||||
public void TestAnd1()
|
||||
{
|
||||
var cmd = BuildScript.Create("abc", "def ghi", false, null, null) & BuildScript.Create("odasa", null, false, null, null);
|
||||
var cmd = BuildScript.Create("abc", "def ghi", false, null, null) & BuildScript.Create("codeql", null, false, null, null);
|
||||
|
||||
actions.RunProcess["abc def ghi"] = 1;
|
||||
cmd.Run(actions, StartCallback, EndCallback);
|
||||
@@ -269,14 +269,14 @@ namespace Semmle.Autobuild.CSharp.Tests
|
||||
[Fact]
|
||||
public void TestAnd2()
|
||||
{
|
||||
var cmd = BuildScript.Create("odasa", null, false, null, null) & BuildScript.Create("abc", "def ghi", false, null, null);
|
||||
var cmd = BuildScript.Create("codeql", null, false, null, null) & BuildScript.Create("abc", "def ghi", false, null, null);
|
||||
|
||||
actions.RunProcess["abc def ghi"] = 1;
|
||||
actions.RunProcess["odasa "] = 0;
|
||||
actions.RunProcess["codeql"] = 0;
|
||||
cmd.Run(actions, StartCallback, EndCallback);
|
||||
|
||||
Assert.Equal("odasa ", actions.RunProcessIn[0]);
|
||||
Assert.Equal("odasa ", startCallbackIn[0]);
|
||||
Assert.Equal("codeql", actions.RunProcessIn[0]);
|
||||
Assert.Equal("codeql", startCallbackIn[0]);
|
||||
Assert.Equal("", endCallbackIn[0]);
|
||||
Assert.Equal(0, endCallbackReturn[0]);
|
||||
|
||||
@@ -289,14 +289,14 @@ namespace Semmle.Autobuild.CSharp.Tests
|
||||
[Fact]
|
||||
public void TestOr1()
|
||||
{
|
||||
var cmd = BuildScript.Create("odasa", null, false, null, null) | BuildScript.Create("abc", "def ghi", false, null, null);
|
||||
var cmd = BuildScript.Create("codeql", null, false, null, null) | BuildScript.Create("abc", "def ghi", false, null, null);
|
||||
|
||||
actions.RunProcess["abc def ghi"] = 1;
|
||||
actions.RunProcess["odasa "] = 0;
|
||||
actions.RunProcess["codeql"] = 0;
|
||||
cmd.Run(actions, StartCallback, EndCallback);
|
||||
|
||||
Assert.Equal("odasa ", actions.RunProcessIn[0]);
|
||||
Assert.Equal("odasa ", startCallbackIn[0]);
|
||||
Assert.Equal("codeql", actions.RunProcessIn[0]);
|
||||
Assert.Equal("codeql", startCallbackIn[0]);
|
||||
Assert.Equal("", endCallbackIn[0]);
|
||||
Assert.Equal(0, endCallbackReturn[0]);
|
||||
Assert.Equal(1, endCallbackReturn.Count);
|
||||
@@ -305,10 +305,10 @@ namespace Semmle.Autobuild.CSharp.Tests
|
||||
[Fact]
|
||||
public void TestOr2()
|
||||
{
|
||||
var cmd = BuildScript.Create("abc", "def ghi", false, null, null) | BuildScript.Create("odasa", null, false, null, null);
|
||||
var cmd = BuildScript.Create("abc", "def ghi", false, null, null) | BuildScript.Create("codeql", null, false, null, null);
|
||||
|
||||
actions.RunProcess["abc def ghi"] = 1;
|
||||
actions.RunProcess["odasa "] = 0;
|
||||
actions.RunProcess["codeql"] = 0;
|
||||
cmd.Run(actions, StartCallback, EndCallback);
|
||||
|
||||
Assert.Equal("abc def ghi", actions.RunProcessIn[0]);
|
||||
@@ -316,8 +316,8 @@ namespace Semmle.Autobuild.CSharp.Tests
|
||||
Assert.Equal("", endCallbackIn[0]);
|
||||
Assert.Equal(1, endCallbackReturn[0]);
|
||||
|
||||
Assert.Equal("odasa ", actions.RunProcessIn[1]);
|
||||
Assert.Equal("odasa ", startCallbackIn[1]);
|
||||
Assert.Equal("codeql", actions.RunProcessIn[1]);
|
||||
Assert.Equal("codeql", startCallbackIn[1]);
|
||||
Assert.Equal("", endCallbackIn[1]);
|
||||
Assert.Equal(0, endCallbackReturn[1]);
|
||||
}
|
||||
@@ -385,9 +385,6 @@ namespace Semmle.Autobuild.CSharp.Tests
|
||||
actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_ROOT"] = $@"C:\codeql\{codeqlUpperLanguage.ToLowerInvariant()}";
|
||||
actions.GetEnvironmentVariable["CODEQL_JAVA_HOME"] = @"C:\codeql\tools\java";
|
||||
actions.GetEnvironmentVariable["CODEQL_PLATFORM"] = isWindows ? "win64" : "linux64";
|
||||
actions.GetEnvironmentVariable["SEMMLE_DIST"] = @"C:\odasa";
|
||||
actions.GetEnvironmentVariable["SEMMLE_JAVA_HOME"] = @"C:\odasa\tools\java";
|
||||
actions.GetEnvironmentVariable["SEMMLE_PLATFORM_TOOLS"] = @"C:\odasa\tools";
|
||||
actions.GetEnvironmentVariable["LGTM_INDEX_VSTOOLS_VERSION"] = vsToolsVersion;
|
||||
actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_ARGUMENTS"] = msBuildArguments;
|
||||
actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_PLATFORM"] = msBuildPlatform;
|
||||
@@ -416,7 +413,7 @@ namespace Semmle.Autobuild.CSharp.Tests
|
||||
actions.RunProcess["cmd.exe /C dotnet --info"] = 0;
|
||||
actions.RunProcess[@"cmd.exe /C dotnet clean C:\Project\test.csproj"] = 0;
|
||||
actions.RunProcess[@"cmd.exe /C dotnet restore C:\Project\test.csproj"] = 0;
|
||||
actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --auto dotnet build --no-incremental /p:UseSharedCompilation=false C:\Project\test.csproj"] = 0;
|
||||
actions.RunProcess[@"cmd.exe /C dotnet build --no-incremental /p:UseSharedCompilation=false C:\Project\test.csproj"] = 0;
|
||||
actions.FileExists["csharp.log"] = true;
|
||||
actions.FileExists[@"C:\Project\test.csproj"] = true;
|
||||
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
|
||||
@@ -443,7 +440,7 @@ namespace Semmle.Autobuild.CSharp.Tests
|
||||
actions.RunProcess["dotnet --info"] = 0;
|
||||
actions.RunProcess[@"dotnet clean C:\Project/test.csproj"] = 0;
|
||||
actions.RunProcess[@"dotnet restore C:\Project/test.csproj"] = 0;
|
||||
actions.RunProcess[@"C:\odasa/tools/odasa index --auto dotnet build --no-incremental /p:UseSharedCompilation=false C:\Project/test.csproj"] = 0;
|
||||
actions.RunProcess[@"dotnet build --no-incremental /p:UseSharedCompilation=false C:\Project/test.csproj"] = 0;
|
||||
actions.FileExists["csharp.log"] = true;
|
||||
actions.FileExists[@"C:\Project/test.csproj"] = true;
|
||||
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
|
||||
@@ -601,7 +598,7 @@ namespace Semmle.Autobuild.CSharp.Tests
|
||||
[Fact]
|
||||
public void TestLinuxBuildCommand()
|
||||
{
|
||||
actions.RunProcess[@"C:\odasa/tools/odasa index --auto ""./build.sh --skip-tests"""] = 0;
|
||||
actions.RunProcess["./build.sh --skip-tests"] = 0;
|
||||
actions.FileExists["csharp.log"] = true;
|
||||
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
|
||||
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
|
||||
@@ -622,8 +619,8 @@ namespace Semmle.Autobuild.CSharp.Tests
|
||||
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
|
||||
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
|
||||
actions.RunProcess[@"/bin/chmod u+x C:\Project/build/build.sh"] = 0;
|
||||
actions.RunProcess[@"C:\odasa/tools/odasa index --auto C:\Project/build/build.sh"] = 0;
|
||||
actions.RunProcessWorkingDirectory[@"C:\odasa/tools/odasa index --auto C:\Project/build/build.sh"] = @"C:\Project/build";
|
||||
actions.RunProcess[@"C:\Project/build/build.sh"] = 0;
|
||||
actions.RunProcessWorkingDirectory[@"C:\Project/build/build.sh"] = @"C:\Project/build";
|
||||
actions.FileExists["csharp.log"] = true;
|
||||
|
||||
var autobuilder = CreateAutoBuilder(false);
|
||||
@@ -639,8 +636,8 @@ namespace Semmle.Autobuild.CSharp.Tests
|
||||
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
|
||||
|
||||
actions.RunProcess[@"/bin/chmod u+x C:\Project/build.sh"] = 0;
|
||||
actions.RunProcess[@"C:\odasa/tools/odasa index --auto C:\Project/build.sh"] = 0;
|
||||
actions.RunProcessWorkingDirectory[@"C:\odasa/tools/odasa index --auto C:\Project/build.sh"] = @"C:\Project";
|
||||
actions.RunProcess[@"C:\Project/build.sh"] = 0;
|
||||
actions.RunProcessWorkingDirectory[@"C:\Project/build.sh"] = @"C:\Project";
|
||||
actions.FileExists["csharp.log"] = false;
|
||||
|
||||
var autobuilder = CreateAutoBuilder(false);
|
||||
@@ -656,8 +653,8 @@ namespace Semmle.Autobuild.CSharp.Tests
|
||||
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
|
||||
|
||||
actions.RunProcess[@"/bin/chmod u+x C:\Project/build.sh"] = 0;
|
||||
actions.RunProcess[@"C:\odasa/tools/odasa index --auto C:\Project/build.sh"] = 5;
|
||||
actions.RunProcessWorkingDirectory[@"C:\odasa/tools/odasa index --auto C:\Project/build.sh"] = @"C:\Project";
|
||||
actions.RunProcess[@"C:\Project/build.sh"] = 5;
|
||||
actions.RunProcessWorkingDirectory[@"C:\Project/build.sh"] = @"C:\Project";
|
||||
actions.FileExists["csharp.log"] = true;
|
||||
|
||||
var autobuilder = CreateAutoBuilder(false);
|
||||
@@ -671,8 +668,8 @@ namespace Semmle.Autobuild.CSharp.Tests
|
||||
actions.EnumerateDirectories[@"C:\Project"] = "";
|
||||
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
|
||||
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
|
||||
actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --auto C:\Project\build.bat"] = 0;
|
||||
actions.RunProcessWorkingDirectory[@"cmd.exe /C C:\odasa\tools\odasa index --auto C:\Project\build.bat"] = @"C:\Project";
|
||||
actions.RunProcess[@"cmd.exe /C C:\Project\build.bat"] = 0;
|
||||
actions.RunProcessWorkingDirectory[@"cmd.exe /C C:\Project\build.bat"] = @"C:\Project";
|
||||
actions.FileExists["csharp.log"] = true;
|
||||
|
||||
var autobuilder = CreateAutoBuilder(true);
|
||||
@@ -686,10 +683,10 @@ namespace Semmle.Autobuild.CSharp.Tests
|
||||
actions.EnumerateDirectories[@"C:\Project"] = "";
|
||||
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
|
||||
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
|
||||
actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --auto C:\Project\build.bat"] = 1;
|
||||
actions.RunProcessWorkingDirectory[@"cmd.exe /C C:\odasa\tools\odasa index --auto C:\Project\build.bat"] = @"C:\Project";
|
||||
actions.RunProcess[@"cmd.exe /C C:\Project\build.bat"] = 1;
|
||||
actions.RunProcessWorkingDirectory[@"cmd.exe /C C:\Project\build.bat"] = @"C:\Project";
|
||||
actions.RunProcess[@"cmd.exe /C C:\codeql\tools\java\bin\java -jar C:\codeql\csharp\tools\extractor-asp.jar ."] = 0;
|
||||
actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --xml --extensions config"] = 0;
|
||||
actions.RunProcess[@"cmd.exe /C C:\codeql\tools\codeql index --xml --extensions config"] = 0;
|
||||
actions.FileExists["csharp.log"] = true;
|
||||
|
||||
var autobuilder = CreateAutoBuilder(true, ignoreErrors: "true");
|
||||
@@ -699,9 +696,9 @@ namespace Semmle.Autobuild.CSharp.Tests
|
||||
[Fact]
|
||||
public void TestWindowsCmdIgnoreErrors()
|
||||
{
|
||||
actions.RunProcess["cmd.exe /C C:\\odasa\\tools\\odasa index --auto ^\"build.cmd --skip-tests^\""] = 3;
|
||||
actions.RunProcess["cmd.exe /C ^\"build.cmd --skip-tests^\""] = 3;
|
||||
actions.RunProcess[@"cmd.exe /C C:\codeql\tools\java\bin\java -jar C:\codeql\csharp\tools\extractor-asp.jar ."] = 0;
|
||||
actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --xml --extensions config"] = 0;
|
||||
actions.RunProcess[@"cmd.exe /C C:\codeql\tools\codeql index --xml --extensions config"] = 0;
|
||||
actions.FileExists["csharp.log"] = true;
|
||||
SkipVsWhere();
|
||||
|
||||
@@ -718,9 +715,9 @@ namespace Semmle.Autobuild.CSharp.Tests
|
||||
public void TestWindowCSharpMsBuild()
|
||||
{
|
||||
actions.RunProcess[@"cmd.exe /C C:\Project\.nuget\nuget.exe restore C:\Project\test1.sln -DisableParallelProcessing"] = 0;
|
||||
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program Files ^(x86^)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && C:\\odasa\\tools\\odasa index --auto msbuild C:\\Project\\test1.sln /p:UseSharedCompilation=false /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /p:MvcBuildViews=true /P:Fu=Bar"] = 0;
|
||||
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program Files ^(x86^)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test1.sln /p:UseSharedCompilation=false /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /p:MvcBuildViews=true /P:Fu=Bar"] = 0;
|
||||
actions.RunProcess[@"cmd.exe /C C:\Project\.nuget\nuget.exe restore C:\Project\test2.sln -DisableParallelProcessing"] = 0;
|
||||
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program Files ^(x86^)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && C:\\odasa\\tools\\odasa index --auto msbuild C:\\Project\\test2.sln /p:UseSharedCompilation=false /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /p:MvcBuildViews=true /P:Fu=Bar"] = 0;
|
||||
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program Files ^(x86^)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test2.sln /p:UseSharedCompilation=false /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /p:MvcBuildViews=true /P:Fu=Bar"] = 0;
|
||||
actions.FileExists["csharp.log"] = true;
|
||||
actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = false;
|
||||
actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = false;
|
||||
@@ -749,9 +746,9 @@ namespace Semmle.Autobuild.CSharp.Tests
|
||||
public void TestWindowCSharpMsBuildMultipleSolutions()
|
||||
{
|
||||
actions.RunProcess[@"cmd.exe /C nuget restore C:\Project\test1.csproj -DisableParallelProcessing"] = 0;
|
||||
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program Files ^(x86^)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && C:\\odasa\\tools\\odasa index --auto msbuild C:\\Project\\test1.csproj /p:UseSharedCompilation=false /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /p:MvcBuildViews=true /P:Fu=Bar"] = 0;
|
||||
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program Files ^(x86^)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test1.csproj /p:UseSharedCompilation=false /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /p:MvcBuildViews=true /P:Fu=Bar"] = 0;
|
||||
actions.RunProcess[@"cmd.exe /C nuget restore C:\Project\test2.csproj -DisableParallelProcessing"] = 0;
|
||||
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program Files ^(x86^)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && C:\\odasa\\tools\\odasa index --auto msbuild C:\\Project\\test2.csproj /p:UseSharedCompilation=false /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /p:MvcBuildViews=true /P:Fu=Bar"] = 0;
|
||||
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program Files ^(x86^)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test2.csproj /p:UseSharedCompilation=false /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /p:MvcBuildViews=true /P:Fu=Bar"] = 0;
|
||||
actions.FileExists["csharp.log"] = true;
|
||||
actions.FileExists[@"C:\Project\test1.csproj"] = true;
|
||||
actions.FileExists[@"C:\Project\test2.csproj"] = true;
|
||||
@@ -794,7 +791,7 @@ namespace Semmle.Autobuild.CSharp.Tests
|
||||
public void TestWindowCSharpMsBuildFailed()
|
||||
{
|
||||
actions.RunProcess[@"cmd.exe /C nuget restore C:\Project\test1.sln -DisableParallelProcessing"] = 0;
|
||||
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program Files ^(x86^)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && C:\\odasa\\tools\\odasa index --auto msbuild C:\\Project\\test1.sln /p:UseSharedCompilation=false /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /p:MvcBuildViews=true /P:Fu=Bar"] = 1;
|
||||
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program Files ^(x86^)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test1.sln /p:UseSharedCompilation=false /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /p:MvcBuildViews=true /P:Fu=Bar"] = 1;
|
||||
actions.FileExists["csharp.log"] = true;
|
||||
actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = false;
|
||||
actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = false;
|
||||
@@ -820,8 +817,8 @@ namespace Semmle.Autobuild.CSharp.Tests
|
||||
[Fact]
|
||||
public void TestSkipNugetMsBuild()
|
||||
{
|
||||
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program Files ^(x86^)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && C:\\odasa\\tools\\odasa index --auto msbuild C:\\Project\\test1.sln /p:UseSharedCompilation=false /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /p:MvcBuildViews=true /P:Fu=Bar"] = 0;
|
||||
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program Files ^(x86^)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && C:\\odasa\\tools\\odasa index --auto msbuild C:\\Project\\test2.sln /p:UseSharedCompilation=false /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /p:MvcBuildViews=true /P:Fu=Bar"] = 0;
|
||||
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program Files ^(x86^)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test1.sln /p:UseSharedCompilation=false /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /p:MvcBuildViews=true /P:Fu=Bar"] = 0;
|
||||
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program Files ^(x86^)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\test2.sln /p:UseSharedCompilation=false /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /p:MvcBuildViews=true /P:Fu=Bar"] = 0;
|
||||
actions.FileExists["csharp.log"] = true;
|
||||
actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = false;
|
||||
actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = false;
|
||||
@@ -865,7 +862,7 @@ namespace Semmle.Autobuild.CSharp.Tests
|
||||
actions.RunProcess["dotnet --info"] = 0;
|
||||
actions.RunProcess[@"dotnet clean C:\Project/test.csproj"] = 0;
|
||||
actions.RunProcess[@"dotnet restore C:\Project/test.csproj"] = 0;
|
||||
actions.RunProcess[@"C:\odasa/tools/odasa index --auto dotnet build --no-incremental /p:UseSharedCompilation=false --no-restore C:\Project/test.csproj"] = 0;
|
||||
actions.RunProcess[@"dotnet build --no-incremental /p:UseSharedCompilation=false --no-restore C:\Project/test.csproj"] = 0;
|
||||
actions.FileExists["csharp.log"] = true;
|
||||
actions.FileExists[@"C:\Project/test.csproj"] = true;
|
||||
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
|
||||
@@ -897,7 +894,7 @@ namespace Semmle.Autobuild.CSharp.Tests
|
||||
actions.RunProcess[@"C:\Project/.dotnet/dotnet --info"] = 0;
|
||||
actions.RunProcess[@"C:\Project/.dotnet/dotnet clean C:\Project/test.csproj"] = 0;
|
||||
actions.RunProcess[@"C:\Project/.dotnet/dotnet restore C:\Project/test.csproj"] = 0;
|
||||
actions.RunProcess[@"C:\odasa/tools/odasa index --auto C:\Project/.dotnet/dotnet build --no-incremental /p:UseSharedCompilation=false C:\Project/test.csproj"] = 0;
|
||||
actions.RunProcess[@"C:\Project/.dotnet/dotnet build --no-incremental /p:UseSharedCompilation=false C:\Project/test.csproj"] = 0;
|
||||
actions.FileExists["csharp.log"] = true;
|
||||
actions.FileExists["test.csproj"] = true;
|
||||
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
|
||||
@@ -932,7 +929,7 @@ namespace Semmle.Autobuild.CSharp.Tests
|
||||
actions.RunProcess[@"C:\Project/.dotnet/dotnet --info"] = 0;
|
||||
actions.RunProcess[@"C:\Project/.dotnet/dotnet clean C:\Project/test.csproj"] = 0;
|
||||
actions.RunProcess[@"C:\Project/.dotnet/dotnet restore C:\Project/test.csproj"] = 0;
|
||||
actions.RunProcess[@"C:\odasa/tools/odasa index --auto C:\Project/.dotnet/dotnet build --no-incremental /p:UseSharedCompilation=false C:\Project/test.csproj"] = 0;
|
||||
actions.RunProcess[@"C:\Project/.dotnet/dotnet build --no-incremental /p:UseSharedCompilation=false C:\Project/test.csproj"] = 0;
|
||||
actions.FileExists["csharp.log"] = true;
|
||||
actions.FileExists["test.csproj"] = true;
|
||||
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
|
||||
@@ -963,7 +960,7 @@ namespace Semmle.Autobuild.CSharp.Tests
|
||||
actions.RunProcess[@"cmd.exe /C C:\Project\.dotnet\dotnet --info"] = 0;
|
||||
actions.RunProcess[@"cmd.exe /C C:\Project\.dotnet\dotnet clean C:\Project\test.csproj"] = 0;
|
||||
actions.RunProcess[@"cmd.exe /C C:\Project\.dotnet\dotnet restore C:\Project\test.csproj"] = 0;
|
||||
actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --auto C:\Project\.dotnet\dotnet build --no-incremental /p:UseSharedCompilation=false C:\Project\test.csproj"] = 0;
|
||||
actions.RunProcess[@"cmd.exe /C C:\Project\.dotnet\dotnet build --no-incremental /p:UseSharedCompilation=false C:\Project\test.csproj"] = 0;
|
||||
actions.FileExists["csharp.log"] = true;
|
||||
actions.FileExists[@"C:\Project\test.csproj"] = true;
|
||||
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
|
||||
@@ -1011,7 +1008,7 @@ namespace Semmle.Autobuild.CSharp.Tests
|
||||
{
|
||||
actions.RunProcess[@"cmd.exe /C nuget restore C:\Project\dirs.proj -DisableParallelProcessing"] = 1;
|
||||
actions.RunProcess[@"cmd.exe /C C:\Project\.nuget\nuget.exe restore C:\Project\dirs.proj -DisableParallelProcessing"] = 0;
|
||||
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program Files ^(x86^)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && C:\\odasa\\tools\\odasa index --auto msbuild C:\\Project\\dirs.proj /p:UseSharedCompilation=false /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /p:MvcBuildViews=true /P:Fu=Bar"] = 0;
|
||||
actions.RunProcess["cmd.exe /C CALL ^\"C:\\Program Files ^(x86^)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall.bat^\" && set Platform=&& type NUL && msbuild C:\\Project\\dirs.proj /p:UseSharedCompilation=false /t:Windows /p:Platform=\"x86\" /p:Configuration=\"Debug\" /p:MvcBuildViews=true /P:Fu=Bar"] = 0;
|
||||
actions.FileExists["csharp.log"] = true;
|
||||
actions.FileExists[@"C:\Project\a\test.csproj"] = true;
|
||||
actions.FileExists[@"C:\Project\dirs.proj"] = true;
|
||||
@@ -1055,7 +1052,7 @@ namespace Semmle.Autobuild.CSharp.Tests
|
||||
{
|
||||
actions.RunProcess[@"nuget restore C:\Project/dirs.proj -DisableParallelProcessing"] = 1;
|
||||
actions.RunProcess[@"mono C:\Project/.nuget/nuget.exe restore C:\Project/dirs.proj -DisableParallelProcessing"] = 0;
|
||||
actions.RunProcess[@"C:\odasa/tools/odasa index --auto msbuild C:\Project/dirs.proj /p:UseSharedCompilation=false /t:rebuild /p:MvcBuildViews=true"] = 0;
|
||||
actions.RunProcess[@"msbuild C:\Project/dirs.proj /p:UseSharedCompilation=false /t:rebuild /p:MvcBuildViews=true"] = 0;
|
||||
actions.FileExists["csharp.log"] = true;
|
||||
actions.FileExists[@"C:\Project/a/test.csproj"] = true;
|
||||
actions.FileExists[@"C:\Project/dirs.proj"] = true;
|
||||
|
||||
@@ -240,7 +240,7 @@ namespace Semmle.Autobuild.CSharp
|
||||
private static BuildScript GetBuildScript(Autobuilder builder, string? dotNetPath, IDictionary<string, string>? environment, string projOrSln)
|
||||
{
|
||||
var build = new CommandBuilder(builder.Actions, null, environment);
|
||||
var script = builder.MaybeIndex(build, DotNetCommand(builder.Actions, dotNetPath)).
|
||||
var script = build.RunCommand(DotNetCommand(builder.Actions, dotNetPath)).
|
||||
Argument("build").
|
||||
Argument("--no-incremental");
|
||||
|
||||
|
||||
@@ -17,10 +17,6 @@ namespace Semmle.Autobuild.CSharp
|
||||
{
|
||||
standalone = builder.Actions.PathCombine(builder.CodeQLExtractorLangRoot, "tools", builder.CodeQlPlatform, "Semmle.Extraction.CSharp.Standalone");
|
||||
}
|
||||
else if (builder.SemmlePlatformTools is not null)
|
||||
{
|
||||
standalone = builder.Actions.PathCombine(builder.SemmlePlatformTools, "csharp", "Semmle.Extraction.CSharp.Standalone");
|
||||
}
|
||||
else
|
||||
{
|
||||
return BuildScript.Failure;
|
||||
|
||||
@@ -190,18 +190,14 @@ namespace Semmle.Autobuild.Shared
|
||||
});
|
||||
|
||||
CodeQLExtractorLangRoot = Actions.GetEnvironmentVariable($"CODEQL_EXTRACTOR_{this.Options.Language.UpperCaseName}_ROOT");
|
||||
SemmlePlatformTools = Actions.GetEnvironmentVariable("SEMMLE_PLATFORM_TOOLS");
|
||||
|
||||
CodeQlPlatform = Actions.GetEnvironmentVariable("CODEQL_PLATFORM");
|
||||
|
||||
TrapDir =
|
||||
Actions.GetEnvironmentVariable($"CODEQL_EXTRACTOR_{this.Options.Language.UpperCaseName}_TRAP_DIR") ??
|
||||
Actions.GetEnvironmentVariable("TRAP_FOLDER") ??
|
||||
throw new InvalidEnvironmentException($"The environment variable CODEQL_EXTRACTOR_{this.Options.Language.UpperCaseName}_TRAP_DIR or TRAP_FOLDER has not been set.");
|
||||
throw new InvalidEnvironmentException($"The environment variable CODEQL_EXTRACTOR_{this.Options.Language.UpperCaseName}_TRAP_DIR has not been set.");
|
||||
|
||||
SourceArchiveDir =
|
||||
Actions.GetEnvironmentVariable($"CODEQL_EXTRACTOR_{this.Options.Language.UpperCaseName}_SOURCE_ARCHIVE_DIR") ??
|
||||
Actions.GetEnvironmentVariable("SOURCE_ARCHIVE") ??
|
||||
throw new InvalidEnvironmentException($"The environment variable CODEQL_EXTRACTOR_{this.Options.Language.UpperCaseName}_SOURCE_ARCHIVE_DIR or SOURCE_ARCHIVE has not been set.");
|
||||
}
|
||||
|
||||
@@ -264,34 +260,9 @@ namespace Semmle.Autobuild.Shared
|
||||
/// </summary>
|
||||
public string? CodeQLExtractorLangRoot { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Value of SEMMLE_PLATFORM_TOOLS environment variable.
|
||||
/// </summary>
|
||||
public string? SemmlePlatformTools { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Value of CODEQL_PLATFORM environment variable.
|
||||
/// </summary>
|
||||
public string? CodeQlPlatform { get; }
|
||||
|
||||
/// <summary>
|
||||
/// The absolute path of the odasa executable.
|
||||
/// null if we are running in CodeQL.
|
||||
/// </summary>
|
||||
public string? Odasa
|
||||
{
|
||||
get
|
||||
{
|
||||
var semmleDist = Actions.GetEnvironmentVariable("SEMMLE_DIST");
|
||||
return semmleDist is null ? null : Actions.PathCombine(semmleDist, "tools", "odasa");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Construct a command that executed the given <paramref name="cmd"/> wrapped in
|
||||
/// an <code>odasa --index</code>, unless indexing has been disabled, in which case
|
||||
/// <paramref name="cmd"/> is run directly.
|
||||
/// </summary>
|
||||
public CommandBuilder MaybeIndex(CommandBuilder builder, string cmd) => Odasa is null ? builder.RunCommand(cmd) : builder.IndexCommand(Odasa, cmd);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -58,7 +58,7 @@ namespace Semmle.Autobuild.Shared
|
||||
if (vsTools is not null)
|
||||
command.CallBatFile(vsTools.Path);
|
||||
|
||||
builder.MaybeIndex(command, scriptPath);
|
||||
command.RunCommand(scriptPath);
|
||||
return command.Script;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@
|
||||
var vsTools = MsBuildRule.GetVcVarsBatFile(builder);
|
||||
if (vsTools is not null)
|
||||
command.CallBatFile(vsTools.Path);
|
||||
builder.MaybeIndex(command, builder.Options.BuildCommand);
|
||||
command.RunCommand(builder.Options.BuildCommand);
|
||||
|
||||
return command.Script;
|
||||
});
|
||||
|
||||
@@ -70,7 +70,7 @@ namespace Semmle.Autobuild.Shared
|
||||
this.environment = environment;
|
||||
}
|
||||
|
||||
public override string ToString() => exe + " " + arguments;
|
||||
public override string ToString() => arguments.Length > 0 ? exe + " " + arguments : exe;
|
||||
|
||||
public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack)
|
||||
{
|
||||
|
||||
@@ -45,11 +45,6 @@ namespace Semmle.Autobuild.Shared
|
||||
this.silent = silent;
|
||||
}
|
||||
|
||||
private void OdasaIndex(string odasa)
|
||||
{
|
||||
RunCommand(odasa, "index --auto");
|
||||
}
|
||||
|
||||
public CommandBuilder CallBatFile(string batFile, string? argumentsOpt = null)
|
||||
{
|
||||
NextCommand();
|
||||
@@ -59,21 +54,6 @@ namespace Semmle.Autobuild.Shared
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Perform odasa index on a given command or BAT file.
|
||||
/// </summary>
|
||||
/// <param name="odasa">The odasa executable.</param>
|
||||
/// <param name="command">The command to run.</param>
|
||||
/// <param name="argumentsOpt">Additional arguments.</param>
|
||||
/// <returns>this for chaining calls.</returns>
|
||||
public CommandBuilder IndexCommand(string odasa, string command, string? argumentsOpt = null)
|
||||
{
|
||||
OdasaIndex(odasa);
|
||||
QuoteArgument(command);
|
||||
Argument(argumentsOpt);
|
||||
return this;
|
||||
}
|
||||
|
||||
private static readonly char[] specialChars = { ' ', '\t', '\n', '\v', '\"' };
|
||||
private static readonly char[] cmdMetacharacter = { '(', ')', '%', '!', '^', '\"', '<', '>', '&', '|' };
|
||||
|
||||
|
||||
@@ -95,7 +95,7 @@ namespace Semmle.Autobuild.Shared
|
||||
command.RunCommand("set Platform=&& type NUL", quoteExe: false);
|
||||
}
|
||||
|
||||
builder.MaybeIndex(command, msBuild);
|
||||
command.RunCommand(msBuild);
|
||||
command.QuoteArgument(projectOrSolution.FullPath);
|
||||
|
||||
command.Argument("/p:UseSharedCompilation=false");
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
**/mcs.exe:
|
||||
**/csc.exe:
|
||||
invoke ${env.SEMMLE_PLATFORM_TOOLS}/csharp/Semmle.Extraction.CSharp.Driver
|
||||
prepend --compiler
|
||||
prepend "${compiler}"
|
||||
prepend --cil
|
||||
**/mono*:
|
||||
**/dotnet:
|
||||
invoke ${odasa_tools}/extract-csharp.sh
|
||||
**/msbuild:
|
||||
**/xbuild:
|
||||
replace yes
|
||||
invoke ${compiler}
|
||||
append /p:UseSharedCompilation=false
|
||||
@@ -1,16 +0,0 @@
|
||||
#!/bin/bash
|
||||
echo extract-csharp.sh: Called with arguments: "$@"
|
||||
|
||||
extractor=$SEMMLE_PLATFORM_TOOLS/csharp/Semmle.Extraction.CSharp.Driver
|
||||
|
||||
for i in "$@"
|
||||
do
|
||||
shift
|
||||
if [[ `basename -- "$i"` =~ csc.exe|mcs.exe|csc.dll ]]
|
||||
then
|
||||
echo extract-csharp.sh: exec $extractor --cil $@
|
||||
exec "$extractor" --compiler $i --cil $@
|
||||
fi
|
||||
done
|
||||
|
||||
echo extract-csharp.sh: Not a compiler invocation
|
||||
@@ -1,9 +0,0 @@
|
||||
**\fakes*.exe:
|
||||
**\moles*.exe:
|
||||
order compiler
|
||||
trace no
|
||||
**\csc*.exe:
|
||||
invoke ${env.SEMMLE_PLATFORM_TOOLS}\csharp\Semmle.Extraction.CSharp.Driver.exe
|
||||
prepend --compiler
|
||||
prepend "${compiler}"
|
||||
prepend --cil
|
||||
@@ -20,11 +20,11 @@ namespace Semmle.Extraction.CIL.Driver
|
||||
Console.WriteLine(" path A directory/dll/exe to analyze");
|
||||
}
|
||||
|
||||
private static void ExtractAssembly(Layout layout, string assemblyPath, ILogger logger, CommonOptions options)
|
||||
private static void ExtractAssembly(string assemblyPath, ILogger logger, CommonOptions options)
|
||||
{
|
||||
var sw = new Stopwatch();
|
||||
sw.Start();
|
||||
Analyser.ExtractCIL(layout, assemblyPath, logger, options, out _, out _);
|
||||
Analyser.ExtractCIL(assemblyPath, logger, options, out _, out _);
|
||||
sw.Stop();
|
||||
logger.Log(Severity.Info, " {0} ({1})", assemblyPath, sw.Elapsed);
|
||||
}
|
||||
@@ -38,12 +38,11 @@ namespace Semmle.Extraction.CIL.Driver
|
||||
}
|
||||
|
||||
var options = new ExtractorOptions(args);
|
||||
var layout = new Layout();
|
||||
using var logger = new ConsoleLogger(options.Verbosity);
|
||||
|
||||
var actions = options.AssembliesToExtract
|
||||
.Select(asm => asm.Filename)
|
||||
.Select<string, Action>(filename => () => ExtractAssembly(layout, filename, logger, options))
|
||||
.Select<string, Action>(filename => () => ExtractAssembly(filename, logger, options))
|
||||
.ToArray();
|
||||
|
||||
foreach (var missingRef in options.MissingReferences)
|
||||
|
||||
@@ -25,7 +25,7 @@ namespace Semmle.Extraction.CIL
|
||||
/// <param name="extractPdbs">Whether to extract PDBs.</param>
|
||||
/// <param name="trapFile">The path of the trap file.</param>
|
||||
/// <param name="extracted">Whether the file was extracted (false=cached).</param>
|
||||
public static void ExtractCIL(Layout layout, string assemblyPath, ILogger logger, CommonOptions options, out string trapFile, out bool extracted)
|
||||
public static void ExtractCIL(string assemblyPath, ILogger logger, CommonOptions options, out string trapFile, out bool extracted)
|
||||
{
|
||||
trapFile = "";
|
||||
extracted = false;
|
||||
@@ -35,8 +35,7 @@ namespace Semmle.Extraction.CIL
|
||||
var pathTransformer = new PathTransformer(canonicalPathCache);
|
||||
var extractor = new TracingExtractor(assemblyPath, logger, pathTransformer, options);
|
||||
var transformedAssemblyPath = pathTransformer.Transform(assemblyPath);
|
||||
var project = layout.LookupProjectOrDefault(transformedAssemblyPath);
|
||||
using var trapWriter = project.CreateTrapWriter(logger, transformedAssemblyPath.WithSuffix(".cil"), options.TrapCompression, discardDuplicates: true);
|
||||
using var trapWriter = transformedAssemblyPath.WithSuffix(".cil").CreateTrapWriter(logger, options.TrapCompression, discardDuplicates: true);
|
||||
trapFile = trapWriter.TrapFile;
|
||||
if (!options.Cache || !System.IO.File.Exists(trapFile))
|
||||
{
|
||||
|
||||
@@ -84,7 +84,7 @@ namespace Semmle.Extraction.CSharp.Standalone
|
||||
foreach (var r in DesktopRuntimes)
|
||||
yield return r;
|
||||
|
||||
// A bad choice if it's the self-contained runtime distributed in odasa dist.
|
||||
// A bad choice if it's the self-contained runtime distributed in codeql dist.
|
||||
yield return ExecutingRuntime;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,7 +14,6 @@ namespace Semmle.Extraction.CSharp
|
||||
public void Initialize(CSharpCompilation compilationIn, CommonOptions options)
|
||||
{
|
||||
compilation = compilationIn;
|
||||
layout = new Layout();
|
||||
extractor = new StandaloneExtractor(Logger, PathTransformer, options);
|
||||
this.options = options;
|
||||
LogExtractorInfo(Extraction.Extractor.Version);
|
||||
|
||||
@@ -18,7 +18,6 @@ namespace Semmle.Extraction.CSharp
|
||||
{
|
||||
protected Extraction.Extractor? extractor;
|
||||
protected CSharpCompilation? compilation;
|
||||
protected Layout? layout;
|
||||
protected CommonOptions? options;
|
||||
|
||||
private readonly object progressMutex = new object();
|
||||
@@ -125,8 +124,7 @@ namespace Semmle.Extraction.CSharp
|
||||
|
||||
var assemblyPath = r.FilePath!;
|
||||
var transformedAssemblyPath = PathTransformer.Transform(assemblyPath);
|
||||
var projectLayout = layout.LookupProjectOrDefault(transformedAssemblyPath);
|
||||
using var trapWriter = projectLayout.CreateTrapWriter(Logger, transformedAssemblyPath, options.TrapCompression, discardDuplicates: true);
|
||||
using var trapWriter = transformedAssemblyPath.CreateTrapWriter(Logger, options.TrapCompression, discardDuplicates: true);
|
||||
|
||||
var skipExtraction = options.Cache && File.Exists(trapWriter.TrapFile);
|
||||
|
||||
@@ -178,7 +176,7 @@ namespace Semmle.Extraction.CSharp
|
||||
{
|
||||
var stopwatch = new Stopwatch();
|
||||
stopwatch.Start();
|
||||
CIL.Analyser.ExtractCIL(layout, r.FilePath!, Logger, options, out var trapFile, out var extracted);
|
||||
CIL.Analyser.ExtractCIL(r.FilePath!, Logger, options, out var trapFile, out var extracted);
|
||||
stopwatch.Stop();
|
||||
ReportProgress(r.FilePath, trapFile, stopwatch.Elapsed, extracted ? AnalysisAction.Extracted : AnalysisAction.UpToDate);
|
||||
}
|
||||
@@ -192,44 +190,35 @@ namespace Semmle.Extraction.CSharp
|
||||
var sourcePath = tree.FilePath;
|
||||
var transformedSourcePath = PathTransformer.Transform(sourcePath);
|
||||
|
||||
var projectLayout = layout.LookupProjectOrNull(transformedSourcePath);
|
||||
var excluded = projectLayout is null;
|
||||
var trapPath = excluded ? "" : projectLayout!.GetTrapPath(Logger, transformedSourcePath, options.TrapCompression);
|
||||
var trapPath = transformedSourcePath.GetTrapPath(Logger, options.TrapCompression);
|
||||
var upToDate = false;
|
||||
|
||||
if (!excluded)
|
||||
// compilation.Clone() is used to allow symbols to be garbage collected.
|
||||
using var trapWriter = transformedSourcePath.CreateTrapWriter(Logger, options.TrapCompression, discardDuplicates: false);
|
||||
|
||||
upToDate = options.Fast && FileIsUpToDate(sourcePath, trapWriter.TrapFile);
|
||||
|
||||
if (!upToDate)
|
||||
{
|
||||
// compilation.Clone() is used to allow symbols to be garbage collected.
|
||||
using var trapWriter = projectLayout!.CreateTrapWriter(Logger, transformedSourcePath, options.TrapCompression, discardDuplicates: false);
|
||||
var cx = new Context(extractor, compilation.Clone(), trapWriter, new SourceScope(tree), addAssemblyTrapPrefix);
|
||||
// Ensure that the file itself is populated in case the source file is totally empty
|
||||
var root = tree.GetRoot();
|
||||
Entities.File.Create(cx, root.SyntaxTree.FilePath);
|
||||
|
||||
upToDate = options.Fast && FileIsUpToDate(sourcePath, trapWriter.TrapFile);
|
||||
|
||||
if (!upToDate)
|
||||
var csNode = (CSharpSyntaxNode)root;
|
||||
var directiveVisitor = new DirectiveVisitor(cx);
|
||||
csNode.Accept(directiveVisitor);
|
||||
foreach (var branch in directiveVisitor.BranchesTaken)
|
||||
{
|
||||
var cx = new Context(extractor, compilation.Clone(), trapWriter, new SourceScope(tree), addAssemblyTrapPrefix);
|
||||
// Ensure that the file itself is populated in case the source file is totally empty
|
||||
var root = tree.GetRoot();
|
||||
Entities.File.Create(cx, root.SyntaxTree.FilePath);
|
||||
|
||||
var csNode = (CSharpSyntaxNode)root;
|
||||
var directiveVisitor = new DirectiveVisitor(cx);
|
||||
csNode.Accept(directiveVisitor);
|
||||
foreach (var branch in directiveVisitor.BranchesTaken)
|
||||
{
|
||||
cx.TrapStackSuffix.Add(branch);
|
||||
}
|
||||
csNode.Accept(new CompilationUnitVisitor(cx));
|
||||
cx.PopulateAll();
|
||||
CommentPopulator.ExtractCommentBlocks(cx, cx.CommentGenerator);
|
||||
cx.PopulateAll();
|
||||
cx.TrapStackSuffix.Add(branch);
|
||||
}
|
||||
csNode.Accept(new CompilationUnitVisitor(cx));
|
||||
cx.PopulateAll();
|
||||
CommentPopulator.ExtractCommentBlocks(cx, cx.CommentGenerator);
|
||||
cx.PopulateAll();
|
||||
}
|
||||
|
||||
ReportProgress(sourcePath, trapPath, stopwatch.Elapsed, excluded
|
||||
? AnalysisAction.Excluded
|
||||
: upToDate
|
||||
? AnalysisAction.UpToDate
|
||||
: AnalysisAction.Extracted);
|
||||
ReportProgress(sourcePath, trapPath, stopwatch.Elapsed, upToDate ? AnalysisAction.UpToDate : AnalysisAction.Extracted);
|
||||
}
|
||||
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
|
||||
{
|
||||
|
||||
@@ -452,19 +452,6 @@ namespace Semmle.Extraction.CSharp
|
||||
if (!string.IsNullOrEmpty(codeQlLogDir))
|
||||
return codeQlLogDir;
|
||||
|
||||
var snapshot = Environment.GetEnvironmentVariable("ODASA_SNAPSHOT");
|
||||
if (!string.IsNullOrEmpty(snapshot))
|
||||
return Path.Combine(snapshot, "log");
|
||||
|
||||
var buildErrorDir = Environment.GetEnvironmentVariable("ODASA_BUILD_ERROR_DIR");
|
||||
if (!string.IsNullOrEmpty(buildErrorDir))
|
||||
// Used by `qltest`
|
||||
return buildErrorDir;
|
||||
|
||||
var traps = Environment.GetEnvironmentVariable("TRAP_FOLDER");
|
||||
if (!string.IsNullOrEmpty(traps))
|
||||
return traps;
|
||||
|
||||
return Directory.GetCurrentDirectory();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -40,8 +40,7 @@ namespace Semmle.Extraction.CSharp
|
||||
public static Options CreateWithEnvironment(string[] arguments)
|
||||
{
|
||||
var options = new Options();
|
||||
var extractionOptions = Environment.GetEnvironmentVariable("SEMMLE_EXTRACTOR_OPTIONS") ??
|
||||
Environment.GetEnvironmentVariable("LGTM_INDEX_EXTRACTOR");
|
||||
var extractionOptions = Environment.GetEnvironmentVariable("LGTM_INDEX_EXTRACTOR");
|
||||
|
||||
var argsList = new List<string>(arguments);
|
||||
|
||||
|
||||
@@ -46,7 +46,6 @@ namespace Semmle.Extraction.CSharp
|
||||
{
|
||||
if (!init)
|
||||
throw new InternalError("EndInitialize called without BeginInitialize returning true");
|
||||
this.layout = new Layout();
|
||||
this.options = options;
|
||||
this.compilation = compilation;
|
||||
this.extractor = new TracingExtractor(GetOutputName(compilation, commandLineArguments), Logger, PathTransformer, options);
|
||||
@@ -202,8 +201,7 @@ namespace Semmle.Extraction.CSharp
|
||||
var assemblyPath = ((TracingExtractor?)extractor).OutputPath;
|
||||
var transformedAssemblyPath = PathTransformer.Transform(assemblyPath);
|
||||
var assembly = compilation.Assembly;
|
||||
var projectLayout = layout.LookupProjectOrDefault(transformedAssemblyPath);
|
||||
var trapWriter = projectLayout.CreateTrapWriter(Logger, transformedAssemblyPath, options.TrapCompression, discardDuplicates: false);
|
||||
var trapWriter = transformedAssemblyPath.CreateTrapWriter(Logger, options.TrapCompression, discardDuplicates: false);
|
||||
compilationTrapFile = trapWriter; // Dispose later
|
||||
var cx = new Context(extractor, compilation.Clone(), trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix);
|
||||
|
||||
|
||||
@@ -1,231 +0,0 @@
|
||||
using System.IO;
|
||||
using Xunit;
|
||||
using Semmle.Util.Logging;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace Semmle.Extraction.Tests
|
||||
{
|
||||
internal struct TransformedPathStub : PathTransformer.ITransformedPath
|
||||
{
|
||||
private readonly string value;
|
||||
public TransformedPathStub(string value) => this.value = value;
|
||||
public string Value => value;
|
||||
|
||||
public string Extension => throw new System.NotImplementedException();
|
||||
|
||||
public string NameWithoutExtension => throw new System.NotImplementedException();
|
||||
|
||||
public PathTransformer.ITransformedPath ParentDirectory => throw new System.NotImplementedException();
|
||||
|
||||
public string DatabaseId => throw new System.NotImplementedException();
|
||||
|
||||
public PathTransformer.ITransformedPath WithSuffix(string suffix)
|
||||
{
|
||||
throw new System.NotImplementedException();
|
||||
}
|
||||
}
|
||||
|
||||
public class Layout
|
||||
{
|
||||
private readonly ILogger logger = new LoggerMock();
|
||||
|
||||
[Fact]
|
||||
public void TestDefaultLayout()
|
||||
{
|
||||
var layout = new Semmle.Extraction.Layout(null, null, null);
|
||||
var project = layout.LookupProjectOrNull(new TransformedPathStub("foo.cs"));
|
||||
|
||||
Assert.NotNull(project);
|
||||
|
||||
// All files are mapped when there's no layout file.
|
||||
Assert.True(layout.FileInLayout(new TransformedPathStub("foo.cs")));
|
||||
|
||||
// Test trap filename
|
||||
var tmpDir = Path.GetTempPath();
|
||||
Directory.SetCurrentDirectory(tmpDir);
|
||||
if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
|
||||
{
|
||||
// `Directory.SetCurrentDirectory()` seems to slightly change the path on macOS,
|
||||
// so adjusting it:
|
||||
Assert.NotEqual(Directory.GetCurrentDirectory(), tmpDir);
|
||||
tmpDir = "/private" + tmpDir;
|
||||
// Remove trailing slash:
|
||||
Assert.Equal('/', tmpDir[tmpDir.Length - 1]);
|
||||
tmpDir = tmpDir.Substring(0, tmpDir.Length - 1);
|
||||
Assert.Equal(Directory.GetCurrentDirectory(), tmpDir);
|
||||
}
|
||||
var f1 = project!.GetTrapPath(logger, new TransformedPathStub("foo.cs"), TrapWriter.CompressionMode.Gzip);
|
||||
var g1 = TrapWriter.NestPaths(logger, tmpDir, "foo.cs.trap.gz");
|
||||
Assert.Equal(f1, g1);
|
||||
|
||||
// Test trap file generation
|
||||
var trapwriterFilename = project.GetTrapPath(logger, new TransformedPathStub("foo.cs"), TrapWriter.CompressionMode.Gzip);
|
||||
using (var trapwriter = project.CreateTrapWriter(logger, new TransformedPathStub("foo.cs"), TrapWriter.CompressionMode.Gzip, discardDuplicates: false))
|
||||
{
|
||||
trapwriter.Emit("1=*");
|
||||
Assert.False(File.Exists(trapwriterFilename));
|
||||
}
|
||||
Assert.True(File.Exists(trapwriterFilename));
|
||||
File.Delete(trapwriterFilename);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TestLayoutFile()
|
||||
{
|
||||
File.WriteAllLines("layout.txt", new string[]
|
||||
{
|
||||
"# Section",
|
||||
"TRAP_FOLDER=" + Path.GetFullPath("snapshot\\trap"),
|
||||
"ODASA_DB=snapshot\\db-csharp",
|
||||
"SOURCE_ARCHIVE=" + Path.GetFullPath("snapshot\\archive"),
|
||||
"ODASA_BUILD_ERROR_DIR=snapshot\build-errors",
|
||||
"-foo.cs",
|
||||
"bar.cs",
|
||||
"-excluded",
|
||||
"excluded/foo.cs",
|
||||
"included"
|
||||
});
|
||||
|
||||
var layout = new Semmle.Extraction.Layout(null, null, "layout.txt");
|
||||
|
||||
// Test general pattern matching
|
||||
Assert.True(layout.FileInLayout(new TransformedPathStub("bar.cs")));
|
||||
Assert.False(layout.FileInLayout(new TransformedPathStub("foo.cs")));
|
||||
Assert.False(layout.FileInLayout(new TransformedPathStub("goo.cs")));
|
||||
Assert.False(layout.FileInLayout(new TransformedPathStub("excluded/bar.cs")));
|
||||
Assert.True(layout.FileInLayout(new TransformedPathStub("excluded/foo.cs")));
|
||||
Assert.True(layout.FileInLayout(new TransformedPathStub("included/foo.cs")));
|
||||
|
||||
// Test the trap file
|
||||
var project = layout.LookupProjectOrNull(new TransformedPathStub("bar.cs"));
|
||||
Assert.NotNull(project);
|
||||
var trapwriterFilename = project!.GetTrapPath(logger, new TransformedPathStub("bar.cs"), TrapWriter.CompressionMode.Gzip);
|
||||
Assert.Equal(TrapWriter.NestPaths(logger, Path.GetFullPath("snapshot\\trap"), "bar.cs.trap.gz"),
|
||||
trapwriterFilename);
|
||||
|
||||
// Test the source archive
|
||||
var trapWriter = project.CreateTrapWriter(logger, new TransformedPathStub("bar.cs"), TrapWriter.CompressionMode.Gzip, discardDuplicates: false);
|
||||
trapWriter.Archive("layout.txt", new TransformedPathStub("layout.txt"), System.Text.Encoding.ASCII);
|
||||
var writtenFile = TrapWriter.NestPaths(logger, Path.GetFullPath("snapshot\\archive"), "layout.txt");
|
||||
Assert.True(File.Exists(writtenFile));
|
||||
File.Delete("layout.txt");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TestTrapOverridesLayout()
|
||||
{
|
||||
// When you specify both a trap file and a layout, use the trap file.
|
||||
var layout = new Semmle.Extraction.Layout(Path.GetFullPath("snapshot\\trap"), null, "something.txt");
|
||||
Assert.True(layout.FileInLayout(new TransformedPathStub("bar.cs")));
|
||||
var subProject = layout.LookupProjectOrNull(new TransformedPathStub("foo.cs"));
|
||||
Assert.NotNull(subProject);
|
||||
var f1 = subProject!.GetTrapPath(logger, new TransformedPathStub("foo.cs"), TrapWriter.CompressionMode.Gzip);
|
||||
var g1 = TrapWriter.NestPaths(logger, Path.GetFullPath("snapshot\\trap"), "foo.cs.trap.gz");
|
||||
Assert.Equal(f1, g1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TestMultipleSections()
|
||||
{
|
||||
File.WriteAllLines("layout.txt", new string[]
|
||||
{
|
||||
"# Section 1",
|
||||
"TRAP_FOLDER=" + Path.GetFullPath("snapshot\\trap1"),
|
||||
"ODASA_DB=snapshot\\db-csharp",
|
||||
"SOURCE_ARCHIVE=" + Path.GetFullPath("snapshot\\archive1"),
|
||||
"ODASA_BUILD_ERROR_DIR=snapshot\build-errors",
|
||||
"foo.cs",
|
||||
"# Section 2",
|
||||
"TRAP_FOLDER=" + Path.GetFullPath("snapshot\\trap2"),
|
||||
"ODASA_DB=snapshot\\db-csharp",
|
||||
"SOURCE_ARCHIVE=" + Path.GetFullPath("snapshot\\archive2"),
|
||||
"ODASA_BUILD_ERROR_DIR=snapshot\build-errors",
|
||||
"bar.cs",
|
||||
});
|
||||
|
||||
var layout = new Semmle.Extraction.Layout(null, null, "layout.txt");
|
||||
|
||||
// Use Section 2
|
||||
Assert.True(layout.FileInLayout(new TransformedPathStub("bar.cs")));
|
||||
var subProject = layout.LookupProjectOrNull(new TransformedPathStub("bar.cs"));
|
||||
Assert.NotNull(subProject);
|
||||
var f1 = subProject!.GetTrapPath(logger, new TransformedPathStub("bar.cs"), TrapWriter.CompressionMode.Gzip);
|
||||
var g1 = TrapWriter.NestPaths(logger, Path.GetFullPath("snapshot\\trap2"), "bar.cs.trap.gz");
|
||||
Assert.Equal(f1, g1);
|
||||
|
||||
// Use Section 1
|
||||
Assert.True(layout.FileInLayout(new TransformedPathStub("foo.cs")));
|
||||
subProject = layout.LookupProjectOrNull(new TransformedPathStub("foo.cs"));
|
||||
Assert.NotNull(subProject);
|
||||
var f2 = subProject!.GetTrapPath(logger, new TransformedPathStub("foo.cs"), TrapWriter.CompressionMode.Gzip);
|
||||
var g2 = TrapWriter.NestPaths(logger, Path.GetFullPath("snapshot\\trap1"), "foo.cs.trap.gz");
|
||||
Assert.Equal(f2, g2);
|
||||
|
||||
// boo.dll is not in the layout, so use layout from first section.
|
||||
Assert.False(layout.FileInLayout(new TransformedPathStub("boo.dll")));
|
||||
var f3 = layout.LookupProjectOrDefault(new TransformedPathStub("boo.dll")).GetTrapPath(logger, new TransformedPathStub("boo.dll"), TrapWriter.CompressionMode.Gzip);
|
||||
var g3 = TrapWriter.NestPaths(logger, Path.GetFullPath("snapshot\\trap1"), "boo.dll.trap.gz");
|
||||
Assert.Equal(f3, g3);
|
||||
|
||||
// boo.cs is not in the layout, so return null
|
||||
Assert.False(layout.FileInLayout(new TransformedPathStub("boo.cs")));
|
||||
Assert.Null(layout.LookupProjectOrNull(new TransformedPathStub("boo.cs")));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MissingLayout()
|
||||
{
|
||||
Assert.Throws<Extraction.Layout.InvalidLayoutException>(() =>
|
||||
new Semmle.Extraction.Layout(null, null, "nosuchfile.txt"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EmptyLayout()
|
||||
{
|
||||
File.Create("layout.txt").Close();
|
||||
Assert.Throws<Extraction.Layout.InvalidLayoutException>(() =>
|
||||
new Semmle.Extraction.Layout(null, null, "layout.txt"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void InvalidLayout()
|
||||
{
|
||||
File.WriteAllLines("layout.txt", new string[]
|
||||
{
|
||||
"# Section 1"
|
||||
});
|
||||
|
||||
Assert.Throws<Extraction.Layout.InvalidLayoutException>(() =>
|
||||
new Semmle.Extraction.Layout(null, null, "layout.txt"));
|
||||
}
|
||||
|
||||
private sealed class LoggerMock : ILogger
|
||||
{
|
||||
public void Dispose() { }
|
||||
|
||||
public void Log(Severity s, string text) { }
|
||||
}
|
||||
}
|
||||
|
||||
internal static class TrapWriterTestExtensions
|
||||
{
|
||||
public static void Emit(this TrapWriter trapFile, string s)
|
||||
{
|
||||
trapFile.Emit(new StringTrapEmitter(s));
|
||||
}
|
||||
|
||||
private class StringTrapEmitter : ITrapEmitter
|
||||
{
|
||||
private readonly string content;
|
||||
public StringTrapEmitter(string content)
|
||||
{
|
||||
this.content = content;
|
||||
}
|
||||
|
||||
public void EmitTrap(TextWriter trapFile)
|
||||
{
|
||||
trapFile.Write(content);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -14,7 +14,6 @@ namespace Semmle.Extraction.Tests
|
||||
|
||||
public OptionsTests()
|
||||
{
|
||||
Environment.SetEnvironmentVariable("SEMMLE_EXTRACTOR_OPTIONS", "");
|
||||
Environment.SetEnvironmentVariable("LGTM_INDEX_EXTRACTOR", "");
|
||||
}
|
||||
|
||||
@@ -126,14 +125,14 @@ namespace Semmle.Extraction.Tests
|
||||
[Fact]
|
||||
public void EnvironmentVariables()
|
||||
{
|
||||
Environment.SetEnvironmentVariable("SEMMLE_EXTRACTOR_OPTIONS", "--cil c");
|
||||
Environment.SetEnvironmentVariable("LGTM_INDEX_EXTRACTOR", "--cil c");
|
||||
options = CSharp.Options.CreateWithEnvironment(new string[] { "a", "b" });
|
||||
Assert.True(options.CIL);
|
||||
Assert.Equal("a", options.CompilerArguments[0]);
|
||||
Assert.Equal("b", options.CompilerArguments[1]);
|
||||
Assert.Equal("c", options.CompilerArguments[2]);
|
||||
|
||||
Environment.SetEnvironmentVariable("SEMMLE_EXTRACTOR_OPTIONS", "");
|
||||
Environment.SetEnvironmentVariable("LGTM_INDEX_EXTRACTOR", "");
|
||||
Environment.SetEnvironmentVariable("LGTM_INDEX_EXTRACTOR", "--nocil");
|
||||
options = CSharp.Options.CreateWithEnvironment(new string[] { "--cil" });
|
||||
Assert.False(options.CIL);
|
||||
|
||||
@@ -1,204 +0,0 @@
|
||||
using Semmle.Util.Logging;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
|
||||
namespace Semmle.Extraction
|
||||
{
|
||||
/// <summary>
|
||||
/// An extractor layout file.
|
||||
/// Represents the layout of projects into trap folders and source archives.
|
||||
/// </summary>
|
||||
public sealed class Layout
|
||||
{
|
||||
/// <summary>
|
||||
/// Exception thrown when the layout file is invalid.
|
||||
/// </summary>
|
||||
public class InvalidLayoutException : Exception
|
||||
{
|
||||
public InvalidLayoutException(string file, string message) :
|
||||
base("ODASA_CSHARP_LAYOUT " + file + " " + message)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// List of blocks in the layout file.
|
||||
/// </summary>
|
||||
private readonly List<LayoutBlock> blocks;
|
||||
|
||||
/// <summary>
|
||||
/// A subproject in the layout file.
|
||||
/// </summary>
|
||||
public class SubProject
|
||||
{
|
||||
/// <summary>
|
||||
/// The trap folder, or null for current directory.
|
||||
/// </summary>
|
||||
public string? TRAP_FOLDER { get; }
|
||||
|
||||
/// <summary>
|
||||
/// The source archive, or null to skip.
|
||||
/// </summary>
|
||||
public string? SOURCE_ARCHIVE { get; }
|
||||
|
||||
public SubProject(string? traps, string? archive)
|
||||
{
|
||||
TRAP_FOLDER = traps;
|
||||
SOURCE_ARCHIVE = archive;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the name of the trap file for a given source/assembly file.
|
||||
/// </summary>
|
||||
/// <param name="srcFile">The source file.</param>
|
||||
/// <returns>The full filepath of the trap file.</returns>
|
||||
public string GetTrapPath(ILogger logger, PathTransformer.ITransformedPath srcFile, TrapWriter.CompressionMode trapCompression) =>
|
||||
TrapWriter.TrapPath(logger, TRAP_FOLDER, srcFile, trapCompression);
|
||||
|
||||
/// <summary>
|
||||
/// Creates a trap writer for a given source/assembly file.
|
||||
/// </summary>
|
||||
/// <param name="srcFile">The source file.</param>
|
||||
/// <returns>A newly created TrapWriter.</returns>
|
||||
public TrapWriter CreateTrapWriter(ILogger logger, PathTransformer.ITransformedPath srcFile, TrapWriter.CompressionMode trapCompression, bool discardDuplicates) =>
|
||||
new TrapWriter(logger, srcFile, TRAP_FOLDER, SOURCE_ARCHIVE, trapCompression, discardDuplicates);
|
||||
}
|
||||
|
||||
private readonly SubProject defaultProject;
|
||||
|
||||
/// <summary>
|
||||
/// Finds the suitable directories for a given source file.
|
||||
/// Returns null if not included in the layout.
|
||||
/// </summary>
|
||||
/// <param name="sourceFile">The file to look up.</param>
|
||||
/// <returns>The relevant subproject, or null if not found.</returns>
|
||||
public SubProject? LookupProjectOrNull(PathTransformer.ITransformedPath sourceFile)
|
||||
{
|
||||
if (!useLayoutFile)
|
||||
return defaultProject;
|
||||
|
||||
return blocks
|
||||
.Where(block => block.Matches(sourceFile))
|
||||
.Select(block => block.Directories)
|
||||
.FirstOrDefault();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Finds the suitable directories for a given source file.
|
||||
/// Returns the default project if not included in the layout.
|
||||
/// </summary>
|
||||
/// <param name="sourceFile">The file to look up.</param>
|
||||
/// <returns>The relevant subproject, or DefaultProject if not found.</returns>
|
||||
public SubProject LookupProjectOrDefault(PathTransformer.ITransformedPath sourceFile)
|
||||
{
|
||||
return LookupProjectOrNull(sourceFile) ?? defaultProject;
|
||||
}
|
||||
|
||||
private readonly bool useLayoutFile;
|
||||
|
||||
/// <summary>
|
||||
/// Default constructor reads parameters from the environment.
|
||||
/// </summary>
|
||||
public Layout() : this(
|
||||
Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_TRAP_DIR") ?? Environment.GetEnvironmentVariable("TRAP_FOLDER"),
|
||||
Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR") ?? Environment.GetEnvironmentVariable("SOURCE_ARCHIVE"),
|
||||
Environment.GetEnvironmentVariable("ODASA_CSHARP_LAYOUT"))
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates the project layout. Reads the layout file if specified.
|
||||
/// </summary>
|
||||
/// <param name="traps">Directory for trap files, or null to use layout/current directory.</param>
|
||||
/// <param name="archive">Directory for source archive, or null for layout/no archive.</param>
|
||||
/// <param name="layout">Path of layout file, or null for no layout.</param>
|
||||
/// <exception cref="InvalidLayoutException">Failed to read layout file.</exception>
|
||||
public Layout(string? traps, string? archive, string? layout)
|
||||
{
|
||||
useLayoutFile = string.IsNullOrEmpty(traps) && !string.IsNullOrEmpty(layout);
|
||||
blocks = new List<LayoutBlock>();
|
||||
|
||||
if (useLayoutFile)
|
||||
{
|
||||
ReadLayoutFile(layout!);
|
||||
defaultProject = blocks[0].Directories;
|
||||
}
|
||||
else
|
||||
{
|
||||
defaultProject = new SubProject(traps, archive);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Is the source file included in the layout?
|
||||
/// </summary>
|
||||
/// <param name="path">The absolute path of the file to query.</param>
|
||||
/// <returns>True iff there is no layout file or the layout file specifies the file.</returns>
|
||||
public bool FileInLayout(PathTransformer.ITransformedPath path) => LookupProjectOrNull(path) is not null;
|
||||
|
||||
private void ReadLayoutFile(string layout)
|
||||
{
|
||||
try
|
||||
{
|
||||
var lines = File.ReadAllLines(layout);
|
||||
|
||||
var i = 0;
|
||||
while (!lines[i].StartsWith("#"))
|
||||
i++;
|
||||
while (i < lines.Length)
|
||||
{
|
||||
var block = new LayoutBlock(lines, ref i);
|
||||
blocks.Add(block);
|
||||
}
|
||||
|
||||
if (blocks.Count == 0)
|
||||
throw new InvalidLayoutException(layout, "contains no blocks");
|
||||
}
|
||||
catch (IOException ex)
|
||||
{
|
||||
throw new InvalidLayoutException(layout, ex.Message);
|
||||
}
|
||||
catch (IndexOutOfRangeException)
|
||||
{
|
||||
throw new InvalidLayoutException(layout, "is invalid");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class LayoutBlock
|
||||
{
|
||||
private readonly List<FilePattern> filePatterns = new List<FilePattern>();
|
||||
|
||||
public Layout.SubProject Directories { get; }
|
||||
|
||||
private static string? ReadVariable(string name, string line)
|
||||
{
|
||||
var prefix = name + "=";
|
||||
if (!line.StartsWith(prefix))
|
||||
return null;
|
||||
return line.Substring(prefix.Length).Trim();
|
||||
}
|
||||
|
||||
public LayoutBlock(string[] lines, ref int i)
|
||||
{
|
||||
// first line: #name
|
||||
i++;
|
||||
var trapFolder = ReadVariable("TRAP_FOLDER", lines[i++]);
|
||||
// Don't care about ODASA_DB.
|
||||
ReadVariable("ODASA_DB", lines[i++]);
|
||||
var sourceArchive = ReadVariable("SOURCE_ARCHIVE", lines[i++]);
|
||||
|
||||
Directories = new Layout.SubProject(trapFolder, sourceArchive);
|
||||
// Don't care about ODASA_BUILD_ERROR_DIR.
|
||||
ReadVariable("ODASA_BUILD_ERROR_DIR", lines[i++]);
|
||||
while (i < lines.Length && !lines[i].StartsWith("#"))
|
||||
{
|
||||
filePatterns.Add(new FilePattern(lines[i++]));
|
||||
}
|
||||
}
|
||||
|
||||
public bool Matches(PathTransformer.ITransformedPath path) => FilePattern.Matches(filePatterns, path.Value, out var _);
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using Semmle.Util;
|
||||
using Semmle.Util.Logging;
|
||||
|
||||
namespace Semmle.Extraction
|
||||
{
|
||||
@@ -35,6 +36,20 @@ namespace Semmle.Extraction
|
||||
ITransformedPath WithSuffix(string suffix);
|
||||
|
||||
string DatabaseId { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the name of the trap file for this file.
|
||||
/// </summary>
|
||||
/// <returns>The full filepath of the trap file.</returns>
|
||||
public string GetTrapPath(ILogger logger, TrapWriter.CompressionMode trapCompression) =>
|
||||
TrapWriter.TrapPath(logger, Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"), this, trapCompression);
|
||||
|
||||
/// <summary>
|
||||
/// Creates a trap writer for this file.
|
||||
/// </summary>
|
||||
/// <returns>A newly created TrapWriter.</returns>
|
||||
public TrapWriter CreateTrapWriter(ILogger logger, TrapWriter.CompressionMode trapCompression, bool discardDuplicates) =>
|
||||
new(logger, this, Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"), Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"), trapCompression, discardDuplicates);
|
||||
}
|
||||
|
||||
private struct TransformedPath : ITransformedPath
|
||||
|
||||
@@ -234,9 +234,7 @@ namespace Semmle.Util
|
||||
/// <returns>A new CanonicalPathCache.</returns>
|
||||
public static CanonicalPathCache Create(ILogger logger, int maxCapacity)
|
||||
{
|
||||
var preserveSymlinks =
|
||||
Environment.GetEnvironmentVariable("CODEQL_PRESERVE_SYMLINKS") == "true" ||
|
||||
Environment.GetEnvironmentVariable("SEMMLE_PRESERVE_SYMLINKS") == "true";
|
||||
var preserveSymlinks = Environment.GetEnvironmentVariable("CODEQL_PRESERVE_SYMLINKS") == "true";
|
||||
return Create(logger, maxCapacity, preserveSymlinks ? CanonicalPathCache.Symlinks.Preserve : CanonicalPathCache.Symlinks.Follow);
|
||||
|
||||
}
|
||||
|
||||
@@ -74,7 +74,7 @@ namespace Semmle.Util.Logging
|
||||
}
|
||||
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
|
||||
{
|
||||
Console.Error.WriteLine("SEMMLE: Couldn't initialise C# extractor output: " + ex.Message + "\n" + ex.StackTrace);
|
||||
Console.Error.WriteLine("CodeQL: Couldn't initialise C# extractor output: " + ex.Message + "\n" + ex.StackTrace);
|
||||
Console.Error.Flush();
|
||||
throw;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user