hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-17 01:03:14 +01:00
Code Issues Packages Projects Releases Wiki Activity

C#: Autobuilder fixes

Browse Source 
This commit fixes a few issues that were identified during the last dist upgrade,
and which were introduced/revealed on 836daaf07b.

- Expand environment variables that are passed from `lgtm.yml` to the autobuilder,
  for example `solution: $LGTM_SRC/mysolution.sln`.
- Distinguish between when a build rule is applied automatically and when it is applied
  manually via `lgtm.yml`.
- Catch `FileNotFoundException`s when parsing project files and solution files.
This commit is contained in:
Tom Hvitved
2018-12-04 13:57:01 +01:00
parent a23f7a6b0e
commit ad77afef04
13 changed files with 163 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

120
csharp/autobuilder/Semmle.Autobuild.Tests/BuildScripts.cs
View File

@@ -130,7 +130,7 @@ namespace Semmle.Extraction.Tests
string IBuildActions.PathCombine(params string[] parts)
{
return string.Join('\\', parts.Where(p => !string.IsNullOrWhiteSpace(p)));
return string.Join(IsWindows ? '\\' : '/', parts.Where(p => !string.IsNullOrWhiteSpace(p)));
}
string IBuildActions.GetFullPath(string path) => path;
@@ -146,6 +146,13 @@ namespace Semmle.Extraction.Tests
return xml;
throw new ArgumentException("Missing LoadXml " + filename);
}
public string EnvironmentExpandEnvironmentVariables(string s)
{
foreach (var kvp in GetEnvironmentVariable)
s = s.Replace($"%{kvp.Key}%", kvp.Value);
return s;
}
}
/// <summary>
@@ -394,9 +401,9 @@ namespace Semmle.Extraction.Tests
Actions.RunProcess["dotnet --info"] = 0;
Actions.RunProcess["dotnet clean test.csproj"] = 0;
Actions.RunProcess["dotnet restore test.csproj"] = 0;
Actions.RunProcess[@"C:\odasa\tools\odasa index --auto dotnet build --no-incremental /p:UseSharedCompilation=false test.csproj"] = 0;
Actions.RunProcess[@"C:\odasa\tools\java\bin\java -jar C:\odasa\tools\extractor-asp.jar ."] = 0;
Actions.RunProcess[@"C:\odasa\tools\odasa index --xml --extensions config csproj props xml"] = 0;
Actions.RunProcess[@"C:\odasa/tools/odasa index --auto dotnet build --no-incremental /p:UseSharedCompilation=false test.csproj"] = 0;
Actions.RunProcess[@"C:\odasa\tools\java/bin/java -jar C:\odasa/tools/extractor-asp.jar ."] = 0;
Actions.RunProcess[@"C:\odasa/tools/odasa index --xml --extensions config csproj props xml"] = 0;
Actions.FileExists["csharp.log"] = true;
Actions.FileExists["test.csproj"] = true;
Actions.GetEnvironmentVariable["TRAP_FOLDER"] = null;
@@ -470,6 +477,7 @@ namespace Semmle.Extraction.Tests
[Fact]
public void TestVsWhereSucceeded()
{
Actions.IsWindows = true;
Actions.GetEnvironmentVariable["ProgramFiles(x86)"] = @"C:\Program Files (x86)";
Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = true;
Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = 0;
@@ -487,6 +495,7 @@ namespace Semmle.Extraction.Tests
[Fact]
public void TestVsWhereNotExist()
{
Actions.IsWindows = true;
Actions.GetEnvironmentVariable["ProgramFiles(x86)"] = @"C:\Program Files (x86)";
Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = false;
@@ -497,6 +506,7 @@ namespace Semmle.Extraction.Tests
[Fact]
public void TestVcVarsAllBatFiles()
{
Actions.IsWindows = true;
Actions.GetEnvironmentVariable["ProgramFiles(x86)"] = @"C:\Program Files (x86)";
Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = false;
Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = true;
@@ -511,9 +521,9 @@ namespace Semmle.Extraction.Tests
[Fact]
public void TestLinuxBuildlessExtractionSuccess()
{
Actions.RunProcess[@"C:\odasa\tools\csharp\Semmle.Extraction.CSharp.Standalone --references:."] = 0;
Actions.RunProcess[@"C:\odasa\tools\java\bin\java -jar C:\odasa\tools\extractor-asp.jar ."] = 0;
Actions.RunProcess[@"C:\odasa\tools\odasa index --xml --extensions config csproj props xml"] = 0;
Actions.RunProcess[@"C:\odasa\tools/csharp/Semmle.Extraction.CSharp.Standalone --references:."] = 0;
Actions.RunProcess[@"C:\odasa\tools\java/bin/java -jar C:\odasa/tools/extractor-asp.jar ."] = 0;
Actions.RunProcess[@"C:\odasa/tools/odasa index --xml --extensions config csproj props xml"] = 0;
Actions.FileExists["csharp.log"] = true;
Actions.GetEnvironmentVariable["TRAP_FOLDER"] = null;
Actions.GetEnvironmentVariable["SOURCE_ARCHIVE"] = null;
@@ -527,9 +537,7 @@ namespace Semmle.Extraction.Tests
[Fact]
public void TestLinuxBuildlessExtractionFailed()
{
Actions.RunProcess[@"C:\odasa\tools\csharp\Semmle.Extraction.CSharp.Standalone --references:."] = 10;
Actions.RunProcess[@"C:\odasa\tools\java\bin\java -jar C:\odasa\tools\extractor-asp.jar ."] = 0;
Actions.RunProcess[@"C:\odasa\tools\odasa index --xml --extensions config"] = 0;
Actions.RunProcess[@"C:\odasa\tools/csharp/Semmle.Extraction.CSharp.Standalone --references:."] = 10;
Actions.FileExists["csharp.log"] = true;
Actions.GetEnvironmentVariable["TRAP_FOLDER"] = null;
Actions.GetEnvironmentVariable["SOURCE_ARCHIVE"] = null;
@@ -543,9 +551,9 @@ namespace Semmle.Extraction.Tests
[Fact]
public void TestLinuxBuildlessExtractionSolution()
{
Actions.RunProcess[@"C:\odasa\tools\csharp\Semmle.Extraction.CSharp.Standalone foo.sln --references:."] = 0;
Actions.RunProcess[@"C:\odasa\tools\java\bin\java -jar C:\odasa\tools\extractor-asp.jar ."] = 0;
Actions.RunProcess[@"C:\odasa\tools\odasa index --xml --extensions config csproj props xml"] = 0;
Actions.RunProcess[@"C:\odasa\tools/csharp/Semmle.Extraction.CSharp.Standalone foo.sln --references:."] = 0;
Actions.RunProcess[@"C:\odasa\tools\java/bin/java -jar C:\odasa/tools/extractor-asp.jar ."] = 0;
Actions.RunProcess[@"C:\odasa/tools/odasa index --xml --extensions config csproj props xml"] = 0;
Actions.FileExists["csharp.log"] = true;
Actions.GetEnvironmentVariable["TRAP_FOLDER"] = null;
Actions.GetEnvironmentVariable["SOURCE_ARCHIVE"] = null;
@@ -587,9 +595,9 @@ namespace Semmle.Extraction.Tests
[Fact]
public void TestLinuxBuildCommand()
{
Actions.RunProcess["C:\\odasa\\tools\\odasa index --auto \"./build.sh --skip-tests\""] = 0;
Actions.RunProcess[@"C:\odasa\tools\java\bin\java -jar C:\odasa\tools\extractor-asp.jar ."] = 0;
Actions.RunProcess[@"C:\odasa\tools\odasa index --xml --extensions config csproj props xml"] = 0;
Actions.RunProcess[@"C:\odasa/tools/odasa index --auto ""./build.sh --skip-tests"""] = 0;
Actions.RunProcess[@"C:\odasa\tools\java/bin/java -jar C:\odasa/tools/extractor-asp.jar ."] = 0;
Actions.RunProcess[@"C:\odasa/tools/odasa index --xml --extensions config csproj props xml"] = 0;
Actions.FileExists["csharp.log"] = true;
Actions.GetEnvironmentVariable["TRAP_FOLDER"] = null;
Actions.GetEnvironmentVariable["SOURCE_ARCHIVE"] = null;
@@ -610,10 +618,10 @@ namespace Semmle.Extraction.Tests
Actions.GetEnvironmentVariable["TRAP_FOLDER"] = null;
Actions.GetEnvironmentVariable["SOURCE_ARCHIVE"] = null;
Actions.RunProcess["/bin/chmod u+x build/build.sh"] = 0;
Actions.RunProcess[@"C:\odasa\tools\odasa index --auto build/build.sh"] = 0;
Actions.RunProcessWorkingDirectory[@"C:\odasa\tools\odasa index --auto build/build.sh"] = "build";
Actions.RunProcess[@"C:\odasa\tools\java\bin\java -jar C:\odasa\tools\extractor-asp.jar ."] = 0;
Actions.RunProcess[@"C:\odasa\tools\odasa index --xml --extensions config csproj props xml"] = 0;
Actions.RunProcess[@"C:\odasa/tools/odasa index --auto build/build.sh"] = 0;
Actions.RunProcessWorkingDirectory[@"C:\odasa/tools/odasa index --auto build/build.sh"] = "build";
Actions.RunProcess[@"C:\odasa\tools\java/bin/java -jar C:\odasa/tools/extractor-asp.jar ."] = 0;
Actions.RunProcess[@"C:\odasa/tools/odasa index --xml --extensions config csproj props xml"] = 0;
Actions.FileExists["csharp.log"] = true;
var autobuilder = CreateAutoBuilder("csharp", false);
@@ -629,8 +637,8 @@ namespace Semmle.Extraction.Tests
Actions.GetEnvironmentVariable["SOURCE_ARCHIVE"] = null;
Actions.RunProcess["/bin/chmod u+x build.sh"] = 0;
Actions.RunProcess[@"C:\odasa\tools\odasa index --auto build.sh"] = 0;
Actions.RunProcessWorkingDirectory[@"C:\odasa\tools\odasa index --auto build.sh"] = "";
Actions.RunProcess[@"C:\odasa/tools/odasa index --auto build.sh"] = 0;
Actions.RunProcessWorkingDirectory[@"C:\odasa/tools/odasa index --auto build.sh"] = "";
Actions.FileExists["csharp.log"] = false;
var autobuilder = CreateAutoBuilder("csharp", false);
@@ -646,8 +654,8 @@ namespace Semmle.Extraction.Tests
Actions.GetEnvironmentVariable["SOURCE_ARCHIVE"] = null;
Actions.RunProcess["/bin/chmod u+x build.sh"] = 0;
Actions.RunProcess[@"C:\odasa\tools\odasa index --auto build.sh"] = 5;
Actions.RunProcessWorkingDirectory[@"C:\odasa\tools\odasa index --auto build.sh"] = "";
Actions.RunProcess[@"C:\odasa/tools/odasa index --auto build.sh"] = 5;
Actions.RunProcessWorkingDirectory[@"C:\odasa/tools/odasa index --auto build.sh"] = "";
Actions.FileExists["csharp.log"] = true;
var autobuilder = CreateAutoBuilder("csharp", false);
@@ -796,9 +804,9 @@ namespace Semmle.Extraction.Tests
[Fact]
public void TestSkipNugetBuildless()
{
Actions.RunProcess[@"C:\odasa\tools\csharp\Semmle.Extraction.CSharp.Standalone foo.sln --references:. --skip-nuget"] = 0;
Actions.RunProcess[@"C:\odasa\tools\java\bin\java -jar C:\odasa\tools\extractor-asp.jar ."] = 0;
Actions.RunProcess[@"C:\odasa\tools\odasa index --xml --extensions config csproj props xml"] = 0;
Actions.RunProcess[@"C:\odasa\tools/csharp/Semmle.Extraction.CSharp.Standalone foo.sln --references:. --skip-nuget"] = 0;
Actions.RunProcess[@"C:\odasa\tools\java/bin/java -jar C:\odasa/tools/extractor-asp.jar ."] = 0;
Actions.RunProcess[@"C:\odasa/tools/odasa index --xml --extensions config csproj props xml"] = 0;
Actions.FileExists["csharp.log"] = true;
Actions.GetEnvironmentVariable["TRAP_FOLDER"] = null;
Actions.GetEnvironmentVariable["SOURCE_ARCHIVE"] = null;
@@ -816,9 +824,9 @@ namespace Semmle.Extraction.Tests
Actions.RunProcess["dotnet --info"] = 0;
Actions.RunProcess["dotnet clean test.csproj"] = 0;
Actions.RunProcess["dotnet restore test.csproj"] = 0;
Actions.RunProcess[@"C:\odasa\tools\odasa index --auto dotnet build --no-incremental /p:UseSharedCompilation=false --no-restore test.csproj"] = 0;
Actions.RunProcess[@"C:\odasa\tools\java\bin\java -jar C:\odasa\tools\extractor-asp.jar ."] = 0;
Actions.RunProcess[@"C:\odasa\tools\odasa index --xml --extensions config csproj props xml"] = 0;
Actions.RunProcess[@"C:\odasa/tools/odasa index --auto dotnet build --no-incremental /p:UseSharedCompilation=false --no-restore test.csproj"] = 0;
Actions.RunProcess[@"C:\odasa\tools\java/bin/java -jar C:\odasa/tools/extractor-asp.jar ."] = 0;
Actions.RunProcess[@"C:\odasa/tools/odasa index --xml --extensions config csproj props xml"] = 0;
Actions.FileExists["csharp.log"] = true;
Actions.FileExists["test.csproj"] = true;
Actions.GetEnvironmentVariable["TRAP_FOLDER"] = null;
@@ -846,13 +854,13 @@ namespace Semmle.Extraction.Tests
Actions.RunProcessOut["dotnet --list-sdks"] = "2.1.2 [C:\\Program Files\\dotnet\\sdks]\n2.1.4 [C:\\Program Files\\dotnet\\sdks]";
Actions.RunProcess[@"curl -sO https://dot.net/v1/dotnet-install.sh"] = 0;
Actions.RunProcess[@"chmod u+x dotnet-install.sh"] = 0;
Actions.RunProcess[@"./dotnet-install.sh --channel release --version 2.1.3 --install-dir C:\Project\.dotnet"] = 0;
Actions.RunProcess[@"C:\Project\.dotnet\dotnet --info"] = 0;
Actions.RunProcess[@"C:\Project\.dotnet\dotnet clean test.csproj"] = 0;
Actions.RunProcess[@"C:\Project\.dotnet\dotnet restore test.csproj"] = 0;
Actions.RunProcess[@"C:\odasa\tools\odasa index --auto C:\Project\.dotnet\dotnet build --no-incremental /p:UseSharedCompilation=false test.csproj"] = 0;
Actions.RunProcess[@"C:\odasa\tools\java\bin\java -jar C:\odasa\tools\extractor-asp.jar ."] = 0;
Actions.RunProcess[@"C:\odasa\tools\odasa index --xml --extensions config csproj props xml"] = 0;
Actions.RunProcess[@"./dotnet-install.sh --channel release --version 2.1.3 --install-dir C:\Project/.dotnet"] = 0;
Actions.RunProcess[@"C:\Project/.dotnet/dotnet --info"] = 0;
Actions.RunProcess[@"C:\Project/.dotnet/dotnet clean test.csproj"] = 0;
Actions.RunProcess[@"C:\Project/.dotnet/dotnet restore test.csproj"] = 0;
Actions.RunProcess[@"C:\odasa/tools/odasa index --auto C:\Project/.dotnet/dotnet build --no-incremental /p:UseSharedCompilation=false test.csproj"] = 0;
Actions.RunProcess[@"C:\odasa\tools\java/bin/java -jar C:\odasa/tools/extractor-asp.jar ."] = 0;
Actions.RunProcess[@"C:\odasa/tools/odasa index --xml --extensions config csproj props xml"] = 0;
Actions.FileExists["csharp.log"] = true;
Actions.FileExists["test.csproj"] = true;
Actions.GetEnvironmentVariable["TRAP_FOLDER"] = null;
@@ -881,13 +889,13 @@ namespace Semmle.Extraction.Tests
Actions.RunProcessOut["dotnet --list-sdks"] = "2.1.3 [C:\\Program Files\\dotnet\\sdks]\n2.1.4 [C:\\Program Files\\dotnet\\sdks]";
Actions.RunProcess[@"curl -sO https://dot.net/v1/dotnet-install.sh"] = 0;
Actions.RunProcess[@"chmod u+x dotnet-install.sh"] = 0;
Actions.RunProcess[@"./dotnet-install.sh --channel release --version 2.1.3 --install-dir C:\Project\.dotnet"] = 0;
Actions.RunProcess[@"C:\Project\.dotnet\dotnet --info"] = 0;
Actions.RunProcess[@"C:\Project\.dotnet\dotnet clean test.csproj"] = 0;
Actions.RunProcess[@"C:\Project\.dotnet\dotnet restore test.csproj"] = 0;
Actions.RunProcess[@"C:\odasa\tools\odasa index --auto C:\Project\.dotnet\dotnet build --no-incremental /p:UseSharedCompilation=false test.csproj"] = 0;
Actions.RunProcess[@"C:\odasa\tools\java\bin\java -jar C:\odasa\tools\extractor-asp.jar ."] = 0;
Actions.RunProcess[@"C:\odasa\tools\odasa index --xml --extensions config csproj props xml"] = 0;
Actions.RunProcess[@"./dotnet-install.sh --channel release --version 2.1.3 --install-dir C:\Project/.dotnet"] = 0;
Actions.RunProcess[@"C:\Project/.dotnet/dotnet --info"] = 0;
Actions.RunProcess[@"C:\Project/.dotnet/dotnet clean test.csproj"] = 0;
Actions.RunProcess[@"C:\Project/.dotnet/dotnet restore test.csproj"] = 0;
Actions.RunProcess[@"C:\odasa/tools/odasa index --auto C:\Project/.dotnet/dotnet build --no-incremental /p:UseSharedCompilation=false test.csproj"] = 0;
Actions.RunProcess[@"C:\odasa\tools\java/bin/java -jar C:\odasa/tools/extractor-asp.jar ."] = 0;
Actions.RunProcess[@"C:\odasa/tools/odasa index --xml --extensions config csproj props xml"] = 0;
Actions.FileExists["csharp.log"] = true;
Actions.FileExists["test.csproj"] = true;
Actions.GetEnvironmentVariable["TRAP_FOLDER"] = null;
@@ -988,10 +996,10 @@ namespace Semmle.Extraction.Tests
[Fact]
public void TestDirsProjLinux()
{
Actions.RunProcess[@"mono C:\odasa\tools\csharp\nuget\nuget.exe restore dirs.proj"] = 1;
Actions.RunProcess[@"C:\odasa\tools\odasa index --auto msbuild dirs.proj /p:UseSharedCompilation=false /t:rebuild /p:MvcBuildViews=true"] = 0;
Actions.RunProcess[@"C:\odasa\tools\java\bin\java -jar C:\odasa\tools\extractor-asp.jar ."] = 0;
Actions.RunProcess[@"C:\odasa\tools\odasa index --xml --extensions config csproj props xml"] = 0;
Actions.RunProcess[@"mono C:\odasa\tools/csharp/nuget/nuget.exe restore dirs.proj"] = 1;
Actions.RunProcess[@"C:\odasa/tools/odasa index --auto msbuild dirs.proj /p:UseSharedCompilation=false /t:rebuild /p:MvcBuildViews=true"] = 0;
Actions.RunProcess[@"C:\odasa\tools\java/bin/java -jar C:\odasa/tools/extractor-asp.jar ."] = 0;
Actions.RunProcess[@"C:\odasa/tools/odasa index --xml --extensions config csproj props xml"] = 0;
Actions.FileExists["csharp.log"] = true;
Actions.FileExists["a/test.csproj"] = true;
Actions.FileExists["dirs.proj"] = true;
@@ -1042,5 +1050,21 @@ namespace Semmle.Extraction.Tests
var autobuilder = CreateAutoBuilder("csharp", false);
TestAutobuilderScript(autobuilder, 1, 0);
}
[Fact]
public void TestAsStringWithExpandedEnvVarsWindows()
{
Actions.IsWindows = true;
Actions.GetEnvironmentVariable["LGTM_SRC"] = @"C:\repo";
Assert.Equal(@"C:\repo\test", @"%LGTM_SRC%\test".AsStringWithExpandedEnvVars(Actions));
}
[Fact]
public void TestAsStringWithExpandedEnvVarsLinux()
{
Actions.IsWindows = false;
Actions.GetEnvironmentVariable["LGTM_SRC"] = "/tmp/repo";
Assert.Equal("/tmp/repo/test", "$LGTM_SRC/test".AsStringWithExpandedEnvVars(Actions));
}
}
}

2
csharp/autobuilder/Semmle.Autobuild/AspBuildRule.cs
View File

@@ -7,7 +7,7 @@ namespace Semmle.Autobuild
/// </summary>
class AspBuildRule : IBuildRule
{
public BuildScript Analyse(Autobuilder builder)
public BuildScript Analyse(Autobuilder builder, bool auto)
{
var command = new CommandBuilder(builder.Actions).
RunCommand(builder.Actions.PathCombine(builder.SemmleJavaHome, "bin", "java")).

27
csharp/autobuilder/Semmle.Autobuild/AutobuildOptions.cs
View File

@@ -1,5 +1,6 @@
using System;
using System.Linq;
using System.Text.RegularExpressions;
namespace Semmle.Autobuild
{
@@ -37,14 +38,14 @@ namespace Semmle.Autobuild
{
RootDirectory = actions.GetCurrentDirectory();
VsToolsVersion = actions.GetEnvironmentVariable(prefix + "VSTOOLS_VERSION");
MsBuildArguments = actions.GetEnvironmentVariable(prefix + "MSBUILD_ARGUMENTS");
MsBuildArguments = actions.GetEnvironmentVariable(prefix + "MSBUILD_ARGUMENTS").AsStringWithExpandedEnvVars(actions);
MsBuildPlatform = actions.GetEnvironmentVariable(prefix + "MSBUILD_PLATFORM");
MsBuildConfiguration = actions.GetEnvironmentVariable(prefix + "MSBUILD_CONFIGURATION");
MsBuildTarget = actions.GetEnvironmentVariable(prefix + "MSBUILD_TARGET");
DotNetArguments = actions.GetEnvironmentVariable(prefix + "DOTNET_ARGUMENTS");
DotNetArguments = actions.GetEnvironmentVariable(prefix + "DOTNET_ARGUMENTS").AsStringWithExpandedEnvVars(actions);
DotNetVersion = actions.GetEnvironmentVariable(prefix + "DOTNET_VERSION");
BuildCommand = actions.GetEnvironmentVariable(prefix + "BUILD_COMMAND");
Solution = actions.GetEnvironmentVariable(prefix + "SOLUTION").AsList(new string[0]);
Solution = actions.GetEnvironmentVariable(prefix + "SOLUTION").AsListWithExpandedEnvVars(actions, new string[0]);
IgnoreErrors = actions.GetEnvironmentVariable(prefix + "IGNORE_ERRORS").AsBool("ignore_errors", false);
Buildless = actions.GetEnvironmentVariable(prefix + "BUILDLESS").AsBool("buildless", false);
@@ -92,12 +93,28 @@ namespace Semmle.Autobuild
}
}
public static string[] AsList(this string value, string[] defaultValue)
public static string[] AsListWithExpandedEnvVars(this string value, IBuildActions actions, string[] defaultValue)
{
if (value == null)
return defaultValue;
return value.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries).ToArray();
return value.
Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries).
Select(s => AsStringWithExpandedEnvVars(s, actions)).ToArray();
}
static readonly Regex linuxEnvRegEx = new Regex(@"\$([A-Z_][A-Z_0-9]*)", RegexOptions.Compiled);
public static string AsStringWithExpandedEnvVars(this string value, IBuildActions actions)
{
if (string.IsNullOrEmpty(value))
return value;
// `Environment.ExpandEnvironmentVariables` only works with Windows-style
// environment variables
var windowsStyle = actions.IsWindows()
? value
: linuxEnvRegEx.Replace(value, m => $"%{m.Groups[1].Value}%");
return actions.EnvironmentExpandEnvironmentVariables(windowsStyle);
}
}
}

33
csharp/autobuilder/Semmle.Autobuild/Autobuilder.cs
View File

@@ -16,7 +16,8 @@ namespace Semmle.Autobuild
/// Analyse the files and produce a build script.
/// </summary>
/// <param name="builder">The files and options relating to the build.</param>
BuildScript Analyse(Autobuilder builder);
/// <param name="auto">Whether this build rule is being automatically applied.</param>
BuildScript Analyse(Autobuilder builder, bool auto);
}
/// <summary>
@@ -135,9 +136,9 @@ namespace Semmle.Autobuild
foreach (var solution in options.Solution)
{
if (actions.FileExists(solution))
ret.Add(new Solution(this, solution));
ret.Add(new Solution(this, solution, true));
else
Log(Severity.Error, $"The specified solution file {solution} was not found");
Log(Severity.Error, $"The specified project or solution file {solution} was not found");
}
return ret;
}
@@ -172,7 +173,7 @@ namespace Semmle.Autobuild
return ret;
// Then look for `.sln` files
ret = FindFiles(".sln", f => new Solution(this, f))?.ToList();
ret = FindFiles(".sln", f => new Solution(this, f, false))?.ToList();
if (ret != null)
return ret;
@@ -250,17 +251,17 @@ namespace Semmle.Autobuild
switch (GetCSharpBuildStrategy())
{
case CSharpBuildStrategy.CustomBuildCommand:
attempt = new BuildCommandRule().Analyse(this) & CheckExtractorRun(true);
attempt = new BuildCommandRule().Analyse(this, false) & CheckExtractorRun(true);
break;
case CSharpBuildStrategy.Buildless:
// No need to check that the extractor has been executed in buildless mode
attempt = new StandaloneBuildRule().Analyse(this);
attempt = new StandaloneBuildRule().Analyse(this, false);
break;
case CSharpBuildStrategy.MSBuild:
attempt = new MsBuildRule().Analyse(this) & CheckExtractorRun(true);
attempt = new MsBuildRule().Analyse(this, false) & CheckExtractorRun(true);
break;
case CSharpBuildStrategy.DotNet:
attempt = new DotNetRule().Analyse(this) & CheckExtractorRun(true);
attempt = new DotNetRule().Analyse(this, false) & CheckExtractorRun(true);
break;
case CSharpBuildStrategy.Auto:
var cleanTrapFolder =
@@ -285,11 +286,11 @@ namespace Semmle.Autobuild
attempt =
// First try .NET Core
IntermediateAttempt(new DotNetRule().Analyse(this)) |
IntermediateAttempt(new DotNetRule().Analyse(this, true)) |
// Then MSBuild
(() => IntermediateAttempt(new MsBuildRule().Analyse(this))) |
(() => IntermediateAttempt(new MsBuildRule().Analyse(this, true))) |
// And finally look for a script that might be a build script
(() => new BuildCommandAutoRule().Analyse(this) & CheckExtractorRun(true)) |
(() => new BuildCommandAutoRule().Analyse(this, true) & CheckExtractorRun(true)) |
// All attempts failed: print message
AutobuildFailure();
break;
@@ -297,8 +298,8 @@ namespace Semmle.Autobuild
return
attempt &
(() => new AspBuildRule().Analyse(this)) &
(() => new XmlBuildRule().Analyse(this));
(() => new AspBuildRule().Analyse(this, false)) &
(() => new XmlBuildRule().Analyse(this, false));
}
/// <summary>
@@ -337,13 +338,13 @@ namespace Semmle.Autobuild
BuildScript GetCppBuildScript()
{
if (Options.BuildCommand != null)
return new BuildCommandRule().Analyse(this);
return new BuildCommandRule().Analyse(this, false);
return
// First try MSBuild
new MsBuildRule().Analyse(this) |
new MsBuildRule().Analyse(this, true) |
// Then look for a script that might be a build script
(() => new BuildCommandAutoRule().Analyse(this)) |
(() => new BuildCommandAutoRule().Analyse(this, true)) |
// All attempts failed: print message
AutobuildFailure();
}

8
csharp/autobuilder/Semmle.Autobuild/BuildActions.cs
View File

@@ -113,6 +113,12 @@ namespace Semmle.Autobuild
/// Loads the XML document from <paramref name="filename"/>.
/// </summary>
XmlDocument LoadXml(string filename);
/// <summary>
/// Expand all Windows-style environment variables in <paramref name="s"/>,
/// Environment.ExpandEnvironmentVariables()
/// </summary>
string EnvironmentExpandEnvironmentVariables(string s);
}
/// <summary>
@@ -187,6 +193,8 @@ namespace Semmle.Autobuild
string IBuildActions.GetFullPath(string path) => Path.GetFullPath(path);
public string EnvironmentExpandEnvironmentVariables(string s) => Environment.ExpandEnvironmentVariables(s);
public static readonly IBuildActions Instance = new SystemBuildActions();
}
}

2
csharp/autobuilder/Semmle.Autobuild/BuildCommandAutoRule.cs
View File

@@ -25,7 +25,7 @@ namespace Semmle.Autobuild
"build"
};
public BuildScript Analyse(Autobuilder builder)
public BuildScript Analyse(Autobuilder builder, bool auto)
{
builder.Log(Severity.Info, "Attempting to locate build script");

2
csharp/autobuilder/Semmle.Autobuild/BuildCommandRule.cs
View File

@@ -5,7 +5,7 @@
/// </summary>
class BuildCommandRule : IBuildRule
{
public BuildScript Analyse(Autobuilder builder)
public BuildScript Analyse(Autobuilder builder, bool auto)
{
if (builder.Options.BuildCommand == null)
return BuildScript.Failure;

23
csharp/autobuilder/Semmle.Autobuild/DotNetRule.cs
View File

@@ -14,22 +14,25 @@ namespace Semmle.Autobuild
/// </summary>
class DotNetRule : IBuildRule
{
public BuildScript Analyse(Autobuilder builder)
public BuildScript Analyse(Autobuilder builder, bool auto)
{
if (!builder.ProjectsOrSolutionsToBuild.Any())
return BuildScript.Failure;
var notDotNetProject = builder.ProjectsOrSolutionsToBuild.
SelectMany(p => Enumerators.Singleton(p).Concat(p.IncludedProjects)).
OfType<Project>().
FirstOrDefault(p => !p.DotNetProject);
if (notDotNetProject != null)
if (auto)
{
builder.Log(Severity.Info, "Not using .NET Core because of incompatible project {0}", notDotNetProject);
return BuildScript.Failure;
}
var notDotNetProject = builder.ProjectsOrSolutionsToBuild.
SelectMany(p => Enumerators.Singleton(p).Concat(p.IncludedProjects)).
OfType<Project>().
FirstOrDefault(p => !p.DotNetProject);
if (notDotNetProject != null)
{
builder.Log(Severity.Info, "Not using .NET Core because of incompatible project {0}", notDotNetProject);
return BuildScript.Failure;
}
builder.Log(Severity.Info, "Attempting to build using .NET Core");
builder.Log(Severity.Info, "Attempting to build using .NET Core");
}
return WithDotNet(builder, dotNet =>
{

5
csharp/autobuilder/Semmle.Autobuild/MsBuildRule.cs
View File

@@ -13,12 +13,13 @@ namespace Semmle.Autobuild
/// </summary>
const string MsBuild = "msbuild";
public BuildScript Analyse(Autobuilder builder)
public BuildScript Analyse(Autobuilder builder, bool auto)
{
if (!builder.ProjectsOrSolutionsToBuild.Any())
return BuildScript.Failure;
builder.Log(Severity.Info, "Attempting to build using MSBuild");
if (auto)
builder.Log(Severity.Info, "Attempting to build using MSBuild");
var vsTools = GetVcVarsBatFile(builder);

6
csharp/autobuilder/Semmle.Autobuild/Project.cs
View File

@@ -39,9 +39,9 @@ namespace Semmle.Autobuild
{
projFile = builder.Actions.LoadXml(FullPath);
}
catch (XmlException)
catch (Exception e) when (e is XmlException || e is FileNotFoundException)
{
builder.Log(Severity.Info, $"Skipping project file {path} as it is not a valid XML document.");
builder.Log(Severity.Info, $"Unable to read project file {path}.");
return;
}
@@ -80,7 +80,7 @@ namespace Semmle.Autobuild
var projectFilesIncludes = root.SelectNodes("//msbuild:Project/msbuild:ItemGroup/msbuild:ProjectFiles/@Include", mgr).OfType<XmlNode>();
foreach (var include in projectFileIncludes.Concat(projectFilesIncludes))
{
var includePath = builder.Actions.IsWindows() ? include.Value : include.Value.Replace("\\", "/");
var includePath = builder.Actions.PathCombine(include.Value.Split('\\', StringSplitOptions.RemoveEmptyEntries));
ret.Add(new Project(builder, builder.Actions.PathCombine(Path.GetDirectoryName(this.FullPath), includePath)));
}
return ret;

30
csharp/autobuilder/Semmle.Autobuild/Solution.cs
View File

@@ -4,6 +4,8 @@ using System;
using System.Collections.Generic;
using System.Linq;
using Semmle.Util;
using System.IO;
using Semmle.Util.Logging;
namespace Semmle.Autobuild
{
@@ -55,25 +57,33 @@ namespace Semmle.Autobuild
public string DefaultPlatformName =>
solution == null ? "" : solution.GetDefaultPlatformName();
public Solution(Autobuilder builder, string path) : base(builder, path)
public Solution(Autobuilder builder, string path, bool allowProject) : base(builder, path)
{
try
{
solution = SolutionFile.Parse(FullPath);
includedProjects =
solution.ProjectsInOrder.
Where(p => p.ProjectType == SolutionProjectType.KnownToBeMSBuildFormat).
Select(p => builder.Actions.GetFullPath(FileUtils.ConvertToNative(p.AbsolutePath))).
Select(p => new Project(builder, p)).
ToArray();
}
catch (InvalidProjectFileException)
catch (Exception e) when (e is InvalidProjectFileException || e is FileNotFoundException)
{
// We allow specifying projects as solutions in lgtm.yml, so model
// that scenario as a solution with just that one project
includedProjects = new[] { new Project(builder, path) };
if (allowProject)
{
includedProjects = new[] { new Project(builder, path) };
return;
}
builder.Log(Severity.Info, $"Unable to read solution file {path}.");
includedProjects = new Project[0];
return;
}
includedProjects =
solution.ProjectsInOrder.
Where(p => p.ProjectType == SolutionProjectType.KnownToBeMSBuildFormat).
Select(p => builder.Actions.PathCombine(Path.GetDirectoryName(path), builder.Actions.PathCombine(p.RelativePath.Split('\\', StringSplitOptions.RemoveEmptyEntries)))).
Select(p => new Project(builder, p)).
ToArray();
}
IEnumerable<Version> ToolsVersions => includedProjects.Where(p => p.ValidToolsVersion).Select(p => p.ToolsVersion);

2
csharp/autobuilder/Semmle.Autobuild/StandaloneBuildRule.cs
View File

@@ -7,7 +7,7 @@ namespace Semmle.Autobuild
/// </summary>
class StandaloneBuildRule : IBuildRule
{
public BuildScript Analyse(Autobuilder builder)
public BuildScript Analyse(Autobuilder builder, bool auto)
{
BuildScript GetCommand(string solution)
{

2
csharp/autobuilder/Semmle.Autobuild/XmlBuildRule.cs
View File

@@ -7,7 +7,7 @@ namespace Semmle.Autobuild
/// </summary>
class XmlBuildRule : IBuildRule
{
public BuildScript Analyse(Autobuilder builder)
public BuildScript Analyse(Autobuilder builder, bool auto)
{
var command = new CommandBuilder(builder.Actions).
RunCommand(builder.Odasa).