hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-05-02 04:05:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into python-keyword-only-args

Browse Source
This commit is contained in:
Rasmus Wriedt Larsen
2020-07-02 14:48:32 +02:00
parent b2f8638ff0 eecc3ca5dd
commit 513c2974bd
46 changed files with 1081 additions and 359 deletions
Download Patch File Download Diff File Expand all files Collapse all files

296
cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/BuildScripts.cs Normal file
View File

@@ -0,0 +1,296 @@
using Xunit;
using Semmle.Autobuild.Shared;
using System.Collections.Generic;
using System;
using System.Linq;
using Microsoft.Build.Construction;
using System.Xml;
namespace Semmle.Autobuild.Cpp.Tests
{
/// <summary>
/// Test class to script Autobuilder scenarios.
/// For most methods, it uses two fields:
/// - an IList to capture the the arguments passed to it
/// - an IDictionary of possible return values.
/// </summary>
class TestActions : IBuildActions
{
/// <summary>
/// List of strings passed to FileDelete.
/// </summary>
public IList<string> FileDeleteIn = new List<string>();
void IBuildActions.FileDelete(string file)
{
FileDeleteIn.Add(file);
}
public IList<string> FileExistsIn = new List<string>();
public IDictionary<string, bool> FileExists = new Dictionary<string, bool>();
bool IBuildActions.FileExists(string file)
{
FileExistsIn.Add(file);
if (FileExists.TryGetValue(file, out var ret))
return ret;
if (FileExists.TryGetValue(System.IO.Path.GetFileName(file), out ret))
return ret;
throw new ArgumentException("Missing FileExists " + file);
}
public IList<string> RunProcessIn = new List<string>();
public IDictionary<string, int> RunProcess = new Dictionary<string, int>();
public IDictionary<string, string> RunProcessOut = new Dictionary<string, string>();
public IDictionary<string, string> RunProcessWorkingDirectory = new Dictionary<string, string>();
int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory, IDictionary<string, string>? env, out IList<string> stdOut)
{
var pattern = cmd + " " + args;
RunProcessIn.Add(pattern);
if (RunProcessOut.TryGetValue(pattern, out var str))
stdOut = str.Split("\n");
else
throw new ArgumentException("Missing RunProcessOut " + pattern);
RunProcessWorkingDirectory.TryGetValue(pattern, out var wd);
if (wd != workingDirectory)
throw new ArgumentException("Missing RunProcessWorkingDirectory " + pattern);
if (RunProcess.TryGetValue(pattern, out var ret))
return ret;
throw new ArgumentException("Missing RunProcess " + pattern);
}
int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory, IDictionary<string, string>? env)
{
var pattern = cmd + " " + args;
RunProcessIn.Add(pattern);
RunProcessWorkingDirectory.TryGetValue(pattern, out var wd);
if (wd != workingDirectory)
throw new ArgumentException("Missing RunProcessWorkingDirectory " + pattern);
if (RunProcess.TryGetValue(pattern, out var ret))
return ret;
throw new ArgumentException("Missing RunProcess " + pattern);
}
public IList<string> DirectoryDeleteIn = new List<string>();
void IBuildActions.DirectoryDelete(string dir, bool recursive)
{
DirectoryDeleteIn.Add(dir);
}
public IDictionary<string, bool> DirectoryExists = new Dictionary<string, bool>();
public IList<string> DirectoryExistsIn = new List<string>();
bool IBuildActions.DirectoryExists(string dir)
{
DirectoryExistsIn.Add(dir);
if (DirectoryExists.TryGetValue(dir, out var ret))
return ret;
throw new ArgumentException("Missing DirectoryExists " + dir);
}
public IDictionary<string, string?> GetEnvironmentVariable = new Dictionary<string, string?>();
string? IBuildActions.GetEnvironmentVariable(string name)
{
if (GetEnvironmentVariable.TryGetValue(name, out var ret))
return ret;
throw new ArgumentException("Missing GetEnvironmentVariable " + name);
}
public string GetCurrentDirectory = "";
string IBuildActions.GetCurrentDirectory()
{
return GetCurrentDirectory;
}
public IDictionary<string, string> EnumerateFiles = new Dictionary<string, string>();
IEnumerable<string> IBuildActions.EnumerateFiles(string dir)
{
if (EnumerateFiles.TryGetValue(dir, out var str))
return str.Split("\n");
throw new ArgumentException("Missing EnumerateFiles " + dir);
}
public IDictionary<string, string> EnumerateDirectories = new Dictionary<string, string>();
IEnumerable<string> IBuildActions.EnumerateDirectories(string dir)
{
if (EnumerateDirectories.TryGetValue(dir, out var str))
return string.IsNullOrEmpty(str) ? Enumerable.Empty<string>() : str.Split("\n");
throw new ArgumentException("Missing EnumerateDirectories " + dir);
}
public bool IsWindows;
bool IBuildActions.IsWindows() => IsWindows;
string IBuildActions.PathCombine(params string[] parts)
{
return string.Join(IsWindows ? '\\' : '/', parts.Where(p => !string.IsNullOrWhiteSpace(p)));
}
string IBuildActions.GetFullPath(string path) => path;
void IBuildActions.WriteAllText(string filename, string contents)
{
}
public IDictionary<string, XmlDocument> LoadXml = new Dictionary<string, XmlDocument>();
XmlDocument IBuildActions.LoadXml(string filename)
{
if (LoadXml.TryGetValue(filename, out var xml))
return xml;
throw new ArgumentException("Missing LoadXml " + filename);
}
public string EnvironmentExpandEnvironmentVariables(string s)
{
foreach (var kvp in GetEnvironmentVariable)
s = s.Replace($"%{kvp.Key}%", kvp.Value);
return s;
}
}
/// <summary>
/// A fake solution to build.
/// </summary>
class TestSolution : ISolution
{
public IEnumerable<SolutionConfigurationInSolution> Configurations => throw new NotImplementedException();
public string DefaultConfigurationName => "Release";
public string DefaultPlatformName => "x86";
public string FullPath { get; set; }
public Version ToolsVersion => new Version("14.0");
public IEnumerable<IProjectOrSolution> IncludedProjects => throw new NotImplementedException();
public TestSolution(string path)
{
FullPath = path;
}
}
public class BuildScriptTests
{
TestActions Actions = new TestActions();
// Records the arguments passed to StartCallback.
IList<string> StartCallbackIn = new List<string>();
void StartCallback(string s, bool silent)
{
StartCallbackIn.Add(s);
}
// Records the arguments passed to EndCallback
IList<string> EndCallbackIn = new List<string>();
IList<int> EndCallbackReturn = new List<int>();
void EndCallback(int ret, string s, bool silent)
{
EndCallbackReturn.Add(ret);
EndCallbackIn.Add(s);
}
CppAutobuilder CreateAutoBuilder(bool isWindows,
string? buildless = null, string? solution = null, string? buildCommand = null, string? ignoreErrors = null,
string? msBuildArguments = null, string? msBuildPlatform = null, string? msBuildConfiguration = null, string? msBuildTarget = null,
string? dotnetArguments = null, string? dotnetVersion = null, string? vsToolsVersion = null,
string? nugetRestore = null, string? allSolutions = null,
string cwd = @"C:\Project")
{
string codeqlUpperLanguage = Language.Cpp.UpperCaseName;
Actions.GetEnvironmentVariable[$"CODEQL_AUTOBUILDER_{codeqlUpperLanguage}_NO_INDEXING"] = "false";
Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_TRAP_DIR"] = "";
Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_SOURCE_ARCHIVE_DIR"] = "";
Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_ROOT"] = $@"C:\codeql\{codeqlUpperLanguage.ToLowerInvariant()}";
Actions.GetEnvironmentVariable["CODEQL_JAVA_HOME"] = @"C:\codeql\tools\java";
Actions.GetEnvironmentVariable["SEMMLE_DIST"] = @"C:\odasa";
Actions.GetEnvironmentVariable["SEMMLE_JAVA_HOME"] = @"C:\odasa\tools\java";
Actions.GetEnvironmentVariable["SEMMLE_PLATFORM_TOOLS"] = @"C:\odasa\tools";
Actions.GetEnvironmentVariable["LGTM_INDEX_VSTOOLS_VERSION"] = vsToolsVersion;
Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_ARGUMENTS"] = msBuildArguments;
Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_PLATFORM"] = msBuildPlatform;
Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_CONFIGURATION"] = msBuildConfiguration;
Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_TARGET"] = msBuildTarget;
Actions.GetEnvironmentVariable["LGTM_INDEX_DOTNET_ARGUMENTS"] = dotnetArguments;
Actions.GetEnvironmentVariable["LGTM_INDEX_DOTNET_VERSION"] = dotnetVersion;
Actions.GetEnvironmentVariable["LGTM_INDEX_BUILD_COMMAND"] = buildCommand;
Actions.GetEnvironmentVariable["LGTM_INDEX_SOLUTION"] = solution;
Actions.GetEnvironmentVariable["LGTM_INDEX_IGNORE_ERRORS"] = ignoreErrors;
Actions.GetEnvironmentVariable["LGTM_INDEX_BUILDLESS"] = buildless;
Actions.GetEnvironmentVariable["LGTM_INDEX_ALL_SOLUTIONS"] = allSolutions;
Actions.GetEnvironmentVariable["LGTM_INDEX_NUGET_RESTORE"] = nugetRestore;
Actions.GetEnvironmentVariable["ProgramFiles(x86)"] = isWindows ? @"C:\Program Files (x86)" : null;
Actions.GetCurrentDirectory = cwd;
Actions.IsWindows = isWindows;
var options = new AutobuildOptions(Actions, Language.Cpp);
return new CppAutobuilder(Actions, options);
}
void TestAutobuilderScript(Autobuilder autobuilder, int expectedOutput, int commandsRun)
{
Assert.Equal(expectedOutput, autobuilder.GetBuildScript().Run(Actions, StartCallback, EndCallback));
// Check expected commands actually ran
Assert.Equal(commandsRun, StartCallbackIn.Count);
Assert.Equal(commandsRun, EndCallbackIn.Count);
Assert.Equal(commandsRun, EndCallbackReturn.Count);
var action = Actions.RunProcess.GetEnumerator();
for (int cmd = 0; cmd < commandsRun; ++cmd)
{
Assert.True(action.MoveNext());
Assert.Equal(action.Current.Key, StartCallbackIn[cmd]);
Assert.Equal(action.Current.Value, EndCallbackReturn[cmd]);
}
}
[Fact]
public void TestDefaultCppAutobuilder()
{
Actions.EnumerateFiles[@"C:\Project"] = "";
Actions.EnumerateDirectories[@"C:\Project"] = "";
var autobuilder = CreateAutoBuilder(true);
var script = autobuilder.GetBuildScript();
// Fails due to no solutions present.
Assert.NotEqual(0, script.Run(Actions, StartCallback, EndCallback));
}
[Fact]
public void TestCppAutobuilderSuccess()
{
Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\csharp\nuget\nuget.exe restore C:\Project\test.sln"] = 1;
Actions.RunProcess[@"cmd.exe /C CALL ^""C:\Program Files ^(x86^)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat^"" && set Platform=&& type NUL && C:\odasa\tools\odasa index --auto msbuild C:\Project\test.sln /p:UseSharedCompilation=false /t:rebuild /p:Platform=""x86"" /p:Configuration=""Release"" /p:MvcBuildViews=true"] = 0;
Actions.RunProcessOut[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = "";
Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = 1;
Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationVersion"] = 0;
Actions.RunProcessOut[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationVersion"] = "";
Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = true;
Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat"] = true;
Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat"] = true;
Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat"] = true;
Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = true;
Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.slx";
Actions.EnumerateDirectories[@"C:\Project"] = "";
var autobuilder = CreateAutoBuilder(true);
var solution = new TestSolution(@"C:\Project\test.sln");
autobuilder.ProjectsOrSolutionsToBuild.Add(solution);
TestAutobuilderScript(autobuilder, 0, 2);
}
}
}

25
cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/Semmle.Autobuild.Cpp.Tests.csproj Normal file
View File

@@ -0,0 +1,25 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp3.0</TargetFramework>
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
<RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="System.IO.FileSystem" Version="4.3.0" />
<PackageReference Include="System.IO.FileSystem.Primitives" Version="4.3.0" />
<PackageReference Include="xunit" Version="2.4.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.1">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
</PackageReference>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Semmle.Autobuild.Cpp\Semmle.Autobuild.Cpp.csproj" />
<ProjectReference Include="..\..\..\csharp\autobuilder\Semmle.Autobuild.Shared\Semmle.Autobuild.Shared.csproj" />
</ItemGroup>
</Project>

23
cpp/autobuilder/Semmle.Autobuild.Cpp/CppAutobuilder.cs Normal file
View File

@@ -0,0 +1,23 @@
using Semmle.Autobuild.Shared;
namespace Semmle.Autobuild.Cpp
{
public class CppAutobuilder : Autobuilder
{
public CppAutobuilder(IBuildActions actions, AutobuildOptions options) : base(actions, options) { }
public override BuildScript GetBuildScript()
{
if (Options.BuildCommand != null)
return new BuildCommandRule((_, f) => f(null)).Analyse(this, false);
return
// First try MSBuild
new MsBuildRule().Analyse(this, true) |
// Then look for a script that might be a build script
(() => new BuildCommandAutoRule((_, f) => f(null)).Analyse(this, true)) |
// All attempts failed: print message
AutobuildFailure();
}
}
}

9
csharp/autobuilder/Semmle.Autobuild/Program.cs → cpp/autobuilder/Semmle.Autobuild.Cpp/Program.cs
View File

@@ -1,6 +1,7 @@
using System;
using Semmle.Autobuild.Shared;
namespace Semmle.Autobuild
namespace Semmle.Autobuild.Cpp
{
class Program
{
@@ -10,11 +11,11 @@ namespace Semmle.Autobuild
try
{
var actions = SystemBuildActions.Instance;
var options = new AutobuildOptions(actions);
var options = new AutobuildOptions(actions, Language.Cpp);
try
{
Console.WriteLine($"Semmle autobuilder for {options.Language}");
var builder = new Autobuilder(actions, options);
Console.WriteLine("CodeQL C++ autobuilder");
var builder = new CppAutobuilder(actions, options);
return builder.AttemptBuild();
}
catch(InvalidEnvironmentException ex)

8
csharp/autobuilder/Semmle.Autobuild.Tests/Properties/AssemblyInfo.cs → cpp/autobuilder/Semmle.Autobuild.Cpp/Properties/AssemblyInfo.cs
View File

@@ -4,12 +4,12 @@ using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("Semmle.Autobuild.Tests")]
[assembly: AssemblyTitle("Semmle.Autobuild.Cpp")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("")]
[assembly: AssemblyProduct("Semmle.Extraction.Tests")]
[assembly: AssemblyCopyright("Copyright © 2018")]
[assembly: AssemblyCompany("GitHub")]
[assembly: AssemblyProduct("CodeQL autobuilder for C++")]
[assembly: AssemblyCopyright("Copyright © GitHub 2020")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]

28
cpp/autobuilder/Semmle.Autobuild.Cpp/Semmle.Autobuild.Cpp.csproj Normal file
View File

@@ -0,0 +1,28 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netcoreapp3.0</TargetFramework>
<AssemblyName>Semmle.Autobuild.Cpp</AssemblyName>
<RootNamespace>Semmle.Autobuild.Cpp</RootNamespace>
<ApplicationIcon />
<OutputType>Exe</OutputType>
<StartupObject />
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
<RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<Folder Include="Properties\" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Build" Version="16.0.461" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\..\csharp\extractor\Semmle.Util\Semmle.Util.csproj" />
<ProjectReference Include="..\..\..\csharp\autobuilder\Semmle.Autobuild.Shared\Semmle.Autobuild.Shared.csproj" />
</ItemGroup>
</Project>

28
csharp/CSharp.sln
View File

@@ -11,8 +11,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CSharp",
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CIL", "extractor\Semmle.Extraction.CIL\Semmle.Extraction.CIL.csproj", "{399A1579-68F0-40F4-9A23-F241BA697F9C}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Autobuild", "autobuilder\Semmle.Autobuild\Semmle.Autobuild.csproj", "{5131EF00-0BA9-4436-A3B0-C5CDAB4B194C}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CSharp.Standalone", "extractor\Semmle.Extraction.CSharp.Standalone\Semmle.Extraction.CSharp.Standalone.csproj", "{D00E7D25-0FA0-48EC-B048-CD60CE1B30D8}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.CIL.Driver", "extractor\Semmle.Extraction.CIL.Driver\Semmle.Extraction.CIL.Driver.csproj", "{EFA400B3-C1CE-446F-A4E2-8B44E61EF47C}"
@@ -23,7 +21,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Extraction.Tests", "
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Util.Tests", "extractor\Semmle.Util.Tests\Semmle.Util.Tests.csproj", "{55A620F0-23F6-440D-A5BA-0567613B3C0F}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Semmle.Autobuild.Tests", "autobuilder\Semmle.Autobuild.Tests\Semmle.Autobuild.Tests.csproj", "{CE267461-D762-4F53-A275-685A0A4EC48D}"
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Semmle.Autobuild.Shared", "autobuilder\Semmle.Autobuild.Shared\Semmle.Autobuild.Shared.csproj", "{133F2B5B-FD25-4BD9-B34C-062CC6BB4178}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Semmle.Autobuild.CSharp", "autobuilder\Semmle.Autobuild.CSharp\Semmle.Autobuild.CSharp.csproj", "{F3C07863-3759-4A0B-B777-8A0E0FDB1A41}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Semmle.Autobuild.CSharp.Tests", "autobuilder\Semmle.Autobuild.CSharp.Tests\Semmle.Autobuild.CSharp.Tests.csproj", "{34256E8F-866A-46C1-800E-3DF69FD1DCB7}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -47,10 +49,6 @@ Global
{399A1579-68F0-40F4-9A23-F241BA697F9C}.Debug|Any CPU.Build.0 = Debug|Any CPU
{399A1579-68F0-40F4-9A23-F241BA697F9C}.Release|Any CPU.ActiveCfg = Release|Any CPU
{399A1579-68F0-40F4-9A23-F241BA697F9C}.Release|Any CPU.Build.0 = Release|Any CPU
{5131EF00-0BA9-4436-A3B0-C5CDAB4B194C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{5131EF00-0BA9-4436-A3B0-C5CDAB4B194C}.Debug|Any CPU.Build.0 = Debug|Any CPU
{5131EF00-0BA9-4436-A3B0-C5CDAB4B194C}.Release|Any CPU.ActiveCfg = Release|Any CPU
{5131EF00-0BA9-4436-A3B0-C5CDAB4B194C}.Release|Any CPU.Build.0 = Release|Any CPU
{D00E7D25-0FA0-48EC-B048-CD60CE1B30D8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{D00E7D25-0FA0-48EC-B048-CD60CE1B30D8}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D00E7D25-0FA0-48EC-B048-CD60CE1B30D8}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -69,10 +67,18 @@ Global
{55A620F0-23F6-440D-A5BA-0567613B3C0F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{55A620F0-23F6-440D-A5BA-0567613B3C0F}.Debug|Any CPU.Build.0 = Debug|Any CPU
{55A620F0-23F6-440D-A5BA-0567613B3C0F}.Release|Any CPU.ActiveCfg = Release|Any CPU
{CE267461-D762-4F53-A275-685A0A4EC48D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{CE267461-D762-4F53-A275-685A0A4EC48D}.Debug|Any CPU.Build.0 = Debug|Any CPU
{CE267461-D762-4F53-A275-685A0A4EC48D}.Release|Any CPU.ActiveCfg = Release|Any CPU
{CE267461-D762-4F53-A275-685A0A4EC48D}.Release|Any CPU.Build.0 = Release|Any CPU
{133F2B5B-FD25-4BD9-B34C-062CC6BB4178}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{133F2B5B-FD25-4BD9-B34C-062CC6BB4178}.Debug|Any CPU.Build.0 = Debug|Any CPU
{133F2B5B-FD25-4BD9-B34C-062CC6BB4178}.Release|Any CPU.ActiveCfg = Release|Any CPU
{133F2B5B-FD25-4BD9-B34C-062CC6BB4178}.Release|Any CPU.Build.0 = Release|Any CPU
{F3C07863-3759-4A0B-B777-8A0E0FDB1A41}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{F3C07863-3759-4A0B-B777-8A0E0FDB1A41}.Debug|Any CPU.Build.0 = Debug|Any CPU
{F3C07863-3759-4A0B-B777-8A0E0FDB1A41}.Release|Any CPU.ActiveCfg = Release|Any CPU
{F3C07863-3759-4A0B-B777-8A0E0FDB1A41}.Release|Any CPU.Build.0 = Release|Any CPU
{34256E8F-866A-46C1-800E-3DF69FD1DCB7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{34256E8F-866A-46C1-800E-3DF69FD1DCB7}.Debug|Any CPU.Build.0 = Debug|Any CPU
{34256E8F-866A-46C1-800E-3DF69FD1DCB7}.Release|Any CPU.ActiveCfg = Release|Any CPU
{34256E8F-866A-46C1-800E-3DF69FD1DCB7}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE

104
csharp/autobuilder/Semmle.Autobuild.Tests/BuildScripts.cs → csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs
View File

@@ -1,12 +1,12 @@
using Xunit;
using Semmle.Autobuild;
using Semmle.Autobuild.Shared;
using System.Collections.Generic;
using System;
using System.Linq;
using Microsoft.Build.Construction;
using System.Xml;
namespace Semmle.Extraction.Tests
namespace Semmle.Autobuild.CSharp.Tests
{
/// <summary>
/// Test class to script Autobuilder scenarios.
@@ -80,11 +80,9 @@ namespace Semmle.Extraction.Tests
}
public IDictionary<string, bool> DirectoryExists = new Dictionary<string, bool>();
public IList<string> DirectoryExistsIn = new List<string>();
bool IBuildActions.DirectoryExists(string dir)
{
DirectoryExistsIn.Add(dir);
if (DirectoryExists.TryGetValue(dir, out var ret))
return ret;
throw new ArgumentException("Missing DirectoryExists " + dir);
@@ -333,22 +331,21 @@ namespace Semmle.Extraction.Tests
Assert.Equal(0, BuildScript.Try(BuildScript.Failure).Run(Actions, StartCallback, EndCallback));
}
Autobuilder CreateAutoBuilder(string lgtmLanguage, bool isWindows,
CSharpAutobuilder CreateAutoBuilder(bool isWindows,
string? buildless = null, string? solution = null, string? buildCommand = null, string? ignoreErrors = null,
string? msBuildArguments = null, string? msBuildPlatform = null, string? msBuildConfiguration = null, string? msBuildTarget = null,
string? dotnetArguments = null, string? dotnetVersion = null, string? vsToolsVersion = null,
string? nugetRestore = null, string? allSolutions = null,
string cwd = @"C:\Project")
{
string codeqlUpperLanguage = lgtmLanguage.ToUpper();
string codeqlUpperLanguage = Language.CSharp.UpperCaseName;
Actions.GetEnvironmentVariable[$"CODEQL_AUTOBUILDER_{codeqlUpperLanguage}_NO_INDEXING"] = "false";
Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_TRAP_DIR"] = "";
Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_SOURCE_ARCHIVE_DIR"] = "";
Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_ROOT"] = $@"C:\codeql\{lgtmLanguage}";
Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_ROOT"] = $@"C:\codeql\{codeqlUpperLanguage.ToLowerInvariant()}";
Actions.GetEnvironmentVariable["CODEQL_JAVA_HOME"] = @"C:\codeql\tools\java";
Actions.GetEnvironmentVariable["SEMMLE_DIST"] = @"C:\odasa";
Actions.GetEnvironmentVariable["SEMMLE_JAVA_HOME"] = @"C:\odasa\tools\java";
Actions.GetEnvironmentVariable["LGTM_PROJECT_LANGUAGE"] = lgtmLanguage;
Actions.GetEnvironmentVariable["SEMMLE_PLATFORM_TOOLS"] = @"C:\odasa\tools";
Actions.GetEnvironmentVariable["LGTM_INDEX_VSTOOLS_VERSION"] = vsToolsVersion;
Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_ARGUMENTS"] = msBuildArguments;
@@ -367,8 +364,8 @@ namespace Semmle.Extraction.Tests
Actions.GetCurrentDirectory = cwd;
Actions.IsWindows = isWindows;
var options = new AutobuildOptions(Actions);
return new Autobuilder(Actions, options);
var options = new AutobuildOptions(Actions, Language.CSharp);
return new CSharpAutobuilder(Actions, options);
}
[Fact]
@@ -396,7 +393,7 @@ namespace Semmle.Extraction.Tests
</Project>");
Actions.LoadXml["test.csproj"] = xml;
var autobuilder = CreateAutoBuilder("csharp", true);
var autobuilder = CreateAutoBuilder(true);
TestAutobuilderScript(autobuilder, 0, 6);
}
@@ -428,7 +425,7 @@ Microsoft.NETCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
</Project>");
Actions.LoadXml["test.csproj"] = xml;
var autobuilder = CreateAutoBuilder("csharp", false);
var autobuilder = CreateAutoBuilder(false);
TestAutobuilderScript(autobuilder, 0, 7);
}
@@ -441,47 +438,10 @@ Microsoft.NETCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.cs";
Actions.EnumerateDirectories[@"C:\Project"] = "";
var autobuilder = CreateAutoBuilder("csharp", false);
var autobuilder = CreateAutoBuilder(false);
TestAutobuilderScript(autobuilder, 1, 0);
}
[Fact]
public void TestDefaultCppAutobuilder()
{
Actions.EnumerateFiles[@"C:\Project"] = "";
Actions.EnumerateDirectories[@"C:\Project"] = "";
var autobuilder = CreateAutoBuilder("cpp", true);
var script = autobuilder.GetBuildScript();
// Fails due to no solutions present.
Assert.NotEqual(0, script.Run(Actions, StartCallback, EndCallback));
}
[Fact]
public void TestCppAutobuilderSuccess()
{
Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\csharp\nuget\nuget.exe restore C:\Project\test.sln"] = 1;
Actions.RunProcess[@"cmd.exe /C CALL ^""C:\Program Files ^(x86^)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat^"" && set Platform=&& type NUL && C:\odasa\tools\odasa index --auto msbuild C:\Project\test.sln /p:UseSharedCompilation=false /t:rebuild /p:Platform=""x86"" /p:Configuration=""Release"" /p:MvcBuildViews=true"] = 0;
Actions.RunProcessOut[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = "";
Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = 1;
Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationVersion"] = 0;
Actions.RunProcessOut[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationVersion"] = "";
Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = true;
Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat"] = true;
Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat"] = true;
Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat"] = true;
Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = true;
Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.slx";
Actions.EnumerateDirectories[@"C:\Project"] = "";
var autobuilder = CreateAutoBuilder("cpp", true);
var solution = new TestSolution(@"C:\Project\test.sln");
autobuilder.ProjectsOrSolutionsToBuild.Add(solution);
TestAutobuilderScript(autobuilder, 0, 2);
}
[Fact]
public void TestVsWhereSucceeded()
{
@@ -538,7 +498,7 @@ Microsoft.NETCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
Actions.EnumerateDirectories[@"C:\Project"] = "";
var autobuilder = CreateAutoBuilder("csharp", false, buildless: "true");
var autobuilder = CreateAutoBuilder(false, buildless: "true");
TestAutobuilderScript(autobuilder, 0, 3);
}
@@ -552,7 +512,7 @@ Microsoft.NETCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
Actions.EnumerateDirectories[@"C:\Project"] = "";
var autobuilder = CreateAutoBuilder("csharp", false, buildless: "true");
var autobuilder = CreateAutoBuilder(false, buildless: "true");
TestAutobuilderScript(autobuilder, 10, 1);
}
@@ -568,7 +528,7 @@ Microsoft.NETCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
Actions.EnumerateDirectories[@"C:\Project"] = "";
var autobuilder = CreateAutoBuilder("csharp", false, buildless: "true", solution: "foo.sln");
var autobuilder = CreateAutoBuilder(false, buildless: "true", solution: "foo.sln");
TestAutobuilderScript(autobuilder, 0, 3);
}
@@ -616,7 +576,7 @@ Microsoft.NETCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
SkipVsWhere();
var autobuilder = CreateAutoBuilder("csharp", false, buildCommand: "./build.sh --skip-tests");
var autobuilder = CreateAutoBuilder(false, buildCommand: "./build.sh --skip-tests");
TestAutobuilderScript(autobuilder, 0, 4);
}
@@ -636,7 +596,7 @@ Microsoft.NETCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
Actions.RunProcess[@"C:\odasa/tools/odasa index --xml --extensions config csproj props xml"] = 0;
Actions.FileExists["csharp.log"] = true;
var autobuilder = CreateAutoBuilder("csharp", false);
var autobuilder = CreateAutoBuilder(false);
TestAutobuilderScript(autobuilder, 0, 5);
}
@@ -655,7 +615,7 @@ Microsoft.NETCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
Actions.RunProcessWorkingDirectory[@"C:\odasa/tools/odasa index --auto build.sh"] = "";
Actions.FileExists["csharp.log"] = false;
var autobuilder = CreateAutoBuilder("csharp", false);
var autobuilder = CreateAutoBuilder(false);
TestAutobuilderScript(autobuilder, 1, 3);
}
@@ -674,7 +634,7 @@ Microsoft.NETCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
Actions.RunProcessWorkingDirectory[@"C:\odasa/tools/odasa index --auto build.sh"] = "";
Actions.FileExists["csharp.log"] = true;
var autobuilder = CreateAutoBuilder("csharp", false);
var autobuilder = CreateAutoBuilder(false);
TestAutobuilderScript(autobuilder, 1, 3);
}
@@ -691,7 +651,7 @@ Microsoft.NETCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --xml --extensions config csproj props xml"] = 0;
Actions.FileExists["csharp.log"] = true;
var autobuilder = CreateAutoBuilder("csharp", true);
var autobuilder = CreateAutoBuilder(true);
TestAutobuilderScript(autobuilder, 0, 3);
}
@@ -708,7 +668,7 @@ Microsoft.NETCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\odasa index --xml --extensions config"] = 0;
Actions.FileExists["csharp.log"] = true;
var autobuilder = CreateAutoBuilder("csharp", true, ignoreErrors: "true");
var autobuilder = CreateAutoBuilder(true, ignoreErrors: "true");
TestAutobuilderScript(autobuilder, 1, 1);
}
@@ -726,7 +686,7 @@ Microsoft.NETCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
Actions.EnumerateDirectories[@"C:\Project"] = "";
var autobuilder = CreateAutoBuilder("csharp", true, buildCommand: "build.cmd --skip-tests", ignoreErrors: "true");
var autobuilder = CreateAutoBuilder(true, buildCommand: "build.cmd --skip-tests", ignoreErrors: "true");
TestAutobuilderScript(autobuilder, 3, 1);
}
@@ -751,7 +711,7 @@ Microsoft.NETCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest1.cs\ntest2.cs";
Actions.EnumerateDirectories[@"C:\Project"] = "";
var autobuilder = CreateAutoBuilder("csharp", true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows", msBuildPlatform: "x86", msBuildConfiguration: "Debug",
var autobuilder = CreateAutoBuilder(true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows", msBuildPlatform: "x86", msBuildConfiguration: "Debug",
vsToolsVersion: "12", allSolutions: "true");
var testSolution1 = new TestSolution(@"C:\Project\test1.sln");
var testSolution2 = new TestSolution(@"C:\Project\test2.sln");
@@ -802,7 +762,7 @@ Microsoft.NETCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
</Project>");
Actions.LoadXml["test2.csproj"] = csproj2;
var autobuilder = CreateAutoBuilder("csharp", true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows", msBuildPlatform: "x86", msBuildConfiguration: "Debug",
var autobuilder = CreateAutoBuilder(true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows", msBuildPlatform: "x86", msBuildConfiguration: "Debug",
vsToolsVersion: "12");
TestAutobuilderScript(autobuilder, 0, 6);
@@ -824,7 +784,7 @@ Microsoft.NETCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest1.cs\ntest2.cs";
Actions.EnumerateDirectories[@"C:\Project"] = "";
var autobuilder = CreateAutoBuilder("csharp", true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows", msBuildPlatform: "x86", msBuildConfiguration: "Debug",
var autobuilder = CreateAutoBuilder(true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows", msBuildPlatform: "x86", msBuildConfiguration: "Debug",
vsToolsVersion: "12", allSolutions: "true");
var testSolution1 = new TestSolution(@"C:\Project\test1.sln");
var testSolution2 = new TestSolution(@"C:\Project\test2.sln");
@@ -853,7 +813,7 @@ Microsoft.NETCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest1.cs\ntest2.cs";
Actions.EnumerateDirectories[@"C:\Project"] = "";
var autobuilder = CreateAutoBuilder("csharp", true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows",
var autobuilder = CreateAutoBuilder(true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows",
msBuildPlatform: "x86", msBuildConfiguration: "Debug", vsToolsVersion: "12",
allSolutions: "true", nugetRestore: "false");
var testSolution1 = new TestSolution(@"C:\Project\test1.sln");
@@ -876,7 +836,7 @@ Microsoft.NETCore.App 2.2.5 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
Actions.EnumerateDirectories[@"C:\Project"] = "";
var autobuilder = CreateAutoBuilder("csharp", false, buildless: "true", solution: "foo.sln", nugetRestore: "false");
var autobuilder = CreateAutoBuilder(false, buildless: "true", solution: "foo.sln", nugetRestore: "false");
TestAutobuilderScript(autobuilder, 0, 3);
}
@@ -909,7 +869,7 @@ Microsoft.NETCore.App 2.1.3 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
</Project>");
Actions.LoadXml["test.csproj"] = xml;
var autobuilder = CreateAutoBuilder("csharp", false, dotnetArguments: "--no-restore"); // nugetRestore=false does not work for now.
var autobuilder = CreateAutoBuilder(false, dotnetArguments: "--no-restore"); // nugetRestore=false does not work for now.
TestAutobuilderScript(autobuilder, 0, 7);
}
@@ -948,7 +908,7 @@ Microsoft.NETCore.App 3.0.0 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
</Project>");
Actions.LoadXml["test.csproj"] = xml;
var autobuilder = CreateAutoBuilder("csharp", false, dotnetVersion: "2.1.3");
var autobuilder = CreateAutoBuilder(false, dotnetVersion: "2.1.3");
TestAutobuilderScript(autobuilder, 0, 12);
}
@@ -990,7 +950,7 @@ Microsoft.NETCore.App 2.1.4 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
</Project>");
Actions.LoadXml["test.csproj"] = xml;
var autobuilder = CreateAutoBuilder("csharp", false, dotnetVersion: "2.1.3");
var autobuilder = CreateAutoBuilder(false, dotnetVersion: "2.1.3");
TestAutobuilderScript(autobuilder, 0, 12);
}
@@ -1024,7 +984,7 @@ Microsoft.NETCore.App 2.1.4 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
</Project>");
Actions.LoadXml["test.csproj"] = xml;
var autobuilder = CreateAutoBuilder("csharp", true, dotnetVersion: "2.1.3");
var autobuilder = CreateAutoBuilder(true, dotnetVersion: "2.1.3");
TestAutobuilderScript(autobuilder, 0, 9);
}
@@ -1066,7 +1026,7 @@ Microsoft.NETCore.App 2.1.4 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
</Project>");
Actions.LoadXml["dirs.proj"] = dirsproj;
var autobuilder = CreateAutoBuilder("csharp", true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows", msBuildPlatform: "x86", msBuildConfiguration: "Debug",
var autobuilder = CreateAutoBuilder(true, msBuildArguments: "/P:Fu=Bar", msBuildTarget: "Windows", msBuildPlatform: "x86", msBuildConfiguration: "Debug",
vsToolsVersion: "12", allSolutions: "true");
TestAutobuilderScript(autobuilder, 0, 4);
}
@@ -1103,7 +1063,7 @@ Microsoft.NETCore.App 2.1.4 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
</Project>");
Actions.LoadXml["dirs.proj"] = dirsproj;
var autobuilder = CreateAutoBuilder("csharp", false);
var autobuilder = CreateAutoBuilder(false);
TestAutobuilderScript(autobuilder, 0, 4);
}
@@ -1125,7 +1085,7 @@ Microsoft.NETCore.App 2.1.4 [/usr/local/share/dotnet/shared/Microsoft.NETCore.Ap
</Project>");
Actions.LoadXml["dirs.proj"] = dirsproj1;
var autobuilder = CreateAutoBuilder("csharp", false);
var autobuilder = CreateAutoBuilder(false);
TestAutobuilderScript(autobuilder, 1, 0);
}

4
csharp/autobuilder/Semmle.Autobuild.Tests/Semmle.Autobuild.Tests.csproj → csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/Semmle.Autobuild.CSharp.Tests.csproj
View File

@@ -19,7 +19,7 @@
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Semmle.Autobuild\Semmle.Autobuild.csproj" />
<ProjectReference Include="..\Semmle.Autobuild.CSharp\Semmle.Autobuild.CSharp.csproj" />
<ProjectReference Include="..\Semmle.Autobuild.Shared\Semmle.Autobuild.Shared.csproj" />
</ItemGroup>
</Project>

4
csharp/autobuilder/Semmle.Autobuild/AspBuildRule.cs → csharp/autobuilder/Semmle.Autobuild.CSharp/AspBuildRule.cs
View File

@@ -1,4 +1,6 @@
namespace Semmle.Autobuild
using Semmle.Autobuild.Shared;
namespace Semmle.Autobuild.CSharp
{
/// <summary>
/// ASP extraction.

127
csharp/autobuilder/Semmle.Autobuild.CSharp/CSharpAutobuilder.cs Normal file
View File

@@ -0,0 +1,127 @@
using Semmle.Extraction.CSharp;
using Semmle.Util.Logging;
using Semmle.Autobuild.Shared;
namespace Semmle.Autobuild.CSharp
{
public class CSharpAutobuilder : Autobuilder
{
public CSharpAutobuilder(IBuildActions actions, AutobuildOptions options) : base(actions, options) { }
public override BuildScript GetBuildScript()
{
/// <summary>
/// A script that checks that the C# extractor has been executed.
/// </summary>
BuildScript CheckExtractorRun(bool warnOnFailure) =>
BuildScript.Create(actions =>
{
if (actions.FileExists(Extractor.GetCSharpLogPath()))
return 0;
if (warnOnFailure)
Log(Severity.Error, "No C# code detected during build.");
return 1;
});
var attempt = BuildScript.Failure;
switch (GetCSharpBuildStrategy())
{
case CSharpBuildStrategy.CustomBuildCommand:
attempt = new BuildCommandRule(DotNetRule.WithDotNet).Analyse(this, false) & CheckExtractorRun(true);
break;
case CSharpBuildStrategy.Buildless:
// No need to check that the extractor has been executed in buildless mode
attempt = new StandaloneBuildRule().Analyse(this, false);
break;
case CSharpBuildStrategy.MSBuild:
attempt = new MsBuildRule().Analyse(this, false) & CheckExtractorRun(true);
break;
case CSharpBuildStrategy.DotNet:
attempt = new DotNetRule().Analyse(this, false) & CheckExtractorRun(true);
break;
case CSharpBuildStrategy.Auto:
var cleanTrapFolder =
BuildScript.DeleteDirectory(TrapDir);
var cleanSourceArchive =
BuildScript.DeleteDirectory(SourceArchiveDir);
var tryCleanExtractorArgsLogs =
BuildScript.Create(actions =>
{
foreach (var file in Extractor.GetCSharpArgsLogs())
try
{
actions.FileDelete(file);
}
catch // lgtm[cs/catch-of-all-exceptions] lgtm[cs/empty-catch-block]
{ }
return 0;
});
var attemptExtractorCleanup =
BuildScript.Try(cleanTrapFolder) &
BuildScript.Try(cleanSourceArchive) &
tryCleanExtractorArgsLogs &
BuildScript.DeleteFile(Extractor.GetCSharpLogPath());
/// <summary>
/// Execute script `s` and check that the C# extractor has been executed.
/// If either fails, attempt to cleanup any artifacts produced by the extractor,
/// and exit with code 1, in order to proceed to the next attempt.
/// </summary>
BuildScript IntermediateAttempt(BuildScript s) =>
(s & CheckExtractorRun(false)) |
(attemptExtractorCleanup & BuildScript.Failure);
attempt =
// First try .NET Core
IntermediateAttempt(new DotNetRule().Analyse(this, true)) |
// Then MSBuild
(() => IntermediateAttempt(new MsBuildRule().Analyse(this, true))) |
// And finally look for a script that might be a build script
(() => new BuildCommandAutoRule(DotNetRule.WithDotNet).Analyse(this, true) & CheckExtractorRun(true)) |
// All attempts failed: print message
AutobuildFailure();
break;
}
return
attempt &
(() => new AspBuildRule().Analyse(this, false)) &
(() => new XmlBuildRule().Analyse(this, false));
}
/// <summary>
/// Gets the build strategy that the autobuilder should apply, based on the
/// options in the `lgtm.yml` file.
/// </summary>
CSharpBuildStrategy GetCSharpBuildStrategy()
{
if (Options.BuildCommand != null)
return CSharpBuildStrategy.CustomBuildCommand;
if (Options.Buildless)
return CSharpBuildStrategy.Buildless;
if (Options.MsBuildArguments != null
|| Options.MsBuildConfiguration != null
|| Options.MsBuildPlatform != null
|| Options.MsBuildTarget != null)
return CSharpBuildStrategy.MSBuild;
if (Options.DotNetArguments != null || Options.DotNetVersion != null)
return CSharpBuildStrategy.DotNet;
return CSharpBuildStrategy.Auto;
}
enum CSharpBuildStrategy
{
CustomBuildCommand,
Buildless,
MSBuild,
DotNet,
Auto
}
}
}

3
csharp/autobuilder/Semmle.Autobuild/DotNetRule.cs → csharp/autobuilder/Semmle.Autobuild.CSharp/DotNetRule.cs
View File

@@ -6,8 +6,9 @@ using System.Collections.Generic;
using System.IO;
using Semmle.Util;
using System.Text.RegularExpressions;
using Semmle.Autobuild.Shared;
namespace Semmle.Autobuild
namespace Semmle.Autobuild.CSharp
{
/// <summary>
/// A build rule where the build command is of the form "dotnet build".

33
csharp/autobuilder/Semmle.Autobuild.CSharp/Program.cs Normal file
View File

@@ -0,0 +1,33 @@
using System;
using Semmle.Autobuild.Shared;
namespace Semmle.Autobuild.CSharp
{
class Program
{
static int Main()
{
try
{
var actions = SystemBuildActions.Instance;
var options = new AutobuildOptions(actions, Language.CSharp);
try
{
Console.WriteLine("CodeQL C# autobuilder");
var builder = new CSharpAutobuilder(actions, options);
return builder.AttemptBuild();
}
catch (InvalidEnvironmentException ex)
{
Console.WriteLine("The environment is invalid: {0}", ex.Message);
}
}
catch (ArgumentOutOfRangeException ex)
{
Console.WriteLine("The value \"{0}\" for parameter \"{1}\" is invalid", ex.ActualValue, ex.ParamName);
}
return 1;
}
}
}

32
csharp/autobuilder/Semmle.Autobuild.CSharp/Properties/AssemblyInfo.cs Normal file
View File

@@ -0,0 +1,32 @@
using System.Reflection;
using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("Semmle.Autobuild.CSharp")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("GitHub")]
[assembly: AssemblyProduct("CodeQL autobuilder for C#")]
[assembly: AssemblyCopyright("Copyright © GitHub 2020")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]
// Setting ComVisible to false makes the types in this assembly not visible
// to COM components. If you need to access a type in this assembly from
// COM, set the ComVisible attribute to true on that type.
[assembly: ComVisible(false)]
// Version information for an assembly consists of the following four values:
//
// Major Version
// Minor Version
// Build Number
// Revision
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.0.0")]
[assembly: AssemblyFileVersion("1.0.0.0")]

5
csharp/autobuilder/Semmle.Autobuild/Semmle.Autobuild.csproj → csharp/autobuilder/Semmle.Autobuild.CSharp/Semmle.Autobuild.CSharp.csproj
View File

@@ -2,8 +2,8 @@
<PropertyGroup>
<TargetFramework>netcoreapp3.0</TargetFramework>
<AssemblyName>Semmle.Autobuild</AssemblyName>
<RootNamespace>Semmle.Autobuild</RootNamespace>
<AssemblyName>Semmle.Autobuild.CSharp</AssemblyName>
<RootNamespace>Semmle.Autobuild.CSharp</RootNamespace>
<ApplicationIcon />
<OutputType>Exe</OutputType>
<StartupObject />
@@ -24,6 +24,7 @@
<ItemGroup>
<ProjectReference Include="..\..\extractor\Semmle.Util\Semmle.Util.csproj" />
<ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp\Semmle.Extraction.CSharp.csproj" />
<ProjectReference Include="..\Semmle.Autobuild.Shared\Semmle.Autobuild.Shared.csproj" />
</ItemGroup>
</Project>

4
csharp/autobuilder/Semmle.Autobuild/StandaloneBuildRule.cs → csharp/autobuilder/Semmle.Autobuild.CSharp/StandaloneBuildRule.cs
View File

@@ -1,4 +1,6 @@
namespace Semmle.Autobuild
using Semmle.Autobuild.Shared;
namespace Semmle.Autobuild.CSharp
{
/// <summary>
/// Build using standalone extraction.

4
csharp/autobuilder/Semmle.Autobuild/XmlBuildRule.cs → csharp/autobuilder/Semmle.Autobuild.CSharp/XmlBuildRule.cs
View File

@@ -1,4 +1,6 @@
namespace Semmle.Autobuild
using Semmle.Autobuild.Shared;
namespace Semmle.Autobuild.CSharp
{
/// <summary>
/// XML extraction.

21
csharp/autobuilder/Semmle.Autobuild/AutobuildOptions.cs → csharp/autobuilder/Semmle.Autobuild.Shared/AutobuildOptions.cs
View File

@@ -2,7 +2,7 @@
using System.Linq;
using System.Text.RegularExpressions;
namespace Semmle.Autobuild
namespace Semmle.Autobuild.Shared
{
/// <summary>
/// Encapsulates build options.
@@ -35,7 +35,7 @@ namespace Semmle.Autobuild
/// Reads options from environment variables.
/// Throws ArgumentOutOfRangeException for invalid arguments.
/// </summary>
public AutobuildOptions(IBuildActions actions)
public AutobuildOptions(IBuildActions actions, Language language)
{
RootDirectory = actions.GetCurrentDirectory();
VsToolsVersion = actions.GetEnvironmentVariable(prefix + "VSTOOLS_VERSION");
@@ -53,7 +53,7 @@ namespace Semmle.Autobuild
AllSolutions = actions.GetEnvironmentVariable(prefix + "ALL_SOLUTIONS").AsBool("all_solutions", false);
NugetRestore = actions.GetEnvironmentVariable(prefix + "NUGET_RESTORE").AsBool("nuget_restore", true);
Language = actions.GetEnvironmentVariable("LGTM_PROJECT_LANGUAGE").AsLanguage();
Language = language;
Indexing = !actions.GetEnvironmentVariable($"CODEQL_AUTOBUILDER_{Language.UpperCaseName}_NO_INDEXING").AsBool("no_indexing", false);
}
@@ -81,21 +81,6 @@ namespace Semmle.Autobuild
}
}
public static Language AsLanguage(this string? key)
{
switch (key)
{
case null:
throw new ArgumentException("Environment variable required: LGTM_PROJECT_LANGUAGE");
case "csharp":
return Language.CSharp;
case "cpp":
return Language.Cpp;
default:
throw new ArgumentException("Language key not understood: '" + key + "'");
}
}
public static string[] AsListWithExpandedEnvVars(this string? value, IBuildActions actions, string[] defaultValue)
{
if (value == null)

164
csharp/autobuilder/Semmle.Autobuild/Autobuilder.cs → csharp/autobuilder/Semmle.Autobuild.Shared/Autobuilder.cs
View File

@@ -1,16 +1,15 @@
using Semmle.Extraction.CSharp;
using Semmle.Util.Logging;
using Semmle.Util.Logging;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
namespace Semmle.Autobuild
namespace Semmle.Autobuild.Shared
{
/// <summary>
/// A build rule analyses the files in "builder" and outputs a build script.
/// </summary>
interface IBuildRule
public interface IBuildRule
{
/// <summary>
/// Analyse the files and produce a build script.
@@ -23,7 +22,7 @@ namespace Semmle.Autobuild
/// <summary>
/// Exception indicating that environment variables are missing or invalid.
/// </summary>
class InvalidEnvironmentException : Exception
public class InvalidEnvironmentException : Exception
{
public InvalidEnvironmentException(string m) : base(m) { }
}
@@ -35,7 +34,7 @@ namespace Semmle.Autobuild
/// The overall design is intended to be extensible so that in theory,
/// it should be possible to add new build rules without touching this code.
/// </summary>
public class Autobuilder
public abstract class Autobuilder
{
/// <summary>
/// Full file paths of files found in the project directory, as well as
@@ -126,10 +125,10 @@ namespace Semmle.Autobuild
ToArray();
if (matchingFiles.Length == 0)
return null;
return null;
if (Options.AllSolutions)
return matchingFiles.Select(p => p.ProjectOrSolution);
return matchingFiles.Select(p => p.ProjectOrSolution);
return matchingFiles.
Where(f => f.DistanceFromRoot == matchingFiles[0].DistanceFromRoot).
@@ -188,9 +187,9 @@ namespace Semmle.Autobuild
SemmleDist = Actions.GetEnvironmentVariable("SEMMLE_DIST");
SemmlePlatformTools = Actions.GetEnvironmentVariable("SEMMLE_PLATFORM_TOOLS");
JavaHome =
JavaHome =
Actions.GetEnvironmentVariable("CODEQL_JAVA_HOME") ??
Actions.GetEnvironmentVariable("SEMMLE_JAVA_HOME") ??
Actions.GetEnvironmentVariable("SEMMLE_JAVA_HOME") ??
throw new InvalidEnvironmentException("The environment variable CODEQL_JAVA_HOME or SEMMLE_JAVA_HOME has not been set.");
Distribution =
@@ -209,9 +208,9 @@ namespace Semmle.Autobuild
throw new InvalidEnvironmentException($"The environment variable CODEQL_EXTRACTOR_{this.Options.Language.UpperCaseName}_SOURCE_ARCHIVE_DIR or SOURCE_ARCHIVE has not been set.");
}
private string TrapDir { get; }
protected string TrapDir { get; }
private string SourceArchiveDir { get; }
protected string SourceArchiveDir { get; }
readonly ILogger logger = new ConsoleLogger(Verbosity.Info);
@@ -234,6 +233,7 @@ namespace Semmle.Autobuild
Log(Severity.Info, $"Working directory: {Options.RootDirectory}");
var script = GetBuildScript();
if (Options.IgnoreErrors)
script |= BuildScript.Success;
@@ -253,143 +253,9 @@ namespace Semmle.Autobuild
/// <summary>
/// Returns the build script to use for this project.
/// </summary>
public BuildScript GetBuildScript()
{
var isCSharp = Options.Language == Language.CSharp;
return isCSharp ? GetCSharpBuildScript() : GetCppBuildScript();
}
public abstract BuildScript GetBuildScript();
BuildScript GetCSharpBuildScript()
{
/// <summary>
/// A script that checks that the C# extractor has been executed.
/// </summary>
BuildScript CheckExtractorRun(bool warnOnFailure) =>
BuildScript.Create(actions =>
{
if (actions.FileExists(Extractor.GetCSharpLogPath()))
return 0;
if (warnOnFailure)
Log(Severity.Error, "No C# code detected during build.");
return 1;
});
var attempt = BuildScript.Failure;
switch (GetCSharpBuildStrategy())
{
case CSharpBuildStrategy.CustomBuildCommand:
attempt = new BuildCommandRule().Analyse(this, false) & CheckExtractorRun(true);
break;
case CSharpBuildStrategy.Buildless:
// No need to check that the extractor has been executed in buildless mode
attempt = new StandaloneBuildRule().Analyse(this, false);
break;
case CSharpBuildStrategy.MSBuild:
attempt = new MsBuildRule().Analyse(this, false) & CheckExtractorRun(true);
break;
case CSharpBuildStrategy.DotNet:
attempt = new DotNetRule().Analyse(this, false) & CheckExtractorRun(true);
break;
case CSharpBuildStrategy.Auto:
var cleanTrapFolder =
BuildScript.DeleteDirectory(TrapDir);
var cleanSourceArchive =
BuildScript.DeleteDirectory(SourceArchiveDir);
var tryCleanExtractorArgsLogs =
BuildScript.Create(actions =>
{
foreach (var file in Extractor.GetCSharpArgsLogs())
try
{
actions.FileDelete(file);
}
catch // lgtm[cs/catch-of-all-exceptions] lgtm[cs/empty-catch-block]
{ }
return 0;
});
var attemptExtractorCleanup =
BuildScript.Try(cleanTrapFolder) &
BuildScript.Try(cleanSourceArchive) &
tryCleanExtractorArgsLogs &
BuildScript.DeleteFile(Extractor.GetCSharpLogPath());
/// <summary>
/// Execute script `s` and check that the C# extractor has been executed.
/// If either fails, attempt to cleanup any artifacts produced by the extractor,
/// and exit with code 1, in order to proceed to the next attempt.
/// </summary>
BuildScript IntermediateAttempt(BuildScript s) =>
(s & CheckExtractorRun(false)) |
(attemptExtractorCleanup & BuildScript.Failure);
attempt =
// First try .NET Core
IntermediateAttempt(new DotNetRule().Analyse(this, true)) |
// Then MSBuild
(() => IntermediateAttempt(new MsBuildRule().Analyse(this, true))) |
// And finally look for a script that might be a build script
(() => new BuildCommandAutoRule().Analyse(this, true) & CheckExtractorRun(true)) |
// All attempts failed: print message
AutobuildFailure();
break;
}
return
attempt &
(() => new AspBuildRule().Analyse(this, false)) &
(() => new XmlBuildRule().Analyse(this, false));
}
/// <summary>
/// Gets the build strategy that the autobuilder should apply, based on the
/// options in the `lgtm.yml` file.
/// </summary>
CSharpBuildStrategy GetCSharpBuildStrategy()
{
if (Options.BuildCommand != null)
return CSharpBuildStrategy.CustomBuildCommand;
if (Options.Buildless)
return CSharpBuildStrategy.Buildless;
if (Options.MsBuildArguments != null
|| Options.MsBuildConfiguration != null
|| Options.MsBuildPlatform != null
|| Options.MsBuildTarget != null)
return CSharpBuildStrategy.MSBuild;
if (Options.DotNetArguments != null || Options.DotNetVersion != null)
return CSharpBuildStrategy.DotNet;
return CSharpBuildStrategy.Auto;
}
enum CSharpBuildStrategy
{
CustomBuildCommand,
Buildless,
MSBuild,
DotNet,
Auto
}
BuildScript GetCppBuildScript()
{
if (Options.BuildCommand != null)
return new BuildCommandRule().Analyse(this, false);
return
// First try MSBuild
new MsBuildRule().Analyse(this, true) |
// Then look for a script that might be a build script
(() => new BuildCommandAutoRule().Analyse(this, true)) |
// All attempts failed: print message
AutobuildFailure();
}
BuildScript AutobuildFailure() =>
protected BuildScript AutobuildFailure() =>
BuildScript.Create(actions =>
{
Log(Severity.Error, "Could not auto-detect a suitable build method");
@@ -426,6 +292,6 @@ namespace Semmle.Autobuild
/// an <code>odasa --index</code>, unless indexing has been disabled, in which case
/// <paramref name="cmd"/> is run directly.
/// </summary>
internal CommandBuilder MaybeIndex(CommandBuilder builder, string cmd) => Options.Indexing && !(Odasa is null) ? builder.IndexCommand(Odasa, cmd) : builder.RunCommand(cmd);
public CommandBuilder MaybeIndex(CommandBuilder builder, string cmd) => Options.Indexing && !(Odasa is null) ? builder.IndexCommand(Odasa, cmd) : builder.RunCommand(cmd);
}
}

4
csharp/autobuilder/Semmle.Autobuild/BuildActions.cs → csharp/autobuilder/Semmle.Autobuild.Shared/BuildActions.cs
View File

@@ -5,7 +5,7 @@ using System.Diagnostics;
using System.IO;
using System.Xml;
namespace Semmle.Autobuild
namespace Semmle.Autobuild.Shared
{
/// <summary>
/// Wrapper around system calls so that the build scripts can be unit-tested.
@@ -124,7 +124,7 @@ namespace Semmle.Autobuild
/// <summary>
/// An implementation of IBuildActions that actually performs the requested operations.
/// </summary>
class SystemBuildActions : IBuildActions
public class SystemBuildActions : IBuildActions
{
void IBuildActions.FileDelete(string file) => File.Delete(file);

16
csharp/autobuilder/Semmle.Autobuild/BuildCommandAutoRule.cs → csharp/autobuilder/Semmle.Autobuild.Shared/BuildCommandAutoRule.cs
View File

@@ -1,15 +1,23 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Semmle.Util.Logging;
namespace Semmle.Autobuild
namespace Semmle.Autobuild.Shared
{
/// <summary>
/// Auto-detection of build scripts.
/// </summary>
class BuildCommandAutoRule : IBuildRule
public class BuildCommandAutoRule : IBuildRule
{
private readonly Func<Autobuilder, Func<IDictionary<string, string>?, BuildScript>, BuildScript> withDotNet;
public BuildCommandAutoRule(Func<Autobuilder, Func<IDictionary<string, string>?, BuildScript>, BuildScript> withDotNet)
{
this.withDotNet = withDotNet;
}
readonly IEnumerable<string> winExtensions = new List<string> {
".bat",
".cmd",
@@ -43,7 +51,7 @@ namespace Semmle.Autobuild
string? dir = Path.GetDirectoryName(scriptPath);
// A specific .NET Core version may be required
return chmodScript & DotNetRule.WithDotNet(builder, environment =>
return chmodScript & withDotNet(builder, environment =>
{
var command = new CommandBuilder(builder.Actions, dir, environment);

16
csharp/autobuilder/Semmle.Autobuild/BuildCommandRule.cs → csharp/autobuilder/Semmle.Autobuild.Shared/BuildCommandRule.cs
View File

@@ -1,17 +1,27 @@
namespace Semmle.Autobuild
using System;
using System.Collections.Generic;
namespace Semmle.Autobuild.Shared
{
/// <summary>
/// Execute the build_command rule.
/// </summary>
class BuildCommandRule : IBuildRule
public class BuildCommandRule : IBuildRule
{
private readonly Func<Autobuilder, Func<IDictionary<string, string>?, BuildScript>, BuildScript> withDotNet;
public BuildCommandRule(Func<Autobuilder, Func<IDictionary<string, string>?, BuildScript>, BuildScript> withDotNet)
{
this.withDotNet = withDotNet;
}
public BuildScript Analyse(Autobuilder builder, bool auto)
{
if (builder.Options.BuildCommand == null)
return BuildScript.Failure;
// Custom build commands may require a specific .NET Core version
return DotNetRule.WithDotNet(builder, environment =>
return withDotNet(builder, environment =>
{
var command = new CommandBuilder(builder.Actions, null, environment);

2
csharp/autobuilder/Semmle.Autobuild/BuildScript.cs → csharp/autobuilder/Semmle.Autobuild.Shared/BuildScript.cs
View File

@@ -2,7 +2,7 @@
using System.Collections.Generic;
using System.IO;
namespace Semmle.Autobuild
namespace Semmle.Autobuild.Shared
{
/// <summary>
/// A build script.

2
csharp/autobuilder/Semmle.Autobuild/BuildTools.cs → csharp/autobuilder/Semmle.Autobuild.Shared/BuildTools.cs
View File

@@ -1,7 +1,7 @@
using System.Collections.Generic;
using System.Linq;
namespace Semmle.Autobuild
namespace Semmle.Autobuild.Shared
{
/// <summary>
/// A BAT file used to initialise the appropriate

4
csharp/autobuilder/Semmle.Autobuild/CommandBuilder.cs → csharp/autobuilder/Semmle.Autobuild.Shared/CommandBuilder.cs
View File

@@ -2,12 +2,12 @@ using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace Semmle.Autobuild
namespace Semmle.Autobuild.Shared
{
/// <summary>
/// Utility to construct a build command.
/// </summary>
class CommandBuilder
public class CommandBuilder
{
enum EscapeMode { Process, Cmd };

2
csharp/autobuilder/Semmle.Autobuild/Language.cs → csharp/autobuilder/Semmle.Autobuild.Shared/Language.cs
View File

@@ -1,4 +1,4 @@
namespace Semmle.Autobuild
namespace Semmle.Autobuild.Shared
{
public sealed class Language
{

4
csharp/autobuilder/Semmle.Autobuild/MsBuildRule.cs → csharp/autobuilder/Semmle.Autobuild.Shared/MsBuildRule.cs
View File

@@ -1,12 +1,12 @@
using Semmle.Util.Logging;
using System.Linq;
namespace Semmle.Autobuild
namespace Semmle.Autobuild.Shared
{
/// <summary>
/// A build rule using msbuild.
/// </summary>
class MsBuildRule : IBuildRule
public class MsBuildRule : IBuildRule
{
/// <summary>
/// The name of the msbuild command.

2
csharp/autobuilder/Semmle.Autobuild/Project.cs → csharp/autobuilder/Semmle.Autobuild.Shared/Project.cs
View File

@@ -5,7 +5,7 @@ using System.Linq;
using System.Xml;
using Semmle.Util.Logging;
namespace Semmle.Autobuild
namespace Semmle.Autobuild.Shared
{
/// <summary>
/// Representation of a .proj file, a .csproj file (C#), or a .vcxproj file (C++).

2
csharp/autobuilder/Semmle.Autobuild/ProjectOrSolution.cs → csharp/autobuilder/Semmle.Autobuild.Shared/ProjectOrSolution.cs
View File

@@ -2,7 +2,7 @@
using System.IO;
using System.Linq;
namespace Semmle.Autobuild
namespace Semmle.Autobuild.Shared
{
/// <summary>
/// A file that can be the target in an invocation of `msbuild` or `dotnet build`.

8
csharp/autobuilder/Semmle.Autobuild/Properties/AssemblyInfo.cs → csharp/autobuilder/Semmle.Autobuild.Shared/Properties/AssemblyInfo.cs
View File

@@ -4,12 +4,12 @@ using System.Runtime.InteropServices;
// General Information about an assembly is controlled through the following
// set of attributes. Change these attribute values to modify the information
// associated with an assembly.
[assembly: AssemblyTitle("Semmle.Autobuild")]
[assembly: AssemblyTitle("Semmle.Autobuild.Shared")]
[assembly: AssemblyDescription("")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("Semmle")]
[assembly: AssemblyProduct("Semmle Visual Studio Autobuild")]
[assembly: AssemblyCopyright("Copyright © Semmle 2017")]
[assembly: AssemblyCompany("GitHub")]
[assembly: AssemblyProduct("CodeQL autobuilder")]
[assembly: AssemblyCopyright("Copyright © GitHub 2020")]
[assembly: AssemblyTrademark("")]
[assembly: AssemblyCulture("")]

24
csharp/autobuilder/Semmle.Autobuild.Shared/Semmle.Autobuild.Shared.csproj Normal file
View File

@@ -0,0 +1,24 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netcoreapp3.0</TargetFramework>
<AssemblyName>Semmle.Autobuild.Shared</AssemblyName>
<RootNamespace>Semmle.Autobuild.Shared</RootNamespace>
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
<RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<Folder Include="Properties\" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Build" Version="16.0.461" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\extractor\Semmle.Util\Semmle.Util.csproj" />
</ItemGroup>
</Project>

3
csharp/autobuilder/Semmle.Autobuild/Solution.cs → csharp/autobuilder/Semmle.Autobuild.Shared/Solution.cs
View File

@@ -3,11 +3,10 @@ using Microsoft.Build.Exceptions;
using System;
using System.Collections.Generic;
using System.Linq;
using Semmle.Util;
using System.IO;
using Semmle.Util.Logging;
namespace Semmle.Autobuild
namespace Semmle.Autobuild.Shared
{
/// <summary>
/// A solution file, extension .sln.

33
javascript/ql/src/semmle/javascript/security/dataflow/DomBasedXss.qll
View File

@@ -28,6 +28,17 @@ module DomBasedXss {
guard instanceof SanitizerGuard
}
override predicate isAdditionalStoreStep(
DataFlow::Node pred, DataFlow::SourceNode succ, string prop
) {
exists(DataFlow::PropRead read |
pred = read.getBase() and
succ = read and
read.getPropertyName() = "hash" and
prop = urlSuffixPseudoProperty()
)
}
override predicate isAdditionalLoadStoreStep(
DataFlow::Node pred, DataFlow::Node succ, string predProp, string succProp
) {
@@ -41,15 +52,29 @@ module DomBasedXss {
}
override predicate isAdditionalLoadStep(DataFlow::Node pred, DataFlow::Node succ, string prop) {
exists(DataFlow::MethodCallNode call, string name |
name = "substr" or name = "substring" or name = "slice"
|
call.getMethodName() = name and
exists(DataFlow::MethodCallNode call |
call.getMethodName() = ["substr", "substring", "slice"] and
not call.getArgument(0).getIntValue() = 0 and
pred = call.getReceiver() and
succ = call and
prop = urlSuffixPseudoProperty()
)
or
exists(DataFlow::MethodCallNode call |
call.getMethodName() = "exec" and pred = call.getArgument(0)
or
call.getMethodName() = "match" and pred = call.getReceiver()
|
succ = call and
prop = urlSuffixPseudoProperty()
)
or
exists(StringSplitCall split |
split.getSeparator() = ["#", "?"] and
pred = split.getBaseString() and
succ = split.getASubstringRead(1) and
prop = urlSuffixPseudoProperty()
)
}
override predicate isSanitizerEdge(DataFlow::Node pred, DataFlow::Node succ) {

35
javascript/ql/test/query-tests/Security/CWE-079/Xss.expected
View File

@@ -453,6 +453,23 @@ nodes
| tst.js:414:19:414:31 | target.taint8 |
| tst.js:415:18:415:30 | target.taint8 |
| tst.js:415:18:415:30 | target.taint8 |
| tst.js:422:7:422:46 | payload |
| tst.js:422:17:422:31 | window.location |
| tst.js:422:17:422:31 | window.location |
| tst.js:422:17:422:46 | window. ... bstr(1) |
| tst.js:423:18:423:24 | payload |
| tst.js:423:18:423:24 | payload |
| tst.js:425:7:425:55 | match |
| tst.js:425:15:425:29 | window.location |
| tst.js:425:15:425:29 | window.location |
| tst.js:425:15:425:55 | window. ... (\\w+)/) |
| tst.js:427:20:427:24 | match |
| tst.js:427:20:427:27 | match[1] |
| tst.js:427:20:427:27 | match[1] |
| tst.js:430:18:430:32 | window.location |
| tst.js:430:18:430:32 | window.location |
| tst.js:430:18:430:51 | window. ... '#')[1] |
| tst.js:430:18:430:51 | window. ... '#')[1] |
| typeahead.js:20:13:20:45 | target |
| typeahead.js:20:22:20:38 | document.location |
| typeahead.js:20:22:20:38 | document.location |
@@ -882,6 +899,21 @@ edges
| tst.js:414:19:414:31 | target.taint8 | tst.js:414:19:414:31 | target.taint8 |
| tst.js:414:19:414:31 | target.taint8 | tst.js:415:18:415:30 | target.taint8 |
| tst.js:414:19:414:31 | target.taint8 | tst.js:415:18:415:30 | target.taint8 |
| tst.js:422:7:422:46 | payload | tst.js:423:18:423:24 | payload |
| tst.js:422:7:422:46 | payload | tst.js:423:18:423:24 | payload |
| tst.js:422:17:422:31 | window.location | tst.js:422:17:422:46 | window. ... bstr(1) |
| tst.js:422:17:422:31 | window.location | tst.js:422:17:422:46 | window. ... bstr(1) |
| tst.js:422:17:422:46 | window. ... bstr(1) | tst.js:422:7:422:46 | payload |
| tst.js:425:7:425:55 | match | tst.js:427:20:427:24 | match |
| tst.js:425:15:425:29 | window.location | tst.js:425:15:425:55 | window. ... (\\w+)/) |
| tst.js:425:15:425:29 | window.location | tst.js:425:15:425:55 | window. ... (\\w+)/) |
| tst.js:425:15:425:55 | window. ... (\\w+)/) | tst.js:425:7:425:55 | match |
| tst.js:427:20:427:24 | match | tst.js:427:20:427:27 | match[1] |
| tst.js:427:20:427:24 | match | tst.js:427:20:427:27 | match[1] |
| tst.js:430:18:430:32 | window.location | tst.js:430:18:430:51 | window. ... '#')[1] |
| tst.js:430:18:430:32 | window.location | tst.js:430:18:430:51 | window. ... '#')[1] |
| tst.js:430:18:430:32 | window.location | tst.js:430:18:430:51 | window. ... '#')[1] |
| tst.js:430:18:430:32 | window.location | tst.js:430:18:430:51 | window. ... '#')[1] |
| typeahead.js:20:13:20:45 | target | typeahead.js:21:12:21:17 | target |
| typeahead.js:20:22:20:38 | document.location | typeahead.js:20:22:20:45 | documen ... .search |
| typeahead.js:20:22:20:38 | document.location | typeahead.js:20:22:20:45 | documen ... .search |
@@ -1009,6 +1041,9 @@ edges
| tst.js:403:18:403:30 | target.taint5 | tst.js:387:16:387:32 | document.location | tst.js:403:18:403:30 | target.taint5 | Cross-site scripting vulnerability due to $@. | tst.js:387:16:387:32 | document.location | user-provided value |
| tst.js:412:18:412:30 | target.taint7 | tst.js:387:16:387:32 | document.location | tst.js:412:18:412:30 | target.taint7 | Cross-site scripting vulnerability due to $@. | tst.js:387:16:387:32 | document.location | user-provided value |
| tst.js:415:18:415:30 | target.taint8 | tst.js:387:16:387:32 | document.location | tst.js:415:18:415:30 | target.taint8 | Cross-site scripting vulnerability due to $@. | tst.js:387:16:387:32 | document.location | user-provided value |
| tst.js:423:18:423:24 | payload | tst.js:422:17:422:31 | window.location | tst.js:423:18:423:24 | payload | Cross-site scripting vulnerability due to $@. | tst.js:422:17:422:31 | window.location | user-provided value |
| tst.js:427:20:427:27 | match[1] | tst.js:425:15:425:29 | window.location | tst.js:427:20:427:27 | match[1] | Cross-site scripting vulnerability due to $@. | tst.js:425:15:425:29 | window.location | user-provided value |
| tst.js:430:18:430:51 | window. ... '#')[1] | tst.js:430:18:430:32 | window.location | tst.js:430:18:430:51 | window. ... '#')[1] | Cross-site scripting vulnerability due to $@. | tst.js:430:18:430:32 | window.location | user-provided value |
| typeahead.js:25:18:25:20 | val | typeahead.js:20:22:20:38 | document.location | typeahead.js:25:18:25:20 | val | Cross-site scripting vulnerability due to $@. | typeahead.js:20:22:20:38 | document.location | user-provided value |
| v-html.vue:2:8:2:23 | v-html=tainted | v-html.vue:6:42:6:58 | document.location | v-html.vue:2:8:2:23 | v-html=tainted | Cross-site scripting vulnerability due to $@. | v-html.vue:6:42:6:58 | document.location | user-provided value |
| winjs.js:3:43:3:49 | tainted | winjs.js:2:17:2:33 | document.location | winjs.js:3:43:3:49 | tainted | Cross-site scripting vulnerability due to $@. | winjs.js:2:17:2:33 | document.location | user-provided value |

32
javascript/ql/test/query-tests/Security/CWE-079/XssWithAdditionalSources.expected
View File

@@ -453,6 +453,23 @@ nodes
| tst.js:414:19:414:31 | target.taint8 |
| tst.js:415:18:415:30 | target.taint8 |
| tst.js:415:18:415:30 | target.taint8 |
| tst.js:422:7:422:46 | payload |
| tst.js:422:17:422:31 | window.location |
| tst.js:422:17:422:31 | window.location |
| tst.js:422:17:422:46 | window. ... bstr(1) |
| tst.js:423:18:423:24 | payload |
| tst.js:423:18:423:24 | payload |
| tst.js:425:7:425:55 | match |
| tst.js:425:15:425:29 | window.location |
| tst.js:425:15:425:29 | window.location |
| tst.js:425:15:425:55 | window. ... (\\w+)/) |
| tst.js:427:20:427:24 | match |
| tst.js:427:20:427:27 | match[1] |
| tst.js:427:20:427:27 | match[1] |
| tst.js:430:18:430:32 | window.location |
| tst.js:430:18:430:32 | window.location |
| tst.js:430:18:430:51 | window. ... '#')[1] |
| tst.js:430:18:430:51 | window. ... '#')[1] |
| typeahead.js:9:28:9:30 | loc |
| typeahead.js:9:28:9:30 | loc |
| typeahead.js:10:16:10:18 | loc |
@@ -886,6 +903,21 @@ edges
| tst.js:414:19:414:31 | target.taint8 | tst.js:414:19:414:31 | target.taint8 |
| tst.js:414:19:414:31 | target.taint8 | tst.js:415:18:415:30 | target.taint8 |
| tst.js:414:19:414:31 | target.taint8 | tst.js:415:18:415:30 | target.taint8 |
| tst.js:422:7:422:46 | payload | tst.js:423:18:423:24 | payload |
| tst.js:422:7:422:46 | payload | tst.js:423:18:423:24 | payload |
| tst.js:422:17:422:31 | window.location | tst.js:422:17:422:46 | window. ... bstr(1) |
| tst.js:422:17:422:31 | window.location | tst.js:422:17:422:46 | window. ... bstr(1) |
| tst.js:422:17:422:46 | window. ... bstr(1) | tst.js:422:7:422:46 | payload |
| tst.js:425:7:425:55 | match | tst.js:427:20:427:24 | match |
| tst.js:425:15:425:29 | window.location | tst.js:425:15:425:55 | window. ... (\\w+)/) |
| tst.js:425:15:425:29 | window.location | tst.js:425:15:425:55 | window. ... (\\w+)/) |
| tst.js:425:15:425:55 | window. ... (\\w+)/) | tst.js:425:7:425:55 | match |
| tst.js:427:20:427:24 | match | tst.js:427:20:427:27 | match[1] |
| tst.js:427:20:427:24 | match | tst.js:427:20:427:27 | match[1] |
| tst.js:430:18:430:32 | window.location | tst.js:430:18:430:51 | window. ... '#')[1] |
| tst.js:430:18:430:32 | window.location | tst.js:430:18:430:51 | window. ... '#')[1] |
| tst.js:430:18:430:32 | window.location | tst.js:430:18:430:51 | window. ... '#')[1] |
| tst.js:430:18:430:32 | window.location | tst.js:430:18:430:51 | window. ... '#')[1] |
| typeahead.js:9:28:9:30 | loc | typeahead.js:10:16:10:18 | loc |
| typeahead.js:9:28:9:30 | loc | typeahead.js:10:16:10:18 | loc |
| typeahead.js:9:28:9:30 | loc | typeahead.js:10:16:10:18 | loc |

11
javascript/ql/test/query-tests/Security/CWE-079/tst.js
View File

@@ -418,3 +418,14 @@ function test() {
$('myId').html(target.taint9); // OK
}
function hash2() {
var payload = window.location.hash.substr(1);
document.write(payload); // NOT OK
let match = window.location.hash.match(/hello (\w+)/);
if (match) {
document.write(match[1]); // NOT OK
}
document.write(window.location.hash.split('#')[1]); // NOT OK
}

7
python/ql/src/Security/CWE-079/ReflectedXss.ql
View File

@@ -28,7 +28,12 @@ class ReflectedXssConfiguration extends TaintTracking::Configuration {
source instanceof HttpRequestTaintSource
}
override predicate isSink(TaintTracking::Sink sink) { sink instanceof HttpResponseTaintSink }
override predicate isSink(TaintTracking::Sink sink) {
sink instanceof HttpResponseTaintSink and
not sink instanceof DjangoResponseContent
or
sink instanceof DjangoResponseContentXSSVulnerable
}
}
from ReflectedXssConfiguration config, TaintedPathSource src, TaintedPathSink sink

28
python/ql/src/semmle/python/web/django/Redirect.qll
View File

@@ -11,10 +11,30 @@ private import semmle.python.web.django.Shared
private import semmle.python.web.Http
/**
* Represents an argument to the `django.redirect` function.
* The URL argument for a call to the `django.shortcuts.redirect` function.
*/
class DjangoRedirect extends HttpRedirectTaintSink {
override string toString() { result = "django.redirect" }
class DjangoShortcutsRedirectSink extends HttpRedirectTaintSink {
override string toString() { result = "DjangoShortcutsRedirectSink" }
DjangoRedirect() { this = redirect().getACall().getAnArg() }
DjangoShortcutsRedirectSink() {
this = Value::named("django.shortcuts.redirect").(FunctionValue).getArgumentForCall(_, 0)
}
}
/** DEPRECATED: Use `DjangoShortcutsRedirectSink` instead. */
deprecated class DjangoRedirect = DjangoShortcutsRedirectSink;
/**
* The URL argument when instantiating a Django Redirect Response.
*/
class DjangoRedirectResponseSink extends HttpRedirectTaintSink {
DjangoRedirectResponseSink() {
exists(CallNode call | call = any(DjangoRedirectResponseClass cls).getACall() |
this = call.getArg(0)
or
this = call.getArgByName("redirect_to")
)
}
override string toString() { result = "DjangoRedirectResponseSink" }
}

71
python/ql/src/semmle/python/web/django/Response.qll
View File

@@ -5,37 +5,25 @@ private import semmle.python.web.django.Shared
private import semmle.python.web.Http
/**
* DEPRECATED: This class is internal to the django library modeling, and should
* never be used by anyone.
*
* A django.http.response.Response object
* This isn't really a "taint", but we use the value tracking machinery to
* track the flow of response objects.
*/
class DjangoResponse extends TaintKind {
DjangoResponse() { this = "django.response.HttpResponse" }
deprecated class DjangoResponse = DjangoResponseKind;
/** INTERNAL class used for tracking a django response object. */
private class DjangoResponseKind extends TaintKind {
DjangoResponseKind() { this = "django.response.HttpResponse" }
}
private ClassValue theDjangoHttpResponseClass() {
(
// version 1.x
result = Value::named("django.http.response.HttpResponse")
or
// version 2.x
// https://docs.djangoproject.com/en/2.2/ref/request-response/#httpresponse-objects
result = Value::named("django.http.HttpResponse")
) and
// TODO: does this do anything? when could they be the same???
not result = theDjangoHttpRedirectClass()
}
/** internal class used for tracking a django response. */
/** INTERNAL taint-source used for tracking a django response object. */
private class DjangoResponseSource extends TaintSource {
DjangoResponseSource() {
exists(ClassValue cls |
cls.getASuperType() = theDjangoHttpResponseClass() and
cls.getACall() = this
)
}
DjangoResponseSource() { exists(DjangoContentResponseClass cls | cls.getACall() = this) }
override predicate isSourceOf(TaintKind kind) { kind instanceof DjangoResponse }
override predicate isSourceOf(TaintKind kind) { kind instanceof DjangoResponseKind }
override string toString() { result = "django.http.response.HttpResponse" }
}
@@ -45,7 +33,7 @@ class DjangoResponseWrite extends HttpResponseTaintSink {
DjangoResponseWrite() {
exists(AttrNode meth, CallNode call |
call.getFunction() = meth and
any(DjangoResponse response).taints(meth.getObject("write")) and
any(DjangoResponseKind response).taints(meth.getObject("write")) and
this = call.getArg(0)
)
}
@@ -55,17 +43,16 @@ class DjangoResponseWrite extends HttpResponseTaintSink {
override string toString() { result = "django.Response.write(...)" }
}
/** An argument to initialization of a django response, which is vulnerable to external data (xss) */
/**
* An argument to initialization of a django response.
*/
class DjangoResponseContent extends HttpResponseTaintSink {
DjangoContentResponseClass cls;
CallNode call;
DjangoResponseContent() {
exists(CallNode call, ClassValue cls |
cls.getASuperType() = theDjangoHttpResponseClass() and
call.getFunction().pointsTo(cls)
|
call.getArg(0) = this
or
call.getArgByName("content") = this
)
call = cls.getACall() and
this = cls.getContentArg(call)
}
override predicate sinks(TaintKind kind) { kind instanceof StringKind }
@@ -73,9 +60,25 @@ class DjangoResponseContent extends HttpResponseTaintSink {
override string toString() { result = "django.Response(...)" }
}
/**
* An argument to initialization of a django response, which is vulnerable to external data (XSS).
*/
class DjangoResponseContentXSSVulnerable extends DjangoResponseContent {
override DjangoXSSVulnerableResponseClass cls;
DjangoResponseContentXSSVulnerable() {
not exists(cls.getContentTypeArg(call))
or
exists(StringValue s |
cls.getContentTypeArg(call).pointsTo(s) and
s.getText().matches("text/html%")
)
}
}
class DjangoCookieSet extends CookieSet, CallNode {
DjangoCookieSet() {
any(DjangoResponse r).taints(this.getFunction().(AttrNode).getObject("set_cookie"))
any(DjangoResponseKind r).taints(this.getFunction().(AttrNode).getObject("set_cookie"))
}
override string toString() { result = CallNode.super.toString() }

78
python/ql/src/semmle/python/web/django/Shared.qll
View File

@@ -1,12 +1,84 @@
import python
/** django.shortcuts.redirect */
FunctionValue redirect() { result = Value::named("django.shortcuts.redirect") }
/** DEPRECATED: Use `Value::named("django.shortcuts.redirect")` instead. */
deprecated FunctionValue redirect() { result = Value::named("django.shortcuts.redirect") }
ClassValue theDjangoHttpRedirectClass() {
/** DEPRECATED: Use `DjangoRedirectResponseClass` instead. */
deprecated ClassValue theDjangoHttpRedirectClass() {
// version 1.x
result = Value::named("django.http.response.HttpResponseRedirectBase")
or
// version 2.x
result = Value::named("django.http.HttpResponseRedirectBase")
}
/** A class that is a Django Redirect Response (subclass of `django.http.HttpResponseRedirectBase`). */
class DjangoRedirectResponseClass extends ClassValue {
DjangoRedirectResponseClass() {
exists(ClassValue redirect_base |
// version 1.x
redirect_base = Value::named("django.http.response.HttpResponseRedirectBase")
or
// version 2.x and 3.x
redirect_base = Value::named("django.http.HttpResponseRedirectBase")
|
this.getASuperType() = redirect_base
)
}
}
/**
* A class that is a Django Response, which can contain content.
* A subclass of `django.http.HttpResponse` that is not a `DjangoRedirectResponseClass`.
*/
class DjangoContentResponseClass extends ClassValue {
ClassValue base;
DjangoContentResponseClass() {
(
// version 1.x
base = Value::named("django.http.response.HttpResponse")
or
// version 2.x and 3.x
// https://docs.djangoproject.com/en/2.2/ref/request-response/#httpresponse-objects
base = Value::named("django.http.HttpResponse")
) and
this.getASuperType() = base
}
// The reason these two methods are defined in this class (and not in the Sink
// definition that uses this class), is that if we were to add support for
// `django.http.response.HttpResponseNotAllowed` it would make much more sense to add
// the custom logic in this class (or subclass), than to handle all of it in the sink
// definition.
/** Gets the `content` argument of a `call` to the constructor */
ControlFlowNode getContentArg(CallNode call) { none() }
/** Gets the `content_type` argument of a `call` to the constructor */
ControlFlowNode getContentTypeArg(CallNode call) { none() }
}
/** A class that is a Django Response, and is vulnerable to XSS. */
class DjangoXSSVulnerableResponseClass extends DjangoContentResponseClass {
DjangoXSSVulnerableResponseClass() {
// We want to avoid FPs on subclasses that are not exposed to XSS, for example `JsonResponse`.
// The easiest way is to disregard any subclass that has a special `__init__` method.
// It's not guaranteed to remove all FPs, or not to generate FNs, but compared to our
// previous implementation that would treat 0-th argument to _any_ subclass as a sink,
// this gets us much closer to reality.
this.lookup("__init__") = base.lookup("__init__") and
not this instanceof DjangoRedirectResponseClass
}
override ControlFlowNode getContentArg(CallNode call) {
result = call.getArg(0)
or
result = call.getArgByName("content")
}
override ControlFlowNode getContentTypeArg(CallNode call) {
result = call.getArg(1)
or
result = call.getArgByName("content_type")
}
}

6
python/ql/test/library-tests/web/django/HttpRedirectSinks.expected
View File

@@ -1,2 +1,4 @@
| test_1x.py:13:21:13:24 | django.redirect | externally controlled string |
| test_2x_3x.py:13:21:13:24 | django.redirect | externally controlled string |
| test_1x.py:13:21:13:24 | DjangoShortcutsRedirectSink | externally controlled string |
| test_2x_3x.py:13:21:13:24 | DjangoShortcutsRedirectSink | externally controlled string |
| views_1x.py:99:33:99:55 | DjangoRedirectResponseSink | externally controlled string |
| views_2x_3x.py:120:33:120:55 | DjangoRedirectResponseSink | externally controlled string |

8
python/ql/test/library-tests/web/django/HttpResponseSinks.expected
View File

@@ -8,6 +8,10 @@
| views_1x.py:45:25:45:70 | django.Response(...) | externally controlled string |
| views_1x.py:66:25:66:55 | django.Response(...) | externally controlled string |
| views_1x.py:75:25:75:33 | django.Response(...) | externally controlled string |
| views_1x.py:90:25:90:33 | django.Response(...) | externally controlled string |
| views_1x.py:94:25:94:58 | django.Response(...) | externally controlled string |
| views_1x.py:103:33:103:55 | django.Response(...) | externally controlled string |
| views_1x.py:107:25:107:47 | django.Response(...) | externally controlled string |
| views_2x_3x.py:8:25:8:63 | django.Response(...) | externally controlled string |
| views_2x_3x.py:12:25:12:52 | django.Response(...) | externally controlled string |
| views_2x_3x.py:16:25:16:53 | django.Response(...) | externally controlled string |
@@ -21,3 +25,7 @@
| views_2x_3x.py:82:25:82:69 | django.Response(...) | externally controlled string |
| views_2x_3x.py:85:25:85:64 | django.Response(...) | externally controlled string |
| views_2x_3x.py:88:25:88:32 | django.Response(...) | externally controlled string |
| views_2x_3x.py:111:25:111:33 | django.Response(...) | externally controlled string |
| views_2x_3x.py:115:25:115:58 | django.Response(...) | externally controlled string |
| views_2x_3x.py:124:33:124:55 | django.Response(...) | externally controlled string |
| views_2x_3x.py:128:25:128:47 | django.Response(...) | externally controlled string |

30
python/ql/test/library-tests/web/django/views_1x.py
View File

@@ -1,6 +1,6 @@
"""test of views for Django 1.x"""
from django.conf.urls import patterns, url
from django.http.response import HttpResponse
from django.http.response import HttpResponse, HttpResponseRedirect, JsonResponse, HttpResponseNotFound
from django.views.generic import View
@@ -77,3 +77,31 @@ def kw_args(request):
urlpatterns = [
url(view=kw_args, regex=r'^kw_args$')
]
# Not an XSS sink, since the Content-Type is not "text/html"
# FP reported in https://github.com/github/codeql-python-team/issues/38
def fp_json_response(request):
# implicitly sets Content-Type to "application/json"
return JsonResponse({"foo": request.GET.get("foo")})
# Not an XSS sink, since the Content-Type is not "text/html"
def fp_manual_json_response(request):
json_data = '{"json": "{}"}'.format(request.GET.get("foo"))
return HttpResponse(json_data, content_type="application/json")
# Not an XSS sink, since the Content-Type is not "text/html"
def fp_manual_content_type(reuqest):
return HttpResponse('<img src="0" onerror="alert(1)">', content_type="text/plain")
# XSS FP reported in https://github.com/github/codeql/issues/3466
# Note: This should be a open-redirect sink, but not a XSS sink.
def fp_redirect(request):
return HttpResponseRedirect(request.GET.get("next"))
# Ensure that simple subclasses are still vuln to XSS
def tp_not_found(request):
return HttpResponseNotFound(request.GET.get("name"))
# Ensure we still have a XSS sink when manually setting the content_type to HTML
def tp_manual_response_type(request):
return HttpResponse(request.GET.get("name"), content_type="text/html; charset=utf-8")

40
python/ql/test/library-tests/web/django/views_2x_3x.py
View File

@@ -1,6 +1,6 @@
"""testing views for Django 2.x and 3.x"""
from django.urls import path, re_path
from django.http import HttpResponse
from django.http import HttpResponse, HttpResponseRedirect, JsonResponse, HttpResponseNotFound
from django.views import View
@@ -99,24 +99,30 @@ urlpatterns = [
]
################################################################################
# Not an XSS sink, since the Content-Type is not "text/html"
# FP reported in https://github.com/github/codeql-python-team/issues/38
def fp_json_response(request):
# implicitly sets Content-Type to "application/json"
return JsonResponse({"foo": request.GET.get("foo")})
# Not an XSS sink, since the Content-Type is not "text/html"
def fp_manual_json_response(request):
json_data = '{"json": "{}"}'.format(request.GET.get("foo"))
return HttpResponse(json_data, content_type="application/json")
# We should abort if a decorator is used. As demonstrated below, anything might happen
# Not an XSS sink, since the Content-Type is not "text/html"
def fp_manual_content_type(reuqest):
return HttpResponse('<img src="0" onerror="alert(1)">', content_type="text/plain")
# def reverse_kwargs(f):
# @wraps(f)
# def f_(*args, **kwargs):
# new_kwargs = dict()
# for key, value in kwargs.items():
# new_kwargs[key[::-1]] = value
# return f(*args, **new_kwargs)
# return f_
# XSS FP reported in https://github.com/github/codeql/issues/3466
# Note: This should be a open-redirect sink, but not a XSS sink.
def fp_redirect(request):
return HttpResponseRedirect(request.GET.get("next"))
# @reverse_kwargs
# def decorators_can_do_anything(request, oof, foo=None):
# return HttpResponse('This is a mess'[::-1])
# Ensure that simple subclasses are still vuln to XSS
def tp_not_found(request):
return HttpResponseNotFound(request.GET.get("name"))
# urlpatterns = [
# path('rev/<foo>', decorators_can_do_anything),
# ]
# Ensure we still have a XSS sink when manually setting the content_type to HTML
def tp_manual_response_type(request):
return HttpResponse(request.GET.get("name"), content_type="text/html; charset=utf-8")

2
python/ql/test/query-tests/Security/lib/django/http/__init__.py
View File

@@ -1,2 +1,2 @@
from .response import HttpResponse
from .response import *
from .request import HttpRequest

48
python/ql/test/query-tests/Security/lib/django/http/response.py
View File

@@ -1,2 +1,46 @@
class HttpResponse(object):
pass
class HttpResponseBase(object):
status_code = 200
def __init__(self, content_type=None, status=None, reason=None, charset=None):
pass
class HttpResponse(HttpResponseBase):
def __init__(self, content=b"", *args, **kwargs):
super().__init__(*args, **kwargs)
# Content is a bytestring. See the `content` property methods.
self.content = content
class HttpResponseRedirectBase(HttpResponse):
def __init__(self, redirect_to, *args, **kwargs):
super().__init__(*args, **kwargs)
...
class HttpResponsePermanentRedirect(HttpResponseRedirectBase):
status_code = 301
class HttpResponseRedirect(HttpResponseRedirectBase):
status_code = 302
class HttpResponseNotFound(HttpResponse):
status_code = 404
class JsonResponse(HttpResponse):
def __init__(
self,
data,
encoder=...,
safe=True,
json_dumps_params=None,
**kwargs
):
# fake code to represent what is going on :)
kwargs.setdefault("content_type", "application/json")
data = str(data)
super().__init__(content=data, **kwargs)