hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-16 16:53:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

Revert "C#: Tool status page support"

Browse Source
This commit is contained in:
Asger F
2023-03-03 11:44:42 +01:00
committed by GitHub
parent 7b596f4928
commit 8f0b77d54f
43 changed files with 81 additions and 1452 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/BuildScripts.cs
View File

@@ -1,6 +1,5 @@
using Xunit;
using Semmle.Autobuild.Shared;
using Semmle.Util;
using System.Collections.Generic;
using System;
using System.Linq;
@@ -76,15 +75,6 @@ namespace Semmle.Autobuild.Cpp.Tests
throw new ArgumentException("Missing RunProcess " + pattern);
}
int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory, IDictionary<string, string>? env, BuildOutputHandler onOutput, BuildOutputHandler onError)
{
var ret = (this as IBuildActions).RunProcess(cmd, args, workingDirectory, env, out var stdout);
stdout.ForEach(line => onOutput(line));
return ret;
}
public IList<string> DirectoryDeleteIn = new List<string>();
void IBuildActions.DirectoryDelete(string dir, bool recursive)
@@ -253,7 +243,6 @@ namespace Semmle.Autobuild.Cpp.Tests
Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_TRAP_DIR"] = "";
Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_SOURCE_ARCHIVE_DIR"] = "";
Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_ROOT"] = $@"C:\codeql\{codeqlUpperLanguage.ToLowerInvariant()}";
Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_DIAGNOSTIC_DIR"] = Path.GetTempPath();
Actions.GetEnvironmentVariable["CODEQL_JAVA_HOME"] = @"C:\codeql\tools\java";
Actions.GetEnvironmentVariable["CODEQL_PLATFORM"] = "win64";
Actions.GetEnvironmentVariable["SEMMLE_DIST"] = @"C:\odasa";

3
cpp/autobuilder/Semmle.Autobuild.Cpp/CppAutobuilder.cs
View File

@@ -1,5 +1,4 @@
using Semmle.Autobuild.Shared;
using Semmle.Util;
namespace Semmle.Autobuild.Cpp
{
@@ -22,7 +21,7 @@ namespace Semmle.Autobuild.Cpp
public class CppAutobuilder : Autobuilder<CppAutobuildOptions>
{
public CppAutobuilder(IBuildActions actions, CppAutobuildOptions options) : base(actions, options, new DiagnosticClassifier()) { }
public CppAutobuilder(IBuildActions actions, CppAutobuildOptions options) : base(actions, options) { }
public override BuildScript GetBuildScript()
{

11
csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs
View File

@@ -1,6 +1,5 @@
using Xunit;
using Semmle.Autobuild.Shared;
using Semmle.Util;
using System.Collections.Generic;
using System;
using System.Linq;
@@ -86,15 +85,6 @@ namespace Semmle.Autobuild.CSharp.Tests
return ret;
}
int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory, IDictionary<string, string>? env, BuildOutputHandler onOutput, BuildOutputHandler onError)
{
var ret = (this as IBuildActions).RunProcess(cmd, args, workingDirectory, env, out var stdout);
stdout.ForEach(line => onOutput(line));
return ret;
}
public IList<string> DirectoryDeleteIn { get; } = new List<string>();
void IBuildActions.DirectoryDelete(string dir, bool recursive)
@@ -401,7 +391,6 @@ namespace Semmle.Autobuild.CSharp.Tests
actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_TRAP_DIR"] = "";
actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_SOURCE_ARCHIVE_DIR"] = "";
actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_ROOT"] = $@"C:\codeql\{codeqlUpperLanguage.ToLowerInvariant()}";
actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_DIAGNOSTIC_DIR"] = Path.GetTempPath();
actions.GetEnvironmentVariable["CODEQL_JAVA_HOME"] = @"C:\codeql\tools\java";
actions.GetEnvironmentVariable["CODEQL_PLATFORM"] = isWindows ? "win64" : "linux64";
actions.GetEnvironmentVariable["LGTM_INDEX_VSTOOLS_VERSION"] = vsToolsVersion;

69
csharp/autobuilder/Semmle.Autobuild.CSharp/AutoBuildRule.cs
View File

@@ -1,69 +0,0 @@
using Semmle.Autobuild.Shared;
using Semmle.Extraction.CSharp;
namespace Semmle.Autobuild.CSharp
{
internal class AutoBuildRule : IBuildRule<CSharpAutobuildOptions>
{
private readonly CSharpAutobuilder autobuilder;
public DotNetRule DotNetRule { get; }
public MsBuildRule MsBuildRule { get; }
public BuildCommandAutoRule BuildCommandAutoRule { get; }
public AutoBuildRule(CSharpAutobuilder autobuilder)
{
this.autobuilder = autobuilder;
this.DotNetRule = new DotNetRule();
this.MsBuildRule = new MsBuildRule();
this.BuildCommandAutoRule = new BuildCommandAutoRule(DotNetRule.WithDotNet);
}
public BuildScript Analyse(IAutobuilder<CSharpAutobuildOptions> builder, bool auto)
{
var cleanTrapFolder =
BuildScript.DeleteDirectory(this.autobuilder.TrapDir);
var cleanSourceArchive =
BuildScript.DeleteDirectory(this.autobuilder.SourceArchiveDir);
var tryCleanExtractorArgsLogs =
BuildScript.Create(actions =>
{
foreach (var file in Extractor.GetCSharpArgsLogs())
{
try
{
actions.FileDelete(file);
}
catch // lgtm[cs/catch-of-all-exceptions] lgtm[cs/empty-catch-block]
{ }
}
return 0;
});
var attemptExtractorCleanup =
BuildScript.Try(cleanTrapFolder) &
BuildScript.Try(cleanSourceArchive) &
tryCleanExtractorArgsLogs &
BuildScript.DeleteFile(Extractor.GetCSharpLogPath());
/// <summary>
/// Execute script `s` and check that the C# extractor has been executed.
/// If either fails, attempt to cleanup any artifacts produced by the extractor,
/// and exit with code 1, in order to proceed to the next attempt.
/// </summary>
BuildScript IntermediateAttempt(BuildScript s) =>
(s & this.autobuilder.CheckExtractorRun(false)) |
(attemptExtractorCleanup & BuildScript.Failure);
return
// First try .NET Core
IntermediateAttempt(this.DotNetRule.Analyse(this.autobuilder, true)) |
// Then MSBuild
(() => IntermediateAttempt(this.MsBuildRule.Analyse(this.autobuilder, true))) |
// And finally look for a script that might be a build script
(() => this.BuildCommandAutoRule.Analyse(this.autobuilder, true) & this.autobuilder.CheckExtractorRun(true));
}
}
}

188
csharp/autobuilder/Semmle.Autobuild.CSharp/CSharpAutobuilder.cs
View File

@@ -1,8 +1,6 @@
using Semmle.Extraction.CSharp;
using Semmle.Util.Logging;
using Semmle.Autobuild.Shared;
using Semmle.Util;
using System.Linq;
namespace Semmle.Autobuild.CSharp
{
@@ -31,16 +29,25 @@ namespace Semmle.Autobuild.CSharp
public class CSharpAutobuilder : Autobuilder<CSharpAutobuildOptions>
{
private const string buildCommandDocsUrl =
"https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages";
private readonly AutoBuildRule autoBuildRule;
public CSharpAutobuilder(IBuildActions actions, CSharpAutobuildOptions options) : base(actions, options, new CSharpDiagnosticClassifier()) =>
this.autoBuildRule = new AutoBuildRule(this);
public CSharpAutobuilder(IBuildActions actions, CSharpAutobuildOptions options) : base(actions, options) { }
public override BuildScript GetBuildScript()
{
/// <summary>
/// A script that checks that the C# extractor has been executed.
/// </summary>
BuildScript CheckExtractorRun(bool warnOnFailure) =>
BuildScript.Create(actions =>
{
if (actions.FileExists(Extractor.GetCSharpLogPath()))
return 0;
if (warnOnFailure)
Log(Severity.Error, "No C# code detected during build.");
return 1;
});
var attempt = BuildScript.Failure;
switch (GetCSharpBuildStrategy())
{
@@ -58,9 +65,47 @@ namespace Semmle.Autobuild.CSharp
attempt = new DotNetRule().Analyse(this, false) & CheckExtractorRun(true);
break;
case CSharpBuildStrategy.Auto:
var cleanTrapFolder =
BuildScript.DeleteDirectory(TrapDir);
var cleanSourceArchive =
BuildScript.DeleteDirectory(SourceArchiveDir);
var tryCleanExtractorArgsLogs =
BuildScript.Create(actions =>
{
foreach (var file in Extractor.GetCSharpArgsLogs())
{
try
{
actions.FileDelete(file);
}
catch // lgtm[cs/catch-of-all-exceptions] lgtm[cs/empty-catch-block]
{ }
}
return 0;
});
var attemptExtractorCleanup =
BuildScript.Try(cleanTrapFolder) &
BuildScript.Try(cleanSourceArchive) &
tryCleanExtractorArgsLogs &
BuildScript.DeleteFile(Extractor.GetCSharpLogPath());
/// <summary>
/// Execute script `s` and check that the C# extractor has been executed.
/// If either fails, attempt to cleanup any artifacts produced by the extractor,
/// and exit with code 1, in order to proceed to the next attempt.
/// </summary>
BuildScript IntermediateAttempt(BuildScript s) =>
(s & CheckExtractorRun(false)) |
(attemptExtractorCleanup & BuildScript.Failure);
attempt =
// Attempt a few different build strategies to see if one works
this.autoBuildRule.Analyse(this, true) |
// First try .NET Core
IntermediateAttempt(new DotNetRule().Analyse(this, true)) |
// Then MSBuild
(() => IntermediateAttempt(new MsBuildRule().Analyse(this, true))) |
// And finally look for a script that might be a build script
(() => new BuildCommandAutoRule(DotNetRule.WithDotNet).Analyse(this, true) & CheckExtractorRun(true)) |
// All attempts failed: print message
AutobuildFailure();
break;
@@ -69,127 +114,6 @@ namespace Semmle.Autobuild.CSharp
return attempt;
}
/// <summary>
/// A script that checks that the C# extractor has been executed.
/// </summary>
public BuildScript CheckExtractorRun(bool warnOnFailure) =>
BuildScript.Create(actions =>
{
if (actions.FileExists(Extractor.GetCSharpLogPath()))
return 0;
if (warnOnFailure)
Log(Severity.Error, "No C# code detected during build.");
return 1;
});
protected override void AutobuildFailureDiagnostic()
{
// if `ScriptPath` is not null here, the `BuildCommandAuto` rule was
// run and found at least one script to execute
if (this.autoBuildRule.BuildCommandAutoRule.ScriptPath is not null)
{
var relScriptPath = this.MakeRelative(autoBuildRule.BuildCommandAutoRule.ScriptPath);
// if we found multiple build scripts in the project directory, then we can say
// as much to indicate that we may have picked the wrong one;
// otherwise, we just report that the one script we found didn't work
DiagnosticMessage message =
this.autoBuildRule.BuildCommandAutoRule.CandidatePaths.Count() > 1 ?
new(
this.Options.Language,
"multiple-build-scripts",
"There are multiple potential build scripts",
markdownMessage:
"CodeQL found multiple potential build scripts for your project and " +
$"attempted to run `{relScriptPath}`, which failed. " +
"This may not be the right build script for your project. " +
$"Set up a [manual build command]({buildCommandDocsUrl})."
) :
new(
this.Options.Language,
"script-failure",
"Unable to build project using build script",
markdownMessage:
"CodeQL attempted to build your project using a script located at " +
$"`{relScriptPath}`, which failed. " +
$"Set up a [manual build command]({buildCommandDocsUrl})."
);
AddDiagnostic(message);
}
// project files which don't exist get marked as not .NET core projects, but we don't want
// to show an error for this if the files don't exist
var foundNotDotNetProjects = autoBuildRule.DotNetRule.NotDotNetProjects.Where(
proj => this.Actions.FileExists(proj.FullPath)
);
// both dotnet and msbuild builds require project or solution files; if we haven't found any
// then neither of those rules would've worked
if (this.ProjectsOrSolutionsToBuild.Count == 0)
{
this.AddDiagnostic(new(
this.Options.Language,
"no-projects-or-solutions",
"No project or solutions files found",
markdownMessage:
"CodeQL could not find any project or solution files in your repository. " +
$"Set up a [manual build command]({buildCommandDocsUrl})."
));
}
// show a warning if there are projects which are not compatible with .NET Core, in case that is unintentional
else if (foundNotDotNetProjects.Any())
{
this.AddDiagnostic(new(
this.Options.Language,
"dotnet-incompatible-projects",
"Some projects are incompatible with .NET Core",
severity: DiagnosticMessage.TspSeverity.Warning,
markdownMessage: $"""
CodeQL found some projects which cannot be built with .NET Core:
{autoBuildRule.DotNetRule.NotDotNetProjects.Select(p => this.MakeRelative(p.FullPath)).ToMarkdownList(MarkdownUtil.CodeFormatter, 5)}
"""
));
}
// report any projects that failed to build with .NET Core
if (autoBuildRule.DotNetRule.FailedProjectsOrSolutions.Any())
{
this.AddDiagnostic(new(
this.Options.Language,
"dotnet-build-failure",
"Some projects or solutions failed to build using .NET Core",
markdownMessage: $"""
CodeQL was unable to build the following projects using .NET Core:
{autoBuildRule.DotNetRule.FailedProjectsOrSolutions.Select(p => this.MakeRelative(p.FullPath)).ToMarkdownList(MarkdownUtil.CodeFormatter, 10)}
Set up a [manual build command]({buildCommandDocsUrl}).
"""
));
}
// report any projects that failed to build with MSBuild
if (autoBuildRule.MsBuildRule.FailedProjectsOrSolutions.Any())
{
this.AddDiagnostic(new(
this.Options.Language,
"msbuild-build-failure",
"Some projects or solutions failed to build using MSBuild",
markdownMessage: $"""
CodeQL was unable to build the following projects using MSBuild:
{autoBuildRule.MsBuildRule.FailedProjectsOrSolutions.Select(p => this.MakeRelative(p.FullPath)).ToMarkdownList(MarkdownUtil.CodeFormatter, 10)}
Set up a [manual build command]({buildCommandDocsUrl}).
"""
));
}
}
/// <summary>
/// Gets the build strategy that the autobuilder should apply, based on the
/// options in the `lgtm.yml` file.

133
csharp/autobuilder/Semmle.Autobuild.CSharp/CSharpDiagnosticClassifier.cs
View File

@@ -1,133 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using Semmle.Autobuild.Shared;
using Semmle.Util;
namespace Semmle.Autobuild.CSharp
{
/// <summary>
/// A diagnostic rule which tries to identify missing Xamarin SDKs.
/// </summary>
public class MissingXamarinSdkRule : DiagnosticRule
{
private const string docsUrl = "https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-xamarin-applications";
public class Result : IDiagnosticsResult
{
/// <summary>
/// The name of the SDK that is missing.
/// </summary>
public string SDKName { get; }
public Result(string sdkName)
{
this.SDKName = sdkName;
}
public DiagnosticMessage ToDiagnosticMessage<T>(Autobuilder<T> builder, DiagnosticMessage.TspSeverity? severity = null) where T : AutobuildOptionsShared => new(
builder.Options.Language,
$"missing-xamarin-{this.SDKName.ToLower()}-sdk",
$"Missing Xamarin SDK for {this.SDKName}",
severity: severity ?? DiagnosticMessage.TspSeverity.Error,
markdownMessage: $"[Configure your workflow]({docsUrl}) for this SDK before running CodeQL."
);
}
public MissingXamarinSdkRule() :
base("MSB4019:[^\"]*\"[^\"]*Xamarin\\.(?<sdkName>[^\\.]*)\\.CSharp\\.targets\"")
{
}
public override void Fire(DiagnosticClassifier classifier, Match match)
{
if (!match.Groups.TryGetValue("sdkName", out var sdkName))
throw new ArgumentException("Expected regular expression match to contain sdkName");
var xamarinResults = classifier.Results.OfType<Result>().Where(result =>
result.SDKName.Equals(sdkName.Value)
);
if (!xamarinResults.Any())
classifier.Results.Add(new Result(sdkName.Value));
}
}
public class MissingProjectFileRule : DiagnosticRule
{
private const string runsOnDocsUrl = "https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idruns-on";
private const string checkoutDocsUrl = "https://github.com/actions/checkout#usage";
public class Result : IDiagnosticsResult
{
/// <summary>
/// A set of missing project files.
/// </summary>
public HashSet<string> MissingProjectFiles { get; }
public Result()
{
this.MissingProjectFiles = new HashSet<string>();
}
public DiagnosticMessage ToDiagnosticMessage<T>(Autobuilder<T> builder, DiagnosticMessage.TspSeverity? severity = null) where T : AutobuildOptionsShared => new(
builder.Options.Language,
"missing-project-files",
"Missing project files",
severity: severity ?? DiagnosticMessage.TspSeverity.Warning,
markdownMessage: $"""
Some project files were not found when CodeQL built your project:
{this.MissingProjectFiles.AsEnumerable().Select(p => builder.MakeRelative(p)).ToMarkdownList(MarkdownUtil.CodeFormatter, 5)}
This may lead to subsequent failures. You can check for common causes for missing project files:
- Ensure that the project is built using the {runsOnDocsUrl.ToMarkdownLink("intended operating system")} and that filenames on case-sensitive platforms are correctly specified.
- If your repository uses Git submodules, ensure that those are {checkoutDocsUrl.ToMarkdownLink("checked out")} before the CodeQL action is run.
- If you auto-generate some project files as part of your build process, ensure that these are generated before the CodeQL action is run.
"""
);
}
public MissingProjectFileRule() :
base("MSB3202: The project file \"(?<projectFile>[^\"]+)\" was not found. \\[(?<location>[^\\]]+)\\]")
{
}
public override void Fire(DiagnosticClassifier classifier, Match match)
{
if (!match.Groups.TryGetValue("projectFile", out var projectFile))
throw new ArgumentException("Expected regular expression match to contain projectFile");
if (!match.Groups.TryGetValue("location", out var location))
throw new ArgumentException("Expected regular expression match to contain location");
var result = classifier.Results.OfType<Result>().FirstOrDefault();
// if we do not yet have a result for this rule, create one and add it to the list
// of results the classifier knows about
if (result is null)
{
result = new Result();
classifier.Results.Add(result);
}
// then add the missing project file
result.MissingProjectFiles.Add(projectFile.Value);
}
}
/// <summary>
/// Implements a <see cref="DiagnosticClassifier" /> which applies C#-specific rules to
/// the build output.
/// </summary>
public class CSharpDiagnosticClassifier : DiagnosticClassifier
{
public CSharpDiagnosticClassifier()
{
// add C#-specific rules to this classifier
this.AddRule(new MissingXamarinSdkRule());
this.AddRule(new MissingProjectFileRule());
}
}
}

20
csharp/autobuilder/Semmle.Autobuild.CSharp/DotNetRule.cs
View File

@@ -15,15 +15,6 @@ namespace Semmle.Autobuild.CSharp
/// </summary>
internal class DotNetRule : IBuildRule<CSharpAutobuildOptions>
{
public readonly List<IProjectOrSolution> FailedProjectsOrSolutions = new();
/// <summary>
/// A list of projects which are incompatible with DotNet.
/// </summary>
public IEnumerable<Project<CSharpAutobuildOptions>> NotDotNetProjects { get; private set; }
public DotNetRule() => NotDotNetProjects = new List<Project<CSharpAutobuildOptions>>();
public BuildScript Analyse(IAutobuilder<CSharpAutobuildOptions> builder, bool auto)
{
if (!builder.ProjectsOrSolutionsToBuild.Any())
@@ -31,12 +22,10 @@ namespace Semmle.Autobuild.CSharp
if (auto)
{
NotDotNetProjects = builder.ProjectsOrSolutionsToBuild
var notDotNetProject = builder.ProjectsOrSolutionsToBuild
.SelectMany(p => Enumerators.Singleton(p).Concat(p.IncludedProjects))
.OfType<Project<CSharpAutobuildOptions>>()
.Where(p => !p.DotNetProject);
var notDotNetProject = NotDotNetProjects.FirstOrDefault();
.FirstOrDefault(p => !p.DotNetProject);
if (notDotNetProject is not null)
{
builder.Log(Severity.Info, "Not using .NET Core because of incompatible project {0}", notDotNetProject);
@@ -61,10 +50,7 @@ namespace Semmle.Autobuild.CSharp
var build = GetBuildScript(builder, dotNetPath, environment, projectOrSolution.FullPath);
ret &= BuildScript.Try(clean) & BuildScript.Try(restore) & BuildScript.OnFailure(build, ret =>
{
FailedProjectsOrSolutions.Add(projectOrSolution);
});
ret &= BuildScript.Try(clean) & BuildScript.Try(restore) & build;
}
return ret;
});

1
csharp/autobuilder/Semmle.Autobuild.Shared/AutobuildOptions.cs
View File

@@ -2,7 +2,6 @@
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using Semmle.Util;
namespace Semmle.Autobuild.Shared
{

121
csharp/autobuilder/Semmle.Autobuild.Shared/Autobuilder.cs
View File

@@ -1,4 +1,3 @@
using Semmle.Util;
using Semmle.Util.Logging;
using System;
using System.Collections.Generic;
@@ -190,11 +189,10 @@ namespace Semmle.Autobuild.Shared
/// solution file and tools.
/// </summary>
/// <param name="options">The command line options.</param>
protected Autobuilder(IBuildActions actions, TAutobuildOptions options, DiagnosticClassifier diagnosticClassifier)
protected Autobuilder(IBuildActions actions, TAutobuildOptions options)
{
Actions = actions;
Options = options;
DiagnosticClassifier = diagnosticClassifier;
pathsLazy = new Lazy<IEnumerable<(string, int)>>(() =>
{
@@ -234,53 +232,24 @@ namespace Semmle.Autobuild.Shared
return ret ?? new List<IProjectOrSolution>();
});
CodeQLExtractorLangRoot = Actions.GetEnvironmentVariable(EnvVars.Root(this.Options.Language));
CodeQlPlatform = Actions.GetEnvironmentVariable(EnvVars.Platform);
CodeQLExtractorLangRoot = Actions.GetEnvironmentVariable($"CODEQL_EXTRACTOR_{this.Options.Language.UpperCaseName}_ROOT");
CodeQlPlatform = Actions.GetEnvironmentVariable("CODEQL_PLATFORM");
TrapDir = RequireEnvironmentVariable(EnvVars.TrapDir(this.Options.Language));
SourceArchiveDir = RequireEnvironmentVariable(EnvVars.SourceArchiveDir(this.Options.Language));
DiagnosticsDir = RequireEnvironmentVariable(EnvVars.DiagnosticDir(this.Options.Language));
TrapDir =
Actions.GetEnvironmentVariable($"CODEQL_EXTRACTOR_{this.Options.Language.UpperCaseName}_TRAP_DIR") ??
throw new InvalidEnvironmentException($"The environment variable CODEQL_EXTRACTOR_{this.Options.Language.UpperCaseName}_TRAP_DIR has not been set.");
this.diagnostics = new DiagnosticsStream(Path.Combine(DiagnosticsDir, $"autobuilder-{DateTime.UtcNow:yyyyMMddHHmm}.jsonc"));
SourceArchiveDir =
Actions.GetEnvironmentVariable($"CODEQL_EXTRACTOR_{this.Options.Language.UpperCaseName}_SOURCE_ARCHIVE_DIR") ??
throw new InvalidEnvironmentException($"The environment variable CODEQL_EXTRACTOR_{this.Options.Language.UpperCaseName}_SOURCE_ARCHIVE_DIR has not been set.");
}
/// <summary>
/// Retrieves the value of an environment variable named <paramref name="name"> or throws
/// an exception if no such environment variable has been set.
/// </summary>
/// <param name="name">The name of the environment variable.</param>
/// <returns>The value of the environment variable.</returns>
/// <exception cref="InvalidEnvironmentException">
/// Thrown if the environment variable is not set.
/// </exception>
protected string RequireEnvironmentVariable(string name)
{
return Actions.GetEnvironmentVariable(name) ??
throw new InvalidEnvironmentException($"The environment variable {name} has not been set.");
}
protected string TrapDir { get; }
public string TrapDir { get; }
public string SourceArchiveDir { get; }
public string DiagnosticsDir { get; }
protected DiagnosticClassifier DiagnosticClassifier { get; }
protected string SourceArchiveDir { get; }
private readonly ILogger logger = new ConsoleLogger(Verbosity.Info);
private readonly DiagnosticsStream diagnostics;
/// <summary>
/// Makes <see cref="path" /> relative to the root source directory.
/// </summary>
/// <param name="path">The path which to make relative.</param>
/// <returns>The relative path.</returns>
public string MakeRelative(string path)
{
return Path.GetRelativePath(this.RootDirectory, path);
}
/// <summary>
/// Log a given build event to the console.
/// </summary>
@@ -291,15 +260,6 @@ namespace Semmle.Autobuild.Shared
logger.Log(severity, format, args);
}
/// <summary>
/// Write <paramref name="diagnostic"/> to the diagnostics file.
/// </summary>
/// <param name="diagnostic">The diagnostics entry to write.</param>
public void AddDiagnostic(DiagnosticMessage diagnostic)
{
diagnostics.AddEntry(diagnostic);
}
/// <summary>
/// Attempt to build this project.
/// </summary>
@@ -323,19 +283,7 @@ namespace Semmle.Autobuild.Shared
Log(silent ? Severity.Debug : Severity.Info, $"Exit code {ret}{(string.IsNullOrEmpty(msg) ? "" : $": {msg}")}");
}
var onOutput = BuildOutputHandler(Console.Out);
var onError = BuildOutputHandler(Console.Error);
var buildResult = script.Run(Actions, startCallback, exitCallback, onOutput, onError);
// if the build succeeded, all diagnostics we captured from the build output should be warnings;
// otherwise they should all be errors
var diagSeverity = buildResult == 0 ? DiagnosticMessage.TspSeverity.Warning : DiagnosticMessage.TspSeverity.Error;
this.DiagnosticClassifier.Results
.Select(result => result.ToDiagnosticMessage(this, diagSeverity))
.ForEach(AddDiagnostic);
return buildResult;
return script.Run(Actions, startCallback, exitCallback);
}
/// <summary>
@@ -343,58 +291,13 @@ namespace Semmle.Autobuild.Shared
/// </summary>
public abstract BuildScript GetBuildScript();
/// <summary>
/// Produces a diagnostic for the tool status page that we were unable to automatically
/// build the user's project and that they can manually specify a build command. This
/// can be overriden to implement more specific messages depending on the origin of
/// the failure.
/// </summary>
protected virtual void AutobuildFailureDiagnostic() => AddDiagnostic(new DiagnosticMessage(
this.Options.Language,
"autobuild-failure",
"Unable to build project",
visibility: new DiagnosticMessage.TspVisibility(statusPage: true),
plaintextMessage: """
We were unable to automatically build your project.
Set up a manual build command.
"""
));
/// <summary>
/// Returns a build script that can be run upon autobuild failure.
/// </summary>
/// <returns>
/// A build script that reports that we could not automatically detect a suitable build method.
/// </returns>
protected BuildScript AutobuildFailure() =>
BuildScript.Create(actions =>
{
Log(Severity.Error, "Could not auto-detect a suitable build method");
AutobuildFailureDiagnostic();
return 1;
});
/// <summary>
/// Constructs a <see cref="BuildOutputHandler" /> which uses the <see cref="DiagnosticClassifier" />
/// to classify build output. All data also gets written to <paramref name="writer" />.
/// </summary>
/// <param name="writer">
/// The <see cref="TextWriter" /> to which the build output would have normally been written to.
/// This is normally <see cref="Console.Out" /> or <see cref="Console.Error" />.
/// </param>
/// <returns>The constructed <see cref="BuildOutputHandler" />.</returns>
protected BuildOutputHandler BuildOutputHandler(TextWriter writer) => new(data =>
{
if (data is not null)
{
writer.WriteLine(data);
DiagnosticClassifier.ClassifyLine(data);
}
});
/// <summary>
/// Value of CODEQL_EXTRACTOR_<LANG>_ROOT environment variable.
/// </summary>

35
csharp/autobuilder/Semmle.Autobuild.Shared/BuildActions.cs
View File

@@ -11,26 +11,11 @@ using System.Runtime.InteropServices;
namespace Semmle.Autobuild.Shared
{
public delegate void BuildOutputHandler(string? data);
/// <summary>
/// Wrapper around system calls so that the build scripts can be unit-tested.
/// </summary>
public interface IBuildActions
{
/// <summary>
/// Runs a process, captures its output, and provides it asynchronously.
/// </summary>
/// <param name="exe">The exe to run.</param>
/// <param name="args">The other command line arguments.</param>
/// <param name="workingDirectory">The working directory (<code>null</code> for current directory).</param>
/// <param name="env">Additional environment variables.</param>
/// <param name="onOutput">A handler for stdout output.</param>
/// <param name="onError">A handler for stderr output.</param>
/// <returns>The process exit code.</returns>
int RunProcess(string exe, string args, string? workingDirectory, IDictionary<string, string>? env, BuildOutputHandler onOutput, BuildOutputHandler onError);
/// <summary>
/// Runs a process and captures its output.
/// </summary>
@@ -197,26 +182,6 @@ namespace Semmle.Autobuild.Shared
return pi;
}
int IBuildActions.RunProcess(string exe, string args, string? workingDirectory, System.Collections.Generic.IDictionary<string, string>? env, BuildOutputHandler onOutput, BuildOutputHandler onError)
{
var pi = GetProcessStartInfo(exe, args, workingDirectory, env, true);
using var p = new Process
{
StartInfo = pi
};
p.StartInfo.RedirectStandardError = true;
p.OutputDataReceived += new DataReceivedEventHandler((sender, e) => onOutput(e.Data));
p.ErrorDataReceived += new DataReceivedEventHandler((sender, e) => onError(e.Data));
p.Start();
p.BeginErrorReadLine();
p.BeginOutputReadLine();
p.WaitForExit();
return p.ExitCode;
}
int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory, IDictionary<string, string>? environment)
{
var pi = GetProcessStartInfo(cmd, args, workingDirectory, environment, false);

37
csharp/autobuilder/Semmle.Autobuild.Shared/BuildCommandAutoRule.cs
View File

@@ -11,42 +11,22 @@ namespace Semmle.Autobuild.Shared
{
private readonly WithDotNet<AutobuildOptionsShared> withDotNet;
/// <summary>
/// A list of paths to files in the project directory which we classified as scripts.
/// </summary>
public IEnumerable<string> CandidatePaths { get; private set; }
/// <summary>
/// The path of the script we decided to run, if any.
/// </summary>
public string? ScriptPath { get; private set; }
public BuildCommandAutoRule(WithDotNet<AutobuildOptionsShared> withDotNet)
{
this.withDotNet = withDotNet;
this.CandidatePaths = new List<string>();
}
/// <summary>
/// A list of extensions that we consider to be for scripts on Windows.
/// </summary>
private readonly IEnumerable<string> winExtensions = new List<string> {
".bat",
".cmd",
".exe"
};
/// <summary>
/// A list of extensions that we consider to be for scripts on Linux.
/// </summary>
private readonly IEnumerable<string> linuxExtensions = new List<string> {
"",
".sh"
};
/// <summary>
/// A list of filenames without extensions that we think might be build scripts.
/// </summary>
private readonly IEnumerable<string> buildScripts = new List<string> {
"build"
};
@@ -55,25 +35,18 @@ namespace Semmle.Autobuild.Shared
{
builder.Log(Severity.Info, "Attempting to locate build script");
// a list of extensions for files that we consider to be scripts on the current platform
var extensions = builder.Actions.IsWindows() ? winExtensions : linuxExtensions;
// a list of combined base script names with the current platform's script extensions
// e.g. for Linux: build, build.sh
var scripts = buildScripts.SelectMany(s => extensions.Select(e => s + e));
// search through the files in the project directory for paths which end in one of
// the names given by `scripts`, then order them by their distance from the root
this.CandidatePaths = builder.Paths.Where(p => scripts.Any(p.Item1.ToLower().EndsWith)).OrderBy(p => p.Item2).Select(p => p.Item1);
// pick the first matching path, if there is one
this.ScriptPath = this.CandidatePaths.FirstOrDefault();
var scriptPath = builder.Paths.Where(p => scripts.Any(p.Item1.ToLower().EndsWith)).OrderBy(p => p.Item2).Select(p => p.Item1).FirstOrDefault();
if (this.ScriptPath is null)
if (scriptPath is null)
return BuildScript.Failure;
var chmod = new CommandBuilder(builder.Actions);
chmod.RunCommand("/bin/chmod", $"u+x {this.ScriptPath}");
chmod.RunCommand("/bin/chmod", $"u+x {scriptPath}");
var chmodScript = builder.Actions.IsWindows() ? BuildScript.Success : BuildScript.Try(chmod.Script);
var dir = builder.Actions.GetDirectoryName(this.ScriptPath);
var dir = builder.Actions.GetDirectoryName(scriptPath);
// A specific .NET Core version may be required
return chmodScript & withDotNet(builder, environment =>
@@ -85,7 +58,7 @@ namespace Semmle.Autobuild.Shared
if (vsTools is not null)
command.CallBatFile(vsTools.Path);
command.RunCommand(this.ScriptPath);
command.RunCommand(scriptPath);
return command.Script;
});
}

96
csharp/autobuilder/Semmle.Autobuild.Shared/BuildScript.cs
View File

@@ -46,33 +46,6 @@ namespace Semmle.Autobuild.Shared
/// <returns>The exit code from this build script.</returns>
public abstract int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, out IList<string> stdout);
/// <summary>
/// Runs this build command.
/// </summary>
/// <param name="actions">
/// The interface used to implement the build actions.
/// </param>
/// <param name="startCallback">
/// A call back that is called every time a new process is started. The
/// argument to the call back is a textual representation of the process.
/// </param>
/// <param name="exitCallBack">
/// A call back that is called every time a new process exits. The first
/// argument to the call back is the exit code, and the second argument is
/// an exit message.
/// </param>
/// <param name="onOutput">
/// A handler for data read from stdout.
/// </param>
/// <param name="onError">
/// A handler for data read from stderr.
/// </param>
/// <returns>The exit code from this build script.</returns>
public abstract int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, BuildOutputHandler onOutput, BuildOutputHandler onError);
/// <summary>
/// A build script which executes an external program or script.
/// </summary>
private class BuildCommand : BuildScript
{
private readonly string exe, arguments;
@@ -137,29 +110,8 @@ namespace Semmle.Autobuild.Shared
return ret;
}
public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, BuildOutputHandler onOutput, BuildOutputHandler onError)
{
startCallback(this.ToString(), silent);
var ret = 1;
var retMessage = "";
try
{
ret = actions.RunProcess(exe, arguments, workingDirectory, environment, onOutput, onError);
}
catch (Exception ex)
when (ex is System.ComponentModel.Win32Exception || ex is FileNotFoundException)
{
retMessage = ex.Message;
}
exitCallBack(ret, retMessage, silent);
return ret;
}
}
/// <summary>
/// A build script which runs a C# function.
/// </summary>
private class ReturnBuildCommand : BuildScript
{
private readonly Func<IBuildActions, int> func;
@@ -175,13 +127,8 @@ namespace Semmle.Autobuild.Shared
stdout = Array.Empty<string>();
return func(actions);
}
public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, BuildOutputHandler onOutput, BuildOutputHandler onError) => func(actions);
}
/// <summary>
/// Allows two build scripts to be composed sequentially.
/// </summary>
private class BindBuildScript : BuildScript
{
private readonly BuildScript s1;
@@ -228,32 +175,6 @@ namespace Semmle.Autobuild.Shared
stdout = @out;
return ret2;
}
public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, BuildOutputHandler onOutput, BuildOutputHandler onError)
{
int ret1;
if (s2a is not null)
{
var stdout1 = new List<string>();
var onOutputWrapper = new BuildOutputHandler(data =>
{
if (data is not null)
stdout1.Add(data);
onOutput(data);
});
ret1 = s1.Run(actions, startCallback, exitCallBack, onOutputWrapper, onError);
return s2a(stdout1, ret1).Run(actions, startCallback, exitCallBack, onOutput, onError);
}
if (s2b is not null)
{
ret1 = s1.Run(actions, startCallback, exitCallBack, onOutput, onError);
return s2b(ret1).Run(actions, startCallback, exitCallBack, onOutput, onError);
}
throw new InvalidOperationException("Unexpected error");
}
}
/// <summary>
@@ -339,23 +260,6 @@ namespace Semmle.Autobuild.Shared
/// </summary>
public static BuildScript Try(BuildScript s) => s | Success;
/// <summary>
/// Creates a build script that runs the build script <paramref name="s" />. If
/// running <paramref name="s" /> fails, <paramref name="k" /> is invoked with
/// the exit code.
/// </summary>
/// <param name="s">The build script to run.</param>
/// <param name="k">
/// The callback that is invoked if <paramref name="s" /> failed.
/// </param>
/// <returns>The build script which implements this.</returns>
public static BuildScript OnFailure(BuildScript s, Action<int> k) =>
new BindBuildScript(s, ret => Create(actions =>
{
if (!Succeeded(ret)) k(ret);
return ret;
}));
/// <summary>
/// Creates a build script that deletes the given directory.
/// </summary>

96
csharp/autobuilder/Semmle.Autobuild.Shared/DiagnosticClassifier.cs
View File

@@ -1,96 +0,0 @@
using System.Collections.Generic;
using System.Text.RegularExpressions;
using Semmle.Util;
namespace Semmle.Autobuild.Shared
{
/// <summary>
/// Direct results result from the successful application of a <see cref="DiagnosticRule" />,
/// which can later be converted to a corresponding <see cref="DiagnosticMessage" />.
/// </summary>
public interface IDiagnosticsResult
{
/// <summary>
/// Produces a <see cref="DiagnosticMessage" /> corresponding to this result.
/// </summary>
/// <param name="builder">
/// The autobuilder to use for constructing the base <see cref="DiagnosticMessage" />.
/// </param>
/// <param name="severity">
/// An optional severity value which overrides the default severity of the diagnostic.
/// </param>
/// <returns>The <see cref="DiagnosticMessage" /> corresponding to this result.</returns>
DiagnosticMessage ToDiagnosticMessage<T>(Autobuilder<T> builder, DiagnosticMessage.TspSeverity? severity = null) where T : AutobuildOptionsShared;
}
public class DiagnosticRule
{
/// <summary>
/// The pattern against which this rule matches build output.
/// </summary>
public Regex Pattern { get; }
/// <summary>
/// Constructs a diagnostic rule for the given <paramref name="pattern" />.
/// </summary>
/// <param name="pattern"></param>
public DiagnosticRule(Regex pattern)
{
this.Pattern = pattern;
}
/// <summary>
/// Constructs a diagnostic rule for the given regular expression <paramref name="pattern" />.
/// </summary>
/// <param name="pattern"></param>
public DiagnosticRule(string pattern)
{
this.Pattern = new Regex(pattern, RegexOptions.Compiled);
}
/// <summary>
/// Used by a <see cref="DiagnosticClassifier" /> <paramref name="classifier" /> to
/// signal that the rule has matched some build output with <paramref name="match" />.
/// </summary>
/// <param name="classifier">The classifier which is firing the rule.</param>
/// <param name="match">The <see cref="Match" /> that resulted from applying the rule.</param>
public virtual void Fire(DiagnosticClassifier classifier, Match match) { }
}
public class DiagnosticClassifier
{
private readonly List<DiagnosticRule> rules;
public readonly List<IDiagnosticsResult> Results;
public DiagnosticClassifier()
{
this.rules = new List<DiagnosticRule>();
this.Results = new List<IDiagnosticsResult>();
}
/// <summary>
/// Adds <paramref name="rule" /> to this classifier.
/// </summary>
/// <param name="rule">The rule to add.</param>
protected void AddRule(DiagnosticRule rule)
{
this.rules.Add(rule);
}
/// <summary>
/// Applies all of this classifier's rules to <paramref name="line" /> to see which match.
/// </summary>
/// <param name="line">The line to which the rules should be applied to.</param>
public void ClassifyLine(string line)
{
this.rules.ForEach(rule =>
{
var match = rule.Pattern.Match(line);
if (match.Success)
{
rule.Fire(this, match);
}
});
}
}
}

13
csharp/autobuilder/Semmle.Autobuild.Shared/EnvVars.cs
View File

@@ -1,13 +0,0 @@
using Semmle.Util;
namespace Semmle.Autobuild.Shared
{
public static class EnvVars
{
public const string Platform = "CODEQL_PLATFORM";
public static string Root(Language language) => $"CODEQL_EXTRACTOR_{language.UpperCaseName}_ROOT";
public static string TrapDir(Language language) => $"CODEQL_EXTRACTOR_{language.UpperCaseName}_TRAP_DIR";
public static string SourceArchiveDir(Language language) => $"CODEQL_EXTRACTOR_{language.UpperCaseName}_SOURCE_ARCHIVE_DIR";
public static string DiagnosticDir(Language language) => $"CODEQL_EXTRACTOR_{language.UpperCaseName}_DIAGNOSTIC_DIR";
}
}

2
csharp/extractor/Semmle.Util/Language.cs → csharp/autobuilder/Semmle.Autobuild.Shared/Language.cs
View File

@@ -1,4 +1,4 @@
namespace Semmle.Util
namespace Semmle.Autobuild.Shared
{
public sealed class Language
{

65
csharp/autobuilder/Semmle.Autobuild.Shared/MarkdownUtil.cs
View File

@@ -1,65 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace Semmle.Autobuild.Shared
{
public static class MarkdownUtil
{
/// <summary>
/// Formats items as markdown inline code.
/// </summary>
/// <returns>A function which formats items as markdown inline code.</returns>
public static readonly Func<string, string> CodeFormatter = item => $"`{item}`";
/// <summary>
/// Formats the string as a markdown link.
/// </summary>
/// <param name="link">The URL for the link.</param>
/// <param name="title">The text that is displayed.</param>
/// <returns>A string containing a markdown-formatted link.</returns>
public static string ToMarkdownLink(this string link, string title) => $"[{title}]({link})";
/// <summary>
/// Renders <see cref="projects" /> as a markdown list of the project paths.
/// </summary>
/// <param name="projects">
/// The list of projects whose paths should be rendered as a markdown list.
/// </param>
/// <param name="limit">The maximum number of items to include in the list.</param>
/// <returns>Returns the markdown list as a string.</returns>
public static string ToMarkdownList(this IEnumerable<IProjectOrSolution> projects, int? limit = null)
{
return projects.ToMarkdownList(p => $"`{p.FullPath}`", limit);
}
/// <summary>
/// Renders <see cref="items" /> as a markdown list.
/// </summary>
/// <typeparam name="T">The item type.</typeparam>
/// <param name="items">The list that should be formatted as a markdown list.</param>
/// <param name="formatter">A function which converts individual items into a string representation.</param>
/// <param name="limit">The maximum number of items to include in the list.</param>
/// <returns>Returns the markdown list as a string.</returns>
public static string ToMarkdownList<T>(this IEnumerable<T> items, Func<T, string> formatter, int? limit = null)
{
var sb = new StringBuilder();
// if there is a limit, take at most that many items from the start of the list
var list = limit is not null ? items.Take(limit.Value) : items;
sb.Append(string.Join('\n', list.Select(item => $"- {formatter(item)}")));
// if there were more items than allowed in the list, add an extra item indicating
// how many more items there were
var length = items.Count();
if (limit is not null && length > limit)
{
sb.Append($"\n- and {length - limit} more. View the CodeQL logs for a full list.");
}
return sb.ToString();
}
}
}

16
csharp/autobuilder/Semmle.Autobuild.Shared/MsBuildRule.cs
View File

@@ -1,6 +1,7 @@
using Semmle.Util.Logging;
using System.Collections.Generic;
using System;
using System.Linq;
using System.Runtime.InteropServices;
namespace Semmle.Autobuild.Shared
{
@@ -30,11 +31,6 @@ namespace Semmle.Autobuild.Shared
/// </summary>
public class MsBuildRule : IBuildRule<AutobuildOptionsShared>
{
/// <summary>
/// A list of solutions or projects which failed to build.
/// </summary>
public readonly List<IProjectOrSolution> FailedProjectsOrSolutions = new();
public BuildScript Analyse(IAutobuilder<AutobuildOptionsShared> builder, bool auto)
{
if (!builder.ProjectsOrSolutionsToBuild.Any())
@@ -132,13 +128,7 @@ namespace Semmle.Autobuild.Shared
command.Argument(builder.Options.MsBuildArguments);
// append the build script which invokes msbuild to the overall build script `ret`;
// we insert a check that building the current project or solution was successful:
// if it was not successful, we add it to `FailedProjectsOrSolutions`
ret &= BuildScript.OnFailure(command.Script, ret =>
{
FailedProjectsOrSolutions.Add(projectOrSolution);
});
ret &= command.Script;
}
return ret;

1
csharp/autobuilder/Semmle.Autobuild.Shared/ProjectOrSolution.cs
View File

@@ -1,6 +1,5 @@
using System.Collections.Generic;
using System.Linq;
using Semmle.Util;
namespace Semmle.Autobuild.Shared
{

1
csharp/extractor/Semmle.Util/Semmle.Util.csproj
View File

@@ -15,7 +15,6 @@
<ItemGroup>
<PackageReference Include="Mono.Posix.NETStandard" Version="1.0.0" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.2" />
</ItemGroup>
</Project>

224
csharp/extractor/Semmle.Util/ToolStatusPage.cs
View File

@@ -1,224 +0,0 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
using Newtonsoft.Json.Serialization;
namespace Semmle.Util
{
/// <summary>
/// Represents diagnostic messages for the tool status page.
/// </summary>
public class DiagnosticMessage
{
/// <summary>
/// Represents sources of diagnostic messages.
/// </summary>
public class TspSource
{
/// <summary>
/// An identifier under which it makes sense to group this diagnostic message.
/// This is used to build the SARIF reporting descriptor object.
/// </summary>
public string Id { get; }
/// <summary>
/// Display name for the ID. This is used to build the SARIF reporting descriptor object.
/// </summary>
public string Name { get; }
/// <summary>
/// Name of the CodeQL extractor. This is used to identify which tool component the reporting descriptor object should be nested under in SARIF.
/// </summary>
public string? ExtractorName { get; }
public TspSource(string id, string name, string? extractorName = null)
{
Id = id;
Name = name;
ExtractorName = extractorName;
}
}
/// <summary>
/// Enumerates severity levels for diagnostics.
/// </summary>
[JsonConverter(typeof(StringEnumConverter), typeof(CamelCaseNamingStrategy))]
public enum TspSeverity
{
Note,
Warning,
Error
}
/// <summary>
/// Stores flags indicating where the diagnostic should be displayed.
/// </summary>
public class TspVisibility
{
/// <summary>
/// True if the message should be displayed on the status page (defaults to false).
/// </summary>
public bool? StatusPage { get; }
/// <summary>
/// True if the message should be counted in the diagnostics summary table printed by
/// <c>codeql database analyze</c> (defaults to false).
/// </summary>
public bool? CLISummaryTable { get; }
/// <summary>
/// True if the message should be sent to telemetry (defaults to false).
/// </summary>
public bool? Telemetry { get; }
public TspVisibility(bool? statusPage = null, bool? cliSummaryTable = null, bool? telemetry = null)
{
this.StatusPage = statusPage;
this.CLISummaryTable = cliSummaryTable;
this.Telemetry = telemetry;
}
}
/// <summary>
/// Represents source code locations for diagnostic messages.
/// </summary>
public class TspLocation
{
/// <summary>
/// Path to the affected file if appropriate, relative to the source root.
/// </summary>
public string? File { get; }
/// <summary>
/// The line where the range to which the diagnostic relates to starts.
/// </summary>
public int? StartLine { get; }
/// <summary>
/// The column where the range to which the diagnostic relates to starts.
/// </summary>
public int? StartColumn { get; }
/// <summary>
/// The line where the range to which the diagnostic relates to ends.
/// </summary>
public int? EndLine { get; }
/// <summary>
/// The column where the range to which the diagnostic relates to ends.
/// </summary>
public int? EndColumn { get; }
public TspLocation(string? file = null, int? startLine = null, int? startColumn = null, int? endLine = null, int? endColumn = null)
{
this.File = file;
this.StartLine = startLine;
this.StartColumn = startColumn;
this.EndLine = endLine;
this.EndColumn = endColumn;
}
}
/// <summary>
/// ISO 8601 timestamp.
/// </summary>
public string Timestamp { get; }
/// <summary>
/// The source of the diagnostic message.
/// </summary>
public TspSource Source { get; }
/// <summary>
/// GitHub flavored Markdown formatted message. Should include inline links to any help pages.
/// </summary>
public string? MarkdownMessage { get; }
/// <summary>
/// Plain text message. Used by components where the string processing needed to support
/// Markdown is cumbersome.
/// </summary>
public string? PlaintextMessage { get; }
/// <summary>
/// List of help links intended to supplement <see cref="PlaintextMessage" />.
/// </summary>
public List<string> HelpLinks { get; }
/// <summary>
/// SARIF severity.
/// </summary>
public TspSeverity? Severity { get; }
/// <summary>
/// If true, then this message won't be presented to users.
/// </summary>
public bool Internal { get; }
public TspVisibility Visibility { get; }
public TspLocation Location { get; }
/// <summary>
/// Structured metadata about the diagnostic message.
/// </summary>
public Dictionary<string, object> Attributes { get; }
public DiagnosticMessage(
Language language, string id, string name, string? markdownMessage = null, string? plaintextMessage = null,
TspVisibility? visibility = null, TspLocation? location = null, TspSeverity? severity = TspSeverity.Error,
DateTime? timestamp = null, bool? intrnl = null
)
{
this.Source = new TspSource(
id: $"{language.UpperCaseName.ToLower()}/autobuilder/{id}",
name: name,
extractorName: language.UpperCaseName.ToLower()
);
this.Timestamp = (timestamp ?? DateTime.UtcNow).ToString("o", CultureInfo.InvariantCulture);
this.HelpLinks = new List<string>();
this.Attributes = new Dictionary<string, object>();
this.Severity = severity;
this.Visibility = visibility ?? new TspVisibility(statusPage: true);
this.Location = location ?? new TspLocation();
this.Internal = intrnl ?? false;
this.MarkdownMessage = markdownMessage;
this.PlaintextMessage = plaintextMessage;
}
}
/// <summary>
/// A wrapper around an underlying <see cref="StreamWriter" /> which allows
/// <see cref="DiagnosticMessage" /> objects to be serialized to it.
/// </summary>
public sealed class DiagnosticsStream : IDisposable
{
private readonly JsonSerializer serializer;
private readonly StreamWriter writer;
/// <summary>
/// Initialises a new <see cref="DiagnosticsStream" /> for a file at <paramref name="path" />.
/// </summary>
/// <param name="path">The path to the file that should be created.</param>
public DiagnosticsStream(string path)
{
this.writer = File.CreateText(path);
var contractResolver = new DefaultContractResolver
{
NamingStrategy = new CamelCaseNamingStrategy()
};
serializer = new JsonSerializer
{
ContractResolver = contractResolver,
NullValueHandling = NullValueHandling.Ignore
};
}
/// <summary>
/// Adds <paramref name="message" /> as a new diagnostics entry.
/// </summary>
/// <param name="message">The diagnostics entry to add.</param>
public void AddEntry(DiagnosticMessage message)
{
serializer.Serialize(writer, message);
writer.Flush();
}
/// <summary>
/// Releases all resources used by the <see cref="DiagnosticsStream" /> object.
/// </summary>
public void Dispose()
{
writer.Dispose();
}
}
}

32
csharp/ql/integration-tests/all-platforms/diag_dotnet_incompatible/diagnostics.expected
View File

@@ -1,32 +0,0 @@
{
"attributes": {},
"helpLinks": [],
"internal": false,
"location": {},
"markdownMessage": "CodeQL found some projects which cannot be built with .NET Core:\n\n- `test.csproj`",
"severity": "warning",
"source": {
"extractorName": "csharp",
"id": "csharp/autobuilder/dotnet-incompatible-projects",
"name": "Some projects are incompatible with .NET Core"
},
"visibility": {
"statusPage": true
}
}
{
"attributes": {},
"helpLinks": [],
"internal": false,
"location": {},
"markdownMessage": "CodeQL was unable to build the following projects using MSBuild:\n\n- `test.csproj`\n\nSet up a [manual build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
"severity": "error",
"source": {
"extractorName": "csharp",
"id": "csharp/autobuilder/msbuild-build-failure",
"name": "Some projects or solutions failed to build using MSBuild"
},
"visibility": {
"statusPage": true
}
}

66
csharp/ql/integration-tests/all-platforms/diag_dotnet_incompatible/test.csproj
View File

@@ -1,66 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
<ProjectGuid>{F70CE6B6-1735-4AD2-B1EB-B91FCD1012D1}</ProjectGuid>
<OutputType>Library</OutputType>
<AppDesignerFolder>Properties</AppDesignerFolder>
<RootNamespace>Example</RootNamespace>
<AssemblyName>Example</AssemblyName>
<TargetFrameworkVersion>v4.0</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<TargetFrameworkProfile />
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<DebugSymbols>true</DebugSymbols>
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>bin\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>bin\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<PropertyGroup />
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x86'">
<DebugSymbols>true</DebugSymbols>
<OutputPath>bin\x86\Debug\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<DebugType>full</DebugType>
<PlatformTarget>x86</PlatformTarget>
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|x86'">
<OutputPath>bin\x86\Release\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<Optimize>true</Optimize>
<DebugType>pdbonly</DebugType>
<PlatformTarget>x86</PlatformTarget>
<ErrorReport>prompt</ErrorReport>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
</PropertyGroup>
<ItemGroup>
<Reference Include="System" />
</ItemGroup>
<ItemGroup>
<Compile Include="Program.cs" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">
</Target>
<Target Name="AfterBuild">
</Target>
-->
</Project>

5
csharp/ql/integration-tests/all-platforms/diag_dotnet_incompatible/test.py
View File

@@ -1,5 +0,0 @@
from create_database_utils import *
from diagnostics_test_utils import *
run_codeql_database_create([], db=None, lang="csharp", runFunction=runUnsuccessfully)
check_diagnostics()

32
csharp/ql/integration-tests/all-platforms/diag_missing_project_files/diagnostics.expected
View File

@@ -1,32 +0,0 @@
{
"attributes": {},
"helpLinks": [],
"internal": false,
"location": {},
"markdownMessage": "CodeQL was unable to build the following projects using MSBuild:\n\n- `test.sln`\n\nSet up a [manual build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
"severity": "error",
"source": {
"extractorName": "csharp",
"id": "csharp/autobuilder/msbuild-build-failure",
"name": "Some projects or solutions failed to build using MSBuild"
},
"visibility": {
"statusPage": true
}
}
{
"attributes": {},
"helpLinks": [],
"internal": false,
"location": {},
"markdownMessage": "Some project files were not found when CodeQL built your project:\n\n- `Example.csproj`\n- `Example.Test.csproj`\n\nThis may lead to subsequent failures. You can check for common causes for missing project files:\n\n- Ensure that the project is built using the [intended operating system](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idruns-on) and that filenames on case-sensitive platforms are correctly specified.\n- If your repository uses Git submodules, ensure that those are [checked out](https://github.com/actions/checkout#usage) before the CodeQL action is run.\n- If you auto-generate some project files as part of your build process, ensure that these are generated before the CodeQL action is run.",
"severity": "error",
"source": {
"extractorName": "csharp",
"id": "csharp/autobuilder/missing-project-files",
"name": "Missing project files"
},
"visibility": {
"statusPage": true
}
}

5
csharp/ql/integration-tests/all-platforms/diag_missing_project_files/test.py
View File

@@ -1,5 +0,0 @@
from create_database_utils import *
from diagnostics_test_utils import *
run_codeql_database_create([], db=None, lang="csharp", runFunction=runUnsuccessfully)
check_diagnostics()

26
csharp/ql/integration-tests/all-platforms/diag_missing_project_files/test.sln
View File

@@ -1,26 +0,0 @@
Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 2012
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Example", "Example.csproj", "{F70CE6B6-1735-4AD2-B1EB-B91FCD1012D1}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Example.Test", "Example.Test.csproj", "{F4587B5F-9918-4079-9291-5A08CD9761FB}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|x86 = Debug|x86
Release|x86 = Release|x86
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{F70CE6B6-1735-4AD2-B1EB-B91FCD1012D1}.Debug|x86.ActiveCfg = Debug|x86
{F70CE6B6-1735-4AD2-B1EB-B91FCD1012D1}.Debug|x86.Build.0 = Debug|x86
{F70CE6B6-1735-4AD2-B1EB-B91FCD1012D1}.Release|x86.ActiveCfg = Release|x86
{F70CE6B6-1735-4AD2-B1EB-B91FCD1012D1}.Release|x86.Build.0 = Release|x86
{F4587B5F-9918-4079-9291-5A08CD9761FB}.Debug|x86.ActiveCfg = Debug|x86
{F4587B5F-9918-4079-9291-5A08CD9761FB}.Debug|x86.Build.0 = Debug|x86
{F4587B5F-9918-4079-9291-5A08CD9761FB}.Release|x86.ActiveCfg = Release|x86
{F4587B5F-9918-4079-9291-5A08CD9761FB}.Release|x86.Build.0 = Release|x86
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
EndGlobal

48
csharp/ql/integration-tests/all-platforms/diag_missing_xamarin_sdk/diagnostics.expected
View File

@@ -1,48 +0,0 @@
{
"attributes": {},
"helpLinks": [],
"internal": false,
"location": {},
"markdownMessage": "CodeQL was unable to build the following projects using .NET Core:\n\n- `test.csproj`\n\nSet up a [manual build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
"severity": "error",
"source": {
"extractorName": "csharp",
"id": "csharp/autobuilder/dotnet-build-failure",
"name": "Some projects or solutions failed to build using .NET Core"
},
"visibility": {
"statusPage": true
}
}
{
"attributes": {},
"helpLinks": [],
"internal": false,
"location": {},
"markdownMessage": "CodeQL was unable to build the following projects using MSBuild:\n\n- `test.csproj`\n\nSet up a [manual build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
"severity": "error",
"source": {
"extractorName": "csharp",
"id": "csharp/autobuilder/msbuild-build-failure",
"name": "Some projects or solutions failed to build using MSBuild"
},
"visibility": {
"statusPage": true
}
}
{
"attributes": {},
"helpLinks": [],
"internal": false,
"location": {},
"markdownMessage": "[Configure your workflow](https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-xamarin-applications) for this SDK before running CodeQL.",
"severity": "error",
"source": {
"extractorName": "csharp",
"id": "csharp/autobuilder/missing-xamarin-ios-sdk",
"name": "Missing Xamarin SDK for iOS"
},
"visibility": {
"statusPage": true
}
}

11
csharp/ql/integration-tests/all-platforms/diag_missing_xamarin_sdk/test.csproj
View File

@@ -1,11 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net7.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<LanguageTargets>$(MSBuildExtensionsPath)\Xamarin\iOS\Xamarin.iOS.CSharp.targets</LanguageTargets>
</PropertyGroup>
</Project>

5
csharp/ql/integration-tests/all-platforms/diag_missing_xamarin_sdk/test.py
View File

@@ -1,5 +0,0 @@
from create_database_utils import *
from diagnostics_test_utils import *
run_codeql_database_create([], db=None, lang="csharp", runFunction=runUnsuccessfully)
check_diagnostics()

3
csharp/ql/integration-tests/posix-only/diag_autobuild_script/build.sh
View File

@@ -1,3 +0,0 @@
#!/bin/sh
exit 1

32
csharp/ql/integration-tests/posix-only/diag_autobuild_script/diagnostics.expected
View File

@@ -1,32 +0,0 @@
{
"attributes": {},
"helpLinks": [],
"internal": false,
"location": {},
"markdownMessage": "CodeQL attempted to build your project using a script located at `build.sh`, which failed. Set up a [manual build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
"severity": "error",
"source": {
"extractorName": "csharp",
"id": "csharp/autobuilder/script-failure",
"name": "Unable to build project using build script"
},
"visibility": {
"statusPage": true
}
}
{
"attributes": {},
"helpLinks": [],
"internal": false,
"location": {},
"markdownMessage": "CodeQL could not find any project or solution files in your repository. Set up a [manual build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
"severity": "error",
"source": {
"extractorName": "csharp",
"id": "csharp/autobuilder/no-projects-or-solutions",
"name": "No project or solutions files found"
},
"visibility": {
"statusPage": true
}
}

5
csharp/ql/integration-tests/posix-only/diag_autobuild_script/test.py
View File

@@ -1,5 +0,0 @@
from create_database_utils import *
from diagnostics_test_utils import *
run_codeql_database_create([], db=None, lang="csharp", runFunction=runUnsuccessfully)
check_diagnostics()

3
csharp/ql/integration-tests/posix-only/diag_multiple_scripts/build.sh
View File

@@ -1,3 +0,0 @@
#!/bin/sh
exit 1

32
csharp/ql/integration-tests/posix-only/diag_multiple_scripts/diagnostics.expected
View File

@@ -1,32 +0,0 @@
{
"attributes": {},
"helpLinks": [],
"internal": false,
"location": {},
"markdownMessage": "CodeQL could not find any project or solution files in your repository. Set up a [manual build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
"severity": "error",
"source": {
"extractorName": "csharp",
"id": "csharp/autobuilder/no-projects-or-solutions",
"name": "No project or solutions files found"
},
"visibility": {
"statusPage": true
}
}
{
"attributes": {},
"helpLinks": [],
"internal": false,
"location": {},
"markdownMessage": "CodeQL found multiple potential build scripts for your project and attempted to run `build.sh`, which failed. This may not be the right build script for your project. Set up a [manual build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
"severity": "error",
"source": {
"extractorName": "csharp",
"id": "csharp/autobuilder/multiple-build-scripts",
"name": "There are multiple potential build scripts"
},
"visibility": {
"statusPage": true
}
}

3
csharp/ql/integration-tests/posix-only/diag_multiple_scripts/scripts/build.sh
View File

@@ -1,3 +0,0 @@
#!/bin/sh
exit 1

5
csharp/ql/integration-tests/posix-only/diag_multiple_scripts/test.py
View File

@@ -1,5 +0,0 @@
from create_database_utils import *
from diagnostics_test_utils import *
run_codeql_database_create([], db=None, lang="csharp", runFunction=runUnsuccessfully)
check_diagnostics()

1
csharp/ql/integration-tests/windows-only/diag_autobuild_script/build.bat
View File

@@ -1 +0,0 @@
exit /b 1

32
csharp/ql/integration-tests/windows-only/diag_autobuild_script/diagnostics.expected
View File

@@ -1,32 +0,0 @@
{
"attributes": {},
"helpLinks": [],
"internal": false,
"location": {},
"markdownMessage": "CodeQL attempted to build your project using a script located at `build.bat`, which failed. Set up a [manual build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
"severity": "error",
"source": {
"extractorName": "csharp",
"id": "csharp/autobuilder/script-failure",
"name": "Unable to build project using build script"
},
"visibility": {
"statusPage": true
}
}
{
"attributes": {},
"helpLinks": [],
"internal": false,
"location": {},
"markdownMessage": "CodeQL could not find any project or solution files in your repository. Set up a [manual build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
"severity": "error",
"source": {
"extractorName": "csharp",
"id": "csharp/autobuilder/no-projects-or-solutions",
"name": "No project or solutions files found"
},
"visibility": {
"statusPage": true
}
}

5
csharp/ql/integration-tests/windows-only/diag_autobuild_script/test.py
View File

@@ -1,5 +0,0 @@
from create_database_utils import *
from diagnostics_test_utils import *
run_codeql_database_create([], db=None, lang="csharp", runFunction=runUnsuccessfully)
check_diagnostics()

1
csharp/ql/integration-tests/windows-only/diag_multiple_scripts/build.bat
View File

@@ -1 +0,0 @@
exit /b 1

32
csharp/ql/integration-tests/windows-only/diag_multiple_scripts/diagnostics.expected
View File

@@ -1,32 +0,0 @@
{
"attributes": {},
"helpLinks": [],
"internal": false,
"location": {},
"markdownMessage": "CodeQL could not find any project or solution files in your repository. Set up a [manual build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
"severity": "error",
"source": {
"extractorName": "csharp",
"id": "csharp/autobuilder/no-projects-or-solutions",
"name": "No project or solutions files found"
},
"visibility": {
"statusPage": true
}
}
{
"attributes": {},
"helpLinks": [],
"internal": false,
"location": {},
"markdownMessage": "CodeQL found multiple potential build scripts for your project and attempted to run `build.bat`, which failed. This may not be the right build script for your project. Set up a [manual build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages).",
"severity": "error",
"source": {
"extractorName": "csharp",
"id": "csharp/autobuilder/multiple-build-scripts",
"name": "There are multiple potential build scripts"
},
"visibility": {
"statusPage": true
}
}

1
csharp/ql/integration-tests/windows-only/diag_multiple_scripts/scripts/build.bat
View File

@@ -1 +0,0 @@
exit /b 1

5
csharp/ql/integration-tests/windows-only/diag_multiple_scripts/test.py
View File

@@ -1,5 +0,0 @@
from create_database_utils import *
from diagnostics_test_utils import *
run_codeql_database_create([], db=None, lang="csharp", runFunction=runUnsuccessfully)
check_diagnostics()