using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Semmle.Util;
using Semmle.Util.Logging;
namespace Semmle.Autobuild.Shared
{
///
/// A build rule analyses the files in "builder" and outputs a build script.
///
public interface IBuildRule where TAutobuildOptions : AutobuildOptionsShared
{
///
/// Analyse the files and produce a build script.
///
/// The files and options relating to the build.
/// Whether this build rule is being automatically applied.
BuildScript Analyse(IAutobuilder builder, bool auto);
}
///
/// A delegate used to wrap a build script in an environment where an appropriate
/// version of .NET Core is automatically installed.
///
public delegate BuildScript WithDotNet(IAutobuilder builder, Func?, BuildScript> f) where TAutobuildOptions : AutobuildOptionsShared;
///
/// Exception indicating that environment variables are missing or invalid.
///
public class InvalidEnvironmentException : Exception
{
public InvalidEnvironmentException(string m) : base(m) { }
}
public interface IAutobuilder where TAutobuildOptions : AutobuildOptionsShared
{
///
/// Full file paths of files found in the project directory, as well as
/// their distance from the project root folder. The list is sorted
/// by distance in ascending order.
///
IEnumerable<(string, int)> Paths { get; }
///
/// Gets all paths matching a particular filename, as well as
/// their distance from the project root folder. The list is sorted
/// by distance in ascending order.
///
/// The filename to find.
/// Possibly empty sequence of paths with the given filename.
IEnumerable<(string, int)> GetFilename(string name) =>
Paths.Where(p => Actions.GetFileName(p.Item1) == name);
///
/// List of project/solution files to build.
///
IList ProjectsOrSolutionsToBuild { get; }
///
/// Gets the supplied build configuration.
///
TAutobuildOptions Options { get; }
///
/// The set of build actions used during the autobuilder.
/// Could be real system operations, or a stub for testing.
///
IBuildActions Actions { get; }
///
/// A logger.
///
ILogger Logger { get; }
}
///
/// Main application logic, containing all data
/// gathered from the project and filesystem.
///
/// The overall design is intended to be extensible so that in theory,
/// it should be possible to add new build rules without touching this code.
///
public abstract class Autobuilder : IDisposable, IAutobuilder where TAutobuildOptions : AutobuildOptionsShared
{
///
/// Full file paths of files found in the project directory, as well as
/// their distance from the project root folder. The list is sorted
/// by distance in ascending order.
///
public IEnumerable<(string, int)> Paths => pathsLazy.Value;
private readonly Lazy> pathsLazy;
///
/// Gets a list of paths matching a set of extensions (including the "."),
/// as well as their distance from the project root folder.
/// The list is sorted by distance in ascending order.
///
/// The extensions to find.
/// The files matching the extension.
public IEnumerable<(string, int)> GetExtensions(params string[] extensions) =>
Paths.Where(p => extensions.Contains(Path.GetExtension(p.Item1)));
///
/// Holds if a given path, relative to the root of the source directory
/// was found.
///
/// The relative path.
/// True iff the path was found.
public bool HasRelativePath(string path) => HasPath(Actions.PathCombine(RootDirectory, path));
///
/// List of project/solution files to build.
///
public IList ProjectsOrSolutionsToBuild => projectsOrSolutionsToBuildLazy.Value;
private readonly Lazy> projectsOrSolutionsToBuildLazy;
///
/// Holds if a given path was found.
///
/// The path of the file.
/// True iff the path was found.
public bool HasPath(string path) => Paths.Any(p => path == p.Item1);
///
/// The root of the source directory.
///
private string RootDirectory => Options.RootDirectory;
///
/// Gets the supplied build configuration.
///
public TAutobuildOptions Options { get; }
///
/// The set of build actions used during the autobuilder.
/// Could be real system operations, or a stub for testing.
///
public IBuildActions Actions { get; }
private IEnumerable? FindFiles(string extension, Func> create)
{
var matchingFiles = GetExtensions(extension)
.Select(p => (ProjectOrSolution: create(p.Item1), DistanceFromRoot: p.Item2))
.Where(p => p.ProjectOrSolution.HasLanguage(this.Options.Language))
.ToArray();
if (matchingFiles.Length == 0)
return null;
return matchingFiles
.Where(f => f.DistanceFromRoot == matchingFiles[0].DistanceFromRoot)
.Select(f => f.ProjectOrSolution);
}
///
/// Find all the relevant files and picks the best
/// solution file and tools.
///
/// The command line options.
protected Autobuilder(IBuildActions actions, TAutobuildOptions options, DiagnosticClassifier diagnosticClassifier)
{
Actions = actions;
Options = options;
DiagnosticClassifier = diagnosticClassifier;
pathsLazy = new Lazy>(() => Actions.FindFiles(options.RootDirectory, options.SearchDepth));
projectsOrSolutionsToBuildLazy = new Lazy>(() =>
{
List? ret;
// First look for `.proj` files
ret = FindFiles(".proj", f => new Project(this, f))?.ToList();
if (ret is not null)
return ret;
// Then look for `.sln` files
ret = FindFiles(".sln", f => new Solution(this, f, false))?.ToList();
if (ret is not null)
return ret;
// Finally look for language specific project files, e.g. `.csproj` files
ret = FindFiles(this.Options.Language.ProjectExtension, f => new Project(this, f))?.ToList();
return ret ?? new List();
});
TrapDir = RequireEnvironmentVariable(EnvVars.TrapDir(this.Options.Language));
SourceArchiveDir = RequireEnvironmentVariable(EnvVars.SourceArchiveDir(this.Options.Language));
DiagnosticsDir = RequireEnvironmentVariable(EnvVars.DiagnosticDir(this.Options.Language));
this.diagnostics = actions.CreateDiagnosticsWriter(Path.Combine(DiagnosticsDir, $"autobuilder-{DateTime.UtcNow:yyyyMMddHHmm}-{Environment.ProcessId}.jsonc"));
}
///
/// Retrieves the value of an environment variable named or throws
/// an exception if no such environment variable has been set.
///
/// The name of the environment variable.
/// The value of the environment variable.
///
/// Thrown if the environment variable is not set.
///
protected string RequireEnvironmentVariable(string name)
{
return Actions.GetEnvironmentVariable(name) ??
throw new InvalidEnvironmentException($"The environment variable {name} has not been set.");
}
public string TrapDir { get; }
public string SourceArchiveDir { get; }
public string DiagnosticsDir { get; }
protected DiagnosticClassifier DiagnosticClassifier { get; }
private readonly ILogger logger = new ConsoleLogger(
VerbosityExtensions.ParseVerbosity(
Environment.GetEnvironmentVariable("CODEQL_VERBOSITY"),
logThreadId: false) ?? Verbosity.Info,
logThreadId: false);
public ILogger Logger => logger;
private readonly IDiagnosticsWriter diagnostics;
///
/// Makes relative to the root source directory.
///
/// The path which to make relative.
/// The relative path.
public string MakeRelative(string path)
{
return Path.GetRelativePath(this.RootDirectory, path);
}
///
/// Write to the diagnostics file.
///
/// The diagnostics entry to write.
public void AddDiagnostic(DiagnosticMessage diagnostic)
{
diagnostics.AddEntry(diagnostic);
}
///
/// Attempt to build this project.
///
/// The exit code, 0 for success and non-zero for failures.
public int AttemptBuild()
{
logger.LogInfo($"Working directory: {Options.RootDirectory}");
var script = GetBuildScript();
void startCallback(string s, bool silent)
{
logger.Log(silent ? Severity.Debug : Severity.Info, $"\nRunning {s}");
}
void exitCallback(int ret, string msg, bool silent)
{
logger.Log(silent ? Severity.Debug : Severity.Info, $"Exit code {ret}{(string.IsNullOrEmpty(msg) ? "" : $": {msg}")}");
}
var onOutput = BuildOutputHandler(Console.Out);
var onError = BuildOutputHandler(Console.Error);
var buildResult = script.Run(Actions, startCallback, exitCallback, onOutput, onError);
// if the build succeeded, all diagnostics we captured from the build output should be warnings;
// otherwise they should all be errors
var diagSeverity = buildResult == 0 ? DiagnosticMessage.TspSeverity.Warning : DiagnosticMessage.TspSeverity.Error;
this.DiagnosticClassifier.Results
.Select(result => result.ToDiagnosticMessage(this, diagSeverity))
.ForEach(AddDiagnostic);
return buildResult;
}
///
/// Returns the build script to use for this project.
///
public abstract BuildScript GetBuildScript();
///
/// Produces a diagnostic for the tool status page that we were unable to automatically
/// build the user's project and that they can manually specify a build command. This
/// can be overriden to implement more specific messages depending on the origin of
/// the failure.
///
protected virtual void AutobuildFailureDiagnostic() => AddDiagnostic(new DiagnosticMessage(
this.Options.Language,
"autobuild-failure",
"Unable to build project",
visibility: new DiagnosticMessage.TspVisibility(statusPage: true),
plaintextMessage: """
We were unable to automatically build your project.
Set up a manual build command.
"""
));
///
/// Returns a build script that can be run upon autobuild failure.
///
///
/// A build script that reports that we could not automatically detect a suitable build method.
///
protected BuildScript AutobuildFailure() =>
BuildScript.Create(actions =>
{
logger.LogError("Could not auto-detect a suitable build method");
AutobuildFailureDiagnostic();
return 1;
});
///
/// Constructs a which uses the
/// to classify build output. All data also gets written to .
///
///
/// The to which the build output would have normally been written to.
/// This is normally or .
///
/// The constructed .
protected BuildOutputHandler BuildOutputHandler(TextWriter writer) => new(data =>
{
if (data is not null)
{
writer.WriteLine(data);
DiagnosticClassifier.ClassifyLine(data);
}
});
public void Dispose()
{
Dispose(true);
GC.SuppressFinalize(this);
}
protected virtual void Dispose(bool disposing)
{
if (disposing)
{
diagnostics.Dispose();
}
}
}
}