using Semmle.Util; using Semmle.Util.Logging; using System; using System.Collections.Generic; using System.IO; using System.Linq; namespace Semmle.Autobuild.Shared { /// /// A build rule analyses the files in "builder" and outputs a build script. /// public interface IBuildRule where TAutobuildOptions : AutobuildOptionsShared { /// /// Analyse the files and produce a build script. /// /// The files and options relating to the build. /// Whether this build rule is being automatically applied. BuildScript Analyse(IAutobuilder builder, bool auto); } /// /// A delegate used to wrap a build script in an environment where an appropriate /// version of .NET Core is automatically installed. /// public delegate BuildScript WithDotNet(IAutobuilder builder, Func?, BuildScript> f) where TAutobuildOptions : AutobuildOptionsShared; /// /// Exception indicating that environment variables are missing or invalid. /// public class InvalidEnvironmentException : Exception { public InvalidEnvironmentException(string m) : base(m) { } } public interface IAutobuilder where TAutobuildOptions : AutobuildOptionsShared { /// /// Full file paths of files found in the project directory, as well as /// their distance from the project root folder. The list is sorted /// by distance in ascending order. /// IEnumerable<(string, int)> Paths { get; } /// /// Gets all paths matching a particular filename, as well as /// their distance from the project root folder. The list is sorted /// by distance in ascending order. /// /// The filename to find. /// Possibly empty sequence of paths with the given filename. IEnumerable<(string, int)> GetFilename(string name) => Paths.Where(p => Actions.GetFileName(p.Item1) == name); /// /// List of project/solution files to build. /// IList ProjectsOrSolutionsToBuild { get; } /// /// Gets the supplied build configuration. /// TAutobuildOptions Options { get; } /// /// The set of build actions used during the autobuilder. /// Could be real system operations, or a stub for testing. /// IBuildActions Actions { get; } /// /// Log a given build event to the console. /// /// The format string. /// Inserts to the format string. void Log(Severity severity, string format, params object[] args); /// /// Value of CODEQL_EXTRACTOR__ROOT environment variable. /// string? CodeQLExtractorLangRoot { get; } /// /// Value of CODEQL_PLATFORM environment variable. /// string? CodeQlPlatform { get; } } /// /// Main application logic, containing all data /// gathered from the project and filesystem. /// /// The overall design is intended to be extensible so that in theory, /// it should be possible to add new build rules without touching this code. /// public abstract class Autobuilder : IAutobuilder where TAutobuildOptions : AutobuildOptionsShared { /// /// Full file paths of files found in the project directory, as well as /// their distance from the project root folder. The list is sorted /// by distance in ascending order. /// public IEnumerable<(string, int)> Paths => pathsLazy.Value; private readonly Lazy> pathsLazy; /// /// Gets a list of paths matching a set of extensions (including the "."), /// as well as their distance from the project root folder. /// The list is sorted by distance in ascending order. /// /// The extensions to find. /// The files matching the extension. public IEnumerable<(string, int)> GetExtensions(params string[] extensions) => Paths.Where(p => extensions.Contains(Path.GetExtension(p.Item1))); /// /// Holds if a given path, relative to the root of the source directory /// was found. /// /// The relative path. /// True iff the path was found. public bool HasRelativePath(string path) => HasPath(Actions.PathCombine(RootDirectory, path)); /// /// List of project/solution files to build. /// public IList ProjectsOrSolutionsToBuild => projectsOrSolutionsToBuildLazy.Value; private readonly Lazy> projectsOrSolutionsToBuildLazy; /// /// Holds if a given path was found. /// /// The path of the file. /// True iff the path was found. public bool HasPath(string path) => Paths.Any(p => path == p.Item1); private void FindFiles(string dir, int depth, int maxDepth, IList<(string, int)> results) { foreach (var f in Actions.EnumerateFiles(dir)) { results.Add((f, depth)); } if (depth < maxDepth) { foreach (var d in Actions.EnumerateDirectories(dir)) { FindFiles(d, depth + 1, maxDepth, results); } } } /// /// The root of the source directory. /// private string RootDirectory => Options.RootDirectory; /// /// Gets the supplied build configuration. /// public TAutobuildOptions Options { get; } /// /// The set of build actions used during the autobuilder. /// Could be real system operations, or a stub for testing. /// public IBuildActions Actions { get; } private IEnumerable? FindFiles(string extension, Func> create) { var matchingFiles = GetExtensions(extension) .Select(p => (ProjectOrSolution: create(p.Item1), DistanceFromRoot: p.Item2)) .Where(p => p.ProjectOrSolution.HasLanguage(this.Options.Language)) .ToArray(); if (matchingFiles.Length == 0) return null; if (Options.AllSolutions) return matchingFiles.Select(p => p.ProjectOrSolution); return matchingFiles .Where(f => f.DistanceFromRoot == matchingFiles[0].DistanceFromRoot) .Select(f => f.ProjectOrSolution); } /// /// Find all the relevant files and picks the best /// solution file and tools. /// /// The command line options. protected Autobuilder(IBuildActions actions, TAutobuildOptions options, DiagnosticClassifier diagnosticClassifier) { Actions = actions; Options = options; DiagnosticClassifier = diagnosticClassifier; pathsLazy = new Lazy>(() => { var files = new List<(string, int)>(); FindFiles(options.RootDirectory, 0, options.SearchDepth, files); return files.OrderBy(f => f.Item2).ToArray(); }); projectsOrSolutionsToBuildLazy = new Lazy>(() => { List? ret; if (options.Solution.Any()) { ret = new List(); foreach (var solution in options.Solution) { if (actions.FileExists(solution)) ret.Add(new Solution(this, solution, true)); else Log(Severity.Error, $"The specified project or solution file {solution} was not found"); } return ret; } // First look for `.proj` files ret = FindFiles(".proj", f => new Project(this, f))?.ToList(); if (ret is not null) return ret; // Then look for `.sln` files ret = FindFiles(".sln", f => new Solution(this, f, false))?.ToList(); if (ret is not null) return ret; // Finally look for language specific project files, e.g. `.csproj` files ret = FindFiles(this.Options.Language.ProjectExtension, f => new Project(this, f))?.ToList(); return ret ?? new List(); }); CodeQLExtractorLangRoot = Actions.GetEnvironmentVariable(EnvVars.Root(this.Options.Language)); CodeQlPlatform = Actions.GetEnvironmentVariable(EnvVars.Platform); TrapDir = RequireEnvironmentVariable(EnvVars.TrapDir(this.Options.Language)); SourceArchiveDir = RequireEnvironmentVariable(EnvVars.SourceArchiveDir(this.Options.Language)); DiagnosticsDir = RequireEnvironmentVariable(EnvVars.DiagnosticDir(this.Options.Language)); this.diagnostics = actions.CreateDiagnosticsWriter(Path.Combine(DiagnosticsDir, $"autobuilder-{DateTime.UtcNow:yyyyMMddHHmm}.jsonc")); } /// /// Retrieves the value of an environment variable named or throws /// an exception if no such environment variable has been set. /// /// The name of the environment variable. /// The value of the environment variable. /// /// Thrown if the environment variable is not set. /// protected string RequireEnvironmentVariable(string name) { return Actions.GetEnvironmentVariable(name) ?? throw new InvalidEnvironmentException($"The environment variable {name} has not been set."); } public string TrapDir { get; } public string SourceArchiveDir { get; } public string DiagnosticsDir { get; } protected DiagnosticClassifier DiagnosticClassifier { get; } private readonly ILogger logger = new ConsoleLogger(Verbosity.Info); private readonly IDiagnosticsWriter diagnostics; /// /// Makes relative to the root source directory. /// /// The path which to make relative. /// The relative path. public string MakeRelative(string path) { return Path.GetRelativePath(this.RootDirectory, path); } /// /// Log a given build event to the console. /// /// The format string. /// Inserts to the format string. public void Log(Severity severity, string format, params object[] args) { logger.Log(severity, format, args); } /// /// Write to the diagnostics file. /// /// The diagnostics entry to write. public void AddDiagnostic(DiagnosticMessage diagnostic) { diagnostics.AddEntry(diagnostic); } /// /// Attempt to build this project. /// /// The exit code, 0 for success and non-zero for failures. public int AttemptBuild() { Log(Severity.Info, $"Working directory: {Options.RootDirectory}"); var script = GetBuildScript(); if (Options.IgnoreErrors) script |= BuildScript.Success; void startCallback(string s, bool silent) { Log(silent ? Severity.Debug : Severity.Info, $"\nRunning {s}"); } void exitCallback(int ret, string msg, bool silent) { Log(silent ? Severity.Debug : Severity.Info, $"Exit code {ret}{(string.IsNullOrEmpty(msg) ? "" : $": {msg}")}"); } var onOutput = BuildOutputHandler(Console.Out); var onError = BuildOutputHandler(Console.Error); var buildResult = script.Run(Actions, startCallback, exitCallback, onOutput, onError); // if the build succeeded, all diagnostics we captured from the build output should be warnings; // otherwise they should all be errors var diagSeverity = buildResult == 0 ? DiagnosticMessage.TspSeverity.Warning : DiagnosticMessage.TspSeverity.Error; this.DiagnosticClassifier.Results .Select(result => result.ToDiagnosticMessage(this, diagSeverity)) .ForEach(AddDiagnostic); return buildResult; } /// /// Returns the build script to use for this project. /// public abstract BuildScript GetBuildScript(); /// /// Produces a diagnostic for the tool status page that we were unable to automatically /// build the user's project and that they can manually specify a build command. This /// can be overriden to implement more specific messages depending on the origin of /// the failure. /// protected virtual void AutobuildFailureDiagnostic() => AddDiagnostic(new DiagnosticMessage( this.Options.Language, "autobuild-failure", "Unable to build project", visibility: new DiagnosticMessage.TspVisibility(statusPage: true), plaintextMessage: """ We were unable to automatically build your project. Set up a manual build command. """ )); /// /// Returns a build script that can be run upon autobuild failure. /// /// /// A build script that reports that we could not automatically detect a suitable build method. /// protected BuildScript AutobuildFailure() => BuildScript.Create(actions => { Log(Severity.Error, "Could not auto-detect a suitable build method"); AutobuildFailureDiagnostic(); return 1; }); /// /// Constructs a which uses the /// to classify build output. All data also gets written to . /// /// /// The to which the build output would have normally been written to. /// This is normally or . /// /// The constructed . protected BuildOutputHandler BuildOutputHandler(TextWriter writer) => new(data => { if (data is not null) { writer.WriteLine(data); DiagnosticClassifier.ClassifyLine(data); } }); /// /// Value of CODEQL_EXTRACTOR__ROOT environment variable. /// public string? CodeQLExtractorLangRoot { get; } /// /// Value of CODEQL_PLATFORM environment variable. /// public string? CodeQlPlatform { get; } } }