using Semmle.Extraction.CSharp; using Semmle.Util.Logging; using System; using System.Collections.Generic; using System.IO; using System.Linq; namespace Semmle.Autobuild { /// /// A build rule analyses the files in "builder" and outputs a build script. /// interface IBuildRule { /// /// Analyse the files and produce a build script. /// /// The files and options relating to the build. /// Whether this build rule is being automatically applied. BuildScript Analyse(Autobuilder builder, bool auto); } /// /// Main application logic, containing all data /// gathered from the project and filesystem. /// /// The overall design is intended to be extensible so that in theory, /// it should be possible to add new build rules without touching this code. /// public class Autobuilder { /// /// Full file paths of files found in the project directory, as well as /// their distance from the project root folder. The list is sorted /// by distance in ascending order. /// public IEnumerable<(string, int)> Paths => pathsLazy.Value; readonly Lazy> pathsLazy; /// /// Gets a list of paths matching a set of extensions (including the "."), /// as well as their distance from the project root folder. /// The list is sorted by distance in ascending order. /// /// The extensions to find. /// The files matching the extension. public IEnumerable<(string, int)> GetExtensions(params string[] extensions) => Paths.Where(p => extensions.Contains(Path.GetExtension(p.Item1))); /// /// Gets all paths matching a particular filename, as well as /// their distance from the project root folder. The list is sorted /// by distance in ascending order. /// /// The filename to find. /// Possibly empty sequence of paths with the given filename. public IEnumerable<(string, int)> GetFilename(string name) => Paths.Where(p => Path.GetFileName(p.Item1) == name); /// /// Holds if a given path, relative to the root of the source directory /// was found. /// /// The relative path. /// True iff the path was found. public bool HasRelativePath(string path) => HasPath(Actions.PathCombine(RootDirectory, path)); /// /// List of project/solution files to build. /// public IList ProjectsOrSolutionsToBuild => projectsOrSolutionsToBuildLazy.Value; readonly Lazy> projectsOrSolutionsToBuildLazy; /// /// Holds if a given path was found. /// /// The path of the file. /// True iff the path was found. public bool HasPath(string path) => Paths.Any(p => path == p.Item1); void FindFiles(string dir, int depth, int maxDepth, IList<(string, int)> results) { foreach (var f in Actions.EnumerateFiles(dir)) { results.Add((f, depth)); } if (depth < maxDepth) { foreach (var d in Actions.EnumerateDirectories(dir)) { FindFiles(d, depth + 1, maxDepth, results); } } } /// /// The root of the source directory. /// string RootDirectory => Options.RootDirectory; /// /// Gets the supplied build configuration. /// public AutobuildOptions Options { get; } /// /// The set of build actions used during the autobuilder. /// Could be real system operations, or a stub for testing. /// public IBuildActions Actions { get; } /// /// Find all the relevant files and picks the best /// solution file and tools. /// /// The command line options. public Autobuilder(IBuildActions actions, AutobuildOptions options) { Actions = actions; Options = options; pathsLazy = new Lazy>(() => { var files = new List<(string, int)>(); FindFiles(options.RootDirectory, 0, options.SearchDepth, files); return files.OrderBy(f => f.Item2).ToArray(); }); projectsOrSolutionsToBuildLazy = new Lazy>(() => { List ret; if (options.Solution.Any()) { ret = new List(); foreach (var solution in options.Solution) { if (actions.FileExists(solution)) ret.Add(new Solution(this, solution, true)); else Log(Severity.Error, $"The specified project or solution file {solution} was not found"); } return ret; } IEnumerable FindFiles(string extension, Func create) { var matchingFiles = GetExtensions(extension). Select(p => (ProjectOrSolution: create(p.Item1), DistanceFromRoot: p.Item2)). Where(p => p.ProjectOrSolution.HasLanguage(this.Options.Language)). ToArray(); if (matchingFiles.Length == 0) return null; if (options.AllSolutions) return matchingFiles.Select(p => p.ProjectOrSolution); var firstIsClosest = matchingFiles.Length > 1 && matchingFiles[0].DistanceFromRoot < matchingFiles[1].DistanceFromRoot; if (matchingFiles.Length == 1 || firstIsClosest) return matchingFiles.Select(p => p.ProjectOrSolution).Take(1); var candidates = matchingFiles. Where(f => f.DistanceFromRoot == matchingFiles[0].DistanceFromRoot). Select(f => f.ProjectOrSolution); Log(Severity.Info, $"Found multiple '{extension}' files, giving up: {string.Join(", ", candidates)}."); return new IProjectOrSolution[0]; } // First look for `.proj` files ret = FindFiles(".proj", f => new Project(this, f))?.ToList(); if (ret != null) return ret; // Then look for `.sln` files ret = FindFiles(".sln", f => new Solution(this, f, false))?.ToList(); if (ret != null) return ret; // Finally look for language specific project files, e.g. `.csproj` files ret = FindFiles(this.Options.Language.ProjectExtension, f => new Project(this, f))?.ToList(); return ret ?? new List(); }); SemmleDist = Actions.GetEnvironmentVariable("SEMMLE_DIST"); SemmleJavaHome = Actions.GetEnvironmentVariable("SEMMLE_JAVA_HOME"); SemmlePlatformTools = Actions.GetEnvironmentVariable("SEMMLE_PLATFORM_TOOLS"); if (SemmleDist == null) Log(Severity.Error, "The environment variable SEMMLE_DIST has not been set."); } readonly ILogger logger = new ConsoleLogger(Verbosity.Info); /// /// Log a given build event to the console. /// /// The format string. /// Inserts to the format string. public void Log(Severity severity, string format, params object[] args) { logger.Log(severity, format, args); } /// /// Attempt to build this project. /// /// The exit code, 0 for success and non-zero for failures. public int AttemptBuild() { Log(Severity.Info, $"Working directory: {Options.RootDirectory}"); var script = GetBuildScript(); if (Options.IgnoreErrors) script |= BuildScript.Success; void startCallback(string s) => Log(Severity.Info, $"\nRunning {s}"); void exitCallback(int ret, string msg) => Log(Severity.Info, $"Exit code {ret}{(string.IsNullOrEmpty(msg) ? "" : $": {msg}")}"); return script.Run(Actions, startCallback, exitCallback); } /// /// Returns the build script to use for this project. /// public BuildScript GetBuildScript() { var isCSharp = Options.Language == Language.CSharp; return isCSharp ? GetCSharpBuildScript() : GetCppBuildScript(); } BuildScript GetCSharpBuildScript() { /// /// A script that checks that the C# extractor has been executed. /// BuildScript CheckExtractorRun(bool warnOnFailure) => BuildScript.Create(actions => { if (actions.FileExists(Extractor.GetCSharpLogPath())) return 0; if (warnOnFailure) Log(Severity.Error, "No C# code detected during build."); return 1; }); var attempt = BuildScript.Failure; switch (GetCSharpBuildStrategy()) { case CSharpBuildStrategy.CustomBuildCommand: attempt = new BuildCommandRule().Analyse(this, false) & CheckExtractorRun(true); break; case CSharpBuildStrategy.Buildless: // No need to check that the extractor has been executed in buildless mode attempt = new StandaloneBuildRule().Analyse(this, false); break; case CSharpBuildStrategy.MSBuild: attempt = new MsBuildRule().Analyse(this, false) & CheckExtractorRun(true); break; case CSharpBuildStrategy.DotNet: attempt = new DotNetRule().Analyse(this, false) & CheckExtractorRun(true); break; case CSharpBuildStrategy.Auto: var cleanTrapFolder = BuildScript.DeleteDirectory(Actions.GetEnvironmentVariable("TRAP_FOLDER")); var cleanSourceArchive = BuildScript.DeleteDirectory(Actions.GetEnvironmentVariable("SOURCE_ARCHIVE")); var cleanExtractorLog = BuildScript.DeleteFile(Extractor.GetCSharpLogPath()); var attemptExtractorCleanup = BuildScript.Try(cleanTrapFolder) & BuildScript.Try(cleanSourceArchive) & BuildScript.Try(cleanExtractorLog); /// /// Execute script `s` and check that the C# extractor has been executed. /// If either fails, attempt to cleanup any artifacts produced by the extractor, /// and exit with code 1, in order to proceed to the next attempt. /// BuildScript IntermediateAttempt(BuildScript s) => (s & CheckExtractorRun(false)) | (attemptExtractorCleanup & BuildScript.Failure); attempt = // First try .NET Core IntermediateAttempt(new DotNetRule().Analyse(this, true)) | // Then MSBuild (() => IntermediateAttempt(new MsBuildRule().Analyse(this, true))) | // And finally look for a script that might be a build script (() => new BuildCommandAutoRule().Analyse(this, true) & CheckExtractorRun(true)) | // All attempts failed: print message AutobuildFailure(); break; } return attempt & (() => new AspBuildRule().Analyse(this, false)) & (() => new XmlBuildRule().Analyse(this, false)); } /// /// Gets the build strategy that the autobuilder should apply, based on the /// options in the `lgtm.yml` file. /// CSharpBuildStrategy GetCSharpBuildStrategy() { if (Options.BuildCommand != null) return CSharpBuildStrategy.CustomBuildCommand; if (Options.Buildless) return CSharpBuildStrategy.Buildless; if (Options.MsBuildArguments != null || Options.MsBuildConfiguration != null || Options.MsBuildPlatform != null || Options.MsBuildTarget != null) return CSharpBuildStrategy.MSBuild; if (Options.DotNetArguments != null || Options.DotNetVersion != null) return CSharpBuildStrategy.DotNet; return CSharpBuildStrategy.Auto; } enum CSharpBuildStrategy { CustomBuildCommand, Buildless, MSBuild, DotNet, Auto } BuildScript GetCppBuildScript() { if (Options.BuildCommand != null) return new BuildCommandRule().Analyse(this, false); return // First try MSBuild new MsBuildRule().Analyse(this, true) | // Then look for a script that might be a build script (() => new BuildCommandAutoRule().Analyse(this, true)) | // All attempts failed: print message AutobuildFailure(); } BuildScript AutobuildFailure() => BuildScript.Create(actions => { Log(Severity.Error, "Could not auto-detect a suitable build method"); return 1; }); /// /// Value of SEMMLE_DIST environment variable. /// public string SemmleDist { get; private set; } /// /// Value of SEMMLE_JAVA_HOME environment variable. /// public string SemmleJavaHome { get; private set; } /// /// Value of SEMMLE_PLATFORM_TOOLS environment variable. /// public string SemmlePlatformTools { get; private set; } /// /// The absolute path of the odasa executable. /// public string Odasa => SemmleDist == null ? null : Actions.PathCombine(SemmleDist, "tools", "odasa"); } }