using Semmle.Extraction.CSharp; using Semmle.Util.Logging; using System; using System.Collections.Generic; using System.IO; using System.Linq; namespace Semmle.Autobuild { /// /// A build rule analyses the files in "builder" and outputs a build script. /// interface IBuildRule { /// /// Analyse the files and produce a build script. /// /// The files and options relating to the build. BuildScript Analyse(Autobuilder builder); } /// /// Main application logic, containing all data /// gathered from the project and filesystem. /// /// The overall design is intended to be extensible so that in theory, /// it should be possible to add new build rules without touching this code. /// public class Autobuilder { /// /// Full file paths of files found in the project directory. /// public IEnumerable Paths => pathsLazy.Value; readonly Lazy> pathsLazy; /// /// Gets a list of paths matching a set of extensions /// (including the "."). /// /// The extensions to find. /// The files matching the extension. public IEnumerable GetExtensions(params string[] extensions) => Paths.Where(p => extensions.Contains(Path.GetExtension(p))); /// /// Gets all paths matching a particular filename. /// /// The filename to find. /// Possibly empty sequence of paths with the given filename. public IEnumerable GetFilename(string name) => Paths.Where(p => Path.GetFileName(p) == name); /// /// Holds if a given path, relative to the root of the source directory /// was found. /// /// The relative path. /// True iff the path was found. public bool HasRelativePath(string path) => HasPath(Actions.PathCombine(RootDirectory, path)); /// /// List of solution files to build. /// public IList SolutionsToBuild => solutionsToBuildLazy.Value; readonly Lazy> solutionsToBuildLazy; /// /// Holds if a given path was found. /// /// The path of the file. /// True iff the path was found. public bool HasPath(string path) => Paths.Any(p => path == p); void FindFiles(string dir, int depth, IList results) { foreach (var f in Actions.EnumerateFiles(dir)) { results.Add(f); } if (depth > 1) { foreach (var d in Actions.EnumerateDirectories(dir)) { FindFiles(d, depth - 1, results); } } } /// /// The root of the source directory. /// string RootDirectory => Options.RootDirectory; /// /// Gets the supplied build configuration. /// public AutobuildOptions Options { get; } /// /// The set of build actions used during the autobuilder. /// Could be real system operations, or a stub for testing. /// public IBuildActions Actions { get; } /// /// Find all the relevant files and picks the best /// solution file and tools. /// /// The command line options. public Autobuilder(IBuildActions actions, AutobuildOptions options) { Actions = actions; Options = options; pathsLazy = new Lazy>(() => { var files = new List(); FindFiles(options.RootDirectory, options.SearchDepth, files); return files. OrderBy(s => s.Count(c => c == Path.DirectorySeparatorChar)). ThenBy(s => Path.GetFileName(s).Length). ToArray(); }); solutionsToBuildLazy = new Lazy>(() => { if (options.Solution.Any()) { var ret = new List(); foreach (var solution in options.Solution) { if (actions.FileExists(solution)) ret.Add(new Solution(this, solution)); else Log(Severity.Error, "The specified solution file {0} was not found", solution); } return ret; } var solutions = GetExtensions(".sln"). Select(s => new Solution(this, s)). Where(s => s.ProjectCount > 0). OrderByDescending(s => s.ProjectCount). ThenBy(s => s.Path.Length). ToArray(); foreach (var sln in solutions) { Log(Severity.Info, $"Found {sln.Path} with {sln.ProjectCount} {this.Options.Language} projects, version {sln.ToolsVersion}, config {string.Join(" ", sln.Configurations.Select(c => c.FullName))}"); } return new List(options.AllSolutions ? solutions : solutions.Take(1)); }); SemmleDist = Actions.GetEnvironmentVariable("SEMMLE_DIST"); SemmleJavaHome = Actions.GetEnvironmentVariable("SEMMLE_JAVA_HOME"); SemmlePlatformTools = Actions.GetEnvironmentVariable("SEMMLE_PLATFORM_TOOLS"); if (SemmleDist == null) Log(Severity.Error, "The environment variable SEMMLE_DIST has not been set."); } readonly ILogger logger = new ConsoleLogger(Verbosity.Info); /// /// Log a given build event to the console. /// /// The format string. /// Inserts to the format string. public void Log(Severity severity, string format, params object[] args) { logger.Log(severity, format, args); } /// /// Attempt to build this project. /// /// The exit code, 0 for success and non-zero for failures. public int AttemptBuild() { Log(Severity.Info, $"Working directory: {Options.RootDirectory}"); var script = GetBuildScript(); if (Options.IgnoreErrors) script |= BuildScript.Success; void startCallback(string s) => Log(Severity.Info, $"\nRunning {s}"); void exitCallback(int ret, string msg) => Log(Severity.Info, $"Exit code {ret}{(string.IsNullOrEmpty(msg) ? "" : $": {msg}")}"); return script.Run(Actions, startCallback, exitCallback); } /// /// Returns the build script to use for this project. /// public BuildScript GetBuildScript() { var isCSharp = Options.Language == Language.CSharp; return isCSharp ? GetCSharpBuildScript() : GetCppBuildScript(); } BuildScript GetCSharpBuildScript() { /// /// A script that checks that the C# extractor has been executed. /// BuildScript CheckExtractorRun(bool warnOnFailure) => BuildScript.Create(actions => { if (actions.FileExists(Extractor.GetCSharpLogPath())) return 0; if (warnOnFailure) Log(Severity.Error, "No C# code detected during build."); return 1; }); var attempt = BuildScript.Failure; switch (GetCSharpBuildStrategy()) { case CSharpBuildStrategy.CustomBuildCommand: attempt = new BuildCommandRule().Analyse(this) & CheckExtractorRun(true); break; case CSharpBuildStrategy.Buildless: // No need to check that the extractor has been executed in buildless mode attempt = new StandaloneBuildRule().Analyse(this); break; case CSharpBuildStrategy.MSBuild: attempt = new MsBuildRule().Analyse(this) & CheckExtractorRun(true); break; case CSharpBuildStrategy.DotNet: attempt = new DotNetRule().Analyse(this) & CheckExtractorRun(true); break; case CSharpBuildStrategy.Auto: var cleanTrapFolder = BuildScript.DeleteDirectory(Actions.GetEnvironmentVariable("TRAP_FOLDER")); var cleanSourceArchive = BuildScript.DeleteDirectory(Actions.GetEnvironmentVariable("SOURCE_ARCHIVE")); var cleanExtractorLog = BuildScript.DeleteFile(Extractor.GetCSharpLogPath()); var attemptExtractorCleanup = BuildScript.Try(cleanTrapFolder) & BuildScript.Try(cleanSourceArchive) & BuildScript.Try(cleanExtractorLog); /// /// Execute script `s` and check that the C# extractor has been executed. /// If either fails, attempt to cleanup any artifacts produced by the extractor, /// and exit with code 1, in order to proceed to the next attempt. /// BuildScript IntermediateAttempt(BuildScript s) => (s & CheckExtractorRun(false)) | (attemptExtractorCleanup & BuildScript.Failure); attempt = // First try .NET Core IntermediateAttempt(new DotNetRule().Analyse(this)) | // Then MSBuild (() => IntermediateAttempt(new MsBuildRule().Analyse(this))) | // And finally look for a script that might be a build script (() => new BuildCommandAutoRule().Analyse(this) & CheckExtractorRun(true)) | // All attempts failed: print message AutobuildFailure(); break; } return attempt & (() => new AspBuildRule().Analyse(this)) & (() => new XmlBuildRule().Analyse(this)); } /// /// Gets the build strategy that the autobuilder should apply, based on the /// options in the `lgtm.yml` file. /// CSharpBuildStrategy GetCSharpBuildStrategy() { if (Options.BuildCommand != null) return CSharpBuildStrategy.CustomBuildCommand; if (Options.Buildless) return CSharpBuildStrategy.Buildless; if (Options.MsBuildArguments != null || Options.MsBuildConfiguration != null || Options.MsBuildPlatform != null || Options.MsBuildTarget != null) return CSharpBuildStrategy.MSBuild; if (Options.DotNetArguments != null || Options.DotNetVersion != null) return CSharpBuildStrategy.DotNet; return CSharpBuildStrategy.Auto; } enum CSharpBuildStrategy { CustomBuildCommand, Buildless, MSBuild, DotNet, Auto } BuildScript GetCppBuildScript() { if (Options.BuildCommand != null) return new BuildCommandRule().Analyse(this); return // First try MSBuild new MsBuildRule().Analyse(this) | // Then look for a script that might be a build script (() => new BuildCommandAutoRule().Analyse(this)) | // All attempts failed: print message AutobuildFailure(); } BuildScript AutobuildFailure() => BuildScript.Create(actions => { Log(Severity.Error, "Could not auto-detect a suitable build method"); return 1; }); /// /// Value of SEMMLE_DIST environment variable. /// public string SemmleDist { get; private set; } /// /// Value of SEMMLE_JAVA_HOME environment variable. /// public string SemmleJavaHome { get; private set; } /// /// Value of SEMMLE_PLATFORM_TOOLS environment variable. /// public string SemmlePlatformTools { get; private set; } /// /// The absolute path of the odasa executable. /// public string Odasa => SemmleDist == null ? null : Actions.PathCombine(SemmleDist, "tools", "odasa"); } }