Merge pull request #15762 from hvitved/csharp/fetch-dotnet-dependency-fetching

C#: Fetch .NET in dependency manager instead of autobuilder

csharp/extractor/Semmle.Extraction.CIL.Driver/Program.cs

@@ -46,7 +46,7 @@ namespace Semmle.Extraction.CIL.Driver
                 .ToArray();
 
             foreach (var missingRef in options.MissingReferences)
-                logger.Log(Severity.Info, "  Missing assembly " + missingRef);
+                logger.LogInfo("  Missing assembly " + missingRef);
 
             var sw = new Stopwatch();
             sw.Start();

csharp/extractor/Semmle.Extraction.CIL/Analyser.cs

@@ -45,7 +45,7 @@ namespace Semmle.Extraction.CIL
             }
             catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
             {
-                logger.Log(Severity.Error, string.Format("Exception extracting {0}: {1}", assemblyPath, ex));
+                logger.LogError(string.Format("Exception extracting {0}: {1}", assemblyPath, ex));
             }
         }
     }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -7,6 +7,7 @@ using System.Security.Cryptography;
 using System.Text;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
+
 using Semmle.Util;
 using Semmle.Util.Logging;
 
@@ -27,8 +28,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         private readonly List<string> generatedSources;
         private int dotnetFrameworkVersionVariantCount = 0;
         private int conflictedReferences = 0;
-        private readonly IDependencyOptions options;
         private readonly DirectoryInfo sourceDir;
+        private string? dotnetPath;
         private readonly IDotNet dotnet;
         private readonly FileContent fileContent;
         private readonly TemporaryDirectory packageDirectory;
@@ -39,17 +40,17 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 
         private readonly Lazy<Runtime> runtimeLazy;
         private Runtime Runtime => runtimeLazy.Value;
+        private readonly int threads = EnvironmentVariables.GetDefaultNumberOfThreads();
 
         /// <summary>
         /// Performs C# dependency fetching.
         /// </summary>
         /// <param name="options">Dependency fetching options</param>
         /// <param name="logger">Logger for dependency fetching progress.</param>
-        public DependencyManager(string srcDir, IDependencyOptions options, ILogger logger)
+        public DependencyManager(string srcDir, ILogger logger)
         {
             var startTime = DateTime.Now;
 
-            this.options = options;
             this.logger = logger;
             this.sourceDir = new DirectoryInfo(srcDir);
 
@@ -59,17 +60,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 
             tempWorkingDirectory = new TemporaryDirectory(FileUtils.GetTemporaryWorkingDirectory(out cleanupTempWorkingDirectory));
 
-            try
-            {
-                this.dotnet = DotNet.Make(options, logger, tempWorkingDirectory);
-                runtimeLazy = new Lazy<Runtime>(() => new Runtime(dotnet, logger));
-            }
-            catch
-            {
-                logger.LogError("Missing dotnet CLI");
-                throw;
-            }
-
             logger.LogInfo($"Finding files in {srcDir}...");
 
             var allFiles = GetAllFiles().ToList();
@@ -85,6 +75,33 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 
             logger.LogInfo($"Found {allFiles.Count} files, {nonGeneratedSources.Count} source files, {allProjects.Count} project files, {allSolutions.Count} solution files, {dllPaths.Count} DLLs.");
 
+            void startCallback(string s, bool silent)
+            {
+                logger.Log(silent ? Severity.Debug : Severity.Info, $"\nRunning {s}");
+            }
+
+            void exitCallback(int ret, string msg, bool silent)
+            {
+                logger.Log(silent ? Severity.Debug : Severity.Info, $"Exit code {ret}{(string.IsNullOrEmpty(msg) ? "" : $": {msg}")}");
+            }
+
+            DotNet.WithDotNet(SystemBuildActions.Instance, logger, smallNonBinaryFiles, tempWorkingDirectory.ToString(), shouldCleanUp: false, ensureDotNetAvailable: true, version: null, installDir =>
+            {
+                this.dotnetPath = installDir;
+                return BuildScript.Success;
+            }).Run(SystemBuildActions.Instance, startCallback, exitCallback);
+
+            try
+            {
+                this.dotnet = DotNet.Make(logger, dotnetPath, tempWorkingDirectory);
+                runtimeLazy = new Lazy<Runtime>(() => new Runtime(dotnet));
+            }
+            catch
+            {
+                logger.LogError("Missing dotnet CLI");
+                throw;
+            }
+
             RestoreNugetPackages(allNonBinaryFiles, allProjects, allSolutions, dllPaths);
             // Find DLLs in the .Net / Asp.Net Framework
             // This needs to come after the nuget restore, because the nuget restore might fetch the .NET Core/Framework reference assemblies.
@@ -570,15 +587,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             }
         }
 
-        public DependencyManager(string srcDir) : this(srcDir, DependencyOptions.Default, new ConsoleLogger(Verbosity.Info, logThreadId: true)) { }
-
         private IEnumerable<FileInfo> GetAllFiles()
         {
             IEnumerable<FileInfo> files = sourceDir.GetFiles("*.*", new EnumerationOptions { RecurseSubdirectories = true });
 
-            if (options.DotNetPath != null)
+            if (dotnetPath != null)
             {
-                files = files.Where(f => !f.FullName.StartsWith(options.DotNetPath, StringComparison.OrdinalIgnoreCase));
+                files = files.Where(f => !f.FullName.StartsWith(dotnetPath, StringComparison.OrdinalIgnoreCase));
             }
 
             files = files.Where(f =>
@@ -590,12 +605,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                         return true;
                     }
 
-                    logger.Log(Severity.Warning, $"File {f.FullName} could not be processed.");
+                    logger.LogWarning($"File {f.FullName} could not be processed.");
                     return false;
                 }
                 catch (Exception ex)
                 {
-                    logger.Log(Severity.Warning, $"File {f.FullName} could not be processed: {ex.Message}");
+                    logger.LogWarning($"File {f.FullName} could not be processed: {ex.Message}");
                     return false;
                 }
             });
@@ -651,7 +666,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                 }
                 catch (AssemblyLoadException)
                 {
-                    logger.Log(Severity.Warning, $"Could not load assembly information from {usedReference.Key}");
+                    logger.LogWarning($"Could not load assembly information from {usedReference.Key}");
                 }
             }
 
@@ -738,7 +753,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 
         private void AnalyseSolutions(IEnumerable<string> solutions)
         {
-            Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, solutionFile =>
+            Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = threads }, solutionFile =>
             {
                 try
                 {
@@ -828,7 +843,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             var successCount = 0;
             var assetFiles = new List<string>();
             var sync = new object();
-            Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, project =>
+            Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = threads }, project =>
             {
                 logger.LogInfo($"Restoring project {project}...");
                 var res = dotnet.Restore(new(project, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
@@ -928,7 +943,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             var successCount = 0;
             var sync = new object();
 
-            Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, package =>
+            Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = threads }, package =>
             {
                 var success = TryRestorePackageManually(package.Name, nugetConfig, package.PackageReferenceSource);
                 if (!success)

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyOptions.cs

@@ -1,32 +0,0 @@
-using System;
-using System.Linq;
-using System.Collections.Generic;
-using Semmle.Util;
-
-namespace Semmle.Extraction.CSharp.DependencyFetching
-{
-    /// <summary>
-    /// Dependency fetching related options.
-    /// </summary>
-    public interface IDependencyOptions
-    {
-        /// <summary>
-        /// The number of threads to use.
-        /// </summary>
-        int Threads { get; }
-
-        /// <summary>
-        /// The path to the local ".dotnet" directory, if any.
-        /// </summary>
-        string? DotNetPath { get; }
-    }
-
-    public class DependencyOptions : IDependencyOptions
-    {
-        public static IDependencyOptions Default => new DependencyOptions();
-
-        public int Threads { get; set; } = EnvironmentVariables.GetDefaultNumberOfThreads();
-
-        public string? DotNetPath { get; set; } = null;
-    }
-}

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs

@@ -2,7 +2,9 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
-using System.Text.RegularExpressions;
+
+using Newtonsoft.Json.Linq;
+
 using Semmle.Util;
 using Semmle.Util.Logging;
 
@@ -11,29 +13,26 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
     /// <summary>
     /// Utilities to run the "dotnet" command.
     /// </summary>
-    internal partial class DotNet : IDotNet
+    public partial class DotNet : IDotNet
     {
         private readonly IDotNetCliInvoker dotnetCliInvoker;
-        private readonly ILogger logger;
         private readonly TemporaryDirectory? tempWorkingDirectory;
 
         private DotNet(IDotNetCliInvoker dotnetCliInvoker, ILogger logger, TemporaryDirectory? tempWorkingDirectory = null)
         {
-            this.logger = logger;
             this.tempWorkingDirectory = tempWorkingDirectory;
             this.dotnetCliInvoker = dotnetCliInvoker;
             Info();
         }
 
-        private DotNet(IDependencyOptions options, ILogger logger, TemporaryDirectory tempWorkingDirectory) : this(new DotNetCliInvoker(logger, Path.Combine(options.DotNetPath ?? string.Empty, "dotnet")), logger, tempWorkingDirectory) { }
+        private DotNet(ILogger logger, string? dotNetPath, TemporaryDirectory tempWorkingDirectory) : this(new DotNetCliInvoker(logger, Path.Combine(dotNetPath ?? string.Empty, "dotnet")), logger, tempWorkingDirectory) { }
 
         internal static IDotNet Make(IDotNetCliInvoker dotnetCliInvoker, ILogger logger) => new DotNet(dotnetCliInvoker, logger);
 
-        public static IDotNet Make(IDependencyOptions options, ILogger logger, TemporaryDirectory tempWorkingDirectory) => new DotNet(options, logger, tempWorkingDirectory);
+        public static IDotNet Make(ILogger logger, string? dotNetPath, TemporaryDirectory tempWorkingDirectory) => new DotNet(logger, dotNetPath, tempWorkingDirectory);
 
         private void Info()
         {
-            // TODO: make sure the below `dotnet` version is matching the one specified in global.json
             var res = dotnetCliInvoker.RunCommand("--info");
             if (!res)
             {
@@ -108,5 +107,191 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             var args = $"exec {execArgs}";
             return dotnetCliInvoker.RunCommand(args);
         }
+
+        // The version number should be kept in sync with the version .NET version used for building the application.
+        public const string LatestDotNetSdkVersion = "8.0.101";
+
+        /// <summary>
+        /// Returns a script for downloading relevant versions of the
+        /// .NET SDK. The SDK(s) will be installed at <code>installDir</code>
+        /// (provided that the script succeeds).
+        /// </summary>
+        private static BuildScript DownloadDotNet(IBuildActions actions, ILogger logger, IEnumerable<string> files, string tempWorkingDirectory, bool shouldCleanUp, string installDir, string? version, bool ensureDotNetAvailable)
+        {
+            if (!string.IsNullOrEmpty(version))
+                // Specific version requested
+                return DownloadDotNetVersion(actions, logger, tempWorkingDirectory, shouldCleanUp, installDir, [version]);
+
+            // Download versions mentioned in `global.json` files
+            // See https://docs.microsoft.com/en-us/dotnet/core/tools/global-json
+            var versions = new List<string>();
+
+            foreach (var path in files.Where(p => p.EndsWith("global.json", StringComparison.Ordinal)))
+            {
+                try
+                {
+                    var o = JObject.Parse(File.ReadAllText(path));
+                    versions.Add((string)o?["sdk"]?["version"]!);
+                }
+                catch
+                {
+                    // not a valid `global.json` file
+                    continue;
+                }
+            }
+
+            if (versions.Count > 0)
+            {
+                return DownloadDotNetVersion(actions, logger, tempWorkingDirectory, shouldCleanUp, installDir, versions);
+            }
+
+            if (ensureDotNetAvailable)
+            {
+                return DownloadDotNetVersion(actions, logger, tempWorkingDirectory, shouldCleanUp, installDir, [LatestDotNetSdkVersion], needExactVersion: false);
+            }
+
+            return BuildScript.Failure;
+        }
+
+        /// <summary>
+        /// Returns a script for downloading specific .NET SDK versions, if the
+        /// versions are not already installed.
+        ///
+        /// See https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet-install-script.
+        /// </summary>
+        private static BuildScript DownloadDotNetVersion(IBuildActions actions, ILogger logger, string tempWorkingDirectory, bool shouldCleanUp, string path, IEnumerable<string> versions, bool needExactVersion = true)
+        {
+            if (!versions.Any())
+            {
+                logger.LogInfo("No .NET SDK versions requested.");
+                return BuildScript.Failure;
+            }
+
+            return BuildScript.Bind(GetInstalledSdksScript(actions), (sdks, sdksRet) =>
+            {
+                if (
+                    needExactVersion &&
+                    sdksRet == 0 &&
+                    // quadratic; should be OK, given that both `version` and `sdks` are expected to be small
+                    versions.All(version => sdks.Any(sdk => sdk.StartsWith(version + " ", StringComparison.Ordinal))))
+                {
+                    // The requested SDKs are already installed, so no need to reinstall
+                    return BuildScript.Failure;
+                }
+                else if (!needExactVersion && sdksRet == 0 && sdks.Count > 0)
+                {
+                    // there's at least one SDK installed, so no need to reinstall
+                    return BuildScript.Failure;
+                }
+                else if (!needExactVersion && sdksRet != 0)
+                {
+                    logger.LogInfo("No .NET SDK found.");
+                }
+
+                BuildScript prelude;
+                BuildScript postlude;
+                Func<string, BuildScript> getInstall;
+
+                if (actions.IsWindows())
+                {
+                    prelude = BuildScript.Success;
+                    postlude = BuildScript.Success;
+
+                    getInstall = version =>
+                    {
+                        var psCommand = $"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; &([scriptblock]::Create((Invoke-WebRequest -UseBasicParsing 'https://dot.net/v1/dotnet-install.ps1'))) -Version {version} -InstallDir {path}";
+
+                        BuildScript GetInstall(string pwsh) =>
+                            new CommandBuilder(actions).
+                            RunCommand(pwsh).
+                            Argument("-NoProfile").
+                            Argument("-ExecutionPolicy").
+                            Argument("unrestricted").
+                            Argument("-Command").
+                            Argument("\"" + psCommand + "\"").
+                            Script;
+
+                        return GetInstall("pwsh") | GetInstall("powershell");
+                    };
+                }
+                else
+                {
+                    var dotnetInstallPath = actions.PathCombine(tempWorkingDirectory, ".dotnet", "dotnet-install.sh");
+
+                    var downloadDotNetInstallSh = BuildScript.DownloadFile(
+                        "https://dot.net/v1/dotnet-install.sh",
+                        dotnetInstallPath,
+                        e => logger.LogWarning($"Failed to download 'dotnet-install.sh': {e.Message}"));
+
+                    var chmod = new CommandBuilder(actions).
+                        RunCommand("chmod").
+                        Argument("u+x").
+                        Argument(dotnetInstallPath);
+
+                    prelude = downloadDotNetInstallSh & chmod.Script;
+                    postlude = shouldCleanUp ? BuildScript.DeleteFile(dotnetInstallPath) : BuildScript.Success;
+
+                    getInstall = version => new CommandBuilder(actions).
+                        RunCommand(dotnetInstallPath).
+                        Argument("--channel").
+                        Argument("release").
+                        Argument("--version").
+                        Argument(version).
+                        Argument("--install-dir").
+                        Argument(path).Script;
+                }
+
+                var installScript = prelude & BuildScript.Failure;
+
+                var attempted = new HashSet<string>();
+                foreach (var version in versions)
+                {
+                    if (!attempted.Add(version))
+                        continue;
+
+                    installScript = BuildScript.Bind(installScript, combinedExit =>
+                    {
+                        logger.LogInfo($"Attempting to download .NET {version}");
+
+                        // When there are multiple versions requested, we want to try to fetch them all, reporting
+                        // a successful exit code when at least one of them succeeds
+                        return combinedExit != 0 ? getInstall(version) : BuildScript.Bind(getInstall(version), _ => BuildScript.Success);
+                    });
+                }
+
+                return installScript & postlude;
+            });
+        }
+
+        private static BuildScript GetInstalledSdksScript(IBuildActions actions)
+        {
+            var listSdks = new CommandBuilder(actions, silent: true).
+                RunCommand("dotnet").
+                Argument("--list-sdks");
+            return listSdks.Script;
+        }
+
+        /// <summary>
+        /// Returns a script that attempts to download relevant version(s) of the
+        /// .NET SDK, followed by running the script generated by <paramref name="f"/>.
+        ///
+        /// The argument to <paramref name="f"/> is the path to the directory in which the
+        /// .NET SDK(s) were installed.
+        /// </summary>
+        public static BuildScript WithDotNet(IBuildActions actions, ILogger logger, IEnumerable<string> files, string tempWorkingDirectory, bool shouldCleanUp, bool ensureDotNetAvailable, string? version, Func<string?, BuildScript> f)
+        {
+            var installDir = actions.PathCombine(tempWorkingDirectory, ".dotnet");
+            var installScript = DownloadDotNet(actions, logger, files, tempWorkingDirectory, shouldCleanUp, installDir, version, ensureDotNetAvailable);
+            return BuildScript.Bind(installScript, installed =>
+            {
+                if (installed != 0)
+                {
+                    // The .NET SDK was not installed, either because the installation failed or because it was already installed.
+                    installDir = null;
+                }
+
+                return f(installDir);
+            });
+        }
     }
-}
+}

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FilePathFilter.cs

@@ -55,12 +55,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                 }
                 else
                 {
-                    logger.Log(Severity.Info, $"Invalid filter: {filter}");
+                    logger.LogInfo($"Invalid filter: {filter}");
                     continue;
                 }
 
                 var regex = new FilePattern(filterText).RegexPattern;
-                logger.Log(Severity.Info, $"Filtering {(include ? "in" : "out")} files matching '{regex}'. Original glob filter: '{filter}'");
+                logger.LogInfo($"Filtering {(include ? "in" : "out")} files matching '{regex}'. Original glob filter: '{filter}'");
                 pathFilters.Add(new PathFilter(new Regex(regex, RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline), include));
             }
 
@@ -91,7 +91,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 
                 if (!include)
                 {
-                    logger.Log(Severity.Info, $"Excluding '{f.FileInfo.FullName}'");
+                    logger.LogInfo($"Excluding '{f.FileInfo.FullName}'");
                 }
 
                 return include;

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs

@@ -5,7 +5,7 @@ using System.Text.RegularExpressions;
 
 namespace Semmle.Extraction.CSharp.DependencyFetching
 {
-    internal interface IDotNet
+    public interface IDotNet
     {
         RestoreResult Restore(RestoreSettings restoreSettings);
         bool New(string folder);
@@ -15,9 +15,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         bool Exec(string execArgs);
     }
 
-    internal record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, string? PathToNugetConfig = null, bool ForceReevaluation = false);
+    public record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, string? PathToNugetConfig = null, bool ForceReevaluation = false);
 
-    internal partial record class RestoreResult(bool Success, IList<string> Output)
+    public partial record class RestoreResult(bool Success, IList<string> Output)
     {
         private readonly Lazy<IEnumerable<string>> assetsFilePaths = new(() => GetFirstGroupOnMatch(AssetsFileRegex(), Output));
         public IEnumerable<string> AssetsFilePaths => Success ? assetsFilePaths.Value : Array.Empty<string>();

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Runtime.cs

@@ -18,14 +18,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         private const string aspNetCoreApp = "Microsoft.AspNetCore.App";
 
         private readonly IDotNet dotNet;
-        private readonly ILogger logger;
         private readonly Lazy<Dictionary<string, DotNetVersion>> newestRuntimes;
         private Dictionary<string, DotNetVersion> NewestRuntimes => newestRuntimes.Value;
 
-        public Runtime(IDotNet dotNet, ILogger logger)
+        public Runtime(IDotNet dotNet)
         {
             this.dotNet = dotNet;
-            this.logger = logger;
             this.newestRuntimes = new(GetNewestRuntimes);
         }
 

csharp/extractor/Semmle.Extraction.CSharp.DependencyStubGenerator/Program.cs

@@ -3,7 +3,7 @@ using Semmle.Extraction.CSharp.StubGenerator;
 using Semmle.Util.Logging;
 
 var logger = new ConsoleLogger(Verbosity.Info, logThreadId: false);
-using var dependencyManager = new DependencyManager(".", DependencyOptions.Default, logger);
+using var dependencyManager = new DependencyManager(".", logger);
 StubGenerator.GenerateStubs(logger, dependencyManager.ReferenceFiles, "codeql_csharp_stubs");
 
 return 0;

csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs

@@ -138,7 +138,7 @@ namespace Semmle.Extraction.CSharp.Standalone
 
             using var logger = new ConsoleLogger(options.Verbosity, logThreadId: true);
             logger.Log(Severity.Info, "Extracting C# in buildless mode");
-            using var dependencyManager = new DependencyManager(options.SrcDir, options.Dependencies, logger);
+            using var dependencyManager = new DependencyManager(options.SrcDir, logger);
 
             if (!dependencyManager.AllSourceFiles.Any())
             {

csharp/extractor/Semmle.Extraction.CSharp.Standalone/Options.cs

@@ -23,18 +23,6 @@ namespace Semmle.Extraction.CSharp.Standalone
             }
         }
 
-        public override bool HandleOption(string key, string value)
-        {
-            switch (key)
-            {
-                case "dotnet":
-                    dependencies.DotNetPath = value;
-                    return true;
-                default:
-                    return base.HandleOption(key, value);
-            }
-        }
-
         public override bool HandleArgument(string arg)
         {
             return true;
@@ -51,12 +39,6 @@ namespace Semmle.Extraction.CSharp.Standalone
         /// </summary>
         public string SrcDir { get; } = Directory.GetCurrentDirectory();
 
-        private readonly DependencyOptions dependencies = new DependencyOptions();
-        /// <summary>
-        /// Dependency fetching related options.
-        /// </summary>
-        public IDependencyOptions Dependencies => dependencies;
-
         /// <summary>
         /// Whether errors were encountered parsing the arguments.
         /// </summary>

csharp/extractor/Semmle.Extraction.CSharp.StubGenerator/StubGenerator.cs

@@ -36,7 +36,7 @@ public static class StubGenerator
             references.Add((reference, path));
         });
 
-        logger.Log(Severity.Info, $"Generating stubs for {references.Count} assemblies.");
+        logger.LogInfo($"Generating stubs for {references.Count} assemblies.");
 
         var compilation = CSharpCompilation.Create(
             "stubgenerator.dll",
@@ -50,7 +50,7 @@ public static class StubGenerator
         });
 
         stopWatch.Stop();
-        logger.Log(Severity.Info, $"Stub generation took {stopWatch.Elapsed}.");
+        logger.LogInfo($"Stub generation took {stopWatch.Elapsed}.");
 
         return stubPaths.ToArray();
     }

csharp/extractor/Semmle.Extraction.Tests/Runtime.cs

@@ -47,7 +47,7 @@ namespace Semmle.Extraction.Tests
                 "Microsoft.NETCore.App 7.0.2 [/path/dotnet/shared/Microsoft.NETCore.App]"
                 };
             var dotnet = new DotNetStub(listedRuntimes, null!);
-            var runtime = new Runtime(dotnet, new LoggerStub());
+            var runtime = new Runtime(dotnet);
 
             // Execute
             var runtimes = runtime.GetNewestRuntimes();
@@ -73,7 +73,7 @@ namespace Semmle.Extraction.Tests
                 "Microsoft.NETCore.App 8.0.0-preview.5.23280.8 [/path/dotnet/shared/Microsoft.NETCore.App]"
             };
             var dotnet = new DotNetStub(listedRuntimes, null!);
-            var runtime = new Runtime(dotnet, new LoggerStub());
+            var runtime = new Runtime(dotnet);
 
             // Execute
             var runtimes = runtime.GetNewestRuntimes();
@@ -96,7 +96,7 @@ namespace Semmle.Extraction.Tests
                 "Microsoft.NETCore.App 8.0.0-preview.5.23280.8 [/path/dotnet/shared/Microsoft.NETCore.App]"
             };
             var dotnet = new DotNetStub(listedRuntimes, null!);
-            var runtime = new Runtime(dotnet, new LoggerStub());
+            var runtime = new Runtime(dotnet);
 
             // Execute
             var runtimes = runtime.GetNewestRuntimes();
@@ -125,7 +125,7 @@ namespace Semmle.Extraction.Tests
                 @"Microsoft.WindowsDesktop.App 7.0.4 [C:\Program Files\dotnet\shared\Microsoft.WindowsDesktop.App]"
             };
             var dotnet = new DotNetStub(listedRuntimes, null!);
-            var runtime = new Runtime(dotnet, new LoggerStub());
+            var runtime = new Runtime(dotnet);
 
             // Execute
             var runtimes = runtime.GetNewestRuntimes();

csharp/extractor/Semmle.Extraction/Extractor/Extractor.cs

@@ -43,7 +43,7 @@ namespace Semmle.Extraction
                     ++Errors;
                     if (Errors == maxErrors)
                     {
-                        Logger.Log(Severity.Info, "  Stopping logging after {0} errors", Errors);
+                        Logger.LogInfo("  Stopping logging after {0} errors", Errors);
                     }
                 }
 

csharp/extractor/Semmle.Extraction/TrapWriter.cs

@@ -227,7 +227,7 @@ namespace Semmle.Extraction
             {
                 // If this happened, it was probably because the same file was compiled multiple times.
                 // In any case, this is not a fatal error.
-                logger.Log(Severity.Warning, "Problem archiving " + dest + ": " + ex);
+                logger.LogWarning("Problem archiving " + dest + ": " + ex);
             }
         }
 

csharp/extractor/Semmle.Util/BuildActions.cs

@@ -0,0 +1,316 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
+using System.IO;
+using System.Linq;
+using System.Runtime.InteropServices;
+using System.Xml;
+using Semmle.Util;
+
+namespace Semmle.Util
+{
+    public delegate void BuildOutputHandler(string? data);
+
+    /// <summary>
+    /// Wrapper around system calls so that the build scripts can be unit-tested.
+    /// </summary>
+    public interface IBuildActions
+    {
+
+        /// <summary>
+        /// Runs a process, captures its output, and provides it asynchronously.
+        /// </summary>
+        /// <param name="exe">The exe to run.</param>
+        /// <param name="args">The other command line arguments.</param>
+        /// <param name="workingDirectory">The working directory (<code>null</code> for current directory).</param>
+        /// <param name="env">Additional environment variables.</param>
+        /// <param name="onOutput">A handler for stdout output.</param>
+        /// <param name="onError">A handler for stderr output.</param>
+        /// <returns>The process exit code.</returns>
+        int RunProcess(string exe, string args, string? workingDirectory, IDictionary<string, string>? env, BuildOutputHandler onOutput, BuildOutputHandler onError);
+
+        /// <summary>
+        /// Runs a process and captures its output.
+        /// </summary>
+        /// <param name="exe">The exe to run.</param>
+        /// <param name="args">The other command line arguments.</param>
+        /// <param name="workingDirectory">The working directory (<code>null</code> for current directory).</param>
+        /// <param name="env">Additional environment variables.</param>
+        /// <param name="stdOut">The lines of stdout.</param>
+        /// <returns>The process exit code.</returns>
+        int RunProcess(string exe, string args, string? workingDirectory, IDictionary<string, string>? env, out IList<string> stdOut);
+
+        /// <summary>
+        /// Runs a process but does not capture its output.
+        /// </summary>
+        /// <param name="exe">The exe to run.</param>
+        /// <param name="args">The other command line arguments.</param>
+        /// <param name="workingDirectory">The working directory (<code>null</code> for current directory).</param>
+        /// <param name="env">Additional environment variables.</param>
+        /// <returns>The process exit code.</returns>
+        int RunProcess(string exe, string args, string? workingDirectory, IDictionary<string, string>? env);
+
+        /// <summary>
+        /// Tests whether a file exists, File.Exists().
+        /// </summary>
+        /// <param name="file">The filename.</param>
+        /// <returns>True iff the file exists.</returns>
+        bool FileExists(string file);
+
+        /// <summary>
+        /// Tests whether a directory exists, Directory.Exists().
+        /// </summary>
+        /// <param name="dir">The directory name.</param>
+        /// <returns>True iff the directory exists.</returns>
+        bool DirectoryExists(string dir);
+
+        /// <summary>
+        /// Deletes a file, File.Delete().
+        /// </summary>
+        /// <param name="file">The filename.</param>
+        void FileDelete(string file);
+
+        /// <summary>
+        /// Deletes a directory, Directory.Delete().
+        /// </summary>
+        void DirectoryDelete(string dir, bool recursive);
+
+        /// <summary>
+        /// Creates all directories and subdirectories in the specified path unless they already exist.
+        /// </summary>
+        void CreateDirectory(string path);
+
+        /// <summary>
+        /// Gets an environment variable, Environment.GetEnvironmentVariable().
+        /// </summary>
+        /// <param name="name">The name of the variable.</param>
+        /// <returns>The string value, or null if the variable is not defined.</returns>
+        string? GetEnvironmentVariable(string name);
+
+        /// <summary>
+        /// Gets the current directory, Directory.GetCurrentDirectory().
+        /// </summary>
+        /// <returns>The current directory.</returns>
+        string GetCurrentDirectory();
+
+        /// <summary>
+        /// Enumerates files in a directory, Directory.EnumerateFiles().
+        /// </summary>
+        /// <param name="dir">The directory to enumerate.</param>
+        /// <returns>A list of filenames, or an empty list.</returns>
+        IEnumerable<string> EnumerateFiles(string dir);
+
+        /// <summary>
+        /// Enumerates the directories in a directory, Directory.EnumerateDirectories().
+        /// </summary>
+        /// <param name="dir">The directory to enumerate.</param>
+        /// <returns>List of subdirectories, or empty list.</returns>
+        IEnumerable<string> EnumerateDirectories(string dir);
+
+        /// <summary>
+        /// True if we are running on Windows.
+        /// </summary>
+        bool IsWindows();
+
+        /// <summary>
+        /// Gets a value indicating whether we are running on macOS.
+        /// </summary>
+        /// <returns>True if we are running on macOS.</returns>
+        bool IsMacOs();
+
+        /// <summary>
+        /// Gets a value indicating whether we are running on Apple Silicon.
+        /// </summary>
+        /// <returns>True if we are running on Apple Silicon.</returns>
+        bool IsRunningOnAppleSilicon();
+
+        /// <summary>
+        /// Combine path segments, Path.Combine().
+        /// </summary>
+        /// <param name="parts">The parts of the path.</param>
+        /// <returns>The combined path.</returns>
+        string PathCombine(params string[] parts);
+
+        /// <summary>
+        /// Gets the full path for <paramref name="path"/>, Path.GetFullPath().
+        /// </summary>
+        string GetFullPath(string path);
+
+        /// <summary>
+        /// Returns the file name and extension of the specified path string.
+        /// </summary>
+        [return: NotNullIfNotNull(nameof(path))]
+        string? GetFileName(string? path);
+
+        /// <summary>
+        /// Returns the directory information for the specified path string.
+        /// </summary>
+        string? GetDirectoryName(string? path);
+
+        /// <summary>
+        /// Writes contents to file, File.WriteAllText().
+        /// </summary>
+        /// <param name="filename">The filename.</param>
+        /// <param name="contents">The text.</param>
+        void WriteAllText(string filename, string contents);
+
+        /// <summary>
+        /// Loads the XML document from <paramref name="filename"/>.
+        /// </summary>
+        XmlDocument LoadXml(string filename);
+
+        string EnvironmentExpandEnvironmentVariables(string s);
+
+        /// <summary>
+        /// Downloads the resource with the specified URI to a local file.
+        /// </summary>
+        void DownloadFile(string address, string fileName);
+
+        /// <summary>
+        /// Creates an <see cref="IDiagnosticsWriter" /> for the given <paramref name="filename" />.
+        /// </summary>
+        /// <param name="filename">
+        /// The path suggesting where the diagnostics should be written to.
+        /// </param>
+        /// <returns>
+        /// A <see cref="IDiagnosticsWriter" /> to which diagnostic entries can be added.
+        /// </returns>
+        IDiagnosticsWriter CreateDiagnosticsWriter(string filename);
+    }
+
+    /// <summary>
+    /// An implementation of IBuildActions that actually performs the requested operations.
+    /// </summary>
+    public class SystemBuildActions : IBuildActions
+    {
+        void IBuildActions.FileDelete(string file) => File.Delete(file);
+
+        bool IBuildActions.FileExists(string file) => File.Exists(file);
+
+        private static ProcessStartInfo GetProcessStartInfo(string exe, string arguments, string? workingDirectory, IDictionary<string, string>? environment)
+        {
+            var pi = new ProcessStartInfo(exe, arguments)
+            {
+                UseShellExecute = false,
+                RedirectStandardOutput = true
+            };
+            if (workingDirectory is not null)
+                pi.WorkingDirectory = workingDirectory;
+
+            environment?.ForEach(kvp => pi.Environment[kvp.Key] = kvp.Value);
+
+            return pi;
+        }
+
+        int IBuildActions.RunProcess(string exe, string args, string? workingDirectory, System.Collections.Generic.IDictionary<string, string>? env, BuildOutputHandler onOutput, BuildOutputHandler onError)
+        {
+            var pi = GetProcessStartInfo(exe, args, workingDirectory, env);
+            pi.RedirectStandardError = true;
+
+            return pi.ReadOutput(out _, onOut: s => onOutput(s), onError: s => onError(s));
+        }
+
+        int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory, IDictionary<string, string>? environment)
+        {
+            var pi = GetProcessStartInfo(cmd, args, workingDirectory, environment);
+            return pi.ReadOutput(out _, onOut: Console.WriteLine, onError: null);
+        }
+
+        int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory, IDictionary<string, string>? environment, out IList<string> stdOut)
+        {
+            var pi = GetProcessStartInfo(cmd, args, workingDirectory, environment);
+            return pi.ReadOutput(out stdOut, onOut: null, onError: null);
+        }
+
+        void IBuildActions.DirectoryDelete(string dir, bool recursive) => Directory.Delete(dir, recursive);
+
+        bool IBuildActions.DirectoryExists(string dir) => Directory.Exists(dir);
+
+        void IBuildActions.CreateDirectory(string path) => Directory.CreateDirectory(path);
+
+        string? IBuildActions.GetEnvironmentVariable(string name) => Environment.GetEnvironmentVariable(name);
+
+        string IBuildActions.GetCurrentDirectory() => Directory.GetCurrentDirectory();
+
+        IEnumerable<string> IBuildActions.EnumerateFiles(string dir) => Directory.EnumerateFiles(dir);
+
+        IEnumerable<string> IBuildActions.EnumerateDirectories(string dir) => Directory.EnumerateDirectories(dir);
+
+        bool IBuildActions.IsWindows() => Win32.IsWindows();
+
+        bool IBuildActions.IsMacOs() => RuntimeInformation.IsOSPlatform(OSPlatform.OSX);
+
+        bool IBuildActions.IsRunningOnAppleSilicon()
+        {
+            var thisBuildActions = (IBuildActions)this;
+
+            if (!thisBuildActions.IsMacOs())
+            {
+                return false;
+            }
+
+            try
+            {
+                thisBuildActions.RunProcess("sysctl", "machdep.cpu.brand_string", workingDirectory: null, env: null, out var stdOut);
+                return stdOut?.Any(s => s?.ToLowerInvariant().Contains("apple") == true) ?? false;
+            }
+            catch (Exception)
+            {
+                return false;
+            }
+        }
+
+        string IBuildActions.PathCombine(params string[] parts) => Path.Combine(parts);
+
+        void IBuildActions.WriteAllText(string filename, string contents) => File.WriteAllText(filename, contents);
+
+        XmlDocument IBuildActions.LoadXml(string filename)
+        {
+            var ret = new XmlDocument();
+            ret.Load(filename);
+            return ret;
+        }
+
+        string IBuildActions.GetFullPath(string path) => Path.GetFullPath(path);
+
+        string? IBuildActions.GetFileName(string? path) => Path.GetFileName(path);
+
+        string? IBuildActions.GetDirectoryName(string? path) => Path.GetDirectoryName(path);
+
+        public string EnvironmentExpandEnvironmentVariables(string s) => Environment.ExpandEnvironmentVariables(s);
+
+        public void DownloadFile(string address, string fileName) =>
+            FileUtils.DownloadFile(address, fileName);
+
+        public IDiagnosticsWriter CreateDiagnosticsWriter(string filename) => new DiagnosticsStream(filename);
+
+        public static IBuildActions Instance { get; } = new SystemBuildActions();
+    }
+
+    public static class BuildActionExtensions
+    {
+        private static void FindFiles(this IBuildActions actions, string dir, int depth, int? maxDepth, IList<(string, int)> results)
+        {
+            foreach (var f in actions.EnumerateFiles(dir))
+            {
+                results.Add((f, depth));
+            }
+
+            if (depth < maxDepth)
+            {
+                foreach (var d in actions.EnumerateDirectories(dir))
+                {
+                    actions.FindFiles(d, depth + 1, maxDepth, results);
+                }
+            }
+        }
+
+        public static (string path, int depth)[] FindFiles(this IBuildActions actions, string dir, int? maxDepth)
+        {
+            var results = new List<(string, int)>();
+            actions.FindFiles(dir, 0, maxDepth, results);
+            return results.OrderBy(f => f.Item2).ToArray();
+        }
+    }
+}

csharp/extractor/Semmle.Util/BuildScript.cs

@@ -0,0 +1,399 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+
+namespace Semmle.Util
+{
+    /// <summary>
+    /// A build script.
+    /// </summary>
+    public abstract class BuildScript
+    {
+        /// <summary>
+        /// Run this build script.
+        /// </summary>
+        /// <param name="actions">
+        /// The interface used to implement the build actions.
+        /// </param>
+        /// <param name="startCallback">
+        /// A call back that is called every time a new process is started. The
+        /// argument to the call back is a textual representation of the process.
+        /// </param>
+        /// <param name="exitCallBack">
+        /// A call back that is called every time a new process exits. The first
+        /// argument to the call back is the exit code, and the second argument is
+        /// an exit message.
+        /// </param>
+        /// <returns>The exit code from this build script.</returns>
+        public abstract int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack);
+
+        /// <summary>
+        /// Run this build command.
+        /// </summary>
+        /// <param name="actions">
+        /// The interface used to implement the build actions.
+        /// </param>
+        /// <param name="startCallback">
+        /// A call back that is called every time a new process is started. The
+        /// argument to the call back is a textual representation of the process.
+        /// </param>
+        /// <param name="exitCallBack">
+        /// A call back that is called every time a new process exits. The first
+        /// argument to the call back is the exit code, and the second argument is
+        /// an exit message.
+        /// </param>
+        /// <param name="stdout">Contents of standard out.</param>
+        /// <returns>The exit code from this build script.</returns>
+        public abstract int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, out IList<string> stdout);
+
+        /// <summary>
+        /// Runs this build command.
+        /// </summary>
+        /// <param name="actions">
+        /// The interface used to implement the build actions.
+        /// </param>
+        /// <param name="startCallback">
+        /// A call back that is called every time a new process is started. The
+        /// argument to the call back is a textual representation of the process.
+        /// </param>
+        /// <param name="exitCallBack">
+        /// A call back that is called every time a new process exits. The first
+        /// argument to the call back is the exit code, and the second argument is
+        /// an exit message.
+        /// </param>
+        /// <param name="onOutput">
+        /// A handler for data read from stdout.
+        /// </param>
+        /// <param name="onError">
+        /// A handler for data read from stderr.
+        /// </param>
+        /// <returns>The exit code from this build script.</returns>
+        public abstract int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, BuildOutputHandler onOutput, BuildOutputHandler onError);
+
+        /// <summary>
+        /// A build script which executes an external program or script.
+        /// </summary>
+        private class BuildCommand : BuildScript
+        {
+            private readonly string exe, arguments;
+            private readonly string? workingDirectory;
+            private readonly IDictionary<string, string>? environment;
+            private readonly bool silent;
+
+            /// <summary>
+            /// Create a simple build command.
+            /// </summary>
+            /// <param name="exe">The executable to run.</param>
+            /// <param name="argumentsOpt">The arguments to the executable, or null.</param>
+            /// <param name="silent">Whether this command should run silently.</param>
+            /// <param name="workingDirectory">The working directory (<code>null</code> for current directory).</param>
+            /// <param name="environment">Additional environment variables.</param>
+            public BuildCommand(string exe, string? argumentsOpt, bool silent, string? workingDirectory = null, IDictionary<string, string>? environment = null)
+            {
+                this.exe = exe;
+                this.arguments = argumentsOpt ?? "";
+                this.silent = silent;
+                this.workingDirectory = workingDirectory;
+                this.environment = environment;
+            }
+
+            public override string ToString() => arguments.Length > 0 ? exe + " " + arguments : exe;
+
+            public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack)
+            {
+                startCallback(this.ToString(), silent);
+                var ret = 1;
+                var retMessage = "";
+                try
+                {
+                    ret = actions.RunProcess(exe, arguments, workingDirectory, environment);
+                }
+                catch (Exception ex)
+                    when (ex is System.ComponentModel.Win32Exception || ex is FileNotFoundException)
+                {
+                    retMessage = ex.Message;
+                }
+
+                exitCallBack(ret, retMessage, silent);
+                return ret;
+            }
+
+            public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, out IList<string> stdout)
+            {
+                startCallback(this.ToString(), silent);
+                var ret = 1;
+                var retMessage = "";
+                try
+                {
+                    ret = actions.RunProcess(exe, arguments, workingDirectory, environment, out stdout);
+                }
+                catch (Exception ex)
+                    when (ex is System.ComponentModel.Win32Exception || ex is FileNotFoundException)
+                {
+                    retMessage = ex.Message;
+                    stdout = Array.Empty<string>();
+                }
+                exitCallBack(ret, retMessage, silent);
+                return ret;
+            }
+
+            public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, BuildOutputHandler onOutput, BuildOutputHandler onError)
+            {
+                startCallback(this.ToString(), silent);
+                var ret = 1;
+                var retMessage = "";
+                try
+                {
+                    ret = actions.RunProcess(exe, arguments, workingDirectory, environment, onOutput, onError);
+                }
+                catch (Exception ex)
+                    when (ex is System.ComponentModel.Win32Exception || ex is FileNotFoundException)
+                {
+                    retMessage = ex.Message;
+                }
+                exitCallBack(ret, retMessage, silent);
+                return ret;
+            }
+
+        }
+
+        /// <summary>
+        /// A build script which runs a C# function.
+        /// </summary>
+        private class ReturnBuildCommand : BuildScript
+        {
+            private readonly Func<IBuildActions, int> func;
+            public ReturnBuildCommand(Func<IBuildActions, int> func)
+            {
+                this.func = func;
+            }
+
+            public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack) => func(actions);
+
+            public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, out IList<string> stdout)
+            {
+                stdout = Array.Empty<string>();
+                return func(actions);
+            }
+
+            public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, BuildOutputHandler onOutput, BuildOutputHandler onError) => func(actions);
+        }
+
+        /// <summary>
+        /// Allows two build scripts to be composed sequentially.
+        /// </summary>
+        private class BindBuildScript : BuildScript
+        {
+            private readonly BuildScript s1;
+            private readonly Func<IList<string>, int, BuildScript>? s2a;
+            private readonly Func<int, BuildScript>? s2b;
+
+            public BindBuildScript(BuildScript s1, Func<IList<string>, int, BuildScript> s2)
+            {
+                this.s1 = s1;
+                this.s2a = s2;
+            }
+
+            public BindBuildScript(BuildScript s1, Func<int, BuildScript> s2)
+            {
+                this.s1 = s1;
+                this.s2b = s2;
+            }
+
+            public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack)
+            {
+                int ret1;
+                if (s2a is not null)
+                {
+                    ret1 = s1.Run(actions, startCallback, exitCallBack, out var stdout1);
+                    return s2a(stdout1, ret1).Run(actions, startCallback, exitCallBack);
+                }
+
+                if (s2b is not null)
+                {
+                    ret1 = s1.Run(actions, startCallback, exitCallBack);
+                    return s2b(ret1).Run(actions, startCallback, exitCallBack);
+                }
+
+                throw new InvalidOperationException("Unexpected error");
+            }
+
+            public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, out IList<string> stdout)
+            {
+                var ret1 = s1.Run(actions, startCallback, exitCallBack, out var stdout1);
+                var ret2 = (s2a is not null ? s2a(stdout1, ret1) : s2b!(ret1)).Run(actions, startCallback, exitCallBack, out var stdout2);
+                var @out = new List<string>();
+                @out.AddRange(stdout1);
+                @out.AddRange(stdout2);
+                stdout = @out;
+                return ret2;
+            }
+
+            public override int Run(IBuildActions actions, Action<string, bool> startCallback, Action<int, string, bool> exitCallBack, BuildOutputHandler onOutput, BuildOutputHandler onError)
+            {
+                int ret1;
+                if (s2a is not null)
+                {
+                    var stdout1 = new List<string>();
+                    var onOutputWrapper = new BuildOutputHandler(data =>
+                    {
+                        if (data is not null)
+                            stdout1.Add(data);
+
+                        onOutput(data);
+                    });
+                    ret1 = s1.Run(actions, startCallback, exitCallBack, onOutputWrapper, onError);
+                    return s2a(stdout1, ret1).Run(actions, startCallback, exitCallBack, onOutput, onError);
+                }
+
+                if (s2b is not null)
+                {
+                    ret1 = s1.Run(actions, startCallback, exitCallBack, onOutput, onError);
+                    return s2b(ret1).Run(actions, startCallback, exitCallBack, onOutput, onError);
+                }
+
+                throw new InvalidOperationException("Unexpected error");
+            }
+        }
+
+        /// <summary>
+        /// Creates a simple build script that runs the specified exe.
+        /// </summary>
+        /// <param name="argumentsOpt">The arguments to the executable, or null.</param>
+        /// <param name="silent">Whether the executable should run silently.</param>
+        /// <param name="workingDirectory">The working directory (<code>null</code> for current directory).</param>
+        /// <param name="environment">Additional environment variables.</param>
+        public static BuildScript Create(string exe, string? argumentsOpt, bool silent, string? workingDirectory, IDictionary<string, string>? environment) =>
+            new BuildCommand(exe, argumentsOpt, silent, workingDirectory, environment);
+
+        /// <summary>
+        /// Creates a simple build script that runs the specified function.
+        /// </summary>
+        public static BuildScript Create(Func<IBuildActions, int> func) =>
+            new ReturnBuildCommand(func);
+
+        /// <summary>
+        /// Creates a build script that downloads the specified file.
+        /// </summary>
+        public static BuildScript DownloadFile(string address, string fileName, Action<Exception> exceptionCallback) =>
+            Create(actions =>
+            {
+                if (actions.GetDirectoryName(fileName) is string dir && !string.IsNullOrWhiteSpace(dir))
+                    actions.CreateDirectory(dir);
+                try
+                {
+                    actions.DownloadFile(address, fileName);
+                    return 0;
+                }
+                catch (Exception e)
+                {
+                    exceptionCallback(e);
+                    return 1;
+                }
+            });
+
+        /// <summary>
+        /// Creates a build script that runs <paramref name="s1"/>, followed by running the script
+        /// produced by <paramref name="s2"/> on the exit code from <paramref name="s1"/>.
+        /// </summary>
+        public static BuildScript Bind(BuildScript s1, Func<int, BuildScript> s2) =>
+            new BindBuildScript(s1, s2);
+
+        /// <summary>
+        /// Creates a build script that runs <paramref name="s1"/>, followed by running the script
+        /// produced by <paramref name="s2"/> on the exit code and standard output from
+        /// <paramref name="s1"/>.
+        /// </summary>
+        public static BuildScript Bind(BuildScript s1, Func<IList<string>, int, BuildScript> s2) =>
+            new BindBuildScript(s1, s2);
+
+        private const int successCode = 0;
+        /// <summary>
+        /// The empty build script that always returns exit code 0.
+        /// </summary>
+        public static BuildScript Success { get; } = Create(actions => successCode);
+
+        private const int failureCode = 1;
+        /// <summary>
+        /// The empty build script that always returns exit code 1.
+        /// </summary>
+        public static BuildScript Failure { get; } = Create(actions => failureCode);
+
+        private static bool Succeeded(int i) => i == successCode;
+
+        public static BuildScript operator &(BuildScript s1, BuildScript s2) =>
+            new BindBuildScript(s1, ret1 => Succeeded(ret1) ? s2 : Create(actions => ret1));
+
+        public static BuildScript operator &(BuildScript s1, Func<BuildScript> s2) =>
+            new BindBuildScript(s1, ret1 => Succeeded(ret1) ? s2() : Create(actions => ret1));
+
+        public static BuildScript operator |(BuildScript s1, BuildScript s2) =>
+            new BindBuildScript(s1, ret1 => Succeeded(ret1) ? Success : s2);
+
+        public static BuildScript operator |(BuildScript s1, Func<BuildScript> s2) =>
+            new BindBuildScript(s1, ret1 => Succeeded(ret1) ? Success : s2());
+
+        /// <summary>
+        /// Creates a build script that attempts to run the build script <paramref name="s"/>,
+        /// always returning a successful exit code.
+        /// </summary>
+        public static BuildScript Try(BuildScript s) => s | Success;
+
+        /// <summary>
+        /// Creates a build script that runs the build script <paramref name="s" />. If
+        /// running <paramref name="s" /> fails, <paramref name="k" /> is invoked with
+        /// the exit code.
+        /// </summary>
+        /// <param name="s">The build script to run.</param>
+        /// <param name="k">
+        /// The callback that is invoked if <paramref name="s" /> failed.
+        /// </param>
+        /// <returns>The build script which implements this.</returns>
+        public static BuildScript OnFailure(BuildScript s, Action<int> k) =>
+            new BindBuildScript(s, ret => Create(actions =>
+            {
+                if (!Succeeded(ret)) k(ret);
+                return ret;
+            }));
+
+        /// <summary>
+        /// Creates a build script that deletes the given directory.
+        /// </summary>
+        public static BuildScript DeleteDirectory(string dir) =>
+            Create(actions =>
+            {
+                if (string.IsNullOrEmpty(dir) || !actions.DirectoryExists(dir))
+                    return failureCode;
+
+                try
+                {
+                    actions.DirectoryDelete(dir, true);
+                }
+                catch  // lgtm[cs/catch-of-all-exceptions]
+                {
+                    return failureCode;
+                }
+                return successCode;
+            });
+
+        /// <summary>
+        /// Creates a build script that deletes the given file.
+        /// </summary>
+        public static BuildScript DeleteFile(string file) =>
+            Create(actions =>
+            {
+                if (string.IsNullOrEmpty(file) || !actions.FileExists(file))
+                    return failureCode;
+
+                try
+                {
+                    actions.FileDelete(file);
+                }
+                catch  // lgtm[cs/catch-of-all-exceptions]
+                {
+                    return failureCode;
+                }
+                return successCode;
+            });
+    }
+}

csharp/extractor/Semmle.Util/CanonicalPathCache.cs

@@ -268,7 +268,7 @@ namespace Semmle.Util
                     catch  // lgtm[cs/catch-of-all-exceptions]
                     {
                         // Failed to late-bind a suitable library.
-                        logger.Log(Severity.Warning, "Preserving symlinks in canonical paths");
+                        logger.LogWarning("Preserving symlinks in canonical paths");
                         pathStrategy = new QueryDirectoryStrategy();
                     }
                     break;

csharp/extractor/Semmle.Util/CommandBuilder.cs

@@ -0,0 +1,197 @@
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace Semmle.Util
+{
+    /// <summary>
+    /// Utility to construct a build command.
+    /// </summary>
+    public class CommandBuilder
+    {
+        private enum EscapeMode { Process, Cmd };
+
+        private readonly StringBuilder arguments;
+        private bool firstCommand;
+        private string? executable;
+        private readonly EscapeMode escapingMode;
+        private readonly string? workingDirectory;
+        private readonly IDictionary<string, string>? environment;
+        private readonly bool silent;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="T:Semmle.Autobuild.CommandBuilder"/> class.
+        /// </summary>
+        /// <param name="workingDirectory">The working directory (<code>null</code> for current directory).</param>
+        /// <param name="environment">Additional environment variables.</param>
+        /// <param name="silent">Whether this command should be run silently.</param>
+        public CommandBuilder(IBuildActions actions, string? workingDirectory = null, IDictionary<string, string>? environment = null, bool silent = false)
+        {
+            arguments = new StringBuilder();
+            if (actions.IsWindows())
+            {
+                executable = "cmd.exe";
+                arguments.Append("/C");
+                escapingMode = EscapeMode.Cmd;
+            }
+            else
+            {
+                escapingMode = EscapeMode.Process;
+            }
+
+            firstCommand = true;
+            this.workingDirectory = workingDirectory;
+            this.environment = environment;
+            this.silent = silent;
+        }
+
+        public CommandBuilder CallBatFile(string batFile, string? argumentsOpt = null)
+        {
+            NextCommand();
+            arguments.Append(" CALL");
+            QuoteArgument(batFile);
+            Argument(argumentsOpt);
+            return this;
+        }
+
+        private static readonly char[] specialChars = { ' ', '\t', '\n', '\v', '\"' };
+        private static readonly char[] cmdMetacharacter = { '(', ')', '%', '!', '^', '\"', '<', '>', '&', '|', ' ' };
+
+        /// <summary>
+        /// Appends the given argument to the command line.
+        /// </summary>
+        /// <param name="argument">The argument to append.</param>
+        /// <param name="force">Whether to always quote the argument.</param>
+        /// <param name="cmd">Whether to escape for cmd.exe</param>
+        ///
+        /// <remarks>
+        /// This implementation is copied from
+        /// https://blogs.msdn.microsoft.com/twistylittlepassagesallalike/2011/04/23/everyone-quotes-command-line-arguments-the-wrong-way/
+        /// </remarks>
+        private void ArgvQuote(string argument, bool force)
+        {
+            var cmd = escapingMode == EscapeMode.Cmd;
+            if (!force &&
+                !string.IsNullOrEmpty(argument) &&
+                argument.IndexOfAny(specialChars) == -1)
+            {
+                arguments.Append(argument);
+            }
+            else
+            {
+                if (cmd)
+                    arguments.Append('^');
+
+                arguments.Append('\"');
+
+                for (var it = 0; ; ++it)
+                {
+                    var numBackslashes = 0;
+                    while (it != argument.Length && argument[it] == '\\')
+                    {
+                        ++it;
+                        ++numBackslashes;
+                    }
+
+                    if (it == argument.Length)
+                    {
+                        arguments.Append('\\', numBackslashes * 2);
+                        break;
+                    }
+
+                    if (argument[it] == '\"')
+                    {
+                        arguments.Append('\\', numBackslashes * 2 + 1);
+                        if (cmd)
+                            arguments.Append('^');
+                        arguments.Append(arguments[it]);
+                    }
+                    else
+                    {
+                        arguments.Append('\\', numBackslashes);
+                        if (cmd && cmdMetacharacter.Any(c => c == argument[it]))
+                            arguments.Append('^');
+
+                        arguments.Append(argument[it]);
+                    }
+                }
+
+                if (cmd)
+                    arguments.Append('^');
+
+                arguments.Append('\"');
+            }
+        }
+
+        public CommandBuilder QuoteArgument(string argumentsOpt)
+        {
+            if (argumentsOpt is not null)
+            {
+                NextArgument();
+                ArgvQuote(argumentsOpt, false);
+            }
+            return this;
+        }
+
+        private void NextArgument()
+        {
+            if (arguments.Length > 0)
+                arguments.Append(' ');
+        }
+
+        public CommandBuilder Argument(string? argumentsOpt)
+        {
+            if (argumentsOpt is not null)
+            {
+                NextArgument();
+                arguments.Append(argumentsOpt);
+            }
+            return this;
+        }
+
+        private void NextCommand()
+        {
+            if (firstCommand)
+                firstCommand = false;
+            else
+                arguments.Append(" &&");
+        }
+
+        public CommandBuilder RunCommand(string exe, string? argumentsOpt = null, bool quoteExe = true)
+        {
+            var (exe0, arg0) =
+                escapingMode == EscapeMode.Process && exe.EndsWith(".exe", System.StringComparison.Ordinal)
+                                          ? ("mono", exe) // Linux
+                                          : (exe, null);
+
+            NextCommand();
+            if (executable is null)
+            {
+                executable = exe0;
+            }
+            else
+            {
+                if (quoteExe)
+                    QuoteArgument(exe0);
+                else
+                    Argument(exe0);
+            }
+            Argument(arg0);
+            Argument(argumentsOpt);
+            return this;
+        }
+
+        /// <summary>
+        /// Returns a build script that contains just this command.
+        /// </summary>
+        public BuildScript Script
+        {
+            get
+            {
+                if (executable is null)
+                    throw new System.InvalidOperationException("executable is null");
+                return BuildScript.Create(executable, arguments.ToString(), silent, workingDirectory, environment);
+            }
+        }
+    }
+}

csharp/extractor/Semmle.Util/FileUtils.cs

@@ -133,14 +133,14 @@ namespace Semmle.Util
                 var directoryName = Path.GetDirectoryName(nested);
                 if (directoryName is null)
                 {
-                    logger.Log(Severity.Warning, "Failed to get directory name from path '" + nested + "'.");
+                    logger.LogWarning("Failed to get directory name from path '" + nested + "'.");
                     throw new InvalidOperationException();
                 }
                 Directory.CreateDirectory(directoryName);
             }
             catch (PathTooLongException)
             {
-                logger.Log(Severity.Warning, "Failed to create parent directory of '" + nested + "': Path too long.");
+                logger.LogWarning("Failed to create parent directory of '" + nested + "': Path too long.");
                 throw;
             }
             return nested;