C#: Fetch .NET in dependency manager instead of autobuilder

cpp/autobuilder/Semmle.Autobuild.Cpp/Program.cs

@@ -1,5 +1,7 @@
 using System;
+
 using Semmle.Autobuild.Shared;
+using Semmle.Util;
 
 namespace Semmle.Autobuild.Cpp
 {

csharp/CSharp.sln

@@ -34,6 +34,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Semmle.Autobuild.CSharp.Tes
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Semmle.Extraction.CSharp.DependencyStubGenerator", "extractor\Semmle.Extraction.CSharp.DependencyStubGenerator\Semmle.Extraction.CSharp.DependencyStubGenerator.csproj", "{0EDA21A3-ADD8-4C10-B494-58B12B526B76}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Semmle.Autobuild.Cpp", "..\cpp\autobuilder\Semmle.Autobuild.Cpp\Semmle.Autobuild.Cpp.csproj", "{125C4FB7-34DA-442A-9095-3EA1514270CD}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Semmle.Autobuild.Cpp.Tests", "..\cpp\autobuilder\Semmle.Autobuild.Cpp.Tests\Semmle.Autobuild.Cpp.Tests.csproj", "{72F369B7-0707-401A-802F-D526F272F9EE}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -102,6 +106,14 @@ Global
 		{0EDA21A3-ADD8-4C10-B494-58B12B526B76}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{0EDA21A3-ADD8-4C10-B494-58B12B526B76}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{0EDA21A3-ADD8-4C10-B494-58B12B526B76}.Release|Any CPU.Build.0 = Release|Any CPU
+		{125C4FB7-34DA-442A-9095-3EA1514270CD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{125C4FB7-34DA-442A-9095-3EA1514270CD}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{125C4FB7-34DA-442A-9095-3EA1514270CD}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{125C4FB7-34DA-442A-9095-3EA1514270CD}.Release|Any CPU.Build.0 = Release|Any CPU
+		{72F369B7-0707-401A-802F-D526F272F9EE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{72F369B7-0707-401A-802F-D526F272F9EE}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{72F369B7-0707-401A-802F-D526F272F9EE}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{72F369B7-0707-401A-802F-D526F272F9EE}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs

@@ -558,8 +558,6 @@ namespace Semmle.Autobuild.CSharp.Tests
         [Fact]
         public void TestLinuxBuildlessExtractionSuccess()
         {
-            actions.RunProcess["dotnet --list-sdks"] = 0;
-            actions.RunProcessOut["dotnet --list-sdks"] = "any version";
             actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 0;
             actions.FileExists["csharp.log"] = true;
             actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
@@ -569,14 +567,12 @@ namespace Semmle.Autobuild.CSharp.Tests
             actions.EnumerateDirectories[@"C:\Project"] = "";
 
             var autobuilder = CreateAutoBuilder(false, buildless: "true");
-            TestAutobuilderScript(autobuilder, 0, 2);
+            TestAutobuilderScript(autobuilder, 0, 1);
         }
 
         [Fact]
         public void TestLinuxBuildlessExtractionFailed()
         {
-            actions.RunProcess["dotnet --list-sdks"] = 0;
-            actions.RunProcessOut["dotnet --list-sdks"] = "any version";
             actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 10;
             actions.FileExists["csharp.log"] = true;
             actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
@@ -586,14 +582,12 @@ namespace Semmle.Autobuild.CSharp.Tests
             actions.EnumerateDirectories[@"C:\Project"] = "";
 
             var autobuilder = CreateAutoBuilder(false, buildless: "true");
-            TestAutobuilderScript(autobuilder, 10, 2);
+            TestAutobuilderScript(autobuilder, 10, 1);
         }
 
         [Fact]
         public void TestLinuxBuildlessExtractionSolution()
         {
-            actions.RunProcess["dotnet --list-sdks"] = 0;
-            actions.RunProcessOut["dotnet --list-sdks"] = "any version";
             actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 0;
             actions.FileExists["csharp.log"] = true;
             actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
@@ -603,28 +597,7 @@ namespace Semmle.Autobuild.CSharp.Tests
             actions.EnumerateDirectories[@"C:\Project"] = "";
 
             var autobuilder = CreateAutoBuilder(false, buildless: "true");
-            TestAutobuilderScript(autobuilder, 0, 2);
+            TestAutobuilderScript(autobuilder, 0, 1);
-        }
-
-        [Fact]
-        public void TestLinuxBuildlessExtractionNoDotnet()
-        {
-            actions.RunProcess["dotnet --list-sdks"] = 1;
-            actions.RunProcessOut["dotnet --list-sdks"] = "";
-            actions.RunProcess[@"chmod u+x scratch/.dotnet/dotnet-install.sh"] = 0;
-            actions.RunProcess[@"scratch/.dotnet/dotnet-install.sh --channel release --version 8.0.101 --install-dir scratch/.dotnet"] = 0;
-            actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone --dotnet scratch/.dotnet"] = 0;
-            actions.FileExists["csharp.log"] = true;
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
-            actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
-            actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
-            actions.EnumerateDirectories[@"C:\Project"] = "";
-            actions.DownloadFiles.Add(("https://dot.net/v1/dotnet-install.sh", "scratch/.dotnet/dotnet-install.sh"));
-            actions.CreateDirectories.Add(@"scratch/.dotnet");
-
-            var autobuilder = CreateAutoBuilder(false, buildless: "true");
-            TestAutobuilderScript(autobuilder, 0, 4);
         }
 
         private void SkipVsWhere()
@@ -915,8 +888,6 @@ namespace Semmle.Autobuild.CSharp.Tests
         [Fact]
         public void TestSkipNugetBuildless()
         {
-            actions.RunProcess["dotnet --list-sdks"] = 0;
-            actions.RunProcessOut["dotnet --list-sdks"] = "any version";
             actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 0;
             actions.FileExists["csharp.log"] = true;
             actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
@@ -926,7 +897,7 @@ namespace Semmle.Autobuild.CSharp.Tests
             actions.EnumerateDirectories[@"C:\Project"] = "";
 
             var autobuilder = CreateAutoBuilder(false, buildless: "true");
-            TestAutobuilderScript(autobuilder, 0, 2);
+            TestAutobuilderScript(autobuilder, 0, 1);
         }
 
 

csharp/autobuilder/Semmle.Autobuild.CSharp/CSharpAutobuilder.cs

@@ -50,11 +50,8 @@ namespace Semmle.Autobuild.CSharp
                     attempt = new BuildCommandRule(DotNetRule.WithDotNet).Analyse(this, false) & CheckExtractorRun(true);
                     break;
                 case CSharpBuildStrategy.Buildless:
-                    attempt = DotNetRule.WithDotNet(this, ensureDotNetAvailable: true, (dotNetPath, env) =>
+                    // No need to check that the extractor has been executed in buildless mode
-                        {
+                    attempt = new StandaloneBuildRule().Analyse(this, false);
-                            // No need to check that the extractor has been executed in buildless mode
-                            return new StandaloneBuildRule(dotNetPath).Analyse(this, false);
-                        });
                     break;
                 case CSharpBuildStrategy.MSBuild:
                     attempt = new MsBuildRule().Analyse(this, false) & CheckExtractorRun(true);
@@ -84,7 +81,7 @@ namespace Semmle.Autobuild.CSharp
                     return 0;
 
                 if (warnOnFailure)
-                    Log(Severity.Error, "No C# code detected during build.");
+                    Logger.LogError("No C# code detected during build.");
 
                 return 1;
             });

csharp/autobuilder/Semmle.Autobuild.CSharp/Constants.cs

@@ -1,8 +0,0 @@
-namespace Semmle.Autobuild.CSharp
-{
-    internal static class Constants
-    {
-        // The version number should be kept in sync with the version .NET version used for building the application.
-        public const string LatestDotNetSdkVersion = "8.0.101";
-    }
-}

csharp/autobuilder/Semmle.Autobuild.CSharp/DotNetRule.cs

@@ -5,7 +5,7 @@ using System.Linq;
 using Semmle.Util;
 using Semmle.Util.Logging;
 using Semmle.Autobuild.Shared;
-using Newtonsoft.Json.Linq;
+using Semmle.Extraction.CSharp.DependencyFetching;
 
 namespace Semmle.Autobuild.CSharp
 {
@@ -39,11 +39,11 @@ namespace Semmle.Autobuild.CSharp
 
                 if (notDotNetProject is not null)
                 {
-                    builder.Log(Severity.Info, "Not using .NET Core because of incompatible project {0}", notDotNetProject);
+                    builder.Logger.Log(Severity.Info, "Not using .NET Core because of incompatible project {0}", notDotNetProject);
                     return BuildScript.Failure;
                 }
 
-                builder.Log(Severity.Info, "Attempting to build using .NET Core");
+                builder.Logger.LogInfo("Attempting to build using .NET Core");
             }
 
             return WithDotNet(builder, ensureDotNetAvailable: false, (dotNetPath, environment) =>
@@ -81,29 +81,22 @@ namespace Semmle.Autobuild.CSharp
         /// </summary>
         public static BuildScript WithDotNet(IAutobuilder<AutobuildOptionsShared> builder, bool ensureDotNetAvailable, Func<string?, IDictionary<string, string>?, BuildScript> f)
         {
-            var installDir = builder.Actions.PathCombine(FileUtils.GetTemporaryWorkingDirectory(builder.Actions.GetEnvironmentVariable, builder.Options.Language.UpperCaseName, out var _), ".dotnet");
+            var temp = FileUtils.GetTemporaryWorkingDirectory(builder.Actions.GetEnvironmentVariable, builder.Options.Language.UpperCaseName, out var shouldCleanUp);
-            var installScript = DownloadDotNet(builder, installDir, ensureDotNetAvailable);
+            return DotNet.WithDotNet(builder.Actions, builder.Logger, builder.Paths.Select(x => x.Item1), temp, shouldCleanUp, ensureDotNetAvailable, builder.Options.DotNetVersion, installDir =>
-            return BuildScript.Bind(installScript, installed =>
             {
                 var env = new Dictionary<string, string>
                 {
                     { "DOTNET_SKIP_FIRST_TIME_EXPERIENCE", "true" },
                     { "MSBUILDDISABLENODEREUSE", "1" }
                 };
-                if (installed == 0)
+                if (installDir is not null)
                 {
-                    // The installation succeeded, so use the newly installed .NET Core
+                    // The installation succeeded, so use the newly installed .NET
                     var path = builder.Actions.GetEnvironmentVariable("PATH");
                     var delim = builder.Actions.IsWindows() ? ";" : ":";
-                    env.Add("DOTNET_MULTILEVEL_LOOKUP", "false"); // prevent look up of other .NET Core SDKs
+                    env.Add("DOTNET_MULTILEVEL_LOOKUP", "false"); // prevent look up of other .NET SDKs
                     env.Add("PATH", installDir + delim + path);
                 }
-                else
-                {
-                    // The .NET SDK was not installed, either because the installation failed or because it was already installed.
-                    installDir = null;
-                }
-
                 return f(installDir, env);
             });
         }
@@ -119,146 +112,6 @@ namespace Semmle.Autobuild.CSharp
         public static BuildScript WithDotNet(IAutobuilder<AutobuildOptionsShared> builder, Func<IDictionary<string, string>?, BuildScript> f)
             => WithDotNet(builder, ensureDotNetAvailable: false, (_, env) => f(env));
 
-        /// <summary>
-        /// Returns a script for downloading relevant versions of the
-        /// .NET Core SDK. The SDK(s) will be installed at <code>installDir</code>
-        /// (provided that the script succeeds).
-        /// </summary>
-        private static BuildScript DownloadDotNet(IAutobuilder<AutobuildOptionsShared> builder, string installDir, bool ensureDotNetAvailable)
-        {
-            if (!string.IsNullOrEmpty(builder.Options.DotNetVersion))
-                // Specific version supplied in configuration: always use that
-                return DownloadDotNetVersion(builder, installDir, builder.Options.DotNetVersion);
-
-            // Download versions mentioned in `global.json` files
-            // See https://docs.microsoft.com/en-us/dotnet/core/tools/global-json
-            var installScript = BuildScript.Success;
-            var validGlobalJson = false;
-            foreach (var path in builder.Paths.Select(p => p.Item1).Where(p => p.EndsWith("global.json", StringComparison.Ordinal)))
-            {
-                string version;
-                try
-                {
-                    var o = JObject.Parse(File.ReadAllText(path));
-                    version = (string)(o?["sdk"]?["version"]!);
-                }
-                catch  // lgtm[cs/catch-of-all-exceptions]
-                {
-                    // not a valid global.json file
-                    continue;
-                }
-
-                installScript &= DownloadDotNetVersion(builder, installDir, version);
-                validGlobalJson = true;
-            }
-
-            if (validGlobalJson)
-            {
-                return installScript;
-            }
-
-            if (ensureDotNetAvailable)
-            {
-                return DownloadDotNetVersion(builder, installDir, Constants.LatestDotNetSdkVersion, needExactVersion: false);
-            }
-
-            return BuildScript.Failure;
-        }
-
-        /// <summary>
-        /// Returns a script for downloading a specific .NET Core SDK version, if the
-        /// version is not already installed.
-        ///
-        /// See https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet-install-script.
-        /// </summary>
-        private static BuildScript DownloadDotNetVersion(IAutobuilder<AutobuildOptionsShared> builder, string path, string version, bool needExactVersion = true)
-        {
-            return BuildScript.Bind(GetInstalledSdksScript(builder.Actions), (sdks, sdksRet) =>
-                {
-                    if (needExactVersion && sdksRet == 0 && sdks.Count == 1 && sdks[0].StartsWith(version + " ", StringComparison.Ordinal))
-                    {
-                        // The requested SDK is already installed (and no other SDKs are installed), so
-                        // no need to reinstall
-                        return BuildScript.Failure;
-                    }
-                    else if (!needExactVersion && sdksRet == 0 && sdks.Count > 0)
-                    {
-                        // there's at least one SDK installed, so no need to reinstall
-                        return BuildScript.Failure;
-                    }
-                    else if (!needExactVersion && sdksRet != 0)
-                    {
-                        builder.Log(Severity.Info, "No .NET Core SDK found.");
-                    }
-
-                    builder.Log(Severity.Info, "Attempting to download .NET Core {0}", version);
-
-                    if (builder.Actions.IsWindows())
-                    {
-
-                        var psCommand = $"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; &([scriptblock]::Create((Invoke-WebRequest -UseBasicParsing 'https://dot.net/v1/dotnet-install.ps1'))) -Version {version} -InstallDir {path}";
-
-                        BuildScript GetInstall(string pwsh) =>
-                            new CommandBuilder(builder.Actions).
-                            RunCommand(pwsh).
-                            Argument("-NoProfile").
-                            Argument("-ExecutionPolicy").
-                            Argument("unrestricted").
-                            Argument("-Command").
-                            Argument("\"" + psCommand + "\"").
-                            Script;
-
-                        return GetInstall("pwsh") | GetInstall("powershell");
-                    }
-                    else
-                    {
-                        var dotnetInstallPath = builder.Actions.PathCombine(FileUtils.GetTemporaryWorkingDirectory(
-                            builder.Actions.GetEnvironmentVariable,
-                            builder.Options.Language.UpperCaseName,
-                            out var shouldCleanUp), ".dotnet", "dotnet-install.sh");
-
-                        var downloadDotNetInstallSh = BuildScript.DownloadFile(
-                            "https://dot.net/v1/dotnet-install.sh",
-                            dotnetInstallPath,
-                            e => builder.Log(Severity.Warning, $"Failed to download 'dotnet-install.sh': {e.Message}"));
-
-                        var chmod = new CommandBuilder(builder.Actions).
-                            RunCommand("chmod").
-                            Argument("u+x").
-                            Argument(dotnetInstallPath);
-
-                        var install = new CommandBuilder(builder.Actions).
-                            RunCommand(dotnetInstallPath).
-                            Argument("--channel").
-                            Argument("release").
-                            Argument("--version").
-                            Argument(version).
-                            Argument("--install-dir").
-                            Argument(path);
-
-                        var buildScript = downloadDotNetInstallSh & chmod.Script & install.Script;
-
-                        if (shouldCleanUp)
-                        {
-                            var removeScript = new CommandBuilder(builder.Actions).
-                                RunCommand("rm").
-                                Argument(dotnetInstallPath);
-                            buildScript &= removeScript.Script;
-                        }
-
-                        return buildScript;
-                    }
-                });
-        }
-
-        private static BuildScript GetInstalledSdksScript(IBuildActions actions)
-        {
-            var listSdks = new CommandBuilder(actions, silent: true).
-                RunCommand("dotnet").
-                Argument("--list-sdks");
-            return listSdks.Script;
-        }
-
         private static string DotNetCommand(IBuildActions actions, string? dotNetPath) =>
             dotNetPath is not null ? actions.PathCombine(dotNetPath, "dotnet") : "dotnet";
 

csharp/autobuilder/Semmle.Autobuild.CSharp/Semmle.Autobuild.CSharp.csproj

@@ -20,6 +20,7 @@
 	<ItemGroup>
 		<ProjectReference Include="..\..\extractor\Semmle.Util\Semmle.Util.csproj" />
 		<ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp\Semmle.Extraction.CSharp.csproj" />
+		<ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp.DependencyFetching\Semmle.Extraction.CSharp.DependencyFetching.csproj" />
 		<ProjectReference Include="..\Semmle.Autobuild.Shared\Semmle.Autobuild.Shared.csproj" />
 	</ItemGroup>
 </Project>

csharp/autobuilder/Semmle.Autobuild.CSharp/StandaloneBuildRule.cs

@@ -8,13 +8,6 @@ namespace Semmle.Autobuild.CSharp
     /// </summary>
     internal class StandaloneBuildRule : IBuildRule<CSharpAutobuildOptions>
     {
-        private readonly string? dotNetPath;
-
-        internal StandaloneBuildRule(string? dotNetPath)
-        {
-            this.dotNetPath = dotNetPath;
-        }
-
         public BuildScript Analyse(IAutobuilder<CSharpAutobuildOptions> builder, bool auto)
         {
             if (builder.CodeQLExtractorLangRoot is null
@@ -27,12 +20,6 @@ namespace Semmle.Autobuild.CSharp
             var cmd = new CommandBuilder(builder.Actions);
             cmd.RunCommand(standalone);
 
-            if (!string.IsNullOrEmpty(this.dotNetPath))
-            {
-                cmd.Argument("--dotnet");
-                cmd.QuoteArgument(this.dotNetPath);
-            }
-
             return cmd.Script;
         }
     }

csharp/autobuilder/Semmle.Autobuild.Shared/Autobuilder.cs

@@ -70,11 +70,9 @@ namespace Semmle.Autobuild.Shared
         IBuildActions Actions { get; }
 
         /// <summary>
-        /// Log a given build event to the console.
+        /// A logger.
         /// </summary>
-        /// <param name="format">The format string.</param>
+        ILogger Logger { get; }
-        /// <param name="args">Inserts to the format string.</param>
-        void Log(Severity severity, string format, params object[] args);
 
         /// <summary>
         /// Value of CODEQL_EXTRACTOR_<LANG>_ROOT environment variable.
@@ -135,21 +133,7 @@ namespace Semmle.Autobuild.Shared
         /// <returns>True iff the path was found.</returns>
         public bool HasPath(string path) => Paths.Any(p => path == p.Item1);
 
-        private void FindFiles(string dir, int depth, int maxDepth, IList<(string, int)> results)
-        {
-            foreach (var f in Actions.EnumerateFiles(dir))
-            {
-                results.Add((f, depth));
-            }
 
-            if (depth < maxDepth)
-            {
-                foreach (var d in Actions.EnumerateDirectories(dir))
-                {
-                    FindFiles(d, depth + 1, maxDepth, results);
-                }
-            }
-        }
 
         /// <summary>
         /// The root of the source directory.
@@ -196,12 +180,7 @@ namespace Semmle.Autobuild.Shared
             Options = options;
             DiagnosticClassifier = diagnosticClassifier;
 
-            pathsLazy = new Lazy<IEnumerable<(string, int)>>(() =>
+            pathsLazy = new Lazy<IEnumerable<(string, int)>>(() => Actions.FindFiles(options.RootDirectory, options.SearchDepth));
-            {
-                var files = new List<(string, int)>();
-                FindFiles(options.RootDirectory, 0, options.SearchDepth, files);
-                return files.OrderBy(f => f.Item2).ToArray();
-            });
 
             projectsOrSolutionsToBuildLazy = new Lazy<IList<IProjectOrSolution>>(() =>
             {
@@ -214,7 +193,7 @@ namespace Semmle.Autobuild.Shared
                         if (actions.FileExists(solution))
                             ret.Add(new Solution<TAutobuildOptions>(this, solution, true));
                         else
-                            Log(Severity.Error, $"The specified project or solution file {solution} was not found");
+                            logger.LogError($"The specified project or solution file {solution} was not found");
                     }
                     return ret;
                 }
@@ -273,6 +252,8 @@ namespace Semmle.Autobuild.Shared
                 logThreadId: false) ?? Verbosity.Info,
             logThreadId: false);
 
+        public ILogger Logger => logger;
+
         private readonly IDiagnosticsWriter diagnostics;
 
         /// <summary>
@@ -285,16 +266,6 @@ namespace Semmle.Autobuild.Shared
             return Path.GetRelativePath(this.RootDirectory, path);
         }
 
-        /// <summary>
-        /// Log a given build event to the console.
-        /// </summary>
-        /// <param name="format">The format string.</param>
-        /// <param name="args">Inserts to the format string.</param>
-        public void Log(Severity severity, string format, params object[] args)
-        {
-            logger.Log(severity, format, args);
-        }
-
         /// <summary>
         /// Write <paramref name="diagnostic"/> to the diagnostics file.
         /// </summary>
@@ -310,7 +281,7 @@ namespace Semmle.Autobuild.Shared
         /// <returns>The exit code, 0 for success and non-zero for failures.</returns>
         public int AttemptBuild()
         {
-            Log(Severity.Info, $"Working directory: {Options.RootDirectory}");
+            logger.LogInfo($"Working directory: {Options.RootDirectory}");
 
             var script = GetBuildScript();
 
@@ -319,12 +290,12 @@ namespace Semmle.Autobuild.Shared
 
             void startCallback(string s, bool silent)
             {
-                Log(silent ? Severity.Debug : Severity.Info, $"\nRunning {s}");
+                logger.Log(silent ? Severity.Debug : Severity.Info, $"\nRunning {s}");
             }
 
             void exitCallback(int ret, string msg, bool silent)
             {
-                Log(silent ? Severity.Debug : Severity.Info, $"Exit code {ret}{(string.IsNullOrEmpty(msg) ? "" : $": {msg}")}");
+                logger.Log(silent ? Severity.Debug : Severity.Info, $"Exit code {ret}{(string.IsNullOrEmpty(msg) ? "" : $": {msg}")}");
             }
 
             var onOutput = BuildOutputHandler(Console.Out);
@@ -374,7 +345,7 @@ namespace Semmle.Autobuild.Shared
         protected BuildScript AutobuildFailure() =>
             BuildScript.Create(actions =>
                 {
-                    Log(Severity.Error, "Could not auto-detect a suitable build method");
+                    logger.LogError("Could not auto-detect a suitable build method");
 
                     AutobuildFailureDiagnostic();
 

csharp/autobuilder/Semmle.Autobuild.Shared/BuildCommandAutoRule.cs

@@ -55,7 +55,7 @@ namespace Semmle.Autobuild.Shared
 
         public BuildScript Analyse(IAutobuilder<AutobuildOptionsShared> builder, bool auto)
         {
-            builder.Log(Severity.Info, "Attempting to locate build script");
+            builder.Logger.LogInfo("Attempting to locate build script");
 
             // a list of extensions for files that we consider to be scripts on the current platform
             var extensions = builder.Actions.IsWindows() ? winExtensions : linuxExtensions;

csharp/autobuilder/Semmle.Autobuild.Shared/MsBuildRule.cs

@@ -40,7 +40,7 @@ namespace Semmle.Autobuild.Shared
                 return BuildScript.Failure;
 
             if (auto)
-                builder.Log(Severity.Info, "Attempting to build using MSBuild");
+                builder.Logger.LogInfo("Attempting to build using MSBuild");
 
             var vsTools = GetVcVarsBatFile(builder);
 
@@ -54,7 +54,7 @@ namespace Semmle.Autobuild.Shared
 
             if (vsTools is null && builder.Actions.IsWindows())
             {
-                builder.Log(Severity.Warning, "Could not find a suitable version of VsDevCmd.bat/vcvarsall.bat");
+                builder.Logger.LogWarning("Could not find a suitable version of VsDevCmd.bat/vcvarsall.bat");
             }
 
             // Use `nuget.exe` from source code repo, if present, otherwise first attempt with global
@@ -165,18 +165,18 @@ namespace Semmle.Autobuild.Shared
                 {
                     foreach (var b in BuildTools.VcVarsAllBatFiles(builder.Actions))
                     {
-                        builder.Log(Severity.Info, "Found {0} version {1}", b.Path, b.ToolsVersion);
+                        builder.Logger.Log(Severity.Info, "Found {0} version {1}", b.Path, b.ToolsVersion);
                     }
 
                     vsTools = BuildTools.FindCompatibleVcVars(builder.Actions, msToolsVersion);
                     if (vsTools is null)
-                        builder.Log(Severity.Warning, "Could not find build tools matching version {0}", msToolsVersion);
+                        builder.Logger.LogWarning("Could not find build tools matching version {0}", msToolsVersion);
                     else
-                        builder.Log(Severity.Info, "Setting Visual Studio tools to {0}", vsTools.Path);
+                        builder.Logger.Log(Severity.Info, "Setting Visual Studio tools to {0}", vsTools.Path);
                 }
                 else
                 {
-                    builder.Log(Severity.Error, "The format of vstools_version is incorrect. Please specify an integer.");
+                    builder.Logger.LogError("The format of vstools_version is incorrect. Please specify an integer.");
                 }
             }
 
@@ -189,18 +189,18 @@ namespace Semmle.Autobuild.Shared
         private static BuildScript DownloadNugetExe<TAutobuildOptions>(IAutobuilder<TAutobuildOptions> builder, string path) where TAutobuildOptions : AutobuildOptionsShared =>
             BuildScript.Create(_ =>
             {
-                builder.Log(Severity.Info, "Attempting to download nuget.exe");
+                builder.Logger.LogInfo("Attempting to download nuget.exe");
                 return 0;
             })
             &
             BuildScript.DownloadFile(
                 FileUtils.NugetExeUrl,
                 path,
-                e => builder.Log(Severity.Warning, $"Failed to download 'nuget.exe': {e.Message}"))
+                e => builder.Logger.LogWarning($"Failed to download 'nuget.exe': {e.Message}"))
             &
             BuildScript.Create(_ =>
             {
-                builder.Log(Severity.Info, $"Successfully downloaded {path}");
+                builder.Logger.LogInfo($"Successfully downloaded {path}");
                 return 0;
             });
     }

csharp/autobuilder/Semmle.Autobuild.Shared/Project.cs

@@ -41,7 +41,7 @@ namespace Semmle.Autobuild.Shared
             }
             catch (Exception ex) when (ex is XmlException || ex is FileNotFoundException)
             {
-                builder.Log(Severity.Info, $"Unable to read project file {path}.");
+                builder.Logger.LogInfo($"Unable to read project file {path}.");
                 return;
             }
 
@@ -66,7 +66,7 @@ namespace Semmle.Autobuild.Shared
                     catch  // lgtm[cs/catch-of-all-exceptions]
                            // Generic catch clause - Version constructor throws about 5 different exceptions.
                     {
-                        builder.Log(Severity.Warning, "Project {0} has invalid tools version {1}", path, toolsVersion);
+                        builder.Logger.Log(Severity.Warning, "Project {0} has invalid tools version {1}", path, toolsVersion);
                     }
                 }
 

csharp/autobuilder/Semmle.Autobuild.Shared/Solution.cs

@@ -73,7 +73,7 @@ namespace Semmle.Autobuild.Shared
                     return;
                 }
 
-                builder.Log(Severity.Info, $"Unable to read solution file {path}.");
+                builder.Logger.LogInfo($"Unable to read solution file {path}.");
                 includedProjects = Array.Empty<Project<TAutobuildOptions>>();
                 return;
             }

csharp/extractor/Semmle.Extraction.CIL.Driver/Program.cs

@@ -46,7 +46,7 @@ namespace Semmle.Extraction.CIL.Driver
                 .ToArray();
 
             foreach (var missingRef in options.MissingReferences)
-                logger.Log(Severity.Info, "  Missing assembly " + missingRef);
+                logger.LogInfo("  Missing assembly " + missingRef);
 
             var sw = new Stopwatch();
             sw.Start();

csharp/extractor/Semmle.Extraction.CIL/Analyser.cs

@@ -45,7 +45,7 @@ namespace Semmle.Extraction.CIL
             }
             catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
             {
-                logger.Log(Severity.Error, string.Format("Exception extracting {0}: {1}", assemblyPath, ex));
+                logger.LogError(string.Format("Exception extracting {0}: {1}", assemblyPath, ex));
             }
         }
     }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -7,6 +7,7 @@ using System.Security.Cryptography;
 using System.Text;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
+
 using Semmle.Util;
 using Semmle.Util.Logging;
 
@@ -27,8 +28,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         private readonly List<string> generatedSources;
         private int dotnetFrameworkVersionVariantCount = 0;
         private int conflictedReferences = 0;
-        private readonly IDependencyOptions options;
         private readonly DirectoryInfo sourceDir;
+        private string? dotnetPath;
         private readonly IDotNet dotnet;
         private readonly FileContent fileContent;
         private readonly TemporaryDirectory packageDirectory;
@@ -45,11 +46,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         /// </summary>
         /// <param name="options">Dependency fetching options</param>
         /// <param name="logger">Logger for dependency fetching progress.</param>
-        public DependencyManager(string srcDir, IDependencyOptions options, ILogger logger)
+        public DependencyManager(string srcDir, ILogger logger)
         {
             var startTime = DateTime.Now;
 
-            this.options = options;
             this.logger = logger;
             this.sourceDir = new DirectoryInfo(srcDir);
 
@@ -59,17 +59,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 
             tempWorkingDirectory = new TemporaryDirectory(FileUtils.GetTemporaryWorkingDirectory(out cleanupTempWorkingDirectory));
 
-            try
-            {
-                this.dotnet = DotNet.Make(options, logger, tempWorkingDirectory);
-                runtimeLazy = new Lazy<Runtime>(() => new Runtime(dotnet, logger));
-            }
-            catch
-            {
-                logger.LogError("Missing dotnet CLI");
-                throw;
-            }
-
             logger.LogInfo($"Finding files in {srcDir}...");
 
             var allFiles = GetAllFiles().ToList();
@@ -85,6 +74,33 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 
             logger.LogInfo($"Found {allFiles.Count} files, {nonGeneratedSources.Count} source files, {allProjects.Count} project files, {allSolutions.Count} solution files, {dllPaths.Count} DLLs.");
 
+            void startCallback(string s, bool silent)
+            {
+                logger.Log(silent ? Severity.Debug : Severity.Info, $"\nRunning {s}");
+            }
+
+            void exitCallback(int ret, string msg, bool silent)
+            {
+                logger.Log(silent ? Severity.Debug : Severity.Info, $"Exit code {ret}{(string.IsNullOrEmpty(msg) ? "" : $": {msg}")}");
+            }
+
+            DotNet.WithDotNet(SystemBuildActions.Instance, logger, smallNonBinaryFiles, tempWorkingDirectory.ToString(), shouldCleanUp: false, ensureDotNetAvailable: true, version: null, installDir =>
+            {
+                this.dotnetPath = installDir;
+                return BuildScript.Success;
+            }).Run(SystemBuildActions.Instance, startCallback, exitCallback);
+
+            try
+            {
+                this.dotnet = DotNet.Make(logger, dotnetPath, tempWorkingDirectory);
+                runtimeLazy = new Lazy<Runtime>(() => new Runtime(dotnet));
+            }
+            catch
+            {
+                logger.LogError("Missing dotnet CLI");
+                throw;
+            }
+
             RestoreNugetPackages(allNonBinaryFiles, allProjects, allSolutions, dllPaths);
             // Find DLLs in the .Net / Asp.Net Framework
             // This needs to come after the nuget restore, because the nuget restore might fetch the .NET Core/Framework reference assemblies.
@@ -570,15 +586,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             }
         }
 
-        public DependencyManager(string srcDir) : this(srcDir, DependencyOptions.Default, new ConsoleLogger(Verbosity.Info, logThreadId: true)) { }
-
         private IEnumerable<FileInfo> GetAllFiles()
         {
             IEnumerable<FileInfo> files = sourceDir.GetFiles("*.*", new EnumerationOptions { RecurseSubdirectories = true });
 
-            if (options.DotNetPath != null)
+            if (dotnetPath != null)
             {
-                files = files.Where(f => !f.FullName.StartsWith(options.DotNetPath, StringComparison.OrdinalIgnoreCase));
+                files = files.Where(f => !f.FullName.StartsWith(dotnetPath, StringComparison.OrdinalIgnoreCase));
             }
 
             files = files.Where(f =>
@@ -590,12 +604,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                         return true;
                     }
 
-                    logger.Log(Severity.Warning, $"File {f.FullName} could not be processed.");
+                    logger.LogWarning($"File {f.FullName} could not be processed.");
                     return false;
                 }
                 catch (Exception ex)
                 {
-                    logger.Log(Severity.Warning, $"File {f.FullName} could not be processed: {ex.Message}");
+                    logger.LogWarning($"File {f.FullName} could not be processed: {ex.Message}");
                     return false;
                 }
             });
@@ -651,7 +665,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                 }
                 catch (AssemblyLoadException)
                 {
-                    logger.Log(Severity.Warning, $"Could not load assembly information from {usedReference.Key}");
+                    logger.LogWarning($"Could not load assembly information from {usedReference.Key}");
                 }
             }
 
@@ -738,7 +752,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 
         private void AnalyseSolutions(IEnumerable<string> solutions)
         {
-            Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, solutionFile =>
+            Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = EnvironmentVariables.GetDefaultNumberOfThreads() }, solutionFile =>
             {
                 try
                 {
@@ -828,7 +842,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             var successCount = 0;
             var assetFiles = new List<string>();
             var sync = new object();
-            Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, project =>
+            Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = EnvironmentVariables.GetDefaultNumberOfThreads() }, project =>
             {
                 logger.LogInfo($"Restoring project {project}...");
                 var res = dotnet.Restore(new(project, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
@@ -928,7 +942,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             var successCount = 0;
             var sync = new object();
 
-            Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, package =>
+            Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = EnvironmentVariables.GetDefaultNumberOfThreads() }, package =>
             {
                 var success = TryRestorePackageManually(package.Name, nugetConfig, package.PackageReferenceSource);
                 if (!success)

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyOptions.cs

@@ -1,32 +0,0 @@
-using System;
-using System.Linq;
-using System.Collections.Generic;
-using Semmle.Util;
-
-namespace Semmle.Extraction.CSharp.DependencyFetching
-{
-    /// <summary>
-    /// Dependency fetching related options.
-    /// </summary>
-    public interface IDependencyOptions
-    {
-        /// <summary>
-        /// The number of threads to use.
-        /// </summary>
-        int Threads { get; }
-
-        /// <summary>
-        /// The path to the local ".dotnet" directory, if any.
-        /// </summary>
-        string? DotNetPath { get; }
-    }
-
-    public class DependencyOptions : IDependencyOptions
-    {
-        public static IDependencyOptions Default => new DependencyOptions();
-
-        public int Threads { get; set; } = EnvironmentVariables.GetDefaultNumberOfThreads();
-
-        public string? DotNetPath { get; set; } = null;
-    }
-}

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs

@@ -2,7 +2,9 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
-using System.Text.RegularExpressions;
+
+using Newtonsoft.Json.Linq;
+
 using Semmle.Util;
 using Semmle.Util.Logging;
 
@@ -11,29 +13,26 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
     /// <summary>
     /// Utilities to run the "dotnet" command.
     /// </summary>
-    internal partial class DotNet : IDotNet
+    public partial class DotNet : IDotNet
     {
         private readonly IDotNetCliInvoker dotnetCliInvoker;
-        private readonly ILogger logger;
         private readonly TemporaryDirectory? tempWorkingDirectory;
 
         private DotNet(IDotNetCliInvoker dotnetCliInvoker, ILogger logger, TemporaryDirectory? tempWorkingDirectory = null)
         {
-            this.logger = logger;
             this.tempWorkingDirectory = tempWorkingDirectory;
             this.dotnetCliInvoker = dotnetCliInvoker;
             Info();
         }
 
-        private DotNet(IDependencyOptions options, ILogger logger, TemporaryDirectory tempWorkingDirectory) : this(new DotNetCliInvoker(logger, Path.Combine(options.DotNetPath ?? string.Empty, "dotnet")), logger, tempWorkingDirectory) { }
+        private DotNet(ILogger logger, string? dotNetPath, TemporaryDirectory tempWorkingDirectory) : this(new DotNetCliInvoker(logger, Path.Combine(dotNetPath ?? string.Empty, "dotnet")), logger, tempWorkingDirectory) { }
 
         internal static IDotNet Make(IDotNetCliInvoker dotnetCliInvoker, ILogger logger) => new DotNet(dotnetCliInvoker, logger);
 
-        public static IDotNet Make(IDependencyOptions options, ILogger logger, TemporaryDirectory tempWorkingDirectory) => new DotNet(options, logger, tempWorkingDirectory);
+        public static IDotNet Make(ILogger logger, string? dotNetPath, TemporaryDirectory tempWorkingDirectory) => new DotNet(logger, dotNetPath, tempWorkingDirectory);
 
         private void Info()
         {
-            // TODO: make sure the below `dotnet` version is matching the one specified in global.json
             var res = dotnetCliInvoker.RunCommand("--info");
             if (!res)
             {
@@ -108,5 +107,165 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             var args = $"exec {execArgs}";
             return dotnetCliInvoker.RunCommand(args);
         }
+
+        // The version number should be kept in sync with the version .NET version used for building the application.
+        public const string LatestDotNetSdkVersion = "8.0.101";
+
+        /// <summary>
+        /// Returns a script for downloading relevant versions of the
+        /// .NET SDK. The SDK(s) will be installed at <code>installDir</code>
+        /// (provided that the script succeeds).
+        /// </summary>
+        private static BuildScript DownloadDotNet(IBuildActions actions, ILogger logger, IEnumerable<string> files, string tempWorkingDirectory, bool shouldCleanUp, string installDir, string? version, bool ensureDotNetAvailable)
+        {
+            if (!string.IsNullOrEmpty(version))
+                // Specific version requested
+                return DownloadDotNetVersion(actions, logger, tempWorkingDirectory, shouldCleanUp, installDir, version);
+
+            // Download versions mentioned in `global.json` files
+            // See https://docs.microsoft.com/en-us/dotnet/core/tools/global-json
+            var installScript = BuildScript.Success;
+            var validGlobalJson = false;
+
+            foreach (var path in files.Where(p => p.EndsWith("global.json", StringComparison.Ordinal)))
+            {
+                try
+                {
+                    var o = JObject.Parse(File.ReadAllText(path));
+                    version = (string)(o?["sdk"]?["version"]!);
+                }
+                catch
+                {
+                    // not a valid global.json file
+                    continue;
+                }
+
+                installScript &= DownloadDotNetVersion(actions, logger, tempWorkingDirectory, shouldCleanUp, installDir, version);
+                validGlobalJson = true;
+            }
+
+            if (validGlobalJson)
+            {
+                return installScript;
+            }
+
+            if (ensureDotNetAvailable)
+            {
+                return DownloadDotNetVersion(actions, logger, tempWorkingDirectory, shouldCleanUp, installDir, LatestDotNetSdkVersion, needExactVersion: false);
+            }
+
+            return BuildScript.Failure;
+        }
+
+        /// <summary>
+        /// Returns a script for downloading a specific .NET SDK version, if the
+        /// version is not already installed.
+        ///
+        /// See https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet-install-script.
+        /// </summary>
+        private static BuildScript DownloadDotNetVersion(IBuildActions actions, ILogger logger, string tempWorkingDirectory, bool shouldCleanUp, string path, string version, bool needExactVersion = true)
+        {
+            return BuildScript.Bind(GetInstalledSdksScript(actions), (sdks, sdksRet) =>
+                {
+                    if (needExactVersion && sdksRet == 0 && sdks.Count == 1 && sdks[0].StartsWith(version + " ", StringComparison.Ordinal))
+                    {
+                        // The requested SDK is already installed (and no other SDKs are installed), so
+                        // no need to reinstall
+                        return BuildScript.Failure;
+                    }
+                    else if (!needExactVersion && sdksRet == 0 && sdks.Count > 0)
+                    {
+                        // there's at least one SDK installed, so no need to reinstall
+                        return BuildScript.Failure;
+                    }
+                    else if (!needExactVersion && sdksRet != 0)
+                    {
+                        logger.LogInfo("No .NET SDK found.");
+                    }
+
+                    logger.LogInfo($"Attempting to download .NET {version}");
+
+                    if (actions.IsWindows())
+                    {
+
+                        var psCommand = $"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; &([scriptblock]::Create((Invoke-WebRequest -UseBasicParsing 'https://dot.net/v1/dotnet-install.ps1'))) -Version {version} -InstallDir {path}";
+
+                        BuildScript GetInstall(string pwsh) =>
+                            new CommandBuilder(actions).
+                            RunCommand(pwsh).
+                            Argument("-NoProfile").
+                            Argument("-ExecutionPolicy").
+                            Argument("unrestricted").
+                            Argument("-Command").
+                            Argument("\"" + psCommand + "\"").
+                            Script;
+
+                        return GetInstall("pwsh") | GetInstall("powershell");
+                    }
+                    else
+                    {
+                        var dotnetInstallPath = actions.PathCombine(tempWorkingDirectory, ".dotnet", "dotnet-install.sh");
+
+                        var downloadDotNetInstallSh = BuildScript.DownloadFile(
+                            "https://dot.net/v1/dotnet-install.sh",
+                            dotnetInstallPath,
+                            e => logger.LogWarning($"Failed to download 'dotnet-install.sh': {e.Message}"));
+
+                        var chmod = new CommandBuilder(actions).
+                            RunCommand("chmod").
+                            Argument("u+x").
+                            Argument(dotnetInstallPath);
+
+                        var install = new CommandBuilder(actions).
+                            RunCommand(dotnetInstallPath).
+                            Argument("--channel").
+                            Argument("release").
+                            Argument("--version").
+                            Argument(version).
+                            Argument("--install-dir").
+                            Argument(path);
+
+                        var buildScript = downloadDotNetInstallSh & chmod.Script & install.Script;
+
+                        if (shouldCleanUp)
+                        {
+                            buildScript &= BuildScript.DeleteFile(dotnetInstallPath);
+                        }
+
+                        return buildScript;
+                    }
+                });
+        }
+
+        private static BuildScript GetInstalledSdksScript(IBuildActions actions)
+        {
+            var listSdks = new CommandBuilder(actions, silent: true).
+                RunCommand("dotnet").
+                Argument("--list-sdks");
+            return listSdks.Script;
+        }
+
+        /// <summary>
+        /// Returns a script that attempts to download relevant version(s) of the
+        /// .NET SDK, followed by running the script generated by <paramref name="f"/>.
+        ///
+        /// The argument to <paramref name="f"/> is the path to the directory in which the
+        /// .NET SDK(s) were installed.
+        /// </summary>
+        public static BuildScript WithDotNet(IBuildActions actions, ILogger logger, IEnumerable<string> files, string tempWorkingDirectory, bool shouldCleanUp, bool ensureDotNetAvailable, string? version, Func<string?, BuildScript> f)
+        {
+            var installDir = actions.PathCombine(tempWorkingDirectory, ".dotnet");
+            var installScript = DownloadDotNet(actions, logger, files, tempWorkingDirectory, shouldCleanUp, installDir, version, ensureDotNetAvailable);
+            return BuildScript.Bind(installScript, installed =>
+            {
+                if (installed != 0)
+                {
+                    // The .NET SDK was not installed, either because the installation failed or because it was already installed.
+                    installDir = null;
+                }
+
+                return f(installDir);
+            });
+        }
     }
-}
+}

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FilePathFilter.cs

@@ -55,12 +55,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                 }
                 else
                 {
-                    logger.Log(Severity.Info, $"Invalid filter: {filter}");
+                    logger.LogInfo($"Invalid filter: {filter}");
                     continue;
                 }
 
                 var regex = new FilePattern(filterText).RegexPattern;
-                logger.Log(Severity.Info, $"Filtering {(include ? "in" : "out")} files matching '{regex}'. Original glob filter: '{filter}'");
+                logger.LogInfo($"Filtering {(include ? "in" : "out")} files matching '{regex}'. Original glob filter: '{filter}'");
                 pathFilters.Add(new PathFilter(new Regex(regex, RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline), include));
             }
 
@@ -91,7 +91,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 
                 if (!include)
                 {
-                    logger.Log(Severity.Info, $"Excluding '{f.FileInfo.FullName}'");
+                    logger.LogInfo($"Excluding '{f.FileInfo.FullName}'");
                 }
 
                 return include;

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs

@@ -5,7 +5,7 @@ using System.Text.RegularExpressions;
 
 namespace Semmle.Extraction.CSharp.DependencyFetching
 {
-    internal interface IDotNet
+    public interface IDotNet
     {
         RestoreResult Restore(RestoreSettings restoreSettings);
         bool New(string folder);
@@ -15,9 +15,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         bool Exec(string execArgs);
     }
 
-    internal record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, string? PathToNugetConfig = null, bool ForceReevaluation = false);
+    public record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, string? PathToNugetConfig = null, bool ForceReevaluation = false);
 
-    internal partial record class RestoreResult(bool Success, IList<string> Output)
+    public partial record class RestoreResult(bool Success, IList<string> Output)
     {
         private readonly Lazy<IEnumerable<string>> assetsFilePaths = new(() => GetFirstGroupOnMatch(AssetsFileRegex(), Output));
         public IEnumerable<string> AssetsFilePaths => Success ? assetsFilePaths.Value : Array.Empty<string>();

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Runtime.cs

@@ -18,14 +18,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         private const string aspNetCoreApp = "Microsoft.AspNetCore.App";
 
         private readonly IDotNet dotNet;
-        private readonly ILogger logger;
         private readonly Lazy<Dictionary<string, DotNetVersion>> newestRuntimes;
         private Dictionary<string, DotNetVersion> NewestRuntimes => newestRuntimes.Value;
 
-        public Runtime(IDotNet dotNet, ILogger logger)
+        public Runtime(IDotNet dotNet)
         {
             this.dotNet = dotNet;
-            this.logger = logger;
             this.newestRuntimes = new(GetNewestRuntimes);
         }
 

csharp/extractor/Semmle.Extraction.CSharp.DependencyStubGenerator/Program.cs

@@ -3,7 +3,7 @@ using Semmle.Extraction.CSharp.StubGenerator;
 using Semmle.Util.Logging;
 
 var logger = new ConsoleLogger(Verbosity.Info, logThreadId: false);
-using var dependencyManager = new DependencyManager(".", DependencyOptions.Default, logger);
+using var dependencyManager = new DependencyManager(".", logger);
 StubGenerator.GenerateStubs(logger, dependencyManager.ReferenceFiles, "codeql_csharp_stubs");
 
 return 0;

csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs

@@ -138,7 +138,7 @@ namespace Semmle.Extraction.CSharp.Standalone
 
             using var logger = new ConsoleLogger(options.Verbosity, logThreadId: true);
             logger.Log(Severity.Info, "Extracting C# in buildless mode");
-            using var dependencyManager = new DependencyManager(options.SrcDir, options.Dependencies, logger);
+            using var dependencyManager = new DependencyManager(options.SrcDir, logger);
 
             if (!dependencyManager.AllSourceFiles.Any())
             {

csharp/extractor/Semmle.Extraction.CSharp.Standalone/Options.cs

@@ -23,18 +23,6 @@ namespace Semmle.Extraction.CSharp.Standalone
             }
         }
 
-        public override bool HandleOption(string key, string value)
-        {
-            switch (key)
-            {
-                case "dotnet":
-                    dependencies.DotNetPath = value;
-                    return true;
-                default:
-                    return base.HandleOption(key, value);
-            }
-        }
-
         public override bool HandleArgument(string arg)
         {
             return true;
@@ -51,12 +39,6 @@ namespace Semmle.Extraction.CSharp.Standalone
         /// </summary>
         public string SrcDir { get; } = Directory.GetCurrentDirectory();
 
-        private readonly DependencyOptions dependencies = new DependencyOptions();
-        /// <summary>
-        /// Dependency fetching related options.
-        /// </summary>
-        public IDependencyOptions Dependencies => dependencies;
-
         /// <summary>
         /// Whether errors were encountered parsing the arguments.
         /// </summary>

csharp/extractor/Semmle.Extraction.CSharp.StubGenerator/StubGenerator.cs

@@ -36,7 +36,7 @@ public static class StubGenerator
             references.Add((reference, path));
         });
 
-        logger.Log(Severity.Info, $"Generating stubs for {references.Count} assemblies.");
+        logger.LogInfo($"Generating stubs for {references.Count} assemblies.");
 
         var compilation = CSharpCompilation.Create(
             "stubgenerator.dll",
@@ -50,7 +50,7 @@ public static class StubGenerator
         });
 
         stopWatch.Stop();
-        logger.Log(Severity.Info, $"Stub generation took {stopWatch.Elapsed}.");
+        logger.LogInfo($"Stub generation took {stopWatch.Elapsed}.");
 
         return stubPaths.ToArray();
     }

csharp/extractor/Semmle.Extraction.Tests/Runtime.cs

@@ -47,7 +47,7 @@ namespace Semmle.Extraction.Tests
                 "Microsoft.NETCore.App 7.0.2 [/path/dotnet/shared/Microsoft.NETCore.App]"
                 };
             var dotnet = new DotNetStub(listedRuntimes, null!);
-            var runtime = new Runtime(dotnet, new LoggerStub());
+            var runtime = new Runtime(dotnet);
 
             // Execute
             var runtimes = runtime.GetNewestRuntimes();
@@ -73,7 +73,7 @@ namespace Semmle.Extraction.Tests
                 "Microsoft.NETCore.App 8.0.0-preview.5.23280.8 [/path/dotnet/shared/Microsoft.NETCore.App]"
             };
             var dotnet = new DotNetStub(listedRuntimes, null!);
-            var runtime = new Runtime(dotnet, new LoggerStub());
+            var runtime = new Runtime(dotnet);
 
             // Execute
             var runtimes = runtime.GetNewestRuntimes();
@@ -96,7 +96,7 @@ namespace Semmle.Extraction.Tests
                 "Microsoft.NETCore.App 8.0.0-preview.5.23280.8 [/path/dotnet/shared/Microsoft.NETCore.App]"
             };
             var dotnet = new DotNetStub(listedRuntimes, null!);
-            var runtime = new Runtime(dotnet, new LoggerStub());
+            var runtime = new Runtime(dotnet);
 
             // Execute
             var runtimes = runtime.GetNewestRuntimes();
@@ -125,7 +125,7 @@ namespace Semmle.Extraction.Tests
                 @"Microsoft.WindowsDesktop.App 7.0.4 [C:\Program Files\dotnet\shared\Microsoft.WindowsDesktop.App]"
             };
             var dotnet = new DotNetStub(listedRuntimes, null!);
-            var runtime = new Runtime(dotnet, new LoggerStub());
+            var runtime = new Runtime(dotnet);
 
             // Execute
             var runtimes = runtime.GetNewestRuntimes();

csharp/extractor/Semmle.Extraction/Extractor/Extractor.cs

@@ -43,7 +43,7 @@ namespace Semmle.Extraction
                     ++Errors;
                     if (Errors == maxErrors)
                     {
-                        Logger.Log(Severity.Info, "  Stopping logging after {0} errors", Errors);
+                        Logger.LogInfo("  Stopping logging after {0} errors", Errors);
                     }
                 }
 

csharp/extractor/Semmle.Extraction/TrapWriter.cs

@@ -227,7 +227,7 @@ namespace Semmle.Extraction
             {
                 // If this happened, it was probably because the same file was compiled multiple times.
                 // In any case, this is not a fatal error.
-                logger.Log(Severity.Warning, "Problem archiving " + dest + ": " + ex);
+                logger.LogWarning("Problem archiving " + dest + ": " + ex);
             }
         }
 

csharp/extractor/Semmle.Util/BuildActions.cs

@@ -287,4 +287,30 @@ namespace Semmle.Util
 
         public static IBuildActions Instance { get; } = new SystemBuildActions();
     }
+
+    public static class BuildActionExtensions
+    {
+        private static void FindFiles(this IBuildActions actions, string dir, int depth, int? maxDepth, IList<(string, int)> results)
+        {
+            foreach (var f in actions.EnumerateFiles(dir))
+            {
+                results.Add((f, depth));
+            }
+
+            if (depth < maxDepth)
+            {
+                foreach (var d in actions.EnumerateDirectories(dir))
+                {
+                    actions.FindFiles(d, depth + 1, maxDepth, results);
+                }
+            }
+        }
+
+        public static (string path, int depth)[] FindFiles(this IBuildActions actions, string dir, int? maxDepth)
+        {
+            var results = new List<(string, int)>();
+            actions.FindFiles(dir, 0, maxDepth, results);
+            return results.OrderBy(f => f.Item2).ToArray();
+        }
+    }
 }

csharp/extractor/Semmle.Util/CanonicalPathCache.cs

@@ -268,7 +268,7 @@ namespace Semmle.Util
                     catch  // lgtm[cs/catch-of-all-exceptions]
                     {
                         // Failed to late-bind a suitable library.
-                        logger.Log(Severity.Warning, "Preserving symlinks in canonical paths");
+                        logger.LogWarning("Preserving symlinks in canonical paths");
                         pathStrategy = new QueryDirectoryStrategy();
                     }
                     break;

csharp/extractor/Semmle.Util/FileUtils.cs

@@ -133,14 +133,14 @@ namespace Semmle.Util
                 var directoryName = Path.GetDirectoryName(nested);
                 if (directoryName is null)
                 {
-                    logger.Log(Severity.Warning, "Failed to get directory name from path '" + nested + "'.");
+                    logger.LogWarning("Failed to get directory name from path '" + nested + "'.");
                     throw new InvalidOperationException();
                 }
                 Directory.CreateDirectory(directoryName);
             }
             catch (PathTooLongException)
             {
-                logger.Log(Severity.Warning, "Failed to create parent directory of '" + nested + "': Path too long.");
+                logger.LogWarning("Failed to create parent directory of '" + nested + "': Path too long.");
                 throw;
             }
             return nested;