C#: Better handle multiple global.json files

csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs

@@ -970,12 +970,10 @@ namespace Semmle.Autobuild.CSharp.Tests
             actions.RunProcess["dotnet --list-sdks"] = 0;
             actions.RunProcessOut["dotnet --list-sdks"] = @"2.1.3 [C:\Program Files\dotnet\sdks]
 2.1.4 [C:\Program Files\dotnet\sdks]";
-            actions.RunProcess[@"chmod u+x scratch/.dotnet/dotnet-install.sh"] = 0;
+            actions.RunProcess[@"dotnet --info"] = 0;
-            actions.RunProcess[@"scratch/.dotnet/dotnet-install.sh --channel release --version 2.1.3 --install-dir scratch/.dotnet"] = 0;
+            actions.RunProcess[@"dotnet clean C:\Project/test.csproj"] = 0;
-            actions.RunProcess[@"scratch/.dotnet/dotnet --info"] = 0;
+            actions.RunProcess[@"dotnet restore C:\Project/test.csproj"] = 0;
-            actions.RunProcess[@"scratch/.dotnet/dotnet clean C:\Project/test.csproj"] = 0;
+            actions.RunProcess[@"dotnet build --no-incremental C:\Project/test.csproj"] = 0;
-            actions.RunProcess[@"scratch/.dotnet/dotnet restore C:\Project/test.csproj"] = 0;
-            actions.RunProcess[@"scratch/.dotnet/dotnet build --no-incremental C:\Project/test.csproj"] = 0;
             actions.FileExists["csharp.log"] = true;
             actions.FileExists["test.csproj"] = true;
             actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
@@ -993,17 +991,15 @@ namespace Semmle.Autobuild.CSharp.Tests
 
 </Project>");
             actions.LoadXml[@"C:\Project/test.csproj"] = xml;
-            actions.DownloadFiles.Add(("https://dot.net/v1/dotnet-install.sh", "scratch/.dotnet/dotnet-install.sh"));
-            actions.CreateDirectories.Add(@"scratch/.dotnet");
 
             var autobuilder = CreateAutoBuilder(false, dotnetVersion: "2.1.3");
-            TestAutobuilderScript(autobuilder, 0, 7);
+            TestAutobuilderScript(autobuilder, 0, 5);
         }
 
         private void TestDotnetVersionWindows(Action action, int commandsRun)
         {
             actions.RunProcess["cmd.exe /C dotnet --list-sdks"] = 0;
-            actions.RunProcessOut["cmd.exe /C dotnet --list-sdks"] = "2.1.3 [C:\\Program Files\\dotnet\\sdks]\n2.1.4 [C:\\Program Files\\dotnet\\sdks]";
+            actions.RunProcessOut["cmd.exe /C dotnet --list-sdks"] = "2.1.4 [C:\\Program Files\\dotnet\\sdks]";
             action();
             actions.RunProcess[@"cmd.exe /C scratch\.dotnet\dotnet --info"] = 0;
             actions.RunProcess[@"cmd.exe /C scratch\.dotnet\dotnet clean C:\Project\test.csproj"] = 0;

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs

@@ -120,74 +120,85 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         {
             if (!string.IsNullOrEmpty(version))
                 // Specific version requested
-                return DownloadDotNetVersion(actions, logger, tempWorkingDirectory, shouldCleanUp, installDir, version);
+                return DownloadDotNetVersion(actions, logger, tempWorkingDirectory, shouldCleanUp, installDir, [version]);
 
             // Download versions mentioned in `global.json` files
             // See https://docs.microsoft.com/en-us/dotnet/core/tools/global-json
-            var installScript = BuildScript.Success;
+            var versions = new List<string>();
-            var validGlobalJson = false;
 
             foreach (var path in files.Where(p => p.EndsWith("global.json", StringComparison.Ordinal)))
             {
                 try
                 {
                     var o = JObject.Parse(File.ReadAllText(path));
-                    version = (string)(o?["sdk"]?["version"]!);
+                    versions.Add((string)o?["sdk"]?["version"]!);
                 }
                 catch
                 {
-                    // not a valid global.json file
+                    // not a valid `global.json` file
                     continue;
                 }
-
-                installScript &= DownloadDotNetVersion(actions, logger, tempWorkingDirectory, shouldCleanUp, installDir, version);
-                validGlobalJson = true;
             }
 
-            if (validGlobalJson)
+            if (versions.Count > 0)
             {
-                return installScript;
+                return DownloadDotNetVersion(actions, logger, tempWorkingDirectory, shouldCleanUp, installDir, versions);
             }
 
             if (ensureDotNetAvailable)
             {
-                return DownloadDotNetVersion(actions, logger, tempWorkingDirectory, shouldCleanUp, installDir, LatestDotNetSdkVersion, needExactVersion: false);
+                return DownloadDotNetVersion(actions, logger, tempWorkingDirectory, shouldCleanUp, installDir, [LatestDotNetSdkVersion], needExactVersion: false);
             }
 
             return BuildScript.Failure;
         }
 
         /// <summary>
-        /// Returns a script for downloading a specific .NET SDK version, if the
+        /// Returns a script for downloading specific .NET SDK versions, if the
-        /// version is not already installed.
+        /// versions are not already installed.
         ///
         /// See https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet-install-script.
         /// </summary>
-        private static BuildScript DownloadDotNetVersion(IBuildActions actions, ILogger logger, string tempWorkingDirectory, bool shouldCleanUp, string path, string version, bool needExactVersion = true)
+        private static BuildScript DownloadDotNetVersion(IBuildActions actions, ILogger logger, string tempWorkingDirectory, bool shouldCleanUp, string path, IEnumerable<string> versions, bool needExactVersion = true)
         {
+            if (!versions.Any())
+            {
+                logger.LogInfo("No .NET SDK versions requested.");
+                return BuildScript.Failure;
+            }
+
             return BuildScript.Bind(GetInstalledSdksScript(actions), (sdks, sdksRet) =>
+            {
+                if (
+                    needExactVersion &&
+                    sdksRet == 0 &&
+                    // quadratic; should be OK, given that both `version` and `sdks` are expected to be small
+                    versions.All(version => sdks.Any(sdk => sdk.StartsWith(version + " ", StringComparison.Ordinal))))
                 {
-                    if (needExactVersion && sdksRet == 0 && sdks.Count == 1 && sdks[0].StartsWith(version + " ", StringComparison.Ordinal))
+                    // The requested SDKs are already installed, so no need to reinstall
-                    {
+                    return BuildScript.Failure;
-                        // The requested SDK is already installed (and no other SDKs are installed), so
+                }
-                        // no need to reinstall
+                else if (!needExactVersion && sdksRet == 0 && sdks.Count > 0)
-                        return BuildScript.Failure;
+                {
-                    }
+                    // there's at least one SDK installed, so no need to reinstall
-                    else if (!needExactVersion && sdksRet == 0 && sdks.Count > 0)
+                    return BuildScript.Failure;
-                    {
+                }
-                        // there's at least one SDK installed, so no need to reinstall
+                else if (!needExactVersion && sdksRet != 0)
-                        return BuildScript.Failure;
+                {
-                    }
+                    logger.LogInfo("No .NET SDK found.");
-                    else if (!needExactVersion && sdksRet != 0)
+                }
-                    {
-                        logger.LogInfo("No .NET SDK found.");
-                    }
 
-                    logger.LogInfo($"Attempting to download .NET {version}");
+                BuildScript prelude;
+                BuildScript postlude;
+                Func<string, BuildScript> getInstall;
 
-                    if (actions.IsWindows())
+                if (actions.IsWindows())
+                {
+                    prelude = BuildScript.Success;
+                    postlude = BuildScript.Success;
+
+                    getInstall = version =>
                     {
-
                         var psCommand = $"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; &([scriptblock]::Create((Invoke-WebRequest -UseBasicParsing 'https://dot.net/v1/dotnet-install.ps1'))) -Version {version} -InstallDir {path}";
 
                         BuildScript GetInstall(string pwsh) =>
@@ -201,40 +212,55 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                             Script;
 
                         return GetInstall("pwsh") | GetInstall("powershell");
-                    }
+                    };
-                    else
+                }
+                else
+                {
+                    var dotnetInstallPath = actions.PathCombine(tempWorkingDirectory, ".dotnet", "dotnet-install.sh");
+
+                    var downloadDotNetInstallSh = BuildScript.DownloadFile(
+                        "https://dot.net/v1/dotnet-install.sh",
+                        dotnetInstallPath,
+                        e => logger.LogWarning($"Failed to download 'dotnet-install.sh': {e.Message}"));
+
+                    var chmod = new CommandBuilder(actions).
+                        RunCommand("chmod").
+                        Argument("u+x").
+                        Argument(dotnetInstallPath);
+
+                    prelude = downloadDotNetInstallSh & chmod.Script;
+                    postlude = shouldCleanUp ? BuildScript.DeleteFile(dotnetInstallPath) : BuildScript.Success;
+
+                    getInstall = version => new CommandBuilder(actions).
+                        RunCommand(dotnetInstallPath).
+                        Argument("--channel").
+                        Argument("release").
+                        Argument("--version").
+                        Argument(version).
+                        Argument("--install-dir").
+                        Argument(path).Script;
+                }
+
+                var installScript = prelude & BuildScript.Failure;
+
+                var attempted = new HashSet<string>();
+                foreach (var version in versions)
+                {
+                    if (!attempted.Add(version))
+                        continue;
+
+                    installScript = BuildScript.Bind(installScript, combinedExit =>
                     {
-                        var dotnetInstallPath = actions.PathCombine(tempWorkingDirectory, ".dotnet", "dotnet-install.sh");
+                        logger.LogInfo($"Attempting to download .NET {version}");
 
-                        var downloadDotNetInstallSh = BuildScript.DownloadFile(
+                        // When there are multiple versions requested, we want to try to fetch them all, reporting
-                            "https://dot.net/v1/dotnet-install.sh",
+                        // a successful exit code when at least one of them succeeds
-                            dotnetInstallPath,
+                        return combinedExit != 0 ? getInstall(version) : BuildScript.Bind(getInstall(version), _ => BuildScript.Success);
-                            e => logger.LogWarning($"Failed to download 'dotnet-install.sh': {e.Message}"));
+                    });
+                }
 
-                        var chmod = new CommandBuilder(actions).
+                return installScript & postlude;
-                            RunCommand("chmod").
+            });
-                            Argument("u+x").
-                            Argument(dotnetInstallPath);
-
-                        var install = new CommandBuilder(actions).
-                            RunCommand(dotnetInstallPath).
-                            Argument("--channel").
-                            Argument("release").
-                            Argument("--version").
-                            Argument(version).
-                            Argument("--install-dir").
-                            Argument(path);
-
-                        var buildScript = downloadDotNetInstallSh & chmod.Script & install.Script;
-
-                        if (shouldCleanUp)
-                        {
-                            buildScript &= BuildScript.DeleteFile(dotnetInstallPath);
-                        }
-
-                        return buildScript;
-                    }
-                });
         }
 
         private static BuildScript GetInstalledSdksScript(IBuildActions actions)