using System; using System.Collections.Generic; using System.Collections.ObjectModel; using System.IO; using System.Linq; using System.Threading; using Newtonsoft.Json.Linq; using Semmle.Util; using Semmle.Util.Logging; namespace Semmle.Extraction.CSharp.DependencyFetching { /// /// Utilities to run the "dotnet" command. /// public partial class DotNet : IDotNet { private readonly IDotNetCliInvoker dotnetCliInvoker; private readonly ILogger logger; private readonly TemporaryDirectory? tempWorkingDirectory; private DotNet(IDotNetCliInvoker dotnetCliInvoker, ILogger logger, bool runDotnetInfo, TemporaryDirectory? tempWorkingDirectory = null) { this.tempWorkingDirectory = tempWorkingDirectory; this.dotnetCliInvoker = dotnetCliInvoker; this.logger = logger; if (runDotnetInfo) { Info(); } } private DotNet(ILogger logger, string? dotNetPath, TemporaryDirectory tempWorkingDirectory, DependabotProxy? dependabotProxy) : this(new DotNetCliInvoker(logger, Path.Combine(dotNetPath ?? string.Empty, "dotnet"), dependabotProxy), logger, dotNetPath is null, tempWorkingDirectory) { } internal static IDotNet Make(IDotNetCliInvoker dotnetCliInvoker, ILogger logger, bool runDotnetInfo) => new DotNet(dotnetCliInvoker, logger, runDotnetInfo); public static IDotNet Make(ILogger logger, string? dotNetPath, TemporaryDirectory tempWorkingDirectory, DependabotProxy? dependabotProxy) => new DotNet(logger, dotNetPath, tempWorkingDirectory, dependabotProxy); private static void HandleRetryExitCode143(string dotnet, int attempt, ILogger logger) { logger.LogWarning($"Running '{dotnet} --info' failed with exit code 143. Retrying..."); var sleep = Math.Pow(2, attempt) * 1000; Thread.Sleep((int)sleep); } private void Info() { // Allow up to four attempts (with up to three retries) to run `dotnet --info`, to mitigate transient issues for (int attempt = 0; attempt < 4; attempt++) { var exitCode = dotnetCliInvoker.RunCommandExitCode("--info", silent: false); switch (exitCode) { case 0: return; case 143 when attempt < 3: HandleRetryExitCode143(dotnetCliInvoker.Exec, attempt, logger); continue; default: throw new Exception($"{dotnetCliInvoker.Exec} --info failed with exit code {exitCode}."); } } } private string GetRestoreArgs(RestoreSettings restoreSettings) { var args = $"restore --no-dependencies \"{restoreSettings.File}\" --packages \"{restoreSettings.PackageDirectory}\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal"; if (restoreSettings.ForceDotnetRefAssemblyFetching) { // Ugly hack: we set the TargetFrameworkRootPath and NetCoreTargetingPackRoot properties to an empty folder: var path = ".empty"; if (tempWorkingDirectory != null) { path = Path.Combine(tempWorkingDirectory.ToString(), "emptyFakeDotnetRoot"); Directory.CreateDirectory(path); } args += $" /p:TargetFrameworkRootPath=\"{path}\" /p:NetCoreTargetingPackRoot=\"{path}\" /p:AllowMissingPrunePackageData=true"; } if (restoreSettings.PathToNugetConfig != null) { args += $" --configfile \"{restoreSettings.PathToNugetConfig}\""; } if (restoreSettings.ForceReevaluation) { args += " --force"; } if (restoreSettings.TargetWindows) { args += " /p:EnableWindowsTargeting=true"; } if (restoreSettings.ExtraArgs is not null) { args += $" {restoreSettings.ExtraArgs}"; } return args; } public RestoreResult Restore(RestoreSettings restoreSettings) { var args = GetRestoreArgs(restoreSettings); var success = dotnetCliInvoker.RunCommand(args, out var output); return new(success, output); } public bool New(string folder) { var args = $"new console --no-restore --output \"{folder}\""; return dotnetCliInvoker.RunCommand(args); } public bool AddPackage(string folder, string package) { var args = $"add \"{folder}\" package \"{package}\" --no-restore"; return dotnetCliInvoker.RunCommand(args); } public IList GetListedRuntimes() => GetResultList("--list-runtimes"); public IList GetListedSdks() => GetResultList("--list-sdks"); private IList GetResultList(string args, string? workingDirectory = null, bool silent = true) { if (dotnetCliInvoker.RunCommand(args, workingDirectory, out var results, silent)) { return results; } logger.LogWarning($"Running 'dotnet {args}' failed."); return []; } public bool Exec(string execArgs) { var args = $"exec {execArgs}"; return dotnetCliInvoker.RunCommand(args); } private const string nugetListSourceCommand = "nuget list source --format Short"; public IList GetNugetFeeds(string nugetConfig) { logger.LogInfo($"Getting NuGet feeds from '{nugetConfig}'..."); return GetResultList($"{nugetListSourceCommand} --configfile \"{nugetConfig}\""); } public IList GetNugetFeedsFromFolder(string folderPath) { logger.LogInfo($"Getting NuGet feeds in folder '{folderPath}'..."); return GetResultList(nugetListSourceCommand, folderPath); } // The version number should be kept in sync with the version .NET version used for building the application. public const string LatestDotNetSdkVersion = "10.0.100"; public static ReadOnlyDictionary MinimalEnvironment => IDotNetCliInvoker.MinimalEnvironment; /// /// Returns a script for downloading relevant versions of the /// .NET SDK. The SDK(s) will be installed at installDir /// (provided that the script succeeds). /// private static BuildScript DownloadDotNet(IBuildActions actions, ILogger logger, IEnumerable files, string tempWorkingDirectory, bool shouldCleanUp, string installDir, string? version, bool ensureDotNetAvailable) { if (!string.IsNullOrEmpty(version)) // Specific version requested return DownloadDotNetVersion(actions, logger, tempWorkingDirectory, shouldCleanUp, installDir, [version]); // Download versions mentioned in `global.json` files // See https://docs.microsoft.com/en-us/dotnet/core/tools/global-json var versions = new List(); foreach (var path in files.Where(p => string.Equals(FileUtils.SafeGetFileName(p, logger), "global.json", StringComparison.OrdinalIgnoreCase))) { try { var o = JObject.Parse(File.ReadAllText(path)); var v = (string?)o?["sdk"]?["version"]; if (v is not null) { versions.Add(v); } } catch { // not a valid `global.json` file logger.LogInfo($"Couldn't find .NET SDK version in '{path}'."); continue; } } if (versions.Count > 0) { return DownloadDotNetVersion(actions, logger, tempWorkingDirectory, shouldCleanUp, installDir, versions) | // if neither of the versions succeed, try the latest version DownloadDotNetVersion(actions, logger, tempWorkingDirectory, shouldCleanUp, installDir, [LatestDotNetSdkVersion], needExactVersion: false); } if (ensureDotNetAvailable) { return DownloadDotNetVersion(actions, logger, tempWorkingDirectory, shouldCleanUp, installDir, [LatestDotNetSdkVersion], needExactVersion: false); } return BuildScript.Failure; } /// /// Returns a script for running `dotnet --info`, with retries on exit code 143. /// public static BuildScript InfoScript(IBuildActions actions, string dotnet, IDictionary? environment, ILogger logger) { var info = new CommandBuilder(actions, null, environment). RunCommand(dotnet). Argument("--info"); var script = info.Script; for (var attempt = 0; attempt < 4; attempt++) { var attemptCopy = attempt; // Capture in local variable script = BuildScript.Bind(script, ret => { switch (ret) { case 0: return BuildScript.Success; case 143 when attemptCopy < 3: HandleRetryExitCode143(dotnet, attemptCopy, logger); return info.Script; default: return BuildScript.Failure; } }); } return script; } /// /// Returns a script for downloading specific .NET SDK versions, if the /// versions are not already installed. /// /// See https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet-install-script. /// private static BuildScript DownloadDotNetVersion(IBuildActions actions, ILogger logger, string tempWorkingDirectory, bool shouldCleanUp, string path, IEnumerable versions, bool needExactVersion = true) { if (!versions.Any()) { logger.LogInfo("No .NET SDK versions requested."); return BuildScript.Failure; } return BuildScript.Bind(GetInstalledSdksScript(actions), (sdks, sdksRet) => { if ( needExactVersion && sdksRet == 0 && // quadratic; should be OK, given that both `version` and `sdks` are expected to be small versions.All(version => sdks.Any(sdk => sdk.StartsWith(version + " ", StringComparison.Ordinal)))) { // The requested SDKs are already installed, so no need to reinstall return BuildScript.Failure; } else if (!needExactVersion && sdksRet == 0 && sdks.Count > 0) { // there's at least one SDK installed, so no need to reinstall return BuildScript.Failure; } else if (!needExactVersion && sdksRet != 0) { logger.LogInfo("No .NET SDK found."); } BuildScript prelude; BuildScript postlude; Func getInstall; if (actions.IsWindows()) { prelude = BuildScript.Success; postlude = BuildScript.Success; getInstall = version => { var psCommand = $"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; &([scriptblock]::Create((Invoke-WebRequest -UseBasicParsing 'https://dot.net/v1/dotnet-install.ps1'))) -Version {version} -InstallDir {path}"; BuildScript GetInstall(string pwsh) => new CommandBuilder(actions). RunCommand(pwsh). Argument("-NoProfile"). Argument("-ExecutionPolicy"). Argument("unrestricted"). Argument("-Command"). Argument($"\"{psCommand}\""). Script; return GetInstall("pwsh") | GetInstall("powershell"); }; } else { var dotnetInstallPath = actions.PathCombine(tempWorkingDirectory, ".dotnet", "dotnet-install.sh"); var downloadDotNetInstallSh = BuildScript.DownloadFile( "https://dot.net/v1/dotnet-install.sh", dotnetInstallPath, e => logger.LogWarning($"Failed to download 'dotnet-install.sh': {e.Message}"), logger); var chmod = new CommandBuilder(actions). RunCommand("chmod"). Argument("u+x"). Argument(dotnetInstallPath); prelude = downloadDotNetInstallSh & chmod.Script; postlude = shouldCleanUp ? BuildScript.DeleteFile(dotnetInstallPath) : BuildScript.Success; getInstall = version => { var cb = new CommandBuilder(actions). RunCommand(dotnetInstallPath). Argument("--channel"). Argument("release"). Argument("--version"). Argument(version); // Request ARM64 architecture on Apple Silicon machines if (actions.IsRunningOnAppleSilicon()) { cb.Argument("--architecture"). Argument("arm64"); } return cb.Argument("--install-dir"). Argument(path).Script; }; } var dotnetInfo = InfoScript(actions, actions.PathCombine(path, "dotnet"), MinimalEnvironment.ToDictionary(), logger); Func getInstallAndVerify = version => // run `dotnet --info` after install, to check that it executes successfully getInstall(version) & dotnetInfo; var installScript = prelude & BuildScript.Failure; var attempted = new HashSet(); foreach (var version in versions) { if (!attempted.Add(version)) continue; installScript = BuildScript.Bind(installScript, combinedExit => { logger.LogInfo($"Attempting to download .NET {version}"); // When there are multiple versions requested, we want to try to fetch them all, reporting // a successful exit code when at least one of them succeeds return combinedExit != 0 ? getInstallAndVerify(version) : BuildScript.Bind(getInstallAndVerify(version), _ => BuildScript.Success); }); } return installScript & postlude; }); } private static BuildScript GetInstalledSdksScript(IBuildActions actions) { var listSdks = new CommandBuilder(actions, silent: true, environment: MinimalEnvironment). RunCommand("dotnet"). Argument("--list-sdks"); return listSdks.Script; } /// /// Returns a script that attempts to download relevant version(s) of the /// .NET SDK, followed by running the script generated by . /// /// The argument to is the path to the directory in which the /// .NET SDK(s) were installed. /// public static BuildScript WithDotNet(IBuildActions actions, ILogger logger, IEnumerable files, string tempWorkingDirectory, bool shouldCleanUp, bool ensureDotNetAvailable, string? version, Func f) { var installDir = actions.PathCombine(tempWorkingDirectory, ".dotnet"); var installScript = DownloadDotNet(actions, logger, files, tempWorkingDirectory, shouldCleanUp, installDir, version, ensureDotNetAvailable); return BuildScript.Bind(installScript, installed => { if (installed != 0) { // The .NET SDK was not installed, either because the installation failed or because it was already installed. installDir = null; } return f(installDir); }); } } }