C#: Fetch .NET in dependency manager instead of autobuilder

This commit is contained in:
Tom Hvitved
2024-02-29 10:33:24 +01:00
parent c6426d8858
commit 62382fc5a2
31 changed files with 306 additions and 373 deletions

View File

@@ -558,8 +558,6 @@ namespace Semmle.Autobuild.CSharp.Tests
[Fact]
public void TestLinuxBuildlessExtractionSuccess()
{
actions.RunProcess["dotnet --list-sdks"] = 0;
actions.RunProcessOut["dotnet --list-sdks"] = "any version";
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 0;
actions.FileExists["csharp.log"] = true;
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
@@ -569,14 +567,12 @@ namespace Semmle.Autobuild.CSharp.Tests
actions.EnumerateDirectories[@"C:\Project"] = "";
var autobuilder = CreateAutoBuilder(false, buildless: "true");
TestAutobuilderScript(autobuilder, 0, 2);
TestAutobuilderScript(autobuilder, 0, 1);
}
[Fact]
public void TestLinuxBuildlessExtractionFailed()
{
actions.RunProcess["dotnet --list-sdks"] = 0;
actions.RunProcessOut["dotnet --list-sdks"] = "any version";
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 10;
actions.FileExists["csharp.log"] = true;
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
@@ -586,14 +582,12 @@ namespace Semmle.Autobuild.CSharp.Tests
actions.EnumerateDirectories[@"C:\Project"] = "";
var autobuilder = CreateAutoBuilder(false, buildless: "true");
TestAutobuilderScript(autobuilder, 10, 2);
TestAutobuilderScript(autobuilder, 10, 1);
}
[Fact]
public void TestLinuxBuildlessExtractionSolution()
{
actions.RunProcess["dotnet --list-sdks"] = 0;
actions.RunProcessOut["dotnet --list-sdks"] = "any version";
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 0;
actions.FileExists["csharp.log"] = true;
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
@@ -603,28 +597,7 @@ namespace Semmle.Autobuild.CSharp.Tests
actions.EnumerateDirectories[@"C:\Project"] = "";
var autobuilder = CreateAutoBuilder(false, buildless: "true");
TestAutobuilderScript(autobuilder, 0, 2);
}
[Fact]
public void TestLinuxBuildlessExtractionNoDotnet()
{
actions.RunProcess["dotnet --list-sdks"] = 1;
actions.RunProcessOut["dotnet --list-sdks"] = "";
actions.RunProcess[@"chmod u+x scratch/.dotnet/dotnet-install.sh"] = 0;
actions.RunProcess[@"scratch/.dotnet/dotnet-install.sh --channel release --version 8.0.101 --install-dir scratch/.dotnet"] = 0;
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone --dotnet scratch/.dotnet"] = 0;
actions.FileExists["csharp.log"] = true;
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
actions.EnumerateDirectories[@"C:\Project"] = "";
actions.DownloadFiles.Add(("https://dot.net/v1/dotnet-install.sh", "scratch/.dotnet/dotnet-install.sh"));
actions.CreateDirectories.Add(@"scratch/.dotnet");
var autobuilder = CreateAutoBuilder(false, buildless: "true");
TestAutobuilderScript(autobuilder, 0, 4);
TestAutobuilderScript(autobuilder, 0, 1);
}
private void SkipVsWhere()
@@ -915,8 +888,6 @@ namespace Semmle.Autobuild.CSharp.Tests
[Fact]
public void TestSkipNugetBuildless()
{
actions.RunProcess["dotnet --list-sdks"] = 0;
actions.RunProcessOut["dotnet --list-sdks"] = "any version";
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 0;
actions.FileExists["csharp.log"] = true;
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
@@ -926,7 +897,7 @@ namespace Semmle.Autobuild.CSharp.Tests
actions.EnumerateDirectories[@"C:\Project"] = "";
var autobuilder = CreateAutoBuilder(false, buildless: "true");
TestAutobuilderScript(autobuilder, 0, 2);
TestAutobuilderScript(autobuilder, 0, 1);
}

View File

@@ -50,11 +50,8 @@ namespace Semmle.Autobuild.CSharp
attempt = new BuildCommandRule(DotNetRule.WithDotNet).Analyse(this, false) & CheckExtractorRun(true);
break;
case CSharpBuildStrategy.Buildless:
attempt = DotNetRule.WithDotNet(this, ensureDotNetAvailable: true, (dotNetPath, env) =>
{
// No need to check that the extractor has been executed in buildless mode
return new StandaloneBuildRule(dotNetPath).Analyse(this, false);
});
// No need to check that the extractor has been executed in buildless mode
attempt = new StandaloneBuildRule().Analyse(this, false);
break;
case CSharpBuildStrategy.MSBuild:
attempt = new MsBuildRule().Analyse(this, false) & CheckExtractorRun(true);
@@ -84,7 +81,7 @@ namespace Semmle.Autobuild.CSharp
return 0;
if (warnOnFailure)
Log(Severity.Error, "No C# code detected during build.");
Logger.LogError("No C# code detected during build.");
return 1;
});

View File

@@ -1,8 +0,0 @@
namespace Semmle.Autobuild.CSharp
{
internal static class Constants
{
// The version number should be kept in sync with the version .NET version used for building the application.
public const string LatestDotNetSdkVersion = "8.0.101";
}
}

View File

@@ -5,7 +5,7 @@ using System.Linq;
using Semmle.Util;
using Semmle.Util.Logging;
using Semmle.Autobuild.Shared;
using Newtonsoft.Json.Linq;
using Semmle.Extraction.CSharp.DependencyFetching;
namespace Semmle.Autobuild.CSharp
{
@@ -39,11 +39,11 @@ namespace Semmle.Autobuild.CSharp
if (notDotNetProject is not null)
{
builder.Log(Severity.Info, "Not using .NET Core because of incompatible project {0}", notDotNetProject);
builder.Logger.Log(Severity.Info, "Not using .NET Core because of incompatible project {0}", notDotNetProject);
return BuildScript.Failure;
}
builder.Log(Severity.Info, "Attempting to build using .NET Core");
builder.Logger.LogInfo("Attempting to build using .NET Core");
}
return WithDotNet(builder, ensureDotNetAvailable: false, (dotNetPath, environment) =>
@@ -81,29 +81,22 @@ namespace Semmle.Autobuild.CSharp
/// </summary>
public static BuildScript WithDotNet(IAutobuilder<AutobuildOptionsShared> builder, bool ensureDotNetAvailable, Func<string?, IDictionary<string, string>?, BuildScript> f)
{
var installDir = builder.Actions.PathCombine(FileUtils.GetTemporaryWorkingDirectory(builder.Actions.GetEnvironmentVariable, builder.Options.Language.UpperCaseName, out var _), ".dotnet");
var installScript = DownloadDotNet(builder, installDir, ensureDotNetAvailable);
return BuildScript.Bind(installScript, installed =>
var temp = FileUtils.GetTemporaryWorkingDirectory(builder.Actions.GetEnvironmentVariable, builder.Options.Language.UpperCaseName, out var shouldCleanUp);
return DotNet.WithDotNet(builder.Actions, builder.Logger, builder.Paths.Select(x => x.Item1), temp, shouldCleanUp, ensureDotNetAvailable, builder.Options.DotNetVersion, installDir =>
{
var env = new Dictionary<string, string>
{
{ "DOTNET_SKIP_FIRST_TIME_EXPERIENCE", "true" },
{ "MSBUILDDISABLENODEREUSE", "1" }
};
if (installed == 0)
if (installDir is not null)
{
// The installation succeeded, so use the newly installed .NET Core
// The installation succeeded, so use the newly installed .NET
var path = builder.Actions.GetEnvironmentVariable("PATH");
var delim = builder.Actions.IsWindows() ? ";" : ":";
env.Add("DOTNET_MULTILEVEL_LOOKUP", "false"); // prevent look up of other .NET Core SDKs
env.Add("DOTNET_MULTILEVEL_LOOKUP", "false"); // prevent look up of other .NET SDKs
env.Add("PATH", installDir + delim + path);
}
else
{
// The .NET SDK was not installed, either because the installation failed or because it was already installed.
installDir = null;
}
return f(installDir, env);
});
}
@@ -119,146 +112,6 @@ namespace Semmle.Autobuild.CSharp
public static BuildScript WithDotNet(IAutobuilder<AutobuildOptionsShared> builder, Func<IDictionary<string, string>?, BuildScript> f)
=> WithDotNet(builder, ensureDotNetAvailable: false, (_, env) => f(env));
/// <summary>
/// Returns a script for downloading relevant versions of the
/// .NET Core SDK. The SDK(s) will be installed at <code>installDir</code>
/// (provided that the script succeeds).
/// </summary>
private static BuildScript DownloadDotNet(IAutobuilder<AutobuildOptionsShared> builder, string installDir, bool ensureDotNetAvailable)
{
if (!string.IsNullOrEmpty(builder.Options.DotNetVersion))
// Specific version supplied in configuration: always use that
return DownloadDotNetVersion(builder, installDir, builder.Options.DotNetVersion);
// Download versions mentioned in `global.json` files
// See https://docs.microsoft.com/en-us/dotnet/core/tools/global-json
var installScript = BuildScript.Success;
var validGlobalJson = false;
foreach (var path in builder.Paths.Select(p => p.Item1).Where(p => p.EndsWith("global.json", StringComparison.Ordinal)))
{
string version;
try
{
var o = JObject.Parse(File.ReadAllText(path));
version = (string)(o?["sdk"]?["version"]!);
}
catch // lgtm[cs/catch-of-all-exceptions]
{
// not a valid global.json file
continue;
}
installScript &= DownloadDotNetVersion(builder, installDir, version);
validGlobalJson = true;
}
if (validGlobalJson)
{
return installScript;
}
if (ensureDotNetAvailable)
{
return DownloadDotNetVersion(builder, installDir, Constants.LatestDotNetSdkVersion, needExactVersion: false);
}
return BuildScript.Failure;
}
/// <summary>
/// Returns a script for downloading a specific .NET Core SDK version, if the
/// version is not already installed.
///
/// See https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet-install-script.
/// </summary>
private static BuildScript DownloadDotNetVersion(IAutobuilder<AutobuildOptionsShared> builder, string path, string version, bool needExactVersion = true)
{
return BuildScript.Bind(GetInstalledSdksScript(builder.Actions), (sdks, sdksRet) =>
{
if (needExactVersion && sdksRet == 0 && sdks.Count == 1 && sdks[0].StartsWith(version + " ", StringComparison.Ordinal))
{
// The requested SDK is already installed (and no other SDKs are installed), so
// no need to reinstall
return BuildScript.Failure;
}
else if (!needExactVersion && sdksRet == 0 && sdks.Count > 0)
{
// there's at least one SDK installed, so no need to reinstall
return BuildScript.Failure;
}
else if (!needExactVersion && sdksRet != 0)
{
builder.Log(Severity.Info, "No .NET Core SDK found.");
}
builder.Log(Severity.Info, "Attempting to download .NET Core {0}", version);
if (builder.Actions.IsWindows())
{
var psCommand = $"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; &([scriptblock]::Create((Invoke-WebRequest -UseBasicParsing 'https://dot.net/v1/dotnet-install.ps1'))) -Version {version} -InstallDir {path}";
BuildScript GetInstall(string pwsh) =>
new CommandBuilder(builder.Actions).
RunCommand(pwsh).
Argument("-NoProfile").
Argument("-ExecutionPolicy").
Argument("unrestricted").
Argument("-Command").
Argument("\"" + psCommand + "\"").
Script;
return GetInstall("pwsh") | GetInstall("powershell");
}
else
{
var dotnetInstallPath = builder.Actions.PathCombine(FileUtils.GetTemporaryWorkingDirectory(
builder.Actions.GetEnvironmentVariable,
builder.Options.Language.UpperCaseName,
out var shouldCleanUp), ".dotnet", "dotnet-install.sh");
var downloadDotNetInstallSh = BuildScript.DownloadFile(
"https://dot.net/v1/dotnet-install.sh",
dotnetInstallPath,
e => builder.Log(Severity.Warning, $"Failed to download 'dotnet-install.sh': {e.Message}"));
var chmod = new CommandBuilder(builder.Actions).
RunCommand("chmod").
Argument("u+x").
Argument(dotnetInstallPath);
var install = new CommandBuilder(builder.Actions).
RunCommand(dotnetInstallPath).
Argument("--channel").
Argument("release").
Argument("--version").
Argument(version).
Argument("--install-dir").
Argument(path);
var buildScript = downloadDotNetInstallSh & chmod.Script & install.Script;
if (shouldCleanUp)
{
var removeScript = new CommandBuilder(builder.Actions).
RunCommand("rm").
Argument(dotnetInstallPath);
buildScript &= removeScript.Script;
}
return buildScript;
}
});
}
private static BuildScript GetInstalledSdksScript(IBuildActions actions)
{
var listSdks = new CommandBuilder(actions, silent: true).
RunCommand("dotnet").
Argument("--list-sdks");
return listSdks.Script;
}
private static string DotNetCommand(IBuildActions actions, string? dotNetPath) =>
dotNetPath is not null ? actions.PathCombine(dotNetPath, "dotnet") : "dotnet";

View File

@@ -20,6 +20,7 @@
<ItemGroup>
<ProjectReference Include="..\..\extractor\Semmle.Util\Semmle.Util.csproj" />
<ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp\Semmle.Extraction.CSharp.csproj" />
<ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp.DependencyFetching\Semmle.Extraction.CSharp.DependencyFetching.csproj" />
<ProjectReference Include="..\Semmle.Autobuild.Shared\Semmle.Autobuild.Shared.csproj" />
</ItemGroup>
</Project>

View File

@@ -8,13 +8,6 @@ namespace Semmle.Autobuild.CSharp
/// </summary>
internal class StandaloneBuildRule : IBuildRule<CSharpAutobuildOptions>
{
private readonly string? dotNetPath;
internal StandaloneBuildRule(string? dotNetPath)
{
this.dotNetPath = dotNetPath;
}
public BuildScript Analyse(IAutobuilder<CSharpAutobuildOptions> builder, bool auto)
{
if (builder.CodeQLExtractorLangRoot is null
@@ -27,12 +20,6 @@ namespace Semmle.Autobuild.CSharp
var cmd = new CommandBuilder(builder.Actions);
cmd.RunCommand(standalone);
if (!string.IsNullOrEmpty(this.dotNetPath))
{
cmd.Argument("--dotnet");
cmd.QuoteArgument(this.dotNetPath);
}
return cmd.Script;
}
}

View File

@@ -70,11 +70,9 @@ namespace Semmle.Autobuild.Shared
IBuildActions Actions { get; }
/// <summary>
/// Log a given build event to the console.
/// A logger.
/// </summary>
/// <param name="format">The format string.</param>
/// <param name="args">Inserts to the format string.</param>
void Log(Severity severity, string format, params object[] args);
ILogger Logger { get; }
/// <summary>
/// Value of CODEQL_EXTRACTOR_<LANG>_ROOT environment variable.
@@ -135,21 +133,7 @@ namespace Semmle.Autobuild.Shared
/// <returns>True iff the path was found.</returns>
public bool HasPath(string path) => Paths.Any(p => path == p.Item1);
private void FindFiles(string dir, int depth, int maxDepth, IList<(string, int)> results)
{
foreach (var f in Actions.EnumerateFiles(dir))
{
results.Add((f, depth));
}
if (depth < maxDepth)
{
foreach (var d in Actions.EnumerateDirectories(dir))
{
FindFiles(d, depth + 1, maxDepth, results);
}
}
}
/// <summary>
/// The root of the source directory.
@@ -196,12 +180,7 @@ namespace Semmle.Autobuild.Shared
Options = options;
DiagnosticClassifier = diagnosticClassifier;
pathsLazy = new Lazy<IEnumerable<(string, int)>>(() =>
{
var files = new List<(string, int)>();
FindFiles(options.RootDirectory, 0, options.SearchDepth, files);
return files.OrderBy(f => f.Item2).ToArray();
});
pathsLazy = new Lazy<IEnumerable<(string, int)>>(() => Actions.FindFiles(options.RootDirectory, options.SearchDepth));
projectsOrSolutionsToBuildLazy = new Lazy<IList<IProjectOrSolution>>(() =>
{
@@ -214,7 +193,7 @@ namespace Semmle.Autobuild.Shared
if (actions.FileExists(solution))
ret.Add(new Solution<TAutobuildOptions>(this, solution, true));
else
Log(Severity.Error, $"The specified project or solution file {solution} was not found");
logger.LogError($"The specified project or solution file {solution} was not found");
}
return ret;
}
@@ -273,6 +252,8 @@ namespace Semmle.Autobuild.Shared
logThreadId: false) ?? Verbosity.Info,
logThreadId: false);
public ILogger Logger => logger;
private readonly IDiagnosticsWriter diagnostics;
/// <summary>
@@ -285,16 +266,6 @@ namespace Semmle.Autobuild.Shared
return Path.GetRelativePath(this.RootDirectory, path);
}
/// <summary>
/// Log a given build event to the console.
/// </summary>
/// <param name="format">The format string.</param>
/// <param name="args">Inserts to the format string.</param>
public void Log(Severity severity, string format, params object[] args)
{
logger.Log(severity, format, args);
}
/// <summary>
/// Write <paramref name="diagnostic"/> to the diagnostics file.
/// </summary>
@@ -310,7 +281,7 @@ namespace Semmle.Autobuild.Shared
/// <returns>The exit code, 0 for success and non-zero for failures.</returns>
public int AttemptBuild()
{
Log(Severity.Info, $"Working directory: {Options.RootDirectory}");
logger.LogInfo($"Working directory: {Options.RootDirectory}");
var script = GetBuildScript();
@@ -319,12 +290,12 @@ namespace Semmle.Autobuild.Shared
void startCallback(string s, bool silent)
{
Log(silent ? Severity.Debug : Severity.Info, $"\nRunning {s}");
logger.Log(silent ? Severity.Debug : Severity.Info, $"\nRunning {s}");
}
void exitCallback(int ret, string msg, bool silent)
{
Log(silent ? Severity.Debug : Severity.Info, $"Exit code {ret}{(string.IsNullOrEmpty(msg) ? "" : $": {msg}")}");
logger.Log(silent ? Severity.Debug : Severity.Info, $"Exit code {ret}{(string.IsNullOrEmpty(msg) ? "" : $": {msg}")}");
}
var onOutput = BuildOutputHandler(Console.Out);
@@ -374,7 +345,7 @@ namespace Semmle.Autobuild.Shared
protected BuildScript AutobuildFailure() =>
BuildScript.Create(actions =>
{
Log(Severity.Error, "Could not auto-detect a suitable build method");
logger.LogError("Could not auto-detect a suitable build method");
AutobuildFailureDiagnostic();

View File

@@ -55,7 +55,7 @@ namespace Semmle.Autobuild.Shared
public BuildScript Analyse(IAutobuilder<AutobuildOptionsShared> builder, bool auto)
{
builder.Log(Severity.Info, "Attempting to locate build script");
builder.Logger.LogInfo("Attempting to locate build script");
// a list of extensions for files that we consider to be scripts on the current platform
var extensions = builder.Actions.IsWindows() ? winExtensions : linuxExtensions;

View File

@@ -40,7 +40,7 @@ namespace Semmle.Autobuild.Shared
return BuildScript.Failure;
if (auto)
builder.Log(Severity.Info, "Attempting to build using MSBuild");
builder.Logger.LogInfo("Attempting to build using MSBuild");
var vsTools = GetVcVarsBatFile(builder);
@@ -54,7 +54,7 @@ namespace Semmle.Autobuild.Shared
if (vsTools is null && builder.Actions.IsWindows())
{
builder.Log(Severity.Warning, "Could not find a suitable version of VsDevCmd.bat/vcvarsall.bat");
builder.Logger.LogWarning("Could not find a suitable version of VsDevCmd.bat/vcvarsall.bat");
}
// Use `nuget.exe` from source code repo, if present, otherwise first attempt with global
@@ -165,18 +165,18 @@ namespace Semmle.Autobuild.Shared
{
foreach (var b in BuildTools.VcVarsAllBatFiles(builder.Actions))
{
builder.Log(Severity.Info, "Found {0} version {1}", b.Path, b.ToolsVersion);
builder.Logger.Log(Severity.Info, "Found {0} version {1}", b.Path, b.ToolsVersion);
}
vsTools = BuildTools.FindCompatibleVcVars(builder.Actions, msToolsVersion);
if (vsTools is null)
builder.Log(Severity.Warning, "Could not find build tools matching version {0}", msToolsVersion);
builder.Logger.LogWarning("Could not find build tools matching version {0}", msToolsVersion);
else
builder.Log(Severity.Info, "Setting Visual Studio tools to {0}", vsTools.Path);
builder.Logger.Log(Severity.Info, "Setting Visual Studio tools to {0}", vsTools.Path);
}
else
{
builder.Log(Severity.Error, "The format of vstools_version is incorrect. Please specify an integer.");
builder.Logger.LogError("The format of vstools_version is incorrect. Please specify an integer.");
}
}
@@ -189,18 +189,18 @@ namespace Semmle.Autobuild.Shared
private static BuildScript DownloadNugetExe<TAutobuildOptions>(IAutobuilder<TAutobuildOptions> builder, string path) where TAutobuildOptions : AutobuildOptionsShared =>
BuildScript.Create(_ =>
{
builder.Log(Severity.Info, "Attempting to download nuget.exe");
builder.Logger.LogInfo("Attempting to download nuget.exe");
return 0;
})
&
BuildScript.DownloadFile(
FileUtils.NugetExeUrl,
path,
e => builder.Log(Severity.Warning, $"Failed to download 'nuget.exe': {e.Message}"))
e => builder.Logger.LogWarning($"Failed to download 'nuget.exe': {e.Message}"))
&
BuildScript.Create(_ =>
{
builder.Log(Severity.Info, $"Successfully downloaded {path}");
builder.Logger.LogInfo($"Successfully downloaded {path}");
return 0;
});
}

View File

@@ -41,7 +41,7 @@ namespace Semmle.Autobuild.Shared
}
catch (Exception ex) when (ex is XmlException || ex is FileNotFoundException)
{
builder.Log(Severity.Info, $"Unable to read project file {path}.");
builder.Logger.LogInfo($"Unable to read project file {path}.");
return;
}
@@ -66,7 +66,7 @@ namespace Semmle.Autobuild.Shared
catch // lgtm[cs/catch-of-all-exceptions]
// Generic catch clause - Version constructor throws about 5 different exceptions.
{
builder.Log(Severity.Warning, "Project {0} has invalid tools version {1}", path, toolsVersion);
builder.Logger.Log(Severity.Warning, "Project {0} has invalid tools version {1}", path, toolsVersion);
}
}

View File

@@ -73,7 +73,7 @@ namespace Semmle.Autobuild.Shared
return;
}
builder.Log(Severity.Info, $"Unable to read solution file {path}.");
builder.Logger.LogInfo($"Unable to read solution file {path}.");
includedProjects = Array.Empty<Project<TAutobuildOptions>>();
return;
}