hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-17 01:03:14 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #13999 from github/mbg/csharp/standalone/dotnet-version

Browse Source 
C# Standalone: Install .NET SDK specified in `global.json`
This commit is contained in:
Michael B. Gale
2023-09-07 11:30:53 +01:00
committed by GitHub
parent ec0529d68c ccbc6f446a
commit 38892bb51b
10 changed files with 136 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs
View File

@@ -557,7 +557,7 @@ namespace Semmle.Autobuild.CSharp.Tests
[Fact]
public void TestLinuxBuildlessExtractionSuccess()
{
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone --references:."] = 0;
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 0;
actions.FileExists["csharp.log"] = true;
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
@@ -571,7 +571,7 @@ namespace Semmle.Autobuild.CSharp.Tests
[Fact]
public void TestLinuxBuildlessExtractionFailed()
{
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone --references:."] = 10;
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 10;
actions.FileExists["csharp.log"] = true;
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
@@ -585,7 +585,7 @@ namespace Semmle.Autobuild.CSharp.Tests
[Fact]
public void TestLinuxBuildlessExtractionSolution()
{
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone foo.sln --references:."] = 0;
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone foo.sln"] = 0;
actions.FileExists["csharp.log"] = true;
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
@@ -873,7 +873,7 @@ namespace Semmle.Autobuild.CSharp.Tests
[Fact]
public void TestSkipNugetBuildless()
{
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone foo.sln --references:. --skip-nuget"] = 0;
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone foo.sln --skip-nuget"] = 0;
actions.FileExists["csharp.log"] = true;
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";

7
csharp/autobuilder/Semmle.Autobuild.CSharp/CSharpAutobuilder.cs
View File

@@ -48,8 +48,11 @@ namespace Semmle.Autobuild.CSharp
attempt = new BuildCommandRule(DotNetRule.WithDotNet).Analyse(this, false) & CheckExtractorRun(true);
break;
case CSharpBuildStrategy.Buildless:
// No need to check that the extractor has been executed in buildless mode
attempt = new StandaloneBuildRule().Analyse(this, false);
attempt = DotNetRule.WithDotNet(this, (dotNetPath, env) =>
{
// No need to check that the extractor has been executed in buildless mode
return new StandaloneBuildRule(dotNetPath).Analyse(this, false);
});
break;
case CSharpBuildStrategy.MSBuild:
attempt = new MsBuildRule().Analyse(this, false) & CheckExtractorRun(true);

11
csharp/autobuilder/Semmle.Autobuild.CSharp/DotNetRule.cs
View File

@@ -70,7 +70,16 @@ namespace Semmle.Autobuild.CSharp
});
}
private static BuildScript WithDotNet(IAutobuilder<AutobuildOptionsShared> builder, Func<string?, IDictionary<string, string>?, BuildScript> f)
/// <summary>
/// Returns a script that attempts to download relevant version(s) of the
/// .NET Core SDK, followed by running the script generated by <paramref name="f"/>.
///
/// The arguments to <paramref name="f"/> are the path to the directory in which the
/// .NET Core SDK(s) were installed and any additional required environment
/// variables needed by the installed .NET Core (<code>null</code> when no variables
/// are needed).
/// </summary>
public static BuildScript WithDotNet(IAutobuilder<AutobuildOptionsShared> builder, Func<string?, IDictionary<string, string>?, BuildScript> f)
{
var installDir = builder.Actions.PathCombine(builder.Options.RootDirectory, ".dotnet");
var installScript = DownloadDotNet(builder, installDir);

15
csharp/autobuilder/Semmle.Autobuild.CSharp/StandaloneBuildRule.cs
View File

@@ -8,6 +8,13 @@ namespace Semmle.Autobuild.CSharp
/// </summary>
internal class StandaloneBuildRule : IBuildRule<CSharpAutobuildOptions>
{
private readonly string? dotNetPath;
internal StandaloneBuildRule(string? dotNetPath)
{
this.dotNetPath = dotNetPath;
}
public BuildScript Analyse(IAutobuilder<CSharpAutobuildOptions> builder, bool auto)
{
BuildScript GetCommand(string? solution)
@@ -28,13 +35,17 @@ namespace Semmle.Autobuild.CSharp
if (solution is not null)
cmd.QuoteArgument(solution);
cmd.Argument("--references:.");
if (!builder.Options.NugetRestore)
{
cmd.Argument("--skip-nuget");
}
if (!string.IsNullOrEmpty(this.dotNetPath))
{
cmd.Argument("--dotnet");
cmd.QuoteArgument(this.dotNetPath);
}
return cmd.Script;
}

20
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs
View File

@@ -15,14 +15,24 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// <summary>
/// Locate all reference files and index them.
/// </summary>
/// <param name="dirs">Directories to search.</param>
/// <param name="paths">
/// Paths to search. Directories are searched recursively. Files are added directly to the
/// assembly cache.
/// </param>
/// <param name="progressMonitor">Callback for progress.</param>
public AssemblyCache(IEnumerable<string> dirs, ProgressMonitor progressMonitor)
public AssemblyCache(IEnumerable<string> paths, ProgressMonitor progressMonitor)
{
foreach (var dir in dirs)
foreach (var path in paths)
{
progressMonitor.FindingFiles(dir);
AddReferenceDirectory(dir);
if (File.Exists(path))
{
pendingDllsToIndex.Enqueue(path);
}
else
{
progressMonitor.FindingFiles(path);
AddReferenceDirectory(path);
}
}
IndexReferences();
}

22
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
View File

@@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
@@ -31,6 +31,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private readonly FileContent fileContent;
private readonly TemporaryDirectory packageDirectory;
private TemporaryDirectory? razorWorkingDirectory;
private readonly Git git;
/// <summary>
@@ -48,7 +49,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
try
{
this.dotnet = new DotNet(progressMonitor);
this.dotnet = new DotNet(options, progressMonitor);
}
catch
{
@@ -68,7 +69,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
? new[] { options.SolutionFile }
: allFiles.SelectFileNamesByExtension(".sln");
var dllDirNames = options.DllDirs.Select(Path.GetFullPath).ToList();
// If DLL reference paths are specified on the command-line, use those to discover
// assemblies. Otherwise (the default), query the git CLI to determine which DLL files
// are tracked as part of the repository.
this.git = new Git(this.progressMonitor);
var dllDirNames = options.DllDirs.Count == 0 ? this.git.ListFiles("*.dll") : options.DllDirs.Select(Path.GetFullPath).ToList();
// Find DLLs in the .Net / Asp.Net Framework
if (options.ScanNetFrameworkDlls)
@@ -98,13 +103,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
progressMonitor.MissingNuGet();
}
// TODO: remove the below when the required SDK is installed
using (new FileRenamer(sourceDir.GetFiles("global.json", SearchOption.AllDirectories)))
{
Restore(solutions);
Restore(allProjects);
DownloadMissingPackages(allFiles);
}
Restore(solutions);
Restore(allProjects);
DownloadMissingPackages(allFiles);
}
assemblyCache = new AssemblyCache(dllDirNames, progressMonitor);
@@ -157,7 +158,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
progressMonitor.LogInfo($"Found {views.Length} cshtml and razor files.");
// TODO: use SDK specified in global.json
var sdk = new Sdk(dotnet).GetNewestSdk();
if (sdk != null)
{

7
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyOptions.cs
View File

@@ -51,6 +51,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// The number of threads to use.
/// </summary>
int Threads { get; }
/// <summary>
/// The path to the local ".dotnet" directory, if any.
/// </summary>
string? DotNetPath { get; }
}
public class DependencyOptions : IDependencyOptions
@@ -73,5 +78,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
Excludes.Any(path.Contains);
public int Threads { get; set; } = EnvironmentVariables.GetDefaultNumberOfThreads();
public string? DotNetPath { get; set; } = null;
}
}

12
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs
View File

@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using Semmle.Util;
namespace Semmle.Extraction.CSharp.DependencyFetching
@@ -10,12 +11,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// </summary>
internal class DotNet : IDotNet
{
private const string dotnet = "dotnet";
private readonly ProgressMonitor progressMonitor;
private readonly string dotnet;
public DotNet(ProgressMonitor progressMonitor)
public DotNet(IDependencyOptions options, ProgressMonitor progressMonitor)
{
this.progressMonitor = progressMonitor;
this.dotnet = Path.Combine(options.DotNetPath ?? string.Empty, "dotnet");
Info();
}
@@ -29,7 +31,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
}
private static ProcessStartInfo MakeDotnetStartInfo(string args, bool redirectStandardOutput) =>
private ProcessStartInfo MakeDotnetStartInfo(string args, bool redirectStandardOutput) =>
new ProcessStartInfo(dotnet, args)
{
UseShellExecute = false,
@@ -39,7 +41,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private bool RunCommand(string args)
{
progressMonitor.RunningProcess($"{dotnet} {args}");
using var proc = Process.Start(MakeDotnetStartInfo(args, redirectStandardOutput: false));
using var proc = Process.Start(this.MakeDotnetStartInfo(args, redirectStandardOutput: false));
proc?.WaitForExit();
var exitCode = proc?.ExitCode ?? -1;
if (exitCode != 0)
@@ -77,7 +79,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private IList<string> GetListed(string args, string artifact)
{
progressMonitor.RunningProcess($"{dotnet} {args}");
var pi = MakeDotnetStartInfo(args, redirectStandardOutput: true);
var pi = this.MakeDotnetStartInfo(args, redirectStandardOutput: true);
var exitCode = pi.ReadOutput(out var artifacts);
if (exitCode != 0)
{

61
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Git.cs Normal file
View File

@@ -0,0 +1,61 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
/// <summary>
/// Utilities for querying information from the git CLI.
/// </summary>
internal class Git
{
private readonly ProgressMonitor progressMonitor;
private const string git = "git";
public Git(ProgressMonitor progressMonitor)
{
this.progressMonitor = progressMonitor;
}
/// <summary>
/// Lists all files matching <paramref name="pattern"/> which are tracked in the
/// current git repository.
/// </summary>
/// <param name="pattern">The file pattern.</param>
/// <returns>A list of all tracked files which match <paramref name="pattern"/>.</returns>
/// <exception cref="Exception"></exception>
public List<string> ListFiles(string pattern)
{
var results = new List<string>();
var args = string.Join(' ', "ls-files", $"\"{pattern}\"");
progressMonitor.RunningProcess($"{git} {args}");
var pi = new ProcessStartInfo(git, args)
{
UseShellExecute = false,
RedirectStandardOutput = true
};
using var p = new Process() { StartInfo = pi };
p.OutputDataReceived += new DataReceivedEventHandler((sender, e) =>
{
if (!string.IsNullOrWhiteSpace(e.Data))
{
results.Add(e.Data);
}
});
p.Start();
p.BeginOutputReadLine();
p.WaitForExit();
if (p.ExitCode != 0)
{
progressMonitor.CommandFailed(git, args, p.ExitCode);
throw new Exception($"{git} {args} failed");
}
return results;
}
}
}

3
csharp/extractor/Semmle.Extraction.CSharp.Standalone/Options.cs
View File

@@ -50,6 +50,9 @@ namespace Semmle.Extraction.CSharp.Standalone
case "references":
dependencies.DllDirs.Add(value);
return true;
case "dotnet":
dependencies.DotNetPath = value;
return true;
default:
return base.HandleOption(key, value);
}