hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-16 16:53:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add managed thread ID to extractor log messages

Browse Source
This commit is contained in:
Tamas Vajk
2023-09-19 11:34:43 +02:00
parent dfd7f1e78b
commit edc93dfeb7
14 changed files with 85 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
csharp/autobuilder/Semmle.Autobuild.Shared/Autobuilder.cs
View File

@@ -267,7 +267,7 @@ namespace Semmle.Autobuild.Shared
protected DiagnosticClassifier DiagnosticClassifier { get; }
private readonly ILogger logger = new ConsoleLogger(Verbosity.Info);
private readonly ILogger logger = new ConsoleLogger(Verbosity.Info, logThreadId: false);
private readonly IDiagnosticsWriter diagnostics;

2
csharp/autobuilder/Semmle.Autobuild.Shared/BuildActions.cs
View File

@@ -237,7 +237,7 @@ namespace Semmle.Autobuild.Shared
int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory, IDictionary<string, string>? environment, out IList<string> stdOut)
{
var pi = GetProcessStartInfo(cmd, args, workingDirectory, environment, true);
return pi.ReadOutput(out stdOut);
return pi.ReadOutput(out stdOut, printToConsole: false);
}
void IBuildActions.DirectoryDelete(string dir, bool recursive) => Directory.Delete(dir, recursive);

2
csharp/extractor/Semmle.Extraction.CIL.Driver/Program.cs
View File

@@ -38,7 +38,7 @@ namespace Semmle.Extraction.CIL.Driver
}
var options = new ExtractorOptions(args);
using var logger = new ConsoleLogger(options.Verbosity);
using var logger = new ConsoleLogger(options.Verbosity, logThreadId: false);
var actions = options.AssembliesToExtract
.Select(asm => asm.Filename)

76
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
View File

@@ -219,7 +219,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
}
public DependencyManager(string srcDir) : this(srcDir, DependencyOptions.Default, new ConsoleLogger(Verbosity.Info)) { }
public DependencyManager(string srcDir) : this(srcDir, DependencyOptions.Default, new ConsoleLogger(Verbosity.Info, logThreadId: true)) { }
private IEnumerable<FileInfo> GetAllFiles()
{
@@ -430,8 +430,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
private bool RestoreProject(string project, out string stdout, string? pathToNugetConfig = null) =>
dotnet.RestoreProjectToDirectory(project, packageDirectory.DirInfo.FullName, out stdout, pathToNugetConfig);
private bool RestoreProject(string project, string? pathToNugetConfig = null) =>
dotnet.RestoreProjectToDirectory(project, packageDirectory.DirInfo.FullName, pathToNugetConfig);
private bool RestoreSolution(string solution, out IEnumerable<string> projects) =>
dotnet.RestoreSolutionToDirectory(solution, packageDirectory.DirInfo.FullName, out projects);
@@ -454,25 +454,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// <summary>
/// Executes `dotnet restore` on all projects in projects.
/// This is done in parallel for performance reasons.
/// To ensure that output is not interleaved, the output of each
/// restore is collected and printed.
/// </summary>
/// <param name="projects">A list of paths to project files.</param>
private void RestoreProjects(IEnumerable<string> projects)
{
var stdoutLines = projects
.AsParallel()
.WithDegreeOfParallelism(options.Threads)
.Select(project =>
{
RestoreProject(project, out var stdout);
return stdout;
})
.ToList();
foreach (var line in stdoutLines)
Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, project =>
{
Console.WriteLine(line);
}
RestoreProject(project);
});
}
private void DownloadMissingPackages(List<FileInfo> allFiles)
@@ -500,38 +489,29 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
.Select(d => Path.GetFileName(d).ToLowerInvariant());
var notYetDownloadedPackages = fileContent.AllPackages.Except(alreadyDownloadedPackages);
var stdoutLines = notYetDownloadedPackages
.AsParallel()
.WithDegreeOfParallelism(options.Threads)
.Select(package =>
{
progressMonitor.NugetInstall(package);
using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package));
var success = dotnet.New(tempDir.DirInfo.FullName, out var stdout1);
if (!success)
{
return new[] { stdout1 };
}
success = dotnet.AddPackage(tempDir.DirInfo.FullName, package, out var stdout2);
if (!success)
{
return new[] { stdout1, stdout2 };
}
success = RestoreProject(tempDir.DirInfo.FullName, out var stdout3, nugetConfig);
// TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package.
if (!success)
{
progressMonitor.FailedToRestoreNugetPackage(package);
}
return new[] { stdout1, stdout2, stdout3 };
})
.ToList();
foreach (var line in stdoutLines.SelectMany(l => l))
Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, package =>
{
Console.WriteLine(line);
}
progressMonitor.NugetInstall(package);
using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package));
var success = dotnet.New(tempDir.DirInfo.FullName);
if (!success)
{
return;
}
success = dotnet.AddPackage(tempDir.DirInfo.FullName, package);
if (!success)
{
return;
}
success = RestoreProject(tempDir.DirInfo.FullName, nugetConfig);
// TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package.
if (!success)
{
progressMonitor.FailedToRestoreNugetPackage(package);
}
});
}
private void AnalyseSolutions(IEnumerable<string> solutions)

22
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs
View File

@@ -37,17 +37,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
}
private bool RunCommand(string args, out string stdout)
{
var success = dotnetCliInvoker.RunCommand(args, out var output);
stdout = string.Join("\n", output);
return success;
}
private static string GetRestoreArgs(string projectOrSolutionFile, string packageDirectory) =>
$"restore --no-dependencies \"{projectOrSolutionFile}\" --packages \"{packageDirectory}\" /p:DisableImplicitNuGetFallbackFolder=true";
public bool RestoreProjectToDirectory(string projectFile, string packageDirectory, out string stdout, string? pathToNugetConfig = null)
public bool RestoreProjectToDirectory(string projectFile, string packageDirectory, string? pathToNugetConfig = null)
{
var args = GetRestoreArgs(projectFile, packageDirectory);
if (pathToNugetConfig != null)
@@ -55,7 +48,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
args += $" --configfile \"{pathToNugetConfig}\"";
}
return RunCommand(args, out stdout);
return dotnetCliInvoker.RunCommand(args);
}
public bool RestoreSolutionToDirectory(string solutionFile, string packageDirectory, out IEnumerable<string> projects)
@@ -76,16 +69,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
return false;
}
public bool New(string folder, out string stdout)
public bool New(string folder)
{
var args = $"new console --no-restore --output \"{folder}\"";
return RunCommand(args, out stdout);
return dotnetCliInvoker.RunCommand(args);
}
public bool AddPackage(string folder, string package, out string stdout)
public bool AddPackage(string folder, string package)
{
var args = $"add \"{folder}\" package \"{package}\" --no-restore";
return RunCommand(args, out stdout);
return dotnetCliInvoker.RunCommand(args);
}
public IList<string> GetListedRuntimes() => GetListed("--list-runtimes", "runtime");
@@ -94,9 +87,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private IList<string> GetListed(string args, string artifact)
{
if (dotnetCliInvoker.RunCommand(args, out IList<string> artifacts))
if (dotnetCliInvoker.RunCommand(args, out var artifacts))
{
progressMonitor.LogInfo($"Found {artifact}s: {string.Join("\n", artifacts)}");
return artifacts;
}
return new List<string>();

14
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs
View File

@@ -19,23 +19,23 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
this.Exec = exec;
}
private ProcessStartInfo MakeDotnetStartInfo(string args, bool redirectStandardOutput)
private ProcessStartInfo MakeDotnetStartInfo(string args)
{
var startInfo = new ProcessStartInfo(Exec, args)
{
UseShellExecute = false,
RedirectStandardOutput = redirectStandardOutput
RedirectStandardOutput = true
};
// Set the .NET CLI language to English to avoid localized output.
startInfo.EnvironmentVariables["DOTNET_CLI_UI_LANGUAGE"] = "en";
return startInfo;
}
private bool RunCommandAux(string args, bool redirectStandardOutput, out IList<string> output)
private bool RunCommandAux(string args, out IList<string> output)
{
progressMonitor.RunningProcess($"{Exec} {args}");
var pi = MakeDotnetStartInfo(args, redirectStandardOutput);
var exitCode = pi.ReadOutput(out output);
var pi = MakeDotnetStartInfo(args);
var exitCode = pi.ReadOutput(out output, true);
if (exitCode != 0)
{
progressMonitor.CommandFailed(Exec, args, exitCode);
@@ -45,9 +45,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
public bool RunCommand(string args) =>
RunCommandAux(args, redirectStandardOutput: false, out _);
RunCommandAux(args, out _);
public bool RunCommand(string args, out IList<string> output) =>
RunCommandAux(args, redirectStandardOutput: true, out output);
RunCommandAux(args, out output);
}
}

6
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs
View File

@@ -4,10 +4,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal interface IDotNet
{
bool RestoreProjectToDirectory(string project, string directory, out string stdout, string? pathToNugetConfig = null);
bool RestoreProjectToDirectory(string project, string directory, string? pathToNugetConfig = null);
bool RestoreSolutionToDirectory(string solutionFile, string packageDirectory, out IEnumerable<string> projects);
bool New(string folder, out string stdout);
bool AddPackage(string folder, string package, out string stdout);
bool New(string folder);
bool AddPackage(string folder, string package);
IList<string> GetListedRuntimes();
IList<string> GetListedSdks();
bool Exec(string execArgs);

2
csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs
View File

@@ -119,7 +119,7 @@ namespace Semmle.Extraction.CSharp.Standalone
var stopwatch = new Stopwatch();
stopwatch.Start();
using var logger = new ConsoleLogger(options.Verbosity);
using var logger = new ConsoleLogger(options.Verbosity, logThreadId: true);
logger.Log(Severity.Info, "Running C# standalone extractor");
using var a = new Analysis(logger, options);
var sourceFileCount = a.Extraction.Sources.Count;

5
csharp/extractor/Semmle.Extraction.CSharp.Standalone/Options.cs
View File

@@ -2,6 +2,7 @@ using System.IO;
using Semmle.Util;
using Semmle.Util.Logging;
using Semmle.Extraction.CSharp.DependencyFetching;
using System;
namespace Semmle.Extraction.CSharp.Standalone
{
@@ -64,7 +65,7 @@ namespace Semmle.Extraction.CSharp.Standalone
var fi = new FileInfo(dependencies.SolutionFile);
if (!fi.Exists)
{
System.Console.WriteLine("Error: The solution {0} does not exist", fi.FullName);
System.Console.WriteLine($"[{Environment.CurrentManagedThreadId:D3}] Error: The solution {fi.FullName} does not exist");
Errors = true;
}
return true;
@@ -72,7 +73,7 @@ namespace Semmle.Extraction.CSharp.Standalone
public override void InvalidArgument(string argument)
{
System.Console.WriteLine($"Error: Invalid argument {argument}");
System.Console.WriteLine($"[{Environment.CurrentManagedThreadId:D3}] Error: Invalid argument {argument}");
Errors = true;
}

4
csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs
View File

@@ -71,9 +71,9 @@ namespace Semmle.Extraction.CSharp
public static ILogger MakeLogger(Verbosity verbosity, bool includeConsole)
{
var fileLogger = new FileLogger(verbosity, GetCSharpLogPath());
var fileLogger = new FileLogger(verbosity, GetCSharpLogPath(), logThreadId: true);
return includeConsole
? new CombinedLogger(new ConsoleLogger(verbosity), fileLogger)
? new CombinedLogger(new ConsoleLogger(verbosity, logThreadId: true), fileLogger)
: (ILogger)fileLogger;
}

8
csharp/extractor/Semmle.Extraction.Tests/DotNet.cs
View File

@@ -99,7 +99,7 @@ namespace Semmle.Extraction.Tests
var dotnet = MakeDotnet(dotnetCliInvoker);
// Execute
dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages", out var _);
dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages");
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();
@@ -114,7 +114,7 @@ namespace Semmle.Extraction.Tests
var dotnet = MakeDotnet(dotnetCliInvoker);
// Execute
dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages", out var _, "myconfig.config");
dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages", "myconfig.config");
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();
@@ -164,7 +164,7 @@ namespace Semmle.Extraction.Tests
var dotnet = MakeDotnet(dotnetCliInvoker);
// Execute
dotnet.New("myfolder", out var _);
dotnet.New("myfolder");
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();
@@ -179,7 +179,7 @@ namespace Semmle.Extraction.Tests
var dotnet = MakeDotnet(dotnetCliInvoker);
// Execute
dotnet.AddPackage("myfolder", "mypackage", out var _);
dotnet.AddPackage("myfolder", "mypackage");
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();

18
csharp/extractor/Semmle.Extraction.Tests/Runtime.cs
View File

@@ -15,23 +15,11 @@ namespace Semmle.Extraction.Tests
this.runtimes = runtimes;
this.sdks = sdks;
}
public bool AddPackage(string folder, string package, out string stdout)
{
stdout = "";
return true;
}
public bool AddPackage(string folder, string package) => true;
public bool New(string folder, out string stdout)
{
stdout = "";
return true;
}
public bool New(string folder) => true;
public bool RestoreProjectToDirectory(string project, string directory, out string stdout, string? pathToNugetConfig = null)
{
stdout = "";
return true;
}
public bool RestoreProjectToDirectory(string project, string directory, string? pathToNugetConfig = null) => true;
public bool RestoreSolutionToDirectory(string solution, string directory, out IEnumerable<string> projects)
{

18
csharp/extractor/Semmle.Util/Logger.cs
View File

@@ -59,10 +59,12 @@ namespace Semmle.Util.Logging
{
private readonly StreamWriter writer;
private readonly Verbosity verbosity;
private readonly bool logThreadId;
public FileLogger(Verbosity verbosity, string outputFile)
public FileLogger(Verbosity verbosity, string outputFile, bool logThreadId)
{
this.verbosity = verbosity;
this.logThreadId = logThreadId;
try
{
@@ -93,7 +95,10 @@ namespace Semmle.Util.Logging
public void Log(Severity s, string text)
{
if (verbosity.Includes(s))
writer.WriteLine(GetSeverityPrefix(s) + text);
{
var threadId = this.logThreadId ? $"[{Environment.CurrentManagedThreadId:D3}] " : "";
writer.WriteLine(threadId + GetSeverityPrefix(s) + text);
}
}
}
@@ -103,10 +108,12 @@ namespace Semmle.Util.Logging
public sealed class ConsoleLogger : ILogger
{
private readonly Verbosity verbosity;
private readonly bool logThreadId;
public ConsoleLogger(Verbosity verbosity)
public ConsoleLogger(Verbosity verbosity, bool logThreadId)
{
this.verbosity = verbosity;
this.logThreadId = logThreadId;
}
public void Dispose() { }
@@ -136,7 +143,10 @@ namespace Semmle.Util.Logging
public void Log(Severity s, string text)
{
if (verbosity.Includes(s))
GetConsole(s).WriteLine(GetSeverityPrefix(s) + text);
{
var threadId = this.logThreadId ? $"[{Environment.CurrentManagedThreadId:D3}] " : "";
GetConsole(s).WriteLine(threadId + GetSeverityPrefix(s) + text);
}
}
}

13
csharp/extractor/Semmle.Util/ProcessStartInfoExtensions.cs
View File

@@ -1,4 +1,5 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.Diagnostics;
namespace Semmle.Util
@@ -7,9 +8,11 @@ namespace Semmle.Util
{
/// <summary>
/// Runs this process, and returns the exit code, as well as the contents
/// of stdout in <paramref name="stdout"/>.
/// of stdout in <paramref name="stdout"/>. If <paramref name="printToConsole"/>
/// is true, then stdout is printed to the console and each line is prefixed
/// with the thread id.
/// </summary>
public static int ReadOutput(this ProcessStartInfo pi, out IList<string> stdout)
public static int ReadOutput(this ProcessStartInfo pi, out IList<string> stdout, bool printToConsole)
{
stdout = new List<string>();
using var process = Process.Start(pi);
@@ -27,6 +30,10 @@ namespace Semmle.Util
s = process.StandardOutput.ReadLine();
if (s is not null)
{
if (printToConsole)
{
Console.WriteLine($"[{Environment.CurrentManagedThreadId:D3}] {s}");
}
stdout.Add(s);
}
}