From b8c8f52efc8fd6226c1fcc7fcea0ee745ff8fe6d Mon Sep 17 00:00:00 2001 From: Tamas Vajk Date: Thu, 25 Jan 2024 17:20:47 +0100 Subject: [PATCH] C#: Introduce extractor option for logging verbosity --- .../Semmle.Autobuild.Shared/Autobuilder.cs | 6 +++- csharp/codeql-extractor.yml | 18 +++++++++++ .../Semmle.Extraction.CIL.Driver/Program.cs | 2 +- .../Extractor.cs | 4 +-- .../Extractor/Extractor.cs | 2 +- .../Semmle.Extraction.Tests/Options.cs | 31 +++++++++++++++++++ csharp/extractor/Semmle.Extraction/Options.cs | 30 ++++++++++++++++++ .../Logging/VerbosityExtensions.cs | 31 ++++++++++++++++++- 8 files changed, 118 insertions(+), 6 deletions(-) diff --git a/csharp/autobuilder/Semmle.Autobuild.Shared/Autobuilder.cs b/csharp/autobuilder/Semmle.Autobuild.Shared/Autobuilder.cs index c6cf3e32b7e..d6757418875 100644 --- a/csharp/autobuilder/Semmle.Autobuild.Shared/Autobuilder.cs +++ b/csharp/autobuilder/Semmle.Autobuild.Shared/Autobuilder.cs @@ -267,7 +267,11 @@ namespace Semmle.Autobuild.Shared protected DiagnosticClassifier DiagnosticClassifier { get; } - private readonly ILogger logger = new ConsoleLogger(Verbosity.Info, logThreadId: false); + private readonly ILogger logger = new ConsoleLogger( + VerbosityExtensions.ParseVerbosity( + Environment.GetEnvironmentVariable("CODEQL_VERBOSITY"), + logThreadId: false) ?? Verbosity.Info, + logThreadId: false); private readonly IDiagnosticsWriter diagnostics; diff --git a/csharp/codeql-extractor.yml b/csharp/codeql-extractor.yml index 0a3e22f1ff9..1990b2080e7 100644 --- a/csharp/codeql-extractor.yml +++ b/csharp/codeql-extractor.yml @@ -48,3 +48,21 @@ options: The default is 'true'. type: string pattern: "^(false|true)$" + logging: + title: Options pertaining to logging. + type: object + properties: + verbosity: + title: Extractor logging verbosity level. + description: > + Controls the level of verbosity of the extractor. + The supported levels are (in order of increasing verbosity): + - off + - errors + - warnings + - info or progress + - debug or progress+ + - trace or progress++ + - progress+++ + type: string + pattern: "^(off|errors|warnings|(info|progress)|(debug|progress\\+)|(trace|progress\\+\\+)|progress\\+\\+\\+)$" diff --git a/csharp/extractor/Semmle.Extraction.CIL.Driver/Program.cs b/csharp/extractor/Semmle.Extraction.CIL.Driver/Program.cs index b5420d8bb67..8fcc6fefcb3 100644 --- a/csharp/extractor/Semmle.Extraction.CIL.Driver/Program.cs +++ b/csharp/extractor/Semmle.Extraction.CIL.Driver/Program.cs @@ -38,7 +38,7 @@ namespace Semmle.Extraction.CIL.Driver } var options = new ExtractorOptions(args); - using ILogger logger = new ConsoleLogger(options.LegacyVerbosity, logThreadId: false); + using ILogger logger = new ConsoleLogger(options.Verbosity, logThreadId: false); var actions = options.AssembliesToExtract .Select(asm => asm.Filename) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs index f9874344b37..e9dbf3525ee 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs @@ -136,7 +136,7 @@ namespace Semmle.Extraction.CSharp.Standalone var stopwatch = new Stopwatch(); stopwatch.Start(); - using var logger = new ConsoleLogger(options.LegacyVerbosity, logThreadId: true); + using var logger = new ConsoleLogger(options.Verbosity, logThreadId: true); logger.Log(Severity.Info, "Running C# standalone extractor"); using var a = new Analysis(logger, options); var sourceFileCount = a.Extraction.Sources.Count; @@ -147,7 +147,7 @@ namespace Semmle.Extraction.CSharp.Standalone return ExitCode.Errors; } - using var fileLogger = CSharp.Extractor.MakeLogger(options.LegacyVerbosity, false); + using var fileLogger = CSharp.Extractor.MakeLogger(options.Verbosity, false); logger.Log(Severity.Info, ""); logger.Log(Severity.Info, "Extracting..."); diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs index 0b051fdc849..5d5bc5860f4 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs @@ -97,7 +97,7 @@ namespace Semmle.Extraction.CSharp var options = Options.CreateWithEnvironment(args); Entities.Compilation.Settings = (Directory.GetCurrentDirectory(), options.CompilerArguments.ToArray()); - using var logger = MakeLogger(options.LegacyVerbosity, options.Console); + using var logger = MakeLogger(options.Verbosity, options.Console); var canonicalPathCache = CanonicalPathCache.Create(logger, 1000); var pathTransformer = new PathTransformer(canonicalPathCache); diff --git a/csharp/extractor/Semmle.Extraction.Tests/Options.cs b/csharp/extractor/Semmle.Extraction.Tests/Options.cs index 08d54431e79..81b750368ca 100644 --- a/csharp/extractor/Semmle.Extraction.Tests/Options.cs +++ b/csharp/extractor/Semmle.Extraction.Tests/Options.cs @@ -103,6 +103,37 @@ namespace Semmle.Extraction.Tests Assert.Throws(() => CSharp.Options.CreateWithEnvironment(new string[] { "--verbosity", "X" })); } + + private const string extractorVariableName = "CODEQL_EXTRACTOR_CSHARP_OPTION_LOGGING_VERBOSITY"; + private const string cliVariableName = "CODEQL_VERBOSITY"; + + private void CheckVerbosity(string? extractor, string? cli, Verbosity expected) + { + var currentExtractorVerbosity = Environment.GetEnvironmentVariable(extractorVariableName); + var currentCliVerbosity = Environment.GetEnvironmentVariable(cliVariableName); + try + { + Environment.SetEnvironmentVariable(extractorVariableName, extractor); + Environment.SetEnvironmentVariable(cliVariableName, cli); + + options = CSharp.Options.CreateWithEnvironment(new string[] { "--verbose" }); + Assert.Equal(expected, options.Verbosity); + } + finally + { + Environment.SetEnvironmentVariable(extractorVariableName, currentExtractorVerbosity); + Environment.SetEnvironmentVariable(cliVariableName, currentCliVerbosity); + } + } + + [Fact] + public void VerbosityTests_WithExtractorOption() + { + CheckVerbosity("progress+++", "progress++", Verbosity.All); + CheckVerbosity(null, "progress++", Verbosity.Trace); + CheckVerbosity(null, null, Verbosity.Debug); + } + [Fact] public void Console() { diff --git a/csharp/extractor/Semmle.Extraction/Options.cs b/csharp/extractor/Semmle.Extraction/Options.cs index dae79e26f10..576d7a4762e 100644 --- a/csharp/extractor/Semmle.Extraction/Options.cs +++ b/csharp/extractor/Semmle.Extraction/Options.cs @@ -20,6 +20,36 @@ namespace Semmle.Extraction /// public Verbosity LegacyVerbosity { get; protected set; } = Verbosity.Info; + private Verbosity? verbosity = null; + public Verbosity Verbosity + { + get + { + if (verbosity != null) + { + return verbosity.Value; + } + + var envVarValue = EnvironmentVariables.GetExtractorOption("LOGGING_VERBOSITY"); + verbosity = VerbosityExtensions.ParseVerbosity(envVarValue, logThreadId: true); + if (verbosity != null) + { + return verbosity.Value; + } + + envVarValue = Environment.GetEnvironmentVariable("CODEQL_VERBOSITY"); + verbosity = VerbosityExtensions.ParseVerbosity(envVarValue, logThreadId: true); + if (verbosity != null) + { + return verbosity.Value; + } + + // This only works, because we already parsed the provided options, so `LegacyVerbosity` is already set (or it still has the default value). + verbosity = LegacyVerbosity; + return verbosity.Value; + } + } + /// /// Whether to output to the console. /// diff --git a/csharp/extractor/Semmle.Util/Logging/VerbosityExtensions.cs b/csharp/extractor/Semmle.Util/Logging/VerbosityExtensions.cs index 66a76b8a030..96cc9abe9d8 100644 --- a/csharp/extractor/Semmle.Util/Logging/VerbosityExtensions.cs +++ b/csharp/extractor/Semmle.Util/Logging/VerbosityExtensions.cs @@ -2,7 +2,7 @@ using System; namespace Semmle.Util.Logging { - internal static class VerbosityExtensions + public static class VerbosityExtensions { /// /// Whether a message with the given severity must be included @@ -26,5 +26,34 @@ namespace Semmle.Util.Logging throw new ArgumentOutOfRangeException(nameof(s)); } } + + public static Verbosity? ParseVerbosity(string? str, bool logThreadId) + { + if (str == null) + { + return null; + } + + Verbosity? verbosity = str.ToLowerInvariant() switch + { + "off" => Verbosity.Off, + "errors" => Verbosity.Error, + "warnings" => Verbosity.Warning, + "info" or "progress" => Verbosity.Info, + "debug" or "progress+" => Verbosity.Debug, + "trace" or "progress++" => Verbosity.Trace, + "progress+++" => Verbosity.All, + _ => null + }; + + if (verbosity == null && str != null) + { + // We don't have a logger when this setting is parsed, so writing it to the console: + var prefix = logThreadId ? $"[{Environment.CurrentManagedThreadId:D3}] " : ""; + Console.WriteLine($"{prefix}Error: Invalid verbosity level: '{str}'"); + } + + return verbosity; + } } }