From 15c1fd942558d2a5d5023dc5f17dfdd8dedd9757 Mon Sep 17 00:00:00 2001 From: Tamas Vajk Date: Thu, 25 Apr 2024 10:39:53 +0200 Subject: [PATCH] C#: Improve log messages --- .../DependencyManager.cs | 1 + .../DotNet.cs | 4 +-- .../DotNetCliInvoker.cs | 5 ++-- .../FileProvider.cs | 29 ++++++++++++------- .../NugetExeWrapper.cs | 24 ++++++--------- .../NugetPackageRestorer.cs | 4 +-- .../Extractor.cs | 9 ++++-- 7 files changed, 42 insertions(+), 34 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs index d0e68defac1..d5450affd93 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs @@ -191,6 +191,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching private HashSet AddFrameworkDlls(HashSet dllLocations) { + logger.LogInfo("Adding .NET Framework DLLs"); var frameworkLocations = new HashSet(); var frameworkReferences = Environment.GetEnvironmentVariable(EnvironmentVariableNames.DotnetFrameworkReferences); diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs index 95e8fe0675f..f1f1862049a 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs @@ -91,9 +91,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching return dotnetCliInvoker.RunCommand(args); } - public IList GetListedRuntimes() => GetResultList("--list-runtimes", null, false); + public IList GetListedRuntimes() => GetResultList("--list-runtimes", null, true); - public IList GetListedSdks() => GetResultList("--list-sdks", null, false); + public IList GetListedSdks() => GetResultList("--list-sdks", null, true); private IList GetResultList(string args, string? workingDirectory = null, bool silent = true) { diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs index 1802521e28d..12678021227 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs @@ -19,6 +19,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { this.logger = logger; this.Exec = exec; + logger.LogInfo($"Using .NET CLI executable: '{Exec}'"); } private ProcessStartInfo MakeDotnetStartInfo(string args, string? workingDirectory) @@ -43,7 +44,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching private bool RunCommandAux(string args, string? workingDirectory, out IList output, bool silent) { var dirLog = string.IsNullOrWhiteSpace(workingDirectory) ? "" : $" in {workingDirectory}"; - logger.LogInfo($"Running {Exec} {args}{dirLog}"); + logger.LogInfo($"Running '{Exec} {args}'{dirLog}"); var pi = MakeDotnetStartInfo(args, workingDirectory); var threadId = Environment.CurrentManagedThreadId; void onOut(string s) => logger.Log(silent ? Severity.Debug : Severity.Info, s, threadId); @@ -51,7 +52,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching var exitCode = pi.ReadOutput(out output, onOut, onError); if (exitCode != 0) { - logger.LogError($"Command {Exec} {args}{dirLog} failed with exit code {exitCode}"); + logger.LogError($"Command '{Exec} {args}'{dirLog} failed with exit code {exitCode}"); return false; } return true; diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileProvider.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileProvider.cs index f5ba20cd888..7b88a1fc1a2 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileProvider.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileProvider.cs @@ -21,6 +21,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching private readonly Lazy dlls; private readonly Lazy nugetConfigs; private readonly Lazy globalJsons; + private readonly Lazy packagesConfigs; private readonly Lazy razorViews; private readonly Lazy resources; private readonly Lazy rootNugetConfig; @@ -32,31 +33,38 @@ namespace Semmle.Extraction.CSharp.DependencyFetching all = GetAllFiles(); allNonBinary = new Lazy(() => all.Where(f => !binaryFileExtensions.Contains(f.Extension.ToLowerInvariant())).ToArray()); - smallNonBinary = new Lazy(() => - { - var ret = SelectSmallFiles(allNonBinary.Value).SelectFileNames().ToArray(); - logger.LogInfo($"Found {ret.Length} small non-binary files in {SourceDir}."); - return ret; - }); + smallNonBinary = new Lazy(() => ReturnAndLogFiles("small non-binary", SelectSmallFiles(allNonBinary.Value).SelectFileNames().ToArray())); sources = new Lazy(() => SelectTextFileNamesByExtension("source", ".cs")); projects = new Lazy(() => SelectTextFileNamesByExtension("project", ".csproj")); solutions = new Lazy(() => SelectTextFileNamesByExtension("solution", ".sln")); dlls = new Lazy(() => SelectBinaryFileNamesByExtension("DLL", ".dll")); - nugetConfigs = new Lazy(() => allNonBinary.Value.SelectFileNamesByName("nuget.config").ToArray()); - globalJsons = new Lazy(() => allNonBinary.Value.SelectFileNamesByName("global.json").ToArray()); + nugetConfigs = new Lazy(() => SelectTextFileNamesByName("nuget.config")); + globalJsons = new Lazy(() => SelectTextFileNamesByName("global.json")); + packagesConfigs = new Lazy(() => SelectTextFileNamesByName("packages.config")); razorViews = new Lazy(() => SelectTextFileNamesByExtension("razor view", ".cshtml", ".razor")); resources = new Lazy(() => SelectTextFileNamesByExtension("resource", ".resx")); rootNugetConfig = new Lazy(() => all.SelectRootFiles(SourceDir).SelectFileNamesByName("nuget.config").FirstOrDefault()); } - private string[] SelectTextFileNamesByExtension(string filetype, params string[] extensions) + private string[] ReturnAndLogFiles(string filetype, IEnumerable files) { - var ret = allNonBinary.Value.SelectFileNamesByExtension(extensions).ToArray(); + var ret = files.ToArray(); logger.LogInfo($"Found {ret.Length} {filetype} files in {SourceDir}."); return ret; } + private string[] SelectTextFileNamesByExtension(string filetype, params string[] extensions) + => ReturnAndLogFiles(filetype, allNonBinary.Value.SelectFileNamesByExtension(extensions)); + + private string[] SelectTextFileNamesByName(string name) + { + var ret = allNonBinary.Value.SelectFileNamesByName(name).ToArray(); + var ending = ret.Length == 0 ? "." : $": {string.Join(", ", ret.OrderBy(s => s))}."; + logger.LogInfo($"Found {ret.Length} {name} files in {SourceDir}{ending}"); + return ret; + } + private string[] SelectBinaryFileNamesByExtension(string filetype, params string[] extensions) { var ret = all.SelectFileNamesByExtension(extensions).ToArray(); @@ -117,6 +125,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching public ICollection NugetConfigs => nugetConfigs.Value; public string? RootNugetConfig => rootNugetConfig.Value; public IEnumerable GlobalJsons => globalJsons.Value; + public ICollection PackagesConfigs => packagesConfigs.Value; public ICollection RazorViews => razorViews.Value; public ICollection Resources => resources.Value; } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs index 01b4eb6b5f4..8537e4b5e0e 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs @@ -20,9 +20,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// /// The list of package files. /// - private readonly FileInfo[] packageFiles; + private readonly ICollection packageFiles; - public int PackageCount => packageFiles.Length; + public int PackageCount => packageFiles.Count; private readonly string? backupNugetConfig; private readonly string? nugetConfigPath; @@ -37,23 +37,21 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// /// Create the package manager for a specified source tree. /// - public NugetExeWrapper(string sourceDir, TemporaryDirectory packageDirectory, Util.Logging.ILogger logger) + public NugetExeWrapper(FileProvider fileProvider, TemporaryDirectory packageDirectory, Util.Logging.ILogger logger) { this.packageDirectory = packageDirectory; this.logger = logger; - packageFiles = new DirectoryInfo(sourceDir) - .EnumerateFiles("packages.config", SearchOption.AllDirectories) - .ToArray(); + packageFiles = fileProvider.PackagesConfigs; - if (packageFiles.Length > 0) + if (packageFiles.Count > 0) { - logger.LogInfo($"Found {packageFiles.Length} packages.config files, trying to use nuget.exe for package restore"); - nugetExe = ResolveNugetExe(sourceDir); + logger.LogInfo($"Found packages.config files, trying to use nuget.exe for package restore"); + nugetExe = ResolveNugetExe(fileProvider.SourceDir.FullName); if (HasNoPackageSource()) { // We only modify or add a top level nuget.config file - nugetConfigPath = Path.Combine(sourceDir, "nuget.config"); + nugetConfigPath = Path.Combine(fileProvider.SourceDir.FullName, "nuget.config"); try { if (File.Exists(nugetConfigPath)) @@ -86,10 +84,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } } } - else - { - logger.LogInfo("Found no packages.config file"); - } } /// @@ -195,7 +189,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// public int InstallPackages() { - return packageFiles.Count(package => TryRestoreNugetPackage(package.FullName)); + return packageFiles.Count(package => TryRestoreNugetPackage(package)); } private bool HasNoPackageSource() diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs index 735e4a676c6..baf23a0d0cc 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs @@ -105,7 +105,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching : [unresponsiveMissingPackageLocation]; } - using (var nuget = new NugetExeWrapper(fileProvider.SourceDir.FullName, legacyPackageDirectory, logger)) + using (var nuget = new NugetExeWrapper(fileProvider, legacyPackageDirectory, logger)) { var count = nuget.InstallPackages(); @@ -178,7 +178,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching logger.LogInfo($"No fallback Nuget feeds specified. Using default feed: {PublicNugetOrgFeed}"); } - logger.LogInfo($"Checking fallback Nuget feed reachability on feeds: {string.Join(", ", fallbackFeeds.OrderBy(f => f))}"); + logger.LogInfo($"Checking fallback Nuget feed reachability on feeds: {string.Join(", ", fallbackFeeds.OrderBy(f => f))}"); var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback: true); var reachableFallbackFeeds = fallbackFeeds.Where(feed => IsFeedReachable(feed, initialTimeout, tryCount, allowExceptions: false)).ToList(); if (reachableFallbackFeeds.Count == 0) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs index fac8523989e..63d4ff0e83a 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs @@ -121,17 +121,20 @@ namespace Semmle.Extraction.CSharp.Standalone public void MissingType(string type) { - logger.Log(Severity.Debug, "Missing type {0}", type); + logger.LogDebug($"Missing type {type}"); } public void MissingNamespace(string @namespace) { - logger.Log(Severity.Info, "Missing namespace {0}", @namespace); + logger.LogInfo($"Missing namespace {@namespace}"); } public void MissingSummary(int missingTypes, int missingNamespaces) { - logger.Log(Severity.Info, "Failed to resolve {0} types in {1} namespaces", missingTypes, missingNamespaces); + if (missingTypes > 0 || missingNamespaces > 0) + { + logger.LogInfo($"Failed to resolve {missingTypes} types in {missingNamespaces} namespaces"); + } } }