From 13a8168c8e62b93ef284488ccc770c04d1cb6b0f Mon Sep 17 00:00:00 2001 From: Tamas Vajk Date: Wed, 24 Jan 2024 11:16:07 +0100 Subject: [PATCH 1/4] C#: Improve log messages in standalone extractor --- .../AssemblyCache.cs | 37 ++- .../AssemblyInfo.cs | 15 -- .../DependencyManager.cs | 244 +++++++++--------- .../DotNetCliInvoker.cs | 16 +- .../FileContent.cs | 3 +- .../NugetPackages.cs | 42 +-- .../ProgressMonitor.cs | 111 +------- .../Razor.cs | 14 +- .../Semmle.Extraction.Tests/LoggerStub.cs | 2 +- .../Semmle.Util.Tests/CanonicalPathCache.cs | 2 +- .../extractor/Semmle.Util.Tests/FileUtils.cs | 2 +- csharp/extractor/Semmle.Util/Logger.cs | 24 +- 12 files changed, 194 insertions(+), 318 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs index 3a124d13e0e..270dee28d4c 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs @@ -21,17 +21,18 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// Callback for progress. public AssemblyCache(IEnumerable paths, IEnumerable frameworkPaths, ProgressMonitor progressMonitor) { + this.progressMonitor = progressMonitor; foreach (var path in paths) { if (File.Exists(path)) { - pendingDllsToIndex.Enqueue(path); + dllsToIndex.Add(path); continue; } if (Directory.Exists(path)) { - progressMonitor.FindingFiles(path); + progressMonitor.LogInfo($"Finding reference DLLs in {path}..."); AddReferenceDirectory(path); } else @@ -52,7 +53,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { foreach (var dll in new DirectoryInfo(dir).EnumerateFiles("*.dll", SearchOption.AllDirectories)) { - pendingDllsToIndex.Enqueue(dll.FullName); + dllsToIndex.Add(dll.FullName); } } @@ -62,12 +63,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// private void IndexReferences(IEnumerable frameworkPaths) { + progressMonitor.LogInfo($"Indexing {dllsToIndex.Count} assemblies..."); + // Read all of the files - foreach (var filename in pendingDllsToIndex) + foreach (var filename in dllsToIndex) { IndexReference(filename); } + progressMonitor.LogInfo($"Read {assemblyInfoByFileName.Count} assembly infos"); + foreach (var info in assemblyInfoByFileName.Values .OrderBy(info => info.Name) .OrderAssemblyInfosByPreference(frameworkPaths)) @@ -83,25 +88,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { try { + progressMonitor.LogDebug($"Reading assembly info from {filename}"); var info = AssemblyInfo.ReadFromFile(filename); assemblyInfoByFileName[filename] = info; } catch (AssemblyLoadException) { - failedAssemblyInfoFileNames.Add(filename); + progressMonitor.LogInfo($"Couldn't read assembly info from {filename}"); } } - /// - /// The number of DLLs which are assemblies. - /// - public int AssemblyCount => assemblyInfoByFileName.Count; - - /// - /// The number of DLLs which weren't assemblies. (E.g. C++). - /// - public int NonAssemblyCount => failedAssemblyInfoFileNames.Count; - /// /// Given an assembly id, determine its full info. /// @@ -113,8 +109,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching if (failedAssemblyInfoIds.Contains(id)) throw new AssemblyLoadException(); - string assemblyName; - (id, assemblyName) = AssemblyInfo.ComputeSanitizedAssemblyInfo(id); + (id, var assemblyName) = AssemblyInfo.ComputeSanitizedAssemblyInfo(id); // Look up the id in our references map. if (assemblyInfoById.TryGetValue(id, out var result)) @@ -164,17 +159,15 @@ namespace Semmle.Extraction.CSharp.DependencyFetching throw new AssemblyLoadException(); } - private readonly Queue pendingDllsToIndex = new Queue(); + private readonly List dllsToIndex = new List(); private readonly Dictionary assemblyInfoByFileName = new Dictionary(); - // List of DLLs which are not assemblies. - // We probably don't need to keep this - private readonly List failedAssemblyInfoFileNames = new List(); - // Map from assembly id (in various formats) to the full info. private readonly Dictionary assemblyInfoById = new Dictionary(); private readonly HashSet failedAssemblyInfoIds = new HashSet(); + + private readonly ProgressMonitor progressMonitor; } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyInfo.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyInfo.cs index d3522025e27..49328f6a78f 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyInfo.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyInfo.cs @@ -120,21 +120,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching NetCoreVersion = netCoreVersion; } - /// - /// Get AssemblyInfo from a loaded Assembly. - /// - /// The assembly. - /// Info about the assembly. - public static AssemblyInfo MakeFromAssembly(Assembly assembly) - { - if (assembly.FullName is null) - { - throw new InvalidOperationException("Assembly with empty full name is not expected."); - } - - return new AssemblyInfo(assembly.FullName, assembly.Location); - } - /// /// Returns the id and name of the assembly that would be created from the received id. /// diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs index 710de21c5d0..c4a1f3a0a71 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs @@ -18,11 +18,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { private readonly AssemblyCache assemblyCache; private readonly ProgressMonitor progressMonitor; + + // Only used as a set, but ConcurrentDictionary is the only concurrent set in .NET. private readonly IDictionary usedReferences = new ConcurrentDictionary(); - private readonly IDictionary sources = new ConcurrentDictionary(); private readonly IDictionary unresolvedReferences = new ConcurrentDictionary(); - private int failedProjects; - private int succeededProjects; private readonly List nonGeneratedSources; private readonly List generatedSources; private int conflictedReferences = 0; @@ -65,12 +64,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } catch { - progressMonitor.MissingDotNet(); + progressMonitor.LogError("Missing dotnet CLI"); throw; } - this.progressMonitor.FindingFiles(srcDir); - + progressMonitor.LogInfo($"Finding files in {srcDir}..."); var allFiles = GetAllFiles().ToList(); var binaryFileExtensions = new HashSet(new[] { ".dll", ".exe" }); // TODO: add more binary file extensions. @@ -79,10 +77,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching this.fileContent = new FileContent(progressMonitor, smallNonBinaryFiles); this.nonGeneratedSources = allNonBinaryFiles.SelectFileNamesByExtension(".cs").ToList(); this.generatedSources = new(); - var allProjects = allNonBinaryFiles.SelectFileNamesByExtension(".csproj"); - var allSolutions = allNonBinaryFiles.SelectFileNamesByExtension(".sln"); + var allProjects = allNonBinaryFiles.SelectFileNamesByExtension(".csproj").ToList(); + var allSolutions = allNonBinaryFiles.SelectFileNamesByExtension(".sln").ToList(); var dllPaths = allFiles.SelectFileNamesByExtension(".dll").ToHashSet(); + progressMonitor.LogInfo($"Found {allFiles.Count} files, {nonGeneratedSources.Count} source files, {allProjects.Count} project files, {allSolutions.Count} solution files, {dllPaths.Count} DLLs."); + RestoreNugetPackages(allNonBinaryFiles, allProjects, allSolutions, dllPaths); // Find DLLs in the .Net / Asp.Net Framework // This needs to come after the nuget restore, because the nuget restore might fetch the .NET Core/Framework reference assemblies. @@ -102,33 +102,35 @@ namespace Semmle.Extraction.CSharp.DependencyFetching // Output the findings foreach (var r in usedReferences.Keys.OrderBy(r => r)) { - progressMonitor.ResolvedReference(r); + progressMonitor.LogInfo($"Resolved reference {r}"); } foreach (var r in unresolvedReferences.OrderBy(r => r.Key)) { - progressMonitor.UnresolvedReference(r.Key, r.Value); + progressMonitor.LogInfo($"Unresolved reference {r.Key} in project {r.Value}"); } var webViewExtractionOption = Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_STANDALONE_EXTRACT_WEB_VIEWS"); if (bool.TryParse(webViewExtractionOption, out var shouldExtractWebViews) && shouldExtractWebViews) { + progressMonitor.LogInfo("Generating source files from cshtml and razor files..."); GenerateSourceFilesFromWebViews(allNonBinaryFiles); } GenerateSourceFileFromImplicitUsings(); - progressMonitor.Summary( - AllSourceFiles.Count(), - ProjectSourceFiles.Count(), - MissingSourceFiles.Count(), - ReferenceFiles.Count(), - UnresolvedReferences.Count(), - conflictedReferences, - succeededProjects + failedProjects, - failedProjects, - DateTime.Now - startTime); + const int align = 6; + progressMonitor.LogInfo(""); + progressMonitor.LogInfo("Build analysis summary:"); + progressMonitor.LogInfo($"{this.nonGeneratedSources.Count,align} source files in the filesystem"); + progressMonitor.LogInfo($"{this.generatedSources.Count,align} generated source files"); + progressMonitor.LogInfo($"{allSolutions.Count,align} solution files"); + progressMonitor.LogInfo($"{allProjects.Count,align} project files in the filesystem"); + progressMonitor.LogInfo($"{usedReferences.Keys.Count,align} resolved references"); + progressMonitor.LogInfo($"{unresolvedReferences.Count,align} unresolved references"); + progressMonitor.LogInfo($"{conflictedReferences,align} resolved assembly conflicts"); + progressMonitor.LogInfo($"Build analysis completed in {DateTime.Now - startTime}"); } private HashSet AddFrameworkDlls(HashSet dllPaths) @@ -152,7 +154,17 @@ namespace Semmle.Extraction.CSharp.DependencyFetching var nugetPackageDlls = legacyPackageDirectory.DirInfo.GetFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = true }); var nugetPackageDllPaths = nugetPackageDlls.Select(f => f.FullName).ToHashSet(); var excludedPaths = nugetPackageDllPaths - .Where(path => IsPathInSubfolder(path, legacyPackageDirectory.DirInfo.FullName, "tools")); + .Where(path => IsPathInSubfolder(path, legacyPackageDirectory.DirInfo.FullName, "tools")) + .ToList(); + + if (nugetPackageDllPaths.Count > 0) + { + progressMonitor.LogInfo($"Restored {nugetPackageDllPaths.Count} Nuget DLLs."); + } + if (excludedPaths.Count > 0) + { + progressMonitor.LogInfo($"Excluding {excludedPaths.Count} Nuget DLLs."); + } foreach (var excludedPath in excludedPaths) { @@ -162,9 +174,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching nugetPackageDllPaths.ExceptWith(excludedPaths); dllPaths.UnionWith(nugetPackageDllPaths); } - catch (FileNotFoundException) + catch (Exception) { - progressMonitor.MissingNuGet(); + progressMonitor.LogError("Failed to restore Nuget packages with nuget.exe"); } var restoredProjects = RestoreSolutions(allSolutions, out var assets1); @@ -220,7 +232,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching if (isInAnalyzersFolder) { usedReferences.Remove(filename); - progressMonitor.RemovedReference(filename); + progressMonitor.LogInfo($"Removed analyzer reference {filename}"); } } } @@ -248,7 +260,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching dllPaths.Add(selectedFrameworkFolder); frameworkLocations.Add(selectedFrameworkFolder); - progressMonitor.LogInfo($"Found {frameworkType} DLLs in NuGet packages at {selectedFrameworkFolder}. Not adding installation directory."); + progressMonitor.LogInfo($"Found {frameworkType} DLLs in NuGet packages at {selectedFrameworkFolder}."); } private void AddNetFrameworkDlls(ISet dllPaths, ISet frameworkLocations) @@ -300,11 +312,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } var packagePathPrefix = Path.Combine(packageFolder, packagePrefix.ToLowerInvariant()); - var toRemove = dllPaths.Where(s => s.ToLowerInvariant().StartsWith(packagePathPrefix)); + var toRemove = dllPaths.Where(s => s.StartsWith(packagePathPrefix, StringComparison.InvariantCultureIgnoreCase)); foreach (var path in toRemove) { dllPaths.Remove(path); - progressMonitor.RemovedReference(path); + progressMonitor.LogInfo($"Removed reference {path}"); } } @@ -346,18 +358,26 @@ namespace Semmle.Extraction.CSharp.DependencyFetching .FullName; } - private IEnumerable GetAllPackageDirectories() + private ICollection GetAllPackageDirectories() { return new DirectoryInfo(packageDirectory.DirInfo.FullName) .EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false }) - .Select(d => d.Name); + .Select(d => d.Name) + .ToList(); } - private void LogAllUnusedPackages(DependencyContainer dependencies) => - GetAllPackageDirectories() + private void LogAllUnusedPackages(DependencyContainer dependencies) + { + var allPackageDirectories = GetAllPackageDirectories(); + + progressMonitor.LogInfo($"Restored {allPackageDirectories.Count} packages"); + progressMonitor.LogInfo($"Found {dependencies.Packages.Count} packages in project.asset.json files"); + + allPackageDirectories .Where(package => !dependencies.Packages.Contains(package)) .Order() .ForEach(package => progressMonitor.LogInfo($"Unused package: {package}")); + } private void GenerateSourceFileFromImplicitUsings() { @@ -380,6 +400,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching usings.UnionWith(fileContent.CustomImplicitUsings); + progressMonitor.LogInfo($"Generating source file for implicit usings. Namespaces: {string.Join(", ", usings.OrderBy(u => u))}"); + if (usings.Count > 0) { var tempDir = GetTemporaryWorkingDirectory("implicitUsings"); @@ -401,29 +423,28 @@ namespace Semmle.Extraction.CSharp.DependencyFetching private void GenerateSourceFilesFromWebViews(List allFiles) { - progressMonitor.LogInfo($"Generating source files from cshtml and razor files."); - var views = allFiles.SelectFileNamesByExtension(".cshtml", ".razor").ToArray(); - - if (views.Length > 0) + if (views.Length == 0) { - progressMonitor.LogInfo($"Found {views.Length} cshtml and razor files."); + return; + } - var sdk = new Sdk(dotnet).GetNewestSdk(); - if (sdk != null) + progressMonitor.LogInfo($"Found {views.Length} cshtml and razor files."); + + var sdk = new Sdk(dotnet).GetNewestSdk(); + if (sdk != null) + { + try { - try - { - var razor = new Razor(sdk, dotnet, progressMonitor); - var targetDir = GetTemporaryWorkingDirectory("razor"); - var generatedFiles = razor.GenerateFiles(views, usedReferences.Keys, targetDir); - this.generatedSources.AddRange(generatedFiles); - } - catch (Exception ex) - { - // It's okay, we tried our best to generate source files from cshtml files. - progressMonitor.LogInfo($"Failed to generate source files from cshtml files: {ex.Message}"); - } + var razor = new Razor(sdk, dotnet, progressMonitor); + var targetDir = GetTemporaryWorkingDirectory("razor"); + var generatedFiles = razor.GenerateFiles(views, usedReferences.Keys, targetDir); + this.generatedSources.AddRange(generatedFiles); + } + catch (Exception ex) + { + // It's okay, we tried our best to generate source files from cshtml files. + progressMonitor.LogInfo($"Failed to generate source files from cshtml files: {ex.Message}"); } } } @@ -499,7 +520,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// private void ResolveConflicts(IEnumerable frameworkPaths) { - var sortedReferences = new List(); + var sortedReferences = new List(usedReferences.Count); foreach (var usedReference in usedReferences) { try @@ -509,7 +530,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } catch (AssemblyLoadException) { - progressMonitor.Log(Util.Logging.Severity.Warning, $"Could not load assembly information from {usedReference.Key}"); + progressMonitor.Log(Severity.Warning, $"Could not load assembly information from {usedReference.Key}"); } } @@ -517,6 +538,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching .OrderAssemblyInfosByPreference(frameworkPaths) .ToList(); + progressMonitor.LogInfo($"Reference list contains {sortedReferences.Count} assemblies"); + var finalAssemblyList = new Dictionary(); // Pick the highest version for each assembly name @@ -532,15 +555,23 @@ namespace Semmle.Extraction.CSharp.DependencyFetching UseReference(r); } + progressMonitor.LogInfo($"After conflict resolution, reference list contains {finalAssemblyList.Count} assemblies"); + // Report the results foreach (var r in sortedReferences) { var resolvedInfo = finalAssemblyList[r.Name]; if (resolvedInfo.Version != r.Version || resolvedInfo.NetCoreVersion != r.NetCoreVersion) { - progressMonitor.ResolvedConflict(r.Id, resolvedInfo.Id + resolvedInfo.NetCoreVersion is null ? "" : $" (.NET Core {resolvedInfo.NetCoreVersion})"); + var asm = resolvedInfo.Id + (resolvedInfo.NetCoreVersion is null ? "" : $" (.NET Core {resolvedInfo.NetCoreVersion})"); + progressMonitor.LogInfo($"Resolved {r.Id} as {asm}"); ++conflictedReferences; } + + if (r != resolvedInfo) + { + progressMonitor.LogDebug($"Resolved {r.Id} as {resolvedInfo.Id} from {resolvedInfo.Filename}"); + } } } @@ -550,22 +581,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// The filename of the reference. private void UseReference(string reference) => usedReferences[reference] = true; - /// - /// Store that a particular source file is used (by a project file). - /// - /// The source file. - private void UseSource(FileInfo sourceFile) => sources[sourceFile.FullName] = sourceFile.Exists; - /// /// The list of resolved reference files. /// public IEnumerable ReferenceFiles => usedReferences.Keys; - /// - /// The list of source files used in projects. - /// - public IEnumerable ProjectSourceFiles => sources.Where(s => s.Value).Select(s => s.Key); - /// /// All of the generated source files in the source directory. /// @@ -581,12 +601,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// public IEnumerable UnresolvedReferences => unresolvedReferences.Select(r => r.Key); - /// - /// List of source files which were mentioned in project files but - /// do not exist on the file system. - /// - public IEnumerable MissingSourceFiles => sources.Where(s => !s.Value).Select(s => s.Key); - /// /// Record that a particular reference couldn't be resolved. /// Note that this records at most one project file per missing reference. @@ -595,23 +609,31 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// The project file making the reference. private void UnresolvedReference(string id, string projectFile) => unresolvedReferences[id] = projectFile; - /// - /// Reads all the source files and references from the given list of projects. - /// - /// The list of projects to analyse. - private void AnalyseProjectFiles(IEnumerable projectFiles) + private void AnalyseSolutions(IEnumerable solutions) { - foreach (var proj in projectFiles) + Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, solutionFile => { - AnalyseProject(proj); - } + try + { + var sln = new SolutionFile(solutionFile); + progressMonitor.LogInfo($"Analyzing {solutionFile}..."); + foreach (var proj in sln.Projects.Select(p => new FileInfo(p))) + { + AnalyseProject(proj); + } + } + catch (Microsoft.Build.Exceptions.InvalidProjectFileException ex) + { + progressMonitor.LogInfo($"Couldn't read solution file {solutionFile}: {ex.BaseMessage}"); + } + }); } private void AnalyseProject(FileInfo project) { if (!project.Exists) { - progressMonitor.MissingProject(project.FullName); + progressMonitor.LogInfo($"Couldn't read project file {project.FullName}"); return; } @@ -631,23 +653,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching UnresolvedReference(@ref, project.FullName); } } - - foreach (var src in csProj.Sources) - { - // Make a note of which source files the projects use. - // This information doesn't affect the build but is dumped - // as diagnostic output. - UseSource(new FileInfo(src)); - } - - ++succeededProjects; } catch (Exception ex) // lgtm[cs/catch-of-all-exceptions] { - ++failedProjects; - progressMonitor.FailedProjectFile(project.FullName, ex.Message); + progressMonitor.LogInfo($"Couldn't read project file {project.FullName}: {ex.Message}"); } - } /// @@ -664,7 +674,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching var assetFiles = new List(); var projects = solutions.SelectMany(solution => { - dotnet.RestoreSolutionToDirectory(solution, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: true, out var restoredProjects, out var a); + progressMonitor.LogInfo($"Restoring solution {solution}..."); + var success = dotnet.RestoreSolutionToDirectory(solution, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: true, out var restoredProjects, out var a); assetFiles.AddRange(a); return restoredProjects; }); @@ -683,7 +694,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching var assetFiles = new List(); Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, project => { - dotnet.RestoreProjectToDirectory(project, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: true, out var a, out var _); + progressMonitor.LogInfo($"Restoring project {project}..."); + var success = dotnet.RestoreProjectToDirectory(project, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: true, out var a, out var _); assetFiles.AddRange(a); }); assets = assetFiles; @@ -691,18 +703,30 @@ namespace Semmle.Extraction.CSharp.DependencyFetching private void DownloadMissingPackages(List allFiles, ISet dllPaths) { + var alreadyDownloadedPackages = Directory.GetDirectories(packageDirectory.DirInfo.FullName) + .Select(d => Path.GetFileName(d).ToLowerInvariant()); + var notYetDownloadedPackages = fileContent.AllPackages + .Except(alreadyDownloadedPackages) + .ToList(); + if (notYetDownloadedPackages.Count == 0) + { + return; + } + + progressMonitor.LogInfo($"Found {notYetDownloadedPackages.Count} packages that are not yet restored"); + var nugetConfigs = allFiles.SelectFileNamesByName("nuget.config").ToArray(); string? nugetConfig = null; if (nugetConfigs.Length > 1) { - progressMonitor.MultipleNugetConfig(nugetConfigs); + progressMonitor.LogInfo($"Found multiple nuget.config files: {string.Join(", ", nugetConfigs)}."); nugetConfig = allFiles .SelectRootFiles(sourceDir) .SelectFileNamesByName("nuget.config") .FirstOrDefault(); if (nugetConfig == null) { - progressMonitor.NoTopLevelNugetConfig(); + progressMonitor.LogInfo("Could not find a top-level nuget.config file."); } } else @@ -710,13 +734,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching nugetConfig = nugetConfigs.FirstOrDefault(); } - var alreadyDownloadedPackages = Directory.GetDirectories(packageDirectory.DirInfo.FullName) - .Select(d => Path.GetFileName(d).ToLowerInvariant()); - var notYetDownloadedPackages = fileContent.AllPackages.Except(alreadyDownloadedPackages); + if (nugetConfig != null) + { + progressMonitor.LogInfo($"Using nuget.config file {nugetConfig}."); + } Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, package => { - progressMonitor.NugetInstall(package); + progressMonitor.LogInfo($"Restoring package {package}..."); using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package, "missingpackages_workingdir")); var success = dotnet.New(tempDir.DirInfo.FullName); if (!success) @@ -743,7 +768,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching if (!success) { - progressMonitor.FailedToRestoreNugetPackage(package); + progressMonitor.LogInfo($"Failed to restore nuget package {package}"); } } }); @@ -751,23 +776,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching dllPaths.Add(missingPackageDirectory.DirInfo.FullName); } - private void AnalyseSolutions(IEnumerable solutions) - { - Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, solutionFile => - { - try - { - var sln = new SolutionFile(solutionFile); - progressMonitor.AnalysingSolution(solutionFile); - AnalyseProjectFiles(sln.Projects.Select(p => new FileInfo(p)).Where(p => p.Exists)); - } - catch (Microsoft.Build.Exceptions.InvalidProjectFileException ex) - { - progressMonitor.FailedProjectFile(solutionFile, ex.BaseMessage); - } - }); - } - public void Dispose(TemporaryDirectory? dir, string name) { try diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs index 5e7c2b60803..db338e3c9e5 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs @@ -35,21 +35,15 @@ namespace Semmle.Extraction.CSharp.DependencyFetching private bool RunCommandAux(string args, out IList output) { - progressMonitor.RunningProcess($"{Exec} {args}"); + progressMonitor.LogInfo($"Running {Exec} {args}"); var pi = MakeDotnetStartInfo(args); - var threadId = $"[{Environment.CurrentManagedThreadId:D3}]"; - void onOut(string s) - { - Console.Out.WriteLine($"{threadId} {s}"); - } - void onError(string s) - { - Console.Error.WriteLine($"{threadId} {s}"); - } + var threadId = Environment.CurrentManagedThreadId; + void onOut(string s) => progressMonitor.LogInfo(s, threadId); + void onError(string s) => progressMonitor.LogError(s, threadId); var exitCode = pi.ReadOutput(out output, onOut, onError); if (exitCode != 0) { - progressMonitor.CommandFailed(Exec, args, exitCode); + progressMonitor.LogError($"Command {Exec} {args} failed with exit code {exitCode}"); return false; } return true; diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs index 578ae81ebe2..04961c06143 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs @@ -192,7 +192,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } catch (Exception ex) { - progressMonitor.FailedToReadFile(file, ex); + progressMonitor.LogInfo($"Failed to read file {file}"); + progressMonitor.LogDebug($"Failed to read file {file}, exception: {ex}"); } } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackages.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackages.cs index 541853faf38..b0f4803ddce 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackages.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackages.cs @@ -42,6 +42,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching if (packageFiles.Length > 0) { + progressMonitor.LogInfo($"Found {packageFiles.Length} packages.config files, trying to use nuget.exe for package restore"); nugetExe = ResolveNugetExe(sourceDir); } else @@ -65,14 +66,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching var nuget = Path.Combine(directory, "nuget", "nuget.exe"); if (File.Exists(nuget)) { - progressMonitor.FoundNuGet(nuget); + progressMonitor.LogInfo($"Found nuget.exe at {nuget}"); return nuget; } - else - { - progressMonitor.LogInfo($"Nuget.exe could not be found at {nuget}"); - return DownloadNugetExe(sourceDir); - } + + return DownloadNugetExe(sourceDir); } private string DownloadNugetExe(string sourceDir) @@ -83,7 +81,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching // Nuget.exe already exists in the .nuget directory. if (File.Exists(nuget)) { - progressMonitor.FoundNuGet(nuget); + progressMonitor.LogInfo($"Found nuget.exe at {nuget}"); return nuget; } @@ -108,7 +106,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// The package file. private void RestoreNugetPackage(string package) { - progressMonitor.NugetInstall(package); + progressMonitor.LogInfo($"Restoring file {package}..."); /* Use nuget.exe to install a package. * Note that there is a clutch of NuGet assemblies which could be used to @@ -135,29 +133,17 @@ namespace Semmle.Extraction.CSharp.DependencyFetching UseShellExecute = false }; - try + var threadId = Environment.CurrentManagedThreadId; + void onOut(string s) => progressMonitor.LogInfo(s, threadId); + void onError(string s) => progressMonitor.LogError(s, threadId); + var exitCode = pi.ReadOutput(out var _, onOut, onError); + if (exitCode != 0) { - using var p = Process.Start(pi); - - if (p is null) - { - progressMonitor.FailedNugetCommand(pi.FileName, pi.Arguments, "Couldn't start process."); - return; - } - - var output = p.StandardOutput.ReadToEnd(); - var error = p.StandardError.ReadToEnd(); - - p.WaitForExit(); - if (p.ExitCode != 0) - { - progressMonitor.FailedNugetCommand(pi.FileName, pi.Arguments, output + error); - } + progressMonitor.LogError($"Command {pi.FileName} {pi.Arguments} failed with exit code {exitCode}"); } - catch (Exception ex) - when (ex is System.ComponentModel.Win32Exception || ex is FileNotFoundException) + else { - progressMonitor.FailedNugetCommand(pi.FileName, pi.Arguments, ex.Message); + progressMonitor.LogInfo($"Restored file {package}"); } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/ProgressMonitor.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/ProgressMonitor.cs index 7505c9a2785..b034c5b1240 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/ProgressMonitor.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/ProgressMonitor.cs @@ -1,5 +1,4 @@ -using System; -using Semmle.Util.Logging; +using Semmle.Util.Logging; namespace Semmle.Extraction.CSharp.DependencyFetching { @@ -15,113 +14,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching public void Log(Severity severity, string message) => logger.Log(severity, message); - public void LogInfo(string message) => - logger.Log(Severity.Info, message); + public void LogInfo(string message, int? threadId = null) => + logger.Log(Severity.Info, message, threadId); public void LogDebug(string message) => logger.Log(Severity.Debug, message); - private void LogError(string message) => - logger.Log(Severity.Error, message); - - public void FindingFiles(string dir) => - LogInfo($"Finding files in {dir}..."); - - public void IndexingReferences(int count) - { - LogInfo("Indexing..."); - LogDebug($"Indexing {count} DLLs..."); - } - - public void UnresolvedReference(string id, string project) - { - LogInfo($"Unresolved reference {id}"); - LogDebug($"Unresolved {id} referenced by {project}"); - } - - public void AnalysingSolution(string filename) => - LogInfo($"Analyzing {filename}..."); - - public void FailedProjectFile(string filename, string reason) => - LogInfo($"Couldn't read project file {filename}: {reason}"); - - public void FailedNugetCommand(string exe, string args, string message) - { - LogInfo($"Command failed: {exe} {args}"); - LogInfo($" {message}"); - } - - public void NugetInstall(string package) => - LogInfo($"Restoring {package}..."); - - public void ResolvedReference(string filename) => - LogInfo($"Resolved reference {filename}"); - - public void RemovedReference(string filename) => - LogInfo($"Removed reference {filename}"); - - public void Summary(int existingSources, int usedSources, int missingSources, - int references, int unresolvedReferences, - int resolvedConflicts, int totalProjects, int failedProjects, - TimeSpan analysisTime) - { - const int align = 6; - LogInfo(""); - LogInfo("Build analysis summary:"); - LogInfo($"{existingSources,align} source files in the filesystem"); - LogInfo($"{usedSources,align} source files in project files"); - LogInfo($"{missingSources,align} sources missing from project files"); - LogInfo($"{references,align} resolved references"); - LogInfo($"{unresolvedReferences,align} unresolved references"); - LogInfo($"{resolvedConflicts,align} resolved assembly conflicts"); - LogInfo($"{totalProjects,align} projects"); - LogInfo($"{failedProjects,align} missing/failed projects"); - LogInfo($"Build analysis completed in {analysisTime}"); - } - - public void ResolvedConflict(string asm1, string asm2) => - LogDebug($"Resolved {asm1} as {asm2}"); - - public void MissingProject(string projectFile) => - LogInfo($"Solution is missing {projectFile}"); - - public void CommandFailed(string exe, string arguments, int exitCode) => - LogError($"Command {exe} {arguments} failed with exit code {exitCode}"); - - public void MissingNuGet() => - LogError("Missing nuget.exe"); - - public void FoundNuGet(string path) => - LogInfo($"Found nuget.exe at {path}"); - - public void MissingDotNet() => - LogError("Missing dotnet CLI"); - - public void RunningProcess(string command) => - LogInfo($"Running {command}"); - - public void FailedToRestoreNugetPackage(string package) => - LogInfo($"Failed to restore nuget package {package}"); - - public void FailedToReadFile(string file, Exception ex) - { - LogInfo($"Failed to read file {file}"); - LogDebug($"Failed to read file {file}, exception: {ex}"); - } - - public void MultipleNugetConfig(string[] nugetConfigs) => - LogInfo($"Found multiple nuget.config files: {string.Join(", ", nugetConfigs)}."); - - internal void NoTopLevelNugetConfig() => - LogInfo("Could not find a top-level nuget.config file."); - - internal void RazorSourceGeneratorMissing(string fullPath) => - LogInfo($"Razor source generator folder {fullPath} does not exist."); - - internal void CscMissing(string cscPath) => - LogInfo($"Csc.exe not found at {cscPath}."); - - internal void RazorCscArgs(string args) => - LogInfo($"Running CSC to generate Razor source files. Args: {args}."); + public void LogError(string message, int? threadId = null) => + logger.Log(Severity.Error, message, threadId); } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Razor.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Razor.cs index 9d910835f90..be9274982ed 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Razor.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Razor.cs @@ -22,16 +22,18 @@ namespace Semmle.Extraction.CSharp.DependencyFetching this.dotNet = dotNet; sourceGeneratorFolder = Path.Combine(this.sdk.FullPath, "Sdks", "Microsoft.NET.Sdk.Razor", "source-generators"); + this.progressMonitor.LogInfo($"Razor source generator folder: {sourceGeneratorFolder}"); if (!Directory.Exists(sourceGeneratorFolder)) { - this.progressMonitor.RazorSourceGeneratorMissing(sourceGeneratorFolder); + this.progressMonitor.LogInfo($"Razor source generator folder {sourceGeneratorFolder} does not exist."); throw new Exception($"Razor source generator folder {sourceGeneratorFolder} does not exist."); } cscPath = Path.Combine(this.sdk.FullPath, "Roslyn", "bincore", "csc.dll"); + this.progressMonitor.LogInfo($"Razor source generator CSC: {cscPath}"); if (!File.Exists(cscPath)) { - this.progressMonitor.CscMissing(cscPath); + this.progressMonitor.LogInfo($"Csc.exe not found at {cscPath}."); throw new Exception($"csc.dll {cscPath} does not exist."); } } @@ -85,7 +87,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching var argsString = args.ToString(); - progressMonitor.RazorCscArgs(argsString); + progressMonitor.LogInfo($"Running CSC to generate Razor source files with arguments: {argsString}."); using (var sw = new StreamWriter(cscArgsPath)) { @@ -94,7 +96,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching dotNet.Exec($"\"{cscPath}\" /noconfig @\"{cscArgsPath}\""); - return Directory.GetFiles(outputFolder, "*.*", new EnumerationOptions { RecurseSubdirectories = true }); + var files = Directory.GetFiles(outputFolder, "*.*", new EnumerationOptions { RecurseSubdirectories = true }); + + progressMonitor.LogInfo($"Generated {files.Length} source files from cshtml files."); + + return files; } finally { diff --git a/csharp/extractor/Semmle.Extraction.Tests/LoggerStub.cs b/csharp/extractor/Semmle.Extraction.Tests/LoggerStub.cs index 78e85b8d60f..9120de8be45 100644 --- a/csharp/extractor/Semmle.Extraction.Tests/LoggerStub.cs +++ b/csharp/extractor/Semmle.Extraction.Tests/LoggerStub.cs @@ -4,7 +4,7 @@ namespace Semmle.Extraction.Tests { internal class LoggerStub : ILogger { - public void Log(Severity severity, string message) { } + public void Log(Severity severity, string message, int? threadId = null) { } public void Dispose() { } } diff --git a/csharp/extractor/Semmle.Util.Tests/CanonicalPathCache.cs b/csharp/extractor/Semmle.Util.Tests/CanonicalPathCache.cs index 1a50fd87679..313b949810d 100644 --- a/csharp/extractor/Semmle.Util.Tests/CanonicalPathCache.cs +++ b/csharp/extractor/Semmle.Util.Tests/CanonicalPathCache.cs @@ -176,7 +176,7 @@ namespace SemmleTests.Semmle.Util { public void Dispose() { } - public void Log(Severity s, string text) { } + public void Log(Severity s, string text, int? threadId = null) { } } } } diff --git a/csharp/extractor/Semmle.Util.Tests/FileUtils.cs b/csharp/extractor/Semmle.Util.Tests/FileUtils.cs index cbc82d4b814..c0e5e91db78 100644 --- a/csharp/extractor/Semmle.Util.Tests/FileUtils.cs +++ b/csharp/extractor/Semmle.Util.Tests/FileUtils.cs @@ -57,7 +57,7 @@ namespace SemmleTests.Semmle.Util { public void Dispose() { } - public void Log(Severity s, string text) { } + public void Log(Severity s, string text, int? threadId = null) { } } } } diff --git a/csharp/extractor/Semmle.Util/Logger.cs b/csharp/extractor/Semmle.Util/Logger.cs index 747d43368b8..5194feb3937 100644 --- a/csharp/extractor/Semmle.Util/Logger.cs +++ b/csharp/extractor/Semmle.Util/Logger.cs @@ -37,7 +37,7 @@ namespace Semmle.Util.Logging /// /// Log the given text with the given severity. /// - void Log(Severity s, string text); + void Log(Severity s, string text, int? threadId = null); } public static class LoggerExtensions @@ -92,12 +92,14 @@ namespace Semmle.Util.Logging return "[" + s.ToString().ToUpper() + "] "; } - public void Log(Severity s, string text) + public void Log(Severity s, string text, int? threadId = null) { if (verbosity.Includes(s)) { - var threadId = this.logThreadId ? $"[{Environment.CurrentManagedThreadId:D3}] " : ""; - writer.WriteLine(threadId + GetSeverityPrefix(s) + text); + threadId ??= Environment.CurrentManagedThreadId; + + var prefix = this.logThreadId ? $"[{threadId:D3}] " : ""; + writer.WriteLine(prefix + GetSeverityPrefix(s) + text); } } } @@ -140,12 +142,14 @@ namespace Semmle.Util.Logging } } - public void Log(Severity s, string text) + public void Log(Severity s, string text, int? threadId = null) { if (verbosity.Includes(s)) { - var threadId = this.logThreadId ? $"[{Environment.CurrentManagedThreadId:D3}] " : ""; - GetConsole(s).WriteLine(threadId + GetSeverityPrefix(s) + text); + threadId ??= Environment.CurrentManagedThreadId; + + var prefix = this.logThreadId ? $"[{threadId:D3}] " : ""; + GetConsole(s).WriteLine(prefix + GetSeverityPrefix(s) + text); } } } @@ -170,10 +174,10 @@ namespace Semmle.Util.Logging logger2.Dispose(); } - public void Log(Severity s, string text) + public void Log(Severity s, string text, int? threadId = null) { - logger1.Log(s, text); - logger2.Log(s, text); + logger1.Log(s, text, threadId); + logger2.Log(s, text, threadId); } } From d742cd3e44dfd81ee86a359f011cf130f7da7477 Mon Sep 17 00:00:00 2001 From: Tamas Vajk Date: Wed, 24 Jan 2024 11:20:14 +0100 Subject: [PATCH 2/4] C#: Remove progress monitor from dependency fetcher, use logger directly --- .../AssemblyCache.cs | 21 +-- .../Assets.cs | 29 ++-- .../DependencyManager.cs | 128 +++++++++--------- .../DotNet.cs | 13 +- .../DotNetCliInvoker.cs | 15 +- .../FileContent.cs | 13 +- .../FileInfoExtensions.cs | 5 +- .../FilePathFilter.cs | 12 +- .../NugetPackages.cs | 29 ++-- .../ProgressMonitor.cs | 26 ---- .../Razor.cs | 21 +-- .../Semmle.Extraction.Tests/Assets.cs | 16 +-- .../Semmle.Extraction.Tests/DotNet.cs | 2 +- .../Semmle.Extraction.Tests/FileContent.cs | 2 +- .../Semmle.Extraction.Tests/FilePathFilter.cs | 46 ++++--- csharp/extractor/Semmle.Util/Logger.cs | 8 ++ 16 files changed, 190 insertions(+), 196 deletions(-) delete mode 100644 csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/ProgressMonitor.cs diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs index 270dee28d4c..3d6672560b6 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs @@ -1,6 +1,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using Semmle.Util.Logging; namespace Semmle.Extraction.CSharp.DependencyFetching { @@ -18,10 +19,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// Paths to search. Directories are searched recursively. Files are added directly to the /// assembly cache. /// - /// Callback for progress. - public AssemblyCache(IEnumerable paths, IEnumerable frameworkPaths, ProgressMonitor progressMonitor) + /// Callback for progress. + public AssemblyCache(IEnumerable paths, IEnumerable frameworkPaths, ILogger logger) { - this.progressMonitor = progressMonitor; + this.logger = logger; foreach (var path in paths) { if (File.Exists(path)) @@ -32,12 +33,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching if (Directory.Exists(path)) { - progressMonitor.LogInfo($"Finding reference DLLs in {path}..."); + logger.LogInfo($"Finding reference DLLs in {path}..."); AddReferenceDirectory(path); } else { - progressMonitor.LogInfo("AssemblyCache: Path not found: " + path); + logger.LogInfo("AssemblyCache: Path not found: " + path); } } IndexReferences(frameworkPaths); @@ -63,7 +64,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// private void IndexReferences(IEnumerable frameworkPaths) { - progressMonitor.LogInfo($"Indexing {dllsToIndex.Count} assemblies..."); + logger.LogInfo($"Indexing {dllsToIndex.Count} assemblies..."); // Read all of the files foreach (var filename in dllsToIndex) @@ -71,7 +72,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching IndexReference(filename); } - progressMonitor.LogInfo($"Read {assemblyInfoByFileName.Count} assembly infos"); + logger.LogInfo($"Read {assemblyInfoByFileName.Count} assembly infos"); foreach (var info in assemblyInfoByFileName.Values .OrderBy(info => info.Name) @@ -88,13 +89,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { try { - progressMonitor.LogDebug($"Reading assembly info from {filename}"); + logger.LogDebug($"Reading assembly info from {filename}"); var info = AssemblyInfo.ReadFromFile(filename); assemblyInfoByFileName[filename] = info; } catch (AssemblyLoadException) { - progressMonitor.LogInfo($"Couldn't read assembly info from {filename}"); + logger.LogInfo($"Couldn't read assembly info from {filename}"); } } @@ -168,6 +169,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching private readonly HashSet failedAssemblyInfoIds = new HashSet(); - private readonly ProgressMonitor progressMonitor; + private readonly ILogger logger; } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Assets.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Assets.cs index 38db871d1b6..a59991b4b83 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Assets.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Assets.cs @@ -5,6 +5,7 @@ using System.IO; using System.Linq; using Newtonsoft.Json.Linq; using Semmle.Util; +using Semmle.Util.Logging; namespace Semmle.Extraction.CSharp.DependencyFetching { @@ -13,11 +14,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// internal class Assets { - private readonly ProgressMonitor progressMonitor; + private readonly ILogger logger; - internal Assets(ProgressMonitor progressMonitor) + internal Assets(ILogger logger) { - this.progressMonitor = progressMonitor; + this.logger = logger; } /// @@ -35,7 +36,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// /// Add the package dependencies from the assets file to dependencies. - /// + /// /// Parse a part of the JSon assets file and add the paths /// to the dependencies required for compilation (and collect /// information about used packages). @@ -60,7 +61,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// } /// } /// } - /// + /// /// Adds the following dependencies /// Paths: { /// "castle.core/4.4.1/lib/netstandard1.5/Castle.Core.dll", @@ -85,7 +86,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching if (references is null) { - progressMonitor.LogDebug("No references found in the targets section in the assets file."); + logger.LogDebug("No references found in the targets section in the assets file."); return; } @@ -157,7 +158,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching if (frameworks is null) { - progressMonitor.LogDebug("No framework section in assets.json."); + logger.LogDebug("No framework section in assets.json."); return; } @@ -171,7 +172,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching if (references is null) { - progressMonitor.LogDebug("No framework references in assets.json."); + logger.LogDebug("No framework references in assets.json."); return; } @@ -196,12 +197,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } catch (Exception e) { - progressMonitor.LogDebug($"Failed to parse assets file (unexpected error): {e.Message}"); + logger.LogDebug($"Failed to parse assets file (unexpected error): {e.Message}"); return false; } } - private static bool TryReadAllText(string path, ProgressMonitor progressMonitor, [NotNullWhen(returnValue: true)] out string? content) + private static bool TryReadAllText(string path, ILogger logger, [NotNullWhen(returnValue: true)] out string? content) { try { @@ -210,19 +211,19 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } catch (Exception e) { - progressMonitor.LogInfo($"Failed to read assets file '{path}': {e.Message}"); + logger.LogInfo($"Failed to read assets file '{path}': {e.Message}"); content = null; return false; } } - public static DependencyContainer GetCompilationDependencies(ProgressMonitor progressMonitor, IEnumerable assets) + public static DependencyContainer GetCompilationDependencies(ILogger logger, IEnumerable assets) { - var parser = new Assets(progressMonitor); + var parser = new Assets(logger); var dependencies = new DependencyContainer(); assets.ForEach(asset => { - if (TryReadAllText(asset, progressMonitor, out var json)) + if (TryReadAllText(asset, logger, out var json)) { parser.TryParse(json, dependencies); } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs index c4a1f3a0a71..d7f8d3c7441 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs @@ -17,7 +17,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching public sealed class DependencyManager : IDisposable { private readonly AssemblyCache assemblyCache; - private readonly ProgressMonitor progressMonitor; + private readonly ILogger logger; // Only used as a set, but ConcurrentDictionary is the only concurrent set in .NET. private readonly IDictionary usedReferences = new ConcurrentDictionary(); @@ -48,7 +48,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching var startTime = DateTime.Now; this.options = options; - this.progressMonitor = new ProgressMonitor(logger); + this.logger = logger; this.sourceDir = new DirectoryInfo(srcDir); packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName, "packages")); @@ -59,36 +59,36 @@ namespace Semmle.Extraction.CSharp.DependencyFetching try { - this.dotnet = DotNet.Make(options, progressMonitor, tempWorkingDirectory); + this.dotnet = DotNet.Make(options, logger, tempWorkingDirectory); runtimeLazy = new Lazy(() => new Runtime(dotnet)); } catch { - progressMonitor.LogError("Missing dotnet CLI"); + logger.LogError("Missing dotnet CLI"); throw; } - progressMonitor.LogInfo($"Finding files in {srcDir}..."); + logger.LogInfo($"Finding files in {srcDir}..."); var allFiles = GetAllFiles().ToList(); var binaryFileExtensions = new HashSet(new[] { ".dll", ".exe" }); // TODO: add more binary file extensions. var allNonBinaryFiles = allFiles.Where(f => !binaryFileExtensions.Contains(f.Extension.ToLowerInvariant())).ToList(); - var smallNonBinaryFiles = allNonBinaryFiles.SelectSmallFiles(progressMonitor).SelectFileNames(); - this.fileContent = new FileContent(progressMonitor, smallNonBinaryFiles); + var smallNonBinaryFiles = allNonBinaryFiles.SelectSmallFiles(logger).SelectFileNames(); + this.fileContent = new FileContent(logger, smallNonBinaryFiles); this.nonGeneratedSources = allNonBinaryFiles.SelectFileNamesByExtension(".cs").ToList(); this.generatedSources = new(); var allProjects = allNonBinaryFiles.SelectFileNamesByExtension(".csproj").ToList(); var allSolutions = allNonBinaryFiles.SelectFileNamesByExtension(".sln").ToList(); var dllPaths = allFiles.SelectFileNamesByExtension(".dll").ToHashSet(); - progressMonitor.LogInfo($"Found {allFiles.Count} files, {nonGeneratedSources.Count} source files, {allProjects.Count} project files, {allSolutions.Count} solution files, {dllPaths.Count} DLLs."); + logger.LogInfo($"Found {allFiles.Count} files, {nonGeneratedSources.Count} source files, {allProjects.Count} project files, {allSolutions.Count} solution files, {dllPaths.Count} DLLs."); RestoreNugetPackages(allNonBinaryFiles, allProjects, allSolutions, dllPaths); // Find DLLs in the .Net / Asp.Net Framework // This needs to come after the nuget restore, because the nuget restore might fetch the .NET Core/Framework reference assemblies. var frameworkLocations = AddFrameworkDlls(dllPaths); - assemblyCache = new AssemblyCache(dllPaths, frameworkLocations, progressMonitor); + assemblyCache = new AssemblyCache(dllPaths, frameworkLocations, logger); AnalyseSolutions(allSolutions); foreach (var filename in assemblyCache.AllAssemblies.Select(a => a.Filename)) @@ -102,35 +102,36 @@ namespace Semmle.Extraction.CSharp.DependencyFetching // Output the findings foreach (var r in usedReferences.Keys.OrderBy(r => r)) { - progressMonitor.LogInfo($"Resolved reference {r}"); + logger.LogInfo($"Resolved reference {r}"); } foreach (var r in unresolvedReferences.OrderBy(r => r.Key)) { - progressMonitor.LogInfo($"Unresolved reference {r.Key} in project {r.Value}"); + logger.LogInfo($"Unresolved reference {r.Key} in project {r.Value}"); } var webViewExtractionOption = Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_STANDALONE_EXTRACT_WEB_VIEWS"); if (bool.TryParse(webViewExtractionOption, out var shouldExtractWebViews) && shouldExtractWebViews) { - progressMonitor.LogInfo("Generating source files from cshtml and razor files..."); + logger.LogInfo("Generating source files from cshtml and razor files..."); GenerateSourceFilesFromWebViews(allNonBinaryFiles); } GenerateSourceFileFromImplicitUsings(); const int align = 6; - progressMonitor.LogInfo(""); - progressMonitor.LogInfo("Build analysis summary:"); - progressMonitor.LogInfo($"{this.nonGeneratedSources.Count,align} source files in the filesystem"); - progressMonitor.LogInfo($"{this.generatedSources.Count,align} generated source files"); - progressMonitor.LogInfo($"{allSolutions.Count,align} solution files"); - progressMonitor.LogInfo($"{allProjects.Count,align} project files in the filesystem"); - progressMonitor.LogInfo($"{usedReferences.Keys.Count,align} resolved references"); - progressMonitor.LogInfo($"{unresolvedReferences.Count,align} unresolved references"); - progressMonitor.LogInfo($"{conflictedReferences,align} resolved assembly conflicts"); - progressMonitor.LogInfo($"Build analysis completed in {DateTime.Now - startTime}"); + logger.LogInfo(""); + logger.LogInfo("Build analysis summary:"); + logger.LogInfo($"{this.nonGeneratedSources.Count,align} source files in the filesystem"); + logger.LogInfo($"{this.generatedSources.Count,align} generated source files"); + logger.LogInfo($"{allSolutions.Count,align} solution files"); + logger.LogInfo($"{allProjects.Count,align} project files in the filesystem"); + logger.LogInfo($"{usedReferences.Keys.Count,align} resolved references"); + logger.LogInfo($"{unresolvedReferences.Count,align} unresolved references"); + logger.LogInfo($"{conflictedReferences,align} resolved assembly conflicts"); + logger.LogInfo($"Build analysis completed in {DateTime.Now - startTime}"); + } private HashSet AddFrameworkDlls(HashSet dllPaths) @@ -148,7 +149,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { try { - var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, progressMonitor); + var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, logger); nuget.InstallPackages(); var nugetPackageDlls = legacyPackageDirectory.DirInfo.GetFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = true }); @@ -159,16 +160,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching if (nugetPackageDllPaths.Count > 0) { - progressMonitor.LogInfo($"Restored {nugetPackageDllPaths.Count} Nuget DLLs."); + logger.LogInfo($"Restored {nugetPackageDllPaths.Count} Nuget DLLs."); } if (excludedPaths.Count > 0) { - progressMonitor.LogInfo($"Excluding {excludedPaths.Count} Nuget DLLs."); + logger.LogInfo($"Excluding {excludedPaths.Count} Nuget DLLs."); } foreach (var excludedPath in excludedPaths) { - progressMonitor.LogInfo($"Excluded Nuget DLL: {excludedPath}"); + logger.LogInfo($"Excluded Nuget DLL: {excludedPath}"); } nugetPackageDllPaths.ExceptWith(excludedPaths); @@ -176,14 +177,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } catch (Exception) { - progressMonitor.LogError("Failed to restore Nuget packages with nuget.exe"); + logger.LogError("Failed to restore Nuget packages with nuget.exe"); } var restoredProjects = RestoreSolutions(allSolutions, out var assets1); var projects = allProjects.Except(restoredProjects); RestoreProjects(projects, out var assets2); - var dependencies = Assets.GetCompilationDependencies(progressMonitor, assets1.Union(assets2)); + var dependencies = Assets.GetCompilationDependencies(logger, assets1.Union(assets2)); var paths = dependencies .Paths @@ -232,7 +233,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching if (isInAnalyzersFolder) { usedReferences.Remove(filename); - progressMonitor.LogInfo($"Removed analyzer reference {filename}"); + logger.LogInfo($"Removed analyzer reference {filename}"); } } } @@ -248,19 +249,19 @@ namespace Semmle.Extraction.CSharp.DependencyFetching if (versionFolders.Length > 1) { var versions = string.Join(", ", versionFolders.Select(d => d.Name)); - progressMonitor.LogInfo($"Found multiple {frameworkType} DLLs in NuGet packages at {frameworkPath}. Using the latest version ({versionFolders[0].Name}) from: {versions}."); + logger.LogInfo($"Found multiple {frameworkType} DLLs in NuGet packages at {frameworkPath}. Using the latest version ({versionFolders[0].Name}) from: {versions}."); } var selectedFrameworkFolder = versionFolders.FirstOrDefault()?.FullName; if (selectedFrameworkFolder is null) { - progressMonitor.LogInfo($"Found {frameworkType} DLLs in NuGet packages at {frameworkPath}, but no version folder was found."); + logger.LogInfo($"Found {frameworkType} DLLs in NuGet packages at {frameworkPath}, but no version folder was found."); selectedFrameworkFolder = frameworkPath; } dllPaths.Add(selectedFrameworkFolder); frameworkLocations.Add(selectedFrameworkFolder); - progressMonitor.LogInfo($"Found {frameworkType} DLLs in NuGet packages at {selectedFrameworkFolder}."); + logger.LogInfo($"Found {frameworkType} DLLs in NuGet packages at {selectedFrameworkFolder}."); } private void AddNetFrameworkDlls(ISet dllPaths, ISet frameworkLocations) @@ -298,7 +299,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching runtimeLocation ??= Runtime.ExecutingRuntime; - progressMonitor.LogInfo($".NET runtime location selected: {runtimeLocation}"); + logger.LogInfo($".NET runtime location selected: {runtimeLocation}"); dllPaths.Add(runtimeLocation); frameworkLocations.Add(runtimeLocation); } @@ -316,7 +317,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching foreach (var path in toRemove) { dllPaths.Remove(path); - progressMonitor.LogInfo($"Removed reference {path}"); + logger.LogInfo($"Removed reference {path}"); } } @@ -336,7 +337,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching if (Runtime.AspNetCoreRuntime is string aspNetCoreRuntime) { - progressMonitor.LogInfo($"ASP.NET runtime location selected: {aspNetCoreRuntime}"); + logger.LogInfo($"ASP.NET runtime location selected: {aspNetCoreRuntime}"); dllPaths.Add(aspNetCoreRuntime); frameworkLocations.Add(aspNetCoreRuntime); } @@ -370,13 +371,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { var allPackageDirectories = GetAllPackageDirectories(); - progressMonitor.LogInfo($"Restored {allPackageDirectories.Count} packages"); - progressMonitor.LogInfo($"Found {dependencies.Packages.Count} packages in project.asset.json files"); + logger.LogInfo($"Restored {allPackageDirectories.Count} packages"); + logger.LogInfo($"Found {dependencies.Packages.Count} packages in project.asset.json files"); allPackageDirectories .Where(package => !dependencies.Packages.Contains(package)) .Order() - .ForEach(package => progressMonitor.LogInfo($"Unused package: {package}")); + .ForEach(package => logger.LogInfo($"Unused package: {package}")); } private void GenerateSourceFileFromImplicitUsings() @@ -400,7 +401,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching usings.UnionWith(fileContent.CustomImplicitUsings); - progressMonitor.LogInfo($"Generating source file for implicit usings. Namespaces: {string.Join(", ", usings.OrderBy(u => u))}"); + logger.LogInfo($"Generating source file for implicit usings. Namespaces: {string.Join(", ", usings.OrderBy(u => u))}"); if (usings.Count > 0) { @@ -429,14 +430,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching return; } - progressMonitor.LogInfo($"Found {views.Length} cshtml and razor files."); + logger.LogInfo($"Found {views.Length} cshtml and razor files."); var sdk = new Sdk(dotnet).GetNewestSdk(); if (sdk != null) { try { - var razor = new Razor(sdk, dotnet, progressMonitor); + var razor = new Razor(sdk, dotnet, logger); var targetDir = GetTemporaryWorkingDirectory("razor"); var generatedFiles = razor.GenerateFiles(views, usedReferences.Keys, targetDir); this.generatedSources.AddRange(generatedFiles); @@ -444,7 +445,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching catch (Exception ex) { // It's okay, we tried our best to generate source files from cshtml files. - progressMonitor.LogInfo($"Failed to generate source files from cshtml files: {ex.Message}"); + logger.LogInfo($"Failed to generate source files from cshtml files: {ex.Message}"); } } } @@ -469,17 +470,17 @@ namespace Semmle.Extraction.CSharp.DependencyFetching return true; } - progressMonitor.Log(Severity.Warning, $"File {f.FullName} could not be processed."); + logger.Log(Severity.Warning, $"File {f.FullName} could not be processed."); return false; } catch (Exception ex) { - progressMonitor.Log(Severity.Warning, $"File {f.FullName} could not be processed: {ex.Message}"); + logger.Log(Severity.Warning, $"File {f.FullName} could not be processed: {ex.Message}"); return false; } }); - files = new FilePathFilter(sourceDir, progressMonitor).Filter(files); + files = new FilePathFilter(sourceDir, logger).Filter(files); return files; } @@ -530,7 +531,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } catch (AssemblyLoadException) { - progressMonitor.Log(Severity.Warning, $"Could not load assembly information from {usedReference.Key}"); + logger.Log(Severity.Warning, $"Could not load assembly information from {usedReference.Key}"); } } @@ -538,7 +539,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching .OrderAssemblyInfosByPreference(frameworkPaths) .ToList(); - progressMonitor.LogInfo($"Reference list contains {sortedReferences.Count} assemblies"); + logger.LogInfo($"Reference list contains {sortedReferences.Count} assemblies"); var finalAssemblyList = new Dictionary(); @@ -555,7 +556,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching UseReference(r); } - progressMonitor.LogInfo($"After conflict resolution, reference list contains {finalAssemblyList.Count} assemblies"); + logger.LogInfo($"After conflict resolution, reference list contains {finalAssemblyList.Count} assemblies"); // Report the results foreach (var r in sortedReferences) @@ -564,13 +565,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching if (resolvedInfo.Version != r.Version || resolvedInfo.NetCoreVersion != r.NetCoreVersion) { var asm = resolvedInfo.Id + (resolvedInfo.NetCoreVersion is null ? "" : $" (.NET Core {resolvedInfo.NetCoreVersion})"); - progressMonitor.LogInfo($"Resolved {r.Id} as {asm}"); + logger.LogInfo($"Resolved {r.Id} as {asm}"); + ++conflictedReferences; } if (r != resolvedInfo) { - progressMonitor.LogDebug($"Resolved {r.Id} as {resolvedInfo.Id} from {resolvedInfo.Filename}"); + logger.LogDebug($"Resolved {r.Id} as {resolvedInfo.Id} from {resolvedInfo.Filename}"); } } } @@ -616,7 +618,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching try { var sln = new SolutionFile(solutionFile); - progressMonitor.LogInfo($"Analyzing {solutionFile}..."); + logger.LogInfo($"Analyzing {solutionFile}..."); foreach (var proj in sln.Projects.Select(p => new FileInfo(p))) { AnalyseProject(proj); @@ -624,7 +626,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } catch (Microsoft.Build.Exceptions.InvalidProjectFileException ex) { - progressMonitor.LogInfo($"Couldn't read solution file {solutionFile}: {ex.BaseMessage}"); + logger.LogInfo($"Couldn't read solution file {solutionFile}: {ex.BaseMessage}"); } }); } @@ -633,7 +635,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { if (!project.Exists) { - progressMonitor.LogInfo($"Couldn't read project file {project.FullName}"); + logger.LogInfo($"Couldn't read project file {project.FullName}"); return; } @@ -656,7 +658,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } catch (Exception ex) // lgtm[cs/catch-of-all-exceptions] { - progressMonitor.LogInfo($"Couldn't read project file {project.FullName}: {ex.Message}"); + logger.LogInfo($"Couldn't read project file {project.FullName}: {ex.Message}"); } } @@ -674,7 +676,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching var assetFiles = new List(); var projects = solutions.SelectMany(solution => { - progressMonitor.LogInfo($"Restoring solution {solution}..."); + logger.LogInfo($"Restoring solution {solution}..."); var success = dotnet.RestoreSolutionToDirectory(solution, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: true, out var restoredProjects, out var a); assetFiles.AddRange(a); return restoredProjects; @@ -694,7 +696,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching var assetFiles = new List(); Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, project => { - progressMonitor.LogInfo($"Restoring project {project}..."); + logger.LogInfo($"Restoring project {project}..."); var success = dotnet.RestoreProjectToDirectory(project, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: true, out var a, out var _); assetFiles.AddRange(a); }); @@ -713,20 +715,20 @@ namespace Semmle.Extraction.CSharp.DependencyFetching return; } - progressMonitor.LogInfo($"Found {notYetDownloadedPackages.Count} packages that are not yet restored"); + logger.LogInfo($"Found {notYetDownloadedPackages.Count} packages that are not yet restored"); var nugetConfigs = allFiles.SelectFileNamesByName("nuget.config").ToArray(); string? nugetConfig = null; if (nugetConfigs.Length > 1) { - progressMonitor.LogInfo($"Found multiple nuget.config files: {string.Join(", ", nugetConfigs)}."); + logger.LogInfo($"Found multiple nuget.config files: {string.Join(", ", nugetConfigs)}."); nugetConfig = allFiles .SelectRootFiles(sourceDir) .SelectFileNamesByName("nuget.config") .FirstOrDefault(); if (nugetConfig == null) { - progressMonitor.LogInfo("Could not find a top-level nuget.config file."); + logger.LogInfo("Could not find a top-level nuget.config file."); } } else @@ -736,12 +738,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching if (nugetConfig != null) { - progressMonitor.LogInfo($"Using nuget.config file {nugetConfig}."); + logger.LogInfo($"Using nuget.config file {nugetConfig}."); } Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, package => { - progressMonitor.LogInfo($"Restoring package {package}..."); + logger.LogInfo($"Restoring package {package}..."); using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package, "missingpackages_workingdir")); var success = dotnet.New(tempDir.DirInfo.FullName); if (!success) @@ -768,7 +770,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching if (!success) { - progressMonitor.LogInfo($"Failed to restore nuget package {package}"); + logger.LogInfo($"Failed to restore nuget package {package}"); } } }); @@ -784,7 +786,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } catch (Exception exc) { - progressMonitor.LogInfo($"Couldn't delete {name} directory {exc.Message}"); + logger.LogInfo($"Couldn't delete {name} directory {exc.Message}"); } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs index 9b53d47a242..fb5e0f2cd15 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs @@ -4,6 +4,7 @@ using System.IO; using System.Linq; using System.Text.RegularExpressions; using Semmle.Util; +using Semmle.Util.Logging; namespace Semmle.Extraction.CSharp.DependencyFetching { @@ -13,22 +14,22 @@ namespace Semmle.Extraction.CSharp.DependencyFetching internal partial class DotNet : IDotNet { private readonly IDotNetCliInvoker dotnetCliInvoker; - private readonly ProgressMonitor progressMonitor; + private readonly ILogger logger; private readonly TemporaryDirectory? tempWorkingDirectory; - private DotNet(IDotNetCliInvoker dotnetCliInvoker, ProgressMonitor progressMonitor, TemporaryDirectory? tempWorkingDirectory = null) + private DotNet(IDotNetCliInvoker dotnetCliInvoker, ILogger logger, TemporaryDirectory? tempWorkingDirectory = null) { - this.progressMonitor = progressMonitor; + this.logger = logger; this.tempWorkingDirectory = tempWorkingDirectory; this.dotnetCliInvoker = dotnetCliInvoker; Info(); } - private DotNet(IDependencyOptions options, ProgressMonitor progressMonitor, TemporaryDirectory tempWorkingDirectory) : this(new DotNetCliInvoker(progressMonitor, Path.Combine(options.DotNetPath ?? string.Empty, "dotnet")), progressMonitor, tempWorkingDirectory) { } + private DotNet(IDependencyOptions options, ILogger logger, TemporaryDirectory tempWorkingDirectory) : this(new DotNetCliInvoker(logger, Path.Combine(options.DotNetPath ?? string.Empty, "dotnet")), logger, tempWorkingDirectory) { } - internal static IDotNet Make(IDotNetCliInvoker dotnetCliInvoker, ProgressMonitor progressMonitor) => new DotNet(dotnetCliInvoker, progressMonitor); + internal static IDotNet Make(IDotNetCliInvoker dotnetCliInvoker, ILogger logger) => new DotNet(dotnetCliInvoker, logger); - public static IDotNet Make(IDependencyOptions options, ProgressMonitor progressMonitor, TemporaryDirectory tempWorkingDirectory) => new DotNet(options, progressMonitor, tempWorkingDirectory); + public static IDotNet Make(IDependencyOptions options, ILogger logger, TemporaryDirectory tempWorkingDirectory) => new DotNet(options, logger, tempWorkingDirectory); private void Info() { diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs index db338e3c9e5..a39124d77b9 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; using Semmle.Util; +using Semmle.Util.Logging; namespace Semmle.Extraction.CSharp.DependencyFetching { @@ -10,13 +11,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// internal sealed class DotNetCliInvoker : IDotNetCliInvoker { - private readonly ProgressMonitor progressMonitor; + private readonly ILogger logger; public string Exec { get; } - public DotNetCliInvoker(ProgressMonitor progressMonitor, string exec) + public DotNetCliInvoker(ILogger logger, string exec) { - this.progressMonitor = progressMonitor; + this.logger = logger; this.Exec = exec; } @@ -35,15 +36,15 @@ namespace Semmle.Extraction.CSharp.DependencyFetching private bool RunCommandAux(string args, out IList output) { - progressMonitor.LogInfo($"Running {Exec} {args}"); + logger.LogInfo($"Running {Exec} {args}"); var pi = MakeDotnetStartInfo(args); var threadId = Environment.CurrentManagedThreadId; - void onOut(string s) => progressMonitor.LogInfo(s, threadId); - void onError(string s) => progressMonitor.LogError(s, threadId); + void onOut(string s) => logger.LogInfo(s, threadId); + void onError(string s) => logger.LogError(s, threadId); var exitCode = pi.ReadOutput(out output, onOut, onError); if (exitCode != 0) { - progressMonitor.LogError($"Command {Exec} {args} failed with exit code {exitCode}"); + logger.LogError($"Command {Exec} {args} failed with exit code {exitCode}"); return false; } return true; diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs index 04961c06143..c06eaec270f 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.IO; using System.Text.RegularExpressions; using Semmle.Util; +using Semmle.Util.Logging; namespace Semmle.Extraction.CSharp.DependencyFetching { @@ -15,7 +16,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching // internal partial class FileContent { - private readonly ProgressMonitor progressMonitor; + private readonly ILogger logger; private readonly IUnsafeFileReader unsafeFileReader; private readonly IEnumerable files; private readonly HashSet allPackages = new HashSet(); @@ -90,18 +91,18 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } } - internal FileContent(ProgressMonitor progressMonitor, + internal FileContent(ILogger logger, IEnumerable files, IUnsafeFileReader unsafeFileReader) { - this.progressMonitor = progressMonitor; + this.logger = logger; this.files = files; this.unsafeFileReader = unsafeFileReader; this.initialize = new Initializer(DoInitialize); } - public FileContent(ProgressMonitor progressMonitor, IEnumerable files) : this(progressMonitor, files, new UnsafeFileReader()) + public FileContent(ILogger logger, IEnumerable files) : this(logger, files, new UnsafeFileReader()) { } private static string GetGroup(ReadOnlySpan input, ValueMatch valueMatch, string groupPrefix, bool toLower) @@ -192,8 +193,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } catch (Exception ex) { - progressMonitor.LogInfo($"Failed to read file {file}"); - progressMonitor.LogDebug($"Failed to read file {file}, exception: {ex}"); + logger.LogInfo($"Failed to read file {file}"); + logger.LogDebug($"Failed to read file {file}, exception: {ex}"); } } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileInfoExtensions.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileInfoExtensions.cs index 1d285d03d04..e68ad8c0616 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileInfoExtensions.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileInfoExtensions.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; +using Semmle.Util.Logging; namespace Semmle.Extraction.CSharp.DependencyFetching { @@ -13,14 +14,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching public static IEnumerable SelectRootFiles(this IEnumerable files, DirectoryInfo dir) => files.Where(file => file.DirectoryName == dir.FullName); - internal static IEnumerable SelectSmallFiles(this IEnumerable files, ProgressMonitor progressMonitor) + internal static IEnumerable SelectSmallFiles(this IEnumerable files, ILogger logger) { const int oneMb = 1_048_576; return files.Where(file => { if (file.Length > oneMb) { - progressMonitor.LogDebug($"Skipping {file.FullName} because it is bigger than 1MB."); + logger.LogDebug($"Skipping {file.FullName} because it is bigger than 1MB."); return false; } return true; diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FilePathFilter.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FilePathFilter.cs index 39f761d08b9..24d92e0f068 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FilePathFilter.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FilePathFilter.cs @@ -12,12 +12,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching public class FilePathFilter { private readonly string rootFolder; - private readonly IProgressMonitor progressMonitor; + private readonly ILogger logger; - public FilePathFilter(DirectoryInfo sourceDir, IProgressMonitor progressMonitor) + public FilePathFilter(DirectoryInfo sourceDir, ILogger logger) { rootFolder = FileUtils.ConvertToUnix(sourceDir.FullName.ToLowerInvariant()); - this.progressMonitor = progressMonitor; + this.logger = logger; } private class FileInclusion(string path, bool include) @@ -55,12 +55,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } else { - progressMonitor.Log(Severity.Info, $"Invalid filter: {filter}"); + logger.Log(Severity.Info, $"Invalid filter: {filter}"); continue; } var regex = new FilePattern(filterText).RegexPattern; - progressMonitor.Log(Severity.Info, $"Filtering {(include ? "in" : "out")} files matching '{regex}'. Original glob filter: '{filter}'"); + logger.Log(Severity.Info, $"Filtering {(include ? "in" : "out")} files matching '{regex}'. Original glob filter: '{filter}'"); pathFilters.Add(new PathFilter(new Regex(regex, RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline), include)); } @@ -91,7 +91,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching if (!include) { - progressMonitor.Log(Severity.Info, $"Excluding '{f.FileInfo.FullName}'"); + logger.Log(Severity.Info, $"Excluding '{f.FileInfo.FullName}'"); } return include; diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackages.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackages.cs index b0f4803ddce..52e5ccc4ff5 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackages.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackages.cs @@ -2,6 +2,7 @@ using System.Diagnostics; using System.IO; using System.Linq; +using Microsoft.Build.Framework; using Semmle.Util; namespace Semmle.Extraction.CSharp.DependencyFetching @@ -14,7 +15,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching internal class NugetPackages { private readonly string? nugetExe; - private readonly ProgressMonitor progressMonitor; + private readonly Util.Logging.ILogger logger; /// /// The list of package files. @@ -31,10 +32,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// /// Create the package manager for a specified source tree. /// - public NugetPackages(string sourceDir, TemporaryDirectory packageDirectory, ProgressMonitor progressMonitor) + public NugetPackages(string sourceDir, TemporaryDirectory packageDirectory, Util.Logging.ILogger logger) { this.packageDirectory = packageDirectory; - this.progressMonitor = progressMonitor; + this.logger = logger; packageFiles = new DirectoryInfo(sourceDir) .EnumerateFiles("packages.config", SearchOption.AllDirectories) @@ -42,12 +43,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching if (packageFiles.Length > 0) { - progressMonitor.LogInfo($"Found {packageFiles.Length} packages.config files, trying to use nuget.exe for package restore"); + logger.LogInfo($"Found {packageFiles.Length} packages.config files, trying to use nuget.exe for package restore"); nugetExe = ResolveNugetExe(sourceDir); } else { - progressMonitor.LogInfo("Found no packages.config file"); + logger.LogInfo("Found no packages.config file"); } } @@ -66,7 +67,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching var nuget = Path.Combine(directory, "nuget", "nuget.exe"); if (File.Exists(nuget)) { - progressMonitor.LogInfo($"Found nuget.exe at {nuget}"); + logger.LogInfo($"Found nuget.exe at {nuget}"); return nuget; } @@ -81,16 +82,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching // Nuget.exe already exists in the .nuget directory. if (File.Exists(nuget)) { - progressMonitor.LogInfo($"Found nuget.exe at {nuget}"); + logger.LogInfo($"Found nuget.exe at {nuget}"); return nuget; } Directory.CreateDirectory(directory); - progressMonitor.LogInfo("Attempting to download nuget.exe"); + logger.LogInfo("Attempting to download nuget.exe"); try { FileUtils.DownloadFile(FileUtils.NugetExeUrl, nuget); - progressMonitor.LogInfo($"Downloaded nuget.exe to {nuget}"); + logger.LogInfo($"Downloaded nuget.exe to {nuget}"); return nuget; } catch @@ -106,7 +107,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching /// The package file. private void RestoreNugetPackage(string package) { - progressMonitor.LogInfo($"Restoring file {package}..."); + logger.LogInfo($"Restoring file {package}..."); /* Use nuget.exe to install a package. * Note that there is a clutch of NuGet assemblies which could be used to @@ -134,16 +135,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching }; var threadId = Environment.CurrentManagedThreadId; - void onOut(string s) => progressMonitor.LogInfo(s, threadId); - void onError(string s) => progressMonitor.LogError(s, threadId); + void onOut(string s) => logger.LogInfo(s, threadId); + void onError(string s) => logger.LogError(s, threadId); var exitCode = pi.ReadOutput(out var _, onOut, onError); if (exitCode != 0) { - progressMonitor.LogError($"Command {pi.FileName} {pi.Arguments} failed with exit code {exitCode}"); + logger.LogError($"Command {pi.FileName} {pi.Arguments} failed with exit code {exitCode}"); } else { - progressMonitor.LogInfo($"Restored file {package}"); + logger.LogInfo($"Restored file {package}"); } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/ProgressMonitor.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/ProgressMonitor.cs deleted file mode 100644 index b034c5b1240..00000000000 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/ProgressMonitor.cs +++ /dev/null @@ -1,26 +0,0 @@ -using Semmle.Util.Logging; - -namespace Semmle.Extraction.CSharp.DependencyFetching -{ - internal class ProgressMonitor : IProgressMonitor - { - private readonly ILogger logger; - - public ProgressMonitor(ILogger logger) - { - this.logger = logger; - } - - public void Log(Severity severity, string message) => - logger.Log(severity, message); - - public void LogInfo(string message, int? threadId = null) => - logger.Log(Severity.Info, message, threadId); - - public void LogDebug(string message) => - logger.Log(Severity.Debug, message); - - public void LogError(string message, int? threadId = null) => - logger.Log(Severity.Error, message, threadId); - } -} diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Razor.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Razor.cs index be9274982ed..a1c96cc964e 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Razor.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Razor.cs @@ -4,36 +4,37 @@ using System.IO; using System.Text; using System.Linq; using Semmle.Util; +using Semmle.Util.Logging; namespace Semmle.Extraction.CSharp.DependencyFetching { internal class Razor { private readonly DotNetVersion sdk; - private readonly ProgressMonitor progressMonitor; + private readonly ILogger logger; private readonly IDotNet dotNet; private readonly string sourceGeneratorFolder; private readonly string cscPath; - public Razor(DotNetVersion sdk, IDotNet dotNet, ProgressMonitor progressMonitor) + public Razor(DotNetVersion sdk, IDotNet dotNet, ILogger logger) { this.sdk = sdk; - this.progressMonitor = progressMonitor; + this.logger = logger; this.dotNet = dotNet; sourceGeneratorFolder = Path.Combine(this.sdk.FullPath, "Sdks", "Microsoft.NET.Sdk.Razor", "source-generators"); - this.progressMonitor.LogInfo($"Razor source generator folder: {sourceGeneratorFolder}"); + this.logger.LogInfo($"Razor source generator folder: {sourceGeneratorFolder}"); if (!Directory.Exists(sourceGeneratorFolder)) { - this.progressMonitor.LogInfo($"Razor source generator folder {sourceGeneratorFolder} does not exist."); + this.logger.LogInfo($"Razor source generator folder {sourceGeneratorFolder} does not exist."); throw new Exception($"Razor source generator folder {sourceGeneratorFolder} does not exist."); } cscPath = Path.Combine(this.sdk.FullPath, "Roslyn", "bincore", "csc.dll"); - this.progressMonitor.LogInfo($"Razor source generator CSC: {cscPath}"); + this.logger.LogInfo($"Razor source generator CSC: {cscPath}"); if (!File.Exists(cscPath)) { - this.progressMonitor.LogInfo($"Csc.exe not found at {cscPath}."); + this.logger.LogInfo($"Csc.exe not found at {cscPath}."); throw new Exception($"csc.dll {cscPath} does not exist."); } } @@ -65,7 +66,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching { GenerateAnalyzerConfig(cshtmls, analyzerConfig); - progressMonitor.LogInfo($"Analyzer config content: {File.ReadAllText(analyzerConfig)}"); + logger.LogInfo($"Analyzer config content: {File.ReadAllText(analyzerConfig)}"); var args = new StringBuilder(); args.Append($"/target:exe /generatedfilesout:\"{outputFolder}\" /out:\"{dllPath}\" /analyzerconfig:\"{analyzerConfig}\" "); @@ -87,7 +88,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching var argsString = args.ToString(); - progressMonitor.LogInfo($"Running CSC to generate Razor source files with arguments: {argsString}."); + logger.LogInfo($"Running CSC to generate Razor source files with arguments: {argsString}."); using (var sw = new StreamWriter(cscArgsPath)) { @@ -98,7 +99,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching var files = Directory.GetFiles(outputFolder, "*.*", new EnumerationOptions { RecurseSubdirectories = true }); - progressMonitor.LogInfo($"Generated {files.Length} source files from cshtml files."); + logger.LogInfo($"Generated {files.Length} source files from cshtml files."); return files; } diff --git a/csharp/extractor/Semmle.Extraction.Tests/Assets.cs b/csharp/extractor/Semmle.Extraction.Tests/Assets.cs index 85e56b6de64..3406788bd1c 100644 --- a/csharp/extractor/Semmle.Extraction.Tests/Assets.cs +++ b/csharp/extractor/Semmle.Extraction.Tests/Assets.cs @@ -12,7 +12,7 @@ namespace Semmle.Extraction.Tests public void TestAssets1() { // Setup - var assets = new Assets(new ProgressMonitor(new LoggerStub())); + var assets = new Assets(new LoggerStub()); var json = assetsJson1; var dependencies = new DependencyContainer(); @@ -43,7 +43,7 @@ namespace Semmle.Extraction.Tests public void TestAssetsFailure() { // Setup - var assets = new Assets(new ProgressMonitor(new LoggerStub())); + var assets = new Assets(new LoggerStub()); var json = "garbage data"; var dependencies = new DependencyContainer(); @@ -59,7 +59,7 @@ namespace Semmle.Extraction.Tests public void TestAssetsNet70() { // Setup - var assets = new Assets(new ProgressMonitor(new LoggerStub())); + var assets = new Assets(new LoggerStub()); var json = assetsNet70; var dependencies = new DependencyContainer(); @@ -90,7 +90,7 @@ namespace Semmle.Extraction.Tests public void TestAssetsNet48() { // Setup - var assets = new Assets(new ProgressMonitor(new LoggerStub())); + var assets = new Assets(new LoggerStub()); var json = assetsNet48; var dependencies = new DependencyContainer(); @@ -117,7 +117,7 @@ namespace Semmle.Extraction.Tests public void TestAssetsNetstandard21() { // Setup - var assets = new Assets(new ProgressMonitor(new LoggerStub())); + var assets = new Assets(new LoggerStub()); var json = assetsNetstandard21; var dependencies = new DependencyContainer(); @@ -145,7 +145,7 @@ namespace Semmle.Extraction.Tests public void TestAssetsNetStandard16() { // Setup - var assets = new Assets(new ProgressMonitor(new LoggerStub())); + var assets = new Assets(new LoggerStub()); var json = assetsNetstandard16; var dependencies = new DependencyContainer(); @@ -177,7 +177,7 @@ namespace Semmle.Extraction.Tests public void TestAssetsNetcoreapp20() { // Setup - var assets = new Assets(new ProgressMonitor(new LoggerStub())); + var assets = new Assets(new LoggerStub()); var json = assetsNetcoreapp20; var dependencies = new DependencyContainer(); @@ -205,7 +205,7 @@ namespace Semmle.Extraction.Tests public void TestAssetsNetcoreapp31() { // Setup - var assets = new Assets(new ProgressMonitor(new LoggerStub())); + var assets = new Assets(new LoggerStub()); var json = assetsNetcoreapp31; var dependencies = new DependencyContainer(); diff --git a/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs b/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs index a9721f8dec2..6598d3d62b6 100644 --- a/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs +++ b/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs @@ -38,7 +38,7 @@ namespace Semmle.Extraction.Tests public class DotNetTests { private static IDotNet MakeDotnet(IDotNetCliInvoker dotnetCliInvoker) => - DotNet.Make(dotnetCliInvoker, new ProgressMonitor(new LoggerStub())); + DotNet.Make(dotnetCliInvoker, new LoggerStub()); private static IList MakeDotnetRestoreOutput() => new List { diff --git a/csharp/extractor/Semmle.Extraction.Tests/FileContent.cs b/csharp/extractor/Semmle.Extraction.Tests/FileContent.cs index 206453e132c..ba934120c45 100644 --- a/csharp/extractor/Semmle.Extraction.Tests/FileContent.cs +++ b/csharp/extractor/Semmle.Extraction.Tests/FileContent.cs @@ -25,7 +25,7 @@ namespace Semmle.Extraction.Tests internal class TestFileContent : FileContent { - public TestFileContent(IEnumerable lines) : base(new ProgressMonitor(new LoggerStub()), + public TestFileContent(IEnumerable lines) : base(new LoggerStub(), new List() { "test1.cs" }, new UnsafeFileReaderStub(lines)) { } diff --git a/csharp/extractor/Semmle.Extraction.Tests/FilePathFilter.cs b/csharp/extractor/Semmle.Extraction.Tests/FilePathFilter.cs index dd32da3361c..efb117a263d 100644 --- a/csharp/extractor/Semmle.Extraction.Tests/FilePathFilter.cs +++ b/csharp/extractor/Semmle.Extraction.Tests/FilePathFilter.cs @@ -10,17 +10,19 @@ namespace Semmle.Extraction.Tests { public class FilePathFilterTest { - private class ProgressMonitorStub : IProgressMonitor + private class LoggerStub : ILogger { public List Messages { get; } = []; - public void Log(Severity severity, string message) + public void Log(Severity s, string text, int? threadId = null) { - Messages.Add(message); + Messages.Add(text); } + + public void Dispose() { } } - private static (FilePathFilter TestSubject, ProgressMonitorStub progressMonitor, IEnumerable Files) TestSetup() + private static (FilePathFilter TestSubject, LoggerStub Logger, IEnumerable Files) TestSetup() { return TestSetup("/a/b", [ @@ -32,15 +34,15 @@ namespace Semmle.Extraction.Tests ]); } - private static (FilePathFilter TestSubject, ProgressMonitorStub progressMonitor, IEnumerable Files) TestSetup(string root, IEnumerable paths) + private static (FilePathFilter TestSubject, LoggerStub Logger, IEnumerable Files) TestSetup(string root, IEnumerable paths) { root = GetPlatformSpecifixPath(root); paths = GetPlatformSpecifixPaths(paths); - var progressMonitor = new ProgressMonitorStub(); + var logger = new LoggerStub(); - var filePathFilter = new FilePathFilter(new DirectoryInfo(root), progressMonitor); - return (filePathFilter, progressMonitor, paths.Select(p => new FileInfo(p))); + var filePathFilter = new FilePathFilter(new DirectoryInfo(root), logger); + return (filePathFilter, logger, paths.Select(p => new FileInfo(p))); } private static string GetPlatformSpecifixPath(string file) @@ -67,20 +69,20 @@ namespace Semmle.Extraction.Tests [Fact] public void TestNoFilter() { - (var testSubject, var progressMonitor, var files) = TestSetup(); + (var testSubject, var logger, var files) = TestSetup(); Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", null); var filtered = testSubject.Filter(files); AssertFileInfoEquivalence(files, filtered); - Assert.Equivalent(Array.Empty(), progressMonitor.Messages, strict: true); + Assert.Equivalent(Array.Empty(), logger.Messages, strict: true); } [Fact] public void TestFiltersWithOnlyInclude() { - (var testSubject, var progressMonitor, var files) = TestSetup(); + (var testSubject, var logger, var files) = TestSetup(); Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """ include:c/d @@ -104,13 +106,13 @@ namespace Semmle.Extraction.Tests "Filtering in files matching '^c/d.*'. Original glob filter: 'include:c/d'", "Filtering in files matching '^c/x/y.*'. Original glob filter: 'include:c/x/y'" }; - Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false); + Assert.Equivalent(expectedRegexMessages, logger.Messages, strict: false); } [Fact] public void TestFiltersWithOnlyExclude() { - (var testSubject, var progressMonitor, var files) = TestSetup(); + (var testSubject, var logger, var files) = TestSetup(); Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """ exclude:c/d/e @@ -130,13 +132,13 @@ namespace Semmle.Extraction.Tests { "Filtering out files matching '^c/d/e.*'. Original glob filter: 'exclude:c/d/e'" }; - Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false); + Assert.Equivalent(expectedRegexMessages, logger.Messages, strict: false); } [Fact] public void TestFiltersWithIncludeExclude() { - (var testSubject, var progressMonitor, var files) = TestSetup(); + (var testSubject, var logger, var files) = TestSetup(); Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """ include:c/x @@ -157,13 +159,13 @@ namespace Semmle.Extraction.Tests "Filtering in files matching '^c/x.*'. Original glob filter: 'include:c/x'", "Filtering out files matching '^c/x/z.*'. Original glob filter: 'exclude:c/x/z'" }; - Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false); + Assert.Equivalent(expectedRegexMessages, logger.Messages, strict: false); } [Fact] public void TestFiltersWithIncludeExcludeExcludeFirst() { - (var testSubject, var progressMonitor, var files) = TestSetup(); + (var testSubject, var logger, var files) = TestSetup(); // NOTE: the ordering DOES matter, later filters takes priority, so the exclude will end up not mattering at all. Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """ @@ -186,13 +188,13 @@ namespace Semmle.Extraction.Tests "Filtering in files matching '^c/x.*'. Original glob filter: 'include:c/x'", "Filtering out files matching '^c/x/z.*'. Original glob filter: 'exclude:c/x/z'" }; - Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false); + Assert.Equivalent(expectedRegexMessages, logger.Messages, strict: false); } [Fact] public void TestFiltersWithIncludeExcludeComplexPatterns1() { - (var testSubject, var progressMonitor, var files) = TestSetup(); + (var testSubject, var logger, var files) = TestSetup(); Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """ include:c/**/i.* @@ -218,13 +220,13 @@ namespace Semmle.Extraction.Tests "Filtering in files matching '^c/d/.*/[^/]*\\.cs.*'. Original glob filter: 'include:c/d/**/*.cs'", "Filtering out files matching '^.*/z/i\\.cs.*'. Original glob filter: 'exclude:**/z/i.cs'" }; - Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false); + Assert.Equivalent(expectedRegexMessages, logger.Messages, strict: false); } [Fact] public void TestFiltersWithIncludeExcludeComplexPatterns2() { - (var testSubject, var progressMonitor, var files) = TestSetup(); + (var testSubject, var logger, var files) = TestSetup(); Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """ include:**/i.* @@ -245,7 +247,7 @@ namespace Semmle.Extraction.Tests "Filtering in files matching '^.*/i\\.[^/]*.*'. Original glob filter: 'include:**/i.*'", "Filtering out files matching '^.*/z/i\\.cs.*'. Original glob filter: 'exclude:**/z/i.cs'" }; - Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false); + Assert.Equivalent(expectedRegexMessages, logger.Messages, strict: false); } } } diff --git a/csharp/extractor/Semmle.Util/Logger.cs b/csharp/extractor/Semmle.Util/Logger.cs index 5194feb3937..5e9aa3e219f 100644 --- a/csharp/extractor/Semmle.Util/Logger.cs +++ b/csharp/extractor/Semmle.Util/Logger.cs @@ -38,6 +38,14 @@ namespace Semmle.Util.Logging /// Log the given text with the given severity. /// void Log(Severity s, string text, int? threadId = null); + + void LogError(string text, int? threadId = null) => Log(Severity.Error, text, threadId); + + void LogWarning(string text, int? threadId = null) => Log(Severity.Warning, text, threadId); + + void LogInfo(string text, int? threadId = null) => Log(Severity.Info, text, threadId); + + void LogDebug(string text, int? threadId = null) => Log(Severity.Debug, text, threadId); } public static class LoggerExtensions From 30095e3179258362c7a42209efb3fb9a973cf9a4 Mon Sep 17 00:00:00 2001 From: Tamas Vajk Date: Wed, 24 Jan 2024 13:27:06 +0100 Subject: [PATCH 3/4] Refactor `dotnet restore` calls --- .../DependencyManager.cs | 20 ++++---- .../DotNet.cs | 48 ++++--------------- .../IDotNet.cs | 32 ++++++++++++- .../Semmle.Extraction.Tests/DotNet.cs | 38 +++++++-------- .../Semmle.Extraction.Tests/Runtime.cs | 14 +----- 5 files changed, 70 insertions(+), 82 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs index d7f8d3c7441..4f975e7aa0d 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs @@ -677,9 +677,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching var projects = solutions.SelectMany(solution => { logger.LogInfo($"Restoring solution {solution}..."); - var success = dotnet.RestoreSolutionToDirectory(solution, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: true, out var restoredProjects, out var a); - assetFiles.AddRange(a); - return restoredProjects; + var res = dotnet.Restore(new(solution, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true)); + assetFiles.AddRange(res.AssetsFilePaths); + return res.RestoredProjects; }); assets = assetFiles; return projects; @@ -697,8 +697,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, project => { logger.LogInfo($"Restoring project {project}..."); - var success = dotnet.RestoreProjectToDirectory(project, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: true, out var a, out var _); - assetFiles.AddRange(a); + var res = dotnet.Restore(new(project, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true)); + assetFiles.AddRange(res.AssetsFilePaths); }); assets = assetFiles; } @@ -757,18 +757,18 @@ namespace Semmle.Extraction.CSharp.DependencyFetching return; } - success = dotnet.RestoreProjectToDirectory(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: false, out var _, out var outputLines, pathToNugetConfig: nugetConfig); - if (!success) + var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig)); + if (!res.Success) { - if (outputLines?.Any(s => s.Contains("NU1301")) == true) + if (res.HasNugetPackageSourceError) { // Restore could not be completed because the listed source is unavailable. Try without the nuget.config: - success = dotnet.RestoreProjectToDirectory(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: false, out var _, out var _, pathToNugetConfig: null, force: true); + res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true)); } // TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package. - if (!success) + if (!res.Success) { logger.LogInfo($"Failed to restore nuget package {package}"); } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs index fb5e0f2cd15..37c028920a8 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs @@ -41,11 +41,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching } } - private string GetRestoreArgs(string projectOrSolutionFile, string packageDirectory, bool forceDotnetRefAssemblyFetching) + private string GetRestoreArgs(RestoreSettings restoreSettings) { - var args = $"restore --no-dependencies \"{projectOrSolutionFile}\" --packages \"{packageDirectory}\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal"; + var args = $"restore --no-dependencies \"{restoreSettings.File}\" --packages \"{restoreSettings.PackageDirectory}\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal"; - if (forceDotnetRefAssemblyFetching) + if (restoreSettings.ForceDotnetRefAssemblyFetching) { // Ugly hack: we set the TargetFrameworkRootPath and NetCoreTargetingPackRoot properties to an empty folder: var path = ".empty"; @@ -58,46 +58,24 @@ namespace Semmle.Extraction.CSharp.DependencyFetching args += $" /p:TargetFrameworkRootPath=\"{path}\" /p:NetCoreTargetingPackRoot=\"{path}\""; } - return args; - } - - private static IEnumerable GetFirstGroupOnMatch(Regex regex, IEnumerable lines) => - lines - .Select(line => regex.Match(line)) - .Where(match => match.Success) - .Select(match => match.Groups[1].Value); - - private static IEnumerable GetAssetsFilePaths(IEnumerable lines) => - GetFirstGroupOnMatch(AssetsFileRegex(), lines); - - private static IEnumerable GetRestoredProjects(IEnumerable lines) => - GetFirstGroupOnMatch(RestoredProjectRegex(), lines); - - public bool RestoreProjectToDirectory(string projectFile, string packageDirectory, bool forceDotnetRefAssemblyFetching, out IEnumerable assets, out IList outputLines, string? pathToNugetConfig = null, bool force = false) - { - var args = GetRestoreArgs(projectFile, packageDirectory, forceDotnetRefAssemblyFetching); - if (pathToNugetConfig != null) + if (restoreSettings.PathToNugetConfig != null) { - args += $" --configfile \"{pathToNugetConfig}\""; + args += $" --configfile \"{restoreSettings.PathToNugetConfig}\""; } - if (force) + if (restoreSettings.ForceReevaluation) { args += " --force"; } - var success = dotnetCliInvoker.RunCommand(args, out outputLines); - assets = success ? GetAssetsFilePaths(outputLines) : Array.Empty(); - return success; + return args; } - public bool RestoreSolutionToDirectory(string solutionFile, string packageDirectory, bool forceDotnetRefAssemblyFetching, out IEnumerable projects, out IEnumerable assets) + public RestoreResult Restore(RestoreSettings restoreSettings) { - var args = GetRestoreArgs(solutionFile, packageDirectory, forceDotnetRefAssemblyFetching); + var args = GetRestoreArgs(restoreSettings); var success = dotnetCliInvoker.RunCommand(args, out var output); - projects = success ? GetRestoredProjects(output) : Array.Empty(); - assets = success ? GetAssetsFilePaths(output) : Array.Empty(); - return success; + return new(success, output); } public bool New(string folder) @@ -130,11 +108,5 @@ namespace Semmle.Extraction.CSharp.DependencyFetching var args = $"exec {execArgs}"; return dotnetCliInvoker.RunCommand(args); } - - [GeneratedRegex("Restored\\s+(.+\\.csproj)", RegexOptions.Compiled)] - private static partial Regex RestoredProjectRegex(); - - [GeneratedRegex("[Assets\\sfile\\shas\\snot\\schanged.\\sSkipping\\sassets\\sfile\\swriting.|Writing\\sassets\\sfile\\sto\\sdisk.]\\sPath:\\s(.+)", RegexOptions.Compiled)] - private static partial Regex AssetsFileRegex(); } } diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs index 841fd0be879..5904248b34f 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs @@ -1,15 +1,43 @@ +using System; using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; namespace Semmle.Extraction.CSharp.DependencyFetching { internal interface IDotNet { - bool RestoreProjectToDirectory(string project, string directory, bool forceDotnetRefAssemblyFetching, out IEnumerable assets, out IList outputLines, string? pathToNugetConfig = null, bool force = false); - bool RestoreSolutionToDirectory(string solutionFile, string packageDirectory, bool forceDotnetRefAssemblyFetching, out IEnumerable projects, out IEnumerable assets); + RestoreResult Restore(RestoreSettings restoreSettings); bool New(string folder); bool AddPackage(string folder, string package); IList GetListedRuntimes(); IList GetListedSdks(); bool Exec(string execArgs); } + + internal record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, string? PathToNugetConfig = null, bool ForceReevaluation = false); + + internal partial record class RestoreResult(bool Success, IList Output) + { + private readonly Lazy> assetsFilePaths = new(() => GetFirstGroupOnMatch(AssetsFileRegex(), Output)); + public IEnumerable AssetsFilePaths => Success ? assetsFilePaths.Value : Array.Empty(); + + private readonly Lazy> restoredProjects = new(() => GetFirstGroupOnMatch(RestoredProjectRegex(), Output)); + public IEnumerable RestoredProjects => Success ? restoredProjects.Value : Array.Empty(); + + private readonly Lazy hasNugetPackageSourceError = new(() => Output.Any(s => s.Contains("NU1301"))); + public bool HasNugetPackageSourceError => hasNugetPackageSourceError.Value; + + private static IEnumerable GetFirstGroupOnMatch(Regex regex, IEnumerable lines) => + lines + .Select(line => regex.Match(line)) + .Where(match => match.Success) + .Select(match => match.Groups[1].Value); + + [GeneratedRegex("Restored\\s+(.+\\.csproj)", RegexOptions.Compiled)] + private static partial Regex RestoredProjectRegex(); + + [GeneratedRegex("[Assets\\sfile\\shas\\snot\\schanged.\\sSkipping\\sassets\\sfile\\swriting.|Writing\\sassets\\sfile\\sto\\sdisk.]\\sPath:\\s(.+)", RegexOptions.Compiled)] + private static partial Regex AssetsFileRegex(); + } } diff --git a/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs b/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs index 6598d3d62b6..289d9efbba4 100644 --- a/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs +++ b/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs @@ -101,7 +101,7 @@ namespace Semmle.Extraction.Tests var dotnet = MakeDotnet(dotnetCliInvoker); // Execute - dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages", false, out var assets, out var _); + dotnet.Restore(new("myproject.csproj", "mypackages", false)); // Verify var lastArgs = dotnetCliInvoker.GetLastArgs(); @@ -116,14 +116,14 @@ namespace Semmle.Extraction.Tests var dotnet = MakeDotnet(dotnetCliInvoker); // Execute - dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages", false, out var assets, out var _, pathToNugetConfig: "myconfig.config"); + var res = dotnet.Restore(new("myproject.csproj", "mypackages", false, "myconfig.config")); // Verify var lastArgs = dotnetCliInvoker.GetLastArgs(); Assert.Equal("restore --no-dependencies \"myproject.csproj\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal --configfile \"myconfig.config\"", lastArgs); - Assert.Equal(2, assets.Count()); - Assert.Contains("/path/to/project.assets.json", assets); - Assert.Contains("/path/to/project2.assets.json", assets); + Assert.Equal(2, res.AssetsFilePaths.Count()); + Assert.Contains("/path/to/project.assets.json", res.AssetsFilePaths); + Assert.Contains("/path/to/project2.assets.json", res.AssetsFilePaths); } [Fact] @@ -134,14 +134,14 @@ namespace Semmle.Extraction.Tests var dotnet = MakeDotnet(dotnetCliInvoker); // Execute - dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages", false, out var assets, out var _, pathToNugetConfig: "myconfig.config", force: true); + var res = dotnet.Restore(new("myproject.csproj", "mypackages", false, "myconfig.config", true)); // Verify var lastArgs = dotnetCliInvoker.GetLastArgs(); Assert.Equal("restore --no-dependencies \"myproject.csproj\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal --configfile \"myconfig.config\" --force", lastArgs); - Assert.Equal(2, assets.Count()); - Assert.Contains("/path/to/project.assets.json", assets); - Assert.Contains("/path/to/project2.assets.json", assets); + Assert.Equal(2, res.AssetsFilePaths.Count()); + Assert.Contains("/path/to/project.assets.json", res.AssetsFilePaths); + Assert.Contains("/path/to/project2.assets.json", res.AssetsFilePaths); } [Fact] @@ -152,17 +152,17 @@ namespace Semmle.Extraction.Tests var dotnet = MakeDotnet(dotnetCliInvoker); // Execute - dotnet.RestoreSolutionToDirectory("mysolution.sln", "mypackages", false, out var projects, out var assets); + var res = dotnet.Restore(new("mysolution.sln", "mypackages", false)); // Verify var lastArgs = dotnetCliInvoker.GetLastArgs(); Assert.Equal("restore --no-dependencies \"mysolution.sln\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal", lastArgs); - Assert.Equal(2, projects.Count()); - Assert.Contains("/path/to/project.csproj", projects); - Assert.Contains("/path/to/project2.csproj", projects); - Assert.Equal(2, assets.Count()); - Assert.Contains("/path/to/project.assets.json", assets); - Assert.Contains("/path/to/project2.assets.json", assets); + Assert.Equal(2, res.RestoredProjects.Count()); + Assert.Contains("/path/to/project.csproj", res.RestoredProjects); + Assert.Contains("/path/to/project2.csproj", res.RestoredProjects); + Assert.Equal(2, res.AssetsFilePaths.Count()); + Assert.Contains("/path/to/project.assets.json", res.AssetsFilePaths); + Assert.Contains("/path/to/project2.assets.json", res.AssetsFilePaths); } [Fact] @@ -174,13 +174,13 @@ namespace Semmle.Extraction.Tests dotnetCliInvoker.Success = false; // Execute - dotnet.RestoreSolutionToDirectory("mysolution.sln", "mypackages", false, out var projects, out var assets); + var res = dotnet.Restore(new("mysolution.sln", "mypackages", false)); // Verify var lastArgs = dotnetCliInvoker.GetLastArgs(); Assert.Equal("restore --no-dependencies \"mysolution.sln\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal", lastArgs); - Assert.Empty(projects); - Assert.Empty(assets); + Assert.Empty(res.RestoredProjects); + Assert.Empty(res.AssetsFilePaths); } [Fact] diff --git a/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs b/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs index 03df84fe462..17bc477bde8 100644 --- a/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs +++ b/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs @@ -19,19 +19,7 @@ namespace Semmle.Extraction.Tests public bool New(string folder) => true; - public bool RestoreProjectToDirectory(string project, string directory, bool forceDotnetRefAssemblyFetching, out IEnumerable assets, out IList outputLines, string? pathToNugetConfig = null, bool force = false) - { - assets = Array.Empty(); - outputLines = Array.Empty(); - return true; - } - - public bool RestoreSolutionToDirectory(string solution, string directory, bool forceDotnetRefAssemblyFetching, out IEnumerable projects, out IEnumerable assets) - { - projects = Array.Empty(); - assets = Array.Empty(); - return true; - } + public RestoreResult Restore(RestoreSettings restoreSettings) => new(true, Array.Empty()); public IList GetListedRuntimes() => runtimes; From 199b0578be4211938ec2eea99a1462cf40e1cf8d Mon Sep 17 00:00:00 2001 From: Tamas Vajk Date: Wed, 24 Jan 2024 15:23:38 +0100 Subject: [PATCH 4/4] C#: Log number of restored dotnet framework variants --- .../DependencyManager.cs | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs index 4f975e7aa0d..2dea2390a04 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs @@ -24,6 +24,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching private readonly IDictionary unresolvedReferences = new ConcurrentDictionary(); private readonly List nonGeneratedSources; private readonly List generatedSources; + private int dotnetFrameworkVersionVariantCount = 0; private int conflictedReferences = 0; private readonly IDependencyOptions options; private readonly DirectoryInfo sourceDir; @@ -123,15 +124,15 @@ namespace Semmle.Extraction.CSharp.DependencyFetching const int align = 6; logger.LogInfo(""); logger.LogInfo("Build analysis summary:"); - logger.LogInfo($"{this.nonGeneratedSources.Count,align} source files in the filesystem"); - logger.LogInfo($"{this.generatedSources.Count,align} generated source files"); + logger.LogInfo($"{nonGeneratedSources.Count,align} source files in the filesystem"); + logger.LogInfo($"{generatedSources.Count,align} generated source files"); logger.LogInfo($"{allSolutions.Count,align} solution files"); logger.LogInfo($"{allProjects.Count,align} project files in the filesystem"); logger.LogInfo($"{usedReferences.Keys.Count,align} resolved references"); logger.LogInfo($"{unresolvedReferences.Count,align} unresolved references"); logger.LogInfo($"{conflictedReferences,align} resolved assembly conflicts"); + logger.LogInfo($"{dotnetFrameworkVersionVariantCount,align} restored .NET framework variants"); logger.LogInfo($"Build analysis completed in {DateTime.Now - startTime}"); - } private HashSet AddFrameworkDlls(HashSet dllPaths) @@ -241,11 +242,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching private void SelectNewestFrameworkPath(string frameworkPath, string frameworkType, ISet dllPaths, ISet frameworkLocations) { - var versionFolders = new DirectoryInfo(frameworkPath) - .EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false }) - .OrderByDescending(d => d.Name) // TODO: Improve sorting to handle pre-release versions. - .ToArray(); - + var versionFolders = GetPackageVersionSubDirectories(frameworkPath); if (versionFolders.Length > 1) { var versions = string.Join(", ", versionFolders.Select(d => d.Name)); @@ -264,18 +261,34 @@ namespace Semmle.Extraction.CSharp.DependencyFetching logger.LogInfo($"Found {frameworkType} DLLs in NuGet packages at {selectedFrameworkFolder}."); } + private static DirectoryInfo[] GetPackageVersionSubDirectories(string packagePath) + { + return new DirectoryInfo(packagePath) + .EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false }) + .OrderByDescending(d => d.Name) // TODO: Improve sorting to handle pre-release versions. + .ToArray(); + } + private void AddNetFrameworkDlls(ISet dllPaths, ISet frameworkLocations) { // Multiple dotnet framework packages could be present. // The order of the packages is important, we're adding the first one that is present in the nuget cache. var packagesInPrioOrder = FrameworkPackageNames.NetFrameworks; - var frameworkPath = packagesInPrioOrder + var frameworkPaths = packagesInPrioOrder .Select((s, index) => (Index: index, Path: GetPackageDirectory(s))) - .FirstOrDefault(pair => pair.Path is not null); + .Where(pair => pair.Path is not null) + .ToArray(); + + var frameworkPath = frameworkPaths.FirstOrDefault(); if (frameworkPath.Path is not null) { + foreach (var fp in frameworkPaths) + { + dotnetFrameworkVersionVariantCount += GetPackageVersionSubDirectories(fp.Path!).Length; + } + SelectNewestFrameworkPath(frameworkPath.Path, ".NET Framework", dllPaths, frameworkLocations); for (var i = frameworkPath.Index + 1; i < packagesInPrioOrder.Length; i++)