hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-16 16:53:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

C#: Remove progress monitor from dependency fetcher, use logger directly

Browse Source
This commit is contained in:
Tamas Vajk
2024-01-24 11:20:14 +01:00
parent 13a8168c8e
commit d742cd3e44
16 changed files with 190 additions and 196 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs
View File

@@ -1,6 +1,7 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
@@ -18,10 +19,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// Paths to search. Directories are searched recursively. Files are added directly to the
/// assembly cache.
/// </param>
/// <param name="progressMonitor">Callback for progress.</param>
public AssemblyCache(IEnumerable<string> paths, IEnumerable<string> frameworkPaths, ProgressMonitor progressMonitor)
/// <param name="logger">Callback for progress.</param>
public AssemblyCache(IEnumerable<string> paths, IEnumerable<string> frameworkPaths, ILogger logger)
{
this.progressMonitor = progressMonitor;
this.logger = logger;
foreach (var path in paths)
{
if (File.Exists(path))
@@ -32,12 +33,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (Directory.Exists(path))
{
progressMonitor.LogInfo($"Finding reference DLLs in {path}...");
logger.LogInfo($"Finding reference DLLs in {path}...");
AddReferenceDirectory(path);
}
else
{
progressMonitor.LogInfo("AssemblyCache: Path not found: " + path);
logger.LogInfo("AssemblyCache: Path not found: " + path);
}
}
IndexReferences(frameworkPaths);
@@ -63,7 +64,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// </summary>
private void IndexReferences(IEnumerable<string> frameworkPaths)
{
progressMonitor.LogInfo($"Indexing {dllsToIndex.Count} assemblies...");
logger.LogInfo($"Indexing {dllsToIndex.Count} assemblies...");
// Read all of the files
foreach (var filename in dllsToIndex)
@@ -71,7 +72,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
IndexReference(filename);
}
progressMonitor.LogInfo($"Read {assemblyInfoByFileName.Count} assembly infos");
logger.LogInfo($"Read {assemblyInfoByFileName.Count} assembly infos");
foreach (var info in assemblyInfoByFileName.Values
.OrderBy(info => info.Name)
@@ -88,13 +89,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
try
{
progressMonitor.LogDebug($"Reading assembly info from {filename}");
logger.LogDebug($"Reading assembly info from {filename}");
var info = AssemblyInfo.ReadFromFile(filename);
assemblyInfoByFileName[filename] = info;
}
catch (AssemblyLoadException)
{
progressMonitor.LogInfo($"Couldn't read assembly info from {filename}");
logger.LogInfo($"Couldn't read assembly info from {filename}");
}
}
@@ -168,6 +169,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private readonly HashSet<string> failedAssemblyInfoIds = new HashSet<string>();
private readonly ProgressMonitor progressMonitor;
private readonly ILogger logger;
}
}

25
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Assets.cs
View File

@@ -5,6 +5,7 @@ using System.IO;
using System.Linq;
using Newtonsoft.Json.Linq;
using Semmle.Util;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
@@ -13,11 +14,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// </summary>
internal class Assets
{
private readonly ProgressMonitor progressMonitor;
private readonly ILogger logger;
internal Assets(ProgressMonitor progressMonitor)
internal Assets(ILogger logger)
{
this.progressMonitor = progressMonitor;
this.logger = logger;
}
/// <summary>
@@ -85,7 +86,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (references is null)
{
progressMonitor.LogDebug("No references found in the targets section in the assets file.");
logger.LogDebug("No references found in the targets section in the assets file.");
return;
}
@@ -157,7 +158,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (frameworks is null)
{
progressMonitor.LogDebug("No framework section in assets.json.");
logger.LogDebug("No framework section in assets.json.");
return;
}
@@ -171,7 +172,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (references is null)
{
progressMonitor.LogDebug("No framework references in assets.json.");
logger.LogDebug("No framework references in assets.json.");
return;
}
@@ -196,12 +197,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
catch (Exception e)
{
progressMonitor.LogDebug($"Failed to parse assets file (unexpected error): {e.Message}");
logger.LogDebug($"Failed to parse assets file (unexpected error): {e.Message}");
return false;
}
}
private static bool TryReadAllText(string path, ProgressMonitor progressMonitor, [NotNullWhen(returnValue: true)] out string? content)
private static bool TryReadAllText(string path, ILogger logger, [NotNullWhen(returnValue: true)] out string? content)
{
try
{
@@ -210,19 +211,19 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
catch (Exception e)
{
progressMonitor.LogInfo($"Failed to read assets file '{path}': {e.Message}");
logger.LogInfo($"Failed to read assets file '{path}': {e.Message}");
content = null;
return false;
}
}
public static DependencyContainer GetCompilationDependencies(ProgressMonitor progressMonitor, IEnumerable<string> assets)
public static DependencyContainer GetCompilationDependencies(ILogger logger, IEnumerable<string> assets)
{
var parser = new Assets(progressMonitor);
var parser = new Assets(logger);
var dependencies = new DependencyContainer();
assets.ForEach(asset =>
{
if (TryReadAllText(asset, progressMonitor, out var json))
if (TryReadAllText(asset, logger, out var json))
{
parser.TryParse(json, dependencies);
}

128
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
View File

@@ -17,7 +17,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
public sealed class DependencyManager : IDisposable
{
private readonly AssemblyCache assemblyCache;
private readonly ProgressMonitor progressMonitor;
private readonly ILogger logger;
// Only used as a set, but ConcurrentDictionary is the only concurrent set in .NET.
private readonly IDictionary<string, bool> usedReferences = new ConcurrentDictionary<string, bool>();
@@ -48,7 +48,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var startTime = DateTime.Now;
this.options = options;
this.progressMonitor = new ProgressMonitor(logger);
this.logger = logger;
this.sourceDir = new DirectoryInfo(srcDir);
packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName, "packages"));
@@ -59,36 +59,36 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
try
{
this.dotnet = DotNet.Make(options, progressMonitor, tempWorkingDirectory);
this.dotnet = DotNet.Make(options, logger, tempWorkingDirectory);
runtimeLazy = new Lazy<Runtime>(() => new Runtime(dotnet));
}
catch
{
progressMonitor.LogError("Missing dotnet CLI");
logger.LogError("Missing dotnet CLI");
throw;
}
progressMonitor.LogInfo($"Finding files in {srcDir}...");
logger.LogInfo($"Finding files in {srcDir}...");
var allFiles = GetAllFiles().ToList();
var binaryFileExtensions = new HashSet<string>(new[] { ".dll", ".exe" }); // TODO: add more binary file extensions.
var allNonBinaryFiles = allFiles.Where(f => !binaryFileExtensions.Contains(f.Extension.ToLowerInvariant())).ToList();
var smallNonBinaryFiles = allNonBinaryFiles.SelectSmallFiles(progressMonitor).SelectFileNames();
this.fileContent = new FileContent(progressMonitor, smallNonBinaryFiles);
var smallNonBinaryFiles = allNonBinaryFiles.SelectSmallFiles(logger).SelectFileNames();
this.fileContent = new FileContent(logger, smallNonBinaryFiles);
this.nonGeneratedSources = allNonBinaryFiles.SelectFileNamesByExtension(".cs").ToList();
this.generatedSources = new();
var allProjects = allNonBinaryFiles.SelectFileNamesByExtension(".csproj").ToList();
var allSolutions = allNonBinaryFiles.SelectFileNamesByExtension(".sln").ToList();
var dllPaths = allFiles.SelectFileNamesByExtension(".dll").ToHashSet();
progressMonitor.LogInfo($"Found {allFiles.Count} files, {nonGeneratedSources.Count} source files, {allProjects.Count} project files, {allSolutions.Count} solution files, {dllPaths.Count} DLLs.");
logger.LogInfo($"Found {allFiles.Count} files, {nonGeneratedSources.Count} source files, {allProjects.Count} project files, {allSolutions.Count} solution files, {dllPaths.Count} DLLs.");
RestoreNugetPackages(allNonBinaryFiles, allProjects, allSolutions, dllPaths);
// Find DLLs in the .Net / Asp.Net Framework
// This needs to come after the nuget restore, because the nuget restore might fetch the .NET Core/Framework reference assemblies.
var frameworkLocations = AddFrameworkDlls(dllPaths);
assemblyCache = new AssemblyCache(dllPaths, frameworkLocations, progressMonitor);
assemblyCache = new AssemblyCache(dllPaths, frameworkLocations, logger);
AnalyseSolutions(allSolutions);
foreach (var filename in assemblyCache.AllAssemblies.Select(a => a.Filename))
@@ -102,35 +102,36 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
// Output the findings
foreach (var r in usedReferences.Keys.OrderBy(r => r))
{
progressMonitor.LogInfo($"Resolved reference {r}");
logger.LogInfo($"Resolved reference {r}");
}
foreach (var r in unresolvedReferences.OrderBy(r => r.Key))
{
progressMonitor.LogInfo($"Unresolved reference {r.Key} in project {r.Value}");
logger.LogInfo($"Unresolved reference {r.Key} in project {r.Value}");
}
var webViewExtractionOption = Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_STANDALONE_EXTRACT_WEB_VIEWS");
if (bool.TryParse(webViewExtractionOption, out var shouldExtractWebViews) &&
shouldExtractWebViews)
{
progressMonitor.LogInfo("Generating source files from cshtml and razor files...");
logger.LogInfo("Generating source files from cshtml and razor files...");
GenerateSourceFilesFromWebViews(allNonBinaryFiles);
}
GenerateSourceFileFromImplicitUsings();
const int align = 6;
progressMonitor.LogInfo("");
progressMonitor.LogInfo("Build analysis summary:");
progressMonitor.LogInfo($"{this.nonGeneratedSources.Count,align} source files in the filesystem");
progressMonitor.LogInfo($"{this.generatedSources.Count,align} generated source files");
progressMonitor.LogInfo($"{allSolutions.Count,align} solution files");
progressMonitor.LogInfo($"{allProjects.Count,align} project files in the filesystem");
progressMonitor.LogInfo($"{usedReferences.Keys.Count,align} resolved references");
progressMonitor.LogInfo($"{unresolvedReferences.Count,align} unresolved references");
progressMonitor.LogInfo($"{conflictedReferences,align} resolved assembly conflicts");
progressMonitor.LogInfo($"Build analysis completed in {DateTime.Now - startTime}");
logger.LogInfo("");
logger.LogInfo("Build analysis summary:");
logger.LogInfo($"{this.nonGeneratedSources.Count,align} source files in the filesystem");
logger.LogInfo($"{this.generatedSources.Count,align} generated source files");
logger.LogInfo($"{allSolutions.Count,align} solution files");
logger.LogInfo($"{allProjects.Count,align} project files in the filesystem");
logger.LogInfo($"{usedReferences.Keys.Count,align} resolved references");
logger.LogInfo($"{unresolvedReferences.Count,align} unresolved references");
logger.LogInfo($"{conflictedReferences,align} resolved assembly conflicts");
logger.LogInfo($"Build analysis completed in {DateTime.Now - startTime}");
}
private HashSet<string> AddFrameworkDlls(HashSet<string> dllPaths)
@@ -148,7 +149,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
try
{
var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, progressMonitor);
var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, logger);
nuget.InstallPackages();
var nugetPackageDlls = legacyPackageDirectory.DirInfo.GetFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = true });
@@ -159,16 +160,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (nugetPackageDllPaths.Count > 0)
{
progressMonitor.LogInfo($"Restored {nugetPackageDllPaths.Count} Nuget DLLs.");
logger.LogInfo($"Restored {nugetPackageDllPaths.Count} Nuget DLLs.");
}
if (excludedPaths.Count > 0)
{
progressMonitor.LogInfo($"Excluding {excludedPaths.Count} Nuget DLLs.");
logger.LogInfo($"Excluding {excludedPaths.Count} Nuget DLLs.");
}
foreach (var excludedPath in excludedPaths)
{
progressMonitor.LogInfo($"Excluded Nuget DLL: {excludedPath}");
logger.LogInfo($"Excluded Nuget DLL: {excludedPath}");
}
nugetPackageDllPaths.ExceptWith(excludedPaths);
@@ -176,14 +177,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
catch (Exception)
{
progressMonitor.LogError("Failed to restore Nuget packages with nuget.exe");
logger.LogError("Failed to restore Nuget packages with nuget.exe");
}
var restoredProjects = RestoreSolutions(allSolutions, out var assets1);
var projects = allProjects.Except(restoredProjects);
RestoreProjects(projects, out var assets2);
var dependencies = Assets.GetCompilationDependencies(progressMonitor, assets1.Union(assets2));
var dependencies = Assets.GetCompilationDependencies(logger, assets1.Union(assets2));
var paths = dependencies
.Paths
@@ -232,7 +233,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (isInAnalyzersFolder)
{
usedReferences.Remove(filename);
progressMonitor.LogInfo($"Removed analyzer reference {filename}");
logger.LogInfo($"Removed analyzer reference {filename}");
}
}
}
@@ -248,19 +249,19 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (versionFolders.Length > 1)
{
var versions = string.Join(", ", versionFolders.Select(d => d.Name));
progressMonitor.LogInfo($"Found multiple {frameworkType} DLLs in NuGet packages at {frameworkPath}. Using the latest version ({versionFolders[0].Name}) from: {versions}.");
logger.LogInfo($"Found multiple {frameworkType} DLLs in NuGet packages at {frameworkPath}. Using the latest version ({versionFolders[0].Name}) from: {versions}.");
}
var selectedFrameworkFolder = versionFolders.FirstOrDefault()?.FullName;
if (selectedFrameworkFolder is null)
{
progressMonitor.LogInfo($"Found {frameworkType} DLLs in NuGet packages at {frameworkPath}, but no version folder was found.");
logger.LogInfo($"Found {frameworkType} DLLs in NuGet packages at {frameworkPath}, but no version folder was found.");
selectedFrameworkFolder = frameworkPath;
}
dllPaths.Add(selectedFrameworkFolder);
frameworkLocations.Add(selectedFrameworkFolder);
progressMonitor.LogInfo($"Found {frameworkType} DLLs in NuGet packages at {selectedFrameworkFolder}.");
logger.LogInfo($"Found {frameworkType} DLLs in NuGet packages at {selectedFrameworkFolder}.");
}
private void AddNetFrameworkDlls(ISet<string> dllPaths, ISet<string> frameworkLocations)
@@ -298,7 +299,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
runtimeLocation ??= Runtime.ExecutingRuntime;
progressMonitor.LogInfo($".NET runtime location selected: {runtimeLocation}");
logger.LogInfo($".NET runtime location selected: {runtimeLocation}");
dllPaths.Add(runtimeLocation);
frameworkLocations.Add(runtimeLocation);
}
@@ -316,7 +317,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
foreach (var path in toRemove)
{
dllPaths.Remove(path);
progressMonitor.LogInfo($"Removed reference {path}");
logger.LogInfo($"Removed reference {path}");
}
}
@@ -336,7 +337,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (Runtime.AspNetCoreRuntime is string aspNetCoreRuntime)
{
progressMonitor.LogInfo($"ASP.NET runtime location selected: {aspNetCoreRuntime}");
logger.LogInfo($"ASP.NET runtime location selected: {aspNetCoreRuntime}");
dllPaths.Add(aspNetCoreRuntime);
frameworkLocations.Add(aspNetCoreRuntime);
}
@@ -370,13 +371,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
var allPackageDirectories = GetAllPackageDirectories();
progressMonitor.LogInfo($"Restored {allPackageDirectories.Count} packages");
progressMonitor.LogInfo($"Found {dependencies.Packages.Count} packages in project.asset.json files");
logger.LogInfo($"Restored {allPackageDirectories.Count} packages");
logger.LogInfo($"Found {dependencies.Packages.Count} packages in project.asset.json files");
allPackageDirectories
.Where(package => !dependencies.Packages.Contains(package))
.Order()
.ForEach(package => progressMonitor.LogInfo($"Unused package: {package}"));
.ForEach(package => logger.LogInfo($"Unused package: {package}"));
}
private void GenerateSourceFileFromImplicitUsings()
@@ -400,7 +401,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
usings.UnionWith(fileContent.CustomImplicitUsings);
progressMonitor.LogInfo($"Generating source file for implicit usings. Namespaces: {string.Join(", ", usings.OrderBy(u => u))}");
logger.LogInfo($"Generating source file for implicit usings. Namespaces: {string.Join(", ", usings.OrderBy(u => u))}");
if (usings.Count > 0)
{
@@ -429,14 +430,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
return;
}
progressMonitor.LogInfo($"Found {views.Length} cshtml and razor files.");
logger.LogInfo($"Found {views.Length} cshtml and razor files.");
var sdk = new Sdk(dotnet).GetNewestSdk();
if (sdk != null)
{
try
{
var razor = new Razor(sdk, dotnet, progressMonitor);
var razor = new Razor(sdk, dotnet, logger);
var targetDir = GetTemporaryWorkingDirectory("razor");
var generatedFiles = razor.GenerateFiles(views, usedReferences.Keys, targetDir);
this.generatedSources.AddRange(generatedFiles);
@@ -444,7 +445,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
catch (Exception ex)
{
// It's okay, we tried our best to generate source files from cshtml files.
progressMonitor.LogInfo($"Failed to generate source files from cshtml files: {ex.Message}");
logger.LogInfo($"Failed to generate source files from cshtml files: {ex.Message}");
}
}
}
@@ -469,17 +470,17 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
return true;
}
progressMonitor.Log(Severity.Warning, $"File {f.FullName} could not be processed.");
logger.Log(Severity.Warning, $"File {f.FullName} could not be processed.");
return false;
}
catch (Exception ex)
{
progressMonitor.Log(Severity.Warning, $"File {f.FullName} could not be processed: {ex.Message}");
logger.Log(Severity.Warning, $"File {f.FullName} could not be processed: {ex.Message}");
return false;
}
});
files = new FilePathFilter(sourceDir, progressMonitor).Filter(files);
files = new FilePathFilter(sourceDir, logger).Filter(files);
return files;
}
@@ -530,7 +531,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
catch (AssemblyLoadException)
{
progressMonitor.Log(Severity.Warning, $"Could not load assembly information from {usedReference.Key}");
logger.Log(Severity.Warning, $"Could not load assembly information from {usedReference.Key}");
}
}
@@ -538,7 +539,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
.OrderAssemblyInfosByPreference(frameworkPaths)
.ToList();
progressMonitor.LogInfo($"Reference list contains {sortedReferences.Count} assemblies");
logger.LogInfo($"Reference list contains {sortedReferences.Count} assemblies");
var finalAssemblyList = new Dictionary<string, AssemblyInfo>();
@@ -555,7 +556,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
UseReference(r);
}
progressMonitor.LogInfo($"After conflict resolution, reference list contains {finalAssemblyList.Count} assemblies");
logger.LogInfo($"After conflict resolution, reference list contains {finalAssemblyList.Count} assemblies");
// Report the results
foreach (var r in sortedReferences)
@@ -564,13 +565,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (resolvedInfo.Version != r.Version || resolvedInfo.NetCoreVersion != r.NetCoreVersion)
{
var asm = resolvedInfo.Id + (resolvedInfo.NetCoreVersion is null ? "" : $" (.NET Core {resolvedInfo.NetCoreVersion})");
progressMonitor.LogInfo($"Resolved {r.Id} as {asm}");
logger.LogInfo($"Resolved {r.Id} as {asm}");
++conflictedReferences;
}
if (r != resolvedInfo)
{
progressMonitor.LogDebug($"Resolved {r.Id} as {resolvedInfo.Id} from {resolvedInfo.Filename}");
logger.LogDebug($"Resolved {r.Id} as {resolvedInfo.Id} from {resolvedInfo.Filename}");
}
}
}
@@ -616,7 +618,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
try
{
var sln = new SolutionFile(solutionFile);
progressMonitor.LogInfo($"Analyzing {solutionFile}...");
logger.LogInfo($"Analyzing {solutionFile}...");
foreach (var proj in sln.Projects.Select(p => new FileInfo(p)))
{
AnalyseProject(proj);
@@ -624,7 +626,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
catch (Microsoft.Build.Exceptions.InvalidProjectFileException ex)
{
progressMonitor.LogInfo($"Couldn't read solution file {solutionFile}: {ex.BaseMessage}");
logger.LogInfo($"Couldn't read solution file {solutionFile}: {ex.BaseMessage}");
}
});
}
@@ -633,7 +635,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
if (!project.Exists)
{
progressMonitor.LogInfo($"Couldn't read project file {project.FullName}");
logger.LogInfo($"Couldn't read project file {project.FullName}");
return;
}
@@ -656,7 +658,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{
progressMonitor.LogInfo($"Couldn't read project file {project.FullName}: {ex.Message}");
logger.LogInfo($"Couldn't read project file {project.FullName}: {ex.Message}");
}
}
@@ -674,7 +676,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var assetFiles = new List<string>();
var projects = solutions.SelectMany(solution =>
{
progressMonitor.LogInfo($"Restoring solution {solution}...");
logger.LogInfo($"Restoring solution {solution}...");
var success = dotnet.RestoreSolutionToDirectory(solution, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: true, out var restoredProjects, out var a);
assetFiles.AddRange(a);
return restoredProjects;
@@ -694,7 +696,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var assetFiles = new List<string>();
Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, project =>
{
progressMonitor.LogInfo($"Restoring project {project}...");
logger.LogInfo($"Restoring project {project}...");
var success = dotnet.RestoreProjectToDirectory(project, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: true, out var a, out var _);
assetFiles.AddRange(a);
});
@@ -713,20 +715,20 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
return;
}
progressMonitor.LogInfo($"Found {notYetDownloadedPackages.Count} packages that are not yet restored");
logger.LogInfo($"Found {notYetDownloadedPackages.Count} packages that are not yet restored");
var nugetConfigs = allFiles.SelectFileNamesByName("nuget.config").ToArray();
string? nugetConfig = null;
if (nugetConfigs.Length > 1)
{
progressMonitor.LogInfo($"Found multiple nuget.config files: {string.Join(", ", nugetConfigs)}.");
logger.LogInfo($"Found multiple nuget.config files: {string.Join(", ", nugetConfigs)}.");
nugetConfig = allFiles
.SelectRootFiles(sourceDir)
.SelectFileNamesByName("nuget.config")
.FirstOrDefault();
if (nugetConfig == null)
{
progressMonitor.LogInfo("Could not find a top-level nuget.config file.");
logger.LogInfo("Could not find a top-level nuget.config file.");
}
}
else
@@ -736,12 +738,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (nugetConfig != null)
{
progressMonitor.LogInfo($"Using nuget.config file {nugetConfig}.");
logger.LogInfo($"Using nuget.config file {nugetConfig}.");
}
Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, package =>
{
progressMonitor.LogInfo($"Restoring package {package}...");
logger.LogInfo($"Restoring package {package}...");
using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package, "missingpackages_workingdir"));
var success = dotnet.New(tempDir.DirInfo.FullName);
if (!success)
@@ -768,7 +770,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (!success)
{
progressMonitor.LogInfo($"Failed to restore nuget package {package}");
logger.LogInfo($"Failed to restore nuget package {package}");
}
}
});
@@ -784,7 +786,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
catch (Exception exc)
{
progressMonitor.LogInfo($"Couldn't delete {name} directory {exc.Message}");
logger.LogInfo($"Couldn't delete {name} directory {exc.Message}");
}
}

13
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs
View File

@@ -4,6 +4,7 @@ using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using Semmle.Util;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
@@ -13,22 +14,22 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
internal partial class DotNet : IDotNet
{
private readonly IDotNetCliInvoker dotnetCliInvoker;
private readonly ProgressMonitor progressMonitor;
private readonly ILogger logger;
private readonly TemporaryDirectory? tempWorkingDirectory;
private DotNet(IDotNetCliInvoker dotnetCliInvoker, ProgressMonitor progressMonitor, TemporaryDirectory? tempWorkingDirectory = null)
private DotNet(IDotNetCliInvoker dotnetCliInvoker, ILogger logger, TemporaryDirectory? tempWorkingDirectory = null)
{
this.progressMonitor = progressMonitor;
this.logger = logger;
this.tempWorkingDirectory = tempWorkingDirectory;
this.dotnetCliInvoker = dotnetCliInvoker;
Info();
}
private DotNet(IDependencyOptions options, ProgressMonitor progressMonitor, TemporaryDirectory tempWorkingDirectory) : this(new DotNetCliInvoker(progressMonitor, Path.Combine(options.DotNetPath ?? string.Empty, "dotnet")), progressMonitor, tempWorkingDirectory) { }
private DotNet(IDependencyOptions options, ILogger logger, TemporaryDirectory tempWorkingDirectory) : this(new DotNetCliInvoker(logger, Path.Combine(options.DotNetPath ?? string.Empty, "dotnet")), logger, tempWorkingDirectory) { }
internal static IDotNet Make(IDotNetCliInvoker dotnetCliInvoker, ProgressMonitor progressMonitor) => new DotNet(dotnetCliInvoker, progressMonitor);
internal static IDotNet Make(IDotNetCliInvoker dotnetCliInvoker, ILogger logger) => new DotNet(dotnetCliInvoker, logger);
public static IDotNet Make(IDependencyOptions options, ProgressMonitor progressMonitor, TemporaryDirectory tempWorkingDirectory) => new DotNet(options, progressMonitor, tempWorkingDirectory);
public static IDotNet Make(IDependencyOptions options, ILogger logger, TemporaryDirectory tempWorkingDirectory) => new DotNet(options, logger, tempWorkingDirectory);
private void Info()
{

15
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs
View File

@@ -2,6 +2,7 @@ using System;
using System.Collections.Generic;
using System.Diagnostics;
using Semmle.Util;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
@@ -10,13 +11,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// </summary>
internal sealed class DotNetCliInvoker : IDotNetCliInvoker
{
private readonly ProgressMonitor progressMonitor;
private readonly ILogger logger;
public string Exec { get; }
public DotNetCliInvoker(ProgressMonitor progressMonitor, string exec)
public DotNetCliInvoker(ILogger logger, string exec)
{
this.progressMonitor = progressMonitor;
this.logger = logger;
this.Exec = exec;
}
@@ -35,15 +36,15 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private bool RunCommandAux(string args, out IList<string> output)
{
progressMonitor.LogInfo($"Running {Exec} {args}");
logger.LogInfo($"Running {Exec} {args}");
var pi = MakeDotnetStartInfo(args);
var threadId = Environment.CurrentManagedThreadId;
void onOut(string s) => progressMonitor.LogInfo(s, threadId);
void onError(string s) => progressMonitor.LogError(s, threadId);
void onOut(string s) => logger.LogInfo(s, threadId);
void onError(string s) => logger.LogError(s, threadId);
var exitCode = pi.ReadOutput(out output, onOut, onError);
if (exitCode != 0)
{
progressMonitor.LogError($"Command {Exec} {args} failed with exit code {exitCode}");
logger.LogError($"Command {Exec} {args} failed with exit code {exitCode}");
return false;
}
return true;

13
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs
View File

@@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.IO;
using System.Text.RegularExpressions;
using Semmle.Util;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
@@ -15,7 +16,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
// </summary>
internal partial class FileContent
{
private readonly ProgressMonitor progressMonitor;
private readonly ILogger logger;
private readonly IUnsafeFileReader unsafeFileReader;
private readonly IEnumerable<string> files;
private readonly HashSet<string> allPackages = new HashSet<string>();
@@ -90,18 +91,18 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
}
internal FileContent(ProgressMonitor progressMonitor,
internal FileContent(ILogger logger,
IEnumerable<string> files,
IUnsafeFileReader unsafeFileReader)
{
this.progressMonitor = progressMonitor;
this.logger = logger;
this.files = files;
this.unsafeFileReader = unsafeFileReader;
this.initialize = new Initializer(DoInitialize);
}
public FileContent(ProgressMonitor progressMonitor, IEnumerable<string> files) : this(progressMonitor, files, new UnsafeFileReader())
public FileContent(ILogger logger, IEnumerable<string> files) : this(logger, files, new UnsafeFileReader())
{ }
private static string GetGroup(ReadOnlySpan<char> input, ValueMatch valueMatch, string groupPrefix, bool toLower)
@@ -192,8 +193,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
catch (Exception ex)
{
progressMonitor.LogInfo($"Failed to read file {file}");
progressMonitor.LogDebug($"Failed to read file {file}, exception: {ex}");
logger.LogInfo($"Failed to read file {file}");
logger.LogDebug($"Failed to read file {file}, exception: {ex}");
}
}
}

5
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileInfoExtensions.cs
View File

@@ -2,6 +2,7 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
@@ -13,14 +14,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
public static IEnumerable<FileInfo> SelectRootFiles(this IEnumerable<FileInfo> files, DirectoryInfo dir) =>
files.Where(file => file.DirectoryName == dir.FullName);
internal static IEnumerable<FileInfo> SelectSmallFiles(this IEnumerable<FileInfo> files, ProgressMonitor progressMonitor)
internal static IEnumerable<FileInfo> SelectSmallFiles(this IEnumerable<FileInfo> files, ILogger logger)
{
const int oneMb = 1_048_576;
return files.Where(file =>
{
if (file.Length > oneMb)
{
progressMonitor.LogDebug($"Skipping {file.FullName} because it is bigger than 1MB.");
logger.LogDebug($"Skipping {file.FullName} because it is bigger than 1MB.");
return false;
}
return true;

12
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FilePathFilter.cs
View File

@@ -12,12 +12,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
public class FilePathFilter
{
private readonly string rootFolder;
private readonly IProgressMonitor progressMonitor;
private readonly ILogger logger;
public FilePathFilter(DirectoryInfo sourceDir, IProgressMonitor progressMonitor)
public FilePathFilter(DirectoryInfo sourceDir, ILogger logger)
{
rootFolder = FileUtils.ConvertToUnix(sourceDir.FullName.ToLowerInvariant());
this.progressMonitor = progressMonitor;
this.logger = logger;
}
private class FileInclusion(string path, bool include)
@@ -55,12 +55,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
else
{
progressMonitor.Log(Severity.Info, $"Invalid filter: {filter}");
logger.Log(Severity.Info, $"Invalid filter: {filter}");
continue;
}
var regex = new FilePattern(filterText).RegexPattern;
progressMonitor.Log(Severity.Info, $"Filtering {(include ? "in" : "out")} files matching '{regex}'. Original glob filter: '{filter}'");
logger.Log(Severity.Info, $"Filtering {(include ? "in" : "out")} files matching '{regex}'. Original glob filter: '{filter}'");
pathFilters.Add(new PathFilter(new Regex(regex, RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline), include));
}
@@ -91,7 +91,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (!include)
{
progressMonitor.Log(Severity.Info, $"Excluding '{f.FileInfo.FullName}'");
logger.Log(Severity.Info, $"Excluding '{f.FileInfo.FullName}'");
}
return include;

29
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackages.cs
View File

@@ -2,6 +2,7 @@
using System.Diagnostics;
using System.IO;
using System.Linq;
using Microsoft.Build.Framework;
using Semmle.Util;
namespace Semmle.Extraction.CSharp.DependencyFetching
@@ -14,7 +15,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
internal class NugetPackages
{
private readonly string? nugetExe;
private readonly ProgressMonitor progressMonitor;
private readonly Util.Logging.ILogger logger;
/// <summary>
/// The list of package files.
@@ -31,10 +32,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// <summary>
/// Create the package manager for a specified source tree.
/// </summary>
public NugetPackages(string sourceDir, TemporaryDirectory packageDirectory, ProgressMonitor progressMonitor)
public NugetPackages(string sourceDir, TemporaryDirectory packageDirectory, Util.Logging.ILogger logger)
{
this.packageDirectory = packageDirectory;
this.progressMonitor = progressMonitor;
this.logger = logger;
packageFiles = new DirectoryInfo(sourceDir)
.EnumerateFiles("packages.config", SearchOption.AllDirectories)
@@ -42,12 +43,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (packageFiles.Length > 0)
{
progressMonitor.LogInfo($"Found {packageFiles.Length} packages.config files, trying to use nuget.exe for package restore");
logger.LogInfo($"Found {packageFiles.Length} packages.config files, trying to use nuget.exe for package restore");
nugetExe = ResolveNugetExe(sourceDir);
}
else
{
progressMonitor.LogInfo("Found no packages.config file");
logger.LogInfo("Found no packages.config file");
}
}
@@ -66,7 +67,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var nuget = Path.Combine(directory, "nuget", "nuget.exe");
if (File.Exists(nuget))
{
progressMonitor.LogInfo($"Found nuget.exe at {nuget}");
logger.LogInfo($"Found nuget.exe at {nuget}");
return nuget;
}
@@ -81,16 +82,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
// Nuget.exe already exists in the .nuget directory.
if (File.Exists(nuget))
{
progressMonitor.LogInfo($"Found nuget.exe at {nuget}");
logger.LogInfo($"Found nuget.exe at {nuget}");
return nuget;
}
Directory.CreateDirectory(directory);
progressMonitor.LogInfo("Attempting to download nuget.exe");
logger.LogInfo("Attempting to download nuget.exe");
try
{
FileUtils.DownloadFile(FileUtils.NugetExeUrl, nuget);
progressMonitor.LogInfo($"Downloaded nuget.exe to {nuget}");
logger.LogInfo($"Downloaded nuget.exe to {nuget}");
return nuget;
}
catch
@@ -106,7 +107,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// <param name="package">The package file.</param>
private void RestoreNugetPackage(string package)
{
progressMonitor.LogInfo($"Restoring file {package}...");
logger.LogInfo($"Restoring file {package}...");
/* Use nuget.exe to install a package.
* Note that there is a clutch of NuGet assemblies which could be used to
@@ -134,16 +135,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
};
var threadId = Environment.CurrentManagedThreadId;
void onOut(string s) => progressMonitor.LogInfo(s, threadId);
void onError(string s) => progressMonitor.LogError(s, threadId);
void onOut(string s) => logger.LogInfo(s, threadId);
void onError(string s) => logger.LogError(s, threadId);
var exitCode = pi.ReadOutput(out var _, onOut, onError);
if (exitCode != 0)
{
progressMonitor.LogError($"Command {pi.FileName} {pi.Arguments} failed with exit code {exitCode}");
logger.LogError($"Command {pi.FileName} {pi.Arguments} failed with exit code {exitCode}");
}
else
{
progressMonitor.LogInfo($"Restored file {package}");
logger.LogInfo($"Restored file {package}");
}
}

26
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/ProgressMonitor.cs
View File

@@ -1,26 +0,0 @@
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal class ProgressMonitor : IProgressMonitor
{
private readonly ILogger logger;
public ProgressMonitor(ILogger logger)
{
this.logger = logger;
}
public void Log(Severity severity, string message) =>
logger.Log(severity, message);
public void LogInfo(string message, int? threadId = null) =>
logger.Log(Severity.Info, message, threadId);
public void LogDebug(string message) =>
logger.Log(Severity.Debug, message);
public void LogError(string message, int? threadId = null) =>
logger.Log(Severity.Error, message, threadId);
}
}

21
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Razor.cs
View File

@@ -4,36 +4,37 @@ using System.IO;
using System.Text;
using System.Linq;
using Semmle.Util;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal class Razor
{
private readonly DotNetVersion sdk;
private readonly ProgressMonitor progressMonitor;
private readonly ILogger logger;
private readonly IDotNet dotNet;
private readonly string sourceGeneratorFolder;
private readonly string cscPath;
public Razor(DotNetVersion sdk, IDotNet dotNet, ProgressMonitor progressMonitor)
public Razor(DotNetVersion sdk, IDotNet dotNet, ILogger logger)
{
this.sdk = sdk;
this.progressMonitor = progressMonitor;
this.logger = logger;
this.dotNet = dotNet;
sourceGeneratorFolder = Path.Combine(this.sdk.FullPath, "Sdks", "Microsoft.NET.Sdk.Razor", "source-generators");
this.progressMonitor.LogInfo($"Razor source generator folder: {sourceGeneratorFolder}");
this.logger.LogInfo($"Razor source generator folder: {sourceGeneratorFolder}");
if (!Directory.Exists(sourceGeneratorFolder))
{
this.progressMonitor.LogInfo($"Razor source generator folder {sourceGeneratorFolder} does not exist.");
this.logger.LogInfo($"Razor source generator folder {sourceGeneratorFolder} does not exist.");
throw new Exception($"Razor source generator folder {sourceGeneratorFolder} does not exist.");
}
cscPath = Path.Combine(this.sdk.FullPath, "Roslyn", "bincore", "csc.dll");
this.progressMonitor.LogInfo($"Razor source generator CSC: {cscPath}");
this.logger.LogInfo($"Razor source generator CSC: {cscPath}");
if (!File.Exists(cscPath))
{
this.progressMonitor.LogInfo($"Csc.exe not found at {cscPath}.");
this.logger.LogInfo($"Csc.exe not found at {cscPath}.");
throw new Exception($"csc.dll {cscPath} does not exist.");
}
}
@@ -65,7 +66,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
GenerateAnalyzerConfig(cshtmls, analyzerConfig);
progressMonitor.LogInfo($"Analyzer config content: {File.ReadAllText(analyzerConfig)}");
logger.LogInfo($"Analyzer config content: {File.ReadAllText(analyzerConfig)}");
var args = new StringBuilder();
args.Append($"/target:exe /generatedfilesout:\"{outputFolder}\" /out:\"{dllPath}\" /analyzerconfig:\"{analyzerConfig}\" ");
@@ -87,7 +88,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var argsString = args.ToString();
progressMonitor.LogInfo($"Running CSC to generate Razor source files with arguments: {argsString}.");
logger.LogInfo($"Running CSC to generate Razor source files with arguments: {argsString}.");
using (var sw = new StreamWriter(cscArgsPath))
{
@@ -98,7 +99,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var files = Directory.GetFiles(outputFolder, "*.*", new EnumerationOptions { RecurseSubdirectories = true });
progressMonitor.LogInfo($"Generated {files.Length} source files from cshtml files.");
logger.LogInfo($"Generated {files.Length} source files from cshtml files.");
return files;
}

16
csharp/extractor/Semmle.Extraction.Tests/Assets.cs
View File

@@ -12,7 +12,7 @@ namespace Semmle.Extraction.Tests
public void TestAssets1()
{
// Setup
var assets = new Assets(new ProgressMonitor(new LoggerStub()));
var assets = new Assets(new LoggerStub());
var json = assetsJson1;
var dependencies = new DependencyContainer();
@@ -43,7 +43,7 @@ namespace Semmle.Extraction.Tests
public void TestAssetsFailure()
{
// Setup
var assets = new Assets(new ProgressMonitor(new LoggerStub()));
var assets = new Assets(new LoggerStub());
var json = "garbage data";
var dependencies = new DependencyContainer();
@@ -59,7 +59,7 @@ namespace Semmle.Extraction.Tests
public void TestAssetsNet70()
{
// Setup
var assets = new Assets(new ProgressMonitor(new LoggerStub()));
var assets = new Assets(new LoggerStub());
var json = assetsNet70;
var dependencies = new DependencyContainer();
@@ -90,7 +90,7 @@ namespace Semmle.Extraction.Tests
public void TestAssetsNet48()
{
// Setup
var assets = new Assets(new ProgressMonitor(new LoggerStub()));
var assets = new Assets(new LoggerStub());
var json = assetsNet48;
var dependencies = new DependencyContainer();
@@ -117,7 +117,7 @@ namespace Semmle.Extraction.Tests
public void TestAssetsNetstandard21()
{
// Setup
var assets = new Assets(new ProgressMonitor(new LoggerStub()));
var assets = new Assets(new LoggerStub());
var json = assetsNetstandard21;
var dependencies = new DependencyContainer();
@@ -145,7 +145,7 @@ namespace Semmle.Extraction.Tests
public void TestAssetsNetStandard16()
{
// Setup
var assets = new Assets(new ProgressMonitor(new LoggerStub()));
var assets = new Assets(new LoggerStub());
var json = assetsNetstandard16;
var dependencies = new DependencyContainer();
@@ -177,7 +177,7 @@ namespace Semmle.Extraction.Tests
public void TestAssetsNetcoreapp20()
{
// Setup
var assets = new Assets(new ProgressMonitor(new LoggerStub()));
var assets = new Assets(new LoggerStub());
var json = assetsNetcoreapp20;
var dependencies = new DependencyContainer();
@@ -205,7 +205,7 @@ namespace Semmle.Extraction.Tests
public void TestAssetsNetcoreapp31()
{
// Setup
var assets = new Assets(new ProgressMonitor(new LoggerStub()));
var assets = new Assets(new LoggerStub());
var json = assetsNetcoreapp31;
var dependencies = new DependencyContainer();

2
csharp/extractor/Semmle.Extraction.Tests/DotNet.cs
View File

@@ -38,7 +38,7 @@ namespace Semmle.Extraction.Tests
public class DotNetTests
{
private static IDotNet MakeDotnet(IDotNetCliInvoker dotnetCliInvoker) =>
DotNet.Make(dotnetCliInvoker, new ProgressMonitor(new LoggerStub()));
DotNet.Make(dotnetCliInvoker, new LoggerStub());
private static IList<string> MakeDotnetRestoreOutput() =>
new List<string> {

2
csharp/extractor/Semmle.Extraction.Tests/FileContent.cs
View File

@@ -25,7 +25,7 @@ namespace Semmle.Extraction.Tests
internal class TestFileContent : FileContent
{
public TestFileContent(IEnumerable<string> lines) : base(new ProgressMonitor(new LoggerStub()),
public TestFileContent(IEnumerable<string> lines) : base(new LoggerStub(),
new List<string>() { "test1.cs" },
new UnsafeFileReaderStub(lines))
{ }

48
csharp/extractor/Semmle.Extraction.Tests/FilePathFilter.cs
View File

@@ -10,17 +10,19 @@ namespace Semmle.Extraction.Tests
{
public class FilePathFilterTest
{
private class ProgressMonitorStub : IProgressMonitor
private class LoggerStub : ILogger
{
public List<string> Messages { get; } = [];
public void Log(Severity severity, string message)
public void Log(Severity s, string text, int? threadId = null)
{
Messages.Add(message);
}
Messages.Add(text);
}
private static (FilePathFilter TestSubject, ProgressMonitorStub progressMonitor, IEnumerable<FileInfo> Files) TestSetup()
public void Dispose() { }
}
private static (FilePathFilter TestSubject, LoggerStub Logger, IEnumerable<FileInfo> Files) TestSetup()
{
return TestSetup("/a/b",
[
@@ -32,15 +34,15 @@ namespace Semmle.Extraction.Tests
]);
}
private static (FilePathFilter TestSubject, ProgressMonitorStub progressMonitor, IEnumerable<FileInfo> Files) TestSetup(string root, IEnumerable<string> paths)
private static (FilePathFilter TestSubject, LoggerStub Logger, IEnumerable<FileInfo> Files) TestSetup(string root, IEnumerable<string> paths)
{
root = GetPlatformSpecifixPath(root);
paths = GetPlatformSpecifixPaths(paths);
var progressMonitor = new ProgressMonitorStub();
var logger = new LoggerStub();
var filePathFilter = new FilePathFilter(new DirectoryInfo(root), progressMonitor);
return (filePathFilter, progressMonitor, paths.Select(p => new FileInfo(p)));
var filePathFilter = new FilePathFilter(new DirectoryInfo(root), logger);
return (filePathFilter, logger, paths.Select(p => new FileInfo(p)));
}
private static string GetPlatformSpecifixPath(string file)
@@ -67,20 +69,20 @@ namespace Semmle.Extraction.Tests
[Fact]
public void TestNoFilter()
{
(var testSubject, var progressMonitor, var files) = TestSetup();
(var testSubject, var logger, var files) = TestSetup();
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", null);
var filtered = testSubject.Filter(files);
AssertFileInfoEquivalence(files, filtered);
Assert.Equivalent(Array.Empty<string>(), progressMonitor.Messages, strict: true);
Assert.Equivalent(Array.Empty<string>(), logger.Messages, strict: true);
}
[Fact]
public void TestFiltersWithOnlyInclude()
{
(var testSubject, var progressMonitor, var files) = TestSetup();
(var testSubject, var logger, var files) = TestSetup();
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
include:c/d
@@ -104,13 +106,13 @@ namespace Semmle.Extraction.Tests
"Filtering in files matching '^c/d.*'. Original glob filter: 'include:c/d'",
"Filtering in files matching '^c/x/y.*'. Original glob filter: 'include:c/x/y'"
};
Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false);
Assert.Equivalent(expectedRegexMessages, logger.Messages, strict: false);
}
[Fact]
public void TestFiltersWithOnlyExclude()
{
(var testSubject, var progressMonitor, var files) = TestSetup();
(var testSubject, var logger, var files) = TestSetup();
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
exclude:c/d/e
@@ -130,13 +132,13 @@ namespace Semmle.Extraction.Tests
{
"Filtering out files matching '^c/d/e.*'. Original glob filter: 'exclude:c/d/e'"
};
Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false);
Assert.Equivalent(expectedRegexMessages, logger.Messages, strict: false);
}
[Fact]
public void TestFiltersWithIncludeExclude()
{
(var testSubject, var progressMonitor, var files) = TestSetup();
(var testSubject, var logger, var files) = TestSetup();
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
include:c/x
@@ -157,13 +159,13 @@ namespace Semmle.Extraction.Tests
"Filtering in files matching '^c/x.*'. Original glob filter: 'include:c/x'",
"Filtering out files matching '^c/x/z.*'. Original glob filter: 'exclude:c/x/z'"
};
Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false);
Assert.Equivalent(expectedRegexMessages, logger.Messages, strict: false);
}
[Fact]
public void TestFiltersWithIncludeExcludeExcludeFirst()
{
(var testSubject, var progressMonitor, var files) = TestSetup();
(var testSubject, var logger, var files) = TestSetup();
// NOTE: the ordering DOES matter, later filters takes priority, so the exclude will end up not mattering at all.
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
@@ -186,13 +188,13 @@ namespace Semmle.Extraction.Tests
"Filtering in files matching '^c/x.*'. Original glob filter: 'include:c/x'",
"Filtering out files matching '^c/x/z.*'. Original glob filter: 'exclude:c/x/z'"
};
Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false);
Assert.Equivalent(expectedRegexMessages, logger.Messages, strict: false);
}
[Fact]
public void TestFiltersWithIncludeExcludeComplexPatterns1()
{
(var testSubject, var progressMonitor, var files) = TestSetup();
(var testSubject, var logger, var files) = TestSetup();
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
include:c/**/i.*
@@ -218,13 +220,13 @@ namespace Semmle.Extraction.Tests
"Filtering in files matching '^c/d/.*/[^/]*\\.cs.*'. Original glob filter: 'include:c/d/**/*.cs'",
"Filtering out files matching '^.*/z/i\\.cs.*'. Original glob filter: 'exclude:**/z/i.cs'"
};
Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false);
Assert.Equivalent(expectedRegexMessages, logger.Messages, strict: false);
}
[Fact]
public void TestFiltersWithIncludeExcludeComplexPatterns2()
{
(var testSubject, var progressMonitor, var files) = TestSetup();
(var testSubject, var logger, var files) = TestSetup();
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
include:**/i.*
@@ -245,7 +247,7 @@ namespace Semmle.Extraction.Tests
"Filtering in files matching '^.*/i\\.[^/]*.*'. Original glob filter: 'include:**/i.*'",
"Filtering out files matching '^.*/z/i\\.cs.*'. Original glob filter: 'exclude:**/z/i.cs'"
};
Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false);
Assert.Equivalent(expectedRegexMessages, logger.Messages, strict: false);
}
}
}

8
csharp/extractor/Semmle.Util/Logger.cs
View File

@@ -38,6 +38,14 @@ namespace Semmle.Util.Logging
/// Log the given text with the given severity.
/// </summary>
void Log(Severity s, string text, int? threadId = null);
void LogError(string text, int? threadId = null) => Log(Severity.Error, text, threadId);
void LogWarning(string text, int? threadId = null) => Log(Severity.Warning, text, threadId);
void LogInfo(string text, int? threadId = null) => Log(Severity.Info, text, threadId);
void LogDebug(string text, int? threadId = null) => Log(Severity.Debug, text, threadId);
}
public static class LoggerExtensions