hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-16 16:53:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #15424 from tamasvajk/standalone/logging

Browse Source 
C#: Improve log messages in buildless mode + some cleanup/refactoring
This commit is contained in:
Tamás Vajk
2024-01-25 11:08:50 +01:00
committed by GitHub
parent 930f1b50b9 199b0578be
commit 01b89508a8
22 changed files with 395 additions and 524 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs
View File

@@ -1,6 +1,7 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
@@ -18,25 +19,26 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// Paths to search. Directories are searched recursively. Files are added directly to the
/// assembly cache.
/// </param>
/// <param name="progressMonitor">Callback for progress.</param>
public AssemblyCache(IEnumerable<string> paths, IEnumerable<string> frameworkPaths, ProgressMonitor progressMonitor)
/// <param name="logger">Callback for progress.</param>
public AssemblyCache(IEnumerable<string> paths, IEnumerable<string> frameworkPaths, ILogger logger)
{
this.logger = logger;
foreach (var path in paths)
{
if (File.Exists(path))
{
pendingDllsToIndex.Enqueue(path);
dllsToIndex.Add(path);
continue;
}
if (Directory.Exists(path))
{
progressMonitor.FindingFiles(path);
logger.LogInfo($"Finding reference DLLs in {path}...");
AddReferenceDirectory(path);
}
else
{
progressMonitor.LogInfo("AssemblyCache: Path not found: " + path);
logger.LogInfo("AssemblyCache: Path not found: " + path);
}
}
IndexReferences(frameworkPaths);
@@ -52,7 +54,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
foreach (var dll in new DirectoryInfo(dir).EnumerateFiles("*.dll", SearchOption.AllDirectories))
{
pendingDllsToIndex.Enqueue(dll.FullName);
dllsToIndex.Add(dll.FullName);
}
}
@@ -62,12 +64,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// </summary>
private void IndexReferences(IEnumerable<string> frameworkPaths)
{
logger.LogInfo($"Indexing {dllsToIndex.Count} assemblies...");
// Read all of the files
foreach (var filename in pendingDllsToIndex)
foreach (var filename in dllsToIndex)
{
IndexReference(filename);
}
logger.LogInfo($"Read {assemblyInfoByFileName.Count} assembly infos");
foreach (var info in assemblyInfoByFileName.Values
.OrderBy(info => info.Name)
.OrderAssemblyInfosByPreference(frameworkPaths))
@@ -83,25 +89,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
try
{
logger.LogDebug($"Reading assembly info from {filename}");
var info = AssemblyInfo.ReadFromFile(filename);
assemblyInfoByFileName[filename] = info;
}
catch (AssemblyLoadException)
{
failedAssemblyInfoFileNames.Add(filename);
logger.LogInfo($"Couldn't read assembly info from {filename}");
}
}
/// <summary>
/// The number of DLLs which are assemblies.
/// </summary>
public int AssemblyCount => assemblyInfoByFileName.Count;
/// <summary>
/// The number of DLLs which weren't assemblies. (E.g. C++).
/// </summary>
public int NonAssemblyCount => failedAssemblyInfoFileNames.Count;
/// <summary>
/// Given an assembly id, determine its full info.
/// </summary>
@@ -113,8 +110,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (failedAssemblyInfoIds.Contains(id))
throw new AssemblyLoadException();
string assemblyName;
(id, assemblyName) = AssemblyInfo.ComputeSanitizedAssemblyInfo(id);
(id, var assemblyName) = AssemblyInfo.ComputeSanitizedAssemblyInfo(id);
// Look up the id in our references map.
if (assemblyInfoById.TryGetValue(id, out var result))
@@ -164,17 +160,15 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
throw new AssemblyLoadException();
}
private readonly Queue<string> pendingDllsToIndex = new Queue<string>();
private readonly List<string> dllsToIndex = new List<string>();
private readonly Dictionary<string, AssemblyInfo> assemblyInfoByFileName = new Dictionary<string, AssemblyInfo>();
// List of DLLs which are not assemblies.
// We probably don't need to keep this
private readonly List<string> failedAssemblyInfoFileNames = new List<string>();
// Map from assembly id (in various formats) to the full info.
private readonly Dictionary<string, AssemblyInfo> assemblyInfoById = new Dictionary<string, AssemblyInfo>();
private readonly HashSet<string> failedAssemblyInfoIds = new HashSet<string>();
private readonly ILogger logger;
}
}

15
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyInfo.cs
View File

@@ -120,21 +120,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
NetCoreVersion = netCoreVersion;
}
/// <summary>
/// Get AssemblyInfo from a loaded Assembly.
/// </summary>
/// <param name="assembly">The assembly.</param>
/// <returns>Info about the assembly.</returns>
public static AssemblyInfo MakeFromAssembly(Assembly assembly)
{
if (assembly.FullName is null)
{
throw new InvalidOperationException("Assembly with empty full name is not expected.");
}
return new AssemblyInfo(assembly.FullName, assembly.Location);
}
/// <summary>
/// Returns the id and name of the assembly that would be created from the received id.
/// </summary>

25
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Assets.cs
View File

@@ -5,6 +5,7 @@ using System.IO;
using System.Linq;
using Newtonsoft.Json.Linq;
using Semmle.Util;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
@@ -13,11 +14,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// </summary>
internal class Assets
{
private readonly ProgressMonitor progressMonitor;
private readonly ILogger logger;
internal Assets(ProgressMonitor progressMonitor)
internal Assets(ILogger logger)
{
this.progressMonitor = progressMonitor;
this.logger = logger;
}
/// <summary>
@@ -85,7 +86,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (references is null)
{
progressMonitor.LogDebug("No references found in the targets section in the assets file.");
logger.LogDebug("No references found in the targets section in the assets file.");
return;
}
@@ -157,7 +158,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (frameworks is null)
{
progressMonitor.LogDebug("No framework section in assets.json.");
logger.LogDebug("No framework section in assets.json.");
return;
}
@@ -171,7 +172,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (references is null)
{
progressMonitor.LogDebug("No framework references in assets.json.");
logger.LogDebug("No framework references in assets.json.");
return;
}
@@ -196,12 +197,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
catch (Exception e)
{
progressMonitor.LogDebug($"Failed to parse assets file (unexpected error): {e.Message}");
logger.LogDebug($"Failed to parse assets file (unexpected error): {e.Message}");
return false;
}
}
private static bool TryReadAllText(string path, ProgressMonitor progressMonitor, [NotNullWhen(returnValue: true)] out string? content)
private static bool TryReadAllText(string path, ILogger logger, [NotNullWhen(returnValue: true)] out string? content)
{
try
{
@@ -210,19 +211,19 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
catch (Exception e)
{
progressMonitor.LogInfo($"Failed to read assets file '{path}': {e.Message}");
logger.LogInfo($"Failed to read assets file '{path}': {e.Message}");
content = null;
return false;
}
}
public static DependencyContainer GetCompilationDependencies(ProgressMonitor progressMonitor, IEnumerable<string> assets)
public static DependencyContainer GetCompilationDependencies(ILogger logger, IEnumerable<string> assets)
{
var parser = new Assets(progressMonitor);
var parser = new Assets(logger);
var dependencies = new DependencyContainer();
assets.ForEach(asset =>
{
if (TryReadAllText(asset, progressMonitor, out var json))
if (TryReadAllText(asset, logger, out var json))
{
parser.TryParse(json, dependencies);
}

299
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
View File

@@ -17,14 +17,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
public sealed class DependencyManager : IDisposable
{
private readonly AssemblyCache assemblyCache;
private readonly ProgressMonitor progressMonitor;
private readonly ILogger logger;
// Only used as a set, but ConcurrentDictionary is the only concurrent set in .NET.
private readonly IDictionary<string, bool> usedReferences = new ConcurrentDictionary<string, bool>();
private readonly IDictionary<string, bool> sources = new ConcurrentDictionary<string, bool>();
private readonly IDictionary<string, string> unresolvedReferences = new ConcurrentDictionary<string, string>();
private int failedProjects;
private int succeededProjects;
private readonly List<string> nonGeneratedSources;
private readonly List<string> generatedSources;
private int dotnetFrameworkVersionVariantCount = 0;
private int conflictedReferences = 0;
private readonly IDependencyOptions options;
private readonly DirectoryInfo sourceDir;
@@ -49,7 +49,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var startTime = DateTime.Now;
this.options = options;
this.progressMonitor = new ProgressMonitor(logger);
this.logger = logger;
this.sourceDir = new DirectoryInfo(srcDir);
packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName, "packages"));
@@ -60,35 +60,36 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
try
{
this.dotnet = DotNet.Make(options, progressMonitor, tempWorkingDirectory);
this.dotnet = DotNet.Make(options, logger, tempWorkingDirectory);
runtimeLazy = new Lazy<Runtime>(() => new Runtime(dotnet));
}
catch
{
progressMonitor.MissingDotNet();
logger.LogError("Missing dotnet CLI");
throw;
}
this.progressMonitor.FindingFiles(srcDir);
logger.LogInfo($"Finding files in {srcDir}...");
var allFiles = GetAllFiles().ToList();
var binaryFileExtensions = new HashSet<string>(new[] { ".dll", ".exe" }); // TODO: add more binary file extensions.
var allNonBinaryFiles = allFiles.Where(f => !binaryFileExtensions.Contains(f.Extension.ToLowerInvariant())).ToList();
var smallNonBinaryFiles = allNonBinaryFiles.SelectSmallFiles(progressMonitor).SelectFileNames();
this.fileContent = new FileContent(progressMonitor, smallNonBinaryFiles);
var smallNonBinaryFiles = allNonBinaryFiles.SelectSmallFiles(logger).SelectFileNames();
this.fileContent = new FileContent(logger, smallNonBinaryFiles);
this.nonGeneratedSources = allNonBinaryFiles.SelectFileNamesByExtension(".cs").ToList();
this.generatedSources = new();
var allProjects = allNonBinaryFiles.SelectFileNamesByExtension(".csproj");
var allSolutions = allNonBinaryFiles.SelectFileNamesByExtension(".sln");
var allProjects = allNonBinaryFiles.SelectFileNamesByExtension(".csproj").ToList();
var allSolutions = allNonBinaryFiles.SelectFileNamesByExtension(".sln").ToList();
var dllPaths = allFiles.SelectFileNamesByExtension(".dll").ToHashSet();
logger.LogInfo($"Found {allFiles.Count} files, {nonGeneratedSources.Count} source files, {allProjects.Count} project files, {allSolutions.Count} solution files, {dllPaths.Count} DLLs.");
RestoreNugetPackages(allNonBinaryFiles, allProjects, allSolutions, dllPaths);
// Find DLLs in the .Net / Asp.Net Framework
// This needs to come after the nuget restore, because the nuget restore might fetch the .NET Core/Framework reference assemblies.
var frameworkLocations = AddFrameworkDlls(dllPaths);
assemblyCache = new AssemblyCache(dllPaths, frameworkLocations, progressMonitor);
assemblyCache = new AssemblyCache(dllPaths, frameworkLocations, logger);
AnalyseSolutions(allSolutions);
foreach (var filename in assemblyCache.AllAssemblies.Select(a => a.Filename))
@@ -102,33 +103,36 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
// Output the findings
foreach (var r in usedReferences.Keys.OrderBy(r => r))
{
progressMonitor.ResolvedReference(r);
logger.LogInfo($"Resolved reference {r}");
}
foreach (var r in unresolvedReferences.OrderBy(r => r.Key))
{
progressMonitor.UnresolvedReference(r.Key, r.Value);
logger.LogInfo($"Unresolved reference {r.Key} in project {r.Value}");
}
var webViewExtractionOption = Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_STANDALONE_EXTRACT_WEB_VIEWS");
if (bool.TryParse(webViewExtractionOption, out var shouldExtractWebViews) &&
shouldExtractWebViews)
{
logger.LogInfo("Generating source files from cshtml and razor files...");
GenerateSourceFilesFromWebViews(allNonBinaryFiles);
}
GenerateSourceFileFromImplicitUsings();
progressMonitor.Summary(
AllSourceFiles.Count(),
ProjectSourceFiles.Count(),
MissingSourceFiles.Count(),
ReferenceFiles.Count(),
UnresolvedReferences.Count(),
conflictedReferences,
succeededProjects + failedProjects,
failedProjects,
DateTime.Now - startTime);
const int align = 6;
logger.LogInfo("");
logger.LogInfo("Build analysis summary:");
logger.LogInfo($"{nonGeneratedSources.Count,align} source files in the filesystem");
logger.LogInfo($"{generatedSources.Count,align} generated source files");
logger.LogInfo($"{allSolutions.Count,align} solution files");
logger.LogInfo($"{allProjects.Count,align} project files in the filesystem");
logger.LogInfo($"{usedReferences.Keys.Count,align} resolved references");
logger.LogInfo($"{unresolvedReferences.Count,align} unresolved references");
logger.LogInfo($"{conflictedReferences,align} resolved assembly conflicts");
logger.LogInfo($"{dotnetFrameworkVersionVariantCount,align} restored .NET framework variants");
logger.LogInfo($"Build analysis completed in {DateTime.Now - startTime}");
}
private HashSet<string> AddFrameworkDlls(HashSet<string> dllPaths)
@@ -146,32 +150,42 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
try
{
var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, progressMonitor);
var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, logger);
nuget.InstallPackages();
var nugetPackageDlls = legacyPackageDirectory.DirInfo.GetFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = true });
var nugetPackageDllPaths = nugetPackageDlls.Select(f => f.FullName).ToHashSet();
var excludedPaths = nugetPackageDllPaths
.Where(path => IsPathInSubfolder(path, legacyPackageDirectory.DirInfo.FullName, "tools"));
.Where(path => IsPathInSubfolder(path, legacyPackageDirectory.DirInfo.FullName, "tools"))
.ToList();
if (nugetPackageDllPaths.Count > 0)
{
logger.LogInfo($"Restored {nugetPackageDllPaths.Count} Nuget DLLs.");
}
if (excludedPaths.Count > 0)
{
logger.LogInfo($"Excluding {excludedPaths.Count} Nuget DLLs.");
}
foreach (var excludedPath in excludedPaths)
{
progressMonitor.LogInfo($"Excluded Nuget DLL: {excludedPath}");
logger.LogInfo($"Excluded Nuget DLL: {excludedPath}");
}
nugetPackageDllPaths.ExceptWith(excludedPaths);
dllPaths.UnionWith(nugetPackageDllPaths);
}
catch (FileNotFoundException)
catch (Exception)
{
progressMonitor.MissingNuGet();
logger.LogError("Failed to restore Nuget packages with nuget.exe");
}
var restoredProjects = RestoreSolutions(allSolutions, out var assets1);
var projects = allProjects.Except(restoredProjects);
RestoreProjects(projects, out var assets2);
var dependencies = Assets.GetCompilationDependencies(progressMonitor, assets1.Union(assets2));
var dependencies = Assets.GetCompilationDependencies(logger, assets1.Union(assets2));
var paths = dependencies
.Paths
@@ -220,7 +234,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (isInAnalyzersFolder)
{
usedReferences.Remove(filename);
progressMonitor.RemovedReference(filename);
logger.LogInfo($"Removed analyzer reference {filename}");
}
}
}
@@ -228,27 +242,31 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private void SelectNewestFrameworkPath(string frameworkPath, string frameworkType, ISet<string> dllPaths, ISet<string> frameworkLocations)
{
var versionFolders = new DirectoryInfo(frameworkPath)
.EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
.OrderByDescending(d => d.Name) // TODO: Improve sorting to handle pre-release versions.
.ToArray();
var versionFolders = GetPackageVersionSubDirectories(frameworkPath);
if (versionFolders.Length > 1)
{
var versions = string.Join(", ", versionFolders.Select(d => d.Name));
progressMonitor.LogInfo($"Found multiple {frameworkType} DLLs in NuGet packages at {frameworkPath}. Using the latest version ({versionFolders[0].Name}) from: {versions}.");
logger.LogInfo($"Found multiple {frameworkType} DLLs in NuGet packages at {frameworkPath}. Using the latest version ({versionFolders[0].Name}) from: {versions}.");
}
var selectedFrameworkFolder = versionFolders.FirstOrDefault()?.FullName;
if (selectedFrameworkFolder is null)
{
progressMonitor.LogInfo($"Found {frameworkType} DLLs in NuGet packages at {frameworkPath}, but no version folder was found.");
logger.LogInfo($"Found {frameworkType} DLLs in NuGet packages at {frameworkPath}, but no version folder was found.");
selectedFrameworkFolder = frameworkPath;
}
dllPaths.Add(selectedFrameworkFolder);
frameworkLocations.Add(selectedFrameworkFolder);
progressMonitor.LogInfo($"Found {frameworkType} DLLs in NuGet packages at {selectedFrameworkFolder}. Not adding installation directory.");
logger.LogInfo($"Found {frameworkType} DLLs in NuGet packages at {selectedFrameworkFolder}.");
}
private static DirectoryInfo[] GetPackageVersionSubDirectories(string packagePath)
{
return new DirectoryInfo(packagePath)
.EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
.OrderByDescending(d => d.Name) // TODO: Improve sorting to handle pre-release versions.
.ToArray();
}
private void AddNetFrameworkDlls(ISet<string> dllPaths, ISet<string> frameworkLocations)
@@ -257,12 +275,20 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
// The order of the packages is important, we're adding the first one that is present in the nuget cache.
var packagesInPrioOrder = FrameworkPackageNames.NetFrameworks;
var frameworkPath = packagesInPrioOrder
var frameworkPaths = packagesInPrioOrder
.Select((s, index) => (Index: index, Path: GetPackageDirectory(s)))
.FirstOrDefault(pair => pair.Path is not null);
.Where(pair => pair.Path is not null)
.ToArray();
var frameworkPath = frameworkPaths.FirstOrDefault();
if (frameworkPath.Path is not null)
{
foreach (var fp in frameworkPaths)
{
dotnetFrameworkVersionVariantCount += GetPackageVersionSubDirectories(fp.Path!).Length;
}
SelectNewestFrameworkPath(frameworkPath.Path, ".NET Framework", dllPaths, frameworkLocations);
for (var i = frameworkPath.Index + 1; i < packagesInPrioOrder.Length; i++)
@@ -286,7 +312,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
runtimeLocation ??= Runtime.ExecutingRuntime;
progressMonitor.LogInfo($".NET runtime location selected: {runtimeLocation}");
logger.LogInfo($".NET runtime location selected: {runtimeLocation}");
dllPaths.Add(runtimeLocation);
frameworkLocations.Add(runtimeLocation);
}
@@ -300,11 +326,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
var packagePathPrefix = Path.Combine(packageFolder, packagePrefix.ToLowerInvariant());
var toRemove = dllPaths.Where(s => s.ToLowerInvariant().StartsWith(packagePathPrefix));
var toRemove = dllPaths.Where(s => s.StartsWith(packagePathPrefix, StringComparison.InvariantCultureIgnoreCase));
foreach (var path in toRemove)
{
dllPaths.Remove(path);
progressMonitor.RemovedReference(path);
logger.LogInfo($"Removed reference {path}");
}
}
@@ -324,7 +350,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (Runtime.AspNetCoreRuntime is string aspNetCoreRuntime)
{
progressMonitor.LogInfo($"ASP.NET runtime location selected: {aspNetCoreRuntime}");
logger.LogInfo($"ASP.NET runtime location selected: {aspNetCoreRuntime}");
dllPaths.Add(aspNetCoreRuntime);
frameworkLocations.Add(aspNetCoreRuntime);
}
@@ -346,18 +372,26 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
.FullName;
}
private IEnumerable<string> GetAllPackageDirectories()
private ICollection<string> GetAllPackageDirectories()
{
return new DirectoryInfo(packageDirectory.DirInfo.FullName)
.EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
.Select(d => d.Name);
.Select(d => d.Name)
.ToList();
}
private void LogAllUnusedPackages(DependencyContainer dependencies) =>
GetAllPackageDirectories()
private void LogAllUnusedPackages(DependencyContainer dependencies)
{
var allPackageDirectories = GetAllPackageDirectories();
logger.LogInfo($"Restored {allPackageDirectories.Count} packages");
logger.LogInfo($"Found {dependencies.Packages.Count} packages in project.asset.json files");
allPackageDirectories
.Where(package => !dependencies.Packages.Contains(package))
.Order()
.ForEach(package => progressMonitor.LogInfo($"Unused package: {package}"));
.ForEach(package => logger.LogInfo($"Unused package: {package}"));
}
private void GenerateSourceFileFromImplicitUsings()
{
@@ -380,6 +414,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
usings.UnionWith(fileContent.CustomImplicitUsings);
logger.LogInfo($"Generating source file for implicit usings. Namespaces: {string.Join(", ", usings.OrderBy(u => u))}");
if (usings.Count > 0)
{
var tempDir = GetTemporaryWorkingDirectory("implicitUsings");
@@ -401,20 +437,20 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private void GenerateSourceFilesFromWebViews(List<FileInfo> allFiles)
{
progressMonitor.LogInfo($"Generating source files from cshtml and razor files.");
var views = allFiles.SelectFileNamesByExtension(".cshtml", ".razor").ToArray();
if (views.Length > 0)
if (views.Length == 0)
{
progressMonitor.LogInfo($"Found {views.Length} cshtml and razor files.");
return;
}
logger.LogInfo($"Found {views.Length} cshtml and razor files.");
var sdk = new Sdk(dotnet).GetNewestSdk();
if (sdk != null)
{
try
{
var razor = new Razor(sdk, dotnet, progressMonitor);
var razor = new Razor(sdk, dotnet, logger);
var targetDir = GetTemporaryWorkingDirectory("razor");
var generatedFiles = razor.GenerateFiles(views, usedReferences.Keys, targetDir);
this.generatedSources.AddRange(generatedFiles);
@@ -422,8 +458,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
catch (Exception ex)
{
// It's okay, we tried our best to generate source files from cshtml files.
progressMonitor.LogInfo($"Failed to generate source files from cshtml files: {ex.Message}");
}
logger.LogInfo($"Failed to generate source files from cshtml files: {ex.Message}");
}
}
}
@@ -448,17 +483,17 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
return true;
}
progressMonitor.Log(Severity.Warning, $"File {f.FullName} could not be processed.");
logger.Log(Severity.Warning, $"File {f.FullName} could not be processed.");
return false;
}
catch (Exception ex)
{
progressMonitor.Log(Severity.Warning, $"File {f.FullName} could not be processed: {ex.Message}");
logger.Log(Severity.Warning, $"File {f.FullName} could not be processed: {ex.Message}");
return false;
}
});
files = new FilePathFilter(sourceDir, progressMonitor).Filter(files);
files = new FilePathFilter(sourceDir, logger).Filter(files);
return files;
}
@@ -499,7 +534,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// </summary>
private void ResolveConflicts(IEnumerable<string> frameworkPaths)
{
var sortedReferences = new List<AssemblyInfo>();
var sortedReferences = new List<AssemblyInfo>(usedReferences.Count);
foreach (var usedReference in usedReferences)
{
try
@@ -509,7 +544,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
catch (AssemblyLoadException)
{
progressMonitor.Log(Util.Logging.Severity.Warning, $"Could not load assembly information from {usedReference.Key}");
logger.Log(Severity.Warning, $"Could not load assembly information from {usedReference.Key}");
}
}
@@ -517,6 +552,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
.OrderAssemblyInfosByPreference(frameworkPaths)
.ToList();
logger.LogInfo($"Reference list contains {sortedReferences.Count} assemblies");
var finalAssemblyList = new Dictionary<string, AssemblyInfo>();
// Pick the highest version for each assembly name
@@ -532,15 +569,24 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
UseReference(r);
}
logger.LogInfo($"After conflict resolution, reference list contains {finalAssemblyList.Count} assemblies");
// Report the results
foreach (var r in sortedReferences)
{
var resolvedInfo = finalAssemblyList[r.Name];
if (resolvedInfo.Version != r.Version || resolvedInfo.NetCoreVersion != r.NetCoreVersion)
{
progressMonitor.ResolvedConflict(r.Id, resolvedInfo.Id + resolvedInfo.NetCoreVersion is null ? "" : $" (.NET Core {resolvedInfo.NetCoreVersion})");
var asm = resolvedInfo.Id + (resolvedInfo.NetCoreVersion is null ? "" : $" (.NET Core {resolvedInfo.NetCoreVersion})");
logger.LogInfo($"Resolved {r.Id} as {asm}");
++conflictedReferences;
}
if (r != resolvedInfo)
{
logger.LogDebug($"Resolved {r.Id} as {resolvedInfo.Id} from {resolvedInfo.Filename}");
}
}
}
@@ -550,22 +596,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// <param name="reference">The filename of the reference.</param>
private void UseReference(string reference) => usedReferences[reference] = true;
/// <summary>
/// Store that a particular source file is used (by a project file).
/// </summary>
/// <param name="sourceFile">The source file.</param>
private void UseSource(FileInfo sourceFile) => sources[sourceFile.FullName] = sourceFile.Exists;
/// <summary>
/// The list of resolved reference files.
/// </summary>
public IEnumerable<string> ReferenceFiles => usedReferences.Keys;
/// <summary>
/// The list of source files used in projects.
/// </summary>
public IEnumerable<string> ProjectSourceFiles => sources.Where(s => s.Value).Select(s => s.Key);
/// <summary>
/// All of the generated source files in the source directory.
/// </summary>
@@ -581,12 +616,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// </summary>
public IEnumerable<string> UnresolvedReferences => unresolvedReferences.Select(r => r.Key);
/// <summary>
/// List of source files which were mentioned in project files but
/// do not exist on the file system.
/// </summary>
public IEnumerable<string> MissingSourceFiles => sources.Where(s => !s.Value).Select(s => s.Key);
/// <summary>
/// Record that a particular reference couldn't be resolved.
/// Note that this records at most one project file per missing reference.
@@ -595,23 +624,31 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// <param name="projectFile">The project file making the reference.</param>
private void UnresolvedReference(string id, string projectFile) => unresolvedReferences[id] = projectFile;
/// <summary>
/// Reads all the source files and references from the given list of projects.
/// </summary>
/// <param name="projectFiles">The list of projects to analyse.</param>
private void AnalyseProjectFiles(IEnumerable<FileInfo> projectFiles)
private void AnalyseSolutions(IEnumerable<string> solutions)
{
foreach (var proj in projectFiles)
Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, solutionFile =>
{
try
{
var sln = new SolutionFile(solutionFile);
logger.LogInfo($"Analyzing {solutionFile}...");
foreach (var proj in sln.Projects.Select(p => new FileInfo(p)))
{
AnalyseProject(proj);
}
}
catch (Microsoft.Build.Exceptions.InvalidProjectFileException ex)
{
logger.LogInfo($"Couldn't read solution file {solutionFile}: {ex.BaseMessage}");
}
});
}
private void AnalyseProject(FileInfo project)
{
if (!project.Exists)
{
progressMonitor.MissingProject(project.FullName);
logger.LogInfo($"Couldn't read project file {project.FullName}");
return;
}
@@ -631,23 +668,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
UnresolvedReference(@ref, project.FullName);
}
}
foreach (var src in csProj.Sources)
{
// Make a note of which source files the projects use.
// This information doesn't affect the build but is dumped
// as diagnostic output.
UseSource(new FileInfo(src));
}
++succeededProjects;
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{
++failedProjects;
progressMonitor.FailedProjectFile(project.FullName, ex.Message);
logger.LogInfo($"Couldn't read project file {project.FullName}: {ex.Message}");
}
}
/// <summary>
@@ -664,9 +689,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var assetFiles = new List<string>();
var projects = solutions.SelectMany(solution =>
{
dotnet.RestoreSolutionToDirectory(solution, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: true, out var restoredProjects, out var a);
assetFiles.AddRange(a);
return restoredProjects;
logger.LogInfo($"Restoring solution {solution}...");
var res = dotnet.Restore(new(solution, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
assetFiles.AddRange(res.AssetsFilePaths);
return res.RestoredProjects;
});
assets = assetFiles;
return projects;
@@ -683,26 +709,39 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var assetFiles = new List<string>();
Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, project =>
{
dotnet.RestoreProjectToDirectory(project, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: true, out var a, out var _);
assetFiles.AddRange(a);
logger.LogInfo($"Restoring project {project}...");
var res = dotnet.Restore(new(project, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
assetFiles.AddRange(res.AssetsFilePaths);
});
assets = assetFiles;
}
private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPaths)
{
var alreadyDownloadedPackages = Directory.GetDirectories(packageDirectory.DirInfo.FullName)
.Select(d => Path.GetFileName(d).ToLowerInvariant());
var notYetDownloadedPackages = fileContent.AllPackages
.Except(alreadyDownloadedPackages)
.ToList();
if (notYetDownloadedPackages.Count == 0)
{
return;
}
logger.LogInfo($"Found {notYetDownloadedPackages.Count} packages that are not yet restored");
var nugetConfigs = allFiles.SelectFileNamesByName("nuget.config").ToArray();
string? nugetConfig = null;
if (nugetConfigs.Length > 1)
{
progressMonitor.MultipleNugetConfig(nugetConfigs);
logger.LogInfo($"Found multiple nuget.config files: {string.Join(", ", nugetConfigs)}.");
nugetConfig = allFiles
.SelectRootFiles(sourceDir)
.SelectFileNamesByName("nuget.config")
.FirstOrDefault();
if (nugetConfig == null)
{
progressMonitor.NoTopLevelNugetConfig();
logger.LogInfo("Could not find a top-level nuget.config file.");
}
}
else
@@ -710,13 +749,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
nugetConfig = nugetConfigs.FirstOrDefault();
}
var alreadyDownloadedPackages = Directory.GetDirectories(packageDirectory.DirInfo.FullName)
.Select(d => Path.GetFileName(d).ToLowerInvariant());
var notYetDownloadedPackages = fileContent.AllPackages.Except(alreadyDownloadedPackages);
if (nugetConfig != null)
{
logger.LogInfo($"Using nuget.config file {nugetConfig}.");
}
Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, package =>
{
progressMonitor.NugetInstall(package);
logger.LogInfo($"Restoring package {package}...");
using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package, "missingpackages_workingdir"));
var success = dotnet.New(tempDir.DirInfo.FullName);
if (!success)
@@ -730,20 +770,20 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
return;
}
success = dotnet.RestoreProjectToDirectory(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: false, out var _, out var outputLines, pathToNugetConfig: nugetConfig);
if (!success)
var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig));
if (!res.Success)
{
if (outputLines?.Any(s => s.Contains("NU1301")) == true)
if (res.HasNugetPackageSourceError)
{
// Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
success = dotnet.RestoreProjectToDirectory(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: false, out var _, out var _, pathToNugetConfig: null, force: true);
res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true));
}
// TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package.
if (!success)
if (!res.Success)
{
progressMonitor.FailedToRestoreNugetPackage(package);
logger.LogInfo($"Failed to restore nuget package {package}");
}
}
});
@@ -751,23 +791,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
dllPaths.Add(missingPackageDirectory.DirInfo.FullName);
}
private void AnalyseSolutions(IEnumerable<string> solutions)
{
Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, solutionFile =>
{
try
{
var sln = new SolutionFile(solutionFile);
progressMonitor.AnalysingSolution(solutionFile);
AnalyseProjectFiles(sln.Projects.Select(p => new FileInfo(p)).Where(p => p.Exists));
}
catch (Microsoft.Build.Exceptions.InvalidProjectFileException ex)
{
progressMonitor.FailedProjectFile(solutionFile, ex.BaseMessage);
}
});
}
public void Dispose(TemporaryDirectory? dir, string name)
{
try
@@ -776,7 +799,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
catch (Exception exc)
{
progressMonitor.LogInfo($"Couldn't delete {name} directory {exc.Message}");
logger.LogInfo($"Couldn't delete {name} directory {exc.Message}");
}
}

63
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs
View File

@@ -4,6 +4,7 @@ using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using Semmle.Util;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
@@ -13,22 +14,22 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
internal partial class DotNet : IDotNet
{
private readonly IDotNetCliInvoker dotnetCliInvoker;
private readonly ProgressMonitor progressMonitor;
private readonly ILogger logger;
private readonly TemporaryDirectory? tempWorkingDirectory;
private DotNet(IDotNetCliInvoker dotnetCliInvoker, ProgressMonitor progressMonitor, TemporaryDirectory? tempWorkingDirectory = null)
private DotNet(IDotNetCliInvoker dotnetCliInvoker, ILogger logger, TemporaryDirectory? tempWorkingDirectory = null)
{
this.progressMonitor = progressMonitor;
this.logger = logger;
this.tempWorkingDirectory = tempWorkingDirectory;
this.dotnetCliInvoker = dotnetCliInvoker;
Info();
}
private DotNet(IDependencyOptions options, ProgressMonitor progressMonitor, TemporaryDirectory tempWorkingDirectory) : this(new DotNetCliInvoker(progressMonitor, Path.Combine(options.DotNetPath ?? string.Empty, "dotnet")), progressMonitor, tempWorkingDirectory) { }
private DotNet(IDependencyOptions options, ILogger logger, TemporaryDirectory tempWorkingDirectory) : this(new DotNetCliInvoker(logger, Path.Combine(options.DotNetPath ?? string.Empty, "dotnet")), logger, tempWorkingDirectory) { }
internal static IDotNet Make(IDotNetCliInvoker dotnetCliInvoker, ProgressMonitor progressMonitor) => new DotNet(dotnetCliInvoker, progressMonitor);
internal static IDotNet Make(IDotNetCliInvoker dotnetCliInvoker, ILogger logger) => new DotNet(dotnetCliInvoker, logger);
public static IDotNet Make(IDependencyOptions options, ProgressMonitor progressMonitor, TemporaryDirectory tempWorkingDirectory) => new DotNet(options, progressMonitor, tempWorkingDirectory);
public static IDotNet Make(IDependencyOptions options, ILogger logger, TemporaryDirectory tempWorkingDirectory) => new DotNet(options, logger, tempWorkingDirectory);
private void Info()
{
@@ -40,11 +41,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
}
private string GetRestoreArgs(string projectOrSolutionFile, string packageDirectory, bool forceDotnetRefAssemblyFetching)
private string GetRestoreArgs(RestoreSettings restoreSettings)
{
var args = $"restore --no-dependencies \"{projectOrSolutionFile}\" --packages \"{packageDirectory}\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal";
var args = $"restore --no-dependencies \"{restoreSettings.File}\" --packages \"{restoreSettings.PackageDirectory}\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal";
if (forceDotnetRefAssemblyFetching)
if (restoreSettings.ForceDotnetRefAssemblyFetching)
{
// Ugly hack: we set the TargetFrameworkRootPath and NetCoreTargetingPackRoot properties to an empty folder:
var path = ".empty";
@@ -57,46 +58,24 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
args += $" /p:TargetFrameworkRootPath=\"{path}\" /p:NetCoreTargetingPackRoot=\"{path}\"";
}
return args;
if (restoreSettings.PathToNugetConfig != null)
{
args += $" --configfile \"{restoreSettings.PathToNugetConfig}\"";
}
private static IEnumerable<string> GetFirstGroupOnMatch(Regex regex, IEnumerable<string> lines) =>
lines
.Select(line => regex.Match(line))
.Where(match => match.Success)
.Select(match => match.Groups[1].Value);
private static IEnumerable<string> GetAssetsFilePaths(IEnumerable<string> lines) =>
GetFirstGroupOnMatch(AssetsFileRegex(), lines);
private static IEnumerable<string> GetRestoredProjects(IEnumerable<string> lines) =>
GetFirstGroupOnMatch(RestoredProjectRegex(), lines);
public bool RestoreProjectToDirectory(string projectFile, string packageDirectory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> assets, out IList<string> outputLines, string? pathToNugetConfig = null, bool force = false)
{
var args = GetRestoreArgs(projectFile, packageDirectory, forceDotnetRefAssemblyFetching);
if (pathToNugetConfig != null)
{
args += $" --configfile \"{pathToNugetConfig}\"";
}
if (force)
if (restoreSettings.ForceReevaluation)
{
args += " --force";
}
var success = dotnetCliInvoker.RunCommand(args, out outputLines);
assets = success ? GetAssetsFilePaths(outputLines) : Array.Empty<string>();
return success;
return args;
}
public bool RestoreSolutionToDirectory(string solutionFile, string packageDirectory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> projects, out IEnumerable<string> assets)
public RestoreResult Restore(RestoreSettings restoreSettings)
{
var args = GetRestoreArgs(solutionFile, packageDirectory, forceDotnetRefAssemblyFetching);
var args = GetRestoreArgs(restoreSettings);
var success = dotnetCliInvoker.RunCommand(args, out var output);
projects = success ? GetRestoredProjects(output) : Array.Empty<string>();
assets = success ? GetAssetsFilePaths(output) : Array.Empty<string>();
return success;
return new(success, output);
}
public bool New(string folder)
@@ -129,11 +108,5 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var args = $"exec {execArgs}";
return dotnetCliInvoker.RunCommand(args);
}
[GeneratedRegex("Restored\\s+(.+\\.csproj)", RegexOptions.Compiled)]
private static partial Regex RestoredProjectRegex();
[GeneratedRegex("[Assets\\sfile\\shas\\snot\\schanged.\\sSkipping\\sassets\\sfile\\swriting.|Writing\\sassets\\sfile\\sto\\sdisk.]\\sPath:\\s(.+)", RegexOptions.Compiled)]
private static partial Regex AssetsFileRegex();
}
}

23
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs
View File

@@ -2,6 +2,7 @@ using System;
using System.Collections.Generic;
using System.Diagnostics;
using Semmle.Util;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
@@ -10,13 +11,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// </summary>
internal sealed class DotNetCliInvoker : IDotNetCliInvoker
{
private readonly ProgressMonitor progressMonitor;
private readonly ILogger logger;
public string Exec { get; }
public DotNetCliInvoker(ProgressMonitor progressMonitor, string exec)
public DotNetCliInvoker(ILogger logger, string exec)
{
this.progressMonitor = progressMonitor;
this.logger = logger;
this.Exec = exec;
}
@@ -35,21 +36,15 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private bool RunCommandAux(string args, out IList<string> output)
{
progressMonitor.RunningProcess($"{Exec} {args}");
logger.LogInfo($"Running {Exec} {args}");
var pi = MakeDotnetStartInfo(args);
var threadId = $"[{Environment.CurrentManagedThreadId:D3}]";
void onOut(string s)
{
Console.Out.WriteLine($"{threadId} {s}");
}
void onError(string s)
{
Console.Error.WriteLine($"{threadId} {s}");
}
var threadId = Environment.CurrentManagedThreadId;
void onOut(string s) => logger.LogInfo(s, threadId);
void onError(string s) => logger.LogError(s, threadId);
var exitCode = pi.ReadOutput(out output, onOut, onError);
if (exitCode != 0)
{
progressMonitor.CommandFailed(Exec, args, exitCode);
logger.LogError($"Command {Exec} {args} failed with exit code {exitCode}");
return false;
}
return true;

12
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs
View File

@@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.IO;
using System.Text.RegularExpressions;
using Semmle.Util;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
@@ -15,7 +16,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
// </summary>
internal partial class FileContent
{
private readonly ProgressMonitor progressMonitor;
private readonly ILogger logger;
private readonly IUnsafeFileReader unsafeFileReader;
private readonly IEnumerable<string> files;
private readonly HashSet<string> allPackages = new HashSet<string>();
@@ -90,18 +91,18 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
}
internal FileContent(ProgressMonitor progressMonitor,
internal FileContent(ILogger logger,
IEnumerable<string> files,
IUnsafeFileReader unsafeFileReader)
{
this.progressMonitor = progressMonitor;
this.logger = logger;
this.files = files;
this.unsafeFileReader = unsafeFileReader;
this.initialize = new Initializer(DoInitialize);
}
public FileContent(ProgressMonitor progressMonitor, IEnumerable<string> files) : this(progressMonitor, files, new UnsafeFileReader())
public FileContent(ILogger logger, IEnumerable<string> files) : this(logger, files, new UnsafeFileReader())
{ }
private static string GetGroup(ReadOnlySpan<char> input, ValueMatch valueMatch, string groupPrefix, bool toLower)
@@ -192,7 +193,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
catch (Exception ex)
{
progressMonitor.FailedToReadFile(file, ex);
logger.LogInfo($"Failed to read file {file}");
logger.LogDebug($"Failed to read file {file}, exception: {ex}");
}
}
}

5
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileInfoExtensions.cs
View File

@@ -2,6 +2,7 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
@@ -13,14 +14,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
public static IEnumerable<FileInfo> SelectRootFiles(this IEnumerable<FileInfo> files, DirectoryInfo dir) =>
files.Where(file => file.DirectoryName == dir.FullName);
internal static IEnumerable<FileInfo> SelectSmallFiles(this IEnumerable<FileInfo> files, ProgressMonitor progressMonitor)
internal static IEnumerable<FileInfo> SelectSmallFiles(this IEnumerable<FileInfo> files, ILogger logger)
{
const int oneMb = 1_048_576;
return files.Where(file =>
{
if (file.Length > oneMb)
{
progressMonitor.LogDebug($"Skipping {file.FullName} because it is bigger than 1MB.");
logger.LogDebug($"Skipping {file.FullName} because it is bigger than 1MB.");
return false;
}
return true;

12
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FilePathFilter.cs
View File

@@ -12,12 +12,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
public class FilePathFilter
{
private readonly string rootFolder;
private readonly IProgressMonitor progressMonitor;
private readonly ILogger logger;
public FilePathFilter(DirectoryInfo sourceDir, IProgressMonitor progressMonitor)
public FilePathFilter(DirectoryInfo sourceDir, ILogger logger)
{
rootFolder = FileUtils.ConvertToUnix(sourceDir.FullName.ToLowerInvariant());
this.progressMonitor = progressMonitor;
this.logger = logger;
}
private class FileInclusion(string path, bool include)
@@ -55,12 +55,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
else
{
progressMonitor.Log(Severity.Info, $"Invalid filter: {filter}");
logger.Log(Severity.Info, $"Invalid filter: {filter}");
continue;
}
var regex = new FilePattern(filterText).RegexPattern;
progressMonitor.Log(Severity.Info, $"Filtering {(include ? "in" : "out")} files matching '{regex}'. Original glob filter: '{filter}'");
logger.Log(Severity.Info, $"Filtering {(include ? "in" : "out")} files matching '{regex}'. Original glob filter: '{filter}'");
pathFilters.Add(new PathFilter(new Regex(regex, RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline), include));
}
@@ -91,7 +91,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (!include)
{
progressMonitor.Log(Severity.Info, $"Excluding '{f.FileInfo.FullName}'");
logger.Log(Severity.Info, $"Excluding '{f.FileInfo.FullName}'");
}
return include;

32
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs
View File

@@ -1,15 +1,43 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal interface IDotNet
{
bool RestoreProjectToDirectory(string project, string directory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> assets, out IList<string> outputLines, string? pathToNugetConfig = null, bool force = false);
bool RestoreSolutionToDirectory(string solutionFile, string packageDirectory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> projects, out IEnumerable<string> assets);
RestoreResult Restore(RestoreSettings restoreSettings);
bool New(string folder);
bool AddPackage(string folder, string package);
IList<string> GetListedRuntimes();
IList<string> GetListedSdks();
bool Exec(string execArgs);
}
internal record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, string? PathToNugetConfig = null, bool ForceReevaluation = false);
internal partial record class RestoreResult(bool Success, IList<string> Output)
{
private readonly Lazy<IEnumerable<string>> assetsFilePaths = new(() => GetFirstGroupOnMatch(AssetsFileRegex(), Output));
public IEnumerable<string> AssetsFilePaths => Success ? assetsFilePaths.Value : Array.Empty<string>();
private readonly Lazy<IEnumerable<string>> restoredProjects = new(() => GetFirstGroupOnMatch(RestoredProjectRegex(), Output));
public IEnumerable<string> RestoredProjects => Success ? restoredProjects.Value : Array.Empty<string>();
private readonly Lazy<bool> hasNugetPackageSourceError = new(() => Output.Any(s => s.Contains("NU1301")));
public bool HasNugetPackageSourceError => hasNugetPackageSourceError.Value;
private static IEnumerable<string> GetFirstGroupOnMatch(Regex regex, IEnumerable<string> lines) =>
lines
.Select(line => regex.Match(line))
.Where(match => match.Success)
.Select(match => match.Groups[1].Value);
[GeneratedRegex("Restored\\s+(.+\\.csproj)", RegexOptions.Compiled)]
private static partial Regex RestoredProjectRegex();
[GeneratedRegex("[Assets\\sfile\\shas\\snot\\schanged.\\sSkipping\\sassets\\sfile\\swriting.|Writing\\sassets\\sfile\\sto\\sdisk.]\\sPath:\\s(.+)", RegexOptions.Compiled)]
private static partial Regex AssetsFileRegex();
}
}

53
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackages.cs
View File

@@ -2,6 +2,7 @@
using System.Diagnostics;
using System.IO;
using System.Linq;
using Microsoft.Build.Framework;
using Semmle.Util;
namespace Semmle.Extraction.CSharp.DependencyFetching
@@ -14,7 +15,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
internal class NugetPackages
{
private readonly string? nugetExe;
private readonly ProgressMonitor progressMonitor;
private readonly Util.Logging.ILogger logger;
/// <summary>
/// The list of package files.
@@ -31,10 +32,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// <summary>
/// Create the package manager for a specified source tree.
/// </summary>
public NugetPackages(string sourceDir, TemporaryDirectory packageDirectory, ProgressMonitor progressMonitor)
public NugetPackages(string sourceDir, TemporaryDirectory packageDirectory, Util.Logging.ILogger logger)
{
this.packageDirectory = packageDirectory;
this.progressMonitor = progressMonitor;
this.logger = logger;
packageFiles = new DirectoryInfo(sourceDir)
.EnumerateFiles("packages.config", SearchOption.AllDirectories)
@@ -42,11 +43,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (packageFiles.Length > 0)
{
logger.LogInfo($"Found {packageFiles.Length} packages.config files, trying to use nuget.exe for package restore");
nugetExe = ResolveNugetExe(sourceDir);
}
else
{
progressMonitor.LogInfo("Found no packages.config file");
logger.LogInfo("Found no packages.config file");
}
}
@@ -65,15 +67,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var nuget = Path.Combine(directory, "nuget", "nuget.exe");
if (File.Exists(nuget))
{
progressMonitor.FoundNuGet(nuget);
logger.LogInfo($"Found nuget.exe at {nuget}");
return nuget;
}
else
{
progressMonitor.LogInfo($"Nuget.exe could not be found at {nuget}");
return DownloadNugetExe(sourceDir);
}
}
private string DownloadNugetExe(string sourceDir)
{
@@ -83,16 +82,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
// Nuget.exe already exists in the .nuget directory.
if (File.Exists(nuget))
{
progressMonitor.FoundNuGet(nuget);
logger.LogInfo($"Found nuget.exe at {nuget}");
return nuget;
}
Directory.CreateDirectory(directory);
progressMonitor.LogInfo("Attempting to download nuget.exe");
logger.LogInfo("Attempting to download nuget.exe");
try
{
FileUtils.DownloadFile(FileUtils.NugetExeUrl, nuget);
progressMonitor.LogInfo($"Downloaded nuget.exe to {nuget}");
logger.LogInfo($"Downloaded nuget.exe to {nuget}");
return nuget;
}
catch
@@ -108,7 +107,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// <param name="package">The package file.</param>
private void RestoreNugetPackage(string package)
{
progressMonitor.NugetInstall(package);
logger.LogInfo($"Restoring file {package}...");
/* Use nuget.exe to install a package.
* Note that there is a clutch of NuGet assemblies which could be used to
@@ -135,29 +134,17 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
UseShellExecute = false
};
try
var threadId = Environment.CurrentManagedThreadId;
void onOut(string s) => logger.LogInfo(s, threadId);
void onError(string s) => logger.LogError(s, threadId);
var exitCode = pi.ReadOutput(out var _, onOut, onError);
if (exitCode != 0)
{
using var p = Process.Start(pi);
if (p is null)
{
progressMonitor.FailedNugetCommand(pi.FileName, pi.Arguments, "Couldn't start process.");
return;
logger.LogError($"Command {pi.FileName} {pi.Arguments} failed with exit code {exitCode}");
}
var output = p.StandardOutput.ReadToEnd();
var error = p.StandardError.ReadToEnd();
p.WaitForExit();
if (p.ExitCode != 0)
else
{
progressMonitor.FailedNugetCommand(pi.FileName, pi.Arguments, output + error);
}
}
catch (Exception ex)
when (ex is System.ComponentModel.Win32Exception || ex is FileNotFoundException)
{
progressMonitor.FailedNugetCommand(pi.FileName, pi.Arguments, ex.Message);
logger.LogInfo($"Restored file {package}");
}
}

127
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/ProgressMonitor.cs
View File

@@ -1,127 +0,0 @@
using System;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal class ProgressMonitor : IProgressMonitor
{
private readonly ILogger logger;
public ProgressMonitor(ILogger logger)
{
this.logger = logger;
}
public void Log(Severity severity, string message) =>
logger.Log(severity, message);
public void LogInfo(string message) =>
logger.Log(Severity.Info, message);
public void LogDebug(string message) =>
logger.Log(Severity.Debug, message);
private void LogError(string message) =>
logger.Log(Severity.Error, message);
public void FindingFiles(string dir) =>
LogInfo($"Finding files in {dir}...");
public void IndexingReferences(int count)
{
LogInfo("Indexing...");
LogDebug($"Indexing {count} DLLs...");
}
public void UnresolvedReference(string id, string project)
{
LogInfo($"Unresolved reference {id}");
LogDebug($"Unresolved {id} referenced by {project}");
}
public void AnalysingSolution(string filename) =>
LogInfo($"Analyzing {filename}...");
public void FailedProjectFile(string filename, string reason) =>
LogInfo($"Couldn't read project file {filename}: {reason}");
public void FailedNugetCommand(string exe, string args, string message)
{
LogInfo($"Command failed: {exe} {args}");
LogInfo($" {message}");
}
public void NugetInstall(string package) =>
LogInfo($"Restoring {package}...");
public void ResolvedReference(string filename) =>
LogInfo($"Resolved reference {filename}");
public void RemovedReference(string filename) =>
LogInfo($"Removed reference {filename}");
public void Summary(int existingSources, int usedSources, int missingSources,
int references, int unresolvedReferences,
int resolvedConflicts, int totalProjects, int failedProjects,
TimeSpan analysisTime)
{
const int align = 6;
LogInfo("");
LogInfo("Build analysis summary:");
LogInfo($"{existingSources,align} source files in the filesystem");
LogInfo($"{usedSources,align} source files in project files");
LogInfo($"{missingSources,align} sources missing from project files");
LogInfo($"{references,align} resolved references");
LogInfo($"{unresolvedReferences,align} unresolved references");
LogInfo($"{resolvedConflicts,align} resolved assembly conflicts");
LogInfo($"{totalProjects,align} projects");
LogInfo($"{failedProjects,align} missing/failed projects");
LogInfo($"Build analysis completed in {analysisTime}");
}
public void ResolvedConflict(string asm1, string asm2) =>
LogDebug($"Resolved {asm1} as {asm2}");
public void MissingProject(string projectFile) =>
LogInfo($"Solution is missing {projectFile}");
public void CommandFailed(string exe, string arguments, int exitCode) =>
LogError($"Command {exe} {arguments} failed with exit code {exitCode}");
public void MissingNuGet() =>
LogError("Missing nuget.exe");
public void FoundNuGet(string path) =>
LogInfo($"Found nuget.exe at {path}");
public void MissingDotNet() =>
LogError("Missing dotnet CLI");
public void RunningProcess(string command) =>
LogInfo($"Running {command}");
public void FailedToRestoreNugetPackage(string package) =>
LogInfo($"Failed to restore nuget package {package}");
public void FailedToReadFile(string file, Exception ex)
{
LogInfo($"Failed to read file {file}");
LogDebug($"Failed to read file {file}, exception: {ex}");
}
public void MultipleNugetConfig(string[] nugetConfigs) =>
LogInfo($"Found multiple nuget.config files: {string.Join(", ", nugetConfigs)}.");
internal void NoTopLevelNugetConfig() =>
LogInfo("Could not find a top-level nuget.config file.");
internal void RazorSourceGeneratorMissing(string fullPath) =>
LogInfo($"Razor source generator folder {fullPath} does not exist.");
internal void CscMissing(string cscPath) =>
LogInfo($"Csc.exe not found at {cscPath}.");
internal void RazorCscArgs(string args) =>
LogInfo($"Running CSC to generate Razor source files. Args: {args}.");
}
}

23
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Razor.cs
View File

@@ -4,34 +4,37 @@ using System.IO;
using System.Text;
using System.Linq;
using Semmle.Util;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal class Razor
{
private readonly DotNetVersion sdk;
private readonly ProgressMonitor progressMonitor;
private readonly ILogger logger;
private readonly IDotNet dotNet;
private readonly string sourceGeneratorFolder;
private readonly string cscPath;
public Razor(DotNetVersion sdk, IDotNet dotNet, ProgressMonitor progressMonitor)
public Razor(DotNetVersion sdk, IDotNet dotNet, ILogger logger)
{
this.sdk = sdk;
this.progressMonitor = progressMonitor;
this.logger = logger;
this.dotNet = dotNet;
sourceGeneratorFolder = Path.Combine(this.sdk.FullPath, "Sdks", "Microsoft.NET.Sdk.Razor", "source-generators");
this.logger.LogInfo($"Razor source generator folder: {sourceGeneratorFolder}");
if (!Directory.Exists(sourceGeneratorFolder))
{
this.progressMonitor.RazorSourceGeneratorMissing(sourceGeneratorFolder);
this.logger.LogInfo($"Razor source generator folder {sourceGeneratorFolder} does not exist.");
throw new Exception($"Razor source generator folder {sourceGeneratorFolder} does not exist.");
}
cscPath = Path.Combine(this.sdk.FullPath, "Roslyn", "bincore", "csc.dll");
this.logger.LogInfo($"Razor source generator CSC: {cscPath}");
if (!File.Exists(cscPath))
{
this.progressMonitor.CscMissing(cscPath);
this.logger.LogInfo($"Csc.exe not found at {cscPath}.");
throw new Exception($"csc.dll {cscPath} does not exist.");
}
}
@@ -63,7 +66,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
GenerateAnalyzerConfig(cshtmls, analyzerConfig);
progressMonitor.LogInfo($"Analyzer config content: {File.ReadAllText(analyzerConfig)}");
logger.LogInfo($"Analyzer config content: {File.ReadAllText(analyzerConfig)}");
var args = new StringBuilder();
args.Append($"/target:exe /generatedfilesout:\"{outputFolder}\" /out:\"{dllPath}\" /analyzerconfig:\"{analyzerConfig}\" ");
@@ -85,7 +88,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var argsString = args.ToString();
progressMonitor.RazorCscArgs(argsString);
logger.LogInfo($"Running CSC to generate Razor source files with arguments: {argsString}.");
using (var sw = new StreamWriter(cscArgsPath))
{
@@ -94,7 +97,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
dotNet.Exec($"\"{cscPath}\" /noconfig @\"{cscArgsPath}\"");
return Directory.GetFiles(outputFolder, "*.*", new EnumerationOptions { RecurseSubdirectories = true });
var files = Directory.GetFiles(outputFolder, "*.*", new EnumerationOptions { RecurseSubdirectories = true });
logger.LogInfo($"Generated {files.Length} source files from cshtml files.");
return files;
}
finally
{

16
csharp/extractor/Semmle.Extraction.Tests/Assets.cs
View File

@@ -12,7 +12,7 @@ namespace Semmle.Extraction.Tests
public void TestAssets1()
{
// Setup
var assets = new Assets(new ProgressMonitor(new LoggerStub()));
var assets = new Assets(new LoggerStub());
var json = assetsJson1;
var dependencies = new DependencyContainer();
@@ -43,7 +43,7 @@ namespace Semmle.Extraction.Tests
public void TestAssetsFailure()
{
// Setup
var assets = new Assets(new ProgressMonitor(new LoggerStub()));
var assets = new Assets(new LoggerStub());
var json = "garbage data";
var dependencies = new DependencyContainer();
@@ -59,7 +59,7 @@ namespace Semmle.Extraction.Tests
public void TestAssetsNet70()
{
// Setup
var assets = new Assets(new ProgressMonitor(new LoggerStub()));
var assets = new Assets(new LoggerStub());
var json = assetsNet70;
var dependencies = new DependencyContainer();
@@ -90,7 +90,7 @@ namespace Semmle.Extraction.Tests
public void TestAssetsNet48()
{
// Setup
var assets = new Assets(new ProgressMonitor(new LoggerStub()));
var assets = new Assets(new LoggerStub());
var json = assetsNet48;
var dependencies = new DependencyContainer();
@@ -117,7 +117,7 @@ namespace Semmle.Extraction.Tests
public void TestAssetsNetstandard21()
{
// Setup
var assets = new Assets(new ProgressMonitor(new LoggerStub()));
var assets = new Assets(new LoggerStub());
var json = assetsNetstandard21;
var dependencies = new DependencyContainer();
@@ -145,7 +145,7 @@ namespace Semmle.Extraction.Tests
public void TestAssetsNetStandard16()
{
// Setup
var assets = new Assets(new ProgressMonitor(new LoggerStub()));
var assets = new Assets(new LoggerStub());
var json = assetsNetstandard16;
var dependencies = new DependencyContainer();
@@ -177,7 +177,7 @@ namespace Semmle.Extraction.Tests
public void TestAssetsNetcoreapp20()
{
// Setup
var assets = new Assets(new ProgressMonitor(new LoggerStub()));
var assets = new Assets(new LoggerStub());
var json = assetsNetcoreapp20;
var dependencies = new DependencyContainer();
@@ -205,7 +205,7 @@ namespace Semmle.Extraction.Tests
public void TestAssetsNetcoreapp31()
{
// Setup
var assets = new Assets(new ProgressMonitor(new LoggerStub()));
var assets = new Assets(new LoggerStub());
var json = assetsNetcoreapp31;
var dependencies = new DependencyContainer();

40
csharp/extractor/Semmle.Extraction.Tests/DotNet.cs
View File

@@ -38,7 +38,7 @@ namespace Semmle.Extraction.Tests
public class DotNetTests
{
private static IDotNet MakeDotnet(IDotNetCliInvoker dotnetCliInvoker) =>
DotNet.Make(dotnetCliInvoker, new ProgressMonitor(new LoggerStub()));
DotNet.Make(dotnetCliInvoker, new LoggerStub());
private static IList<string> MakeDotnetRestoreOutput() =>
new List<string> {
@@ -101,7 +101,7 @@ namespace Semmle.Extraction.Tests
var dotnet = MakeDotnet(dotnetCliInvoker);
// Execute
dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages", false, out var assets, out var _);
dotnet.Restore(new("myproject.csproj", "mypackages", false));
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();
@@ -116,14 +116,14 @@ namespace Semmle.Extraction.Tests
var dotnet = MakeDotnet(dotnetCliInvoker);
// Execute
dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages", false, out var assets, out var _, pathToNugetConfig: "myconfig.config");
var res = dotnet.Restore(new("myproject.csproj", "mypackages", false, "myconfig.config"));
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();
Assert.Equal("restore --no-dependencies \"myproject.csproj\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal --configfile \"myconfig.config\"", lastArgs);
Assert.Equal(2, assets.Count());
Assert.Contains("/path/to/project.assets.json", assets);
Assert.Contains("/path/to/project2.assets.json", assets);
Assert.Equal(2, res.AssetsFilePaths.Count());
Assert.Contains("/path/to/project.assets.json", res.AssetsFilePaths);
Assert.Contains("/path/to/project2.assets.json", res.AssetsFilePaths);
}
[Fact]
@@ -134,14 +134,14 @@ namespace Semmle.Extraction.Tests
var dotnet = MakeDotnet(dotnetCliInvoker);
// Execute
dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages", false, out var assets, out var _, pathToNugetConfig: "myconfig.config", force: true);
var res = dotnet.Restore(new("myproject.csproj", "mypackages", false, "myconfig.config", true));
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();
Assert.Equal("restore --no-dependencies \"myproject.csproj\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal --configfile \"myconfig.config\" --force", lastArgs);
Assert.Equal(2, assets.Count());
Assert.Contains("/path/to/project.assets.json", assets);
Assert.Contains("/path/to/project2.assets.json", assets);
Assert.Equal(2, res.AssetsFilePaths.Count());
Assert.Contains("/path/to/project.assets.json", res.AssetsFilePaths);
Assert.Contains("/path/to/project2.assets.json", res.AssetsFilePaths);
}
[Fact]
@@ -152,17 +152,17 @@ namespace Semmle.Extraction.Tests
var dotnet = MakeDotnet(dotnetCliInvoker);
// Execute
dotnet.RestoreSolutionToDirectory("mysolution.sln", "mypackages", false, out var projects, out var assets);
var res = dotnet.Restore(new("mysolution.sln", "mypackages", false));
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();
Assert.Equal("restore --no-dependencies \"mysolution.sln\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal", lastArgs);
Assert.Equal(2, projects.Count());
Assert.Contains("/path/to/project.csproj", projects);
Assert.Contains("/path/to/project2.csproj", projects);
Assert.Equal(2, assets.Count());
Assert.Contains("/path/to/project.assets.json", assets);
Assert.Contains("/path/to/project2.assets.json", assets);
Assert.Equal(2, res.RestoredProjects.Count());
Assert.Contains("/path/to/project.csproj", res.RestoredProjects);
Assert.Contains("/path/to/project2.csproj", res.RestoredProjects);
Assert.Equal(2, res.AssetsFilePaths.Count());
Assert.Contains("/path/to/project.assets.json", res.AssetsFilePaths);
Assert.Contains("/path/to/project2.assets.json", res.AssetsFilePaths);
}
[Fact]
@@ -174,13 +174,13 @@ namespace Semmle.Extraction.Tests
dotnetCliInvoker.Success = false;
// Execute
dotnet.RestoreSolutionToDirectory("mysolution.sln", "mypackages", false, out var projects, out var assets);
var res = dotnet.Restore(new("mysolution.sln", "mypackages", false));
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();
Assert.Equal("restore --no-dependencies \"mysolution.sln\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal", lastArgs);
Assert.Empty(projects);
Assert.Empty(assets);
Assert.Empty(res.RestoredProjects);
Assert.Empty(res.AssetsFilePaths);
}
[Fact]

2
csharp/extractor/Semmle.Extraction.Tests/FileContent.cs
View File

@@ -25,7 +25,7 @@ namespace Semmle.Extraction.Tests
internal class TestFileContent : FileContent
{
public TestFileContent(IEnumerable<string> lines) : base(new ProgressMonitor(new LoggerStub()),
public TestFileContent(IEnumerable<string> lines) : base(new LoggerStub(),
new List<string>() { "test1.cs" },
new UnsafeFileReaderStub(lines))
{ }

48
csharp/extractor/Semmle.Extraction.Tests/FilePathFilter.cs
View File

@@ -10,17 +10,19 @@ namespace Semmle.Extraction.Tests
{
public class FilePathFilterTest
{
private class ProgressMonitorStub : IProgressMonitor
private class LoggerStub : ILogger
{
public List<string> Messages { get; } = [];
public void Log(Severity severity, string message)
public void Log(Severity s, string text, int? threadId = null)
{
Messages.Add(message);
}
Messages.Add(text);
}
private static (FilePathFilter TestSubject, ProgressMonitorStub progressMonitor, IEnumerable<FileInfo> Files) TestSetup()
public void Dispose() { }
}
private static (FilePathFilter TestSubject, LoggerStub Logger, IEnumerable<FileInfo> Files) TestSetup()
{
return TestSetup("/a/b",
[
@@ -32,15 +34,15 @@ namespace Semmle.Extraction.Tests
]);
}
private static (FilePathFilter TestSubject, ProgressMonitorStub progressMonitor, IEnumerable<FileInfo> Files) TestSetup(string root, IEnumerable<string> paths)
private static (FilePathFilter TestSubject, LoggerStub Logger, IEnumerable<FileInfo> Files) TestSetup(string root, IEnumerable<string> paths)
{
root = GetPlatformSpecifixPath(root);
paths = GetPlatformSpecifixPaths(paths);
var progressMonitor = new ProgressMonitorStub();
var logger = new LoggerStub();
var filePathFilter = new FilePathFilter(new DirectoryInfo(root), progressMonitor);
return (filePathFilter, progressMonitor, paths.Select(p => new FileInfo(p)));
var filePathFilter = new FilePathFilter(new DirectoryInfo(root), logger);
return (filePathFilter, logger, paths.Select(p => new FileInfo(p)));
}
private static string GetPlatformSpecifixPath(string file)
@@ -67,20 +69,20 @@ namespace Semmle.Extraction.Tests
[Fact]
public void TestNoFilter()
{
(var testSubject, var progressMonitor, var files) = TestSetup();
(var testSubject, var logger, var files) = TestSetup();
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", null);
var filtered = testSubject.Filter(files);
AssertFileInfoEquivalence(files, filtered);
Assert.Equivalent(Array.Empty<string>(), progressMonitor.Messages, strict: true);
Assert.Equivalent(Array.Empty<string>(), logger.Messages, strict: true);
}
[Fact]
public void TestFiltersWithOnlyInclude()
{
(var testSubject, var progressMonitor, var files) = TestSetup();
(var testSubject, var logger, var files) = TestSetup();
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
include:c/d
@@ -104,13 +106,13 @@ namespace Semmle.Extraction.Tests
"Filtering in files matching '^c/d.*'. Original glob filter: 'include:c/d'",
"Filtering in files matching '^c/x/y.*'. Original glob filter: 'include:c/x/y'"
};
Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false);
Assert.Equivalent(expectedRegexMessages, logger.Messages, strict: false);
}
[Fact]
public void TestFiltersWithOnlyExclude()
{
(var testSubject, var progressMonitor, var files) = TestSetup();
(var testSubject, var logger, var files) = TestSetup();
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
exclude:c/d/e
@@ -130,13 +132,13 @@ namespace Semmle.Extraction.Tests
{
"Filtering out files matching '^c/d/e.*'. Original glob filter: 'exclude:c/d/e'"
};
Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false);
Assert.Equivalent(expectedRegexMessages, logger.Messages, strict: false);
}
[Fact]
public void TestFiltersWithIncludeExclude()
{
(var testSubject, var progressMonitor, var files) = TestSetup();
(var testSubject, var logger, var files) = TestSetup();
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
include:c/x
@@ -157,13 +159,13 @@ namespace Semmle.Extraction.Tests
"Filtering in files matching '^c/x.*'. Original glob filter: 'include:c/x'",
"Filtering out files matching '^c/x/z.*'. Original glob filter: 'exclude:c/x/z'"
};
Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false);
Assert.Equivalent(expectedRegexMessages, logger.Messages, strict: false);
}
[Fact]
public void TestFiltersWithIncludeExcludeExcludeFirst()
{
(var testSubject, var progressMonitor, var files) = TestSetup();
(var testSubject, var logger, var files) = TestSetup();
// NOTE: the ordering DOES matter, later filters takes priority, so the exclude will end up not mattering at all.
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
@@ -186,13 +188,13 @@ namespace Semmle.Extraction.Tests
"Filtering in files matching '^c/x.*'. Original glob filter: 'include:c/x'",
"Filtering out files matching '^c/x/z.*'. Original glob filter: 'exclude:c/x/z'"
};
Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false);
Assert.Equivalent(expectedRegexMessages, logger.Messages, strict: false);
}
[Fact]
public void TestFiltersWithIncludeExcludeComplexPatterns1()
{
(var testSubject, var progressMonitor, var files) = TestSetup();
(var testSubject, var logger, var files) = TestSetup();
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
include:c/**/i.*
@@ -218,13 +220,13 @@ namespace Semmle.Extraction.Tests
"Filtering in files matching '^c/d/.*/[^/]*\\.cs.*'. Original glob filter: 'include:c/d/**/*.cs'",
"Filtering out files matching '^.*/z/i\\.cs.*'. Original glob filter: 'exclude:**/z/i.cs'"
};
Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false);
Assert.Equivalent(expectedRegexMessages, logger.Messages, strict: false);
}
[Fact]
public void TestFiltersWithIncludeExcludeComplexPatterns2()
{
(var testSubject, var progressMonitor, var files) = TestSetup();
(var testSubject, var logger, var files) = TestSetup();
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
include:**/i.*
@@ -245,7 +247,7 @@ namespace Semmle.Extraction.Tests
"Filtering in files matching '^.*/i\\.[^/]*.*'. Original glob filter: 'include:**/i.*'",
"Filtering out files matching '^.*/z/i\\.cs.*'. Original glob filter: 'exclude:**/z/i.cs'"
};
Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false);
Assert.Equivalent(expectedRegexMessages, logger.Messages, strict: false);
}
}
}

2
csharp/extractor/Semmle.Extraction.Tests/LoggerStub.cs
View File

@@ -4,7 +4,7 @@ namespace Semmle.Extraction.Tests
{
internal class LoggerStub : ILogger
{
public void Log(Severity severity, string message) { }
public void Log(Severity severity, string message, int? threadId = null) { }
public void Dispose() { }
}

14
csharp/extractor/Semmle.Extraction.Tests/Runtime.cs
View File

@@ -19,19 +19,7 @@ namespace Semmle.Extraction.Tests
public bool New(string folder) => true;
public bool RestoreProjectToDirectory(string project, string directory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> assets, out IList<string> outputLines, string? pathToNugetConfig = null, bool force = false)
{
assets = Array.Empty<string>();
outputLines = Array.Empty<string>();
return true;
}
public bool RestoreSolutionToDirectory(string solution, string directory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> projects, out IEnumerable<string> assets)
{
projects = Array.Empty<string>();
assets = Array.Empty<string>();
return true;
}
public RestoreResult Restore(RestoreSettings restoreSettings) => new(true, Array.Empty<string>());
public IList<string> GetListedRuntimes() => runtimes;

2
csharp/extractor/Semmle.Util.Tests/CanonicalPathCache.cs
View File

@@ -176,7 +176,7 @@ namespace SemmleTests.Semmle.Util
{
public void Dispose() { }
public void Log(Severity s, string text) { }
public void Log(Severity s, string text, int? threadId = null) { }
}
}
}

2
csharp/extractor/Semmle.Util.Tests/FileUtils.cs
View File

@@ -57,7 +57,7 @@ namespace SemmleTests.Semmle.Util
{
public void Dispose() { }
public void Log(Severity s, string text) { }
public void Log(Severity s, string text, int? threadId = null) { }
}
}
}

32
csharp/extractor/Semmle.Util/Logger.cs
View File

@@ -37,7 +37,15 @@ namespace Semmle.Util.Logging
/// <summary>
/// Log the given text with the given severity.
/// </summary>
void Log(Severity s, string text);
void Log(Severity s, string text, int? threadId = null);
void LogError(string text, int? threadId = null) => Log(Severity.Error, text, threadId);
void LogWarning(string text, int? threadId = null) => Log(Severity.Warning, text, threadId);
void LogInfo(string text, int? threadId = null) => Log(Severity.Info, text, threadId);
void LogDebug(string text, int? threadId = null) => Log(Severity.Debug, text, threadId);
}
public static class LoggerExtensions
@@ -92,12 +100,14 @@ namespace Semmle.Util.Logging
return "[" + s.ToString().ToUpper() + "] ";
}
public void Log(Severity s, string text)
public void Log(Severity s, string text, int? threadId = null)
{
if (verbosity.Includes(s))
{
var threadId = this.logThreadId ? $"[{Environment.CurrentManagedThreadId:D3}] " : "";
writer.WriteLine(threadId + GetSeverityPrefix(s) + text);
threadId ??= Environment.CurrentManagedThreadId;
var prefix = this.logThreadId ? $"[{threadId:D3}] " : "";
writer.WriteLine(prefix + GetSeverityPrefix(s) + text);
}
}
}
@@ -140,12 +150,14 @@ namespace Semmle.Util.Logging
}
}
public void Log(Severity s, string text)
public void Log(Severity s, string text, int? threadId = null)
{
if (verbosity.Includes(s))
{
var threadId = this.logThreadId ? $"[{Environment.CurrentManagedThreadId:D3}] " : "";
GetConsole(s).WriteLine(threadId + GetSeverityPrefix(s) + text);
threadId ??= Environment.CurrentManagedThreadId;
var prefix = this.logThreadId ? $"[{threadId:D3}] " : "";
GetConsole(s).WriteLine(prefix + GetSeverityPrefix(s) + text);
}
}
}
@@ -170,10 +182,10 @@ namespace Semmle.Util.Logging
logger2.Dispose();
}
public void Log(Severity s, string text)
public void Log(Severity s, string text, int? threadId = null)
{
logger1.Log(s, text);
logger2.Log(s, text);
logger1.Log(s, text, threadId);
logger2.Log(s, text, threadId);
}
}