hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-16 16:53:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #15424 from tamasvajk/standalone/logging

Browse Source 
C#: Improve log messages in buildless mode + some cleanup/refactoring
This commit is contained in:
Tamás Vajk
2024-01-25 11:08:50 +01:00
committed by GitHub
parent 930f1b50b9 199b0578be
commit 01b89508a8
22 changed files with 395 additions and 524 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs
View File

@@ -1,6 +1,7 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.IO; using System.IO;
using System.Linq; using System.Linq;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching namespace Semmle.Extraction.CSharp.DependencyFetching
{ {
@@ -18,25 +19,26 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// Paths to search. Directories are searched recursively. Files are added directly to the /// Paths to search. Directories are searched recursively. Files are added directly to the
/// assembly cache. /// assembly cache.
/// </param> /// </param>
/// <param name="progressMonitor">Callback for progress.</param> /// <param name="logger">Callback for progress.</param>
public AssemblyCache(IEnumerable<string> paths, IEnumerable<string> frameworkPaths, ProgressMonitor progressMonitor) public AssemblyCache(IEnumerable<string> paths, IEnumerable<string> frameworkPaths, ILogger logger)
{ {
this.logger = logger;
foreach (var path in paths) foreach (var path in paths)
{ {
if (File.Exists(path)) if (File.Exists(path))
{ {
pendingDllsToIndex.Enqueue(path); dllsToIndex.Add(path);
continue; continue;
} }
if (Directory.Exists(path)) if (Directory.Exists(path))
{ {
progressMonitor.FindingFiles(path); logger.LogInfo($"Finding reference DLLs in {path}...");
AddReferenceDirectory(path); AddReferenceDirectory(path);
} }
else else
{ {
progressMonitor.LogInfo("AssemblyCache: Path not found: " + path); logger.LogInfo("AssemblyCache: Path not found: " + path);
} }
} }
IndexReferences(frameworkPaths); IndexReferences(frameworkPaths);
@@ -52,7 +54,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{ {
foreach (var dll in new DirectoryInfo(dir).EnumerateFiles("*.dll", SearchOption.AllDirectories)) foreach (var dll in new DirectoryInfo(dir).EnumerateFiles("*.dll", SearchOption.AllDirectories))
{ {
pendingDllsToIndex.Enqueue(dll.FullName); dllsToIndex.Add(dll.FullName);
} }
} }
@@ -62,12 +64,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// </summary> /// </summary>
private void IndexReferences(IEnumerable<string> frameworkPaths) private void IndexReferences(IEnumerable<string> frameworkPaths)
{ {
logger.LogInfo($"Indexing {dllsToIndex.Count} assemblies...");
// Read all of the files // Read all of the files
foreach (var filename in pendingDllsToIndex) foreach (var filename in dllsToIndex)
{ {
IndexReference(filename); IndexReference(filename);
} }
logger.LogInfo($"Read {assemblyInfoByFileName.Count} assembly infos");
foreach (var info in assemblyInfoByFileName.Values foreach (var info in assemblyInfoByFileName.Values
.OrderBy(info => info.Name) .OrderBy(info => info.Name)
.OrderAssemblyInfosByPreference(frameworkPaths)) .OrderAssemblyInfosByPreference(frameworkPaths))
@@ -83,25 +89,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{ {
try try
{ {
logger.LogDebug($"Reading assembly info from {filename}");
var info = AssemblyInfo.ReadFromFile(filename); var info = AssemblyInfo.ReadFromFile(filename);
assemblyInfoByFileName[filename] = info; assemblyInfoByFileName[filename] = info;
} }
catch (AssemblyLoadException) catch (AssemblyLoadException)
{ {
failedAssemblyInfoFileNames.Add(filename); logger.LogInfo($"Couldn't read assembly info from {filename}");
} }
} }
/// <summary>
/// The number of DLLs which are assemblies.
/// </summary>
public int AssemblyCount => assemblyInfoByFileName.Count;
/// <summary>
/// The number of DLLs which weren't assemblies. (E.g. C++).
/// </summary>
public int NonAssemblyCount => failedAssemblyInfoFileNames.Count;
/// <summary> /// <summary>
/// Given an assembly id, determine its full info. /// Given an assembly id, determine its full info.
/// </summary> /// </summary>
@@ -113,8 +110,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (failedAssemblyInfoIds.Contains(id)) if (failedAssemblyInfoIds.Contains(id))
throw new AssemblyLoadException(); throw new AssemblyLoadException();
string assemblyName; (id, var assemblyName) = AssemblyInfo.ComputeSanitizedAssemblyInfo(id);
(id, assemblyName) = AssemblyInfo.ComputeSanitizedAssemblyInfo(id);
// Look up the id in our references map. // Look up the id in our references map.
if (assemblyInfoById.TryGetValue(id, out var result)) if (assemblyInfoById.TryGetValue(id, out var result))
@@ -164,17 +160,15 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
throw new AssemblyLoadException(); throw new AssemblyLoadException();
} }
private readonly Queue<string> pendingDllsToIndex = new Queue<string>(); private readonly List<string> dllsToIndex = new List<string>();
private readonly Dictionary<string, AssemblyInfo> assemblyInfoByFileName = new Dictionary<string, AssemblyInfo>(); private readonly Dictionary<string, AssemblyInfo> assemblyInfoByFileName = new Dictionary<string, AssemblyInfo>();
// List of DLLs which are not assemblies.
// We probably don't need to keep this
private readonly List<string> failedAssemblyInfoFileNames = new List<string>();
// Map from assembly id (in various formats) to the full info. // Map from assembly id (in various formats) to the full info.
private readonly Dictionary<string, AssemblyInfo> assemblyInfoById = new Dictionary<string, AssemblyInfo>(); private readonly Dictionary<string, AssemblyInfo> assemblyInfoById = new Dictionary<string, AssemblyInfo>();
private readonly HashSet<string> failedAssemblyInfoIds = new HashSet<string>(); private readonly HashSet<string> failedAssemblyInfoIds = new HashSet<string>();
private readonly ILogger logger;
} }
} }

15
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyInfo.cs
View File

@@ -120,21 +120,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
NetCoreVersion = netCoreVersion; NetCoreVersion = netCoreVersion;
} }
/// <summary>
/// Get AssemblyInfo from a loaded Assembly.
/// </summary>
/// <param name="assembly">The assembly.</param>
/// <returns>Info about the assembly.</returns>
public static AssemblyInfo MakeFromAssembly(Assembly assembly)
{
if (assembly.FullName is null)
{
throw new InvalidOperationException("Assembly with empty full name is not expected.");
}
return new AssemblyInfo(assembly.FullName, assembly.Location);
}
/// <summary> /// <summary>
/// Returns the id and name of the assembly that would be created from the received id. /// Returns the id and name of the assembly that would be created from the received id.
/// </summary> /// </summary>

25
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Assets.cs
View File

@@ -5,6 +5,7 @@ using System.IO;
using System.Linq; using System.Linq;
using Newtonsoft.Json.Linq; using Newtonsoft.Json.Linq;
using Semmle.Util; using Semmle.Util;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching namespace Semmle.Extraction.CSharp.DependencyFetching
{ {
@@ -13,11 +14,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// </summary> /// </summary>
internal class Assets internal class Assets
{ {
private readonly ProgressMonitor progressMonitor; private readonly ILogger logger;
internal Assets(ProgressMonitor progressMonitor) internal Assets(ILogger logger)
{ {
this.progressMonitor = progressMonitor; this.logger = logger;
} }
/// <summary> /// <summary>
@@ -85,7 +86,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (references is null) if (references is null)
{ {
progressMonitor.LogDebug("No references found in the targets section in the assets file."); logger.LogDebug("No references found in the targets section in the assets file.");
return; return;
} }
@@ -157,7 +158,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (frameworks is null) if (frameworks is null)
{ {
progressMonitor.LogDebug("No framework section in assets.json."); logger.LogDebug("No framework section in assets.json.");
return; return;
} }
@@ -171,7 +172,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (references is null) if (references is null)
{ {
progressMonitor.LogDebug("No framework references in assets.json."); logger.LogDebug("No framework references in assets.json.");
return; return;
} }
@@ -196,12 +197,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
} }
catch (Exception e) catch (Exception e)
{ {
progressMonitor.LogDebug($"Failed to parse assets file (unexpected error): {e.Message}"); logger.LogDebug($"Failed to parse assets file (unexpected error): {e.Message}");
return false; return false;
} }
} }
private static bool TryReadAllText(string path, ProgressMonitor progressMonitor, [NotNullWhen(returnValue: true)] out string? content) private static bool TryReadAllText(string path, ILogger logger, [NotNullWhen(returnValue: true)] out string? content)
{ {
try try
{ {
@@ -210,19 +211,19 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
} }
catch (Exception e) catch (Exception e)
{ {
progressMonitor.LogInfo($"Failed to read assets file '{path}': {e.Message}"); logger.LogInfo($"Failed to read assets file '{path}': {e.Message}");
content = null; content = null;
return false; return false;
} }
} }
public static DependencyContainer GetCompilationDependencies(ProgressMonitor progressMonitor, IEnumerable<string> assets) public static DependencyContainer GetCompilationDependencies(ILogger logger, IEnumerable<string> assets)
{ {
var parser = new Assets(progressMonitor); var parser = new Assets(logger);
var dependencies = new DependencyContainer(); var dependencies = new DependencyContainer();
assets.ForEach(asset => assets.ForEach(asset =>
{ {
if (TryReadAllText(asset, progressMonitor, out var json)) if (TryReadAllText(asset, logger, out var json))
{ {
parser.TryParse(json, dependencies); parser.TryParse(json, dependencies);
} }

299
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
View File

@@ -17,14 +17,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
public sealed class DependencyManager : IDisposable public sealed class DependencyManager : IDisposable
{ {
private readonly AssemblyCache assemblyCache; private readonly AssemblyCache assemblyCache;
private readonly ProgressMonitor progressMonitor; private readonly ILogger logger;
// Only used as a set, but ConcurrentDictionary is the only concurrent set in .NET.
private readonly IDictionary<string, bool> usedReferences = new ConcurrentDictionary<string, bool>(); private readonly IDictionary<string, bool> usedReferences = new ConcurrentDictionary<string, bool>();
private readonly IDictionary<string, bool> sources = new ConcurrentDictionary<string, bool>();
private readonly IDictionary<string, string> unresolvedReferences = new ConcurrentDictionary<string, string>(); private readonly IDictionary<string, string> unresolvedReferences = new ConcurrentDictionary<string, string>();
private int failedProjects;
private int succeededProjects;
private readonly List<string> nonGeneratedSources; private readonly List<string> nonGeneratedSources;
private readonly List<string> generatedSources; private readonly List<string> generatedSources;
private int dotnetFrameworkVersionVariantCount = 0;
private int conflictedReferences = 0; private int conflictedReferences = 0;
private readonly IDependencyOptions options; private readonly IDependencyOptions options;
private readonly DirectoryInfo sourceDir; private readonly DirectoryInfo sourceDir;
@@ -49,7 +49,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var startTime = DateTime.Now; var startTime = DateTime.Now;
this.options = options; this.options = options;
this.progressMonitor = new ProgressMonitor(logger); this.logger = logger;
this.sourceDir = new DirectoryInfo(srcDir); this.sourceDir = new DirectoryInfo(srcDir);
packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName, "packages")); packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName, "packages"));
@@ -60,35 +60,36 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
try try
{ {
this.dotnet = DotNet.Make(options, progressMonitor, tempWorkingDirectory); this.dotnet = DotNet.Make(options, logger, tempWorkingDirectory);
runtimeLazy = new Lazy<Runtime>(() => new Runtime(dotnet)); runtimeLazy = new Lazy<Runtime>(() => new Runtime(dotnet));
} }
catch catch
{ {
progressMonitor.MissingDotNet(); logger.LogError("Missing dotnet CLI");
throw; throw;
} }
this.progressMonitor.FindingFiles(srcDir); logger.LogInfo($"Finding files in {srcDir}...");
var allFiles = GetAllFiles().ToList(); var allFiles = GetAllFiles().ToList();
var binaryFileExtensions = new HashSet<string>(new[] { ".dll", ".exe" }); // TODO: add more binary file extensions. var binaryFileExtensions = new HashSet<string>(new[] { ".dll", ".exe" }); // TODO: add more binary file extensions.
var allNonBinaryFiles = allFiles.Where(f => !binaryFileExtensions.Contains(f.Extension.ToLowerInvariant())).ToList(); var allNonBinaryFiles = allFiles.Where(f => !binaryFileExtensions.Contains(f.Extension.ToLowerInvariant())).ToList();
var smallNonBinaryFiles = allNonBinaryFiles.SelectSmallFiles(progressMonitor).SelectFileNames(); var smallNonBinaryFiles = allNonBinaryFiles.SelectSmallFiles(logger).SelectFileNames();
this.fileContent = new FileContent(progressMonitor, smallNonBinaryFiles); this.fileContent = new FileContent(logger, smallNonBinaryFiles);
this.nonGeneratedSources = allNonBinaryFiles.SelectFileNamesByExtension(".cs").ToList(); this.nonGeneratedSources = allNonBinaryFiles.SelectFileNamesByExtension(".cs").ToList();
this.generatedSources = new(); this.generatedSources = new();
var allProjects = allNonBinaryFiles.SelectFileNamesByExtension(".csproj"); var allProjects = allNonBinaryFiles.SelectFileNamesByExtension(".csproj").ToList();
var allSolutions = allNonBinaryFiles.SelectFileNamesByExtension(".sln"); var allSolutions = allNonBinaryFiles.SelectFileNamesByExtension(".sln").ToList();
var dllPaths = allFiles.SelectFileNamesByExtension(".dll").ToHashSet(); var dllPaths = allFiles.SelectFileNamesByExtension(".dll").ToHashSet();
logger.LogInfo($"Found {allFiles.Count} files, {nonGeneratedSources.Count} source files, {allProjects.Count} project files, {allSolutions.Count} solution files, {dllPaths.Count} DLLs.");
RestoreNugetPackages(allNonBinaryFiles, allProjects, allSolutions, dllPaths); RestoreNugetPackages(allNonBinaryFiles, allProjects, allSolutions, dllPaths);
// Find DLLs in the .Net / Asp.Net Framework // Find DLLs in the .Net / Asp.Net Framework
// This needs to come after the nuget restore, because the nuget restore might fetch the .NET Core/Framework reference assemblies. // This needs to come after the nuget restore, because the nuget restore might fetch the .NET Core/Framework reference assemblies.
var frameworkLocations = AddFrameworkDlls(dllPaths); var frameworkLocations = AddFrameworkDlls(dllPaths);
assemblyCache = new AssemblyCache(dllPaths, frameworkLocations, progressMonitor); assemblyCache = new AssemblyCache(dllPaths, frameworkLocations, logger);
AnalyseSolutions(allSolutions); AnalyseSolutions(allSolutions);
foreach (var filename in assemblyCache.AllAssemblies.Select(a => a.Filename)) foreach (var filename in assemblyCache.AllAssemblies.Select(a => a.Filename))
@@ -102,33 +103,36 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
// Output the findings // Output the findings
foreach (var r in usedReferences.Keys.OrderBy(r => r)) foreach (var r in usedReferences.Keys.OrderBy(r => r))
{ {
progressMonitor.ResolvedReference(r); logger.LogInfo($"Resolved reference {r}");
} }
foreach (var r in unresolvedReferences.OrderBy(r => r.Key)) foreach (var r in unresolvedReferences.OrderBy(r => r.Key))
{ {
progressMonitor.UnresolvedReference(r.Key, r.Value); logger.LogInfo($"Unresolved reference {r.Key} in project {r.Value}");
} }
var webViewExtractionOption = Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_STANDALONE_EXTRACT_WEB_VIEWS"); var webViewExtractionOption = Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_STANDALONE_EXTRACT_WEB_VIEWS");
if (bool.TryParse(webViewExtractionOption, out var shouldExtractWebViews) && if (bool.TryParse(webViewExtractionOption, out var shouldExtractWebViews) &&
shouldExtractWebViews) shouldExtractWebViews)
{ {
logger.LogInfo("Generating source files from cshtml and razor files...");
GenerateSourceFilesFromWebViews(allNonBinaryFiles); GenerateSourceFilesFromWebViews(allNonBinaryFiles);
} }
GenerateSourceFileFromImplicitUsings(); GenerateSourceFileFromImplicitUsings();
progressMonitor.Summary( const int align = 6;
AllSourceFiles.Count(), logger.LogInfo("");
ProjectSourceFiles.Count(), logger.LogInfo("Build analysis summary:");
MissingSourceFiles.Count(), logger.LogInfo($"{nonGeneratedSources.Count,align} source files in the filesystem");
ReferenceFiles.Count(), logger.LogInfo($"{generatedSources.Count,align} generated source files");
UnresolvedReferences.Count(), logger.LogInfo($"{allSolutions.Count,align} solution files");
conflictedReferences, logger.LogInfo($"{allProjects.Count,align} project files in the filesystem");
succeededProjects + failedProjects, logger.LogInfo($"{usedReferences.Keys.Count,align} resolved references");
failedProjects, logger.LogInfo($"{unresolvedReferences.Count,align} unresolved references");
DateTime.Now - startTime); logger.LogInfo($"{conflictedReferences,align} resolved assembly conflicts");
logger.LogInfo($"{dotnetFrameworkVersionVariantCount,align} restored .NET framework variants");
logger.LogInfo($"Build analysis completed in {DateTime.Now - startTime}");
} }
private HashSet<string> AddFrameworkDlls(HashSet<string> dllPaths) private HashSet<string> AddFrameworkDlls(HashSet<string> dllPaths)
@@ -146,32 +150,42 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{ {
try try
{ {
var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, progressMonitor); var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, logger);
nuget.InstallPackages(); nuget.InstallPackages();
var nugetPackageDlls = legacyPackageDirectory.DirInfo.GetFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = true }); var nugetPackageDlls = legacyPackageDirectory.DirInfo.GetFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = true });
var nugetPackageDllPaths = nugetPackageDlls.Select(f => f.FullName).ToHashSet(); var nugetPackageDllPaths = nugetPackageDlls.Select(f => f.FullName).ToHashSet();
var excludedPaths = nugetPackageDllPaths var excludedPaths = nugetPackageDllPaths
.Where(path => IsPathInSubfolder(path, legacyPackageDirectory.DirInfo.FullName, "tools")); .Where(path => IsPathInSubfolder(path, legacyPackageDirectory.DirInfo.FullName, "tools"))
.ToList();
if (nugetPackageDllPaths.Count > 0)
{
logger.LogInfo($"Restored {nugetPackageDllPaths.Count} Nuget DLLs.");
}
if (excludedPaths.Count > 0)
{
logger.LogInfo($"Excluding {excludedPaths.Count} Nuget DLLs.");
}
foreach (var excludedPath in excludedPaths) foreach (var excludedPath in excludedPaths)
{ {
progressMonitor.LogInfo($"Excluded Nuget DLL: {excludedPath}"); logger.LogInfo($"Excluded Nuget DLL: {excludedPath}");
} }
nugetPackageDllPaths.ExceptWith(excludedPaths); nugetPackageDllPaths.ExceptWith(excludedPaths);
dllPaths.UnionWith(nugetPackageDllPaths); dllPaths.UnionWith(nugetPackageDllPaths);
} }
catch (FileNotFoundException) catch (Exception)
{ {
progressMonitor.MissingNuGet(); logger.LogError("Failed to restore Nuget packages with nuget.exe");
} }
var restoredProjects = RestoreSolutions(allSolutions, out var assets1); var restoredProjects = RestoreSolutions(allSolutions, out var assets1);
var projects = allProjects.Except(restoredProjects); var projects = allProjects.Except(restoredProjects);
RestoreProjects(projects, out var assets2); RestoreProjects(projects, out var assets2);
var dependencies = Assets.GetCompilationDependencies(progressMonitor, assets1.Union(assets2)); var dependencies = Assets.GetCompilationDependencies(logger, assets1.Union(assets2));
var paths = dependencies var paths = dependencies
.Paths .Paths
@@ -220,7 +234,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (isInAnalyzersFolder) if (isInAnalyzersFolder)
{ {
usedReferences.Remove(filename); usedReferences.Remove(filename);
progressMonitor.RemovedReference(filename); logger.LogInfo($"Removed analyzer reference {filename}");
} }
} }
} }
@@ -228,27 +242,31 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private void SelectNewestFrameworkPath(string frameworkPath, string frameworkType, ISet<string> dllPaths, ISet<string> frameworkLocations) private void SelectNewestFrameworkPath(string frameworkPath, string frameworkType, ISet<string> dllPaths, ISet<string> frameworkLocations)
{ {
var versionFolders = new DirectoryInfo(frameworkPath) var versionFolders = GetPackageVersionSubDirectories(frameworkPath);
.EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
.OrderByDescending(d => d.Name) // TODO: Improve sorting to handle pre-release versions.
.ToArray();
if (versionFolders.Length > 1) if (versionFolders.Length > 1)
{ {
var versions = string.Join(", ", versionFolders.Select(d => d.Name)); var versions = string.Join(", ", versionFolders.Select(d => d.Name));
progressMonitor.LogInfo($"Found multiple {frameworkType} DLLs in NuGet packages at {frameworkPath}. Using the latest version ({versionFolders[0].Name}) from: {versions}."); logger.LogInfo($"Found multiple {frameworkType} DLLs in NuGet packages at {frameworkPath}. Using the latest version ({versionFolders[0].Name}) from: {versions}.");
} }
var selectedFrameworkFolder = versionFolders.FirstOrDefault()?.FullName; var selectedFrameworkFolder = versionFolders.FirstOrDefault()?.FullName;
if (selectedFrameworkFolder is null) if (selectedFrameworkFolder is null)
{ {
progressMonitor.LogInfo($"Found {frameworkType} DLLs in NuGet packages at {frameworkPath}, but no version folder was found."); logger.LogInfo($"Found {frameworkType} DLLs in NuGet packages at {frameworkPath}, but no version folder was found.");
selectedFrameworkFolder = frameworkPath; selectedFrameworkFolder = frameworkPath;
} }
dllPaths.Add(selectedFrameworkFolder); dllPaths.Add(selectedFrameworkFolder);
frameworkLocations.Add(selectedFrameworkFolder); frameworkLocations.Add(selectedFrameworkFolder);
progressMonitor.LogInfo($"Found {frameworkType} DLLs in NuGet packages at {selectedFrameworkFolder}. Not adding installation directory."); logger.LogInfo($"Found {frameworkType} DLLs in NuGet packages at {selectedFrameworkFolder}.");
}
private static DirectoryInfo[] GetPackageVersionSubDirectories(string packagePath)
{
return new DirectoryInfo(packagePath)
.EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
.OrderByDescending(d => d.Name) // TODO: Improve sorting to handle pre-release versions.
.ToArray();
} }
private void AddNetFrameworkDlls(ISet<string> dllPaths, ISet<string> frameworkLocations) private void AddNetFrameworkDlls(ISet<string> dllPaths, ISet<string> frameworkLocations)
@@ -257,12 +275,20 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
// The order of the packages is important, we're adding the first one that is present in the nuget cache. // The order of the packages is important, we're adding the first one that is present in the nuget cache.
var packagesInPrioOrder = FrameworkPackageNames.NetFrameworks; var packagesInPrioOrder = FrameworkPackageNames.NetFrameworks;
var frameworkPath = packagesInPrioOrder var frameworkPaths = packagesInPrioOrder
.Select((s, index) => (Index: index, Path: GetPackageDirectory(s))) .Select((s, index) => (Index: index, Path: GetPackageDirectory(s)))
.FirstOrDefault(pair => pair.Path is not null); .Where(pair => pair.Path is not null)
.ToArray();
var frameworkPath = frameworkPaths.FirstOrDefault();
if (frameworkPath.Path is not null) if (frameworkPath.Path is not null)
{ {
foreach (var fp in frameworkPaths)
{
dotnetFrameworkVersionVariantCount += GetPackageVersionSubDirectories(fp.Path!).Length;
}
SelectNewestFrameworkPath(frameworkPath.Path, ".NET Framework", dllPaths, frameworkLocations); SelectNewestFrameworkPath(frameworkPath.Path, ".NET Framework", dllPaths, frameworkLocations);
for (var i = frameworkPath.Index + 1; i < packagesInPrioOrder.Length; i++) for (var i = frameworkPath.Index + 1; i < packagesInPrioOrder.Length; i++)
@@ -286,7 +312,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
runtimeLocation ??= Runtime.ExecutingRuntime; runtimeLocation ??= Runtime.ExecutingRuntime;
progressMonitor.LogInfo($".NET runtime location selected: {runtimeLocation}"); logger.LogInfo($".NET runtime location selected: {runtimeLocation}");
dllPaths.Add(runtimeLocation); dllPaths.Add(runtimeLocation);
frameworkLocations.Add(runtimeLocation); frameworkLocations.Add(runtimeLocation);
} }
@@ -300,11 +326,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
} }
var packagePathPrefix = Path.Combine(packageFolder, packagePrefix.ToLowerInvariant()); var packagePathPrefix = Path.Combine(packageFolder, packagePrefix.ToLowerInvariant());
var toRemove = dllPaths.Where(s => s.ToLowerInvariant().StartsWith(packagePathPrefix)); var toRemove = dllPaths.Where(s => s.StartsWith(packagePathPrefix, StringComparison.InvariantCultureIgnoreCase));
foreach (var path in toRemove) foreach (var path in toRemove)
{ {
dllPaths.Remove(path); dllPaths.Remove(path);
progressMonitor.RemovedReference(path); logger.LogInfo($"Removed reference {path}");
} }
} }
@@ -324,7 +350,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (Runtime.AspNetCoreRuntime is string aspNetCoreRuntime) if (Runtime.AspNetCoreRuntime is string aspNetCoreRuntime)
{ {
progressMonitor.LogInfo($"ASP.NET runtime location selected: {aspNetCoreRuntime}"); logger.LogInfo($"ASP.NET runtime location selected: {aspNetCoreRuntime}");
dllPaths.Add(aspNetCoreRuntime); dllPaths.Add(aspNetCoreRuntime);
frameworkLocations.Add(aspNetCoreRuntime); frameworkLocations.Add(aspNetCoreRuntime);
} }
@@ -346,18 +372,26 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
.FullName; .FullName;
} }
private IEnumerable<string> GetAllPackageDirectories() private ICollection<string> GetAllPackageDirectories()
{ {
return new DirectoryInfo(packageDirectory.DirInfo.FullName) return new DirectoryInfo(packageDirectory.DirInfo.FullName)
.EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false }) .EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
.Select(d => d.Name); .Select(d => d.Name)
.ToList();
} }
private void LogAllUnusedPackages(DependencyContainer dependencies) => private void LogAllUnusedPackages(DependencyContainer dependencies)
GetAllPackageDirectories() {
var allPackageDirectories = GetAllPackageDirectories();
logger.LogInfo($"Restored {allPackageDirectories.Count} packages");
logger.LogInfo($"Found {dependencies.Packages.Count} packages in project.asset.json files");
allPackageDirectories
.Where(package => !dependencies.Packages.Contains(package)) .Where(package => !dependencies.Packages.Contains(package))
.Order() .Order()
.ForEach(package => progressMonitor.LogInfo($"Unused package: {package}")); .ForEach(package => logger.LogInfo($"Unused package: {package}"));
}
private void GenerateSourceFileFromImplicitUsings() private void GenerateSourceFileFromImplicitUsings()
{ {
@@ -380,6 +414,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
usings.UnionWith(fileContent.CustomImplicitUsings); usings.UnionWith(fileContent.CustomImplicitUsings);
logger.LogInfo($"Generating source file for implicit usings. Namespaces: {string.Join(", ", usings.OrderBy(u => u))}");
if (usings.Count > 0) if (usings.Count > 0)
{ {
var tempDir = GetTemporaryWorkingDirectory("implicitUsings"); var tempDir = GetTemporaryWorkingDirectory("implicitUsings");
@@ -401,20 +437,20 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private void GenerateSourceFilesFromWebViews(List<FileInfo> allFiles) private void GenerateSourceFilesFromWebViews(List<FileInfo> allFiles)
{ {
progressMonitor.LogInfo($"Generating source files from cshtml and razor files.");
var views = allFiles.SelectFileNamesByExtension(".cshtml", ".razor").ToArray(); var views = allFiles.SelectFileNamesByExtension(".cshtml", ".razor").ToArray();
if (views.Length == 0)
if (views.Length > 0)
{ {
progressMonitor.LogInfo($"Found {views.Length} cshtml and razor files."); return;
}
logger.LogInfo($"Found {views.Length} cshtml and razor files.");
var sdk = new Sdk(dotnet).GetNewestSdk(); var sdk = new Sdk(dotnet).GetNewestSdk();
if (sdk != null) if (sdk != null)
{ {
try try
{ {
var razor = new Razor(sdk, dotnet, progressMonitor); var razor = new Razor(sdk, dotnet, logger);
var targetDir = GetTemporaryWorkingDirectory("razor"); var targetDir = GetTemporaryWorkingDirectory("razor");
var generatedFiles = razor.GenerateFiles(views, usedReferences.Keys, targetDir); var generatedFiles = razor.GenerateFiles(views, usedReferences.Keys, targetDir);
this.generatedSources.AddRange(generatedFiles); this.generatedSources.AddRange(generatedFiles);
@@ -422,8 +458,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
catch (Exception ex) catch (Exception ex)
{ {
// It's okay, we tried our best to generate source files from cshtml files. // It's okay, we tried our best to generate source files from cshtml files.
progressMonitor.LogInfo($"Failed to generate source files from cshtml files: {ex.Message}"); logger.LogInfo($"Failed to generate source files from cshtml files: {ex.Message}");
}
} }
} }
} }
@@ -448,17 +483,17 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
return true; return true;
} }
progressMonitor.Log(Severity.Warning, $"File {f.FullName} could not be processed."); logger.Log(Severity.Warning, $"File {f.FullName} could not be processed.");
return false; return false;
} }
catch (Exception ex) catch (Exception ex)
{ {
progressMonitor.Log(Severity.Warning, $"File {f.FullName} could not be processed: {ex.Message}"); logger.Log(Severity.Warning, $"File {f.FullName} could not be processed: {ex.Message}");
return false; return false;
} }
}); });
files = new FilePathFilter(sourceDir, progressMonitor).Filter(files); files = new FilePathFilter(sourceDir, logger).Filter(files);
return files; return files;
} }
@@ -499,7 +534,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// </summary> /// </summary>
private void ResolveConflicts(IEnumerable<string> frameworkPaths) private void ResolveConflicts(IEnumerable<string> frameworkPaths)
{ {
var sortedReferences = new List<AssemblyInfo>(); var sortedReferences = new List<AssemblyInfo>(usedReferences.Count);
foreach (var usedReference in usedReferences) foreach (var usedReference in usedReferences)
{ {
try try
@@ -509,7 +544,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
} }
catch (AssemblyLoadException) catch (AssemblyLoadException)
{ {
progressMonitor.Log(Util.Logging.Severity.Warning, $"Could not load assembly information from {usedReference.Key}"); logger.Log(Severity.Warning, $"Could not load assembly information from {usedReference.Key}");
} }
} }
@@ -517,6 +552,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
.OrderAssemblyInfosByPreference(frameworkPaths) .OrderAssemblyInfosByPreference(frameworkPaths)
.ToList(); .ToList();
logger.LogInfo($"Reference list contains {sortedReferences.Count} assemblies");
var finalAssemblyList = new Dictionary<string, AssemblyInfo>(); var finalAssemblyList = new Dictionary<string, AssemblyInfo>();
// Pick the highest version for each assembly name // Pick the highest version for each assembly name
@@ -532,15 +569,24 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
UseReference(r); UseReference(r);
} }
logger.LogInfo($"After conflict resolution, reference list contains {finalAssemblyList.Count} assemblies");
// Report the results // Report the results
foreach (var r in sortedReferences) foreach (var r in sortedReferences)
{ {
var resolvedInfo = finalAssemblyList[r.Name]; var resolvedInfo = finalAssemblyList[r.Name];
if (resolvedInfo.Version != r.Version || resolvedInfo.NetCoreVersion != r.NetCoreVersion) if (resolvedInfo.Version != r.Version || resolvedInfo.NetCoreVersion != r.NetCoreVersion)
{ {
progressMonitor.ResolvedConflict(r.Id, resolvedInfo.Id + resolvedInfo.NetCoreVersion is null ? "" : $" (.NET Core {resolvedInfo.NetCoreVersion})"); var asm = resolvedInfo.Id + (resolvedInfo.NetCoreVersion is null ? "" : $" (.NET Core {resolvedInfo.NetCoreVersion})");
logger.LogInfo($"Resolved {r.Id} as {asm}");
++conflictedReferences; ++conflictedReferences;
} }
if (r != resolvedInfo)
{
logger.LogDebug($"Resolved {r.Id} as {resolvedInfo.Id} from {resolvedInfo.Filename}");
}
} }
} }
@@ -550,22 +596,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// <param name="reference">The filename of the reference.</param> /// <param name="reference">The filename of the reference.</param>
private void UseReference(string reference) => usedReferences[reference] = true; private void UseReference(string reference) => usedReferences[reference] = true;
/// <summary>
/// Store that a particular source file is used (by a project file).
/// </summary>
/// <param name="sourceFile">The source file.</param>
private void UseSource(FileInfo sourceFile) => sources[sourceFile.FullName] = sourceFile.Exists;
/// <summary> /// <summary>
/// The list of resolved reference files. /// The list of resolved reference files.
/// </summary> /// </summary>
public IEnumerable<string> ReferenceFiles => usedReferences.Keys; public IEnumerable<string> ReferenceFiles => usedReferences.Keys;
/// <summary>
/// The list of source files used in projects.
/// </summary>
public IEnumerable<string> ProjectSourceFiles => sources.Where(s => s.Value).Select(s => s.Key);
/// <summary> /// <summary>
/// All of the generated source files in the source directory. /// All of the generated source files in the source directory.
/// </summary> /// </summary>
@@ -581,12 +616,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// </summary> /// </summary>
public IEnumerable<string> UnresolvedReferences => unresolvedReferences.Select(r => r.Key); public IEnumerable<string> UnresolvedReferences => unresolvedReferences.Select(r => r.Key);
/// <summary>
/// List of source files which were mentioned in project files but
/// do not exist on the file system.
/// </summary>
public IEnumerable<string> MissingSourceFiles => sources.Where(s => !s.Value).Select(s => s.Key);
/// <summary> /// <summary>
/// Record that a particular reference couldn't be resolved. /// Record that a particular reference couldn't be resolved.
/// Note that this records at most one project file per missing reference. /// Note that this records at most one project file per missing reference.
@@ -595,23 +624,31 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// <param name="projectFile">The project file making the reference.</param> /// <param name="projectFile">The project file making the reference.</param>
private void UnresolvedReference(string id, string projectFile) => unresolvedReferences[id] = projectFile; private void UnresolvedReference(string id, string projectFile) => unresolvedReferences[id] = projectFile;
/// <summary> private void AnalyseSolutions(IEnumerable<string> solutions)
/// Reads all the source files and references from the given list of projects.
/// </summary>
/// <param name="projectFiles">The list of projects to analyse.</param>
private void AnalyseProjectFiles(IEnumerable<FileInfo> projectFiles)
{ {
foreach (var proj in projectFiles) Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, solutionFile =>
{
try
{
var sln = new SolutionFile(solutionFile);
logger.LogInfo($"Analyzing {solutionFile}...");
foreach (var proj in sln.Projects.Select(p => new FileInfo(p)))
{ {
AnalyseProject(proj); AnalyseProject(proj);
} }
} }
catch (Microsoft.Build.Exceptions.InvalidProjectFileException ex)
{
logger.LogInfo($"Couldn't read solution file {solutionFile}: {ex.BaseMessage}");
}
});
}
private void AnalyseProject(FileInfo project) private void AnalyseProject(FileInfo project)
{ {
if (!project.Exists) if (!project.Exists)
{ {
progressMonitor.MissingProject(project.FullName); logger.LogInfo($"Couldn't read project file {project.FullName}");
return; return;
} }
@@ -631,23 +668,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
UnresolvedReference(@ref, project.FullName); UnresolvedReference(@ref, project.FullName);
} }
} }
foreach (var src in csProj.Sources)
{
// Make a note of which source files the projects use.
// This information doesn't affect the build but is dumped
// as diagnostic output.
UseSource(new FileInfo(src));
}
++succeededProjects;
} }
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions] catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{ {
++failedProjects; logger.LogInfo($"Couldn't read project file {project.FullName}: {ex.Message}");
progressMonitor.FailedProjectFile(project.FullName, ex.Message);
} }
} }
/// <summary> /// <summary>
@@ -664,9 +689,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var assetFiles = new List<string>(); var assetFiles = new List<string>();
var projects = solutions.SelectMany(solution => var projects = solutions.SelectMany(solution =>
{ {
dotnet.RestoreSolutionToDirectory(solution, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: true, out var restoredProjects, out var a); logger.LogInfo($"Restoring solution {solution}...");
assetFiles.AddRange(a); var res = dotnet.Restore(new(solution, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
return restoredProjects; assetFiles.AddRange(res.AssetsFilePaths);
return res.RestoredProjects;
}); });
assets = assetFiles; assets = assetFiles;
return projects; return projects;
@@ -683,26 +709,39 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var assetFiles = new List<string>(); var assetFiles = new List<string>();
Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, project => Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, project =>
{ {
dotnet.RestoreProjectToDirectory(project, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: true, out var a, out var _); logger.LogInfo($"Restoring project {project}...");
assetFiles.AddRange(a); var res = dotnet.Restore(new(project, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
assetFiles.AddRange(res.AssetsFilePaths);
}); });
assets = assetFiles; assets = assetFiles;
} }
private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPaths) private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPaths)
{ {
var alreadyDownloadedPackages = Directory.GetDirectories(packageDirectory.DirInfo.FullName)
.Select(d => Path.GetFileName(d).ToLowerInvariant());
var notYetDownloadedPackages = fileContent.AllPackages
.Except(alreadyDownloadedPackages)
.ToList();
if (notYetDownloadedPackages.Count == 0)
{
return;
}
logger.LogInfo($"Found {notYetDownloadedPackages.Count} packages that are not yet restored");
var nugetConfigs = allFiles.SelectFileNamesByName("nuget.config").ToArray(); var nugetConfigs = allFiles.SelectFileNamesByName("nuget.config").ToArray();
string? nugetConfig = null; string? nugetConfig = null;
if (nugetConfigs.Length > 1) if (nugetConfigs.Length > 1)
{ {
progressMonitor.MultipleNugetConfig(nugetConfigs); logger.LogInfo($"Found multiple nuget.config files: {string.Join(", ", nugetConfigs)}.");
nugetConfig = allFiles nugetConfig = allFiles
.SelectRootFiles(sourceDir) .SelectRootFiles(sourceDir)
.SelectFileNamesByName("nuget.config") .SelectFileNamesByName("nuget.config")
.FirstOrDefault(); .FirstOrDefault();
if (nugetConfig == null) if (nugetConfig == null)
{ {
progressMonitor.NoTopLevelNugetConfig(); logger.LogInfo("Could not find a top-level nuget.config file.");
} }
} }
else else
@@ -710,13 +749,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
nugetConfig = nugetConfigs.FirstOrDefault(); nugetConfig = nugetConfigs.FirstOrDefault();
} }
var alreadyDownloadedPackages = Directory.GetDirectories(packageDirectory.DirInfo.FullName) if (nugetConfig != null)
.Select(d => Path.GetFileName(d).ToLowerInvariant()); {
var notYetDownloadedPackages = fileContent.AllPackages.Except(alreadyDownloadedPackages); logger.LogInfo($"Using nuget.config file {nugetConfig}.");
}
Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, package => Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, package =>
{ {
progressMonitor.NugetInstall(package); logger.LogInfo($"Restoring package {package}...");
using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package, "missingpackages_workingdir")); using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package, "missingpackages_workingdir"));
var success = dotnet.New(tempDir.DirInfo.FullName); var success = dotnet.New(tempDir.DirInfo.FullName);
if (!success) if (!success)
@@ -730,20 +770,20 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
return; return;
} }
success = dotnet.RestoreProjectToDirectory(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: false, out var _, out var outputLines, pathToNugetConfig: nugetConfig); var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig));
if (!success) if (!res.Success)
{ {
if (outputLines?.Any(s => s.Contains("NU1301")) == true) if (res.HasNugetPackageSourceError)
{ {
// Restore could not be completed because the listed source is unavailable. Try without the nuget.config: // Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
success = dotnet.RestoreProjectToDirectory(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: false, out var _, out var _, pathToNugetConfig: null, force: true); res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true));
} }
// TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package. // TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package.
if (!success) if (!res.Success)
{ {
progressMonitor.FailedToRestoreNugetPackage(package); logger.LogInfo($"Failed to restore nuget package {package}");
} }
} }
}); });
@@ -751,23 +791,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
dllPaths.Add(missingPackageDirectory.DirInfo.FullName); dllPaths.Add(missingPackageDirectory.DirInfo.FullName);
} }
private void AnalyseSolutions(IEnumerable<string> solutions)
{
Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, solutionFile =>
{
try
{
var sln = new SolutionFile(solutionFile);
progressMonitor.AnalysingSolution(solutionFile);
AnalyseProjectFiles(sln.Projects.Select(p => new FileInfo(p)).Where(p => p.Exists));
}
catch (Microsoft.Build.Exceptions.InvalidProjectFileException ex)
{
progressMonitor.FailedProjectFile(solutionFile, ex.BaseMessage);
}
});
}
public void Dispose(TemporaryDirectory? dir, string name) public void Dispose(TemporaryDirectory? dir, string name)
{ {
try try
@@ -776,7 +799,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
} }
catch (Exception exc) catch (Exception exc)
{ {
progressMonitor.LogInfo($"Couldn't delete {name} directory {exc.Message}"); logger.LogInfo($"Couldn't delete {name} directory {exc.Message}");
} }
} }

63
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs
View File

@@ -4,6 +4,7 @@ using System.IO;
using System.Linq; using System.Linq;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using Semmle.Util; using Semmle.Util;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching namespace Semmle.Extraction.CSharp.DependencyFetching
{ {
@@ -13,22 +14,22 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
internal partial class DotNet : IDotNet internal partial class DotNet : IDotNet
{ {
private readonly IDotNetCliInvoker dotnetCliInvoker; private readonly IDotNetCliInvoker dotnetCliInvoker;
private readonly ProgressMonitor progressMonitor; private readonly ILogger logger;
private readonly TemporaryDirectory? tempWorkingDirectory; private readonly TemporaryDirectory? tempWorkingDirectory;
private DotNet(IDotNetCliInvoker dotnetCliInvoker, ProgressMonitor progressMonitor, TemporaryDirectory? tempWorkingDirectory = null) private DotNet(IDotNetCliInvoker dotnetCliInvoker, ILogger logger, TemporaryDirectory? tempWorkingDirectory = null)
{ {
this.progressMonitor = progressMonitor; this.logger = logger;
this.tempWorkingDirectory = tempWorkingDirectory; this.tempWorkingDirectory = tempWorkingDirectory;
this.dotnetCliInvoker = dotnetCliInvoker; this.dotnetCliInvoker = dotnetCliInvoker;
Info(); Info();
} }
private DotNet(IDependencyOptions options, ProgressMonitor progressMonitor, TemporaryDirectory tempWorkingDirectory) : this(new DotNetCliInvoker(progressMonitor, Path.Combine(options.DotNetPath ?? string.Empty, "dotnet")), progressMonitor, tempWorkingDirectory) { } private DotNet(IDependencyOptions options, ILogger logger, TemporaryDirectory tempWorkingDirectory) : this(new DotNetCliInvoker(logger, Path.Combine(options.DotNetPath ?? string.Empty, "dotnet")), logger, tempWorkingDirectory) { }
internal static IDotNet Make(IDotNetCliInvoker dotnetCliInvoker, ProgressMonitor progressMonitor) => new DotNet(dotnetCliInvoker, progressMonitor); internal static IDotNet Make(IDotNetCliInvoker dotnetCliInvoker, ILogger logger) => new DotNet(dotnetCliInvoker, logger);
public static IDotNet Make(IDependencyOptions options, ProgressMonitor progressMonitor, TemporaryDirectory tempWorkingDirectory) => new DotNet(options, progressMonitor, tempWorkingDirectory); public static IDotNet Make(IDependencyOptions options, ILogger logger, TemporaryDirectory tempWorkingDirectory) => new DotNet(options, logger, tempWorkingDirectory);
private void Info() private void Info()
{ {
@@ -40,11 +41,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
} }
} }
private string GetRestoreArgs(string projectOrSolutionFile, string packageDirectory, bool forceDotnetRefAssemblyFetching) private string GetRestoreArgs(RestoreSettings restoreSettings)
{ {
var args = $"restore --no-dependencies \"{projectOrSolutionFile}\" --packages \"{packageDirectory}\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal"; var args = $"restore --no-dependencies \"{restoreSettings.File}\" --packages \"{restoreSettings.PackageDirectory}\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal";
if (forceDotnetRefAssemblyFetching) if (restoreSettings.ForceDotnetRefAssemblyFetching)
{ {
// Ugly hack: we set the TargetFrameworkRootPath and NetCoreTargetingPackRoot properties to an empty folder: // Ugly hack: we set the TargetFrameworkRootPath and NetCoreTargetingPackRoot properties to an empty folder:
var path = ".empty"; var path = ".empty";
@@ -57,46 +58,24 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
args += $" /p:TargetFrameworkRootPath=\"{path}\" /p:NetCoreTargetingPackRoot=\"{path}\""; args += $" /p:TargetFrameworkRootPath=\"{path}\" /p:NetCoreTargetingPackRoot=\"{path}\"";
} }
return args; if (restoreSettings.PathToNugetConfig != null)
{
args += $" --configfile \"{restoreSettings.PathToNugetConfig}\"";
} }
private static IEnumerable<string> GetFirstGroupOnMatch(Regex regex, IEnumerable<string> lines) => if (restoreSettings.ForceReevaluation)
lines
.Select(line => regex.Match(line))
.Where(match => match.Success)
.Select(match => match.Groups[1].Value);
private static IEnumerable<string> GetAssetsFilePaths(IEnumerable<string> lines) =>
GetFirstGroupOnMatch(AssetsFileRegex(), lines);
private static IEnumerable<string> GetRestoredProjects(IEnumerable<string> lines) =>
GetFirstGroupOnMatch(RestoredProjectRegex(), lines);
public bool RestoreProjectToDirectory(string projectFile, string packageDirectory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> assets, out IList<string> outputLines, string? pathToNugetConfig = null, bool force = false)
{
var args = GetRestoreArgs(projectFile, packageDirectory, forceDotnetRefAssemblyFetching);
if (pathToNugetConfig != null)
{
args += $" --configfile \"{pathToNugetConfig}\"";
}
if (force)
{ {
args += " --force"; args += " --force";
} }
var success = dotnetCliInvoker.RunCommand(args, out outputLines); return args;
assets = success ? GetAssetsFilePaths(outputLines) : Array.Empty<string>();
return success;
} }
public bool RestoreSolutionToDirectory(string solutionFile, string packageDirectory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> projects, out IEnumerable<string> assets) public RestoreResult Restore(RestoreSettings restoreSettings)
{ {
var args = GetRestoreArgs(solutionFile, packageDirectory, forceDotnetRefAssemblyFetching); var args = GetRestoreArgs(restoreSettings);
var success = dotnetCliInvoker.RunCommand(args, out var output); var success = dotnetCliInvoker.RunCommand(args, out var output);
projects = success ? GetRestoredProjects(output) : Array.Empty<string>(); return new(success, output);
assets = success ? GetAssetsFilePaths(output) : Array.Empty<string>();
return success;
} }
public bool New(string folder) public bool New(string folder)
@@ -129,11 +108,5 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var args = $"exec {execArgs}"; var args = $"exec {execArgs}";
return dotnetCliInvoker.RunCommand(args); return dotnetCliInvoker.RunCommand(args);
} }
[GeneratedRegex("Restored\\s+(.+\\.csproj)", RegexOptions.Compiled)]
private static partial Regex RestoredProjectRegex();
[GeneratedRegex("[Assets\\sfile\\shas\\snot\\schanged.\\sSkipping\\sassets\\sfile\\swriting.|Writing\\sassets\\sfile\\sto\\sdisk.]\\sPath:\\s(.+)", RegexOptions.Compiled)]
private static partial Regex AssetsFileRegex();
} }
} }

23
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs
View File

@@ -2,6 +2,7 @@ using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Diagnostics; using System.Diagnostics;
using Semmle.Util; using Semmle.Util;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching namespace Semmle.Extraction.CSharp.DependencyFetching
{ {
@@ -10,13 +11,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// </summary> /// </summary>
internal sealed class DotNetCliInvoker : IDotNetCliInvoker internal sealed class DotNetCliInvoker : IDotNetCliInvoker
{ {
private readonly ProgressMonitor progressMonitor; private readonly ILogger logger;
public string Exec { get; } public string Exec { get; }
public DotNetCliInvoker(ProgressMonitor progressMonitor, string exec) public DotNetCliInvoker(ILogger logger, string exec)
{ {
this.progressMonitor = progressMonitor; this.logger = logger;
this.Exec = exec; this.Exec = exec;
} }
@@ -35,21 +36,15 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private bool RunCommandAux(string args, out IList<string> output) private bool RunCommandAux(string args, out IList<string> output)
{ {
progressMonitor.RunningProcess($"{Exec} {args}"); logger.LogInfo($"Running {Exec} {args}");
var pi = MakeDotnetStartInfo(args); var pi = MakeDotnetStartInfo(args);
var threadId = $"[{Environment.CurrentManagedThreadId:D3}]"; var threadId = Environment.CurrentManagedThreadId;
void onOut(string s) void onOut(string s) => logger.LogInfo(s, threadId);
{ void onError(string s) => logger.LogError(s, threadId);
Console.Out.WriteLine($"{threadId} {s}");
}
void onError(string s)
{
Console.Error.WriteLine($"{threadId} {s}");
}
var exitCode = pi.ReadOutput(out output, onOut, onError); var exitCode = pi.ReadOutput(out output, onOut, onError);
if (exitCode != 0) if (exitCode != 0)
{ {
progressMonitor.CommandFailed(Exec, args, exitCode); logger.LogError($"Command {Exec} {args} failed with exit code {exitCode}");
return false; return false;
} }
return true; return true;

12
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs
View File

@@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.IO; using System.IO;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using Semmle.Util; using Semmle.Util;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching namespace Semmle.Extraction.CSharp.DependencyFetching
{ {
@@ -15,7 +16,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
// </summary> // </summary>
internal partial class FileContent internal partial class FileContent
{ {
private readonly ProgressMonitor progressMonitor; private readonly ILogger logger;
private readonly IUnsafeFileReader unsafeFileReader; private readonly IUnsafeFileReader unsafeFileReader;
private readonly IEnumerable<string> files; private readonly IEnumerable<string> files;
private readonly HashSet<string> allPackages = new HashSet<string>(); private readonly HashSet<string> allPackages = new HashSet<string>();
@@ -90,18 +91,18 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
} }
} }
internal FileContent(ProgressMonitor progressMonitor, internal FileContent(ILogger logger,
IEnumerable<string> files, IEnumerable<string> files,
IUnsafeFileReader unsafeFileReader) IUnsafeFileReader unsafeFileReader)
{ {
this.progressMonitor = progressMonitor; this.logger = logger;
this.files = files; this.files = files;
this.unsafeFileReader = unsafeFileReader; this.unsafeFileReader = unsafeFileReader;
this.initialize = new Initializer(DoInitialize); this.initialize = new Initializer(DoInitialize);
} }
public FileContent(ProgressMonitor progressMonitor, IEnumerable<string> files) : this(progressMonitor, files, new UnsafeFileReader()) public FileContent(ILogger logger, IEnumerable<string> files) : this(logger, files, new UnsafeFileReader())
{ } { }
private static string GetGroup(ReadOnlySpan<char> input, ValueMatch valueMatch, string groupPrefix, bool toLower) private static string GetGroup(ReadOnlySpan<char> input, ValueMatch valueMatch, string groupPrefix, bool toLower)
@@ -192,7 +193,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
} }
catch (Exception ex) catch (Exception ex)
{ {
progressMonitor.FailedToReadFile(file, ex); logger.LogInfo($"Failed to read file {file}");
logger.LogDebug($"Failed to read file {file}, exception: {ex}");
} }
} }
} }

5
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileInfoExtensions.cs
View File

@@ -2,6 +2,7 @@ using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.IO; using System.IO;
using System.Linq; using System.Linq;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching namespace Semmle.Extraction.CSharp.DependencyFetching
{ {
@@ -13,14 +14,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
public static IEnumerable<FileInfo> SelectRootFiles(this IEnumerable<FileInfo> files, DirectoryInfo dir) => public static IEnumerable<FileInfo> SelectRootFiles(this IEnumerable<FileInfo> files, DirectoryInfo dir) =>
files.Where(file => file.DirectoryName == dir.FullName); files.Where(file => file.DirectoryName == dir.FullName);
internal static IEnumerable<FileInfo> SelectSmallFiles(this IEnumerable<FileInfo> files, ProgressMonitor progressMonitor) internal static IEnumerable<FileInfo> SelectSmallFiles(this IEnumerable<FileInfo> files, ILogger logger)
{ {
const int oneMb = 1_048_576; const int oneMb = 1_048_576;
return files.Where(file => return files.Where(file =>
{ {
if (file.Length > oneMb) if (file.Length > oneMb)
{ {
progressMonitor.LogDebug($"Skipping {file.FullName} because it is bigger than 1MB."); logger.LogDebug($"Skipping {file.FullName} because it is bigger than 1MB.");
return false; return false;
} }
return true; return true;

12
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FilePathFilter.cs
View File

@@ -12,12 +12,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
public class FilePathFilter public class FilePathFilter
{ {
private readonly string rootFolder; private readonly string rootFolder;
private readonly IProgressMonitor progressMonitor; private readonly ILogger logger;
public FilePathFilter(DirectoryInfo sourceDir, IProgressMonitor progressMonitor) public FilePathFilter(DirectoryInfo sourceDir, ILogger logger)
{ {
rootFolder = FileUtils.ConvertToUnix(sourceDir.FullName.ToLowerInvariant()); rootFolder = FileUtils.ConvertToUnix(sourceDir.FullName.ToLowerInvariant());
this.progressMonitor = progressMonitor; this.logger = logger;
} }
private class FileInclusion(string path, bool include) private class FileInclusion(string path, bool include)
@@ -55,12 +55,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
} }
else else
{ {
progressMonitor.Log(Severity.Info, $"Invalid filter: {filter}"); logger.Log(Severity.Info, $"Invalid filter: {filter}");
continue; continue;
} }
var regex = new FilePattern(filterText).RegexPattern; var regex = new FilePattern(filterText).RegexPattern;
progressMonitor.Log(Severity.Info, $"Filtering {(include ? "in" : "out")} files matching '{regex}'. Original glob filter: '{filter}'"); logger.Log(Severity.Info, $"Filtering {(include ? "in" : "out")} files matching '{regex}'. Original glob filter: '{filter}'");
pathFilters.Add(new PathFilter(new Regex(regex, RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline), include)); pathFilters.Add(new PathFilter(new Regex(regex, RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline), include));
} }
@@ -91,7 +91,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (!include) if (!include)
{ {
progressMonitor.Log(Severity.Info, $"Excluding '{f.FileInfo.FullName}'"); logger.Log(Severity.Info, $"Excluding '{f.FileInfo.FullName}'");
} }
return include; return include;

32
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs
View File

@@ -1,15 +1,43 @@
using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
namespace Semmle.Extraction.CSharp.DependencyFetching namespace Semmle.Extraction.CSharp.DependencyFetching
{ {
internal interface IDotNet internal interface IDotNet
{ {
bool RestoreProjectToDirectory(string project, string directory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> assets, out IList<string> outputLines, string? pathToNugetConfig = null, bool force = false); RestoreResult Restore(RestoreSettings restoreSettings);
bool RestoreSolutionToDirectory(string solutionFile, string packageDirectory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> projects, out IEnumerable<string> assets);
bool New(string folder); bool New(string folder);
bool AddPackage(string folder, string package); bool AddPackage(string folder, string package);
IList<string> GetListedRuntimes(); IList<string> GetListedRuntimes();
IList<string> GetListedSdks(); IList<string> GetListedSdks();
bool Exec(string execArgs); bool Exec(string execArgs);
} }
internal record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, string? PathToNugetConfig = null, bool ForceReevaluation = false);
internal partial record class RestoreResult(bool Success, IList<string> Output)
{
private readonly Lazy<IEnumerable<string>> assetsFilePaths = new(() => GetFirstGroupOnMatch(AssetsFileRegex(), Output));
public IEnumerable<string> AssetsFilePaths => Success ? assetsFilePaths.Value : Array.Empty<string>();
private readonly Lazy<IEnumerable<string>> restoredProjects = new(() => GetFirstGroupOnMatch(RestoredProjectRegex(), Output));
public IEnumerable<string> RestoredProjects => Success ? restoredProjects.Value : Array.Empty<string>();
private readonly Lazy<bool> hasNugetPackageSourceError = new(() => Output.Any(s => s.Contains("NU1301")));
public bool HasNugetPackageSourceError => hasNugetPackageSourceError.Value;
private static IEnumerable<string> GetFirstGroupOnMatch(Regex regex, IEnumerable<string> lines) =>
lines
.Select(line => regex.Match(line))
.Where(match => match.Success)
.Select(match => match.Groups[1].Value);
[GeneratedRegex("Restored\\s+(.+\\.csproj)", RegexOptions.Compiled)]
private static partial Regex RestoredProjectRegex();
[GeneratedRegex("[Assets\\sfile\\shas\\snot\\schanged.\\sSkipping\\sassets\\sfile\\swriting.|Writing\\sassets\\sfile\\sto\\sdisk.]\\sPath:\\s(.+)", RegexOptions.Compiled)]
private static partial Regex AssetsFileRegex();
}
} }

53
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackages.cs
View File

@@ -2,6 +2,7 @@
using System.Diagnostics; using System.Diagnostics;
using System.IO; using System.IO;
using System.Linq; using System.Linq;
using Microsoft.Build.Framework;
using Semmle.Util; using Semmle.Util;
namespace Semmle.Extraction.CSharp.DependencyFetching namespace Semmle.Extraction.CSharp.DependencyFetching
@@ -14,7 +15,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
internal class NugetPackages internal class NugetPackages
{ {
private readonly string? nugetExe; private readonly string? nugetExe;
private readonly ProgressMonitor progressMonitor; private readonly Util.Logging.ILogger logger;
/// <summary> /// <summary>
/// The list of package files. /// The list of package files.
@@ -31,10 +32,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// <summary> /// <summary>
/// Create the package manager for a specified source tree. /// Create the package manager for a specified source tree.
/// </summary> /// </summary>
public NugetPackages(string sourceDir, TemporaryDirectory packageDirectory, ProgressMonitor progressMonitor) public NugetPackages(string sourceDir, TemporaryDirectory packageDirectory, Util.Logging.ILogger logger)
{ {
this.packageDirectory = packageDirectory; this.packageDirectory = packageDirectory;
this.progressMonitor = progressMonitor; this.logger = logger;
packageFiles = new DirectoryInfo(sourceDir) packageFiles = new DirectoryInfo(sourceDir)
.EnumerateFiles("packages.config", SearchOption.AllDirectories) .EnumerateFiles("packages.config", SearchOption.AllDirectories)
@@ -42,11 +43,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (packageFiles.Length > 0) if (packageFiles.Length > 0)
{ {
logger.LogInfo($"Found {packageFiles.Length} packages.config files, trying to use nuget.exe for package restore");
nugetExe = ResolveNugetExe(sourceDir); nugetExe = ResolveNugetExe(sourceDir);
} }
else else
{ {
progressMonitor.LogInfo("Found no packages.config file"); logger.LogInfo("Found no packages.config file");
} }
} }
@@ -65,15 +67,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var nuget = Path.Combine(directory, "nuget", "nuget.exe"); var nuget = Path.Combine(directory, "nuget", "nuget.exe");
if (File.Exists(nuget)) if (File.Exists(nuget))
{ {
progressMonitor.FoundNuGet(nuget); logger.LogInfo($"Found nuget.exe at {nuget}");
return nuget; return nuget;
} }
else
{
progressMonitor.LogInfo($"Nuget.exe could not be found at {nuget}");
return DownloadNugetExe(sourceDir); return DownloadNugetExe(sourceDir);
} }
}
private string DownloadNugetExe(string sourceDir) private string DownloadNugetExe(string sourceDir)
{ {
@@ -83,16 +82,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
// Nuget.exe already exists in the .nuget directory. // Nuget.exe already exists in the .nuget directory.
if (File.Exists(nuget)) if (File.Exists(nuget))
{ {
progressMonitor.FoundNuGet(nuget); logger.LogInfo($"Found nuget.exe at {nuget}");
return nuget; return nuget;
} }
Directory.CreateDirectory(directory); Directory.CreateDirectory(directory);
progressMonitor.LogInfo("Attempting to download nuget.exe"); logger.LogInfo("Attempting to download nuget.exe");
try try
{ {
FileUtils.DownloadFile(FileUtils.NugetExeUrl, nuget); FileUtils.DownloadFile(FileUtils.NugetExeUrl, nuget);
progressMonitor.LogInfo($"Downloaded nuget.exe to {nuget}"); logger.LogInfo($"Downloaded nuget.exe to {nuget}");
return nuget; return nuget;
} }
catch catch
@@ -108,7 +107,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// <param name="package">The package file.</param> /// <param name="package">The package file.</param>
private void RestoreNugetPackage(string package) private void RestoreNugetPackage(string package)
{ {
progressMonitor.NugetInstall(package); logger.LogInfo($"Restoring file {package}...");
/* Use nuget.exe to install a package. /* Use nuget.exe to install a package.
* Note that there is a clutch of NuGet assemblies which could be used to * Note that there is a clutch of NuGet assemblies which could be used to
@@ -135,29 +134,17 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
UseShellExecute = false UseShellExecute = false
}; };
try var threadId = Environment.CurrentManagedThreadId;
void onOut(string s) => logger.LogInfo(s, threadId);
void onError(string s) => logger.LogError(s, threadId);
var exitCode = pi.ReadOutput(out var _, onOut, onError);
if (exitCode != 0)
{ {
using var p = Process.Start(pi); logger.LogError($"Command {pi.FileName} {pi.Arguments} failed with exit code {exitCode}");
if (p is null)
{
progressMonitor.FailedNugetCommand(pi.FileName, pi.Arguments, "Couldn't start process.");
return;
} }
else
var output = p.StandardOutput.ReadToEnd();
var error = p.StandardError.ReadToEnd();
p.WaitForExit();
if (p.ExitCode != 0)
{ {
progressMonitor.FailedNugetCommand(pi.FileName, pi.Arguments, output + error); logger.LogInfo($"Restored file {package}");
}
}
catch (Exception ex)
when (ex is System.ComponentModel.Win32Exception || ex is FileNotFoundException)
{
progressMonitor.FailedNugetCommand(pi.FileName, pi.Arguments, ex.Message);
} }
} }

127
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/ProgressMonitor.cs
View File

@@ -1,127 +0,0 @@
using System;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal class ProgressMonitor : IProgressMonitor
{
private readonly ILogger logger;
public ProgressMonitor(ILogger logger)
{
this.logger = logger;
}
public void Log(Severity severity, string message) =>
logger.Log(severity, message);
public void LogInfo(string message) =>
logger.Log(Severity.Info, message);
public void LogDebug(string message) =>
logger.Log(Severity.Debug, message);
private void LogError(string message) =>
logger.Log(Severity.Error, message);
public void FindingFiles(string dir) =>
LogInfo($"Finding files in {dir}...");
public void IndexingReferences(int count)
{
LogInfo("Indexing...");
LogDebug($"Indexing {count} DLLs...");
}
public void UnresolvedReference(string id, string project)
{
LogInfo($"Unresolved reference {id}");
LogDebug($"Unresolved {id} referenced by {project}");
}
public void AnalysingSolution(string filename) =>
LogInfo($"Analyzing {filename}...");
public void FailedProjectFile(string filename, string reason) =>
LogInfo($"Couldn't read project file {filename}: {reason}");
public void FailedNugetCommand(string exe, string args, string message)
{
LogInfo($"Command failed: {exe} {args}");
LogInfo($" {message}");
}
public void NugetInstall(string package) =>
LogInfo($"Restoring {package}...");
public void ResolvedReference(string filename) =>
LogInfo($"Resolved reference {filename}");
public void RemovedReference(string filename) =>
LogInfo($"Removed reference {filename}");
public void Summary(int existingSources, int usedSources, int missingSources,
int references, int unresolvedReferences,
int resolvedConflicts, int totalProjects, int failedProjects,
TimeSpan analysisTime)
{
const int align = 6;
LogInfo("");
LogInfo("Build analysis summary:");
LogInfo($"{existingSources,align} source files in the filesystem");
LogInfo($"{usedSources,align} source files in project files");
LogInfo($"{missingSources,align} sources missing from project files");
LogInfo($"{references,align} resolved references");
LogInfo($"{unresolvedReferences,align} unresolved references");
LogInfo($"{resolvedConflicts,align} resolved assembly conflicts");
LogInfo($"{totalProjects,align} projects");
LogInfo($"{failedProjects,align} missing/failed projects");
LogInfo($"Build analysis completed in {analysisTime}");
}
public void ResolvedConflict(string asm1, string asm2) =>
LogDebug($"Resolved {asm1} as {asm2}");
public void MissingProject(string projectFile) =>
LogInfo($"Solution is missing {projectFile}");
public void CommandFailed(string exe, string arguments, int exitCode) =>
LogError($"Command {exe} {arguments} failed with exit code {exitCode}");
public void MissingNuGet() =>
LogError("Missing nuget.exe");
public void FoundNuGet(string path) =>
LogInfo($"Found nuget.exe at {path}");
public void MissingDotNet() =>
LogError("Missing dotnet CLI");
public void RunningProcess(string command) =>
LogInfo($"Running {command}");
public void FailedToRestoreNugetPackage(string package) =>
LogInfo($"Failed to restore nuget package {package}");
public void FailedToReadFile(string file, Exception ex)
{
LogInfo($"Failed to read file {file}");
LogDebug($"Failed to read file {file}, exception: {ex}");
}
public void MultipleNugetConfig(string[] nugetConfigs) =>
LogInfo($"Found multiple nuget.config files: {string.Join(", ", nugetConfigs)}.");
internal void NoTopLevelNugetConfig() =>
LogInfo("Could not find a top-level nuget.config file.");
internal void RazorSourceGeneratorMissing(string fullPath) =>
LogInfo($"Razor source generator folder {fullPath} does not exist.");
internal void CscMissing(string cscPath) =>
LogInfo($"Csc.exe not found at {cscPath}.");
internal void RazorCscArgs(string args) =>
LogInfo($"Running CSC to generate Razor source files. Args: {args}.");
}
}

23
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Razor.cs
View File

@@ -4,34 +4,37 @@ using System.IO;
using System.Text; using System.Text;
using System.Linq; using System.Linq;
using Semmle.Util; using Semmle.Util;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching namespace Semmle.Extraction.CSharp.DependencyFetching
{ {
internal class Razor internal class Razor
{ {
private readonly DotNetVersion sdk; private readonly DotNetVersion sdk;
private readonly ProgressMonitor progressMonitor; private readonly ILogger logger;
private readonly IDotNet dotNet; private readonly IDotNet dotNet;
private readonly string sourceGeneratorFolder; private readonly string sourceGeneratorFolder;
private readonly string cscPath; private readonly string cscPath;
public Razor(DotNetVersion sdk, IDotNet dotNet, ProgressMonitor progressMonitor) public Razor(DotNetVersion sdk, IDotNet dotNet, ILogger logger)
{ {
this.sdk = sdk; this.sdk = sdk;
this.progressMonitor = progressMonitor; this.logger = logger;
this.dotNet = dotNet; this.dotNet = dotNet;
sourceGeneratorFolder = Path.Combine(this.sdk.FullPath, "Sdks", "Microsoft.NET.Sdk.Razor", "source-generators"); sourceGeneratorFolder = Path.Combine(this.sdk.FullPath, "Sdks", "Microsoft.NET.Sdk.Razor", "source-generators");
this.logger.LogInfo($"Razor source generator folder: {sourceGeneratorFolder}");
if (!Directory.Exists(sourceGeneratorFolder)) if (!Directory.Exists(sourceGeneratorFolder))
{ {
this.progressMonitor.RazorSourceGeneratorMissing(sourceGeneratorFolder); this.logger.LogInfo($"Razor source generator folder {sourceGeneratorFolder} does not exist.");
throw new Exception($"Razor source generator folder {sourceGeneratorFolder} does not exist."); throw new Exception($"Razor source generator folder {sourceGeneratorFolder} does not exist.");
} }
cscPath = Path.Combine(this.sdk.FullPath, "Roslyn", "bincore", "csc.dll"); cscPath = Path.Combine(this.sdk.FullPath, "Roslyn", "bincore", "csc.dll");
this.logger.LogInfo($"Razor source generator CSC: {cscPath}");
if (!File.Exists(cscPath)) if (!File.Exists(cscPath))
{ {
this.progressMonitor.CscMissing(cscPath); this.logger.LogInfo($"Csc.exe not found at {cscPath}.");
throw new Exception($"csc.dll {cscPath} does not exist."); throw new Exception($"csc.dll {cscPath} does not exist.");
} }
} }
@@ -63,7 +66,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{ {
GenerateAnalyzerConfig(cshtmls, analyzerConfig); GenerateAnalyzerConfig(cshtmls, analyzerConfig);
progressMonitor.LogInfo($"Analyzer config content: {File.ReadAllText(analyzerConfig)}"); logger.LogInfo($"Analyzer config content: {File.ReadAllText(analyzerConfig)}");
var args = new StringBuilder(); var args = new StringBuilder();
args.Append($"/target:exe /generatedfilesout:\"{outputFolder}\" /out:\"{dllPath}\" /analyzerconfig:\"{analyzerConfig}\" "); args.Append($"/target:exe /generatedfilesout:\"{outputFolder}\" /out:\"{dllPath}\" /analyzerconfig:\"{analyzerConfig}\" ");
@@ -85,7 +88,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var argsString = args.ToString(); var argsString = args.ToString();
progressMonitor.RazorCscArgs(argsString); logger.LogInfo($"Running CSC to generate Razor source files with arguments: {argsString}.");
using (var sw = new StreamWriter(cscArgsPath)) using (var sw = new StreamWriter(cscArgsPath))
{ {
@@ -94,7 +97,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
dotNet.Exec($"\"{cscPath}\" /noconfig @\"{cscArgsPath}\""); dotNet.Exec($"\"{cscPath}\" /noconfig @\"{cscArgsPath}\"");
return Directory.GetFiles(outputFolder, "*.*", new EnumerationOptions { RecurseSubdirectories = true }); var files = Directory.GetFiles(outputFolder, "*.*", new EnumerationOptions { RecurseSubdirectories = true });
logger.LogInfo($"Generated {files.Length} source files from cshtml files.");
return files;
} }
finally finally
{ {

16
csharp/extractor/Semmle.Extraction.Tests/Assets.cs
View File

@@ -12,7 +12,7 @@ namespace Semmle.Extraction.Tests
public void TestAssets1() public void TestAssets1()
{ {
// Setup // Setup
var assets = new Assets(new ProgressMonitor(new LoggerStub())); var assets = new Assets(new LoggerStub());
var json = assetsJson1; var json = assetsJson1;
var dependencies = new DependencyContainer(); var dependencies = new DependencyContainer();
@@ -43,7 +43,7 @@ namespace Semmle.Extraction.Tests
public void TestAssetsFailure() public void TestAssetsFailure()
{ {
// Setup // Setup
var assets = new Assets(new ProgressMonitor(new LoggerStub())); var assets = new Assets(new LoggerStub());
var json = "garbage data"; var json = "garbage data";
var dependencies = new DependencyContainer(); var dependencies = new DependencyContainer();
@@ -59,7 +59,7 @@ namespace Semmle.Extraction.Tests
public void TestAssetsNet70() public void TestAssetsNet70()
{ {
// Setup // Setup
var assets = new Assets(new ProgressMonitor(new LoggerStub())); var assets = new Assets(new LoggerStub());
var json = assetsNet70; var json = assetsNet70;
var dependencies = new DependencyContainer(); var dependencies = new DependencyContainer();
@@ -90,7 +90,7 @@ namespace Semmle.Extraction.Tests
public void TestAssetsNet48() public void TestAssetsNet48()
{ {
// Setup // Setup
var assets = new Assets(new ProgressMonitor(new LoggerStub())); var assets = new Assets(new LoggerStub());
var json = assetsNet48; var json = assetsNet48;
var dependencies = new DependencyContainer(); var dependencies = new DependencyContainer();
@@ -117,7 +117,7 @@ namespace Semmle.Extraction.Tests
public void TestAssetsNetstandard21() public void TestAssetsNetstandard21()
{ {
// Setup // Setup
var assets = new Assets(new ProgressMonitor(new LoggerStub())); var assets = new Assets(new LoggerStub());
var json = assetsNetstandard21; var json = assetsNetstandard21;
var dependencies = new DependencyContainer(); var dependencies = new DependencyContainer();
@@ -145,7 +145,7 @@ namespace Semmle.Extraction.Tests
public void TestAssetsNetStandard16() public void TestAssetsNetStandard16()
{ {
// Setup // Setup
var assets = new Assets(new ProgressMonitor(new LoggerStub())); var assets = new Assets(new LoggerStub());
var json = assetsNetstandard16; var json = assetsNetstandard16;
var dependencies = new DependencyContainer(); var dependencies = new DependencyContainer();
@@ -177,7 +177,7 @@ namespace Semmle.Extraction.Tests
public void TestAssetsNetcoreapp20() public void TestAssetsNetcoreapp20()
{ {
// Setup // Setup
var assets = new Assets(new ProgressMonitor(new LoggerStub())); var assets = new Assets(new LoggerStub());
var json = assetsNetcoreapp20; var json = assetsNetcoreapp20;
var dependencies = new DependencyContainer(); var dependencies = new DependencyContainer();
@@ -205,7 +205,7 @@ namespace Semmle.Extraction.Tests
public void TestAssetsNetcoreapp31() public void TestAssetsNetcoreapp31()
{ {
// Setup // Setup
var assets = new Assets(new ProgressMonitor(new LoggerStub())); var assets = new Assets(new LoggerStub());
var json = assetsNetcoreapp31; var json = assetsNetcoreapp31;
var dependencies = new DependencyContainer(); var dependencies = new DependencyContainer();

40
csharp/extractor/Semmle.Extraction.Tests/DotNet.cs
View File

@@ -38,7 +38,7 @@ namespace Semmle.Extraction.Tests
public class DotNetTests public class DotNetTests
{ {
private static IDotNet MakeDotnet(IDotNetCliInvoker dotnetCliInvoker) => private static IDotNet MakeDotnet(IDotNetCliInvoker dotnetCliInvoker) =>
DotNet.Make(dotnetCliInvoker, new ProgressMonitor(new LoggerStub())); DotNet.Make(dotnetCliInvoker, new LoggerStub());
private static IList<string> MakeDotnetRestoreOutput() => private static IList<string> MakeDotnetRestoreOutput() =>
new List<string> { new List<string> {
@@ -101,7 +101,7 @@ namespace Semmle.Extraction.Tests
var dotnet = MakeDotnet(dotnetCliInvoker); var dotnet = MakeDotnet(dotnetCliInvoker);
// Execute // Execute
dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages", false, out var assets, out var _); dotnet.Restore(new("myproject.csproj", "mypackages", false));
// Verify // Verify
var lastArgs = dotnetCliInvoker.GetLastArgs(); var lastArgs = dotnetCliInvoker.GetLastArgs();
@@ -116,14 +116,14 @@ namespace Semmle.Extraction.Tests
var dotnet = MakeDotnet(dotnetCliInvoker); var dotnet = MakeDotnet(dotnetCliInvoker);
// Execute // Execute
dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages", false, out var assets, out var _, pathToNugetConfig: "myconfig.config"); var res = dotnet.Restore(new("myproject.csproj", "mypackages", false, "myconfig.config"));
// Verify // Verify
var lastArgs = dotnetCliInvoker.GetLastArgs(); var lastArgs = dotnetCliInvoker.GetLastArgs();
Assert.Equal("restore --no-dependencies \"myproject.csproj\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal --configfile \"myconfig.config\"", lastArgs); Assert.Equal("restore --no-dependencies \"myproject.csproj\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal --configfile \"myconfig.config\"", lastArgs);
Assert.Equal(2, assets.Count()); Assert.Equal(2, res.AssetsFilePaths.Count());
Assert.Contains("/path/to/project.assets.json", assets); Assert.Contains("/path/to/project.assets.json", res.AssetsFilePaths);
Assert.Contains("/path/to/project2.assets.json", assets); Assert.Contains("/path/to/project2.assets.json", res.AssetsFilePaths);
} }
[Fact] [Fact]
@@ -134,14 +134,14 @@ namespace Semmle.Extraction.Tests
var dotnet = MakeDotnet(dotnetCliInvoker); var dotnet = MakeDotnet(dotnetCliInvoker);
// Execute // Execute
dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages", false, out var assets, out var _, pathToNugetConfig: "myconfig.config", force: true); var res = dotnet.Restore(new("myproject.csproj", "mypackages", false, "myconfig.config", true));
// Verify // Verify
var lastArgs = dotnetCliInvoker.GetLastArgs(); var lastArgs = dotnetCliInvoker.GetLastArgs();
Assert.Equal("restore --no-dependencies \"myproject.csproj\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal --configfile \"myconfig.config\" --force", lastArgs); Assert.Equal("restore --no-dependencies \"myproject.csproj\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal --configfile \"myconfig.config\" --force", lastArgs);
Assert.Equal(2, assets.Count()); Assert.Equal(2, res.AssetsFilePaths.Count());
Assert.Contains("/path/to/project.assets.json", assets); Assert.Contains("/path/to/project.assets.json", res.AssetsFilePaths);
Assert.Contains("/path/to/project2.assets.json", assets); Assert.Contains("/path/to/project2.assets.json", res.AssetsFilePaths);
} }
[Fact] [Fact]
@@ -152,17 +152,17 @@ namespace Semmle.Extraction.Tests
var dotnet = MakeDotnet(dotnetCliInvoker); var dotnet = MakeDotnet(dotnetCliInvoker);
// Execute // Execute
dotnet.RestoreSolutionToDirectory("mysolution.sln", "mypackages", false, out var projects, out var assets); var res = dotnet.Restore(new("mysolution.sln", "mypackages", false));
// Verify // Verify
var lastArgs = dotnetCliInvoker.GetLastArgs(); var lastArgs = dotnetCliInvoker.GetLastArgs();
Assert.Equal("restore --no-dependencies \"mysolution.sln\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal", lastArgs); Assert.Equal("restore --no-dependencies \"mysolution.sln\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal", lastArgs);
Assert.Equal(2, projects.Count()); Assert.Equal(2, res.RestoredProjects.Count());
Assert.Contains("/path/to/project.csproj", projects); Assert.Contains("/path/to/project.csproj", res.RestoredProjects);
Assert.Contains("/path/to/project2.csproj", projects); Assert.Contains("/path/to/project2.csproj", res.RestoredProjects);
Assert.Equal(2, assets.Count()); Assert.Equal(2, res.AssetsFilePaths.Count());
Assert.Contains("/path/to/project.assets.json", assets); Assert.Contains("/path/to/project.assets.json", res.AssetsFilePaths);
Assert.Contains("/path/to/project2.assets.json", assets); Assert.Contains("/path/to/project2.assets.json", res.AssetsFilePaths);
} }
[Fact] [Fact]
@@ -174,13 +174,13 @@ namespace Semmle.Extraction.Tests
dotnetCliInvoker.Success = false; dotnetCliInvoker.Success = false;
// Execute // Execute
dotnet.RestoreSolutionToDirectory("mysolution.sln", "mypackages", false, out var projects, out var assets); var res = dotnet.Restore(new("mysolution.sln", "mypackages", false));
// Verify // Verify
var lastArgs = dotnetCliInvoker.GetLastArgs(); var lastArgs = dotnetCliInvoker.GetLastArgs();
Assert.Equal("restore --no-dependencies \"mysolution.sln\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal", lastArgs); Assert.Equal("restore --no-dependencies \"mysolution.sln\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal", lastArgs);
Assert.Empty(projects); Assert.Empty(res.RestoredProjects);
Assert.Empty(assets); Assert.Empty(res.AssetsFilePaths);
} }
[Fact] [Fact]

2
csharp/extractor/Semmle.Extraction.Tests/FileContent.cs
View File

@@ -25,7 +25,7 @@ namespace Semmle.Extraction.Tests
internal class TestFileContent : FileContent internal class TestFileContent : FileContent
{ {
public TestFileContent(IEnumerable<string> lines) : base(new ProgressMonitor(new LoggerStub()), public TestFileContent(IEnumerable<string> lines) : base(new LoggerStub(),
new List<string>() { "test1.cs" }, new List<string>() { "test1.cs" },
new UnsafeFileReaderStub(lines)) new UnsafeFileReaderStub(lines))
{ } { }

48
csharp/extractor/Semmle.Extraction.Tests/FilePathFilter.cs
View File

@@ -10,17 +10,19 @@ namespace Semmle.Extraction.Tests
{ {
public class FilePathFilterTest public class FilePathFilterTest
{ {
private class ProgressMonitorStub : IProgressMonitor private class LoggerStub : ILogger
{ {
public List<string> Messages { get; } = []; public List<string> Messages { get; } = [];
public void Log(Severity severity, string message) public void Log(Severity s, string text, int? threadId = null)
{ {
Messages.Add(message); Messages.Add(text);
}
} }
private static (FilePathFilter TestSubject, ProgressMonitorStub progressMonitor, IEnumerable<FileInfo> Files) TestSetup() public void Dispose() { }
}
private static (FilePathFilter TestSubject, LoggerStub Logger, IEnumerable<FileInfo> Files) TestSetup()
{ {
return TestSetup("/a/b", return TestSetup("/a/b",
[ [
@@ -32,15 +34,15 @@ namespace Semmle.Extraction.Tests
]); ]);
} }
private static (FilePathFilter TestSubject, ProgressMonitorStub progressMonitor, IEnumerable<FileInfo> Files) TestSetup(string root, IEnumerable<string> paths) private static (FilePathFilter TestSubject, LoggerStub Logger, IEnumerable<FileInfo> Files) TestSetup(string root, IEnumerable<string> paths)
{ {
root = GetPlatformSpecifixPath(root); root = GetPlatformSpecifixPath(root);
paths = GetPlatformSpecifixPaths(paths); paths = GetPlatformSpecifixPaths(paths);
var progressMonitor = new ProgressMonitorStub(); var logger = new LoggerStub();
var filePathFilter = new FilePathFilter(new DirectoryInfo(root), progressMonitor); var filePathFilter = new FilePathFilter(new DirectoryInfo(root), logger);
return (filePathFilter, progressMonitor, paths.Select(p => new FileInfo(p))); return (filePathFilter, logger, paths.Select(p => new FileInfo(p)));
} }
private static string GetPlatformSpecifixPath(string file) private static string GetPlatformSpecifixPath(string file)
@@ -67,20 +69,20 @@ namespace Semmle.Extraction.Tests
[Fact] [Fact]
public void TestNoFilter() public void TestNoFilter()
{ {
(var testSubject, var progressMonitor, var files) = TestSetup(); (var testSubject, var logger, var files) = TestSetup();
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", null); Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", null);
var filtered = testSubject.Filter(files); var filtered = testSubject.Filter(files);
AssertFileInfoEquivalence(files, filtered); AssertFileInfoEquivalence(files, filtered);
Assert.Equivalent(Array.Empty<string>(), progressMonitor.Messages, strict: true); Assert.Equivalent(Array.Empty<string>(), logger.Messages, strict: true);
} }
[Fact] [Fact]
public void TestFiltersWithOnlyInclude() public void TestFiltersWithOnlyInclude()
{ {
(var testSubject, var progressMonitor, var files) = TestSetup(); (var testSubject, var logger, var files) = TestSetup();
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """ Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
include:c/d include:c/d
@@ -104,13 +106,13 @@ namespace Semmle.Extraction.Tests
"Filtering in files matching '^c/d.*'. Original glob filter: 'include:c/d'", "Filtering in files matching '^c/d.*'. Original glob filter: 'include:c/d'",
"Filtering in files matching '^c/x/y.*'. Original glob filter: 'include:c/x/y'" "Filtering in files matching '^c/x/y.*'. Original glob filter: 'include:c/x/y'"
}; };
Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false); Assert.Equivalent(expectedRegexMessages, logger.Messages, strict: false);
} }
[Fact] [Fact]
public void TestFiltersWithOnlyExclude() public void TestFiltersWithOnlyExclude()
{ {
(var testSubject, var progressMonitor, var files) = TestSetup(); (var testSubject, var logger, var files) = TestSetup();
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """ Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
exclude:c/d/e exclude:c/d/e
@@ -130,13 +132,13 @@ namespace Semmle.Extraction.Tests
{ {
"Filtering out files matching '^c/d/e.*'. Original glob filter: 'exclude:c/d/e'" "Filtering out files matching '^c/d/e.*'. Original glob filter: 'exclude:c/d/e'"
}; };
Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false); Assert.Equivalent(expectedRegexMessages, logger.Messages, strict: false);
} }
[Fact] [Fact]
public void TestFiltersWithIncludeExclude() public void TestFiltersWithIncludeExclude()
{ {
(var testSubject, var progressMonitor, var files) = TestSetup(); (var testSubject, var logger, var files) = TestSetup();
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """ Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
include:c/x include:c/x
@@ -157,13 +159,13 @@ namespace Semmle.Extraction.Tests
"Filtering in files matching '^c/x.*'. Original glob filter: 'include:c/x'", "Filtering in files matching '^c/x.*'. Original glob filter: 'include:c/x'",
"Filtering out files matching '^c/x/z.*'. Original glob filter: 'exclude:c/x/z'" "Filtering out files matching '^c/x/z.*'. Original glob filter: 'exclude:c/x/z'"
}; };
Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false); Assert.Equivalent(expectedRegexMessages, logger.Messages, strict: false);
} }
[Fact] [Fact]
public void TestFiltersWithIncludeExcludeExcludeFirst() public void TestFiltersWithIncludeExcludeExcludeFirst()
{ {
(var testSubject, var progressMonitor, var files) = TestSetup(); (var testSubject, var logger, var files) = TestSetup();
// NOTE: the ordering DOES matter, later filters takes priority, so the exclude will end up not mattering at all. // NOTE: the ordering DOES matter, later filters takes priority, so the exclude will end up not mattering at all.
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """ Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
@@ -186,13 +188,13 @@ namespace Semmle.Extraction.Tests
"Filtering in files matching '^c/x.*'. Original glob filter: 'include:c/x'", "Filtering in files matching '^c/x.*'. Original glob filter: 'include:c/x'",
"Filtering out files matching '^c/x/z.*'. Original glob filter: 'exclude:c/x/z'" "Filtering out files matching '^c/x/z.*'. Original glob filter: 'exclude:c/x/z'"
}; };
Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false); Assert.Equivalent(expectedRegexMessages, logger.Messages, strict: false);
} }
[Fact] [Fact]
public void TestFiltersWithIncludeExcludeComplexPatterns1() public void TestFiltersWithIncludeExcludeComplexPatterns1()
{ {
(var testSubject, var progressMonitor, var files) = TestSetup(); (var testSubject, var logger, var files) = TestSetup();
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """ Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
include:c/**/i.* include:c/**/i.*
@@ -218,13 +220,13 @@ namespace Semmle.Extraction.Tests
"Filtering in files matching '^c/d/.*/[^/]*\\.cs.*'. Original glob filter: 'include:c/d/**/*.cs'", "Filtering in files matching '^c/d/.*/[^/]*\\.cs.*'. Original glob filter: 'include:c/d/**/*.cs'",
"Filtering out files matching '^.*/z/i\\.cs.*'. Original glob filter: 'exclude:**/z/i.cs'" "Filtering out files matching '^.*/z/i\\.cs.*'. Original glob filter: 'exclude:**/z/i.cs'"
}; };
Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false); Assert.Equivalent(expectedRegexMessages, logger.Messages, strict: false);
} }
[Fact] [Fact]
public void TestFiltersWithIncludeExcludeComplexPatterns2() public void TestFiltersWithIncludeExcludeComplexPatterns2()
{ {
(var testSubject, var progressMonitor, var files) = TestSetup(); (var testSubject, var logger, var files) = TestSetup();
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """ Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
include:**/i.* include:**/i.*
@@ -245,7 +247,7 @@ namespace Semmle.Extraction.Tests
"Filtering in files matching '^.*/i\\.[^/]*.*'. Original glob filter: 'include:**/i.*'", "Filtering in files matching '^.*/i\\.[^/]*.*'. Original glob filter: 'include:**/i.*'",
"Filtering out files matching '^.*/z/i\\.cs.*'. Original glob filter: 'exclude:**/z/i.cs'" "Filtering out files matching '^.*/z/i\\.cs.*'. Original glob filter: 'exclude:**/z/i.cs'"
}; };
Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false); Assert.Equivalent(expectedRegexMessages, logger.Messages, strict: false);
} }
} }
} }

2
csharp/extractor/Semmle.Extraction.Tests/LoggerStub.cs
View File

@@ -4,7 +4,7 @@ namespace Semmle.Extraction.Tests
{ {
internal class LoggerStub : ILogger internal class LoggerStub : ILogger
{ {
public void Log(Severity severity, string message) { } public void Log(Severity severity, string message, int? threadId = null) { }
public void Dispose() { } public void Dispose() { }
} }

14
csharp/extractor/Semmle.Extraction.Tests/Runtime.cs
View File

@@ -19,19 +19,7 @@ namespace Semmle.Extraction.Tests
public bool New(string folder) => true; public bool New(string folder) => true;
public bool RestoreProjectToDirectory(string project, string directory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> assets, out IList<string> outputLines, string? pathToNugetConfig = null, bool force = false) public RestoreResult Restore(RestoreSettings restoreSettings) => new(true, Array.Empty<string>());
{
assets = Array.Empty<string>();
outputLines = Array.Empty<string>();
return true;
}
public bool RestoreSolutionToDirectory(string solution, string directory, bool forceDotnetRefAssemblyFetching, out IEnumerable<string> projects, out IEnumerable<string> assets)
{
projects = Array.Empty<string>();
assets = Array.Empty<string>();
return true;
}
public IList<string> GetListedRuntimes() => runtimes; public IList<string> GetListedRuntimes() => runtimes;

2
csharp/extractor/Semmle.Util.Tests/CanonicalPathCache.cs
View File

@@ -176,7 +176,7 @@ namespace SemmleTests.Semmle.Util
{ {
public void Dispose() { } public void Dispose() { }
public void Log(Severity s, string text) { } public void Log(Severity s, string text, int? threadId = null) { }
} }
} }
} }

2
csharp/extractor/Semmle.Util.Tests/FileUtils.cs
View File

@@ -57,7 +57,7 @@ namespace SemmleTests.Semmle.Util
{ {
public void Dispose() { } public void Dispose() { }
public void Log(Severity s, string text) { } public void Log(Severity s, string text, int? threadId = null) { }
} }
} }
} }

32
csharp/extractor/Semmle.Util/Logger.cs
View File

@@ -37,7 +37,15 @@ namespace Semmle.Util.Logging
/// <summary> /// <summary>
/// Log the given text with the given severity. /// Log the given text with the given severity.
/// </summary> /// </summary>
void Log(Severity s, string text); void Log(Severity s, string text, int? threadId = null);
void LogError(string text, int? threadId = null) => Log(Severity.Error, text, threadId);
void LogWarning(string text, int? threadId = null) => Log(Severity.Warning, text, threadId);
void LogInfo(string text, int? threadId = null) => Log(Severity.Info, text, threadId);
void LogDebug(string text, int? threadId = null) => Log(Severity.Debug, text, threadId);
} }
public static class LoggerExtensions public static class LoggerExtensions
@@ -92,12 +100,14 @@ namespace Semmle.Util.Logging
return "[" + s.ToString().ToUpper() + "] "; return "[" + s.ToString().ToUpper() + "] ";
} }
public void Log(Severity s, string text) public void Log(Severity s, string text, int? threadId = null)
{ {
if (verbosity.Includes(s)) if (verbosity.Includes(s))
{ {
var threadId = this.logThreadId ? $"[{Environment.CurrentManagedThreadId:D3}] " : ""; threadId ??= Environment.CurrentManagedThreadId;
writer.WriteLine(threadId + GetSeverityPrefix(s) + text);
var prefix = this.logThreadId ? $"[{threadId:D3}] " : "";
writer.WriteLine(prefix + GetSeverityPrefix(s) + text);
} }
} }
} }
@@ -140,12 +150,14 @@ namespace Semmle.Util.Logging
} }
} }
public void Log(Severity s, string text) public void Log(Severity s, string text, int? threadId = null)
{ {
if (verbosity.Includes(s)) if (verbosity.Includes(s))
{ {
var threadId = this.logThreadId ? $"[{Environment.CurrentManagedThreadId:D3}] " : ""; threadId ??= Environment.CurrentManagedThreadId;
GetConsole(s).WriteLine(threadId + GetSeverityPrefix(s) + text);
var prefix = this.logThreadId ? $"[{threadId:D3}] " : "";
GetConsole(s).WriteLine(prefix + GetSeverityPrefix(s) + text);
} }
} }
} }
@@ -170,10 +182,10 @@ namespace Semmle.Util.Logging
logger2.Dispose(); logger2.Dispose();
} }
public void Log(Severity s, string text) public void Log(Severity s, string text, int? threadId = null)
{ {
logger1.Log(s, text); logger1.Log(s, text, threadId);
logger2.Log(s, text); logger2.Log(s, text, threadId);
} }
} }