diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs
index 3a124d13e0e..3d6672560b6 100644
--- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs
+++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs
@@ -1,6 +1,7 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
+using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
@@ -18,25 +19,26 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// Paths to search. Directories are searched recursively. Files are added directly to the
/// assembly cache.
///
- /// Callback for progress.
- public AssemblyCache(IEnumerable paths, IEnumerable frameworkPaths, ProgressMonitor progressMonitor)
+ /// Callback for progress.
+ public AssemblyCache(IEnumerable paths, IEnumerable frameworkPaths, ILogger logger)
{
+ this.logger = logger;
foreach (var path in paths)
{
if (File.Exists(path))
{
- pendingDllsToIndex.Enqueue(path);
+ dllsToIndex.Add(path);
continue;
}
if (Directory.Exists(path))
{
- progressMonitor.FindingFiles(path);
+ logger.LogInfo($"Finding reference DLLs in {path}...");
AddReferenceDirectory(path);
}
else
{
- progressMonitor.LogInfo("AssemblyCache: Path not found: " + path);
+ logger.LogInfo("AssemblyCache: Path not found: " + path);
}
}
IndexReferences(frameworkPaths);
@@ -52,7 +54,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
foreach (var dll in new DirectoryInfo(dir).EnumerateFiles("*.dll", SearchOption.AllDirectories))
{
- pendingDllsToIndex.Enqueue(dll.FullName);
+ dllsToIndex.Add(dll.FullName);
}
}
@@ -62,12 +64,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
///
private void IndexReferences(IEnumerable frameworkPaths)
{
+ logger.LogInfo($"Indexing {dllsToIndex.Count} assemblies...");
+
// Read all of the files
- foreach (var filename in pendingDllsToIndex)
+ foreach (var filename in dllsToIndex)
{
IndexReference(filename);
}
+ logger.LogInfo($"Read {assemblyInfoByFileName.Count} assembly infos");
+
foreach (var info in assemblyInfoByFileName.Values
.OrderBy(info => info.Name)
.OrderAssemblyInfosByPreference(frameworkPaths))
@@ -83,25 +89,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
try
{
+ logger.LogDebug($"Reading assembly info from {filename}");
var info = AssemblyInfo.ReadFromFile(filename);
assemblyInfoByFileName[filename] = info;
}
catch (AssemblyLoadException)
{
- failedAssemblyInfoFileNames.Add(filename);
+ logger.LogInfo($"Couldn't read assembly info from {filename}");
}
}
- ///
- /// The number of DLLs which are assemblies.
- ///
- public int AssemblyCount => assemblyInfoByFileName.Count;
-
- ///
- /// The number of DLLs which weren't assemblies. (E.g. C++).
- ///
- public int NonAssemblyCount => failedAssemblyInfoFileNames.Count;
-
///
/// Given an assembly id, determine its full info.
///
@@ -113,8 +110,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (failedAssemblyInfoIds.Contains(id))
throw new AssemblyLoadException();
- string assemblyName;
- (id, assemblyName) = AssemblyInfo.ComputeSanitizedAssemblyInfo(id);
+ (id, var assemblyName) = AssemblyInfo.ComputeSanitizedAssemblyInfo(id);
// Look up the id in our references map.
if (assemblyInfoById.TryGetValue(id, out var result))
@@ -164,17 +160,15 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
throw new AssemblyLoadException();
}
- private readonly Queue pendingDllsToIndex = new Queue();
+ private readonly List dllsToIndex = new List();
private readonly Dictionary assemblyInfoByFileName = new Dictionary();
- // List of DLLs which are not assemblies.
- // We probably don't need to keep this
- private readonly List failedAssemblyInfoFileNames = new List();
-
// Map from assembly id (in various formats) to the full info.
private readonly Dictionary assemblyInfoById = new Dictionary();
private readonly HashSet failedAssemblyInfoIds = new HashSet();
+
+ private readonly ILogger logger;
}
}
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyInfo.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyInfo.cs
index d3522025e27..49328f6a78f 100644
--- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyInfo.cs
+++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyInfo.cs
@@ -120,21 +120,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
NetCoreVersion = netCoreVersion;
}
- ///
- /// Get AssemblyInfo from a loaded Assembly.
- ///
- /// The assembly.
- /// Info about the assembly.
- public static AssemblyInfo MakeFromAssembly(Assembly assembly)
- {
- if (assembly.FullName is null)
- {
- throw new InvalidOperationException("Assembly with empty full name is not expected.");
- }
-
- return new AssemblyInfo(assembly.FullName, assembly.Location);
- }
-
///
/// Returns the id and name of the assembly that would be created from the received id.
///
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Assets.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Assets.cs
index 38db871d1b6..a59991b4b83 100644
--- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Assets.cs
+++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Assets.cs
@@ -5,6 +5,7 @@ using System.IO;
using System.Linq;
using Newtonsoft.Json.Linq;
using Semmle.Util;
+using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
@@ -13,11 +14,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
///
internal class Assets
{
- private readonly ProgressMonitor progressMonitor;
+ private readonly ILogger logger;
- internal Assets(ProgressMonitor progressMonitor)
+ internal Assets(ILogger logger)
{
- this.progressMonitor = progressMonitor;
+ this.logger = logger;
}
///
@@ -35,7 +36,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
///
/// Add the package dependencies from the assets file to dependencies.
- ///
+ ///
/// Parse a part of the JSon assets file and add the paths
/// to the dependencies required for compilation (and collect
/// information about used packages).
@@ -60,7 +61,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// }
/// }
/// }
- ///
+ ///
/// Adds the following dependencies
/// Paths: {
/// "castle.core/4.4.1/lib/netstandard1.5/Castle.Core.dll",
@@ -85,7 +86,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (references is null)
{
- progressMonitor.LogDebug("No references found in the targets section in the assets file.");
+ logger.LogDebug("No references found in the targets section in the assets file.");
return;
}
@@ -157,7 +158,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (frameworks is null)
{
- progressMonitor.LogDebug("No framework section in assets.json.");
+ logger.LogDebug("No framework section in assets.json.");
return;
}
@@ -171,7 +172,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (references is null)
{
- progressMonitor.LogDebug("No framework references in assets.json.");
+ logger.LogDebug("No framework references in assets.json.");
return;
}
@@ -196,12 +197,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
catch (Exception e)
{
- progressMonitor.LogDebug($"Failed to parse assets file (unexpected error): {e.Message}");
+ logger.LogDebug($"Failed to parse assets file (unexpected error): {e.Message}");
return false;
}
}
- private static bool TryReadAllText(string path, ProgressMonitor progressMonitor, [NotNullWhen(returnValue: true)] out string? content)
+ private static bool TryReadAllText(string path, ILogger logger, [NotNullWhen(returnValue: true)] out string? content)
{
try
{
@@ -210,19 +211,19 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
catch (Exception e)
{
- progressMonitor.LogInfo($"Failed to read assets file '{path}': {e.Message}");
+ logger.LogInfo($"Failed to read assets file '{path}': {e.Message}");
content = null;
return false;
}
}
- public static DependencyContainer GetCompilationDependencies(ProgressMonitor progressMonitor, IEnumerable assets)
+ public static DependencyContainer GetCompilationDependencies(ILogger logger, IEnumerable assets)
{
- var parser = new Assets(progressMonitor);
+ var parser = new Assets(logger);
var dependencies = new DependencyContainer();
assets.ForEach(asset =>
{
- if (TryReadAllText(asset, progressMonitor, out var json))
+ if (TryReadAllText(asset, logger, out var json))
{
parser.TryParse(json, dependencies);
}
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
index 710de21c5d0..2dea2390a04 100644
--- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
+++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
@@ -17,14 +17,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
public sealed class DependencyManager : IDisposable
{
private readonly AssemblyCache assemblyCache;
- private readonly ProgressMonitor progressMonitor;
+ private readonly ILogger logger;
+
+ // Only used as a set, but ConcurrentDictionary is the only concurrent set in .NET.
private readonly IDictionary usedReferences = new ConcurrentDictionary();
- private readonly IDictionary sources = new ConcurrentDictionary();
private readonly IDictionary unresolvedReferences = new ConcurrentDictionary();
- private int failedProjects;
- private int succeededProjects;
private readonly List nonGeneratedSources;
private readonly List generatedSources;
+ private int dotnetFrameworkVersionVariantCount = 0;
private int conflictedReferences = 0;
private readonly IDependencyOptions options;
private readonly DirectoryInfo sourceDir;
@@ -49,7 +49,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var startTime = DateTime.Now;
this.options = options;
- this.progressMonitor = new ProgressMonitor(logger);
+ this.logger = logger;
this.sourceDir = new DirectoryInfo(srcDir);
packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName, "packages"));
@@ -60,35 +60,36 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
try
{
- this.dotnet = DotNet.Make(options, progressMonitor, tempWorkingDirectory);
+ this.dotnet = DotNet.Make(options, logger, tempWorkingDirectory);
runtimeLazy = new Lazy(() => new Runtime(dotnet));
}
catch
{
- progressMonitor.MissingDotNet();
+ logger.LogError("Missing dotnet CLI");
throw;
}
- this.progressMonitor.FindingFiles(srcDir);
-
+ logger.LogInfo($"Finding files in {srcDir}...");
var allFiles = GetAllFiles().ToList();
var binaryFileExtensions = new HashSet(new[] { ".dll", ".exe" }); // TODO: add more binary file extensions.
var allNonBinaryFiles = allFiles.Where(f => !binaryFileExtensions.Contains(f.Extension.ToLowerInvariant())).ToList();
- var smallNonBinaryFiles = allNonBinaryFiles.SelectSmallFiles(progressMonitor).SelectFileNames();
- this.fileContent = new FileContent(progressMonitor, smallNonBinaryFiles);
+ var smallNonBinaryFiles = allNonBinaryFiles.SelectSmallFiles(logger).SelectFileNames();
+ this.fileContent = new FileContent(logger, smallNonBinaryFiles);
this.nonGeneratedSources = allNonBinaryFiles.SelectFileNamesByExtension(".cs").ToList();
this.generatedSources = new();
- var allProjects = allNonBinaryFiles.SelectFileNamesByExtension(".csproj");
- var allSolutions = allNonBinaryFiles.SelectFileNamesByExtension(".sln");
+ var allProjects = allNonBinaryFiles.SelectFileNamesByExtension(".csproj").ToList();
+ var allSolutions = allNonBinaryFiles.SelectFileNamesByExtension(".sln").ToList();
var dllPaths = allFiles.SelectFileNamesByExtension(".dll").ToHashSet();
+ logger.LogInfo($"Found {allFiles.Count} files, {nonGeneratedSources.Count} source files, {allProjects.Count} project files, {allSolutions.Count} solution files, {dllPaths.Count} DLLs.");
+
RestoreNugetPackages(allNonBinaryFiles, allProjects, allSolutions, dllPaths);
// Find DLLs in the .Net / Asp.Net Framework
// This needs to come after the nuget restore, because the nuget restore might fetch the .NET Core/Framework reference assemblies.
var frameworkLocations = AddFrameworkDlls(dllPaths);
- assemblyCache = new AssemblyCache(dllPaths, frameworkLocations, progressMonitor);
+ assemblyCache = new AssemblyCache(dllPaths, frameworkLocations, logger);
AnalyseSolutions(allSolutions);
foreach (var filename in assemblyCache.AllAssemblies.Select(a => a.Filename))
@@ -102,33 +103,36 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
// Output the findings
foreach (var r in usedReferences.Keys.OrderBy(r => r))
{
- progressMonitor.ResolvedReference(r);
+ logger.LogInfo($"Resolved reference {r}");
}
foreach (var r in unresolvedReferences.OrderBy(r => r.Key))
{
- progressMonitor.UnresolvedReference(r.Key, r.Value);
+ logger.LogInfo($"Unresolved reference {r.Key} in project {r.Value}");
}
var webViewExtractionOption = Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_STANDALONE_EXTRACT_WEB_VIEWS");
if (bool.TryParse(webViewExtractionOption, out var shouldExtractWebViews) &&
shouldExtractWebViews)
{
+ logger.LogInfo("Generating source files from cshtml and razor files...");
GenerateSourceFilesFromWebViews(allNonBinaryFiles);
}
GenerateSourceFileFromImplicitUsings();
- progressMonitor.Summary(
- AllSourceFiles.Count(),
- ProjectSourceFiles.Count(),
- MissingSourceFiles.Count(),
- ReferenceFiles.Count(),
- UnresolvedReferences.Count(),
- conflictedReferences,
- succeededProjects + failedProjects,
- failedProjects,
- DateTime.Now - startTime);
+ const int align = 6;
+ logger.LogInfo("");
+ logger.LogInfo("Build analysis summary:");
+ logger.LogInfo($"{nonGeneratedSources.Count,align} source files in the filesystem");
+ logger.LogInfo($"{generatedSources.Count,align} generated source files");
+ logger.LogInfo($"{allSolutions.Count,align} solution files");
+ logger.LogInfo($"{allProjects.Count,align} project files in the filesystem");
+ logger.LogInfo($"{usedReferences.Keys.Count,align} resolved references");
+ logger.LogInfo($"{unresolvedReferences.Count,align} unresolved references");
+ logger.LogInfo($"{conflictedReferences,align} resolved assembly conflicts");
+ logger.LogInfo($"{dotnetFrameworkVersionVariantCount,align} restored .NET framework variants");
+ logger.LogInfo($"Build analysis completed in {DateTime.Now - startTime}");
}
private HashSet AddFrameworkDlls(HashSet dllPaths)
@@ -146,32 +150,42 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
try
{
- var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, progressMonitor);
+ var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, logger);
nuget.InstallPackages();
var nugetPackageDlls = legacyPackageDirectory.DirInfo.GetFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = true });
var nugetPackageDllPaths = nugetPackageDlls.Select(f => f.FullName).ToHashSet();
var excludedPaths = nugetPackageDllPaths
- .Where(path => IsPathInSubfolder(path, legacyPackageDirectory.DirInfo.FullName, "tools"));
+ .Where(path => IsPathInSubfolder(path, legacyPackageDirectory.DirInfo.FullName, "tools"))
+ .ToList();
+
+ if (nugetPackageDllPaths.Count > 0)
+ {
+ logger.LogInfo($"Restored {nugetPackageDllPaths.Count} Nuget DLLs.");
+ }
+ if (excludedPaths.Count > 0)
+ {
+ logger.LogInfo($"Excluding {excludedPaths.Count} Nuget DLLs.");
+ }
foreach (var excludedPath in excludedPaths)
{
- progressMonitor.LogInfo($"Excluded Nuget DLL: {excludedPath}");
+ logger.LogInfo($"Excluded Nuget DLL: {excludedPath}");
}
nugetPackageDllPaths.ExceptWith(excludedPaths);
dllPaths.UnionWith(nugetPackageDllPaths);
}
- catch (FileNotFoundException)
+ catch (Exception)
{
- progressMonitor.MissingNuGet();
+ logger.LogError("Failed to restore Nuget packages with nuget.exe");
}
var restoredProjects = RestoreSolutions(allSolutions, out var assets1);
var projects = allProjects.Except(restoredProjects);
RestoreProjects(projects, out var assets2);
- var dependencies = Assets.GetCompilationDependencies(progressMonitor, assets1.Union(assets2));
+ var dependencies = Assets.GetCompilationDependencies(logger, assets1.Union(assets2));
var paths = dependencies
.Paths
@@ -220,7 +234,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (isInAnalyzersFolder)
{
usedReferences.Remove(filename);
- progressMonitor.RemovedReference(filename);
+ logger.LogInfo($"Removed analyzer reference {filename}");
}
}
}
@@ -228,27 +242,31 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private void SelectNewestFrameworkPath(string frameworkPath, string frameworkType, ISet dllPaths, ISet frameworkLocations)
{
- var versionFolders = new DirectoryInfo(frameworkPath)
- .EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
- .OrderByDescending(d => d.Name) // TODO: Improve sorting to handle pre-release versions.
- .ToArray();
-
+ var versionFolders = GetPackageVersionSubDirectories(frameworkPath);
if (versionFolders.Length > 1)
{
var versions = string.Join(", ", versionFolders.Select(d => d.Name));
- progressMonitor.LogInfo($"Found multiple {frameworkType} DLLs in NuGet packages at {frameworkPath}. Using the latest version ({versionFolders[0].Name}) from: {versions}.");
+ logger.LogInfo($"Found multiple {frameworkType} DLLs in NuGet packages at {frameworkPath}. Using the latest version ({versionFolders[0].Name}) from: {versions}.");
}
var selectedFrameworkFolder = versionFolders.FirstOrDefault()?.FullName;
if (selectedFrameworkFolder is null)
{
- progressMonitor.LogInfo($"Found {frameworkType} DLLs in NuGet packages at {frameworkPath}, but no version folder was found.");
+ logger.LogInfo($"Found {frameworkType} DLLs in NuGet packages at {frameworkPath}, but no version folder was found.");
selectedFrameworkFolder = frameworkPath;
}
dllPaths.Add(selectedFrameworkFolder);
frameworkLocations.Add(selectedFrameworkFolder);
- progressMonitor.LogInfo($"Found {frameworkType} DLLs in NuGet packages at {selectedFrameworkFolder}. Not adding installation directory.");
+ logger.LogInfo($"Found {frameworkType} DLLs in NuGet packages at {selectedFrameworkFolder}.");
+ }
+
+ private static DirectoryInfo[] GetPackageVersionSubDirectories(string packagePath)
+ {
+ return new DirectoryInfo(packagePath)
+ .EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
+ .OrderByDescending(d => d.Name) // TODO: Improve sorting to handle pre-release versions.
+ .ToArray();
}
private void AddNetFrameworkDlls(ISet dllPaths, ISet frameworkLocations)
@@ -257,12 +275,20 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
// The order of the packages is important, we're adding the first one that is present in the nuget cache.
var packagesInPrioOrder = FrameworkPackageNames.NetFrameworks;
- var frameworkPath = packagesInPrioOrder
+ var frameworkPaths = packagesInPrioOrder
.Select((s, index) => (Index: index, Path: GetPackageDirectory(s)))
- .FirstOrDefault(pair => pair.Path is not null);
+ .Where(pair => pair.Path is not null)
+ .ToArray();
+
+ var frameworkPath = frameworkPaths.FirstOrDefault();
if (frameworkPath.Path is not null)
{
+ foreach (var fp in frameworkPaths)
+ {
+ dotnetFrameworkVersionVariantCount += GetPackageVersionSubDirectories(fp.Path!).Length;
+ }
+
SelectNewestFrameworkPath(frameworkPath.Path, ".NET Framework", dllPaths, frameworkLocations);
for (var i = frameworkPath.Index + 1; i < packagesInPrioOrder.Length; i++)
@@ -286,7 +312,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
runtimeLocation ??= Runtime.ExecutingRuntime;
- progressMonitor.LogInfo($".NET runtime location selected: {runtimeLocation}");
+ logger.LogInfo($".NET runtime location selected: {runtimeLocation}");
dllPaths.Add(runtimeLocation);
frameworkLocations.Add(runtimeLocation);
}
@@ -300,11 +326,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
var packagePathPrefix = Path.Combine(packageFolder, packagePrefix.ToLowerInvariant());
- var toRemove = dllPaths.Where(s => s.ToLowerInvariant().StartsWith(packagePathPrefix));
+ var toRemove = dllPaths.Where(s => s.StartsWith(packagePathPrefix, StringComparison.InvariantCultureIgnoreCase));
foreach (var path in toRemove)
{
dllPaths.Remove(path);
- progressMonitor.RemovedReference(path);
+ logger.LogInfo($"Removed reference {path}");
}
}
@@ -324,7 +350,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (Runtime.AspNetCoreRuntime is string aspNetCoreRuntime)
{
- progressMonitor.LogInfo($"ASP.NET runtime location selected: {aspNetCoreRuntime}");
+ logger.LogInfo($"ASP.NET runtime location selected: {aspNetCoreRuntime}");
dllPaths.Add(aspNetCoreRuntime);
frameworkLocations.Add(aspNetCoreRuntime);
}
@@ -346,18 +372,26 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
.FullName;
}
- private IEnumerable GetAllPackageDirectories()
+ private ICollection GetAllPackageDirectories()
{
return new DirectoryInfo(packageDirectory.DirInfo.FullName)
.EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
- .Select(d => d.Name);
+ .Select(d => d.Name)
+ .ToList();
}
- private void LogAllUnusedPackages(DependencyContainer dependencies) =>
- GetAllPackageDirectories()
+ private void LogAllUnusedPackages(DependencyContainer dependencies)
+ {
+ var allPackageDirectories = GetAllPackageDirectories();
+
+ logger.LogInfo($"Restored {allPackageDirectories.Count} packages");
+ logger.LogInfo($"Found {dependencies.Packages.Count} packages in project.asset.json files");
+
+ allPackageDirectories
.Where(package => !dependencies.Packages.Contains(package))
.Order()
- .ForEach(package => progressMonitor.LogInfo($"Unused package: {package}"));
+ .ForEach(package => logger.LogInfo($"Unused package: {package}"));
+ }
private void GenerateSourceFileFromImplicitUsings()
{
@@ -380,6 +414,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
usings.UnionWith(fileContent.CustomImplicitUsings);
+ logger.LogInfo($"Generating source file for implicit usings. Namespaces: {string.Join(", ", usings.OrderBy(u => u))}");
+
if (usings.Count > 0)
{
var tempDir = GetTemporaryWorkingDirectory("implicitUsings");
@@ -401,29 +437,28 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private void GenerateSourceFilesFromWebViews(List allFiles)
{
- progressMonitor.LogInfo($"Generating source files from cshtml and razor files.");
-
var views = allFiles.SelectFileNamesByExtension(".cshtml", ".razor").ToArray();
-
- if (views.Length > 0)
+ if (views.Length == 0)
{
- progressMonitor.LogInfo($"Found {views.Length} cshtml and razor files.");
+ return;
+ }
- var sdk = new Sdk(dotnet).GetNewestSdk();
- if (sdk != null)
+ logger.LogInfo($"Found {views.Length} cshtml and razor files.");
+
+ var sdk = new Sdk(dotnet).GetNewestSdk();
+ if (sdk != null)
+ {
+ try
{
- try
- {
- var razor = new Razor(sdk, dotnet, progressMonitor);
- var targetDir = GetTemporaryWorkingDirectory("razor");
- var generatedFiles = razor.GenerateFiles(views, usedReferences.Keys, targetDir);
- this.generatedSources.AddRange(generatedFiles);
- }
- catch (Exception ex)
- {
- // It's okay, we tried our best to generate source files from cshtml files.
- progressMonitor.LogInfo($"Failed to generate source files from cshtml files: {ex.Message}");
- }
+ var razor = new Razor(sdk, dotnet, logger);
+ var targetDir = GetTemporaryWorkingDirectory("razor");
+ var generatedFiles = razor.GenerateFiles(views, usedReferences.Keys, targetDir);
+ this.generatedSources.AddRange(generatedFiles);
+ }
+ catch (Exception ex)
+ {
+ // It's okay, we tried our best to generate source files from cshtml files.
+ logger.LogInfo($"Failed to generate source files from cshtml files: {ex.Message}");
}
}
}
@@ -448,17 +483,17 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
return true;
}
- progressMonitor.Log(Severity.Warning, $"File {f.FullName} could not be processed.");
+ logger.Log(Severity.Warning, $"File {f.FullName} could not be processed.");
return false;
}
catch (Exception ex)
{
- progressMonitor.Log(Severity.Warning, $"File {f.FullName} could not be processed: {ex.Message}");
+ logger.Log(Severity.Warning, $"File {f.FullName} could not be processed: {ex.Message}");
return false;
}
});
- files = new FilePathFilter(sourceDir, progressMonitor).Filter(files);
+ files = new FilePathFilter(sourceDir, logger).Filter(files);
return files;
}
@@ -499,7 +534,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
///
private void ResolveConflicts(IEnumerable frameworkPaths)
{
- var sortedReferences = new List();
+ var sortedReferences = new List(usedReferences.Count);
foreach (var usedReference in usedReferences)
{
try
@@ -509,7 +544,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
catch (AssemblyLoadException)
{
- progressMonitor.Log(Util.Logging.Severity.Warning, $"Could not load assembly information from {usedReference.Key}");
+ logger.Log(Severity.Warning, $"Could not load assembly information from {usedReference.Key}");
}
}
@@ -517,6 +552,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
.OrderAssemblyInfosByPreference(frameworkPaths)
.ToList();
+ logger.LogInfo($"Reference list contains {sortedReferences.Count} assemblies");
+
var finalAssemblyList = new Dictionary();
// Pick the highest version for each assembly name
@@ -532,15 +569,24 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
UseReference(r);
}
+ logger.LogInfo($"After conflict resolution, reference list contains {finalAssemblyList.Count} assemblies");
+
// Report the results
foreach (var r in sortedReferences)
{
var resolvedInfo = finalAssemblyList[r.Name];
if (resolvedInfo.Version != r.Version || resolvedInfo.NetCoreVersion != r.NetCoreVersion)
{
- progressMonitor.ResolvedConflict(r.Id, resolvedInfo.Id + resolvedInfo.NetCoreVersion is null ? "" : $" (.NET Core {resolvedInfo.NetCoreVersion})");
+ var asm = resolvedInfo.Id + (resolvedInfo.NetCoreVersion is null ? "" : $" (.NET Core {resolvedInfo.NetCoreVersion})");
+ logger.LogInfo($"Resolved {r.Id} as {asm}");
+
++conflictedReferences;
}
+
+ if (r != resolvedInfo)
+ {
+ logger.LogDebug($"Resolved {r.Id} as {resolvedInfo.Id} from {resolvedInfo.Filename}");
+ }
}
}
@@ -550,22 +596,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// The filename of the reference.
private void UseReference(string reference) => usedReferences[reference] = true;
- ///
- /// Store that a particular source file is used (by a project file).
- ///
- /// The source file.
- private void UseSource(FileInfo sourceFile) => sources[sourceFile.FullName] = sourceFile.Exists;
-
///
/// The list of resolved reference files.
///
public IEnumerable ReferenceFiles => usedReferences.Keys;
- ///
- /// The list of source files used in projects.
- ///
- public IEnumerable ProjectSourceFiles => sources.Where(s => s.Value).Select(s => s.Key);
-
///
/// All of the generated source files in the source directory.
///
@@ -581,12 +616,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
///
public IEnumerable UnresolvedReferences => unresolvedReferences.Select(r => r.Key);
- ///
- /// List of source files which were mentioned in project files but
- /// do not exist on the file system.
- ///
- public IEnumerable MissingSourceFiles => sources.Where(s => !s.Value).Select(s => s.Key);
-
///
/// Record that a particular reference couldn't be resolved.
/// Note that this records at most one project file per missing reference.
@@ -595,23 +624,31 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// The project file making the reference.
private void UnresolvedReference(string id, string projectFile) => unresolvedReferences[id] = projectFile;
- ///
- /// Reads all the source files and references from the given list of projects.
- ///
- /// The list of projects to analyse.
- private void AnalyseProjectFiles(IEnumerable projectFiles)
+ private void AnalyseSolutions(IEnumerable solutions)
{
- foreach (var proj in projectFiles)
+ Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, solutionFile =>
{
- AnalyseProject(proj);
- }
+ try
+ {
+ var sln = new SolutionFile(solutionFile);
+ logger.LogInfo($"Analyzing {solutionFile}...");
+ foreach (var proj in sln.Projects.Select(p => new FileInfo(p)))
+ {
+ AnalyseProject(proj);
+ }
+ }
+ catch (Microsoft.Build.Exceptions.InvalidProjectFileException ex)
+ {
+ logger.LogInfo($"Couldn't read solution file {solutionFile}: {ex.BaseMessage}");
+ }
+ });
}
private void AnalyseProject(FileInfo project)
{
if (!project.Exists)
{
- progressMonitor.MissingProject(project.FullName);
+ logger.LogInfo($"Couldn't read project file {project.FullName}");
return;
}
@@ -631,23 +668,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
UnresolvedReference(@ref, project.FullName);
}
}
-
- foreach (var src in csProj.Sources)
- {
- // Make a note of which source files the projects use.
- // This information doesn't affect the build but is dumped
- // as diagnostic output.
- UseSource(new FileInfo(src));
- }
-
- ++succeededProjects;
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{
- ++failedProjects;
- progressMonitor.FailedProjectFile(project.FullName, ex.Message);
+ logger.LogInfo($"Couldn't read project file {project.FullName}: {ex.Message}");
}
-
}
///
@@ -664,9 +689,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var assetFiles = new List();
var projects = solutions.SelectMany(solution =>
{
- dotnet.RestoreSolutionToDirectory(solution, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: true, out var restoredProjects, out var a);
- assetFiles.AddRange(a);
- return restoredProjects;
+ logger.LogInfo($"Restoring solution {solution}...");
+ var res = dotnet.Restore(new(solution, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
+ assetFiles.AddRange(res.AssetsFilePaths);
+ return res.RestoredProjects;
});
assets = assetFiles;
return projects;
@@ -683,26 +709,39 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var assetFiles = new List();
Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, project =>
{
- dotnet.RestoreProjectToDirectory(project, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: true, out var a, out var _);
- assetFiles.AddRange(a);
+ logger.LogInfo($"Restoring project {project}...");
+ var res = dotnet.Restore(new(project, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
+ assetFiles.AddRange(res.AssetsFilePaths);
});
assets = assetFiles;
}
private void DownloadMissingPackages(List allFiles, ISet dllPaths)
{
+ var alreadyDownloadedPackages = Directory.GetDirectories(packageDirectory.DirInfo.FullName)
+ .Select(d => Path.GetFileName(d).ToLowerInvariant());
+ var notYetDownloadedPackages = fileContent.AllPackages
+ .Except(alreadyDownloadedPackages)
+ .ToList();
+ if (notYetDownloadedPackages.Count == 0)
+ {
+ return;
+ }
+
+ logger.LogInfo($"Found {notYetDownloadedPackages.Count} packages that are not yet restored");
+
var nugetConfigs = allFiles.SelectFileNamesByName("nuget.config").ToArray();
string? nugetConfig = null;
if (nugetConfigs.Length > 1)
{
- progressMonitor.MultipleNugetConfig(nugetConfigs);
+ logger.LogInfo($"Found multiple nuget.config files: {string.Join(", ", nugetConfigs)}.");
nugetConfig = allFiles
.SelectRootFiles(sourceDir)
.SelectFileNamesByName("nuget.config")
.FirstOrDefault();
if (nugetConfig == null)
{
- progressMonitor.NoTopLevelNugetConfig();
+ logger.LogInfo("Could not find a top-level nuget.config file.");
}
}
else
@@ -710,13 +749,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
nugetConfig = nugetConfigs.FirstOrDefault();
}
- var alreadyDownloadedPackages = Directory.GetDirectories(packageDirectory.DirInfo.FullName)
- .Select(d => Path.GetFileName(d).ToLowerInvariant());
- var notYetDownloadedPackages = fileContent.AllPackages.Except(alreadyDownloadedPackages);
+ if (nugetConfig != null)
+ {
+ logger.LogInfo($"Using nuget.config file {nugetConfig}.");
+ }
Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, package =>
{
- progressMonitor.NugetInstall(package);
+ logger.LogInfo($"Restoring package {package}...");
using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package, "missingpackages_workingdir"));
var success = dotnet.New(tempDir.DirInfo.FullName);
if (!success)
@@ -730,20 +770,20 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
return;
}
- success = dotnet.RestoreProjectToDirectory(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: false, out var _, out var outputLines, pathToNugetConfig: nugetConfig);
- if (!success)
+ var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig));
+ if (!res.Success)
{
- if (outputLines?.Any(s => s.Contains("NU1301")) == true)
+ if (res.HasNugetPackageSourceError)
{
// Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
- success = dotnet.RestoreProjectToDirectory(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: false, out var _, out var _, pathToNugetConfig: null, force: true);
+ res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true));
}
// TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package.
- if (!success)
+ if (!res.Success)
{
- progressMonitor.FailedToRestoreNugetPackage(package);
+ logger.LogInfo($"Failed to restore nuget package {package}");
}
}
});
@@ -751,23 +791,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
dllPaths.Add(missingPackageDirectory.DirInfo.FullName);
}
- private void AnalyseSolutions(IEnumerable solutions)
- {
- Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, solutionFile =>
- {
- try
- {
- var sln = new SolutionFile(solutionFile);
- progressMonitor.AnalysingSolution(solutionFile);
- AnalyseProjectFiles(sln.Projects.Select(p => new FileInfo(p)).Where(p => p.Exists));
- }
- catch (Microsoft.Build.Exceptions.InvalidProjectFileException ex)
- {
- progressMonitor.FailedProjectFile(solutionFile, ex.BaseMessage);
- }
- });
- }
-
public void Dispose(TemporaryDirectory? dir, string name)
{
try
@@ -776,7 +799,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
catch (Exception exc)
{
- progressMonitor.LogInfo($"Couldn't delete {name} directory {exc.Message}");
+ logger.LogInfo($"Couldn't delete {name} directory {exc.Message}");
}
}
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs
index 9b53d47a242..37c028920a8 100644
--- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs
+++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs
@@ -4,6 +4,7 @@ using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using Semmle.Util;
+using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
@@ -13,22 +14,22 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
internal partial class DotNet : IDotNet
{
private readonly IDotNetCliInvoker dotnetCliInvoker;
- private readonly ProgressMonitor progressMonitor;
+ private readonly ILogger logger;
private readonly TemporaryDirectory? tempWorkingDirectory;
- private DotNet(IDotNetCliInvoker dotnetCliInvoker, ProgressMonitor progressMonitor, TemporaryDirectory? tempWorkingDirectory = null)
+ private DotNet(IDotNetCliInvoker dotnetCliInvoker, ILogger logger, TemporaryDirectory? tempWorkingDirectory = null)
{
- this.progressMonitor = progressMonitor;
+ this.logger = logger;
this.tempWorkingDirectory = tempWorkingDirectory;
this.dotnetCliInvoker = dotnetCliInvoker;
Info();
}
- private DotNet(IDependencyOptions options, ProgressMonitor progressMonitor, TemporaryDirectory tempWorkingDirectory) : this(new DotNetCliInvoker(progressMonitor, Path.Combine(options.DotNetPath ?? string.Empty, "dotnet")), progressMonitor, tempWorkingDirectory) { }
+ private DotNet(IDependencyOptions options, ILogger logger, TemporaryDirectory tempWorkingDirectory) : this(new DotNetCliInvoker(logger, Path.Combine(options.DotNetPath ?? string.Empty, "dotnet")), logger, tempWorkingDirectory) { }
- internal static IDotNet Make(IDotNetCliInvoker dotnetCliInvoker, ProgressMonitor progressMonitor) => new DotNet(dotnetCliInvoker, progressMonitor);
+ internal static IDotNet Make(IDotNetCliInvoker dotnetCliInvoker, ILogger logger) => new DotNet(dotnetCliInvoker, logger);
- public static IDotNet Make(IDependencyOptions options, ProgressMonitor progressMonitor, TemporaryDirectory tempWorkingDirectory) => new DotNet(options, progressMonitor, tempWorkingDirectory);
+ public static IDotNet Make(IDependencyOptions options, ILogger logger, TemporaryDirectory tempWorkingDirectory) => new DotNet(options, logger, tempWorkingDirectory);
private void Info()
{
@@ -40,11 +41,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
}
- private string GetRestoreArgs(string projectOrSolutionFile, string packageDirectory, bool forceDotnetRefAssemblyFetching)
+ private string GetRestoreArgs(RestoreSettings restoreSettings)
{
- var args = $"restore --no-dependencies \"{projectOrSolutionFile}\" --packages \"{packageDirectory}\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal";
+ var args = $"restore --no-dependencies \"{restoreSettings.File}\" --packages \"{restoreSettings.PackageDirectory}\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal";
- if (forceDotnetRefAssemblyFetching)
+ if (restoreSettings.ForceDotnetRefAssemblyFetching)
{
// Ugly hack: we set the TargetFrameworkRootPath and NetCoreTargetingPackRoot properties to an empty folder:
var path = ".empty";
@@ -57,46 +58,24 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
args += $" /p:TargetFrameworkRootPath=\"{path}\" /p:NetCoreTargetingPackRoot=\"{path}\"";
}
- return args;
- }
-
- private static IEnumerable GetFirstGroupOnMatch(Regex regex, IEnumerable lines) =>
- lines
- .Select(line => regex.Match(line))
- .Where(match => match.Success)
- .Select(match => match.Groups[1].Value);
-
- private static IEnumerable GetAssetsFilePaths(IEnumerable lines) =>
- GetFirstGroupOnMatch(AssetsFileRegex(), lines);
-
- private static IEnumerable GetRestoredProjects(IEnumerable lines) =>
- GetFirstGroupOnMatch(RestoredProjectRegex(), lines);
-
- public bool RestoreProjectToDirectory(string projectFile, string packageDirectory, bool forceDotnetRefAssemblyFetching, out IEnumerable assets, out IList outputLines, string? pathToNugetConfig = null, bool force = false)
- {
- var args = GetRestoreArgs(projectFile, packageDirectory, forceDotnetRefAssemblyFetching);
- if (pathToNugetConfig != null)
+ if (restoreSettings.PathToNugetConfig != null)
{
- args += $" --configfile \"{pathToNugetConfig}\"";
+ args += $" --configfile \"{restoreSettings.PathToNugetConfig}\"";
}
- if (force)
+ if (restoreSettings.ForceReevaluation)
{
args += " --force";
}
- var success = dotnetCliInvoker.RunCommand(args, out outputLines);
- assets = success ? GetAssetsFilePaths(outputLines) : Array.Empty();
- return success;
+ return args;
}
- public bool RestoreSolutionToDirectory(string solutionFile, string packageDirectory, bool forceDotnetRefAssemblyFetching, out IEnumerable projects, out IEnumerable assets)
+ public RestoreResult Restore(RestoreSettings restoreSettings)
{
- var args = GetRestoreArgs(solutionFile, packageDirectory, forceDotnetRefAssemblyFetching);
+ var args = GetRestoreArgs(restoreSettings);
var success = dotnetCliInvoker.RunCommand(args, out var output);
- projects = success ? GetRestoredProjects(output) : Array.Empty();
- assets = success ? GetAssetsFilePaths(output) : Array.Empty();
- return success;
+ return new(success, output);
}
public bool New(string folder)
@@ -129,11 +108,5 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var args = $"exec {execArgs}";
return dotnetCliInvoker.RunCommand(args);
}
-
- [GeneratedRegex("Restored\\s+(.+\\.csproj)", RegexOptions.Compiled)]
- private static partial Regex RestoredProjectRegex();
-
- [GeneratedRegex("[Assets\\sfile\\shas\\snot\\schanged.\\sSkipping\\sassets\\sfile\\swriting.|Writing\\sassets\\sfile\\sto\\sdisk.]\\sPath:\\s(.+)", RegexOptions.Compiled)]
- private static partial Regex AssetsFileRegex();
}
}
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs
index 5e7c2b60803..a39124d77b9 100644
--- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs
+++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNetCliInvoker.cs
@@ -2,6 +2,7 @@ using System;
using System.Collections.Generic;
using System.Diagnostics;
using Semmle.Util;
+using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
@@ -10,13 +11,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
///
internal sealed class DotNetCliInvoker : IDotNetCliInvoker
{
- private readonly ProgressMonitor progressMonitor;
+ private readonly ILogger logger;
public string Exec { get; }
- public DotNetCliInvoker(ProgressMonitor progressMonitor, string exec)
+ public DotNetCliInvoker(ILogger logger, string exec)
{
- this.progressMonitor = progressMonitor;
+ this.logger = logger;
this.Exec = exec;
}
@@ -35,21 +36,15 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private bool RunCommandAux(string args, out IList output)
{
- progressMonitor.RunningProcess($"{Exec} {args}");
+ logger.LogInfo($"Running {Exec} {args}");
var pi = MakeDotnetStartInfo(args);
- var threadId = $"[{Environment.CurrentManagedThreadId:D3}]";
- void onOut(string s)
- {
- Console.Out.WriteLine($"{threadId} {s}");
- }
- void onError(string s)
- {
- Console.Error.WriteLine($"{threadId} {s}");
- }
+ var threadId = Environment.CurrentManagedThreadId;
+ void onOut(string s) => logger.LogInfo(s, threadId);
+ void onError(string s) => logger.LogError(s, threadId);
var exitCode = pi.ReadOutput(out output, onOut, onError);
if (exitCode != 0)
{
- progressMonitor.CommandFailed(Exec, args, exitCode);
+ logger.LogError($"Command {Exec} {args} failed with exit code {exitCode}");
return false;
}
return true;
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs
index 578ae81ebe2..c06eaec270f 100644
--- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs
+++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs
@@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.IO;
using System.Text.RegularExpressions;
using Semmle.Util;
+using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
@@ -15,7 +16,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
//
internal partial class FileContent
{
- private readonly ProgressMonitor progressMonitor;
+ private readonly ILogger logger;
private readonly IUnsafeFileReader unsafeFileReader;
private readonly IEnumerable files;
private readonly HashSet allPackages = new HashSet();
@@ -90,18 +91,18 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
}
- internal FileContent(ProgressMonitor progressMonitor,
+ internal FileContent(ILogger logger,
IEnumerable files,
IUnsafeFileReader unsafeFileReader)
{
- this.progressMonitor = progressMonitor;
+ this.logger = logger;
this.files = files;
this.unsafeFileReader = unsafeFileReader;
this.initialize = new Initializer(DoInitialize);
}
- public FileContent(ProgressMonitor progressMonitor, IEnumerable files) : this(progressMonitor, files, new UnsafeFileReader())
+ public FileContent(ILogger logger, IEnumerable files) : this(logger, files, new UnsafeFileReader())
{ }
private static string GetGroup(ReadOnlySpan input, ValueMatch valueMatch, string groupPrefix, bool toLower)
@@ -192,7 +193,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
catch (Exception ex)
{
- progressMonitor.FailedToReadFile(file, ex);
+ logger.LogInfo($"Failed to read file {file}");
+ logger.LogDebug($"Failed to read file {file}, exception: {ex}");
}
}
}
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileInfoExtensions.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileInfoExtensions.cs
index 1d285d03d04..e68ad8c0616 100644
--- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileInfoExtensions.cs
+++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileInfoExtensions.cs
@@ -2,6 +2,7 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
+using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
@@ -13,14 +14,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
public static IEnumerable SelectRootFiles(this IEnumerable files, DirectoryInfo dir) =>
files.Where(file => file.DirectoryName == dir.FullName);
- internal static IEnumerable SelectSmallFiles(this IEnumerable files, ProgressMonitor progressMonitor)
+ internal static IEnumerable SelectSmallFiles(this IEnumerable files, ILogger logger)
{
const int oneMb = 1_048_576;
return files.Where(file =>
{
if (file.Length > oneMb)
{
- progressMonitor.LogDebug($"Skipping {file.FullName} because it is bigger than 1MB.");
+ logger.LogDebug($"Skipping {file.FullName} because it is bigger than 1MB.");
return false;
}
return true;
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FilePathFilter.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FilePathFilter.cs
index 39f761d08b9..24d92e0f068 100644
--- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FilePathFilter.cs
+++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FilePathFilter.cs
@@ -12,12 +12,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
public class FilePathFilter
{
private readonly string rootFolder;
- private readonly IProgressMonitor progressMonitor;
+ private readonly ILogger logger;
- public FilePathFilter(DirectoryInfo sourceDir, IProgressMonitor progressMonitor)
+ public FilePathFilter(DirectoryInfo sourceDir, ILogger logger)
{
rootFolder = FileUtils.ConvertToUnix(sourceDir.FullName.ToLowerInvariant());
- this.progressMonitor = progressMonitor;
+ this.logger = logger;
}
private class FileInclusion(string path, bool include)
@@ -55,12 +55,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
else
{
- progressMonitor.Log(Severity.Info, $"Invalid filter: {filter}");
+ logger.Log(Severity.Info, $"Invalid filter: {filter}");
continue;
}
var regex = new FilePattern(filterText).RegexPattern;
- progressMonitor.Log(Severity.Info, $"Filtering {(include ? "in" : "out")} files matching '{regex}'. Original glob filter: '{filter}'");
+ logger.Log(Severity.Info, $"Filtering {(include ? "in" : "out")} files matching '{regex}'. Original glob filter: '{filter}'");
pathFilters.Add(new PathFilter(new Regex(regex, RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline), include));
}
@@ -91,7 +91,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (!include)
{
- progressMonitor.Log(Severity.Info, $"Excluding '{f.FileInfo.FullName}'");
+ logger.Log(Severity.Info, $"Excluding '{f.FileInfo.FullName}'");
}
return include;
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs
index 841fd0be879..5904248b34f 100644
--- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs
+++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs
@@ -1,15 +1,43 @@
+using System;
using System.Collections.Generic;
+using System.Linq;
+using System.Text.RegularExpressions;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal interface IDotNet
{
- bool RestoreProjectToDirectory(string project, string directory, bool forceDotnetRefAssemblyFetching, out IEnumerable assets, out IList outputLines, string? pathToNugetConfig = null, bool force = false);
- bool RestoreSolutionToDirectory(string solutionFile, string packageDirectory, bool forceDotnetRefAssemblyFetching, out IEnumerable projects, out IEnumerable assets);
+ RestoreResult Restore(RestoreSettings restoreSettings);
bool New(string folder);
bool AddPackage(string folder, string package);
IList GetListedRuntimes();
IList GetListedSdks();
bool Exec(string execArgs);
}
+
+ internal record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, string? PathToNugetConfig = null, bool ForceReevaluation = false);
+
+ internal partial record class RestoreResult(bool Success, IList Output)
+ {
+ private readonly Lazy> assetsFilePaths = new(() => GetFirstGroupOnMatch(AssetsFileRegex(), Output));
+ public IEnumerable AssetsFilePaths => Success ? assetsFilePaths.Value : Array.Empty();
+
+ private readonly Lazy> restoredProjects = new(() => GetFirstGroupOnMatch(RestoredProjectRegex(), Output));
+ public IEnumerable RestoredProjects => Success ? restoredProjects.Value : Array.Empty();
+
+ private readonly Lazy hasNugetPackageSourceError = new(() => Output.Any(s => s.Contains("NU1301")));
+ public bool HasNugetPackageSourceError => hasNugetPackageSourceError.Value;
+
+ private static IEnumerable GetFirstGroupOnMatch(Regex regex, IEnumerable lines) =>
+ lines
+ .Select(line => regex.Match(line))
+ .Where(match => match.Success)
+ .Select(match => match.Groups[1].Value);
+
+ [GeneratedRegex("Restored\\s+(.+\\.csproj)", RegexOptions.Compiled)]
+ private static partial Regex RestoredProjectRegex();
+
+ [GeneratedRegex("[Assets\\sfile\\shas\\snot\\schanged.\\sSkipping\\sassets\\sfile\\swriting.|Writing\\sassets\\sfile\\sto\\sdisk.]\\sPath:\\s(.+)", RegexOptions.Compiled)]
+ private static partial Regex AssetsFileRegex();
+ }
}
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackages.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackages.cs
index 541853faf38..52e5ccc4ff5 100644
--- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackages.cs
+++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackages.cs
@@ -2,6 +2,7 @@
using System.Diagnostics;
using System.IO;
using System.Linq;
+using Microsoft.Build.Framework;
using Semmle.Util;
namespace Semmle.Extraction.CSharp.DependencyFetching
@@ -14,7 +15,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
internal class NugetPackages
{
private readonly string? nugetExe;
- private readonly ProgressMonitor progressMonitor;
+ private readonly Util.Logging.ILogger logger;
///
/// The list of package files.
@@ -31,10 +32,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
///
/// Create the package manager for a specified source tree.
///
- public NugetPackages(string sourceDir, TemporaryDirectory packageDirectory, ProgressMonitor progressMonitor)
+ public NugetPackages(string sourceDir, TemporaryDirectory packageDirectory, Util.Logging.ILogger logger)
{
this.packageDirectory = packageDirectory;
- this.progressMonitor = progressMonitor;
+ this.logger = logger;
packageFiles = new DirectoryInfo(sourceDir)
.EnumerateFiles("packages.config", SearchOption.AllDirectories)
@@ -42,11 +43,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (packageFiles.Length > 0)
{
+ logger.LogInfo($"Found {packageFiles.Length} packages.config files, trying to use nuget.exe for package restore");
nugetExe = ResolveNugetExe(sourceDir);
}
else
{
- progressMonitor.LogInfo("Found no packages.config file");
+ logger.LogInfo("Found no packages.config file");
}
}
@@ -65,14 +67,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var nuget = Path.Combine(directory, "nuget", "nuget.exe");
if (File.Exists(nuget))
{
- progressMonitor.FoundNuGet(nuget);
+ logger.LogInfo($"Found nuget.exe at {nuget}");
return nuget;
}
- else
- {
- progressMonitor.LogInfo($"Nuget.exe could not be found at {nuget}");
- return DownloadNugetExe(sourceDir);
- }
+
+ return DownloadNugetExe(sourceDir);
}
private string DownloadNugetExe(string sourceDir)
@@ -83,16 +82,16 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
// Nuget.exe already exists in the .nuget directory.
if (File.Exists(nuget))
{
- progressMonitor.FoundNuGet(nuget);
+ logger.LogInfo($"Found nuget.exe at {nuget}");
return nuget;
}
Directory.CreateDirectory(directory);
- progressMonitor.LogInfo("Attempting to download nuget.exe");
+ logger.LogInfo("Attempting to download nuget.exe");
try
{
FileUtils.DownloadFile(FileUtils.NugetExeUrl, nuget);
- progressMonitor.LogInfo($"Downloaded nuget.exe to {nuget}");
+ logger.LogInfo($"Downloaded nuget.exe to {nuget}");
return nuget;
}
catch
@@ -108,7 +107,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// The package file.
private void RestoreNugetPackage(string package)
{
- progressMonitor.NugetInstall(package);
+ logger.LogInfo($"Restoring file {package}...");
/* Use nuget.exe to install a package.
* Note that there is a clutch of NuGet assemblies which could be used to
@@ -135,29 +134,17 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
UseShellExecute = false
};
- try
+ var threadId = Environment.CurrentManagedThreadId;
+ void onOut(string s) => logger.LogInfo(s, threadId);
+ void onError(string s) => logger.LogError(s, threadId);
+ var exitCode = pi.ReadOutput(out var _, onOut, onError);
+ if (exitCode != 0)
{
- using var p = Process.Start(pi);
-
- if (p is null)
- {
- progressMonitor.FailedNugetCommand(pi.FileName, pi.Arguments, "Couldn't start process.");
- return;
- }
-
- var output = p.StandardOutput.ReadToEnd();
- var error = p.StandardError.ReadToEnd();
-
- p.WaitForExit();
- if (p.ExitCode != 0)
- {
- progressMonitor.FailedNugetCommand(pi.FileName, pi.Arguments, output + error);
- }
+ logger.LogError($"Command {pi.FileName} {pi.Arguments} failed with exit code {exitCode}");
}
- catch (Exception ex)
- when (ex is System.ComponentModel.Win32Exception || ex is FileNotFoundException)
+ else
{
- progressMonitor.FailedNugetCommand(pi.FileName, pi.Arguments, ex.Message);
+ logger.LogInfo($"Restored file {package}");
}
}
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/ProgressMonitor.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/ProgressMonitor.cs
deleted file mode 100644
index 7505c9a2785..00000000000
--- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/ProgressMonitor.cs
+++ /dev/null
@@ -1,127 +0,0 @@
-using System;
-using Semmle.Util.Logging;
-
-namespace Semmle.Extraction.CSharp.DependencyFetching
-{
- internal class ProgressMonitor : IProgressMonitor
- {
- private readonly ILogger logger;
-
- public ProgressMonitor(ILogger logger)
- {
- this.logger = logger;
- }
-
- public void Log(Severity severity, string message) =>
- logger.Log(severity, message);
-
- public void LogInfo(string message) =>
- logger.Log(Severity.Info, message);
-
- public void LogDebug(string message) =>
- logger.Log(Severity.Debug, message);
-
- private void LogError(string message) =>
- logger.Log(Severity.Error, message);
-
- public void FindingFiles(string dir) =>
- LogInfo($"Finding files in {dir}...");
-
- public void IndexingReferences(int count)
- {
- LogInfo("Indexing...");
- LogDebug($"Indexing {count} DLLs...");
- }
-
- public void UnresolvedReference(string id, string project)
- {
- LogInfo($"Unresolved reference {id}");
- LogDebug($"Unresolved {id} referenced by {project}");
- }
-
- public void AnalysingSolution(string filename) =>
- LogInfo($"Analyzing {filename}...");
-
- public void FailedProjectFile(string filename, string reason) =>
- LogInfo($"Couldn't read project file {filename}: {reason}");
-
- public void FailedNugetCommand(string exe, string args, string message)
- {
- LogInfo($"Command failed: {exe} {args}");
- LogInfo($" {message}");
- }
-
- public void NugetInstall(string package) =>
- LogInfo($"Restoring {package}...");
-
- public void ResolvedReference(string filename) =>
- LogInfo($"Resolved reference {filename}");
-
- public void RemovedReference(string filename) =>
- LogInfo($"Removed reference {filename}");
-
- public void Summary(int existingSources, int usedSources, int missingSources,
- int references, int unresolvedReferences,
- int resolvedConflicts, int totalProjects, int failedProjects,
- TimeSpan analysisTime)
- {
- const int align = 6;
- LogInfo("");
- LogInfo("Build analysis summary:");
- LogInfo($"{existingSources,align} source files in the filesystem");
- LogInfo($"{usedSources,align} source files in project files");
- LogInfo($"{missingSources,align} sources missing from project files");
- LogInfo($"{references,align} resolved references");
- LogInfo($"{unresolvedReferences,align} unresolved references");
- LogInfo($"{resolvedConflicts,align} resolved assembly conflicts");
- LogInfo($"{totalProjects,align} projects");
- LogInfo($"{failedProjects,align} missing/failed projects");
- LogInfo($"Build analysis completed in {analysisTime}");
- }
-
- public void ResolvedConflict(string asm1, string asm2) =>
- LogDebug($"Resolved {asm1} as {asm2}");
-
- public void MissingProject(string projectFile) =>
- LogInfo($"Solution is missing {projectFile}");
-
- public void CommandFailed(string exe, string arguments, int exitCode) =>
- LogError($"Command {exe} {arguments} failed with exit code {exitCode}");
-
- public void MissingNuGet() =>
- LogError("Missing nuget.exe");
-
- public void FoundNuGet(string path) =>
- LogInfo($"Found nuget.exe at {path}");
-
- public void MissingDotNet() =>
- LogError("Missing dotnet CLI");
-
- public void RunningProcess(string command) =>
- LogInfo($"Running {command}");
-
- public void FailedToRestoreNugetPackage(string package) =>
- LogInfo($"Failed to restore nuget package {package}");
-
- public void FailedToReadFile(string file, Exception ex)
- {
- LogInfo($"Failed to read file {file}");
- LogDebug($"Failed to read file {file}, exception: {ex}");
- }
-
- public void MultipleNugetConfig(string[] nugetConfigs) =>
- LogInfo($"Found multiple nuget.config files: {string.Join(", ", nugetConfigs)}.");
-
- internal void NoTopLevelNugetConfig() =>
- LogInfo("Could not find a top-level nuget.config file.");
-
- internal void RazorSourceGeneratorMissing(string fullPath) =>
- LogInfo($"Razor source generator folder {fullPath} does not exist.");
-
- internal void CscMissing(string cscPath) =>
- LogInfo($"Csc.exe not found at {cscPath}.");
-
- internal void RazorCscArgs(string args) =>
- LogInfo($"Running CSC to generate Razor source files. Args: {args}.");
- }
-}
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Razor.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Razor.cs
index 9d910835f90..a1c96cc964e 100644
--- a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Razor.cs
+++ b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Razor.cs
@@ -4,34 +4,37 @@ using System.IO;
using System.Text;
using System.Linq;
using Semmle.Util;
+using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal class Razor
{
private readonly DotNetVersion sdk;
- private readonly ProgressMonitor progressMonitor;
+ private readonly ILogger logger;
private readonly IDotNet dotNet;
private readonly string sourceGeneratorFolder;
private readonly string cscPath;
- public Razor(DotNetVersion sdk, IDotNet dotNet, ProgressMonitor progressMonitor)
+ public Razor(DotNetVersion sdk, IDotNet dotNet, ILogger logger)
{
this.sdk = sdk;
- this.progressMonitor = progressMonitor;
+ this.logger = logger;
this.dotNet = dotNet;
sourceGeneratorFolder = Path.Combine(this.sdk.FullPath, "Sdks", "Microsoft.NET.Sdk.Razor", "source-generators");
+ this.logger.LogInfo($"Razor source generator folder: {sourceGeneratorFolder}");
if (!Directory.Exists(sourceGeneratorFolder))
{
- this.progressMonitor.RazorSourceGeneratorMissing(sourceGeneratorFolder);
+ this.logger.LogInfo($"Razor source generator folder {sourceGeneratorFolder} does not exist.");
throw new Exception($"Razor source generator folder {sourceGeneratorFolder} does not exist.");
}
cscPath = Path.Combine(this.sdk.FullPath, "Roslyn", "bincore", "csc.dll");
+ this.logger.LogInfo($"Razor source generator CSC: {cscPath}");
if (!File.Exists(cscPath))
{
- this.progressMonitor.CscMissing(cscPath);
+ this.logger.LogInfo($"Csc.exe not found at {cscPath}.");
throw new Exception($"csc.dll {cscPath} does not exist.");
}
}
@@ -63,7 +66,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
GenerateAnalyzerConfig(cshtmls, analyzerConfig);
- progressMonitor.LogInfo($"Analyzer config content: {File.ReadAllText(analyzerConfig)}");
+ logger.LogInfo($"Analyzer config content: {File.ReadAllText(analyzerConfig)}");
var args = new StringBuilder();
args.Append($"/target:exe /generatedfilesout:\"{outputFolder}\" /out:\"{dllPath}\" /analyzerconfig:\"{analyzerConfig}\" ");
@@ -85,7 +88,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var argsString = args.ToString();
- progressMonitor.RazorCscArgs(argsString);
+ logger.LogInfo($"Running CSC to generate Razor source files with arguments: {argsString}.");
using (var sw = new StreamWriter(cscArgsPath))
{
@@ -94,7 +97,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
dotNet.Exec($"\"{cscPath}\" /noconfig @\"{cscArgsPath}\"");
- return Directory.GetFiles(outputFolder, "*.*", new EnumerationOptions { RecurseSubdirectories = true });
+ var files = Directory.GetFiles(outputFolder, "*.*", new EnumerationOptions { RecurseSubdirectories = true });
+
+ logger.LogInfo($"Generated {files.Length} source files from cshtml files.");
+
+ return files;
}
finally
{
diff --git a/csharp/extractor/Semmle.Extraction.Tests/Assets.cs b/csharp/extractor/Semmle.Extraction.Tests/Assets.cs
index 85e56b6de64..3406788bd1c 100644
--- a/csharp/extractor/Semmle.Extraction.Tests/Assets.cs
+++ b/csharp/extractor/Semmle.Extraction.Tests/Assets.cs
@@ -12,7 +12,7 @@ namespace Semmle.Extraction.Tests
public void TestAssets1()
{
// Setup
- var assets = new Assets(new ProgressMonitor(new LoggerStub()));
+ var assets = new Assets(new LoggerStub());
var json = assetsJson1;
var dependencies = new DependencyContainer();
@@ -43,7 +43,7 @@ namespace Semmle.Extraction.Tests
public void TestAssetsFailure()
{
// Setup
- var assets = new Assets(new ProgressMonitor(new LoggerStub()));
+ var assets = new Assets(new LoggerStub());
var json = "garbage data";
var dependencies = new DependencyContainer();
@@ -59,7 +59,7 @@ namespace Semmle.Extraction.Tests
public void TestAssetsNet70()
{
// Setup
- var assets = new Assets(new ProgressMonitor(new LoggerStub()));
+ var assets = new Assets(new LoggerStub());
var json = assetsNet70;
var dependencies = new DependencyContainer();
@@ -90,7 +90,7 @@ namespace Semmle.Extraction.Tests
public void TestAssetsNet48()
{
// Setup
- var assets = new Assets(new ProgressMonitor(new LoggerStub()));
+ var assets = new Assets(new LoggerStub());
var json = assetsNet48;
var dependencies = new DependencyContainer();
@@ -117,7 +117,7 @@ namespace Semmle.Extraction.Tests
public void TestAssetsNetstandard21()
{
// Setup
- var assets = new Assets(new ProgressMonitor(new LoggerStub()));
+ var assets = new Assets(new LoggerStub());
var json = assetsNetstandard21;
var dependencies = new DependencyContainer();
@@ -145,7 +145,7 @@ namespace Semmle.Extraction.Tests
public void TestAssetsNetStandard16()
{
// Setup
- var assets = new Assets(new ProgressMonitor(new LoggerStub()));
+ var assets = new Assets(new LoggerStub());
var json = assetsNetstandard16;
var dependencies = new DependencyContainer();
@@ -177,7 +177,7 @@ namespace Semmle.Extraction.Tests
public void TestAssetsNetcoreapp20()
{
// Setup
- var assets = new Assets(new ProgressMonitor(new LoggerStub()));
+ var assets = new Assets(new LoggerStub());
var json = assetsNetcoreapp20;
var dependencies = new DependencyContainer();
@@ -205,7 +205,7 @@ namespace Semmle.Extraction.Tests
public void TestAssetsNetcoreapp31()
{
// Setup
- var assets = new Assets(new ProgressMonitor(new LoggerStub()));
+ var assets = new Assets(new LoggerStub());
var json = assetsNetcoreapp31;
var dependencies = new DependencyContainer();
diff --git a/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs b/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs
index a9721f8dec2..289d9efbba4 100644
--- a/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs
+++ b/csharp/extractor/Semmle.Extraction.Tests/DotNet.cs
@@ -38,7 +38,7 @@ namespace Semmle.Extraction.Tests
public class DotNetTests
{
private static IDotNet MakeDotnet(IDotNetCliInvoker dotnetCliInvoker) =>
- DotNet.Make(dotnetCliInvoker, new ProgressMonitor(new LoggerStub()));
+ DotNet.Make(dotnetCliInvoker, new LoggerStub());
private static IList MakeDotnetRestoreOutput() =>
new List {
@@ -101,7 +101,7 @@ namespace Semmle.Extraction.Tests
var dotnet = MakeDotnet(dotnetCliInvoker);
// Execute
- dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages", false, out var assets, out var _);
+ dotnet.Restore(new("myproject.csproj", "mypackages", false));
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();
@@ -116,14 +116,14 @@ namespace Semmle.Extraction.Tests
var dotnet = MakeDotnet(dotnetCliInvoker);
// Execute
- dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages", false, out var assets, out var _, pathToNugetConfig: "myconfig.config");
+ var res = dotnet.Restore(new("myproject.csproj", "mypackages", false, "myconfig.config"));
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();
Assert.Equal("restore --no-dependencies \"myproject.csproj\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal --configfile \"myconfig.config\"", lastArgs);
- Assert.Equal(2, assets.Count());
- Assert.Contains("/path/to/project.assets.json", assets);
- Assert.Contains("/path/to/project2.assets.json", assets);
+ Assert.Equal(2, res.AssetsFilePaths.Count());
+ Assert.Contains("/path/to/project.assets.json", res.AssetsFilePaths);
+ Assert.Contains("/path/to/project2.assets.json", res.AssetsFilePaths);
}
[Fact]
@@ -134,14 +134,14 @@ namespace Semmle.Extraction.Tests
var dotnet = MakeDotnet(dotnetCliInvoker);
// Execute
- dotnet.RestoreProjectToDirectory("myproject.csproj", "mypackages", false, out var assets, out var _, pathToNugetConfig: "myconfig.config", force: true);
+ var res = dotnet.Restore(new("myproject.csproj", "mypackages", false, "myconfig.config", true));
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();
Assert.Equal("restore --no-dependencies \"myproject.csproj\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal --configfile \"myconfig.config\" --force", lastArgs);
- Assert.Equal(2, assets.Count());
- Assert.Contains("/path/to/project.assets.json", assets);
- Assert.Contains("/path/to/project2.assets.json", assets);
+ Assert.Equal(2, res.AssetsFilePaths.Count());
+ Assert.Contains("/path/to/project.assets.json", res.AssetsFilePaths);
+ Assert.Contains("/path/to/project2.assets.json", res.AssetsFilePaths);
}
[Fact]
@@ -152,17 +152,17 @@ namespace Semmle.Extraction.Tests
var dotnet = MakeDotnet(dotnetCliInvoker);
// Execute
- dotnet.RestoreSolutionToDirectory("mysolution.sln", "mypackages", false, out var projects, out var assets);
+ var res = dotnet.Restore(new("mysolution.sln", "mypackages", false));
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();
Assert.Equal("restore --no-dependencies \"mysolution.sln\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal", lastArgs);
- Assert.Equal(2, projects.Count());
- Assert.Contains("/path/to/project.csproj", projects);
- Assert.Contains("/path/to/project2.csproj", projects);
- Assert.Equal(2, assets.Count());
- Assert.Contains("/path/to/project.assets.json", assets);
- Assert.Contains("/path/to/project2.assets.json", assets);
+ Assert.Equal(2, res.RestoredProjects.Count());
+ Assert.Contains("/path/to/project.csproj", res.RestoredProjects);
+ Assert.Contains("/path/to/project2.csproj", res.RestoredProjects);
+ Assert.Equal(2, res.AssetsFilePaths.Count());
+ Assert.Contains("/path/to/project.assets.json", res.AssetsFilePaths);
+ Assert.Contains("/path/to/project2.assets.json", res.AssetsFilePaths);
}
[Fact]
@@ -174,13 +174,13 @@ namespace Semmle.Extraction.Tests
dotnetCliInvoker.Success = false;
// Execute
- dotnet.RestoreSolutionToDirectory("mysolution.sln", "mypackages", false, out var projects, out var assets);
+ var res = dotnet.Restore(new("mysolution.sln", "mypackages", false));
// Verify
var lastArgs = dotnetCliInvoker.GetLastArgs();
Assert.Equal("restore --no-dependencies \"mysolution.sln\" --packages \"mypackages\" /p:DisableImplicitNuGetFallbackFolder=true --verbosity normal", lastArgs);
- Assert.Empty(projects);
- Assert.Empty(assets);
+ Assert.Empty(res.RestoredProjects);
+ Assert.Empty(res.AssetsFilePaths);
}
[Fact]
diff --git a/csharp/extractor/Semmle.Extraction.Tests/FileContent.cs b/csharp/extractor/Semmle.Extraction.Tests/FileContent.cs
index 206453e132c..ba934120c45 100644
--- a/csharp/extractor/Semmle.Extraction.Tests/FileContent.cs
+++ b/csharp/extractor/Semmle.Extraction.Tests/FileContent.cs
@@ -25,7 +25,7 @@ namespace Semmle.Extraction.Tests
internal class TestFileContent : FileContent
{
- public TestFileContent(IEnumerable lines) : base(new ProgressMonitor(new LoggerStub()),
+ public TestFileContent(IEnumerable lines) : base(new LoggerStub(),
new List() { "test1.cs" },
new UnsafeFileReaderStub(lines))
{ }
diff --git a/csharp/extractor/Semmle.Extraction.Tests/FilePathFilter.cs b/csharp/extractor/Semmle.Extraction.Tests/FilePathFilter.cs
index dd32da3361c..efb117a263d 100644
--- a/csharp/extractor/Semmle.Extraction.Tests/FilePathFilter.cs
+++ b/csharp/extractor/Semmle.Extraction.Tests/FilePathFilter.cs
@@ -10,17 +10,19 @@ namespace Semmle.Extraction.Tests
{
public class FilePathFilterTest
{
- private class ProgressMonitorStub : IProgressMonitor
+ private class LoggerStub : ILogger
{
public List Messages { get; } = [];
- public void Log(Severity severity, string message)
+ public void Log(Severity s, string text, int? threadId = null)
{
- Messages.Add(message);
+ Messages.Add(text);
}
+
+ public void Dispose() { }
}
- private static (FilePathFilter TestSubject, ProgressMonitorStub progressMonitor, IEnumerable Files) TestSetup()
+ private static (FilePathFilter TestSubject, LoggerStub Logger, IEnumerable Files) TestSetup()
{
return TestSetup("/a/b",
[
@@ -32,15 +34,15 @@ namespace Semmle.Extraction.Tests
]);
}
- private static (FilePathFilter TestSubject, ProgressMonitorStub progressMonitor, IEnumerable Files) TestSetup(string root, IEnumerable paths)
+ private static (FilePathFilter TestSubject, LoggerStub Logger, IEnumerable Files) TestSetup(string root, IEnumerable paths)
{
root = GetPlatformSpecifixPath(root);
paths = GetPlatformSpecifixPaths(paths);
- var progressMonitor = new ProgressMonitorStub();
+ var logger = new LoggerStub();
- var filePathFilter = new FilePathFilter(new DirectoryInfo(root), progressMonitor);
- return (filePathFilter, progressMonitor, paths.Select(p => new FileInfo(p)));
+ var filePathFilter = new FilePathFilter(new DirectoryInfo(root), logger);
+ return (filePathFilter, logger, paths.Select(p => new FileInfo(p)));
}
private static string GetPlatformSpecifixPath(string file)
@@ -67,20 +69,20 @@ namespace Semmle.Extraction.Tests
[Fact]
public void TestNoFilter()
{
- (var testSubject, var progressMonitor, var files) = TestSetup();
+ (var testSubject, var logger, var files) = TestSetup();
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", null);
var filtered = testSubject.Filter(files);
AssertFileInfoEquivalence(files, filtered);
- Assert.Equivalent(Array.Empty(), progressMonitor.Messages, strict: true);
+ Assert.Equivalent(Array.Empty(), logger.Messages, strict: true);
}
[Fact]
public void TestFiltersWithOnlyInclude()
{
- (var testSubject, var progressMonitor, var files) = TestSetup();
+ (var testSubject, var logger, var files) = TestSetup();
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
include:c/d
@@ -104,13 +106,13 @@ namespace Semmle.Extraction.Tests
"Filtering in files matching '^c/d.*'. Original glob filter: 'include:c/d'",
"Filtering in files matching '^c/x/y.*'. Original glob filter: 'include:c/x/y'"
};
- Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false);
+ Assert.Equivalent(expectedRegexMessages, logger.Messages, strict: false);
}
[Fact]
public void TestFiltersWithOnlyExclude()
{
- (var testSubject, var progressMonitor, var files) = TestSetup();
+ (var testSubject, var logger, var files) = TestSetup();
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
exclude:c/d/e
@@ -130,13 +132,13 @@ namespace Semmle.Extraction.Tests
{
"Filtering out files matching '^c/d/e.*'. Original glob filter: 'exclude:c/d/e'"
};
- Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false);
+ Assert.Equivalent(expectedRegexMessages, logger.Messages, strict: false);
}
[Fact]
public void TestFiltersWithIncludeExclude()
{
- (var testSubject, var progressMonitor, var files) = TestSetup();
+ (var testSubject, var logger, var files) = TestSetup();
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
include:c/x
@@ -157,13 +159,13 @@ namespace Semmle.Extraction.Tests
"Filtering in files matching '^c/x.*'. Original glob filter: 'include:c/x'",
"Filtering out files matching '^c/x/z.*'. Original glob filter: 'exclude:c/x/z'"
};
- Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false);
+ Assert.Equivalent(expectedRegexMessages, logger.Messages, strict: false);
}
[Fact]
public void TestFiltersWithIncludeExcludeExcludeFirst()
{
- (var testSubject, var progressMonitor, var files) = TestSetup();
+ (var testSubject, var logger, var files) = TestSetup();
// NOTE: the ordering DOES matter, later filters takes priority, so the exclude will end up not mattering at all.
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
@@ -186,13 +188,13 @@ namespace Semmle.Extraction.Tests
"Filtering in files matching '^c/x.*'. Original glob filter: 'include:c/x'",
"Filtering out files matching '^c/x/z.*'. Original glob filter: 'exclude:c/x/z'"
};
- Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false);
+ Assert.Equivalent(expectedRegexMessages, logger.Messages, strict: false);
}
[Fact]
public void TestFiltersWithIncludeExcludeComplexPatterns1()
{
- (var testSubject, var progressMonitor, var files) = TestSetup();
+ (var testSubject, var logger, var files) = TestSetup();
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
include:c/**/i.*
@@ -218,13 +220,13 @@ namespace Semmle.Extraction.Tests
"Filtering in files matching '^c/d/.*/[^/]*\\.cs.*'. Original glob filter: 'include:c/d/**/*.cs'",
"Filtering out files matching '^.*/z/i\\.cs.*'. Original glob filter: 'exclude:**/z/i.cs'"
};
- Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false);
+ Assert.Equivalent(expectedRegexMessages, logger.Messages, strict: false);
}
[Fact]
public void TestFiltersWithIncludeExcludeComplexPatterns2()
{
- (var testSubject, var progressMonitor, var files) = TestSetup();
+ (var testSubject, var logger, var files) = TestSetup();
Environment.SetEnvironmentVariable("LGTM_INDEX_FILTERS", """
include:**/i.*
@@ -245,7 +247,7 @@ namespace Semmle.Extraction.Tests
"Filtering in files matching '^.*/i\\.[^/]*.*'. Original glob filter: 'include:**/i.*'",
"Filtering out files matching '^.*/z/i\\.cs.*'. Original glob filter: 'exclude:**/z/i.cs'"
};
- Assert.Equivalent(expectedRegexMessages, progressMonitor.Messages, strict: false);
+ Assert.Equivalent(expectedRegexMessages, logger.Messages, strict: false);
}
}
}
diff --git a/csharp/extractor/Semmle.Extraction.Tests/LoggerStub.cs b/csharp/extractor/Semmle.Extraction.Tests/LoggerStub.cs
index 78e85b8d60f..9120de8be45 100644
--- a/csharp/extractor/Semmle.Extraction.Tests/LoggerStub.cs
+++ b/csharp/extractor/Semmle.Extraction.Tests/LoggerStub.cs
@@ -4,7 +4,7 @@ namespace Semmle.Extraction.Tests
{
internal class LoggerStub : ILogger
{
- public void Log(Severity severity, string message) { }
+ public void Log(Severity severity, string message, int? threadId = null) { }
public void Dispose() { }
}
diff --git a/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs b/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs
index 03df84fe462..17bc477bde8 100644
--- a/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs
+++ b/csharp/extractor/Semmle.Extraction.Tests/Runtime.cs
@@ -19,19 +19,7 @@ namespace Semmle.Extraction.Tests
public bool New(string folder) => true;
- public bool RestoreProjectToDirectory(string project, string directory, bool forceDotnetRefAssemblyFetching, out IEnumerable assets, out IList outputLines, string? pathToNugetConfig = null, bool force = false)
- {
- assets = Array.Empty();
- outputLines = Array.Empty();
- return true;
- }
-
- public bool RestoreSolutionToDirectory(string solution, string directory, bool forceDotnetRefAssemblyFetching, out IEnumerable projects, out IEnumerable assets)
- {
- projects = Array.Empty();
- assets = Array.Empty();
- return true;
- }
+ public RestoreResult Restore(RestoreSettings restoreSettings) => new(true, Array.Empty());
public IList GetListedRuntimes() => runtimes;
diff --git a/csharp/extractor/Semmle.Util.Tests/CanonicalPathCache.cs b/csharp/extractor/Semmle.Util.Tests/CanonicalPathCache.cs
index 1a50fd87679..313b949810d 100644
--- a/csharp/extractor/Semmle.Util.Tests/CanonicalPathCache.cs
+++ b/csharp/extractor/Semmle.Util.Tests/CanonicalPathCache.cs
@@ -176,7 +176,7 @@ namespace SemmleTests.Semmle.Util
{
public void Dispose() { }
- public void Log(Severity s, string text) { }
+ public void Log(Severity s, string text, int? threadId = null) { }
}
}
}
diff --git a/csharp/extractor/Semmle.Util.Tests/FileUtils.cs b/csharp/extractor/Semmle.Util.Tests/FileUtils.cs
index cbc82d4b814..c0e5e91db78 100644
--- a/csharp/extractor/Semmle.Util.Tests/FileUtils.cs
+++ b/csharp/extractor/Semmle.Util.Tests/FileUtils.cs
@@ -57,7 +57,7 @@ namespace SemmleTests.Semmle.Util
{
public void Dispose() { }
- public void Log(Severity s, string text) { }
+ public void Log(Severity s, string text, int? threadId = null) { }
}
}
}
diff --git a/csharp/extractor/Semmle.Util/Logger.cs b/csharp/extractor/Semmle.Util/Logger.cs
index 747d43368b8..5e9aa3e219f 100644
--- a/csharp/extractor/Semmle.Util/Logger.cs
+++ b/csharp/extractor/Semmle.Util/Logger.cs
@@ -37,7 +37,15 @@ namespace Semmle.Util.Logging
///
/// Log the given text with the given severity.
///
- void Log(Severity s, string text);
+ void Log(Severity s, string text, int? threadId = null);
+
+ void LogError(string text, int? threadId = null) => Log(Severity.Error, text, threadId);
+
+ void LogWarning(string text, int? threadId = null) => Log(Severity.Warning, text, threadId);
+
+ void LogInfo(string text, int? threadId = null) => Log(Severity.Info, text, threadId);
+
+ void LogDebug(string text, int? threadId = null) => Log(Severity.Debug, text, threadId);
}
public static class LoggerExtensions
@@ -92,12 +100,14 @@ namespace Semmle.Util.Logging
return "[" + s.ToString().ToUpper() + "] ";
}
- public void Log(Severity s, string text)
+ public void Log(Severity s, string text, int? threadId = null)
{
if (verbosity.Includes(s))
{
- var threadId = this.logThreadId ? $"[{Environment.CurrentManagedThreadId:D3}] " : "";
- writer.WriteLine(threadId + GetSeverityPrefix(s) + text);
+ threadId ??= Environment.CurrentManagedThreadId;
+
+ var prefix = this.logThreadId ? $"[{threadId:D3}] " : "";
+ writer.WriteLine(prefix + GetSeverityPrefix(s) + text);
}
}
}
@@ -140,12 +150,14 @@ namespace Semmle.Util.Logging
}
}
- public void Log(Severity s, string text)
+ public void Log(Severity s, string text, int? threadId = null)
{
if (verbosity.Includes(s))
{
- var threadId = this.logThreadId ? $"[{Environment.CurrentManagedThreadId:D3}] " : "";
- GetConsole(s).WriteLine(threadId + GetSeverityPrefix(s) + text);
+ threadId ??= Environment.CurrentManagedThreadId;
+
+ var prefix = this.logThreadId ? $"[{threadId:D3}] " : "";
+ GetConsole(s).WriteLine(prefix + GetSeverityPrefix(s) + text);
}
}
}
@@ -170,10 +182,10 @@ namespace Semmle.Util.Logging
logger2.Dispose();
}
- public void Log(Severity s, string text)
+ public void Log(Severity s, string text, int? threadId = null)
{
- logger1.Log(s, text);
- logger2.Log(s, text);
+ logger1.Log(s, text, threadId);
+ logger2.Log(s, text, threadId);
}
}