using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; using System.Security.Cryptography; using System.Text; using System.Threading.Tasks; using Semmle.Util; using Semmle.Util.Logging; namespace Semmle.Extraction.CSharp.DependencyFetching { /// /// Main implementation of the build analysis. /// public sealed class DependencyManager : IDisposable { private readonly AssemblyCache assemblyCache; private readonly ProgressMonitor progressMonitor; private readonly IDictionary usedReferences = new ConcurrentDictionary(); private readonly IDictionary sources = new ConcurrentDictionary(); private readonly IDictionary unresolvedReferences = new ConcurrentDictionary(); private int failedProjects; private int succeededProjects; private readonly List nonGeneratedSources; private readonly List generatedSources; private int conflictedReferences = 0; private readonly IDependencyOptions options; private readonly DirectoryInfo sourceDir; private readonly IDotNet dotnet; private readonly FileContent fileContent; private readonly TemporaryDirectory packageDirectory; private readonly TemporaryDirectory tempWorkingDirectory; private readonly bool cleanupTempWorkingDirectory; /// /// Performs C# dependency fetching. /// /// Dependency fetching options /// Logger for dependency fetching progress. public DependencyManager(string srcDir, IDependencyOptions options, ILogger logger) { var startTime = DateTime.Now; this.options = options; this.progressMonitor = new ProgressMonitor(logger); this.sourceDir = new DirectoryInfo(srcDir); packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName)); tempWorkingDirectory = new TemporaryDirectory(FileUtils.GetTemporaryWorkingDirectory(out cleanupTempWorkingDirectory)); try { this.dotnet = DotNet.Make(options, progressMonitor, tempWorkingDirectory); } catch { progressMonitor.MissingDotNet(); throw; } this.progressMonitor.FindingFiles(srcDir); var allFiles = GetAllFiles(); var binaryFileExtensions = new HashSet(new[] { ".dll", ".exe" }); // TODO: add more binary file extensions. var allNonBinaryFiles = allFiles.Where(f => !binaryFileExtensions.Contains(f.Extension.ToLowerInvariant())).ToList(); var smallNonBinaryFiles = allNonBinaryFiles.SelectSmallFiles(progressMonitor).SelectFileNames(); this.fileContent = new FileContent(progressMonitor, smallNonBinaryFiles); this.nonGeneratedSources = allNonBinaryFiles.SelectFileNamesByExtension(".cs").ToList(); this.generatedSources = new(); var allProjects = allNonBinaryFiles.SelectFileNamesByExtension(".csproj"); var solutions = options.SolutionFile is not null ? new[] { options.SolutionFile } : allNonBinaryFiles.SelectFileNamesByExtension(".sln"); var dllDirNames = options.DllDirs.Count == 0 ? allFiles.SelectFileNamesByExtension(".dll").ToList() : options.DllDirs.Select(Path.GetFullPath).ToList(); if (options.UseNuGet) { dllDirNames.Add(packageDirectory.DirInfo.FullName); try { var nuget = new NugetPackages(sourceDir.FullName, packageDirectory, progressMonitor); nuget.InstallPackages(); } catch (FileNotFoundException) { progressMonitor.MissingNuGet(); } var restoredProjects = RestoreSolutions(solutions); var projects = allProjects.Except(restoredProjects); RestoreProjects(projects); DownloadMissingPackages(allNonBinaryFiles); } var existsNetCoreRefNugetPackage = false; var existsNetFrameworkRefNugetPackage = false; var existsNetstandardLibRefNugetPackage = false; var existsNetstandardLibNugetPackage = false; // Find DLLs in the .Net / Asp.Net Framework // This block needs to come after the nuget restore, because the nuget restore might fetch the .NET Core/Framework reference assemblies. if (options.ScanNetFrameworkDlls) { existsNetCoreRefNugetPackage = IsNugetPackageAvailable("microsoft.netcore.app.ref"); existsNetFrameworkRefNugetPackage = IsNugetPackageAvailable("microsoft.netframework.referenceassemblies"); existsNetstandardLibRefNugetPackage = IsNugetPackageAvailable("netstandard.library.ref"); existsNetstandardLibNugetPackage = IsNugetPackageAvailable("netstandard.library"); if (existsNetCoreRefNugetPackage || existsNetFrameworkRefNugetPackage || existsNetstandardLibRefNugetPackage || existsNetstandardLibNugetPackage) { progressMonitor.LogInfo("Found .NET Core/Framework DLLs in NuGet packages. Not adding installation directory."); } else { AddNetFrameworkDlls(dllDirNames); } } assemblyCache = new AssemblyCache(dllDirNames, progressMonitor); AnalyseSolutions(solutions); foreach (var filename in assemblyCache.AllAssemblies.Select(a => a.Filename)) { UseReference(filename); } RemoveUnnecessaryNugetPackages(existsNetCoreRefNugetPackage, existsNetFrameworkRefNugetPackage, existsNetstandardLibRefNugetPackage, existsNetstandardLibNugetPackage); ResolveConflicts(); // Output the findings foreach (var r in usedReferences.Keys.OrderBy(r => r)) { progressMonitor.ResolvedReference(r); } foreach (var r in unresolvedReferences.OrderBy(r => r.Key)) { progressMonitor.UnresolvedReference(r.Key, r.Value); } var webViewExtractionOption = Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_STANDALONE_EXTRACT_WEB_VIEWS"); if (bool.TryParse(webViewExtractionOption, out var shouldExtractWebViews) && shouldExtractWebViews) { GenerateSourceFilesFromWebViews(allNonBinaryFiles); } GenerateSourceFileFromImplicitUsings(); progressMonitor.Summary( AllSourceFiles.Count(), ProjectSourceFiles.Count(), MissingSourceFiles.Count(), ReferenceFiles.Count(), UnresolvedReferences.Count(), conflictedReferences, succeededProjects + failedProjects, failedProjects, DateTime.Now - startTime); } private void RemoveUnnecessaryNugetPackages(bool existsNetCoreRefNugetPackage, bool existsNetFrameworkRefNugetPackage, bool existsNetstandardLibRefNugetPackage, bool existsNetstandardLibNugetPackage) { RemoveNugetAnalyzerReferences(); RemoveRuntimeNugetPackageReferences(); if (fileContent.IsNewProjectStructureUsed && !fileContent.UseAspNetCoreDlls) { // This might have been restored by the CLI even though the project isn't an asp.net core one. RemoveNugetPackageReference("microsoft.aspnetcore.app.ref"); } // Multiple dotnet framework packages could be present. We keep only one. // The order of the packages is important, we're keeping the first one that is present in the nuget cache. var packagesInPrioOrder = new (bool isPresent, string prefix)[] { // net7.0, ... net5.0, netcoreapp3.1, netcoreapp3.0 (existsNetCoreRefNugetPackage, "microsoft.netcore.app.ref"), // net48, ..., net20 (existsNetFrameworkRefNugetPackage, "microsoft.netframework.referenceassemblies."), // netstandard2.1 (existsNetstandardLibRefNugetPackage, "netstandard.library.ref"), // netstandard2.0 (existsNetstandardLibNugetPackage, "netstandard.library") }; for (var i = 0; i < packagesInPrioOrder.Length; i++) { var (isPresent, _) = packagesInPrioOrder[i]; if (!isPresent) { continue; } // Package is present, remove all the lower priority packages: for (var j = i + 1; j < packagesInPrioOrder.Length; j++) { var (otherIsPresent, otherPrefix) = packagesInPrioOrder[j]; if (otherIsPresent) { RemoveNugetPackageReference(otherPrefix); } } break; } // TODO: There could be multiple `microsoft.netframework.referenceassemblies` packages, // we could keep the newest one, but this is covered by the conflict resolution logic // (if the file names match) } private void RemoveNugetAnalyzerReferences() { if (!options.UseNuGet) { return; } var packageFolder = packageDirectory.DirInfo.FullName.ToLowerInvariant(); if (packageFolder == null) { return; } foreach (var filename in usedReferences.Keys) { var lowerFilename = filename.ToLowerInvariant(); if (lowerFilename.StartsWith(packageFolder)) { var firstDirectorySeparatorCharIndex = lowerFilename.IndexOf(Path.DirectorySeparatorChar, packageFolder.Length + 1); if (firstDirectorySeparatorCharIndex == -1) { continue; } var secondDirectorySeparatorCharIndex = lowerFilename.IndexOf(Path.DirectorySeparatorChar, firstDirectorySeparatorCharIndex + 1); if (secondDirectorySeparatorCharIndex == -1) { continue; } var subFolderIndex = secondDirectorySeparatorCharIndex + 1; var isInAnalyzersFolder = lowerFilename.IndexOf("analyzers", subFolderIndex) == subFolderIndex; if (isInAnalyzersFolder) { usedReferences.Remove(filename); progressMonitor.RemovedReference(filename); } } } } private void AddNetFrameworkDlls(List dllDirNames) { var runtime = new Runtime(dotnet); string? runtimeLocation = null; if (options.UseSelfContainedDotnet) { runtimeLocation = runtime.ExecutingRuntime; } else if (fileContent.IsNewProjectStructureUsed) { runtimeLocation = runtime.NetCoreRuntime; } else if (fileContent.IsLegacyProjectStructureUsed) { runtimeLocation = runtime.DesktopRuntime; } runtimeLocation ??= runtime.ExecutingRuntime; progressMonitor.LogInfo($".NET runtime location selected: {runtimeLocation}"); dllDirNames.Add(runtimeLocation); if (fileContent.IsNewProjectStructureUsed && fileContent.UseAspNetCoreDlls && runtime.AspNetCoreRuntime is string aspRuntime) { progressMonitor.LogInfo($"ASP.NET runtime location selected: {aspRuntime}"); dllDirNames.Add(aspRuntime); } } private void RemoveRuntimeNugetPackageReferences() { var runtimePackagePrefixes = new[] { "microsoft.netcore.app.runtime", "microsoft.aspnetcore.app.runtime", "microsoft.windowsdesktop.app.runtime", // legacy runtime packages: "runtime.linux-x64.microsoft.netcore.app", "runtime.osx-x64.microsoft.netcore.app", "runtime.win-x64.microsoft.netcore.app", // Internal implementation packages not meant for direct consumption: "runtime." }; RemoveNugetPackageReference(runtimePackagePrefixes); } private void RemoveNugetPackageReference(params string[] packagePrefixes) { if (!options.UseNuGet) { return; } var packageFolder = packageDirectory.DirInfo.FullName.ToLowerInvariant(); if (packageFolder == null) { return; } var packagePathPrefixes = packagePrefixes.Select(p => Path.Combine(packageFolder, p.ToLowerInvariant())); foreach (var filename in usedReferences.Keys) { var lowerFilename = filename.ToLowerInvariant(); if (packagePathPrefixes.Any(prefix => lowerFilename.StartsWith(prefix))) { usedReferences.Remove(filename); progressMonitor.RemovedReference(filename); } } } private bool IsNugetPackageAvailable(string packagePrefix) { if (!options.UseNuGet) { return false; } return new DirectoryInfo(packageDirectory.DirInfo.FullName) .EnumerateDirectories(packagePrefix + "*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false }) .Any(); } private void GenerateSourceFileFromImplicitUsings() { var usings = new HashSet(); if (!fileContent.UseImplicitUsings) { return; } // Hardcoded values from https://learn.microsoft.com/en-us/dotnet/core/project-sdk/overview#implicit-using-directives usings.UnionWith(new[] { "System", "System.Collections.Generic", "System.IO", "System.Linq", "System.Net.Http", "System.Threading", "System.Threading.Tasks" }); if (fileContent.UseAspNetCoreDlls) { usings.UnionWith(new[] { "System.Net.Http.Json", "Microsoft.AspNetCore.Builder", "Microsoft.AspNetCore.Hosting", "Microsoft.AspNetCore.Http", "Microsoft.AspNetCore.Routing", "Microsoft.Extensions.Configuration", "Microsoft.Extensions.DependencyInjection", "Microsoft.Extensions.Hosting", "Microsoft.Extensions.Logging" }); } usings.UnionWith(fileContent.CustomImplicitUsings); if (usings.Count > 0) { var tempDir = GetTemporaryWorkingDirectory("implicitUsings"); var path = Path.Combine(tempDir, "GlobalUsings.g.cs"); using (var writer = new StreamWriter(path)) { writer.WriteLine("// "); writer.WriteLine(""); foreach (var u in usings.OrderBy(u => u)) { writer.WriteLine($"global using global::{u};"); } } this.generatedSources.Add(path); } } private void GenerateSourceFilesFromWebViews(List allFiles) { progressMonitor.LogInfo($"Generating source files from cshtml and razor files."); var views = allFiles.SelectFileNamesByExtension(".cshtml", ".razor").ToArray(); if (views.Length > 0) { progressMonitor.LogInfo($"Found {views.Length} cshtml and razor files."); var sdk = new Sdk(dotnet).GetNewestSdk(); if (sdk != null) { try { var razor = new Razor(sdk, dotnet, progressMonitor); var targetDir = GetTemporaryWorkingDirectory("razor"); var generatedFiles = razor.GenerateFiles(views, usedReferences.Keys, targetDir); this.generatedSources.AddRange(generatedFiles); } catch (Exception ex) { // It's okay, we tried our best to generate source files from cshtml files. progressMonitor.LogInfo($"Failed to generate source files from cshtml files: {ex.Message}"); } } } } public DependencyManager(string srcDir) : this(srcDir, DependencyOptions.Default, new ConsoleLogger(Verbosity.Info, logThreadId: true)) { } private IEnumerable GetAllFiles() { var files = sourceDir.GetFiles("*.*", new EnumerationOptions { RecurseSubdirectories = true }) .Where(d => !options.ExcludesFile(d.FullName)); if (options.DotNetPath != null) { files = files.Where(f => !f.FullName.StartsWith(options.DotNetPath, StringComparison.OrdinalIgnoreCase)); } return files; } /// /// Computes a unique temp directory for the packages associated /// with this source tree. Use a SHA1 of the directory name. /// /// The full path of the temp directory. private static string ComputeTempDirectory(string srcDir) { var bytes = Encoding.Unicode.GetBytes(srcDir); var sha = SHA1.HashData(bytes); var sb = new StringBuilder(); foreach (var b in sha.Take(8)) sb.AppendFormat("{0:x2}", b); return Path.Combine(Path.GetTempPath(), "GitHub", "packages", sb.ToString()); } /// /// Creates a temporary directory with the given subfolder name. /// The created directory might be inside the repo folder, and it is deleted when the object is disposed. /// /// /// private string GetTemporaryWorkingDirectory(string subfolder) { var temp = Path.Combine(tempWorkingDirectory.ToString(), subfolder); Directory.CreateDirectory(temp); return temp; } /// /// Resolves conflicts between all of the resolved references. /// If the same assembly name is duplicated with different versions, /// resolve to the higher version number. /// private void ResolveConflicts() { var sortedReferences = new List(); foreach (var usedReference in usedReferences) { try { var assemblyInfo = assemblyCache.GetAssemblyInfo(usedReference.Key); sortedReferences.Add(assemblyInfo); } catch (AssemblyLoadException) { progressMonitor.Log(Util.Logging.Severity.Warning, $"Could not load assembly information from {usedReference.Key}"); } } var emptyVersion = new Version(0, 0); sortedReferences = sortedReferences .OrderBy(r => r.NetCoreVersion ?? emptyVersion) .ThenBy(r => r.Version ?? emptyVersion) .ThenBy(r => r.Filename) .ToList(); var finalAssemblyList = new Dictionary(); // Pick the highest version for each assembly name foreach (var r in sortedReferences) { finalAssemblyList[r.Name] = r; } // Update the used references list usedReferences.Clear(); foreach (var r in finalAssemblyList.Select(r => r.Value.Filename)) { UseReference(r); } // Report the results foreach (var r in sortedReferences) { var resolvedInfo = finalAssemblyList[r.Name]; if (resolvedInfo.Version != r.Version || resolvedInfo.NetCoreVersion != r.NetCoreVersion) { progressMonitor.ResolvedConflict(r.Id, resolvedInfo.Id + resolvedInfo.NetCoreVersion is null ? "" : $" (.NET Core {resolvedInfo.NetCoreVersion})"); ++conflictedReferences; } } } /// /// Store that a particular reference file is used. /// /// The filename of the reference. private void UseReference(string reference) => usedReferences[reference] = true; /// /// Store that a particular source file is used (by a project file). /// /// The source file. private void UseSource(FileInfo sourceFile) => sources[sourceFile.FullName] = sourceFile.Exists; /// /// The list of resolved reference files. /// public IEnumerable ReferenceFiles => usedReferences.Keys; /// /// The list of source files used in projects. /// public IEnumerable ProjectSourceFiles => sources.Where(s => s.Value).Select(s => s.Key); /// /// All of the generated source files in the source directory. /// public IEnumerable GeneratedSourceFiles => generatedSources; /// /// All of the source files in the source directory. /// public IEnumerable AllSourceFiles => generatedSources.Concat(nonGeneratedSources); /// /// List of assembly IDs which couldn't be resolved. /// public IEnumerable UnresolvedReferences => unresolvedReferences.Select(r => r.Key); /// /// List of source files which were mentioned in project files but /// do not exist on the file system. /// public IEnumerable MissingSourceFiles => sources.Where(s => !s.Value).Select(s => s.Key); /// /// Record that a particular reference couldn't be resolved. /// Note that this records at most one project file per missing reference. /// /// The assembly ID. /// The project file making the reference. private void UnresolvedReference(string id, string projectFile) => unresolvedReferences[id] = projectFile; /// /// Reads all the source files and references from the given list of projects. /// /// The list of projects to analyse. private void AnalyseProjectFiles(IEnumerable projectFiles) { foreach (var proj in projectFiles) { AnalyseProject(proj); } } private void AnalyseProject(FileInfo project) { if (!project.Exists) { progressMonitor.MissingProject(project.FullName); return; } try { var csProj = new CsProjFile(project); foreach (var @ref in csProj.References) { try { var resolved = assemblyCache.ResolveReference(@ref); UseReference(resolved.Filename); } catch (AssemblyLoadException) { UnresolvedReference(@ref, project.FullName); } } foreach (var src in csProj.Sources) { // Make a note of which source files the projects use. // This information doesn't affect the build but is dumped // as diagnostic output. UseSource(new FileInfo(src)); } ++succeededProjects; } catch (Exception ex) // lgtm[cs/catch-of-all-exceptions] { ++failedProjects; progressMonitor.FailedProjectFile(project.FullName, ex.Message); } } private bool RestoreProject(string project, bool forceDotnetRefAssemblyFetching, string? pathToNugetConfig = null) => dotnet.RestoreProjectToDirectory(project, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching, pathToNugetConfig); private bool RestoreSolution(string solution, out IEnumerable projects) => dotnet.RestoreSolutionToDirectory(solution, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: true, out projects); /// /// Executes `dotnet restore` on all solution files in solutions. /// As opposed to RestoreProjects this is not run in parallel using PLINQ /// as `dotnet restore` on a solution already uses multiple threads for restoring /// the projects (this can be disabled with the `--disable-parallel` flag). /// Returns a list of projects that are up to date with respect to restore. /// /// A list of paths to solution files. private IEnumerable RestoreSolutions(IEnumerable solutions) => solutions.SelectMany(solution => { RestoreSolution(solution, out var restoredProjects); return restoredProjects; }); /// /// Executes `dotnet restore` on all projects in projects. /// This is done in parallel for performance reasons. /// /// A list of paths to project files. private void RestoreProjects(IEnumerable projects) { Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, project => { RestoreProject(project, forceDotnetRefAssemblyFetching: true); }); } private void DownloadMissingPackages(List allFiles) { var nugetConfigs = allFiles.SelectFileNamesByName("nuget.config").ToArray(); string? nugetConfig = null; if (nugetConfigs.Length > 1) { progressMonitor.MultipleNugetConfig(nugetConfigs); nugetConfig = allFiles .SelectRootFiles(sourceDir) .SelectFileNamesByName("nuget.config") .FirstOrDefault(); if (nugetConfig == null) { progressMonitor.NoTopLevelNugetConfig(); } } else { nugetConfig = nugetConfigs.FirstOrDefault(); } var alreadyDownloadedPackages = Directory.GetDirectories(packageDirectory.DirInfo.FullName) .Select(d => Path.GetFileName(d).ToLowerInvariant()); var notYetDownloadedPackages = fileContent.AllPackages.Except(alreadyDownloadedPackages); Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, package => { progressMonitor.NugetInstall(package); using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package)); var success = dotnet.New(tempDir.DirInfo.FullName); if (!success) { return; } success = dotnet.AddPackage(tempDir.DirInfo.FullName, package); if (!success) { return; } success = RestoreProject(tempDir.DirInfo.FullName, forceDotnetRefAssemblyFetching: false, pathToNugetConfig: nugetConfig); // TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package. if (!success) { progressMonitor.FailedToRestoreNugetPackage(package); } }); } private void AnalyseSolutions(IEnumerable solutions) { Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, solutionFile => { try { var sln = new SolutionFile(solutionFile); progressMonitor.AnalysingSolution(solutionFile); AnalyseProjectFiles(sln.Projects.Select(p => new FileInfo(p)).Where(p => p.Exists)); } catch (Microsoft.Build.Exceptions.InvalidProjectFileException ex) { progressMonitor.FailedProjectFile(solutionFile, ex.BaseMessage); } }); } public void Dispose() { try { packageDirectory?.Dispose(); } catch (Exception exc) { progressMonitor.LogInfo("Couldn't delete package directory: " + exc.Message); } if (cleanupTempWorkingDirectory) { try { tempWorkingDirectory?.Dispose(); } catch (Exception exc) { progressMonitor.LogInfo("Couldn't delete temporary working directory: " + exc.Message); } } } } }