using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Security.Policy; using Semmle.Util.Logging; namespace Semmle.Extraction.CSharp.DependencyFetching { public class FileProvider { private static readonly HashSet binaryFileExtensions = [".dll", ".exe"]; // TODO: add more binary file extensions. private readonly ILogger logger; private readonly FileInfo[] all; private readonly Lazy allNonBinary; private readonly Lazy smallNonBinary; private readonly Lazy sources; private readonly Lazy projects; private readonly Lazy solutions; private readonly Lazy dlls; private readonly Lazy nugetConfigs; private readonly Lazy nugetExes; private readonly Lazy globalJsons; private readonly Lazy packagesConfigs; private readonly Lazy razorViews; private readonly Lazy resources; private readonly Lazy rootNugetConfig; public FileProvider(DirectoryInfo sourceDir, ILogger logger) { SourceDir = sourceDir; this.logger = logger; all = GetAllFiles(); allNonBinary = new Lazy(() => all.Where(f => !binaryFileExtensions.Contains(f.Extension.ToLowerInvariant())).ToArray()); smallNonBinary = new Lazy(() => ReturnAndLogFiles("small non-binary", SelectSmallFiles(allNonBinary.Value).SelectFileNames().ToArray())); sources = new Lazy(() => SelectTextFileNamesByExtension("source", ".cs")); projects = new Lazy(() => SelectTextFileNamesByExtension("project", ".csproj")); solutions = new Lazy(() => SelectTextFileNamesByExtension("solution", ".sln")); dlls = new Lazy(() => SelectBinaryFileNamesByExtension("DLL", ".dll")); nugetConfigs = new Lazy(() => SelectTextFileNamesByName("nuget.config")); globalJsons = new Lazy(() => SelectTextFileNamesByName("global.json")); packagesConfigs = new Lazy(() => SelectTextFileNamesByName("packages.config")); razorViews = new Lazy(() => SelectTextFileNamesByExtension("razor view", ".cshtml", ".razor")); resources = new Lazy(() => SelectTextFileNamesByExtension("resource", ".resx")); rootNugetConfig = new Lazy(() => all.SelectRootFiles(SourceDir).SelectFileNamesByName("nuget.config").FirstOrDefault()); nugetExes = new Lazy(() => all.SelectFileNamesByName("nuget.exe").ToArray()); } private string[] ReturnAndLogFiles(string filetype, IEnumerable files) { var ret = files.ToArray(); logger.LogInfo($"Found {ret.Length} {filetype} files in {SourceDir}."); return ret; } private string[] SelectTextFileNamesByExtension(string filetype, params string[] extensions) => ReturnAndLogFiles(filetype, allNonBinary.Value.SelectFileNamesByExtension(extensions)); private string[] SelectTextFileNamesByName(string name) { var ret = allNonBinary.Value.SelectFileNamesByName(name).ToArray(); var ending = ret.Length == 0 ? "." : $": {string.Join(", ", ret.OrderBy(s => s))}."; logger.LogInfo($"Found {ret.Length} {name} files in {SourceDir}{ending}"); return ret; } private string[] SelectBinaryFileNamesByExtension(string filetype, params string[] extensions) { var ret = all.SelectFileNamesByExtension(extensions).ToArray(); logger.LogInfo($"Found {ret.Length} {filetype} files in {SourceDir}."); return ret; } private IEnumerable SelectSmallFiles(IEnumerable files) { const int oneMb = 1_048_576; return files.Where(file => { if (file.Length > oneMb) { logger.LogDebug($"Skipping {file.FullName} because it is bigger than 1MB."); return false; } return true; }); } private FileInfo[] GetAllFiles() { logger.LogInfo($"Finding files in {SourceDir}..."); var files = SourceDir.GetFiles("*.*", new EnumerationOptions { RecurseSubdirectories = true }); var filteredFiles = files.Where(f => { try { if (f.Exists) { return true; } logger.LogWarning($"File {f.FullName} could not be processed."); return false; } catch (Exception ex) { logger.LogWarning($"File {f.FullName} could not be processed: {ex.Message}"); return false; } }); var allFiles = new FilePathFilter(SourceDir, logger).Filter(filteredFiles).ToArray(); logger.LogInfo($"Found {allFiles.Length} files in {SourceDir}."); return allFiles; } public DirectoryInfo SourceDir { get; } public IEnumerable SmallNonBinary => smallNonBinary.Value; public IEnumerable Sources => sources.Value; public ICollection Projects => projects.Value; public ICollection Solutions => solutions.Value; public IEnumerable Dlls => dlls.Value; public ICollection NugetConfigs => nugetConfigs.Value; public ICollection NugetExes => nugetExes.Value; public string? RootNugetConfig => rootNugetConfig.Value; public IEnumerable GlobalJsons => globalJsons.Value; public ICollection PackagesConfigs => packagesConfigs.Value; public ICollection RazorViews => razorViews.Value; public ICollection Resources => resources.Value; } }