C#: Generate source file with implicit usings in Standalone

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -30,9 +30,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         private readonly DotNet dotnet;
         private readonly FileContent fileContent;
         private readonly TemporaryDirectory packageDirectory;
-        private TemporaryDirectory? razorWorkingDirectory;
+        private readonly TemporaryDirectory tempWorkingDirectory;
-        private readonly Git git;
-
 
         /// <summary>
         /// Performs C# dependency fetching.
@@ -60,20 +58,21 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             this.progressMonitor.FindingFiles(srcDir);
 
             packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName));
-            var allFiles = GetAllFiles().ToList();
+            tempWorkingDirectory = new TemporaryDirectory(GetTemporaryWorkingDirectory());
-            var smallFiles = allFiles.SelectSmallFiles(progressMonitor).SelectFileNames();
+
-            this.fileContent = new FileContent(progressMonitor, smallFiles);
+            var allFiles = GetAllFiles();
-            this.allSources = allFiles.SelectFileNamesByExtension(".cs").ToList();
+            var binaryFileExtensions = new HashSet<string>(new[] { ".dll", ".exe" }); // TODO: add more binary file extensions.
-            var allProjects = allFiles.SelectFileNamesByExtension(".csproj");
+            var allNonBinaryFiles = allFiles.Where(f => !binaryFileExtensions.Contains(f.Extension.ToLowerInvariant())).ToList();
+            var smallNonBinaryFiles = allNonBinaryFiles.SelectSmallFiles(progressMonitor).SelectFileNames();
+            this.fileContent = new FileContent(progressMonitor, smallNonBinaryFiles);
+            this.allSources = allNonBinaryFiles.SelectFileNamesByExtension(".cs").ToList();
+            var allProjects = allNonBinaryFiles.SelectFileNamesByExtension(".csproj");
             var solutions = options.SolutionFile is not null
                 ? new[] { options.SolutionFile }
-                : allFiles.SelectFileNamesByExtension(".sln");
+                : allNonBinaryFiles.SelectFileNamesByExtension(".sln");
-
+            var dllDirNames = options.DllDirs.Count == 0
-            // If DLL reference paths are specified on the command-line, use those to discover
+                ? allFiles.SelectFileNamesByExtension(".dll").ToList()
-            // assemblies. Otherwise (the default), query the git CLI to determine which DLL files
+                : options.DllDirs.Select(Path.GetFullPath).ToList();
-            // are tracked as part of the repository.
-            this.git = new Git(this.progressMonitor);
-            var dllDirNames = options.DllDirs.Count == 0 ? this.git.ListFiles("*.dll") : options.DllDirs.Select(Path.GetFullPath).ToList();
 
             // Find DLLs in the .Net / Asp.Net Framework
             if (options.ScanNetFrameworkDlls)
@@ -106,7 +105,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                 var restoredProjects = RestoreSolutions(solutions);
                 var projects = allProjects.Except(restoredProjects);
                 RestoreProjects(projects);
-                DownloadMissingPackages(allFiles);
+                DownloadMissingPackages(allNonBinaryFiles);
             }
 
             assemblyCache = new AssemblyCache(dllDirNames, progressMonitor);
@@ -134,9 +133,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             if (bool.TryParse(webViewExtractionOption, out var shouldExtractWebViews) &&
                 shouldExtractWebViews)
             {
-                GenerateSourceFilesFromWebViews(allFiles);
+                GenerateSourceFilesFromWebViews(allNonBinaryFiles);
             }
 
+            GenerateSourceFileFromImplicitUsings();
+
             progressMonitor.Summary(
                 AllSourceFiles.Count(),
                 ProjectSourceFiles.Count(),
@@ -149,6 +150,46 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                 DateTime.Now - startTime);
         }
 
+        private void GenerateSourceFileFromImplicitUsings()
+        {
+            var usings = new HashSet<string>();
+            if (!fileContent.UseImplicitUsings)
+            {
+                return;
+            }
+
+            // Hardcoded values from https://learn.microsoft.com/en-us/dotnet/core/project-sdk/overview#implicit-using-directives
+            usings.UnionWith(new[] { "System", "System.Collections.Generic", "System.IO", "System.Linq", "System.Net.Http", "System.Threading",
+                "System.Threading.Tasks" });
+
+            if (fileContent.UseAspNetDlls)
+            {
+                usings.UnionWith(new[] { "System.Net.Http.Json", "Microsoft.AspNetCore.Builder", "Microsoft.AspNetCore.Hosting",
+                    "Microsoft.AspNetCore.Http", "Microsoft.AspNetCore.Routing", "Microsoft.Extensions.Configuration",
+                    "Microsoft.Extensions.DependencyInjection", "Microsoft.Extensions.Hosting", "Microsoft.Extensions.Logging" });
+            }
+
+            usings.UnionWith(fileContent.CustomImplicitUsings);
+
+            if (usings.Count > 0)
+            {
+                var tempDir = GetTemporaryWorkingDirectory("implicitUsings");
+                var path = Path.Combine(tempDir, "GlobalUsings.g.cs");
+                using (var writer = new StreamWriter(path))
+                {
+                    writer.WriteLine("// <auto-generated/>");
+                    writer.WriteLine("");
+
+                    foreach (var u in usings.OrderBy(u => u))
+                    {
+                        writer.WriteLine($"global using global::{u};");
+                    }
+                }
+
+                this.allSources.Add(path);
+            }
+        }
+
         private void GenerateSourceFilesFromWebViews(List<FileInfo> allFiles)
         {
             progressMonitor.LogInfo($"Generating source files from cshtml and razor files.");
@@ -165,8 +206,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                     try
                     {
                         var razor = new Razor(sdk, dotnet, progressMonitor);
-                        razorWorkingDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName, "razor"));
+                        var targetDir = GetTemporaryWorkingDirectory("razor");
-                        var generatedFiles = razor.GenerateFiles(views, usedReferences.Keys, razorWorkingDirectory.ToString());
+                        var generatedFiles = razor.GenerateFiles(views, usedReferences.Keys, targetDir);
                         this.allSources.AddRange(generatedFiles);
                     }
                     catch (Exception ex)
@@ -180,16 +221,25 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 
         public DependencyManager(string srcDir) : this(srcDir, DependencyOptions.Default, new ConsoleLogger(Verbosity.Info)) { }
 
-        private IEnumerable<FileInfo> GetAllFiles() =>
+        private IEnumerable<FileInfo> GetAllFiles()
-             sourceDir.GetFiles("*.*", new EnumerationOptions { RecurseSubdirectories = true })
+        {
-                .Where(d => d.Extension != ".dll" && !options.ExcludesFile(d.FullName));
+            var files = sourceDir.GetFiles("*.*", new EnumerationOptions { RecurseSubdirectories = true })
+                .Where(d => !options.ExcludesFile(d.FullName));
+
+            if (options.DotNetPath != null)
+            {
+                files = files.Where(f => !f.FullName.StartsWith(options.DotNetPath, StringComparison.OrdinalIgnoreCase));
+            }
+
+            return files;
+        }
 
         /// <summary>
         /// Computes a unique temp directory for the packages associated
         /// with this source tree. Use a SHA1 of the directory name.
         /// </summary>
         /// <returns>The full path of the temp directory.</returns>
-        private static string ComputeTempDirectory(string srcDir, string subfolderName = "packages")
+        private static string ComputeTempDirectory(string srcDir)
         {
             var bytes = Encoding.Unicode.GetBytes(srcDir);
             var sha = SHA1.HashData(bytes);
@@ -197,7 +247,35 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             foreach (var b in sha.Take(8))
                 sb.AppendFormat("{0:x2}", b);
 
-            return Path.Combine(Path.GetTempPath(), "GitHub", subfolderName, sb.ToString());
+            return Path.Combine(Path.GetTempPath(), "GitHub", "packages", sb.ToString());
+        }
+
+        private static string GetTemporaryWorkingDirectory()
+        {
+            var tempFolder = EnvironmentVariables.GetScratchDirectory();
+
+            if (string.IsNullOrEmpty(tempFolder))
+            {
+                var tempPath = Path.GetTempPath();
+                var name = Guid.NewGuid().ToString("N").ToUpper();
+                tempFolder = Path.Combine(tempPath, "GitHub", name);
+            }
+
+            return tempFolder;
+        }
+
+        /// <summary>
+        /// Creates a temporary directory with the given subfolder name.
+        /// The created directory might be inside the repo folder, and it is deleted when the object is disposed.
+        /// </summary>
+        /// <param name="subfolder"></param>
+        /// <returns></returns>
+        private string GetTemporaryWorkingDirectory(string subfolder)
+        {
+            var temp = Path.Combine(tempWorkingDirectory.ToString(), subfolder);
+            Directory.CreateDirectory(temp);
+
+            return temp;
         }
 
         /// <summary>
@@ -424,7 +502,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             foreach (var package in notYetDownloadedPackages)
             {
                 progressMonitor.NugetInstall(package);
-                using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package));
+                using var tempDir = new TemporaryDirectory(GetTemporaryWorkingDirectory(package));
                 var success = dotnet.New(tempDir.DirInfo.FullName);
                 if (!success)
                 {
@@ -467,7 +545,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         public void Dispose()
         {
             packageDirectory?.Dispose();
-            razorWorkingDirectory?.Dispose();
+            tempWorkingDirectory?.Dispose();
         }
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs

@@ -19,6 +19,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         private readonly IUnsafeFileReader unsafeFileReader;
         private readonly IEnumerable<string> files;
         private readonly HashSet<string> allPackages = new HashSet<string>();
+        private readonly HashSet<string> implicitUsingNamespaces = new HashSet<string>();
         private readonly Initializer initialize;
 
         public HashSet<string> AllPackages
@@ -48,6 +49,26 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             }
         }
 
+        private bool useImplicitUsings = false;
+
+        public bool UseImplicitUsings
+        {
+            get
+            {
+                initialize.Run();
+                return useImplicitUsings;
+            }
+        }
+
+        public HashSet<string> CustomImplicitUsings
+        {
+            get
+            {
+                initialize.Run();
+                return implicitUsingNamespaces;
+            }
+        }
+
         internal FileContent(ProgressMonitor progressMonitor,
             IEnumerable<string> files,
             IUnsafeFileReader unsafeFileReader)
@@ -62,7 +83,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
         public FileContent(ProgressMonitor progressMonitor, IEnumerable<string> files) : this(progressMonitor, files, new UnsafeFileReader())
         { }
 
-        private static string GetGroup(ReadOnlySpan<char> input, ValueMatch valueMatch, string groupPrefix)
+        private static string GetGroup(ReadOnlySpan<char> input, ValueMatch valueMatch, string groupPrefix, bool toLower)
         {
             var match = input.Slice(valueMatch.Index, valueMatch.Length);
             var includeIndex = match.IndexOf(groupPrefix, StringComparison.InvariantCultureIgnoreCase);
@@ -76,7 +97,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             var quoteIndex1 = match.IndexOf("\"");
             var quoteIndex2 = match.Slice(quoteIndex1 + 1).IndexOf("\"");
 
-            return match.Slice(quoteIndex1 + 1, quoteIndex2).ToString().ToLowerInvariant();
+            var result = match.Slice(quoteIndex1 + 1, quoteIndex2).ToString();
+
+            if (toLower)
+            {
+                result = result.ToLowerInvariant();
+            }
+
+            return result;
         }
 
         private static bool IsGroupMatch(ReadOnlySpan<char> line, Regex regex, string groupPrefix, string value)
@@ -84,7 +112,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             foreach (var valueMatch in regex.EnumerateMatches(line))
             {
                 // We can't get the group from the ValueMatch, so doing it manually:
-                if (GetGroup(line, valueMatch, groupPrefix) == value.ToLowerInvariant())
+                if (GetGroup(line, valueMatch, groupPrefix, toLower: true) == value.ToLowerInvariant())
                 {
                     return true;
                 }
@@ -105,7 +133,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                         foreach (var valueMatch in PackageReference().EnumerateMatches(line))
                         {
                             // We can't get the group from the ValueMatch, so doing it manually:
-                            var packageName = GetGroup(line, valueMatch, "Include");
+                            var packageName = GetGroup(line, valueMatch, "Include", toLower: true);
                             if (!string.IsNullOrEmpty(packageName))
                             {
                                 allPackages.Add(packageName);
@@ -119,6 +147,23 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                                 IsGroupMatch(line, ProjectSdk(), "Sdk", "Microsoft.NET.Sdk.Web") ||
                                 IsGroupMatch(line, FrameworkReference(), "Include", "Microsoft.AspNetCore.App");
                         }
+
+                        // Determine if implicit usings are used.
+                        if (!useImplicitUsings)
+                        {
+                            useImplicitUsings = line.Contains("<ImplicitUsings>enable</ImplicitUsings>".AsSpan(), StringComparison.Ordinal) ||
+                                                line.Contains("<ImplicitUsings>true</ImplicitUsings>".AsSpan(), StringComparison.Ordinal);
+                        }
+
+                        // Find all custom implicit usings.
+                        foreach (var valueMatch in CustomImplicitUsingDeclarations().EnumerateMatches(line))
+                        {
+                            var ns = GetGroup(line, valueMatch, "Include", toLower: false);
+                            if (!string.IsNullOrEmpty(ns))
+                            {
+                                implicitUsingNamespaces.Add(ns);
+                            }
+                        }
                     }
                 }
                 catch (Exception ex)
@@ -136,6 +181,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 
         [GeneratedRegex("<(.*\\s)?Project.*\\sSdk=\"(.*?)\".*/?>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
         private static partial Regex ProjectSdk();
+
+        [GeneratedRegex("<Using.*\\sInclude=\"(.*?)\".*/?>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
+        private static partial Regex CustomImplicitUsingDeclarations();
     }
 
     internal interface IUnsafeFileReader

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Git.cs

@@ -1,61 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-
-namespace Semmle.Extraction.CSharp.DependencyFetching
-{
-    /// <summary>
-    /// Utilities for querying information from the git CLI.
-    /// </summary>
-    internal class Git
-    {
-        private readonly ProgressMonitor progressMonitor;
-        private const string git = "git";
-
-        public Git(ProgressMonitor progressMonitor)
-        {
-            this.progressMonitor = progressMonitor;
-        }
-
-        /// <summary>
-        /// Lists all files matching <paramref name="pattern"/> which are tracked in the
-        /// current git repository.
-        /// </summary>
-        /// <param name="pattern">The file pattern.</param>
-        /// <returns>A list of all tracked files which match <paramref name="pattern"/>.</returns>
-        /// <exception cref="Exception"></exception>
-        public List<string> ListFiles(string pattern)
-        {
-            var results = new List<string>();
-            var args = string.Join(' ', "ls-files", $"\"{pattern}\"");
-
-            progressMonitor.RunningProcess($"{git} {args}");
-            var pi = new ProcessStartInfo(git, args)
-            {
-                UseShellExecute = false,
-                RedirectStandardOutput = true
-            };
-
-            using var p = new Process() { StartInfo = pi };
-            p.OutputDataReceived += new DataReceivedEventHandler((sender, e) =>
-            {
-                if (!string.IsNullOrWhiteSpace(e.Data))
-                {
-                    results.Add(e.Data);
-                }
-            });
-            p.Start();
-            p.BeginOutputReadLine();
-            p.WaitForExit();
-
-            if (p.ExitCode != 0)
-            {
-                progressMonitor.CommandFailed(git, args, p.ExitCode);
-                throw new Exception($"{git} {args} failed");
-            }
-
-            return results;
-        }
-
-    }
-}

csharp/extractor/Semmle.Util/EnvironmentVariables.cs

@@ -7,6 +7,8 @@ namespace Semmle.Util
         public static string? GetExtractorOption(string name) =>
             Environment.GetEnvironmentVariable($"CODEQL_EXTRACTOR_CSHARP_OPTION_{name.ToUpper()}");
 
+        public static string? GetScratchDirectory() => Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR");
+
         public static int GetDefaultNumberOfThreads()
         {
             if (!int.TryParse(Environment.GetEnvironmentVariable("CODEQL_THREADS"), out var threads) || threads == -1)