C#: Move source code generators to dedicated classes

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/ISourceGenerator.cs

@@ -0,0 +1,12 @@
+using System.Collections.Generic;
+
+namespace Semmle.Extraction.CSharp.DependencyFetching
+{
+    public interface ISourceGenerator
+    {
+        /// <summary>
+        /// Returns the paths to the generated source files.
+        /// </summary>
+        IEnumerable<string> Generate();
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/ImplicitUsingsGenerator.cs

@@ -0,0 +1,69 @@
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using Semmle.Util;
+using Semmle.Util.Logging;
+
+namespace Semmle.Extraction.CSharp.DependencyFetching
+{
+    internal class ImplicitUsingsGenerator : SourceGeneratorBase
+    {
+        private readonly FileContent fileContent;
+
+        public ImplicitUsingsGenerator(FileContent fileContent, ILogger logger, TemporaryDirectory tempWorkingDirectory)
+            : base(logger, tempWorkingDirectory)
+        {
+            this.fileContent = fileContent;
+        }
+
+        public override IEnumerable<string> Generate()
+        {
+            var usings = new HashSet<string>();
+            if (!fileContent.UseImplicitUsings)
+            {
+                logger.LogDebug("Skipping implicit usings generation");
+                return [];
+            }
+
+            // Hardcoded values from https://learn.microsoft.com/en-us/dotnet/core/project-sdk/overview#implicit-using-directives
+            usings.UnionWith([ "System", "System.Collections.Generic", "System.IO", "System.Linq", "System.Net.Http", "System.Threading",
+                "System.Threading.Tasks" ]);
+
+            if (fileContent.UseAspNetCoreDlls)
+            {
+                usings.UnionWith([ "System.Net.Http.Json", "Microsoft.AspNetCore.Builder", "Microsoft.AspNetCore.Hosting",
+                    "Microsoft.AspNetCore.Http", "Microsoft.AspNetCore.Routing", "Microsoft.Extensions.Configuration",
+                    "Microsoft.Extensions.DependencyInjection", "Microsoft.Extensions.Hosting", "Microsoft.Extensions.Logging" ]);
+            }
+
+            if (fileContent.UseWindowsForms)
+            {
+                usings.UnionWith(["System.Drawing", "System.Windows.Forms"]);
+            }
+
+            usings.UnionWith(fileContent.CustomImplicitUsings);
+
+            logger.LogInfo($"Generating source file for implicit usings. Namespaces: {string.Join(", ", usings.OrderBy(u => u))}");
+
+            if (usings.Count > 0)
+            {
+                var tempDir = GetTemporaryWorkingDirectory("implicitUsings");
+                var path = Path.Combine(tempDir, "GlobalUsings.g.cs");
+                using (var writer = new StreamWriter(path))
+                {
+                    writer.WriteLine("// <auto-generated/>");
+                    writer.WriteLine("");
+
+                    foreach (var u in usings.OrderBy(u => u))
+                    {
+                        writer.WriteLine($"global using global::{u};");
+                    }
+                }
+
+                return [path];
+            }
+
+            return [];
+        }
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/Razor.cs

@@ -0,0 +1,131 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+using System.Linq;
+using Semmle.Util;
+using Semmle.Util.Logging;
+
+namespace Semmle.Extraction.CSharp.DependencyFetching
+{
+    internal class Razor
+    {
+        private readonly DotNetVersion sdk;
+        private readonly ILogger logger;
+        private readonly IDotNet dotNet;
+        private readonly string sourceGeneratorFolder;
+        private readonly string cscPath;
+
+        public Razor(DotNetVersion sdk, IDotNet dotNet, ILogger logger)
+        {
+            this.sdk = sdk;
+            this.logger = logger;
+            this.dotNet = dotNet;
+
+            sourceGeneratorFolder = Path.Combine(this.sdk.FullPath, "Sdks", "Microsoft.NET.Sdk.Razor", "source-generators");
+            this.logger.LogInfo($"Razor source generator folder: {sourceGeneratorFolder}");
+            if (!Directory.Exists(sourceGeneratorFolder))
+            {
+                this.logger.LogInfo($"Razor source generator folder {sourceGeneratorFolder} does not exist.");
+                throw new Exception($"Razor source generator folder {sourceGeneratorFolder} does not exist.");
+            }
+
+            cscPath = Path.Combine(this.sdk.FullPath, "Roslyn", "bincore", "csc.dll");
+            this.logger.LogInfo($"Razor source generator CSC: {cscPath}");
+            if (!File.Exists(cscPath))
+            {
+                this.logger.LogInfo($"Csc.exe not found at {cscPath}.");
+                throw new Exception($"csc.dll {cscPath} does not exist.");
+            }
+        }
+
+        private static void GenerateAnalyzerConfig(IEnumerable<string> cshtmls, string analyzerConfigPath)
+        {
+            using var sw = new StreamWriter(analyzerConfigPath);
+            sw.WriteLine("is_global = true");
+
+            foreach (var f in cshtmls.Select(f => f.Replace('\\', '/')))
+            {
+                sw.WriteLine($"\n[{f}]");
+                var base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(f)); // TODO: this should be the relative path of the file.
+                sw.WriteLine($"build_metadata.AdditionalFiles.TargetPath = {base64}");
+            }
+        }
+
+        public IEnumerable<string> GenerateFiles(IEnumerable<string> cshtmls, IEnumerable<string> references, string workingDirectory)
+        {
+            var name = Guid.NewGuid().ToString("N").ToUpper();
+            var tempPath = FileUtils.GetTemporaryWorkingDirectory(out var shouldCleanUp);
+            var analyzerConfig = Path.Combine(tempPath, $"{name}.txt");
+            var dllPath = Path.Combine(tempPath, $"{name}.dll");
+            var cscArgsPath = Path.Combine(tempPath, $"{name}.rsp");
+            var outputFolder = Path.Combine(workingDirectory, name);
+            Directory.CreateDirectory(outputFolder);
+
+            try
+            {
+                logger.LogInfo("Produce analyzer config content.");
+                GenerateAnalyzerConfig(cshtmls, analyzerConfig);
+
+                logger.LogDebug($"Analyzer config content: {File.ReadAllText(analyzerConfig)}");
+
+                var args = new StringBuilder();
+                args.Append($"/target:exe /generatedfilesout:\"{outputFolder}\" /out:\"{dllPath}\" /analyzerconfig:\"{analyzerConfig}\" ");
+
+                foreach (var f in Directory.GetFiles(sourceGeneratorFolder, "*.dll", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive }))
+                {
+                    args.Append($"/analyzer:\"{f}\" ");
+                }
+
+                foreach (var f in cshtmls)
+                {
+                    args.Append($"/additionalfile:\"{f}\" ");
+                }
+
+                foreach (var f in references)
+                {
+                    args.Append($"/reference:\"{f}\" ");
+                }
+
+                var argsString = args.ToString();
+
+                logger.LogInfo($"Running CSC to generate Razor source files.");
+                logger.LogDebug($"Running CSC to generate Razor source files with arguments: {argsString}.");
+
+                using (var sw = new StreamWriter(cscArgsPath))
+                {
+                    sw.Write(argsString);
+                }
+
+                dotNet.Exec($"\"{cscPath}\" /noconfig @\"{cscArgsPath}\"");
+
+                var files = Directory.GetFiles(outputFolder, "*.*", new EnumerationOptions { RecurseSubdirectories = true });
+
+                logger.LogInfo($"Generated {files.Length} source files from cshtml files.");
+
+                return files;
+            }
+            finally
+            {
+                if (shouldCleanUp)
+                {
+                    DeleteFile(analyzerConfig);
+                    DeleteFile(dllPath);
+                    DeleteFile(cscArgsPath);
+                }
+            }
+        }
+
+        private void DeleteFile(string path)
+        {
+            try
+            {
+                File.Delete(path);
+            }
+            catch (Exception exc)
+            {
+                logger.LogWarning($"Failed to delete file {path}: {exc}");
+            }
+        }
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/SourceGeneratorBase.cs

@@ -0,0 +1,33 @@
+using System.Collections.Generic;
+using System.IO;
+using Semmle.Util;
+using Semmle.Util.Logging;
+
+namespace Semmle.Extraction.CSharp.DependencyFetching
+{
+    internal abstract class SourceGeneratorBase : ISourceGenerator
+    {
+        protected readonly ILogger logger;
+        protected readonly TemporaryDirectory tempWorkingDirectory;
+
+        public SourceGeneratorBase(ILogger logger, TemporaryDirectory tempWorkingDirectory)
+        {
+            this.logger = logger;
+            this.tempWorkingDirectory = tempWorkingDirectory;
+        }
+
+        public abstract IEnumerable<string> Generate();
+
+        /// <summary>
+        /// Creates a temporary directory with the given subfolder name.
+        /// The created directory might be inside the repo folder, and it is deleted when the temporary working directory is disposed.
+        /// </summary>
+        protected string GetTemporaryWorkingDirectory(string subfolder)
+        {
+            var temp = Path.Combine(tempWorkingDirectory.ToString(), subfolder);
+            Directory.CreateDirectory(temp);
+
+            return temp;
+        }
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/WebViewGenerator.cs

@@ -0,0 +1,86 @@
+using System;
+using System.Collections.Generic;
+using Semmle.Util;
+using Semmle.Util.Logging;
+
+namespace Semmle.Extraction.CSharp.DependencyFetching
+{
+    internal class WebViewGenerator : SourceGeneratorBase
+    {
+        private readonly FileProvider fileProvider;
+        private readonly FileContent fileContent;
+        private readonly IDotNet dotnet;
+        private readonly ICompilationInfoContainer compilationInfoContainer;
+        private readonly IEnumerable<string> references;
+
+        public WebViewGenerator(
+            FileProvider fileProvider,
+            FileContent fileContent,
+            IDotNet dotnet,
+            ICompilationInfoContainer compilationInfoContainer,
+            ILogger logger,
+            TemporaryDirectory tempWorkingDirectory,
+            IEnumerable<string> references) : base(logger, tempWorkingDirectory)
+        {
+            this.fileProvider = fileProvider;
+            this.fileContent = fileContent;
+            this.dotnet = dotnet;
+            this.compilationInfoContainer = compilationInfoContainer;
+            this.references = references;
+        }
+
+        public override IEnumerable<string> Generate()
+        {
+            var webViewExtractionOption = Environment.GetEnvironmentVariable(EnvironmentVariableNames.WebViewGeneration);
+            if (webViewExtractionOption == null ||
+                bool.TryParse(webViewExtractionOption, out var shouldExtractWebViews) &&
+                shouldExtractWebViews)
+            {
+                compilationInfoContainer.CompilationInfos.Add(("WebView extraction enabled", "1"));
+                return GenerateSourceFilesFromWebViews();
+            }
+
+            compilationInfoContainer.CompilationInfos.Add(("WebView extraction enabled", "0"));
+            return [];
+        }
+
+        private IEnumerable<string> GenerateSourceFilesFromWebViews()
+        {
+            var views = fileProvider.RazorViews;
+            if (views.Count == 0)
+            {
+                logger.LogDebug("No cshtml or razor files found.");
+                return [];
+            }
+
+            logger.LogInfo($"Found {views.Count} cshtml and razor files.");
+
+            if (!fileContent.IsAspNetCoreDetected)
+            {
+                logger.LogInfo("Generating source files from cshtml files is only supported for new (SDK-style) project files");
+                return [];
+            }
+
+            logger.LogInfo("Generating source files from cshtml and razor files...");
+
+            var sdk = new Sdk(dotnet).GetNewestSdk();
+            if (sdk != null)
+            {
+                try
+                {
+                    var razor = new Razor(sdk, dotnet, logger);
+                    var targetDir = GetTemporaryWorkingDirectory("razor");
+                    var generatedFiles = razor.GenerateFiles(views, references, targetDir);
+                    return generatedFiles ?? [];
+                }
+                catch (Exception ex)
+                {
+                    // It's okay, we tried our best to generate source files from cshtml files.
+                    logger.LogInfo($"Failed to generate source files from cshtml files: {ex.Message}");
+                }
+            }
+
+            return [];
+        }
+    }
+}