Run dotnet source generators on files grouped by projects

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/DotnetSourceGeneratorBase.cs

@@ -1,5 +1,7 @@
 using System;
 using System.Collections.Generic;
+using System.IO;
+using System.Linq;
 using Semmle.Util;
 using Semmle.Util.Logging;
 
@@ -31,8 +33,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 
         protected override IEnumerable<string> Run()
         {
-            var additionalFiles = AdditionalFiles;
-            if (additionalFiles.Count == 0)
+            if (AdditionalFiles.Count == 0)
             {
                 logger.LogDebug($"No {FileType} files found. Skipping source generation.");
                 return [];
@@ -44,16 +45,51 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
                 return [];
             }
 
-            logger.LogInfo($"Generating source files from {additionalFiles.Count} {FileType} files...");
+            if (fileProvider.Projects.Count == 0)
+            {
+                logger.LogInfo($"No projects found. Skipping source generation from {FileType} files.");
+                return [];
+            }
+
+            logger.LogInfo($"Generating source files from {AdditionalFiles.Count} {FileType} files...");
+
+            // group additional files by closes project file:
+            var projects = fileProvider.Projects
+                .Select(p => (File: p, Directory: SafeGetDirectoryName(p)))
+                .Where(p => p.Directory.Length > 0);
+
+            var groupedFiles = new Dictionary<string, List<string>>();
+
+            foreach (var additionalFile in AdditionalFiles)
+            {
+                var project = projects
+                    .Where(p => additionalFile.StartsWith(p.Directory))
+                    .OrderByDescending(p => p.Directory.Length)
+                    .FirstOrDefault();
+
+                if (project == default)
+                {
+                    logger.LogDebug($"Failed to find project file for {additionalFile}");
+                    continue;
+                }
+
+                if (!groupedFiles.TryGetValue(project.File, out var files))
+                {
+                    files = [];
+                    groupedFiles[project.File] = files;
+                }
+
+                files.Add(additionalFile);
+            }
 
             try
             {
                 var sdk = new Sdk(dotnet, logger);
                 var sourceGenerator = GetSourceGenerator(sdk);
                 var targetDir = GetTemporaryWorkingDirectory(FileType.ToLowerInvariant());
-                // todo: run the below in a loop, on groups of files belonging to the same project:
-                var generatedFiles = sourceGenerator.RunSourceGenerator(additionalFiles, references, targetDir);
-                return generatedFiles ?? [];
+
+                return groupedFiles
+                    .SelectMany(group => sourceGenerator.RunSourceGenerator(group.Value, group.Key, references, targetDir));
             }
             catch (Exception ex)
             {
@@ -63,6 +99,30 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             }
         }
 
+        private string SafeGetDirectoryName(string fileName)
+        {
+            try
+            {
+                var dir = Path.GetDirectoryName(fileName);
+                if (dir is null)
+                {
+                    return "";
+                }
+
+                if (!dir.EndsWith(Path.DirectorySeparatorChar))
+                {
+                    dir += Path.DirectorySeparatorChar;
+                }
+
+                return dir;
+            }
+            catch (Exception ex)
+            {
+                logger.LogDebug($"Failed to get directory name for {fileName}: {ex.Message}");
+                return "";
+            }
+        }
+
         protected abstract ICollection<string> AdditionalFiles { get; }
 
         protected abstract string FileType { get; }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/DotnetSourceGeneratorWrapper/DotnetSourceGeneratorWrapper.cs

@@ -31,9 +31,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             this.cscPath = sdk.CscPath;
         }
 
-        protected abstract void GenerateAnalyzerConfig(IEnumerable<string> additionalFiles, string analyzerConfigPath);
+        protected abstract void GenerateAnalyzerConfig(IEnumerable<string> additionalFiles, string csprojFile, string analyzerConfigPath);
 
-        public IEnumerable<string> RunSourceGenerator(IEnumerable<string> additionalFiles, IEnumerable<string> references, string targetDir)
+        public IEnumerable<string> RunSourceGenerator(IEnumerable<string> additionalFiles, string csprojFile, IEnumerable<string> references, string targetDir)
         {
             var name = Guid.NewGuid().ToString("N").ToUpper();
             var tempPath = FileUtils.GetTemporaryWorkingDirectory(out var shouldCleanUp);
@@ -46,7 +46,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             try
             {
                 logger.LogInfo("Producing analyzer config content.");
-                GenerateAnalyzerConfig(additionalFiles, analyzerConfig);
+                GenerateAnalyzerConfig(additionalFiles, csprojFile, analyzerConfig);
 
                 logger.LogDebug($"Analyzer config content: {File.ReadAllText(analyzerConfig)}");
 
@@ -86,6 +86,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 
                 return files;
             }
+            catch (Exception ex)
+            {
+                logger.LogInfo($"Failed to generate source files from {FileType} files: {ex.Message}");
+                return [];
+            }
             finally
             {
                 if (shouldCleanUp)

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/DotnetSourceGeneratorWrapper/Razor.cs

@@ -29,7 +29,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             }
         }
 
-        protected override void GenerateAnalyzerConfig(IEnumerable<string> cshtmls, string analyzerConfigPath)
+        protected override void GenerateAnalyzerConfig(IEnumerable<string> cshtmls, string csprojFile, string analyzerConfigPath)
         {
             using var sw = new StreamWriter(analyzerConfigPath);
             sw.WriteLine("is_global = true");

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/DotnetSourceGeneratorWrapper/Resx.cs

@@ -22,11 +22,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
             SourceGeneratorFolder = sourceGeneratorFolder;
         }
 
-        protected override void GenerateAnalyzerConfig(IEnumerable<string> resources, string analyzerConfigPath)
+        protected override void GenerateAnalyzerConfig(IEnumerable<string> resources, string csprojFile, string analyzerConfigPath)
         {
             using var sw = new StreamWriter(analyzerConfigPath);
             sw.WriteLine("is_global = true");
-            sw.WriteLine("build_property.RootNamespace = abc"); // todo: fix the namespace
+
+            var rootNamespace = Path.GetFileNameWithoutExtension(csprojFile);
+            sw.WriteLine($"build_property.RootNamespace = {rootNamespace}");
 
             foreach (var f in resources.Select(f => f.Replace('\\', '/')))
             {

csharp/ql/integration-tests/all-platforms/standalone_resx/Members.expected

@@ -1,8 +1,8 @@
 | Program | Program.<Main>$ |
 | Program | Program.Program |
-| abc.test | abc.test.Culture |
-| abc.test | abc.test.GetResourceString |
-| abc.test | abc.test.Key123 |
-| abc.test | abc.test.Key456 |
-| abc.test | abc.test.ResourceManager |
-| abc.test | abc.test.s_resourceManager |
+| resx.test | resx.test.Culture |
+| resx.test | resx.test.GetResourceString |
+| resx.test | resx.test.Key123 |
+| resx.test | resx.test.Key456 |
+| resx.test | resx.test.ResourceManager |
+| resx.test | resx.test.s_resourceManager |