Merge pull request #13290 from tamasvajk/feature/source-generators

C#: Extract source files generated by source generators

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs

@@ -381,8 +381,17 @@ namespace Semmle.Extraction.CSharp
                 references => ResolveReferences(compilerArguments, analyser, canonicalPathCache, references),
                 (analyser, syntaxTrees) =>
                 {
+                    var paths = compilerArguments.SourceFiles
+                        .Select(src => src.Path)
+                        .ToList();
+
+                    if (compilerArguments.GeneratedFilesOutputDirectory is not null)
+                    {
+                        paths.AddRange(Directory.GetFiles(compilerArguments.GeneratedFilesOutputDirectory, "*.cs", SearchOption.AllDirectories));
+                    }
+
                     return ReadSyntaxTrees(
-                        compilerArguments.SourceFiles.Select(src => canonicalPathCache.GetCanonicalPath(src.Path)),
+                        paths.Select(canonicalPathCache.GetCanonicalPath),
                         analyser,
                         compilerArguments.ParseOptions,
                         compilerArguments.Encoding,

csharp/ql/integration-tests/all-platforms/cshtml/Files.expected

@@ -0,0 +1,6 @@
+| Program.cs:0:0:0:0 | Program.cs |
+| obj/Debug/net7.0/.NETCoreApp,Version=v7.0.AssemblyAttributes.cs:0:0:0:0 | obj/Debug/net7.0/.NETCoreApp,Version=v7.0.AssemblyAttributes.cs |
+| obj/Debug/net7.0/cshtml.AssemblyInfo.cs:0:0:0:0 | obj/Debug/net7.0/cshtml.AssemblyInfo.cs |
+| obj/Debug/net7.0/cshtml.GlobalUsings.g.cs:0:0:0:0 | obj/Debug/net7.0/cshtml.GlobalUsings.g.cs |
+| obj/Debug/net7.0/cshtml.RazorAssemblyInfo.cs:0:0:0:0 | obj/Debug/net7.0/cshtml.RazorAssemblyInfo.cs |
+| obj/Debug/net7.0/generated/Microsoft.NET.Sdk.Razor.SourceGenerators/Microsoft.NET.Sdk.Razor.SourceGenerators.RazorSourceGenerator/Views_Home_Index_cshtml.g.cs:0:0:0:0 | obj/Debug/net7.0/generated/Microsoft.NET.Sdk.Razor.SourceGenerators/Microsoft.NET.Sdk.Razor.SourceGenerators.RazorSourceGenerator/Views_Home_Index_cshtml.g.cs |

csharp/ql/integration-tests/all-platforms/cshtml/Files.ql

@@ -0,0 +1,5 @@
+import csharp
+
+from File f
+where f.fromSource()
+select f

csharp/ql/integration-tests/all-platforms/cshtml/Program.cs

@@ -0,0 +1 @@
+var dummy = "dummy";

csharp/ql/integration-tests/all-platforms/cshtml/Views/Home/Index.cshtml

@@ -0,0 +1,8 @@
+@{
+    ViewData["Title"] = "Home Page";
+}
+
+<div class="text-center">
+    <h1 class="display-4">Welcome</h1>
+    <p>Learn about <a href="https://docs.microsoft.com/aspnet/core">building Web apps with ASP.NET Core</a>.</p>
+</div>

csharp/ql/integration-tests/all-platforms/cshtml/cshtml.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+ <Target Name="DeleteBinObjFolders" BeforeTargets="Clean">
+   <RemoveDir Directories=".\bin" />
+   <RemoveDir Directories=".\obj" />
+ </Target>
+</Project>

csharp/ql/integration-tests/all-platforms/cshtml/test.py

@@ -0,0 +1,3 @@
+from create_database_utils import *
+
+run_codeql_database_create(['dotnet build'], lang="csharp", extra_args=["--extractor-option=cil=false"])

csharp/ql/lib/change-notes/2023-05-30-source-generators.md

@@ -0,0 +1,4 @@
+---
+category: majorAnalysis
+---
+* The extractor has been changed to run after the traced compiler call. This allows inspecting compiler generated files, such as the output of source generators. With this change, `.cshtml` files and their generated `.cshtml.g.cs` counterparts are extracted on dotnet 6 and above.

csharp/tools/tracing-config.lua

@@ -63,7 +63,7 @@ function RegisterExtractorPack(id)
             end
         end
         if match then
-            local injections = { '-p:UseSharedCompilation=false' }
+            local injections = { '-p:UseSharedCompilation=false', '-p:EmitCompilerGeneratedFiles=true' }
             if dotnetRunNeedsSeparator then
                 table.insert(injections, '--')
             end
@@ -118,7 +118,8 @@ function RegisterExtractorPack(id)
                     compilerArguments,
                     nil, {
                     '/p:UseSharedCompilation=false',
-                    '/p:MvcBuildViews=true'
+                    '/p:MvcBuildViews=true',
+                    '/p:EmitCompilerGeneratedFiles=true',
                 })
 
             }
@@ -154,7 +155,7 @@ function RegisterExtractorPack(id)
 
             if seenCompilerCall then
                 return {
-                    order = ORDER_BEFORE,
+                    order = ORDER_AFTER,
                     invocation = {
                         path = AbsolutifyExtractorPath(id, extractor),
                         arguments = {
@@ -194,7 +195,7 @@ function RegisterExtractorPack(id)
 
             if seenCompilerCall then
                 return {
-                    order = ORDER_BEFORE,
+                    order = ORDER_AFTER,
                     invocation = {
                         path = AbsolutifyExtractorPath(id, extractor),
                         arguments = {