hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-17 01:03:14 +01:00
Code Issues Packages Projects Releases Wiki Activity

C#: Add resource generator

Browse Source
This commit is contained in:
Tamas Vajk
2024-04-16 11:25:01 +02:00
parent 407837afc4
commit 79fe5f851b
12 changed files with 241 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
View File

@@ -152,7 +152,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var sourceGenerators = new ISourceGenerator[]
{
new ImplicitUsingsGenerator(fileContent, logger, tempWorkingDirectory),
new WebViewGenerator(fileProvider, fileContent, dotnet, this, logger, tempWorkingDirectory, usedReferences.Keys)
new RazorGenerator(fileProvider, fileContent, dotnet, this, logger, tempWorkingDirectory, usedReferences.Keys),
new ResxGenerator(fileProvider, fileContent, dotnet, this, logger, tempWorkingDirectory, usedReferences.Keys),
};
foreach (var sourceGenerator in sourceGenerators)

5
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs
View File

@@ -2,6 +2,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal class EnvironmentVariableNames
{
/// <summary>
/// Controls whether to generate source files from resources (`.resx`).
/// </summary>
public const string ResourceGeneration = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_EXTRACT_RESOURCES";
/// <summary>
/// Controls whether to generate source files from Asp.Net Core views (`.cshtml`, `.razor`).
/// </summary>

3
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileProvider.cs
View File

@@ -22,6 +22,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private readonly Lazy<string[]> nugetConfigs;
private readonly Lazy<string[]> globalJsons;
private readonly Lazy<string[]> razorViews;
private readonly Lazy<string[]> resources;
private readonly Lazy<string?> rootNugetConfig;
public FileProvider(DirectoryInfo sourceDir, ILogger logger)
@@ -44,6 +45,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
nugetConfigs = new Lazy<string[]>(() => allNonBinary.Value.SelectFileNamesByName("nuget.config").ToArray());
globalJsons = new Lazy<string[]>(() => allNonBinary.Value.SelectFileNamesByName("global.json").ToArray());
razorViews = new Lazy<string[]>(() => SelectTextFileNamesByExtension("razor view", ".cshtml", ".razor"));
resources = new Lazy<string[]>(() => SelectTextFileNamesByExtension("resource", ".resx"));
rootNugetConfig = new Lazy<string?>(() => all.SelectRootFiles(SourceDir).SelectFileNamesByName("nuget.config").FirstOrDefault());
}
@@ -116,5 +118,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
public string? RootNugetConfig => rootNugetConfig.Value;
public IEnumerable<string> GlobalJsons => globalJsons.Value;
public ICollection<string> RazorViews => razorViews.Value;
public ICollection<string> Resources => resources.Value;
}
}

75
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/DotnetSourceGeneratorBase.cs Normal file
View File

@@ -0,0 +1,75 @@
using System;
using System.Collections.Generic;
using Semmle.Util;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal abstract class DotnetSourceGeneratorBase<T> : SourceGeneratorBase where T : DotnetSourceGeneratorWrapper
{
protected readonly FileProvider fileProvider;
protected readonly FileContent fileContent;
protected readonly IDotNet dotnet;
protected readonly ICompilationInfoContainer compilationInfoContainer;
protected readonly IEnumerable<string> references;
public DotnetSourceGeneratorBase(
FileProvider fileProvider,
FileContent fileContent,
IDotNet dotnet,
ICompilationInfoContainer compilationInfoContainer,
ILogger logger,
TemporaryDirectory tempWorkingDirectory,
IEnumerable<string> references) : base(logger, tempWorkingDirectory)
{
this.fileProvider = fileProvider;
this.fileContent = fileContent;
this.dotnet = dotnet;
this.compilationInfoContainer = compilationInfoContainer;
this.references = references;
}
protected override IEnumerable<string> Run()
{
var additionalFiles = AdditionalFiles;
if (additionalFiles.Count == 0)
{
logger.LogDebug($"No {FileType} files found.");
return [];
}
logger.LogInfo($"Found {additionalFiles.Count} {FileType} files.");
if (!fileContent.IsAspNetCoreDetected)
{
logger.LogInfo($"Generating source files from {FileType} files is only supported for new (SDK-style) project files");
return [];
}
logger.LogInfo($"Generating source files from {FileType} files...");
try
{
var sdk = new Sdk(dotnet, logger);
var sourceGenerator = GetSourceGenerator(sdk);
var targetDir = GetTemporaryWorkingDirectory(FileType.ToLowerInvariant());
// todo: run the below in a loop, on groups of files belonging to the same project:
var generatedFiles = sourceGenerator.RunSourceGenerator(additionalFiles, references, targetDir);
return generatedFiles ?? [];
}
catch (Exception ex)
{
// It's okay, we tried our best to generate source files.
logger.LogInfo($"Failed to generate source files from {FileType} files: {ex.Message}");
return [];
}
}
protected abstract ICollection<string> AdditionalFiles { get; }
protected abstract string FileType { get; }
protected abstract T GetSourceGenerator(Sdk sdk);
}
}

17
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/DotnetSourceGeneratorWrapper/DotnetSourceGeneratorWrapper.cs
View File

@@ -10,8 +10,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
internal abstract class DotnetSourceGeneratorWrapper
{
protected readonly ILogger logger;
private readonly Sdk sdk;
protected readonly IDotNet dotnet;
private readonly string cscPath;
protected abstract string SourceGeneratorFolder { get; init; }
protected abstract string FileType { get; }
@@ -22,20 +22,19 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
ILogger logger)
{
this.logger = logger;
this.sdk = sdk;
this.dotnet = dotnet;
if (sdk.CscPath is null)
{
throw new Exception($"Not running {FileType} source generator because CSC path is not available.");
}
this.cscPath = sdk.CscPath;
}
protected abstract void GenerateAnalyzerConfig(IEnumerable<string> additionalFiles, string analyzerConfigPath);
public IEnumerable<string> RunSourceGenerator(IEnumerable<string> additionalFiles, IEnumerable<string> references, string targetDir)
{
if (sdk.CscPath is null)
{
logger.LogWarning("Not running source generator because csc path is not available.");
return [];
}
var name = Guid.NewGuid().ToString("N").ToUpper();
var tempPath = FileUtils.GetTemporaryWorkingDirectory(out var shouldCleanUp);
var analyzerConfig = Path.Combine(tempPath, $"{name}.txt");
@@ -79,7 +78,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
sw.Write(argsString);
}
dotnet.Exec($"\"{sdk.CscPath}\" /noconfig @\"{cscArgsPath}\"");
dotnet.Exec($"\"{cscPath}\" /noconfig @\"{cscArgsPath}\"");
var files = Directory.GetFiles(outputFolder, "*.*", new EnumerationOptions { RecurseSubdirectories = true });

2
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/DotnetSourceGeneratorWrapper/Razor.cs
View File

@@ -9,7 +9,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal class Razor : DotnetSourceGeneratorWrapper
{
protected override string FileType => "cshtml";
protected override string FileType => "Razor";
protected override string SourceGeneratorFolder { get; init; }

37
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/DotnetSourceGeneratorWrapper/Resx.cs Normal file
View File

@@ -0,0 +1,37 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal class Resx : DotnetSourceGeneratorWrapper
{
protected override string FileType => "Resx";
protected override string SourceGeneratorFolder { get; init; }
public Resx(Sdk sdk, IDotNet dotNet, ILogger logger, string? sourceGeneratorFolder) : base(sdk, dotNet, logger)
{
if (sourceGeneratorFolder is null)
{
throw new Exception("No resx source generator folder available.");
}
SourceGeneratorFolder = sourceGeneratorFolder;
}
protected override void GenerateAnalyzerConfig(IEnumerable<string> resources, string analyzerConfigPath)
{
using var sw = new StreamWriter(analyzerConfigPath);
sw.WriteLine("is_global = true");
foreach (var f in resources.Select(f => f.Replace('\\', '/')))
{
sw.WriteLine($"\n[{f}]");
sw.WriteLine($"build_metadata.AdditionalFiles.EmitFormatMethods = true");
}
}
}
}

4
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/ImplicitUsingsGenerator.cs
View File

@@ -16,7 +16,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
this.fileContent = fileContent;
}
public override IEnumerable<string> Generate()
protected override bool IsEnabled() => true;
protected override IEnumerable<string> Run()
{
var usings = new HashSet<string>();
if (!fileContent.UseImplicitUsings)

42
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/RazorGenerator.cs Normal file
View File

@@ -0,0 +1,42 @@
using System;
using System.Collections.Generic;
using Semmle.Util;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal class RazorGenerator : DotnetSourceGeneratorBase<Razor>
{
public RazorGenerator(
FileProvider fileProvider,
FileContent fileContent,
IDotNet dotnet,
ICompilationInfoContainer compilationInfoContainer,
ILogger logger,
TemporaryDirectory tempWorkingDirectory,
IEnumerable<string> references) : base(fileProvider, fileContent, dotnet, compilationInfoContainer, logger, tempWorkingDirectory, references)
{
}
protected override bool IsEnabled()
{
var webViewExtractionOption = Environment.GetEnvironmentVariable(EnvironmentVariableNames.WebViewGeneration);
if (webViewExtractionOption == null ||
bool.TryParse(webViewExtractionOption, out var shouldExtractWebViews) &&
shouldExtractWebViews)
{
compilationInfoContainer.CompilationInfos.Add(("WebView extraction enabled", "1"));
return true;
}
compilationInfoContainer.CompilationInfos.Add(("WebView extraction enabled", "0"));
return false;
}
protected override ICollection<string> AdditionalFiles => fileProvider.RazorViews;
protected override string FileType => "Razor";
protected override Razor GetSourceGenerator(Sdk sdk) => new Razor(sdk, dotnet, logger);
}
}

53
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/ResxGenerator.cs Normal file
View File

@@ -0,0 +1,53 @@
using System;
using System.Collections.Generic;
using Semmle.Util;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal class ResxGenerator : DotnetSourceGeneratorBase<Resx>
{
private readonly string? sourceGeneratorFolder = null;
public ResxGenerator(
FileProvider fileProvider,
FileContent fileContent,
IDotNet dotnet,
ICompilationInfoContainer compilationInfoContainer,
ILogger logger,
TemporaryDirectory tempWorkingDirectory,
IEnumerable<string> references) : base(fileProvider, fileContent, dotnet, compilationInfoContainer, logger, tempWorkingDirectory, references)
{
try
{
// todo: download latest `Microsoft.CodeAnalysis.ResxSourceGenerator` and set `sourceGeneratorFolder`
}
catch (Exception e)
{
logger.LogWarning($"Failed to download source generator: {e.Message}");
sourceGeneratorFolder = null;
}
}
protected override bool IsEnabled()
{
var resourceExtractionOption = Environment.GetEnvironmentVariable(EnvironmentVariableNames.ResourceGeneration);
if (resourceExtractionOption == null ||
bool.TryParse(resourceExtractionOption, out var shouldExtractResources) &&
shouldExtractResources)
{
compilationInfoContainer.CompilationInfos.Add(("Resource extraction enabled", "1"));
return true;
}
compilationInfoContainer.CompilationInfos.Add(("Resource extraction enabled", "0"));
return false;
}
protected override ICollection<string> AdditionalFiles => fileProvider.Resources;
protected override string FileType => "Resx";
protected override Resx GetSourceGenerator(Sdk sdk) => new Resx(sdk, dotnet, logger, sourceGeneratorFolder);
}
}

13
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/SourceGeneratorBase.cs
View File

@@ -16,7 +16,18 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
this.tempWorkingDirectory = tempWorkingDirectory;
}
public abstract IEnumerable<string> Generate();
public IEnumerable<string> Generate()
{
if (!IsEnabled())
{
return [];
}
return Run();
}
protected abstract IEnumerable<string> Run();
protected abstract bool IsEnabled();
/// <summary>
/// Creates a temporary directory with the given subfolder name.

82
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/WebViewGenerator.cs
View File

@@ -1,82 +0,0 @@
using System;
using System.Collections.Generic;
using Semmle.Util;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal class WebViewGenerator : SourceGeneratorBase
{
private readonly FileProvider fileProvider;
private readonly FileContent fileContent;
private readonly IDotNet dotnet;
private readonly ICompilationInfoContainer compilationInfoContainer;
private readonly IEnumerable<string> references;
public WebViewGenerator(
FileProvider fileProvider,
FileContent fileContent,
IDotNet dotnet,
ICompilationInfoContainer compilationInfoContainer,
ILogger logger,
TemporaryDirectory tempWorkingDirectory,
IEnumerable<string> references) : base(logger, tempWorkingDirectory)
{
this.fileProvider = fileProvider;
this.fileContent = fileContent;
this.dotnet = dotnet;
this.compilationInfoContainer = compilationInfoContainer;
this.references = references;
}
public override IEnumerable<string> Generate()
{
var webViewExtractionOption = Environment.GetEnvironmentVariable(EnvironmentVariableNames.WebViewGeneration);
if (webViewExtractionOption == null ||
bool.TryParse(webViewExtractionOption, out var shouldExtractWebViews) &&
shouldExtractWebViews)
{
compilationInfoContainer.CompilationInfos.Add(("WebView extraction enabled", "1"));
return GenerateSourceFilesFromWebViews();
}
compilationInfoContainer.CompilationInfos.Add(("WebView extraction enabled", "0"));
return [];
}
private IEnumerable<string> GenerateSourceFilesFromWebViews()
{
var views = fileProvider.RazorViews;
if (views.Count == 0)
{
logger.LogDebug("No cshtml or razor files found.");
return [];
}
logger.LogInfo($"Found {views.Count} cshtml and razor files.");
if (!fileContent.IsAspNetCoreDetected)
{
logger.LogInfo("Generating source files from cshtml files is only supported for new (SDK-style) project files");
return [];
}
logger.LogInfo("Generating source files from cshtml and razor files...");
try
{
var sdk = new Sdk(dotnet, logger);
var razor = new Razor(sdk, dotnet, logger);
var targetDir = GetTemporaryWorkingDirectory("razor");
var generatedFiles = razor.RunSourceGenerator(views, references, targetDir);
return generatedFiles ?? [];
}
catch (Exception ex)
{
// It's okay, we tried our best to generate source files from cshtml files.
logger.LogInfo($"Failed to generate source files from cshtml files: {ex.Message}");
return [];
}
}
}
}