hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-04-29 02:35:15 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #16225 from tamasvajk/buildless/resx

Browse Source 
C#: Add resource generator
This commit is contained in:
Tamás Vajk
2024-04-24 10:10:45 +02:00
committed by GitHub
parent a1a93c7331 f20812d8ad
commit de58ee5a22
31 changed files with 835 additions and 296 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
View File

@@ -152,7 +152,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var sourceGenerators = new ISourceGenerator[]
{
new ImplicitUsingsGenerator(fileContent, logger, tempWorkingDirectory),
new WebViewGenerator(fileProvider, fileContent, dotnet, this, logger, tempWorkingDirectory, usedReferences.Keys)
new RazorGenerator(fileProvider, fileContent, dotnet, this, logger, tempWorkingDirectory, usedReferences.Keys),
new ResxGenerator(fileProvider, fileContent, dotnet, this, logger, nugetPackageRestorer, tempWorkingDirectory, usedReferences.Keys),
};
foreach (var sourceGenerator in sourceGenerators)
@@ -255,35 +256,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
}
private void SelectNewestFrameworkPath(string frameworkPath, string frameworkType, ISet<AssemblyLookupLocation> dllLocations, ISet<string> frameworkLocations)
{
var versionFolders = GetPackageVersionSubDirectories(frameworkPath);
if (versionFolders.Length > 1)
{
var versions = string.Join(", ", versionFolders.Select(d => d.Name));
logger.LogDebug($"Found multiple {frameworkType} DLLs in NuGet packages at {frameworkPath}. Using the latest version ({versionFolders[0].Name}) from: {versions}.");
}
var selectedFrameworkFolder = versionFolders.FirstOrDefault()?.FullName;
if (selectedFrameworkFolder is null)
{
logger.LogDebug($"Found {frameworkType} DLLs in NuGet packages at {frameworkPath}, but no version folder was found.");
selectedFrameworkFolder = frameworkPath;
}
dllLocations.Add(selectedFrameworkFolder);
frameworkLocations.Add(selectedFrameworkFolder);
logger.LogDebug($"Found {frameworkType} DLLs in NuGet packages at {selectedFrameworkFolder}.");
}
private static DirectoryInfo[] GetPackageVersionSubDirectories(string packagePath)
{
return new DirectoryInfo(packagePath)
.EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
.OrderByDescending(d => d.Name) // TODO: Improve sorting to handle pre-release versions.
.ToArray();
}
private void RemoveFrameworkNugetPackages(ISet<AssemblyLookupLocation> dllLocations, int fromIndex = 0)
{
var packagesInPrioOrder = FrameworkPackageNames.NetFrameworks;
@@ -310,10 +282,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
foreach (var fp in frameworkPaths)
{
dotnetFrameworkVersionVariantCount += GetPackageVersionSubDirectories(fp.Path!).Length;
dotnetFrameworkVersionVariantCount += NugetPackageRestorer.GetOrderedPackageVersionSubDirectories(fp.Path!).Length;
}
SelectNewestFrameworkPath(frameworkPath.Path, ".NET Framework", dllLocations, frameworkLocations);
var folder = nugetPackageRestorer.GetNewestNugetPackageVersionFolder(frameworkPath.Path, ".NET Framework");
dllLocations.Add(folder);
frameworkLocations.Add(folder);
RemoveFrameworkNugetPackages(dllLocations, frameworkPath.Index + 1);
return;
}
@@ -331,7 +305,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (runtimeLocation is null)
{
logger.LogInfo("No .NET Desktop Runtime location found. Attempting to restore the .NET Framework reference assemblies manually.");
runtimeLocation = nugetPackageRestorer.TryRestoreLatestNetFrameworkReferenceAssemblies();
runtimeLocation = nugetPackageRestorer.TryRestore(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies);
}
}
@@ -371,7 +345,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
// First try to find ASP.NET Core assemblies in the NuGet packages
if (GetPackageDirectory(FrameworkPackageNames.AspNetCoreFramework) is string aspNetCorePackage)
{
SelectNewestFrameworkPath(aspNetCorePackage, "ASP.NET Core", dllLocations, frameworkLocations);
var folder = nugetPackageRestorer.GetNewestNugetPackageVersionFolder(aspNetCorePackage, "ASP.NET Core");
dllLocations.Add(folder);
frameworkLocations.Add(folder);
return;
}
@@ -387,7 +363,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
if (GetPackageDirectory(FrameworkPackageNames.WindowsDesktopFramework) is string windowsDesktopApp)
{
SelectNewestFrameworkPath(windowsDesktopApp, "Windows Desktop App", dllLocations, frameworkLocations);
var folder = nugetPackageRestorer.GetNewestNugetPackageVersionFolder(windowsDesktopApp, "Windows Desktop App");
dllLocations.Add(folder);
frameworkLocations.Add(folder);
}
}

5
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs
View File

@@ -2,6 +2,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal class EnvironmentVariableNames
{
/// <summary>
/// Controls whether to generate source files from resources (`.resx`).
/// </summary>
public const string ResourceGeneration = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_EXTRACT_RESOURCES";
/// <summary>
/// Controls whether to generate source files from Asp.Net Core views (`.cshtml`, `.razor`).
/// </summary>

3
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileProvider.cs
View File

@@ -22,6 +22,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private readonly Lazy<string[]> nugetConfigs;
private readonly Lazy<string[]> globalJsons;
private readonly Lazy<string[]> razorViews;
private readonly Lazy<string[]> resources;
private readonly Lazy<string?> rootNugetConfig;
public FileProvider(DirectoryInfo sourceDir, ILogger logger)
@@ -44,6 +45,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
nugetConfigs = new Lazy<string[]>(() => allNonBinary.Value.SelectFileNamesByName("nuget.config").ToArray());
globalJsons = new Lazy<string[]>(() => allNonBinary.Value.SelectFileNamesByName("global.json").ToArray());
razorViews = new Lazy<string[]>(() => SelectTextFileNamesByExtension("razor view", ".cshtml", ".razor"));
resources = new Lazy<string[]>(() => SelectTextFileNamesByExtension("resource", ".resx"));
rootNugetConfig = new Lazy<string?>(() => all.SelectRootFiles(SourceDir).SelectFileNamesByName("nuget.config").FirstOrDefault());
}
@@ -116,5 +118,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
public string? RootNugetConfig => rootNugetConfig.Value;
public IEnumerable<string> GlobalJsons => globalJsons.Value;
public ICollection<string> RazorViews => razorViews.Value;
public ICollection<string> Resources => resources.Value;
}
}

3
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs
View File

@@ -30,6 +30,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private readonly Lazy<bool> hasNugetPackageSourceError = new(() => Output.Any(s => s.Contains("NU1301")));
public bool HasNugetPackageSourceError => hasNugetPackageSourceError.Value;
private readonly Lazy<bool> hasNugetNoStablePackageVersionError = new(() => Output.Any(s => s.Contains("NU1103")));
public bool HasNugetNoStablePackageVersionError => hasNugetNoStablePackageVersionError.Value;
private static IEnumerable<string> GetFirstGroupOnMatch(Regex regex, IEnumerable<string> lines) =>
lines
.Select(line => regex.Match(line))

126
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs
View File

@@ -48,16 +48,48 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
missingPackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath(fileProvider.SourceDir.FullName, "missingpackages"), "missing package", logger);
}
public string? TryRestoreLatestNetFrameworkReferenceAssemblies()
public string? TryRestore(string package)
{
if (TryRestorePackageManually(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies))
if (TryRestorePackageManually(package))
{
return DependencyManager.GetPackageDirectory(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies, missingPackageDirectory.DirInfo);
var packageDir = DependencyManager.GetPackageDirectory(package, missingPackageDirectory.DirInfo);
if (packageDir is not null)
{
return GetNewestNugetPackageVersionFolder(packageDir, package);
}
}
return null;
}
public string GetNewestNugetPackageVersionFolder(string packagePath, string packageFriendlyName)
{
var versionFolders = GetOrderedPackageVersionSubDirectories(packagePath);
if (versionFolders.Length > 1)
{
var versions = string.Join(", ", versionFolders.Select(d => d.Name));
logger.LogDebug($"Found multiple {packageFriendlyName} DLLs in NuGet packages at {packagePath}. Using the latest version ({versionFolders[0].Name}) from: {versions}.");
}
var selectedFrameworkFolder = versionFolders.FirstOrDefault()?.FullName;
if (selectedFrameworkFolder is null)
{
logger.LogDebug($"Found {packageFriendlyName} DLLs in NuGet packages at {packagePath}, but no version folder was found.");
selectedFrameworkFolder = packagePath;
}
logger.LogDebug($"Found {packageFriendlyName} DLLs in NuGet packages at {selectedFrameworkFolder}.");
return selectedFrameworkFolder;
}
public static DirectoryInfo[] GetOrderedPackageVersionSubDirectories(string packagePath)
{
return new DirectoryInfo(packagePath)
.EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
.OrderByDescending(d => d.Name) // TODO: Improve sorting to handle pre-release versions.
.ToArray();
}
public HashSet<AssemblyLookupLocation> Restore()
{
var assemblyLookupLocations = new HashSet<AssemblyLookupLocation>();
@@ -408,7 +440,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
.Select(d => Path.GetFileName(d).ToLowerInvariant());
}
private bool TryRestorePackageManually(string package, string? nugetConfig = null, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj, bool tryWithoutNugetConfig = true)
private bool TryRestorePackageManually(string package, string? nugetConfig = null, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj,
bool tryWithoutNugetConfig = true, bool tryPrereleaseVersion = true)
{
logger.LogInfo($"Restoring package {package}...");
using var tempDir = new TemporaryDirectory(
@@ -430,59 +463,87 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
return false;
}
var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig));
if (!res.Success)
var res = TryRestorePackageManually(package, nugetConfig, tempDir, tryPrereleaseVersion);
if (res.Success)
{
if (tryWithoutNugetConfig && res.HasNugetPackageSourceError && nugetConfig is not null)
{
// Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true));
}
return true;
}
// TODO: the restore might fail, we could retry with
// - a prerelease (*-* instead of *) version of the package,
// - a different target framework moniker.
if (!res.Success)
if (tryWithoutNugetConfig && res.HasNugetPackageSourceError && nugetConfig is not null)
{
logger.LogDebug($"Trying to restore '{package}' without nuget.config.");
// Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
res = TryRestorePackageManually(package, nugetConfig: null, tempDir, tryPrereleaseVersion);
if (res.Success)
{
logger.LogInfo($"Failed to restore nuget package {package}");
return false;
return true;
}
}
return true;
logger.LogInfo($"Failed to restore nuget package {package}");
return false;
}
private RestoreResult TryRestorePackageManually(string package, string? nugetConfig, TemporaryDirectory tempDir, bool tryPrereleaseVersion)
{
var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig, ForceReevaluation: true));
if (!res.Success && tryPrereleaseVersion && res.HasNugetNoStablePackageVersionError)
{
logger.LogDebug($"Failed to restore nuget package {package} because no stable version was found.");
TryChangePackageVersion(tempDir.DirInfo, "*-*");
res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig, ForceReevaluation: true));
if (!res.Success)
{
TryChangePackageVersion(tempDir.DirInfo, "*");
}
}
return res;
}
private void TryChangeTargetFrameworkMoniker(DirectoryInfo tempDir)
{
TryChangeProjectFile(tempDir, TargetFramework(), $"<TargetFramework>{FrameworkPackageNames.LatestNetFrameworkMoniker}</TargetFramework>", "target framework moniker");
}
private void TryChangePackageVersion(DirectoryInfo tempDir, string newVersion)
{
TryChangeProjectFile(tempDir, PackageReferenceVersion(), $"Version=\"{newVersion}\"", "package reference version");
}
private bool TryChangeProjectFile(DirectoryInfo projectDir, Regex pattern, string replacement, string patternName)
{
try
{
logger.LogInfo($"Changing the target framework moniker in {tempDir.FullName}...");
logger.LogDebug($"Changing the {patternName} in {projectDir.FullName}...");
var csprojs = tempDir.GetFiles("*.csproj", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive });
var csprojs = projectDir.GetFiles("*.csproj", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive });
if (csprojs.Length != 1)
{
logger.LogError($"Could not find the .csproj file in {tempDir.FullName}, count = {csprojs.Length}");
return;
logger.LogError($"Could not find the .csproj file in {projectDir.FullName}, count = {csprojs.Length}");
return false;
}
var csproj = csprojs[0];
var content = File.ReadAllText(csproj.FullName);
var matches = TargetFramework().Matches(content);
var matches = pattern.Matches(content);
if (matches.Count == 0)
{
logger.LogError($"Could not find target framework in {csproj.FullName}");
}
else
{
content = TargetFramework().Replace(content, $"<TargetFramework>{FrameworkPackageNames.LatestNetFrameworkMoniker}</TargetFramework>", 1);
File.WriteAllText(csproj.FullName, content);
logger.LogError($"Could not find the {patternName} in {csproj.FullName}");
return false;
}
content = pattern.Replace(content, replacement, 1);
File.WriteAllText(csproj.FullName, content);
return true;
}
catch (Exception exc)
{
logger.LogError($"Failed to update target framework in {tempDir.FullName}: {exc}");
logger.LogError($"Failed to change the {patternName} in {projectDir.FullName}: {exc}");
}
return false;
}
private static async Task ExecuteGetRequest(string address, HttpClient httpClient, CancellationToken cancellationToken)
@@ -664,6 +725,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
[GeneratedRegex(@"<TargetFramework>.*</TargetFramework>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
private static partial Regex TargetFramework();
[GeneratedRegex(@"Version=""(\*|\*-\*)""", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
private static partial Regex PackageReferenceVersion();
[GeneratedRegex(@"^(.+)\.(\d+\.\d+\.\d+(-(.+))?)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
private static partial Regex LegacyNugetPackage();

46
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Sdk.cs
View File

@@ -1,15 +1,31 @@
using System.Collections.Generic;
using Semmle.Util;
using System.Text.RegularExpressions;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using Semmle.Util;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal partial class Sdk
{
private readonly IDotNet dotNet;
private readonly ILogger logger;
private readonly Lazy<string?> cscPath;
public string? CscPath => cscPath.Value;
public Sdk(IDotNet dotNet) => this.dotNet = dotNet;
private readonly Lazy<DotNetVersion?> newestSdkVersion;
public DotNetVersion? Version => newestSdkVersion.Value;
public Sdk(IDotNet dotNet, ILogger logger)
{
this.dotNet = dotNet;
this.logger = logger;
newestSdkVersion = new Lazy<DotNetVersion?>(GetNewestSdkVersion);
cscPath = new Lazy<string?>(GetCscPath);
}
[GeneratedRegex(@"^(\d+\.\d+\.\d+)(-([a-z]+)\.(\d+\.\d+\.\d+))?\s\[(.+)\]$")]
private static partial Regex SdkRegex();
@@ -30,11 +46,31 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
return sdks;
}
public DotNetVersion? GetNewestSdk()
private DotNetVersion? GetNewestSdkVersion()
{
var listed = dotNet.GetListedSdks();
var sdks = ParseSdks(listed);
return sdks.Max();
}
private string? GetCscPath()
{
var version = Version;
if (version is null)
{
logger.LogWarning("No dotnet SDK found.");
return null;
}
var path = Path.Combine(version.FullPath, "Roslyn", "bincore", "csc.dll");
logger.LogDebug($"Source generator CSC: '{path}'");
if (!File.Exists(path))
{
logger.LogWarning($"csc.dll not found at '{path}'.");
return null;
}
return path;
}
}
}

127
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/DotnetSourceGeneratorBase.cs Normal file
View File

@@ -0,0 +1,127 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Semmle.Util;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal abstract class DotnetSourceGeneratorBase<T> : SourceGeneratorBase where T : DotnetSourceGeneratorWrapper
{
protected readonly FileProvider fileProvider;
protected readonly FileContent fileContent;
protected readonly IDotNet dotnet;
protected readonly ICompilationInfoContainer compilationInfoContainer;
protected readonly IEnumerable<string> references;
public DotnetSourceGeneratorBase(
FileProvider fileProvider,
FileContent fileContent,
IDotNet dotnet,
ICompilationInfoContainer compilationInfoContainer,
ILogger logger,
TemporaryDirectory tempWorkingDirectory,
IEnumerable<string> references) : base(logger, tempWorkingDirectory)
{
this.fileProvider = fileProvider;
this.fileContent = fileContent;
this.dotnet = dotnet;
this.compilationInfoContainer = compilationInfoContainer;
this.references = references;
}
protected override IEnumerable<string> Run()
{
if (AdditionalFiles.Count == 0)
{
logger.LogDebug($"No {FileType} files found. Skipping source generation.");
return [];
}
if (!fileContent.IsNewProjectStructureUsed)
{
logger.LogInfo($"Generating source files from {FileType} files is only supported for new (SDK-style) project files");
return [];
}
if (fileProvider.Projects.Count == 0)
{
logger.LogInfo($"No projects found. Skipping source generation from {FileType} files.");
return [];
}
logger.LogInfo($"Generating source files from {AdditionalFiles.Count} {FileType} files...");
// group additional files by closes project file:
var projects = fileProvider.Projects
.Select(p => (File: p, Directory: SafeGetDirectoryName(p)))
.Where(p => p.Directory.Length > 0);
var groupedFiles = new Dictionary<string, List<string>>();
foreach (var additionalFile in AdditionalFiles)
{
var project = projects
.Where(p => additionalFile.StartsWith(p.Directory))
.OrderByDescending(p => p.Directory.Length)
.FirstOrDefault();
if (project == default)
{
logger.LogDebug($"Failed to find project file for {additionalFile}");
continue;
}
groupedFiles.AddAnother(project.File, additionalFile);
}
try
{
var sdk = new Sdk(dotnet, logger);
var sourceGenerator = GetSourceGenerator(sdk);
var targetDir = GetTemporaryWorkingDirectory(FileType.ToLowerInvariant());
return groupedFiles
.SelectMany(group => sourceGenerator.RunSourceGenerator(group.Value, group.Key, references, targetDir));
}
catch (Exception ex)
{
// It's okay, we tried our best to generate source files.
logger.LogInfo($"Failed to generate source files from {FileType} files: {ex.Message}");
return [];
}
}
private string SafeGetDirectoryName(string fileName)
{
try
{
var dir = Path.GetDirectoryName(fileName);
if (dir is null)
{
return "";
}
if (!dir.EndsWith(Path.DirectorySeparatorChar))
{
dir += Path.DirectorySeparatorChar;
}
return dir;
}
catch (Exception ex)
{
logger.LogDebug($"Failed to get directory name for {fileName}: {ex.Message}");
return "";
}
}
protected abstract ICollection<string> AdditionalFiles { get; }
protected abstract string FileType { get; }
protected abstract T GetSourceGenerator(Sdk sdk);
}
}

95
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/DotnetSourceGeneratorWrapper/DotnetSourceGeneratorWrapper.cs Normal file
View File

@@ -0,0 +1,95 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using Semmle.Util;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal abstract class DotnetSourceGeneratorWrapper
{
protected readonly ILogger logger;
protected readonly IDotNet dotnet;
private readonly string cscPath;
protected abstract string SourceGeneratorFolder { get; init; }
protected abstract string FileType { get; }
public DotnetSourceGeneratorWrapper(
Sdk sdk,
IDotNet dotnet,
ILogger logger)
{
this.logger = logger;
this.dotnet = dotnet;
if (sdk.CscPath is null)
{
throw new Exception($"Not running {FileType} source generator because CSC path is not available.");
}
this.cscPath = sdk.CscPath;
}
protected abstract void GenerateAnalyzerConfig(IEnumerable<string> additionalFiles, string csprojFile, string analyzerConfigPath);
public IEnumerable<string> RunSourceGenerator(IEnumerable<string> additionalFiles, string csprojFile, IEnumerable<string> references, string targetDir)
{
try
{
var name = Guid.NewGuid().ToString("N").ToUpper();
using var tempDir = new TemporaryDirectory(Path.Join(FileUtils.GetTemporaryWorkingDirectory(out _), "source-generator"), "source generator temporary", logger);
var analyzerConfigPath = Path.Combine(tempDir.DirInfo.FullName, $"{name}.txt");
var dllPath = Path.Combine(tempDir.DirInfo.FullName, $"{name}.dll");
var cscArgsPath = Path.Combine(tempDir.DirInfo.FullName, $"{name}.rsp");
var outputFolder = Path.Combine(targetDir, name);
Directory.CreateDirectory(outputFolder);
logger.LogInfo("Producing analyzer config content.");
GenerateAnalyzerConfig(additionalFiles, csprojFile, analyzerConfigPath);
logger.LogDebug($"Analyzer config content: {File.ReadAllText(analyzerConfigPath)}");
var args = new StringBuilder();
args.Append($"/target:exe /generatedfilesout:\"{outputFolder}\" /out:\"{dllPath}\" /analyzerconfig:\"{analyzerConfigPath}\" ");
foreach (var f in Directory.GetFiles(SourceGeneratorFolder, "*.dll", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive }))
{
args.Append($"/analyzer:\"{f}\" ");
}
foreach (var f in additionalFiles)
{
args.Append($"/additionalfile:\"{f}\" ");
}
foreach (var f in references)
{
args.Append($"/reference:\"{f}\" ");
}
var argsString = args.ToString();
logger.LogInfo($"Running CSC to generate source files from {FileType} files.");
logger.LogDebug($"Running CSC to generate source files from {FileType} files with arguments: {argsString}.");
using (var sw = new StreamWriter(cscArgsPath))
{
sw.Write(argsString);
}
dotnet.Exec($"\"{cscPath}\" /noconfig @\"{cscArgsPath}\"");
var files = Directory.GetFiles(outputFolder, "*.*", new EnumerationOptions { RecurseSubdirectories = true });
logger.LogInfo($"Generated {files.Length} source files from {FileType} files.");
return files;
}
catch (Exception ex)
{
logger.LogInfo($"Failed to generate source files from {FileType} files: {ex.Message}");
return [];
}
}
}
}

45
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/DotnetSourceGeneratorWrapper/Razor.cs Normal file
View File

@@ -0,0 +1,45 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal sealed class Razor : DotnetSourceGeneratorWrapper
{
protected override string FileType => "Razor";
protected override string SourceGeneratorFolder { get; init; }
public Razor(Sdk sdk, IDotNet dotNet, ILogger logger) : base(sdk, dotNet, logger)
{
var sdkPath = sdk.Version?.FullPath;
if (sdkPath is null)
{
throw new Exception("No SDK path available.");
}
SourceGeneratorFolder = Path.Combine(sdkPath, "Sdks", "Microsoft.NET.Sdk.Razor", "source-generators");
this.logger.LogInfo($"Razor source generator folder: {SourceGeneratorFolder}");
if (!Directory.Exists(SourceGeneratorFolder))
{
throw new Exception($"Razor source generator folder {SourceGeneratorFolder} does not exist.");
}
}
protected override void GenerateAnalyzerConfig(IEnumerable<string> cshtmls, string csprojFile, string analyzerConfigPath)
{
using var sw = new StreamWriter(analyzerConfigPath);
sw.WriteLine("is_global = true");
foreach (var f in cshtmls.Select(f => f.Replace('\\', '/')))
{
sw.WriteLine($"\n[{f}]");
var base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(f)); // TODO: this should be the relative path of the file.
sw.WriteLine($"build_metadata.AdditionalFiles.TargetPath = {base64}");
}
}
}
}

61
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/DotnetSourceGeneratorWrapper/Resx.cs Normal file
View File

@@ -0,0 +1,61 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal sealed partial class Resx : DotnetSourceGeneratorWrapper
{
protected override string FileType => "Resx";
protected override string SourceGeneratorFolder { get; init; }
public Resx(Sdk sdk, IDotNet dotNet, ILogger logger, string? sourceGeneratorFolder) : base(sdk, dotNet, logger)
{
if (sourceGeneratorFolder is null)
{
throw new Exception("No resx source generator folder available.");
}
SourceGeneratorFolder = sourceGeneratorFolder;
}
protected override void GenerateAnalyzerConfig(IEnumerable<string> resources, string csprojFile, string analyzerConfigPath)
{
using var sw = new StreamWriter(analyzerConfigPath);
sw.WriteLine("is_global = true");
var rootNamespace = Path.GetFileNameWithoutExtension(csprojFile);
var matches = File.ReadAllLines(csprojFile)
.Select(line => RootNamespace().Match(line))
.Where(match => match.Success)
.Select(match => match.Groups[1].Value)
.ToArray();
if (matches.Length == 1)
{
logger.LogDebug($"RootNamespace found in {csprojFile}: {matches[0]}");
rootNamespace = matches[0];
}
else if (matches.Length > 1)
{
logger.LogDebug($"Multiple RootNamespace elements found in {csprojFile}. Using the first one.");
rootNamespace = matches[0];
}
sw.WriteLine($"build_property.RootNamespace = {rootNamespace}");
foreach (var f in resources.Select(f => f.Replace('\\', '/')))
{
sw.WriteLine($"\n[{f}]");
sw.WriteLine($"build_metadata.AdditionalFiles.EmitFormatMethods = true");
}
}
[GeneratedRegex(@"<RootNamespace>(.*)</RootNamespace>", RegexOptions.Compiled | RegexOptions.Singleline)]
private static partial Regex RootNamespace();
}
}

4
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/ImplicitUsingsGenerator.cs
View File

@@ -16,7 +16,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
this.fileContent = fileContent;
}
public override IEnumerable<string> Generate()
protected override bool IsEnabled() => true;
protected override IEnumerable<string> Run()
{
var usings = new HashSet<string>();
if (!fileContent.UseImplicitUsings)

131
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/Razor.cs
View File

@@ -1,131 +0,0 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Linq;
using Semmle.Util;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal class Razor
{
private readonly DotNetVersion sdk;
private readonly ILogger logger;
private readonly IDotNet dotNet;
private readonly string sourceGeneratorFolder;
private readonly string cscPath;
public Razor(DotNetVersion sdk, IDotNet dotNet, ILogger logger)
{
this.sdk = sdk;
this.logger = logger;
this.dotNet = dotNet;
sourceGeneratorFolder = Path.Combine(this.sdk.FullPath, "Sdks", "Microsoft.NET.Sdk.Razor", "source-generators");
this.logger.LogInfo($"Razor source generator folder: {sourceGeneratorFolder}");
if (!Directory.Exists(sourceGeneratorFolder))
{
this.logger.LogInfo($"Razor source generator folder {sourceGeneratorFolder} does not exist.");
throw new Exception($"Razor source generator folder {sourceGeneratorFolder} does not exist.");
}
cscPath = Path.Combine(this.sdk.FullPath, "Roslyn", "bincore", "csc.dll");
this.logger.LogInfo($"Razor source generator CSC: {cscPath}");
if (!File.Exists(cscPath))
{
this.logger.LogInfo($"Csc.exe not found at {cscPath}.");
throw new Exception($"csc.dll {cscPath} does not exist.");
}
}
private static void GenerateAnalyzerConfig(IEnumerable<string> cshtmls, string analyzerConfigPath)
{
using var sw = new StreamWriter(analyzerConfigPath);
sw.WriteLine("is_global = true");
foreach (var f in cshtmls.Select(f => f.Replace('\\', '/')))
{
sw.WriteLine($"\n[{f}]");
var base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(f)); // TODO: this should be the relative path of the file.
sw.WriteLine($"build_metadata.AdditionalFiles.TargetPath = {base64}");
}
}
public IEnumerable<string> GenerateFiles(IEnumerable<string> cshtmls, IEnumerable<string> references, string workingDirectory)
{
var name = Guid.NewGuid().ToString("N").ToUpper();
var tempPath = FileUtils.GetTemporaryWorkingDirectory(out var shouldCleanUp);
var analyzerConfig = Path.Combine(tempPath, $"{name}.txt");
var dllPath = Path.Combine(tempPath, $"{name}.dll");
var cscArgsPath = Path.Combine(tempPath, $"{name}.rsp");
var outputFolder = Path.Combine(workingDirectory, name);
Directory.CreateDirectory(outputFolder);
try
{
logger.LogInfo("Produce analyzer config content.");
GenerateAnalyzerConfig(cshtmls, analyzerConfig);
logger.LogDebug($"Analyzer config content: {File.ReadAllText(analyzerConfig)}");
var args = new StringBuilder();
args.Append($"/target:exe /generatedfilesout:\"{outputFolder}\" /out:\"{dllPath}\" /analyzerconfig:\"{analyzerConfig}\" ");
foreach (var f in Directory.GetFiles(sourceGeneratorFolder, "*.dll", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive }))
{
args.Append($"/analyzer:\"{f}\" ");
}
foreach (var f in cshtmls)
{
args.Append($"/additionalfile:\"{f}\" ");
}
foreach (var f in references)
{
args.Append($"/reference:\"{f}\" ");
}
var argsString = args.ToString();
logger.LogInfo($"Running CSC to generate Razor source files.");
logger.LogDebug($"Running CSC to generate Razor source files with arguments: {argsString}.");
using (var sw = new StreamWriter(cscArgsPath))
{
sw.Write(argsString);
}
dotNet.Exec($"\"{cscPath}\" /noconfig @\"{cscArgsPath}\"");
var files = Directory.GetFiles(outputFolder, "*.*", new EnumerationOptions { RecurseSubdirectories = true });
logger.LogInfo($"Generated {files.Length} source files from cshtml files.");
return files;
}
finally
{
if (shouldCleanUp)
{
DeleteFile(analyzerConfig);
DeleteFile(dllPath);
DeleteFile(cscArgsPath);
}
}
}
private void DeleteFile(string path)
{
try
{
File.Delete(path);
}
catch (Exception exc)
{
logger.LogWarning($"Failed to delete file {path}: {exc}");
}
}
}
}

42
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/RazorGenerator.cs Normal file
View File

@@ -0,0 +1,42 @@
using System;
using System.Collections.Generic;
using Semmle.Util;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal class RazorGenerator : DotnetSourceGeneratorBase<Razor>
{
public RazorGenerator(
FileProvider fileProvider,
FileContent fileContent,
IDotNet dotnet,
ICompilationInfoContainer compilationInfoContainer,
ILogger logger,
TemporaryDirectory tempWorkingDirectory,
IEnumerable<string> references) : base(fileProvider, fileContent, dotnet, compilationInfoContainer, logger, tempWorkingDirectory, references)
{
}
protected override bool IsEnabled()
{
var webViewExtractionOption = Environment.GetEnvironmentVariable(EnvironmentVariableNames.WebViewGeneration);
if (webViewExtractionOption == null ||
bool.TryParse(webViewExtractionOption, out var shouldExtractWebViews) &&
shouldExtractWebViews)
{
compilationInfoContainer.CompilationInfos.Add(("WebView extraction enabled", "1"));
return true;
}
compilationInfoContainer.CompilationInfos.Add(("WebView extraction enabled", "0"));
return false;
}
protected override ICollection<string> AdditionalFiles => fileProvider.RazorViews;
protected override string FileType => "Razor";
protected override Razor GetSourceGenerator(Sdk sdk) => new Razor(sdk, dotnet, logger);
}
}

58
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/ResxGenerator.cs Normal file
View File

@@ -0,0 +1,58 @@
using System;
using System.Collections.Generic;
using Semmle.Util;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal class ResxGenerator : DotnetSourceGeneratorBase<Resx>
{
private readonly string? sourceGeneratorFolder = null;
public ResxGenerator(
FileProvider fileProvider,
FileContent fileContent,
IDotNet dotnet,
ICompilationInfoContainer compilationInfoContainer,
ILogger logger,
NugetPackageRestorer nugetPackageRestorer,
TemporaryDirectory tempWorkingDirectory,
IEnumerable<string> references) : base(fileProvider, fileContent, dotnet, compilationInfoContainer, logger, tempWorkingDirectory, references)
{
try
{
// The package is downloaded to `missingpackages`, which is okay, we're already after the DLL collection phase.
var nugetFolder = nugetPackageRestorer.TryRestore("Microsoft.CodeAnalysis.ResxSourceGenerator");
if (nugetFolder is not null)
{
sourceGeneratorFolder = System.IO.Path.Combine(nugetFolder, "analyzers", "dotnet", "cs");
}
}
catch (Exception e)
{
logger.LogWarning($"Failed to download source generator: {e.Message}");
sourceGeneratorFolder = null;
}
}
protected override bool IsEnabled()
{
var resourceExtractionOption = Environment.GetEnvironmentVariable(EnvironmentVariableNames.ResourceGeneration);
if (bool.TryParse(resourceExtractionOption, out var shouldExtractResources) &&
shouldExtractResources)
{
compilationInfoContainer.CompilationInfos.Add(("Resource extraction enabled", "1"));
return true;
}
compilationInfoContainer.CompilationInfos.Add(("Resource extraction enabled", "0"));
return false;
}
protected override ICollection<string> AdditionalFiles => fileProvider.Resources;
protected override string FileType => "Resx";
protected override Resx GetSourceGenerator(Sdk sdk) => new Resx(sdk, dotnet, logger, sourceGeneratorFolder);
}
}

13
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/SourceGeneratorBase.cs
View File

@@ -16,7 +16,18 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
this.tempWorkingDirectory = tempWorkingDirectory;
}
public abstract IEnumerable<string> Generate();
public IEnumerable<string> Generate()
{
if (!IsEnabled())
{
return [];
}
return Run();
}
protected abstract IEnumerable<string> Run();
protected abstract bool IsEnabled();
/// <summary>
/// Creates a temporary directory with the given subfolder name.

86
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/WebViewGenerator.cs
View File

@@ -1,86 +0,0 @@
using System;
using System.Collections.Generic;
using Semmle.Util;
using Semmle.Util.Logging;
namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal class WebViewGenerator : SourceGeneratorBase
{
private readonly FileProvider fileProvider;
private readonly FileContent fileContent;
private readonly IDotNet dotnet;
private readonly ICompilationInfoContainer compilationInfoContainer;
private readonly IEnumerable<string> references;
public WebViewGenerator(
FileProvider fileProvider,
FileContent fileContent,
IDotNet dotnet,
ICompilationInfoContainer compilationInfoContainer,
ILogger logger,
TemporaryDirectory tempWorkingDirectory,
IEnumerable<string> references) : base(logger, tempWorkingDirectory)
{
this.fileProvider = fileProvider;
this.fileContent = fileContent;
this.dotnet = dotnet;
this.compilationInfoContainer = compilationInfoContainer;
this.references = references;
}
public override IEnumerable<string> Generate()
{
var webViewExtractionOption = Environment.GetEnvironmentVariable(EnvironmentVariableNames.WebViewGeneration);
if (webViewExtractionOption == null ||
bool.TryParse(webViewExtractionOption, out var shouldExtractWebViews) &&
shouldExtractWebViews)
{
compilationInfoContainer.CompilationInfos.Add(("WebView extraction enabled", "1"));
return GenerateSourceFilesFromWebViews();
}
compilationInfoContainer.CompilationInfos.Add(("WebView extraction enabled", "0"));
return [];
}
private IEnumerable<string> GenerateSourceFilesFromWebViews()
{
var views = fileProvider.RazorViews;
if (views.Count == 0)
{
logger.LogDebug("No cshtml or razor files found.");
return [];
}
logger.LogInfo($"Found {views.Count} cshtml and razor files.");
if (!fileContent.IsAspNetCoreDetected)
{
logger.LogInfo("Generating source files from cshtml files is only supported for new (SDK-style) project files");
return [];
}
logger.LogInfo("Generating source files from cshtml and razor files...");
var sdk = new Sdk(dotnet).GetNewestSdk();
if (sdk != null)
{
try
{
var razor = new Razor(sdk, dotnet, logger);
var targetDir = GetTemporaryWorkingDirectory("razor");
var generatedFiles = razor.GenerateFiles(views, references, targetDir);
return generatedFiles ?? [];
}
catch (Exception ex)
{
// It's okay, we tried our best to generate source files from cshtml files.
logger.LogInfo($"Failed to generate source files from cshtml files: {ex.Message}");
}
}
return [];
}
}
}

8
csharp/extractor/Semmle.Extraction.Tests/Runtime.cs
View File

@@ -164,10 +164,10 @@ namespace Semmle.Extraction.Tests
"6.0.301 [/usr/local/share/dotnet/sdk7]",
};
var dotnet = new DotNetStub(null!, listedSdks);
var sdk = new Sdk(dotnet);
var sdk = new Sdk(dotnet, new LoggerStub());
// Execute
var version = sdk.GetNewestSdk();
var version = sdk.Version;
// Verify
Assert.NotNull(version);
@@ -186,10 +186,10 @@ namespace Semmle.Extraction.Tests
"7.0.400 [/usr/local/share/dotnet/sdk4]",
};
var dotnet = new DotNetStub(null!, listedSdks);
var sdk = new Sdk(dotnet);
var sdk = new Sdk(dotnet, new LoggerStub());
// Execute
var version = sdk.GetNewestSdk();
var version = sdk.Version;
// Verify
Assert.NotNull(version);

4
csharp/extractor/Semmle.Util/TemporaryDirectory.cs
View File

@@ -15,9 +15,9 @@ namespace Semmle.Util
public DirectoryInfo DirInfo { get; }
public TemporaryDirectory(string name, string userReportedDirectoryPurpose, ILogger logger)
public TemporaryDirectory(string path, string userReportedDirectoryPurpose, ILogger logger)
{
DirInfo = new DirectoryInfo(name);
DirInfo = new DirectoryInfo(path);
DirInfo.Create();
this.userReportedDirectoryPurpose = userReportedDirectoryPurpose;
this.logger = logger;