hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-04-30 19:26:02 +02:00
Code Issues Packages Projects Releases Wiki Activity

Restore and use Microsoft.CodeAnalysis.ResxSourceGenerator

Browse Source
This commit is contained in:
Tamas Vajk
2024-04-17 10:41:41 +02:00
parent 79fe5f851b
commit 1ff4c0daf3
11 changed files with 298 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
View File

@@ -153,7 +153,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
new ImplicitUsingsGenerator(fileContent, logger, tempWorkingDirectory),
new RazorGenerator(fileProvider, fileContent, dotnet, this, logger, tempWorkingDirectory, usedReferences.Keys),
new ResxGenerator(fileProvider, fileContent, dotnet, this, logger, tempWorkingDirectory, usedReferences.Keys),
new ResxGenerator(fileProvider, fileContent, dotnet, this, logger, nugetPackageRestorer, tempWorkingDirectory, usedReferences.Keys),
};
foreach (var sourceGenerator in sourceGenerators)
@@ -256,35 +256,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
}
private void SelectNewestFrameworkPath(string frameworkPath, string frameworkType, ISet<AssemblyLookupLocation> dllLocations, ISet<string> frameworkLocations)
{
var versionFolders = GetPackageVersionSubDirectories(frameworkPath);
if (versionFolders.Length > 1)
{
var versions = string.Join(", ", versionFolders.Select(d => d.Name));
logger.LogDebug($"Found multiple {frameworkType} DLLs in NuGet packages at {frameworkPath}. Using the latest version ({versionFolders[0].Name}) from: {versions}.");
}
var selectedFrameworkFolder = versionFolders.FirstOrDefault()?.FullName;
if (selectedFrameworkFolder is null)
{
logger.LogDebug($"Found {frameworkType} DLLs in NuGet packages at {frameworkPath}, but no version folder was found.");
selectedFrameworkFolder = frameworkPath;
}
dllLocations.Add(selectedFrameworkFolder);
frameworkLocations.Add(selectedFrameworkFolder);
logger.LogDebug($"Found {frameworkType} DLLs in NuGet packages at {selectedFrameworkFolder}.");
}
private static DirectoryInfo[] GetPackageVersionSubDirectories(string packagePath)
{
return new DirectoryInfo(packagePath)
.EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
.OrderByDescending(d => d.Name) // TODO: Improve sorting to handle pre-release versions.
.ToArray();
}
private void RemoveFrameworkNugetPackages(ISet<AssemblyLookupLocation> dllLocations, int fromIndex = 0)
{
var packagesInPrioOrder = FrameworkPackageNames.NetFrameworks;
@@ -311,10 +282,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
foreach (var fp in frameworkPaths)
{
dotnetFrameworkVersionVariantCount += GetPackageVersionSubDirectories(fp.Path!).Length;
dotnetFrameworkVersionVariantCount += NugetPackageRestorer.GetOrderedPackageVersionSubDirectories(fp.Path!).Length;
}
SelectNewestFrameworkPath(frameworkPath.Path, ".NET Framework", dllLocations, frameworkLocations);
var folder = nugetPackageRestorer.GetNewestNugetPackageVersionFolder(frameworkPath.Path, ".NET Framework");
dllLocations.Add(folder);
frameworkLocations.Add(folder);
RemoveFrameworkNugetPackages(dllLocations, frameworkPath.Index + 1);
return;
}
@@ -332,7 +305,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (runtimeLocation is null)
{
logger.LogInfo("No .NET Desktop Runtime location found. Attempting to restore the .NET Framework reference assemblies manually.");
runtimeLocation = nugetPackageRestorer.TryRestoreLatestNetFrameworkReferenceAssemblies();
runtimeLocation = nugetPackageRestorer.TryRestore(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies);
}
}
@@ -372,7 +345,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
// First try to find ASP.NET Core assemblies in the NuGet packages
if (GetPackageDirectory(FrameworkPackageNames.AspNetCoreFramework) is string aspNetCorePackage)
{
SelectNewestFrameworkPath(aspNetCorePackage, "ASP.NET Core", dllLocations, frameworkLocations);
var folder = nugetPackageRestorer.GetNewestNugetPackageVersionFolder(aspNetCorePackage, "ASP.NET Core");
dllLocations.Add(folder);
frameworkLocations.Add(folder);
return;
}
@@ -388,7 +363,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
if (GetPackageDirectory(FrameworkPackageNames.WindowsDesktopFramework) is string windowsDesktopApp)
{
SelectNewestFrameworkPath(windowsDesktopApp, "Windows Desktop App", dllLocations, frameworkLocations);
var folder = nugetPackageRestorer.GetNewestNugetPackageVersionFolder(windowsDesktopApp, "Windows Desktop App");
dllLocations.Add(folder);
frameworkLocations.Add(folder);
}
}

3
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs
View File

@@ -30,6 +30,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private readonly Lazy<bool> hasNugetPackageSourceError = new(() => Output.Any(s => s.Contains("NU1301")));
public bool HasNugetPackageSourceError => hasNugetPackageSourceError.Value;
private readonly Lazy<bool> hasNugetNoStablePackageVersionError = new(() => Output.Any(s => s.Contains("NU1103")));
public bool HasNugetNoStablePackageVersionError => hasNugetNoStablePackageVersionError.Value;
private static IEnumerable<string> GetFirstGroupOnMatch(Regex regex, IEnumerable<string> lines) =>
lines
.Select(line => regex.Match(line))

130
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs
View File

@@ -48,16 +48,48 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
missingPackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath(fileProvider.SourceDir.FullName, "missingpackages"), "missing package", logger);
}
public string? TryRestoreLatestNetFrameworkReferenceAssemblies()
public string? TryRestore(string package)
{
if (TryRestorePackageManually(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies))
if (TryRestorePackageManually(package))
{
return DependencyManager.GetPackageDirectory(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies, missingPackageDirectory.DirInfo);
var packageDir = DependencyManager.GetPackageDirectory(package, missingPackageDirectory.DirInfo);
if (packageDir is not null)
{
return GetNewestNugetPackageVersionFolder(packageDir, package);
}
}
return null;
}
public string GetNewestNugetPackageVersionFolder(string packagePath, string packageFriendlyName)
{
var versionFolders = GetOrderedPackageVersionSubDirectories(packagePath);
if (versionFolders.Length > 1)
{
var versions = string.Join(", ", versionFolders.Select(d => d.Name));
logger.LogDebug($"Found multiple {packageFriendlyName} DLLs in NuGet packages at {packagePath}. Using the latest version ({versionFolders[0].Name}) from: {versions}.");
}
var selectedFrameworkFolder = versionFolders.FirstOrDefault()?.FullName;
if (selectedFrameworkFolder is null)
{
logger.LogDebug($"Found {packageFriendlyName} DLLs in NuGet packages at {packagePath}, but no version folder was found.");
selectedFrameworkFolder = packagePath;
}
logger.LogDebug($"Found {packageFriendlyName} DLLs in NuGet packages at {selectedFrameworkFolder}.");
return selectedFrameworkFolder;
}
public static DirectoryInfo[] GetOrderedPackageVersionSubDirectories(string packagePath)
{
return new DirectoryInfo(packagePath)
.EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
.OrderByDescending(d => d.Name) // TODO: Improve sorting to handle pre-release versions.
.ToArray();
}
public HashSet<AssemblyLookupLocation> Restore()
{
var assemblyLookupLocations = new HashSet<AssemblyLookupLocation>();
@@ -408,7 +440,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
.Select(d => Path.GetFileName(d).ToLowerInvariant());
}
private bool TryRestorePackageManually(string package, string? nugetConfig = null, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj, bool tryWithoutNugetConfig = true)
private bool TryRestorePackageManually(string package, string? nugetConfig = null, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj,
bool tryWithoutNugetConfig = true, bool tryPrereleaseVersion = true)
{
logger.LogInfo($"Restoring package {package}...");
using var tempDir = new TemporaryDirectory(
@@ -430,59 +463,91 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
return false;
}
var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig));
if (!res.Success)
var res = TryRestorePackageManually(package, nugetConfig, tempDir, tryPrereleaseVersion);
if (res.Success)
{
if (tryWithoutNugetConfig && res.HasNugetPackageSourceError && nugetConfig is not null)
{
// Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true));
}
return true;
}
// TODO: the restore might fail, we could retry with
// - a prerelease (*-* instead of *) version of the package,
// - a different target framework moniker.
if (!res.Success)
if (tryWithoutNugetConfig && res.HasNugetPackageSourceError && nugetConfig is not null)
{
logger.LogDebug($"Trying to restore '{package}' without nuget.config.");
// Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
res = TryRestorePackageManually(package, nugetConfig: null, tempDir, tryPrereleaseVersion);
if (res.Success)
{
logger.LogInfo($"Failed to restore nuget package {package}");
return false;
return true;
}
}
return true;
logger.LogInfo($"Failed to restore nuget package {package}");
return false;
}
private RestoreResult TryRestorePackageManually(string package, string? nugetConfig, TemporaryDirectory tempDir, bool tryPrereleaseVersion)
{
var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig, ForceReevaluation: true));
if (!res.Success && tryPrereleaseVersion && res.HasNugetNoStablePackageVersionError)
{
logger.LogDebug($"Failed to restore nuget package {package} because no stable version was found.");
try
{
TryChangePackageVersion(tempDir.DirInfo, "*-*");
res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig, ForceReevaluation: true));
return res;
}
finally
{
TryChangePackageVersion(tempDir.DirInfo, "*");
}
}
return res;
}
private void TryChangeTargetFrameworkMoniker(DirectoryInfo tempDir)
{
TryChangeProjectFile(tempDir, TargetFramework(), $"<TargetFramework>{FrameworkPackageNames.LatestNetFrameworkMoniker}</TargetFramework>", "target framework moniker");
}
private void TryChangePackageVersion(DirectoryInfo tempDir, string newVersion)
{
TryChangeProjectFile(tempDir, PackageReferenceVersion(), $"Version=\"{newVersion}\"", "package reference version");
}
private bool TryChangeProjectFile(DirectoryInfo projectDir, Regex pattern, string replacement, string patternName)
{
try
{
logger.LogInfo($"Changing the target framework moniker in {tempDir.FullName}...");
logger.LogDebug($"Changing the {patternName} in {projectDir.FullName}...");
var csprojs = tempDir.GetFiles("*.csproj", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive });
var csprojs = projectDir.GetFiles("*.csproj", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive });
if (csprojs.Length != 1)
{
logger.LogError($"Could not find the .csproj file in {tempDir.FullName}, count = {csprojs.Length}");
return;
logger.LogError($"Could not find the .csproj file in {projectDir.FullName}, count = {csprojs.Length}");
return false;
}
var csproj = csprojs[0];
var content = File.ReadAllText(csproj.FullName);
var matches = TargetFramework().Matches(content);
var matches = pattern.Matches(content);
if (matches.Count == 0)
{
logger.LogError($"Could not find target framework in {csproj.FullName}");
}
else
{
content = TargetFramework().Replace(content, $"<TargetFramework>{FrameworkPackageNames.LatestNetFrameworkMoniker}</TargetFramework>", 1);
File.WriteAllText(csproj.FullName, content);
logger.LogError($"Could not find the {patternName} in {csproj.FullName}");
return false;
}
content = pattern.Replace(content, replacement, 1);
File.WriteAllText(csproj.FullName, content);
return true;
}
catch (Exception exc)
{
logger.LogError($"Failed to update target framework in {tempDir.FullName}: {exc}");
logger.LogError($"Failed to change the {patternName} in {projectDir.FullName}: {exc}");
}
return false;
}
private static async Task ExecuteGetRequest(string address, HttpClient httpClient, CancellationToken cancellationToken)
@@ -664,6 +729,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
[GeneratedRegex(@"<TargetFramework>.*</TargetFramework>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
private static partial Regex TargetFramework();
[GeneratedRegex(@"Version=""[*|*-*]""", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
private static partial Regex PackageReferenceVersion();
[GeneratedRegex(@"^(.+)\.(\d+\.\d+\.\d+(-(.+))?)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
private static partial Regex LegacyNugetPackage();

8
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/SourceGenerators/ResxGenerator.cs
View File

@@ -15,12 +15,18 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
IDotNet dotnet,
ICompilationInfoContainer compilationInfoContainer,
ILogger logger,
NugetPackageRestorer nugetPackageRestorer,
TemporaryDirectory tempWorkingDirectory,
IEnumerable<string> references) : base(fileProvider, fileContent, dotnet, compilationInfoContainer, logger, tempWorkingDirectory, references)
{
try
{
// todo: download latest `Microsoft.CodeAnalysis.ResxSourceGenerator` and set `sourceGeneratorFolder`
// The package is downloaded to `missingpackages`, which is okay, we're already after the DLL collection phase.
var nugetFolder = nugetPackageRestorer.TryRestore("Microsoft.CodeAnalysis.ResxSourceGenerator");
if (nugetFolder is not null)
{
sourceGeneratorFolder = System.IO.Path.Combine(nugetFolder, "analyzers", "dotnet", "cs");
}
}
catch (Exception e)
{