Restore and use Microsoft.CodeAnalysis.ResxSourceGenerator

This commit is contained in:
Tamas Vajk
2024-04-17 10:41:41 +02:00
parent 79fe5f851b
commit 1ff4c0daf3
11 changed files with 298 additions and 67 deletions

View File

@@ -153,7 +153,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
new ImplicitUsingsGenerator(fileContent, logger, tempWorkingDirectory),
new RazorGenerator(fileProvider, fileContent, dotnet, this, logger, tempWorkingDirectory, usedReferences.Keys),
new ResxGenerator(fileProvider, fileContent, dotnet, this, logger, tempWorkingDirectory, usedReferences.Keys),
new ResxGenerator(fileProvider, fileContent, dotnet, this, logger, nugetPackageRestorer, tempWorkingDirectory, usedReferences.Keys),
};
foreach (var sourceGenerator in sourceGenerators)
@@ -256,35 +256,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
}
private void SelectNewestFrameworkPath(string frameworkPath, string frameworkType, ISet<AssemblyLookupLocation> dllLocations, ISet<string> frameworkLocations)
{
var versionFolders = GetPackageVersionSubDirectories(frameworkPath);
if (versionFolders.Length > 1)
{
var versions = string.Join(", ", versionFolders.Select(d => d.Name));
logger.LogDebug($"Found multiple {frameworkType} DLLs in NuGet packages at {frameworkPath}. Using the latest version ({versionFolders[0].Name}) from: {versions}.");
}
var selectedFrameworkFolder = versionFolders.FirstOrDefault()?.FullName;
if (selectedFrameworkFolder is null)
{
logger.LogDebug($"Found {frameworkType} DLLs in NuGet packages at {frameworkPath}, but no version folder was found.");
selectedFrameworkFolder = frameworkPath;
}
dllLocations.Add(selectedFrameworkFolder);
frameworkLocations.Add(selectedFrameworkFolder);
logger.LogDebug($"Found {frameworkType} DLLs in NuGet packages at {selectedFrameworkFolder}.");
}
private static DirectoryInfo[] GetPackageVersionSubDirectories(string packagePath)
{
return new DirectoryInfo(packagePath)
.EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
.OrderByDescending(d => d.Name) // TODO: Improve sorting to handle pre-release versions.
.ToArray();
}
private void RemoveFrameworkNugetPackages(ISet<AssemblyLookupLocation> dllLocations, int fromIndex = 0)
{
var packagesInPrioOrder = FrameworkPackageNames.NetFrameworks;
@@ -311,10 +282,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
foreach (var fp in frameworkPaths)
{
dotnetFrameworkVersionVariantCount += GetPackageVersionSubDirectories(fp.Path!).Length;
dotnetFrameworkVersionVariantCount += NugetPackageRestorer.GetOrderedPackageVersionSubDirectories(fp.Path!).Length;
}
SelectNewestFrameworkPath(frameworkPath.Path, ".NET Framework", dllLocations, frameworkLocations);
var folder = nugetPackageRestorer.GetNewestNugetPackageVersionFolder(frameworkPath.Path, ".NET Framework");
dllLocations.Add(folder);
frameworkLocations.Add(folder);
RemoveFrameworkNugetPackages(dllLocations, frameworkPath.Index + 1);
return;
}
@@ -332,7 +305,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (runtimeLocation is null)
{
logger.LogInfo("No .NET Desktop Runtime location found. Attempting to restore the .NET Framework reference assemblies manually.");
runtimeLocation = nugetPackageRestorer.TryRestoreLatestNetFrameworkReferenceAssemblies();
runtimeLocation = nugetPackageRestorer.TryRestore(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies);
}
}
@@ -372,7 +345,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
// First try to find ASP.NET Core assemblies in the NuGet packages
if (GetPackageDirectory(FrameworkPackageNames.AspNetCoreFramework) is string aspNetCorePackage)
{
SelectNewestFrameworkPath(aspNetCorePackage, "ASP.NET Core", dllLocations, frameworkLocations);
var folder = nugetPackageRestorer.GetNewestNugetPackageVersionFolder(aspNetCorePackage, "ASP.NET Core");
dllLocations.Add(folder);
frameworkLocations.Add(folder);
return;
}
@@ -388,7 +363,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
if (GetPackageDirectory(FrameworkPackageNames.WindowsDesktopFramework) is string windowsDesktopApp)
{
SelectNewestFrameworkPath(windowsDesktopApp, "Windows Desktop App", dllLocations, frameworkLocations);
var folder = nugetPackageRestorer.GetNewestNugetPackageVersionFolder(windowsDesktopApp, "Windows Desktop App");
dllLocations.Add(folder);
frameworkLocations.Add(folder);
}
}

View File

@@ -30,6 +30,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private readonly Lazy<bool> hasNugetPackageSourceError = new(() => Output.Any(s => s.Contains("NU1301")));
public bool HasNugetPackageSourceError => hasNugetPackageSourceError.Value;
private readonly Lazy<bool> hasNugetNoStablePackageVersionError = new(() => Output.Any(s => s.Contains("NU1103")));
public bool HasNugetNoStablePackageVersionError => hasNugetNoStablePackageVersionError.Value;
private static IEnumerable<string> GetFirstGroupOnMatch(Regex regex, IEnumerable<string> lines) =>
lines
.Select(line => regex.Match(line))

View File

@@ -48,16 +48,48 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
missingPackageDirectory = new TemporaryDirectory(ComputeTempDirectoryPath(fileProvider.SourceDir.FullName, "missingpackages"), "missing package", logger);
}
public string? TryRestoreLatestNetFrameworkReferenceAssemblies()
public string? TryRestore(string package)
{
if (TryRestorePackageManually(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies))
if (TryRestorePackageManually(package))
{
return DependencyManager.GetPackageDirectory(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies, missingPackageDirectory.DirInfo);
var packageDir = DependencyManager.GetPackageDirectory(package, missingPackageDirectory.DirInfo);
if (packageDir is not null)
{
return GetNewestNugetPackageVersionFolder(packageDir, package);
}
}
return null;
}
public string GetNewestNugetPackageVersionFolder(string packagePath, string packageFriendlyName)
{
var versionFolders = GetOrderedPackageVersionSubDirectories(packagePath);
if (versionFolders.Length > 1)
{
var versions = string.Join(", ", versionFolders.Select(d => d.Name));
logger.LogDebug($"Found multiple {packageFriendlyName} DLLs in NuGet packages at {packagePath}. Using the latest version ({versionFolders[0].Name}) from: {versions}.");
}
var selectedFrameworkFolder = versionFolders.FirstOrDefault()?.FullName;
if (selectedFrameworkFolder is null)
{
logger.LogDebug($"Found {packageFriendlyName} DLLs in NuGet packages at {packagePath}, but no version folder was found.");
selectedFrameworkFolder = packagePath;
}
logger.LogDebug($"Found {packageFriendlyName} DLLs in NuGet packages at {selectedFrameworkFolder}.");
return selectedFrameworkFolder;
}
public static DirectoryInfo[] GetOrderedPackageVersionSubDirectories(string packagePath)
{
return new DirectoryInfo(packagePath)
.EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
.OrderByDescending(d => d.Name) // TODO: Improve sorting to handle pre-release versions.
.ToArray();
}
public HashSet<AssemblyLookupLocation> Restore()
{
var assemblyLookupLocations = new HashSet<AssemblyLookupLocation>();
@@ -408,7 +440,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
.Select(d => Path.GetFileName(d).ToLowerInvariant());
}
private bool TryRestorePackageManually(string package, string? nugetConfig = null, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj, bool tryWithoutNugetConfig = true)
private bool TryRestorePackageManually(string package, string? nugetConfig = null, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj,
bool tryWithoutNugetConfig = true, bool tryPrereleaseVersion = true)
{
logger.LogInfo($"Restoring package {package}...");
using var tempDir = new TemporaryDirectory(
@@ -430,59 +463,91 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
return false;
}
var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig));
if (!res.Success)
var res = TryRestorePackageManually(package, nugetConfig, tempDir, tryPrereleaseVersion);
if (res.Success)
{
if (tryWithoutNugetConfig && res.HasNugetPackageSourceError && nugetConfig is not null)
{
// Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true));
}
return true;
}
// TODO: the restore might fail, we could retry with
// - a prerelease (*-* instead of *) version of the package,
// - a different target framework moniker.
if (!res.Success)
if (tryWithoutNugetConfig && res.HasNugetPackageSourceError && nugetConfig is not null)
{
logger.LogDebug($"Trying to restore '{package}' without nuget.config.");
// Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
res = TryRestorePackageManually(package, nugetConfig: null, tempDir, tryPrereleaseVersion);
if (res.Success)
{
logger.LogInfo($"Failed to restore nuget package {package}");
return false;
return true;
}
}
return true;
logger.LogInfo($"Failed to restore nuget package {package}");
return false;
}
private RestoreResult TryRestorePackageManually(string package, string? nugetConfig, TemporaryDirectory tempDir, bool tryPrereleaseVersion)
{
var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig, ForceReevaluation: true));
if (!res.Success && tryPrereleaseVersion && res.HasNugetNoStablePackageVersionError)
{
logger.LogDebug($"Failed to restore nuget package {package} because no stable version was found.");
try
{
TryChangePackageVersion(tempDir.DirInfo, "*-*");
res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig, ForceReevaluation: true));
return res;
}
finally
{
TryChangePackageVersion(tempDir.DirInfo, "*");
}
}
return res;
}
private void TryChangeTargetFrameworkMoniker(DirectoryInfo tempDir)
{
TryChangeProjectFile(tempDir, TargetFramework(), $"<TargetFramework>{FrameworkPackageNames.LatestNetFrameworkMoniker}</TargetFramework>", "target framework moniker");
}
private void TryChangePackageVersion(DirectoryInfo tempDir, string newVersion)
{
TryChangeProjectFile(tempDir, PackageReferenceVersion(), $"Version=\"{newVersion}\"", "package reference version");
}
private bool TryChangeProjectFile(DirectoryInfo projectDir, Regex pattern, string replacement, string patternName)
{
try
{
logger.LogInfo($"Changing the target framework moniker in {tempDir.FullName}...");
logger.LogDebug($"Changing the {patternName} in {projectDir.FullName}...");
var csprojs = tempDir.GetFiles("*.csproj", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive });
var csprojs = projectDir.GetFiles("*.csproj", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive });
if (csprojs.Length != 1)
{
logger.LogError($"Could not find the .csproj file in {tempDir.FullName}, count = {csprojs.Length}");
return;
logger.LogError($"Could not find the .csproj file in {projectDir.FullName}, count = {csprojs.Length}");
return false;
}
var csproj = csprojs[0];
var content = File.ReadAllText(csproj.FullName);
var matches = TargetFramework().Matches(content);
var matches = pattern.Matches(content);
if (matches.Count == 0)
{
logger.LogError($"Could not find target framework in {csproj.FullName}");
}
else
{
content = TargetFramework().Replace(content, $"<TargetFramework>{FrameworkPackageNames.LatestNetFrameworkMoniker}</TargetFramework>", 1);
File.WriteAllText(csproj.FullName, content);
logger.LogError($"Could not find the {patternName} in {csproj.FullName}");
return false;
}
content = pattern.Replace(content, replacement, 1);
File.WriteAllText(csproj.FullName, content);
return true;
}
catch (Exception exc)
{
logger.LogError($"Failed to update target framework in {tempDir.FullName}: {exc}");
logger.LogError($"Failed to change the {patternName} in {projectDir.FullName}: {exc}");
}
return false;
}
private static async Task ExecuteGetRequest(string address, HttpClient httpClient, CancellationToken cancellationToken)
@@ -664,6 +729,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
[GeneratedRegex(@"<TargetFramework>.*</TargetFramework>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
private static partial Regex TargetFramework();
[GeneratedRegex(@"Version=""[*|*-*]""", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
private static partial Regex PackageReferenceVersion();
[GeneratedRegex(@"^(.+)\.(\d+\.\d+\.\d+(-(.+))?)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
private static partial Regex LegacyNugetPackage();

View File

@@ -15,12 +15,18 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
IDotNet dotnet,
ICompilationInfoContainer compilationInfoContainer,
ILogger logger,
NugetPackageRestorer nugetPackageRestorer,
TemporaryDirectory tempWorkingDirectory,
IEnumerable<string> references) : base(fileProvider, fileContent, dotnet, compilationInfoContainer, logger, tempWorkingDirectory, references)
{
try
{
// todo: download latest `Microsoft.CodeAnalysis.ResxSourceGenerator` and set `sourceGeneratorFolder`
// The package is downloaded to `missingpackages`, which is okay, we're already after the DLL collection phase.
var nugetFolder = nugetPackageRestorer.TryRestore("Microsoft.CodeAnalysis.ResxSourceGenerator");
if (nugetFolder is not null)
{
sourceGeneratorFolder = System.IO.Path.Combine(nugetFolder, "analyzers", "dotnet", "cs");
}
}
catch (Exception e)
{

View File

@@ -0,0 +1,3 @@
| Program.cs |
| test-db/working/implicitUsings/GlobalUsings.g.cs |
| test-db/working/resx/[...]/Microsoft.CodeAnalysis.ResxSourceGenerator.CSharp/Microsoft.CodeAnalysis.ResxSourceGenerator.CSharp.CSharpResxGenerator/test.Designer.cs |

View File

@@ -0,0 +1,22 @@
import csharp
private string getPath(File f) {
result = f.getRelativePath() and
not exists(
result
.indexOf("Microsoft.CodeAnalysis.ResxSourceGenerator.CSharp/Microsoft.CodeAnalysis.ResxSourceGenerator.CSharp.CSharpResxGenerator")
)
or
exists(int index |
index =
f.getRelativePath()
.indexOf("Microsoft.CodeAnalysis.ResxSourceGenerator.CSharp/Microsoft.CodeAnalysis.ResxSourceGenerator.CSharp.CSharpResxGenerator") and
result =
f.getRelativePath().substring(0, index - 32 - 2) + "/[...]/" +
f.getRelativePath().substring(index, f.getRelativePath().length())
)
}
from File f
where f.fromSource()
select getPath(f)

View File

@@ -0,0 +1 @@
var dummy = "dummy";

View File

@@ -0,0 +1,5 @@
{
"sdk": {
"version": "8.0.101"
}
}

View File

@@ -0,0 +1,14 @@
<Project Sdk="Microsoft.NET.Sdk.Web">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<Target Name="DeleteBinObjFolders" BeforeTargets="Clean">
<RemoveDir Directories=".\bin" />
<RemoveDir Directories=".\obj" />
</Target>
</Project>

View File

@@ -0,0 +1,4 @@
import os
from create_database_utils import *
run_codeql_database_create(lang="csharp", extra_args=["--build-mode=none"])

View File

@@ -0,0 +1,128 @@
<?xml version="1.0" encoding="utf-8"?>
<root>
<!--
Microsoft ResX Schema
Version 2.0
The primary goals of this format is to allow a simple XML format
that is mostly human readable. The generation and parsing of the
various data types are done through the TypeConverter classes
associated with the data types.
Example:
... ado.net/XML headers & schema ...
<resheader name="resmimetype">text/microsoft-resx</resheader>
<resheader name="version">2.0</resheader>
<resheader name="reader">System.Resources.ResXResourceReader, System.Windows.Forms, ...</resheader>
<resheader name="writer">System.Resources.ResXResourceWriter, System.Windows.Forms, ...</resheader>
<data name="Name1"><value>this is my long string</value><comment>this is a comment</comment></data>
<data name="Color1" type="System.Drawing.Color, System.Drawing">Blue</data>
<data name="Bitmap1" mimetype="application/x-microsoft.net.object.binary.base64">
<value>[base64 mime encoded serialized .NET Framework object]</value>
</data>
<data name="Icon1" type="System.Drawing.Icon, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
<value>[base64 mime encoded string representing a byte array form of the .NET Framework object]</value>
<comment>This is a comment</comment>
</data>
There are any number of "resheader" rows that contain simple
name/value pairs.
Each data row contains a name, and value. The row also contains a
type or mimetype. Type corresponds to a .NET class that support
text/value conversion through the TypeConverter architecture.
Classes that don't support this are serialized and stored with the
mimetype set.
The mimetype is used for serialized objects, and tells the
ResXResourceReader how to depersist the object. This is currently not
extensible. For a given mimetype the value must be set accordingly:
Note - application/x-microsoft.net.object.binary.base64 is the format
that the ResXResourceWriter will generate, however the reader can
read any of the formats listed below.
mimetype: application/x-microsoft.net.object.binary.base64
value : The object must be serialized with
: System.Runtime.Serialization.Formatters.Binary.BinaryFormatter
: and then encoded with base64 encoding.
mimetype: application/x-microsoft.net.object.soap.base64
value : The object must be serialized with
: System.Runtime.Serialization.Formatters.Soap.SoapFormatter
: and then encoded with base64 encoding.
mimetype: application/x-microsoft.net.object.bytearray.base64
value : The object must be serialized into a byte array
: using a System.ComponentModel.TypeConverter
: and then encoded with base64 encoding.
-->
<xsd:schema id="root" xmlns="" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:msdata="urn:schemas-microsoft-com:xml-msdata">
<xsd:import namespace="http://www.w3.org/XML/1998/namespace"/>
<xsd:element name="root" msdata:IsDataSet="true">
<xsd:complexType>
<xsd:choice maxOccurs="unbounded">
<xsd:element name="metadata">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="value" type="xsd:string" minOccurs="0"/>
</xsd:sequence>
<xsd:attribute name="name" use="required" type="xsd:string"/>
<xsd:attribute name="type" type="xsd:string"/>
<xsd:attribute name="mimetype" type="xsd:string"/>
<xsd:attribute ref="xml:space"/>
</xsd:complexType>
</xsd:element>
<xsd:element name="assembly">
<xsd:complexType>
<xsd:attribute name="alias" type="xsd:string"/>
<xsd:attribute name="name" type="xsd:string"/>
</xsd:complexType>
</xsd:element>
<xsd:element name="data">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="value" type="xsd:string" minOccurs="0" msdata:Ordinal="1"/>
<xsd:element name="comment" type="xsd:string" minOccurs="0" msdata:Ordinal="2"/>
</xsd:sequence>
<xsd:attribute name="name" type="xsd:string" use="required" msdata:Ordinal="1"/>
<xsd:attribute name="type" type="xsd:string" msdata:Ordinal="3"/>
<xsd:attribute name="mimetype" type="xsd:string" msdata:Ordinal="4"/>
<xsd:attribute ref="xml:space"/>
</xsd:complexType>
</xsd:element>
<xsd:element name="resheader">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="value" type="xsd:string" minOccurs="0" msdata:Ordinal="1"/>
</xsd:sequence>
<xsd:attribute name="name" type="xsd:string" use="required"/>
</xsd:complexType>
</xsd:element>
</xsd:choice>
</xsd:complexType>
</xsd:element>
</xsd:schema>
<resheader name="resmimetype">
<value>text/microsoft-resx</value>
</resheader>
<resheader name="version">
<value>2.0</value>
</resheader>
<resheader name="reader">
<value>System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
</resheader>
<resheader name="writer">
<value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
</resheader>
<data name="Key123" xml:space="preserve">
<value>Value123</value>
<comment/>
</data>
<data name="Key456" xml:space="preserve">
<value>Value456</value>
<comment/>
</data>
</root>