hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2025-12-17 01:03:14 +01:00
Code Issues Packages Projects Releases Wiki Activity

C#: Improve fallback nuget package restore in buildless

Browse Source 
Fallback cases coming from `<PackageReference />` and `packages.config` are now differentiated. In the latter case we're restoring the package through projects that target `net481`.
This commit is contained in:
Tamas Vajk
2024-02-21 11:46:35 +01:00
parent 70a2d16b1a
commit 0c46b493c3
4 changed files with 73 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
View File

@@ -882,14 +882,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(packageDirectory.DirInfo);
var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames();
var notYetDownloadedPackages = new HashSet<string>(fileContent.AllPackages);
var notYetDownloadedPackages = new HashSet<PackageReference>(fileContent.AllPackages);
foreach (var alreadyDownloadedPackage in alreadyDownloadedPackages)
{
notYetDownloadedPackages.Remove(alreadyDownloadedPackage);
notYetDownloadedPackages.Remove(new(alreadyDownloadedPackage, PackageReferenceSource.SdkCsProj));
}
foreach (var alreadyDownloadedLegacyPackage in alreadyDownloadedLegacyPackages)
{
notYetDownloadedPackages.Remove(alreadyDownloadedLegacyPackage);
notYetDownloadedPackages.Remove(new(alreadyDownloadedLegacyPackage, PackageReferenceSource.PackagesConfig));
}
if (notYetDownloadedPackages.Count == 0)
@@ -930,7 +930,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, package =>
{
var success = TryRestorePackageManually(package, nugetConfig);
var success = TryRestorePackageManually(package.Name, nugetConfig, package.PackageReferenceSource);
if (!success)
{
return;
@@ -947,7 +947,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
dllPaths.Add(missingPackageDirectory.DirInfo.FullName);
}
private bool TryRestorePackageManually(string package, string? nugetConfig)
[GeneratedRegex(@"<TargetFramework>.*</TargetFramework>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
private static partial Regex TargetFramework();
private bool TryRestorePackageManually(string package, string? nugetConfig, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj)
{
logger.LogInfo($"Restoring package {package}...");
using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package, "missingpackages_workingdir"));
@@ -957,6 +960,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
return false;
}
if (packageReferenceSource == PackageReferenceSource.PackagesConfig)
{
TryChangeTargetFrameworkMoniker(tempDir.DirInfo);
}
success = dotnet.AddPackage(tempDir.DirInfo.FullName, package);
if (!success)
{
@@ -972,7 +980,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true));
}
// TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package.
// TODO: the restore might fail, we could retry with
// - a prerelease (*-* instead of *) version of the package,
// - a different target framework moniker.
if (!res.Success)
{
@@ -984,6 +994,38 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
return true;
}
private void TryChangeTargetFrameworkMoniker(DirectoryInfo tempDir)
{
try
{
logger.LogInfo($"Changing the target framework moniker in {tempDir.FullName}...");
var csprojs = tempDir.GetFiles("*.csproj", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive });
if (csprojs.Length != 1)
{
logger.LogError($"Could not find the .csproj file in {tempDir.FullName}, count = {csprojs.Length}");
return;
}
var csproj = csprojs[0];
var content = File.ReadAllText(csproj.FullName);
var matches = TargetFramework().Matches(content);
if (matches.Count == 0)
{
logger.LogError($"Could not find target framework in {csproj.FullName}");
}
else
{
content = TargetFramework().Replace(content, $"<TargetFramework>{FrameworkPackageNames.LatestNetFrameworkMoniker}</TargetFramework>", 1);
File.WriteAllText(csproj.FullName, content);
}
}
catch (Exception exc)
{
logger.LogError($"Failed to update target framework in {tempDir.FullName}: {exc}");
}
}
public void Dispose(TemporaryDirectory? dir, string name)
{
try

24
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs
View File

@@ -19,11 +19,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private readonly ILogger logger;
private readonly IUnsafeFileReader unsafeFileReader;
private readonly IEnumerable<string> files;
private readonly HashSet<string> allPackages = new HashSet<string>();
private readonly HashSet<PackageReference> allPackages = new HashSet<PackageReference>();
private readonly HashSet<string> implicitUsingNamespaces = new HashSet<string>();
private readonly Initializer initialize;
public HashSet<string> AllPackages
public HashSet<PackageReference> AllPackages
{
get
{
@@ -157,7 +157,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
return false;
}
private void AddPackageReference(ReadOnlySpan<char> line, string groupName, Func<Regex> regex)
private void AddPackageReference(ReadOnlySpan<char> line, string groupName, Func<Regex> regex, PackageReferenceSource source)
{
foreach (var valueMatch in regex().EnumerateMatches(line))
{
@@ -165,7 +165,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
var packageName = GetGroup(line, valueMatch, groupName).ToLowerInvariant();
if (!string.IsNullOrEmpty(packageName))
{
allPackages.Add(packageName);
allPackages.Add(new PackageReference(packageName, source));
}
}
}
@@ -181,11 +181,13 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
foreach (ReadOnlySpan<char> line in unsafeFileReader.ReadLines(file))
{
// Find all the packages.
AddPackageReference(line, "Include", PackageReference);
if (isPackagesConfig)
{
AddPackageReference(line, "id", LegacyPackageReference);
AddPackageReference(line, "id", LegacyPackageReference, PackageReferenceSource.PackagesConfig);
}
else
{
AddPackageReference(line, "Include", PackageReference, PackageReferenceSource.SdkCsProj);
}
// Determine if ASP.NET is used.
@@ -268,4 +270,12 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
}
}
public enum PackageReferenceSource
{
SdkCsProj,
PackagesConfig
}
public record PackageReference(string Name, PackageReferenceSource PackageReferenceSource);
}

4
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FrameworkPackageNames.cs
View File

@@ -4,7 +4,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal static class FrameworkPackageNames
{
public static string LatestNetFrameworkReferenceAssemblies { get; } = "microsoft.netframework.referenceassemblies.net481";
public const string LatestNetFrameworkMoniker = "net481";
public static string LatestNetFrameworkReferenceAssemblies { get; } = $"microsoft.netframework.referenceassemblies.{LatestNetFrameworkMoniker}";
public static string AspNetCoreFramework { get; } = "microsoft.aspnetcore.app.ref";

10
csharp/extractor/Semmle.Extraction.Tests/FileContent.cs
View File

@@ -55,9 +55,9 @@ namespace Semmle.Extraction.Tests
// Verify
Assert.False(useAspNetDlls);
Assert.Equal(3, allPackages.Count);
Assert.Contains("DotNetAnalyzers.DocumentationAnalyzers".ToLowerInvariant(), allPackages);
Assert.Contains("Microsoft.CodeAnalysis.NetAnalyzers".ToLowerInvariant(), allPackages);
Assert.Contains("StyleCop.Analyzers".ToLowerInvariant(), allPackages);
Assert.Contains(new PackageReference("DotNetAnalyzers.DocumentationAnalyzers".ToLowerInvariant(), PackageReferenceSource.SdkCsProj), allPackages);
Assert.Contains(new PackageReference("Microsoft.CodeAnalysis.NetAnalyzers".ToLowerInvariant(), PackageReferenceSource.SdkCsProj), allPackages);
Assert.Contains(new PackageReference("StyleCop.Analyzers".ToLowerInvariant(), PackageReferenceSource.SdkCsProj), allPackages);
}
[Fact]
@@ -80,8 +80,8 @@ namespace Semmle.Extraction.Tests
// Verify
Assert.True(useAspNetDlls);
Assert.Equal(2, allPackages.Count);
Assert.Contains("Microsoft.CodeAnalysis.NetAnalyzers".ToLowerInvariant(), allPackages);
Assert.Contains("StyleCop.Analyzers".ToLowerInvariant(), allPackages);
Assert.Contains(new PackageReference("Microsoft.CodeAnalysis.NetAnalyzers".ToLowerInvariant(), PackageReferenceSource.SdkCsProj), allPackages);
Assert.Contains(new PackageReference("StyleCop.Analyzers".ToLowerInvariant(), PackageReferenceSource.SdkCsProj), allPackages);
}
private static void CsProjSettingsTest(string line, bool expected, Func<FileContent, bool> func)