hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-01-05 10:40:21 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #14020 from tamasvajk/fix/dependency-fetching-1

Browse Source 
C#: Fix lazy evaluation of not yet downloaded packages
This commit is contained in:
Michael Nebel
2023-08-23 10:39:29 +02:00
committed by GitHub
parent cf53956d39 c1f167ceee
commit 1f1d48f768
3 changed files with 26 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
View File

@@ -60,7 +60,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName));
this.fileContent = new FileContent(packageDirectory, progressMonitor, () => GetFiles("*.*"));
this.fileContent = new FileContent(progressMonitor, () => GetFiles("*.*"));
this.allSources = GetFiles("*.cs").ToList();
var allProjects = GetFiles("*.csproj");
var solutions = options.SolutionFile is not null
@@ -388,7 +388,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
nugetConfig = nugetConfigs.FirstOrDefault();
}
foreach (var package in fileContent.NotYetDownloadedPackages)
var alreadyDownloadedPackages = Directory.GetDirectories(packageDirectory.DirInfo.FullName)
.Select(d => Path.GetFileName(d)
.ToLowerInvariant());
var notYetDownloadedPackages = fileContent.AllPackages.Except(alreadyDownloadedPackages);
foreach (var package in notYetDownloadedPackages)
{
progressMonitor.NugetInstall(package);
using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package));

25
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs
View File

@@ -12,23 +12,22 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
// This class is used to read a set of files and decide different properties about the
// content (by reading the content of the files only once).
// The implementation is lazy, so the properties are only calculated when
// the first property is accessed.
// the first property is accessed.
// </summary>
internal partial class FileContent
{
private readonly ProgressMonitor progressMonitor;
private readonly IUnsafeFileReader unsafeFileReader;
private readonly Func<IEnumerable<string>> getFiles;
private readonly Func<HashSet<string>> getAlreadyDownloadedPackages;
private readonly HashSet<string> notYetDownloadedPackages = new HashSet<string>();
private readonly HashSet<string> allPackages = new HashSet<string>();
private readonly Initializer initialize;
public HashSet<string> NotYetDownloadedPackages
public HashSet<string> AllPackages
{
get
{
initialize.Run();
return notYetDownloadedPackages;
return allPackages;
}
}
@@ -50,12 +49,10 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
}
internal FileContent(Func<HashSet<string>> getAlreadyDownloadedPackages,
ProgressMonitor progressMonitor,
internal FileContent(ProgressMonitor progressMonitor,
Func<IEnumerable<string>> getFiles,
IUnsafeFileReader unsafeFileReader)
{
this.getAlreadyDownloadedPackages = getAlreadyDownloadedPackages;
this.progressMonitor = progressMonitor;
this.getFiles = getFiles;
this.unsafeFileReader = unsafeFileReader;
@@ -63,10 +60,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
}
public FileContent(TemporaryDirectory packageDirectory, ProgressMonitor progressMonitor, Func<IEnumerable<string>> getFiles) : this(() => Directory.GetDirectories(packageDirectory.DirInfo.FullName)
.Select(d => Path.GetFileName(d)
.ToLowerInvariant())
.ToHashSet(), progressMonitor, getFiles, new UnsafeFileReader())
public FileContent(ProgressMonitor progressMonitor, Func<IEnumerable<string>> getFiles) : this(progressMonitor, getFiles, new UnsafeFileReader())
{ }
private static string GetGroup(ReadOnlySpan<char> input, ValueMatch valueMatch, string groupPrefix)
@@ -101,7 +95,6 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
private void DoInitialize()
{
var alreadyDownloadedPackages = getAlreadyDownloadedPackages();
foreach (var file in getFiles())
{
try
@@ -109,14 +102,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
foreach (ReadOnlySpan<char> line in unsafeFileReader.ReadLines(file))
{
// Find the not yet downloaded packages.
// Find all the packages.
foreach (var valueMatch in PackageReference().EnumerateMatches(line))
{
// We can't get the group from the ValueMatch, so doing it manually:
var packageName = GetGroup(line, valueMatch, "Include");
if (!string.IsNullOrEmpty(packageName) && !alreadyDownloadedPackages.Contains(packageName))
if (!string.IsNullOrEmpty(packageName))
{
notYetDownloadedPackages.Add(packageName);
allPackages.Add(packageName);
}
}

22
csharp/extractor/Semmle.Extraction.Tests/FileContent.cs
View File

@@ -1,5 +1,6 @@
using Xunit;
using System.Collections.Generic;
using System.Linq;
using Semmle.Util.Logging;
using Semmle.Extraction.CSharp.DependencyFetching;
@@ -33,8 +34,7 @@ namespace Semmle.Extraction.Tests
internal class TestFileContent : FileContent
{
public TestFileContent(List<string> lines) : base(() => new HashSet<string>(),
new ProgressMonitor(new LoggerStub()),
public TestFileContent(List<string> lines) : base(new ProgressMonitor(new LoggerStub()),
() => new List<string>() { "test1.cs" },
new UnsafeFileReaderStub(lines))
{ }
@@ -57,15 +57,15 @@ namespace Semmle.Extraction.Tests
var fileContent = new TestFileContent(lines);
// Execute
var notYetDownloadedPackages = fileContent.NotYetDownloadedPackages;
var allPackages = fileContent.AllPackages;
var useAspNetDlls = fileContent.UseAspNetDlls;
// Verify
Assert.False(useAspNetDlls);
Assert.Equal(3, notYetDownloadedPackages.Count);
Assert.Contains("DotNetAnalyzers.DocumentationAnalyzers".ToLowerInvariant(), notYetDownloadedPackages);
Assert.Contains("Microsoft.CodeAnalysis.NetAnalyzers".ToLowerInvariant(), notYetDownloadedPackages);
Assert.Contains("StyleCop.Analyzers".ToLowerInvariant(), notYetDownloadedPackages);
Assert.Equal(3, allPackages.Count);
Assert.Contains("DotNetAnalyzers.DocumentationAnalyzers".ToLowerInvariant(), allPackages);
Assert.Contains("Microsoft.CodeAnalysis.NetAnalyzers".ToLowerInvariant(), allPackages);
Assert.Contains("StyleCop.Analyzers".ToLowerInvariant(), allPackages);
}
[Fact]
@@ -83,13 +83,13 @@ namespace Semmle.Extraction.Tests
// Execute
var useAspNetDlls = fileContent.UseAspNetDlls;
var notYetDownloadedPackages = fileContent.NotYetDownloadedPackages;
var allPackages = fileContent.AllPackages;
// Verify
Assert.True(useAspNetDlls);
Assert.Equal(2, notYetDownloadedPackages.Count);
Assert.Contains("Microsoft.CodeAnalysis.NetAnalyzers".ToLowerInvariant(), notYetDownloadedPackages);
Assert.Contains("StyleCop.Analyzers".ToLowerInvariant(), notYetDownloadedPackages);
Assert.Equal(2, allPackages.Count);
Assert.Contains("Microsoft.CodeAnalysis.NetAnalyzers".ToLowerInvariant(), allPackages);
Assert.Contains("StyleCop.Analyzers".ToLowerInvariant(), allPackages);
}
}
}