hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-04-28 10:15:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

C#: Extract dependency restore telemetry data

Browse Source
This commit is contained in:
Tamas Vajk
2024-02-05 13:09:04 +01:00
parent 6fbbb82f68
commit c2c7826936
13 changed files with 133 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
View File

@@ -133,6 +133,17 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
logger.LogInfo($"{conflictedReferences,align} resolved assembly conflicts");
logger.LogInfo($"{dotnetFrameworkVersionVariantCount,align} restored .NET framework variants");
logger.LogInfo($"Build analysis completed in {DateTime.Now - startTime}");
CompilationInfos.AddRange([
("Source files on filesystem", nonGeneratedSources.Count.ToString()),
("Source files generated", generatedSources.Count.ToString()),
("Solution files on filesystem", allSolutions.Count.ToString()),
("Project files on filesystem", allProjects.Count.ToString()),
("Resolved references", usedReferences.Keys.Count.ToString()),
("Unresolved references", unresolvedReferences.Count.ToString()),
("Resolved assembly conflicts", conflictedReferences.ToString()),
("Restored .NET framework variants", dotnetFrameworkVersionVariantCount.ToString()),
]);
}
private HashSet<string> AddFrameworkDlls(HashSet<string> dllPaths)
@@ -146,12 +157,18 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
return frameworkLocations;
}
private void RestoreNugetPackages(List<FileInfo> allNonBinaryFiles, IEnumerable<string> allProjects, IEnumerable<string> allSolutions, HashSet<string> dllPaths)
private void RestoreNugetPackages(List<FileInfo> allNonBinaryFiles, List<string> allProjects, List<string> allSolutions, HashSet<string> dllPaths)
{
try
{
var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, logger);
nuget.InstallPackages();
var count = nuget.InstallPackages();
if (nuget.PackageCount > 0)
{
CompilationInfos.Add(("packages.config files", nuget.PackageCount.ToString()));
CompilationInfos.Add(("Successfully restored packages.config files", count.ToString()));
}
var nugetPackageDlls = legacyPackageDirectory.DirInfo.GetFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = true });
var nugetPackageDllPaths = nugetPackageDlls.Select(f => f.FullName).ToHashSet();
@@ -629,6 +646,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// </summary>
public IEnumerable<string> UnresolvedReferences => unresolvedReferences.Select(r => r.Key);
/// <summary>
/// List of `(key, value)` tuples, that are stored in the DB for telemetry purposes.
/// </summary>
public List<(string, string)> CompilationInfos { get; } = new List<(string, string)>();
/// <summary>
/// Record that a particular reference couldn't be resolved.
/// Note that this records at most one project file per missing reference.
@@ -697,17 +719,24 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// Returns a list of projects that are up to date with respect to restore.
/// </summary>
/// <param name="solutions">A list of paths to solution files.</param>
private IEnumerable<string> RestoreSolutions(IEnumerable<string> solutions, out IEnumerable<string> assets)
private IEnumerable<string> RestoreSolutions(List<string> solutions, out IEnumerable<string> assets)
{
var successCount = 0;
var assetFiles = new List<string>();
var projects = solutions.SelectMany(solution =>
{
logger.LogInfo($"Restoring solution {solution}...");
var res = dotnet.Restore(new(solution, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
if (res.Success)
{
successCount++;
}
assetFiles.AddRange(res.AssetsFilePaths);
return res.RestoredProjects;
});
}).ToList();
assets = assetFiles;
CompilationInfos.Add(("Successfully restored solution files", successCount.ToString()));
CompilationInfos.Add(("Restored projects through solution files", projects.Count.ToString()));
return projects;
}
@@ -719,14 +748,24 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// <param name="projects">A list of paths to project files.</param>
private void RestoreProjects(IEnumerable<string> projects, out IEnumerable<string> assets)
{
var successCount = 0;
var assetFiles = new List<string>();
var sync = new object();
Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, project =>
{
logger.LogInfo($"Restoring project {project}...");
var res = dotnet.Restore(new(project, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
assetFiles.AddRange(res.AssetsFilePaths);
lock (sync)
{
if (res.Success)
{
successCount++;
}
assetFiles.AddRange(res.AssetsFilePaths);
}
});
assets = assetFiles;
CompilationInfos.Add(("Successfully restored project files", successCount.ToString()));
}
private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPaths)
@@ -767,6 +806,11 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
logger.LogInfo($"Using nuget.config file {nugetConfig}.");
}
CompilationInfos.Add(("Fallback nuget restore", notYetDownloadedPackages.Count.ToString()));
var successCount = 0;
var sync = new object();
Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, package =>
{
logger.LogInfo($"Restoring package {package}...");
@@ -798,9 +842,25 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
logger.LogInfo($"Failed to restore nuget package {package}");
}
else
{
lock (sync)
{
successCount++;
}
}
}
else
{
lock (sync)
{
successCount++;
}
}
});
CompilationInfos.Add(("Successfully ran fallback nuget restore", successCount.ToString()));
dllPaths.Add(missingPackageDirectory.DirInfo.FullName);
}

17
csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackages.cs
View File

@@ -22,6 +22,8 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// </summary>
private readonly FileInfo[] packageFiles;
public int PackageCount => packageFiles.Length;
/// <summary>
/// The computed packages directory.
/// This will be in the Temp location
@@ -105,7 +107,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
/// Restore all files in a specified package.
/// </summary>
/// <param name="package">The package file.</param>
private void RestoreNugetPackage(string package)
private bool RestoreNugetPackage(string package)
{
logger.LogInfo($"Restoring file {package}...");
@@ -141,22 +143,31 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
if (exitCode != 0)
{
logger.LogError($"Command {pi.FileName} {pi.Arguments} failed with exit code {exitCode}");
return false;
}
else
{
logger.LogInfo($"Restored file {package}");
return true;
}
}
/// <summary>
/// Download the packages to the temp folder.
/// </summary>
public void InstallPackages()
public int InstallPackages()
{
var success = 0;
foreach (var package in packageFiles)
{
RestoreNugetPackage(package.FullName);
var result = RestoreNugetPackage(package.FullName);
if (result)
{
success++;
}
}
return success;
}
}
}

19
csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs
View File

@@ -24,8 +24,7 @@ namespace Semmle.Extraction.CSharp.Standalone
private static void AnalyseStandalone(
StandaloneAnalyser analyser,
IEnumerable<string> sources,
IEnumerable<string> referencePaths,
ExtractionInput extractionInput,
CommonOptions options,
IProgressMonitor progressMonitor,
Stopwatch stopwatch)
@@ -35,12 +34,12 @@ namespace Semmle.Extraction.CSharp.Standalone
try
{
CSharp.Extractor.Analyse(stopwatch, analyser, options,
references => GetResolvedReferencesStandalone(referencePaths, references),
(analyser, syntaxTrees) => CSharp.Extractor.ReadSyntaxTrees(sources, analyser, null, null, syntaxTrees),
references => GetResolvedReferencesStandalone(extractionInput.References, references),
(analyser, syntaxTrees) => CSharp.Extractor.ReadSyntaxTrees(extractionInput.Sources, analyser, null, null, syntaxTrees),
(syntaxTrees, references) => CSharpCompilation.Create(
output.Name, syntaxTrees, references, new CSharpCompilationOptions(OutputKind.ConsoleApplication, allowUnsafe: true)
),
(compilation, options) => analyser.Initialize(output.FullName, compilation, options),
(compilation, options) => analyser.Initialize(output.FullName, extractionInput.CompilationInfos, compilation, options),
_ => { },
() =>
{
@@ -73,8 +72,7 @@ namespace Semmle.Extraction.CSharp.Standalone
}
private static void ExtractStandalone(
IEnumerable<string> sources,
IEnumerable<string> referencePaths,
ExtractionInput extractionInput,
IProgressMonitor pm,
ILogger logger,
CommonOptions options)
@@ -88,7 +86,7 @@ namespace Semmle.Extraction.CSharp.Standalone
using var analyser = new StandaloneAnalyser(pm, logger, false, pathTransformer);
try
{
AnalyseStandalone(analyser, sources, referencePaths, options, pm, stopwatch);
AnalyseStandalone(analyser, extractionInput, options, pm, stopwatch);
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{
@@ -131,6 +129,8 @@ namespace Semmle.Extraction.CSharp.Standalone
}
}
public record ExtractionInput(IEnumerable<string> Sources, IEnumerable<string> References, IEnumerable<(string, string)> CompilationInfos);
public static ExitCode Run(Options options)
{
var stopwatch = new Stopwatch();
@@ -152,8 +152,7 @@ namespace Semmle.Extraction.CSharp.Standalone
logger.Log(Severity.Info, "");
logger.Log(Severity.Info, "Extracting...");
ExtractStandalone(
a.Extraction.Sources,
a.References,
new ExtractionInput(a.Extraction.Sources, a.References, a.CompilationInfos),
new ExtractionProgress(logger),
fileLogger,
options);

3
csharp/extractor/Semmle.Extraction.CSharp.Standalone/Program.cs
View File

@@ -30,6 +30,7 @@ namespace Semmle.Extraction.CSharp.Standalone
References = dependencyManager.ReferenceFiles;
Extraction = new Extraction(options.SrcDir);
Extraction.Sources.AddRange(dependencyManager.AllSourceFiles);
CompilationInfos = dependencyManager.CompilationInfos;
}
public IEnumerable<string> References { get; }
@@ -39,6 +40,8 @@ namespace Semmle.Extraction.CSharp.Standalone
/// </summary>
public Extraction Extraction { get; }
public IEnumerable<(string, string)> CompilationInfos { get; }
private readonly DependencyManager dependencyManager;
public void Dispose()

5
csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs
View File

@@ -6,6 +6,7 @@ using System.Linq;
using System.Threading.Tasks;
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.CSharp;
using Semmle.Util;
using Semmle.Util.Logging;
using Semmle.Extraction.CSharp.Populators;
@@ -20,7 +21,7 @@ namespace Semmle.Extraction.CSharp
protected CSharpCompilation? compilation;
protected CommonOptions? options;
private protected Entities.Compilation? compilationEntity;
private IDisposable? compilationTrapFile;
private TrapWriter? compilationTrapFile;
private readonly object progressMutex = new object();
@@ -240,6 +241,8 @@ namespace Semmle.Extraction.CSharp
var cx = new Context(extractor, compilation.Clone(), trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix);
compilationEntity = Entities.Compilation.Create(cx);
extractor.CompilationInfos?.ForEach(ci => trapWriter.Writer.compilation_info(compilationEntity, ci.key, ci.value));
}
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
{

4
csharp/extractor/Semmle.Extraction.CSharp/Extractor/StandaloneAnalyser.cs
View File

@@ -13,10 +13,10 @@ namespace Semmle.Extraction.CSharp
{
}
public void Initialize(string outputPath, CSharpCompilation compilationIn, CommonOptions options)
public void Initialize(string outputPath, IEnumerable<(string, string)> compilationInfos, CSharpCompilation compilationIn, CommonOptions options)
{
compilation = compilationIn;
extractor = new StandaloneExtractor(outputPath, Logger, PathTransformer, options);
extractor = new StandaloneExtractor(outputPath, compilationInfos, Logger, PathTransformer, options);
this.options = options;
LogExtractorInfo(Extraction.Extractor.Version);
SetReferencePaths();

3
csharp/extractor/Semmle.Extraction.CSharp/Tuples.cs
View File

@@ -71,6 +71,9 @@ namespace Semmle.Extraction.CSharp
internal static void compilation_expanded_args(this TextWriter trapFile, Compilation compilation, int index, string arg) =>
trapFile.WriteTuple("compilation_expanded_args", compilation, index, arg);
internal static void compilation_info(this TextWriter trapFile, Compilation compilation, string infoKey, string infoValue) =>
trapFile.WriteTuple("compilation_info", compilation, infoKey, infoValue);
internal static void compilation_compiling_files(this TextWriter trapFile, Compilation compilation, int index, Extraction.Entities.File file) =>
trapFile.WriteTuple("compilation_compiling_files", compilation, index, file);

6
csharp/extractor/Semmle.Extraction/Extractor/Extractor.cs
View File

@@ -1,6 +1,8 @@
using System.Collections.Generic;
using Semmle.Util.Logging;
using CompilationInfo = (string key, string value);
namespace Semmle.Extraction
{
/// <summary>
@@ -10,17 +12,19 @@ namespace Semmle.Extraction
{
public abstract ExtractorMode Mode { get; }
public string OutputPath { get; }
public IEnumerable<CompilationInfo> CompilationInfos { get; }
/// <summary>
/// Creates a new extractor instance for one compilation unit.
/// </summary>
/// <param name="logger">The object used for logging.</param>
/// <param name="pathTransformer">The object used for path transformations.</param>
protected Extractor(string outputPath, ILogger logger, PathTransformer pathTransformer)
protected Extractor(string outputPath, IEnumerable<CompilationInfo> compilationInfos, ILogger logger, PathTransformer pathTransformer)
{
OutputPath = outputPath;
Logger = logger;
PathTransformer = pathTransformer;
CompilationInfos = compilationInfos;
}
// Limit the number of error messages in the log file

3
csharp/extractor/Semmle.Extraction/Extractor/StandaloneExtractor.cs
View File

@@ -1,3 +1,4 @@
using System.Collections.Generic;
using Semmle.Util.Logging;
namespace Semmle.Extraction
@@ -11,7 +12,7 @@ namespace Semmle.Extraction
/// </summary>
/// <param name="logger">The object used for logging.</param>
/// <param name="pathTransformer">The object used for path transformations.</param>
public StandaloneExtractor(string outputPath, ILogger logger, PathTransformer pathTransformer, CommonOptions options) : base(outputPath, logger, pathTransformer)
public StandaloneExtractor(string outputPath, IEnumerable<(string, string)> compilationInfos, ILogger logger, PathTransformer pathTransformer, CommonOptions options) : base(outputPath, compilationInfos, logger, pathTransformer)
{
Mode = ExtractorMode.Standalone;
if (options.QlTest)

3
csharp/extractor/Semmle.Extraction/Extractor/TracingExtractor.cs
View File

@@ -1,3 +1,4 @@
using System.Linq;
using Semmle.Util.Logging;
namespace Semmle.Extraction
@@ -12,7 +13,7 @@ namespace Semmle.Extraction
/// <param name="outputPath">The name of the output DLL/EXE, or null if not specified (standalone extraction).</param>
/// <param name="logger">The object used for logging.</param>
/// <param name="pathTransformer">The object used for path transformations.</param>
public TracingExtractor(string outputPath, ILogger logger, PathTransformer pathTransformer, CommonOptions options) : base(outputPath, logger, pathTransformer)
public TracingExtractor(string outputPath, ILogger logger, PathTransformer pathTransformer, CommonOptions options) : base(outputPath, Enumerable.Empty<(string, string)>(), logger, pathTransformer)
{
Mode = ExtractorMode.None;
if (options.QlTest)

5
csharp/ql/lib/semmle/code/csharp/commons/Compilation.qll
View File

@@ -83,4 +83,9 @@ class Compilation extends @compilation {
/** Gets the elapsed seconds for the entire extractor process. */
float getElapsedSeconds() { compilation_finished(this, _, result) }
/**
* Gets the piece of compilation information with the given key, if any.
*/
string getInfo(string key) { compilation_info(this, key, result) }
}

6
csharp/ql/lib/semmlecode.csharp.dbscheme
View File

@@ -24,6 +24,12 @@ compilations(
string cwd : string ref
);
compilation_info(
int id : @compilation ref,
string info_key: string ref,
string info_value: string ref
)
/**
* The arguments that were passed to the extractor for a compiler
* invocation. If `id` is for the compiler invocation

13
csharp/ql/src/Telemetry/ExtractorInformation.ql
View File

@@ -9,6 +9,18 @@
import csharp
import semmle.code.csharp.commons.Diagnostics
predicate compilationInfo(string key, float value) {
exists(Compilation c, string infoKey, string infoValue | infoValue = c.getInfo(infoKey) |
exists(infoValue.toFloat()) and
key = infoKey and
value = infoValue.toFloat()
or
not exists(infoValue.toFloat()) and
key = infoKey + ": " + infoValue and
value = 1
)
}
predicate fileCount(string key, int value) {
key = "Number of files" and
value = strictcount(File f)
@@ -177,6 +189,7 @@ predicate analyzerAssemblies(string key, float value) {
from string key, float value
where
(
compilationInfo(key, value) or
fileCount(key, value) or
fileCountByExtension(key, value) or
totalNumberOfLines(key, value) or