mirror of
https://github.com/github/codeql.git
synced 2025-12-16 16:53:25 +01:00
382 lines
15 KiB
C#
382 lines
15 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.Diagnostics;
|
|
using System.IO;
|
|
using System.Linq;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
using Microsoft.CodeAnalysis;
|
|
using Microsoft.CodeAnalysis.CSharp;
|
|
using Semmle.Util;
|
|
using Semmle.Util.Logging;
|
|
using Semmle.Extraction.CSharp.Populators;
|
|
using System.Reflection;
|
|
|
|
namespace Semmle.Extraction.CSharp
|
|
{
|
|
/// <summary>
|
|
/// Encapsulates a C# analysis task.
|
|
/// </summary>
|
|
public class Analyser : IDisposable
|
|
{
|
|
public ExtractionContext? ExtractionContext { get; protected set; }
|
|
protected CSharpCompilation? compilation;
|
|
protected CommonOptions? options;
|
|
private protected Entities.Compilation? compilationEntity;
|
|
private IDisposable? compilationTrapFile;
|
|
|
|
// The bulk of the extraction work, potentially executed in parallel.
|
|
protected readonly List<Action> extractionTasks = new List<Action>();
|
|
private int taskCount = 0;
|
|
|
|
private readonly Stopwatch stopWatch = new Stopwatch();
|
|
|
|
private readonly IProgressMonitor progressMonitor;
|
|
|
|
public ILogger Logger { get; }
|
|
|
|
protected readonly bool addAssemblyTrapPrefix;
|
|
|
|
public PathTransformer PathTransformer { get; }
|
|
|
|
public IPathCache PathCache { get; }
|
|
|
|
public IOverlayInfo OverlayInfo { get; }
|
|
|
|
protected Analyser(
|
|
IProgressMonitor pm,
|
|
ILogger logger,
|
|
PathTransformer pathTransformer,
|
|
IPathCache pathCache,
|
|
IOverlayInfo overlayInfo,
|
|
bool addAssemblyTrapPrefix)
|
|
{
|
|
Logger = logger;
|
|
PathTransformer = pathTransformer;
|
|
PathCache = pathCache;
|
|
OverlayInfo = overlayInfo;
|
|
this.addAssemblyTrapPrefix = addAssemblyTrapPrefix;
|
|
this.progressMonitor = pm;
|
|
|
|
Logger.LogInfo($"EXTRACTION STARTING at {DateTime.Now}");
|
|
stopWatch.Start();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Perform an analysis on a source file/syntax tree.
|
|
/// </summary>
|
|
/// <param name="tree">Syntax tree to analyse.</param>
|
|
public void AnalyseTree(SyntaxTree tree)
|
|
{
|
|
extractionTasks.Add(() => DoExtractTree(tree));
|
|
}
|
|
|
|
#nullable disable warnings
|
|
|
|
/// <summary>
|
|
/// Enqueue all reference analysis tasks.
|
|
/// </summary>
|
|
public void AnalyseReferences()
|
|
{
|
|
foreach (var assembly in compilation.References.OfType<PortableExecutableReference>())
|
|
{
|
|
extractionTasks.Add(() => DoAnalyseReferenceAssembly(assembly));
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Constructs the map from assembly string to its filename.
|
|
///
|
|
/// Roslyn doesn't record the relationship between a filename and its assembly
|
|
/// information, so we need to retrieve this information manually.
|
|
/// </summary>
|
|
protected void SetReferencePaths()
|
|
{
|
|
foreach (var reference in compilation.References.OfType<PortableExecutableReference>())
|
|
{
|
|
try
|
|
{
|
|
var refPath = reference.FilePath!;
|
|
|
|
/* This method is significantly faster and more lightweight than using
|
|
* System.Reflection.Assembly.ReflectionOnlyLoadFrom. It is also allows
|
|
* loading the same assembly from different locations.
|
|
*/
|
|
using var pereader = new System.Reflection.PortableExecutable.PEReader(new FileStream(refPath, FileMode.Open, FileAccess.Read, FileShare.Read));
|
|
|
|
var metadata = pereader.GetMetadata();
|
|
string assemblyIdentity;
|
|
unsafe
|
|
{
|
|
var reader = new System.Reflection.Metadata.MetadataReader(metadata.Pointer, metadata.Length);
|
|
var def = reader.GetAssemblyDefinition();
|
|
assemblyIdentity = $"{reader.GetString(def.Name)} {def.Version}";
|
|
}
|
|
ExtractionContext.SetAssemblyFile(assemblyIdentity, refPath);
|
|
|
|
}
|
|
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
|
|
{
|
|
ExtractionContext.Message(new Message("Exception reading reference file", reference.FilePath, null, ex.StackTrace));
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Extract an assembly to a new trap file.
|
|
/// If the trap file exists, skip extraction to avoid duplicating
|
|
/// extraction within the snapshot.
|
|
/// </summary>
|
|
/// <param name="r">The assembly to extract.</param>
|
|
private void DoAnalyseReferenceAssembly(PortableExecutableReference r)
|
|
{
|
|
try
|
|
{
|
|
var stopwatch = new Stopwatch();
|
|
stopwatch.Start();
|
|
|
|
var assemblyPath = r.FilePath!;
|
|
var transformedAssemblyPath = PathTransformer.Transform(assemblyPath);
|
|
using var trapWriter = transformedAssemblyPath.CreateTrapWriter(Logger, options.TrapCompression, discardDuplicates: true);
|
|
|
|
var skipExtraction = options.Cache && File.Exists(trapWriter.TrapFile);
|
|
|
|
var currentTaskId = IncrementTaskCount();
|
|
ReportProgressTaskStarted(currentTaskId, assemblyPath);
|
|
|
|
if (!skipExtraction)
|
|
{
|
|
/* Note on parallel builds:
|
|
*
|
|
* The trap writer and source archiver both perform atomic moves
|
|
* of the file to the final destination.
|
|
*
|
|
* If the same source file or trap file are generated concurrently
|
|
* (by different parallel invocations of the extractor), then
|
|
* last one wins.
|
|
*
|
|
* Specifically, if two assemblies are analysed concurrently in a build,
|
|
* then there is a small amount of duplicated work but the output should
|
|
* still be correct.
|
|
*/
|
|
|
|
if (compilation.GetAssemblyOrModuleSymbol(r) is IAssemblySymbol assembly)
|
|
{
|
|
var cx = new Context(ExtractionContext, compilation, trapWriter, new AssemblyScope(assembly, assemblyPath), OverlayInfo, addAssemblyTrapPrefix);
|
|
|
|
foreach (var module in assembly.Modules)
|
|
{
|
|
AnalyseNamespace(cx, module.GlobalNamespace);
|
|
}
|
|
|
|
Entities.Attribute.ExtractAttributes(cx, assembly, Entities.Assembly.Create(cx, assembly.GetSymbolLocation()));
|
|
|
|
cx.PopulateAll();
|
|
}
|
|
}
|
|
|
|
ReportProgressTaskDone(currentTaskId, assemblyPath, trapWriter.TrapFile, stopwatch.Elapsed, skipExtraction ? AnalysisAction.UpToDate : AnalysisAction.Extracted);
|
|
}
|
|
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
|
|
{
|
|
Logger.LogError($" Unhandled exception analyzing {r.FilePath}: {ex}");
|
|
}
|
|
}
|
|
|
|
private void DoExtractTree(SyntaxTree tree)
|
|
{
|
|
try
|
|
{
|
|
var stopwatch = new Stopwatch();
|
|
stopwatch.Start();
|
|
var sourcePath = BinaryLogExtractionContext.GetAdjustedPath(ExtractionContext, tree.FilePath) ?? tree.FilePath;
|
|
|
|
var transformedSourcePath = PathTransformer.Transform(sourcePath);
|
|
|
|
var trapPath = transformedSourcePath.GetTrapPath(Logger, options.TrapCompression);
|
|
using var trapWriter = transformedSourcePath.CreateTrapWriter(Logger, options.TrapCompression, discardDuplicates: false);
|
|
|
|
var currentTaskId = IncrementTaskCount();
|
|
ReportProgressTaskStarted(currentTaskId, sourcePath);
|
|
|
|
var cx = new Context(ExtractionContext, compilation, trapWriter, new SourceScope(tree), OverlayInfo, addAssemblyTrapPrefix);
|
|
// Ensure that the file itself is populated in case the source file is totally empty
|
|
var root = tree.GetRoot();
|
|
Entities.File.Create(cx, root.SyntaxTree.FilePath);
|
|
|
|
var csNode = (CSharpSyntaxNode)root;
|
|
var directiveVisitor = new DirectiveVisitor(cx);
|
|
csNode.Accept(directiveVisitor);
|
|
foreach (var branch in directiveVisitor.BranchesTaken)
|
|
{
|
|
cx.TrapStackSuffix.Add(branch);
|
|
}
|
|
csNode.Accept(new CompilationUnitVisitor(cx));
|
|
cx.PopulateAll();
|
|
CommentPopulator.ExtractCommentBlocks(cx, cx.CommentGenerator);
|
|
cx.PopulateAll();
|
|
|
|
ReportProgressTaskDone(currentTaskId, sourcePath, trapPath, stopwatch.Elapsed, AnalysisAction.Extracted);
|
|
}
|
|
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
|
|
{
|
|
ExtractionContext.Message(new Message($"Unhandled exception processing syntax tree. {ex.Message}", tree.FilePath, null, ex.StackTrace));
|
|
}
|
|
}
|
|
|
|
private void DoAnalyseCompilation()
|
|
{
|
|
try
|
|
{
|
|
var assemblyPath = ExtractionContext.OutputPath;
|
|
var stopwatch = new Stopwatch();
|
|
stopwatch.Start();
|
|
var currentTaskId = IncrementTaskCount();
|
|
ReportProgressTaskStarted(currentTaskId, assemblyPath);
|
|
|
|
var transformedAssemblyPath = PathTransformer.Transform(assemblyPath);
|
|
var assembly = compilation.Assembly;
|
|
var trapWriter = transformedAssemblyPath.CreateTrapWriter(Logger, options.TrapCompression, discardDuplicates: false);
|
|
compilationTrapFile = trapWriter; // Dispose later
|
|
var cx = new Context(ExtractionContext, compilation, trapWriter, new AssemblyScope(assembly, assemblyPath), OverlayInfo, addAssemblyTrapPrefix);
|
|
|
|
compilationEntity = Entities.Compilation.Create(cx);
|
|
|
|
// Ensure that the empty location is always created.
|
|
Entities.EmptyLocation.Create(cx);
|
|
|
|
ExtractionContext.CompilationInfos.ForEach(ci => trapWriter.Writer.compilation_info(compilationEntity, ci.key, ci.value));
|
|
|
|
ReportProgressTaskDone(currentTaskId, assemblyPath, trapWriter.TrapFile, stopwatch.Elapsed, AnalysisAction.Extracted);
|
|
}
|
|
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
|
|
{
|
|
Logger.LogError($" Unhandled exception analyzing compilation: {ex}");
|
|
}
|
|
}
|
|
|
|
public void LogPerformance(Entities.PerformanceMetrics p) => compilationEntity.PopulatePerformance(p);
|
|
|
|
public void ExtractAggregatedMessages() => compilationEntity.PopulateAggregatedMessages();
|
|
|
|
#nullable restore warnings
|
|
|
|
/// <summary>
|
|
/// Extracts compilation-wide entities, such as compilations and compiler diagnostics.
|
|
/// </summary>
|
|
public void AnalyseCompilation()
|
|
{
|
|
extractionTasks.Add(() => DoAnalyseCompilation());
|
|
}
|
|
|
|
private static void AnalyseNamespace(Context cx, INamespaceSymbol ns)
|
|
{
|
|
foreach (var memberNamespace in ns.GetNamespaceMembers())
|
|
{
|
|
AnalyseNamespace(cx, memberNamespace);
|
|
}
|
|
|
|
foreach (var memberType in ns.GetTypeMembers().ExtractionCandidates())
|
|
{
|
|
Entities.Type.Create(cx, memberType).ExtractRecursive();
|
|
}
|
|
}
|
|
|
|
private int IncrementTaskCount()
|
|
{
|
|
return Interlocked.Increment(ref taskCount);
|
|
}
|
|
|
|
private void ReportProgressTaskStarted(int currentCount, string src)
|
|
{
|
|
progressMonitor.Started(currentCount, extractionTasks.Count, src);
|
|
}
|
|
|
|
private void ReportProgressTaskDone(int currentCount, string src, string output, TimeSpan time, AnalysisAction action)
|
|
{
|
|
progressMonitor.Analysed(currentCount, extractionTasks.Count, src, output, time, action);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Run all extraction tasks.
|
|
/// </summary>
|
|
/// <param name="numberOfThreads">The number of threads to use.</param>
|
|
public void PerformExtraction(int numberOfThreads)
|
|
{
|
|
Parallel.Invoke(
|
|
new ParallelOptions { MaxDegreeOfParallelism = numberOfThreads },
|
|
extractionTasks.ToArray());
|
|
}
|
|
|
|
public virtual void Dispose()
|
|
{
|
|
stopWatch.Stop();
|
|
Logger.LogInfo($" Peak working set = {Process.GetCurrentProcess().PeakWorkingSet64 / (1024 * 1024)} MB");
|
|
|
|
if (TotalErrors > 0)
|
|
Logger.LogInfo($"EXTRACTION FAILED with {TotalErrors} error{(TotalErrors == 1 ? "" : "s")} in {stopWatch.Elapsed}");
|
|
else
|
|
Logger.LogInfo($"EXTRACTION SUCCEEDED in {stopWatch.Elapsed}");
|
|
|
|
compilationTrapFile?.Dispose();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Number of errors encountered during extraction.
|
|
/// </summary>
|
|
private int ExtractorErrors => ExtractionContext?.Errors ?? 0;
|
|
|
|
/// <summary>
|
|
/// Number of errors encountered by the compiler.
|
|
/// </summary>
|
|
public int CompilationErrors { get; set; }
|
|
|
|
/// <summary>
|
|
/// Total number of errors reported.
|
|
/// </summary>
|
|
public int TotalErrors => CompilationErrors + ExtractorErrors;
|
|
|
|
/// <summary>
|
|
/// Logs information about the extractor.
|
|
/// </summary>
|
|
public void LogExtractorInfo()
|
|
{
|
|
Logger.LogInfo($" Extractor: {Environment.GetCommandLineArgs()[0]}");
|
|
Logger.LogInfo($" Extractor version: {Version}");
|
|
Logger.LogInfo($" Current working directory: {Directory.GetCurrentDirectory()}");
|
|
}
|
|
|
|
private static string Version
|
|
{
|
|
get
|
|
{
|
|
// the attribute for the git information are always attached to the entry assembly by our build system
|
|
var assembly = Assembly.GetEntryAssembly();
|
|
var versionString = assembly?.GetCustomAttribute<AssemblyInformationalVersionAttribute>();
|
|
if (versionString == null)
|
|
{
|
|
return "unknown (not built from internal bazel workspace)";
|
|
}
|
|
return versionString.InformationalVersion;
|
|
}
|
|
}
|
|
|
|
private static readonly HashSet<string> errorsToIgnore = new HashSet<string>
|
|
{
|
|
"CS7027", // Code signing failure
|
|
"CS1589", // XML referencing not supported
|
|
"CS1569" // Error writing XML documentation
|
|
};
|
|
|
|
/// <summary>
|
|
/// Retrieves the diagnostics from the compilation, filtering out those that should be ignored.
|
|
/// </summary>
|
|
protected List<Diagnostic> GetFilteredDiagnostics() =>
|
|
compilation is not null
|
|
? compilation.GetDiagnostics()
|
|
.Where(e => e.Severity >= DiagnosticSeverity.Error && !errorsToIgnore.Contains(e.Id))
|
|
.ToList()
|
|
: [];
|
|
}
|
|
}
|