using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.CSharp; using Semmle.Util; using Semmle.Util.Logging; using Semmle.Extraction.CSharp.Populators; using System.Reflection; namespace Semmle.Extraction.CSharp { /// /// Encapsulates a C# analysis task. /// public class Analyser : IDisposable { public ExtractionContext? ExtractionContext { get; protected set; } protected CSharpCompilation? compilation; protected CommonOptions? options; private protected Entities.Compilation? compilationEntity; private IDisposable? compilationTrapFile; // The bulk of the extraction work, potentially executed in parallel. protected readonly List extractionTasks = new List(); private int taskCount = 0; private readonly Stopwatch stopWatch = new Stopwatch(); private readonly IProgressMonitor progressMonitor; public ILogger Logger { get; } protected readonly bool addAssemblyTrapPrefix; public PathTransformer PathTransformer { get; } public IPathCache PathCache { get; } protected Analyser( IProgressMonitor pm, ILogger logger, PathTransformer pathTransformer, IPathCache pathCache, bool addAssemblyTrapPrefix) { Logger = logger; PathTransformer = pathTransformer; PathCache = pathCache; this.addAssemblyTrapPrefix = addAssemblyTrapPrefix; this.progressMonitor = pm; Logger.LogInfo($"EXTRACTION STARTING at {DateTime.Now}"); stopWatch.Start(); } /// /// Perform an analysis on a source file/syntax tree. /// /// Syntax tree to analyse. public void AnalyseTree(SyntaxTree tree) { extractionTasks.Add(() => DoExtractTree(tree)); } #nullable disable warnings /// /// Enqueue all reference analysis tasks. /// public void AnalyseReferences() { foreach (var assembly in compilation.References.OfType()) { extractionTasks.Add(() => DoAnalyseReferenceAssembly(assembly)); } } /// /// Constructs the map from assembly string to its filename. /// /// Roslyn doesn't record the relationship between a filename and its assembly /// information, so we need to retrieve this information manually. /// protected void SetReferencePaths() { foreach (var reference in compilation.References.OfType()) { try { var refPath = reference.FilePath!; /* This method is significantly faster and more lightweight than using * System.Reflection.Assembly.ReflectionOnlyLoadFrom. It is also allows * loading the same assembly from different locations. */ using var pereader = new System.Reflection.PortableExecutable.PEReader(new FileStream(refPath, FileMode.Open, FileAccess.Read, FileShare.Read)); var metadata = pereader.GetMetadata(); string assemblyIdentity; unsafe { var reader = new System.Reflection.Metadata.MetadataReader(metadata.Pointer, metadata.Length); var def = reader.GetAssemblyDefinition(); assemblyIdentity = $"{reader.GetString(def.Name)} {def.Version}"; } ExtractionContext.SetAssemblyFile(assemblyIdentity, refPath); } catch (Exception ex) // lgtm[cs/catch-of-all-exceptions] { ExtractionContext.Message(new Message("Exception reading reference file", reference.FilePath, null, ex.StackTrace)); } } } /// /// Extract an assembly to a new trap file. /// If the trap file exists, skip extraction to avoid duplicating /// extraction within the snapshot. /// /// The assembly to extract. private void DoAnalyseReferenceAssembly(PortableExecutableReference r) { try { var stopwatch = new Stopwatch(); stopwatch.Start(); var assemblyPath = r.FilePath!; var transformedAssemblyPath = PathTransformer.Transform(assemblyPath); using var trapWriter = transformedAssemblyPath.CreateTrapWriter(Logger, options.TrapCompression, discardDuplicates: true); var skipExtraction = options.Cache && File.Exists(trapWriter.TrapFile); var currentTaskId = IncrementTaskCount(); ReportProgressTaskStarted(currentTaskId, assemblyPath); if (!skipExtraction) { /* Note on parallel builds: * * The trap writer and source archiver both perform atomic moves * of the file to the final destination. * * If the same source file or trap file are generated concurrently * (by different parallel invocations of the extractor), then * last one wins. * * Specifically, if two assemblies are analysed concurrently in a build, * then there is a small amount of duplicated work but the output should * still be correct. */ if (compilation.GetAssemblyOrModuleSymbol(r) is IAssemblySymbol assembly) { var cx = new Context(ExtractionContext, compilation, trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix); foreach (var module in assembly.Modules) { AnalyseNamespace(cx, module.GlobalNamespace); } Entities.Attribute.ExtractAttributes(cx, assembly, Entities.Assembly.Create(cx, assembly.GetSymbolLocation())); cx.PopulateAll(); } } ReportProgressTaskDone(currentTaskId, assemblyPath, trapWriter.TrapFile, stopwatch.Elapsed, skipExtraction ? AnalysisAction.UpToDate : AnalysisAction.Extracted); } catch (Exception ex) // lgtm[cs/catch-of-all-exceptions] { Logger.LogError($" Unhandled exception analyzing {r.FilePath}: {ex}"); } } private void DoExtractTree(SyntaxTree tree) { try { var stopwatch = new Stopwatch(); stopwatch.Start(); var sourcePath = BinaryLogExtractionContext.GetAdjustedPath(ExtractionContext, tree.FilePath) ?? tree.FilePath; var transformedSourcePath = PathTransformer.Transform(sourcePath); var trapPath = transformedSourcePath.GetTrapPath(Logger, options.TrapCompression); using var trapWriter = transformedSourcePath.CreateTrapWriter(Logger, options.TrapCompression, discardDuplicates: false); var currentTaskId = IncrementTaskCount(); ReportProgressTaskStarted(currentTaskId, sourcePath); var cx = new Context(ExtractionContext, compilation, trapWriter, new SourceScope(tree), addAssemblyTrapPrefix); // Ensure that the file itself is populated in case the source file is totally empty var root = tree.GetRoot(); Entities.File.Create(cx, root.SyntaxTree.FilePath); var csNode = (CSharpSyntaxNode)root; var directiveVisitor = new DirectiveVisitor(cx); csNode.Accept(directiveVisitor); foreach (var branch in directiveVisitor.BranchesTaken) { cx.TrapStackSuffix.Add(branch); } csNode.Accept(new CompilationUnitVisitor(cx)); cx.PopulateAll(); CommentPopulator.ExtractCommentBlocks(cx, cx.CommentGenerator); cx.PopulateAll(); ReportProgressTaskDone(currentTaskId, sourcePath, trapPath, stopwatch.Elapsed, AnalysisAction.Extracted); } catch (Exception ex) // lgtm[cs/catch-of-all-exceptions] { ExtractionContext.Message(new Message($"Unhandled exception processing syntax tree. {ex.Message}", tree.FilePath, null, ex.StackTrace)); } } private void DoAnalyseCompilation() { try { var assemblyPath = ExtractionContext.OutputPath; var stopwatch = new Stopwatch(); stopwatch.Start(); var currentTaskId = IncrementTaskCount(); ReportProgressTaskStarted(currentTaskId, assemblyPath); var transformedAssemblyPath = PathTransformer.Transform(assemblyPath); var assembly = compilation.Assembly; var trapWriter = transformedAssemblyPath.CreateTrapWriter(Logger, options.TrapCompression, discardDuplicates: false); compilationTrapFile = trapWriter; // Dispose later var cx = new Context(ExtractionContext, compilation, trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix); compilationEntity = Entities.Compilation.Create(cx); ExtractionContext.CompilationInfos.ForEach(ci => trapWriter.Writer.compilation_info(compilationEntity, ci.key, ci.value)); ReportProgressTaskDone(currentTaskId, assemblyPath, trapWriter.TrapFile, stopwatch.Elapsed, AnalysisAction.Extracted); } catch (Exception ex) // lgtm[cs/catch-of-all-exceptions] { Logger.LogError($" Unhandled exception analyzing compilation: {ex}"); } } public void LogPerformance(Entities.PerformanceMetrics p) => compilationEntity.PopulatePerformance(p); public void ExtractAggregatedMessages() => compilationEntity.PopulateAggregatedMessages(); #nullable restore warnings /// /// Extracts compilation-wide entities, such as compilations and compiler diagnostics. /// public void AnalyseCompilation() { extractionTasks.Add(() => DoAnalyseCompilation()); } private static void AnalyseNamespace(Context cx, INamespaceSymbol ns) { foreach (var memberNamespace in ns.GetNamespaceMembers()) { AnalyseNamespace(cx, memberNamespace); } foreach (var memberType in ns.GetTypeMembers().ExtractionCandidates()) { Entities.Type.Create(cx, memberType).ExtractRecursive(); } } private int IncrementTaskCount() { return Interlocked.Increment(ref taskCount); } private void ReportProgressTaskStarted(int currentCount, string src) { progressMonitor.Started(currentCount, extractionTasks.Count, src); } private void ReportProgressTaskDone(int currentCount, string src, string output, TimeSpan time, AnalysisAction action) { progressMonitor.Analysed(currentCount, extractionTasks.Count, src, output, time, action); } /// /// Run all extraction tasks. /// /// The number of threads to use. public void PerformExtraction(int numberOfThreads) { Parallel.Invoke( new ParallelOptions { MaxDegreeOfParallelism = numberOfThreads }, extractionTasks.ToArray()); } public virtual void Dispose() { stopWatch.Stop(); Logger.LogInfo($" Peak working set = {Process.GetCurrentProcess().PeakWorkingSet64 / (1024 * 1024)} MB"); if (TotalErrors > 0) Logger.LogInfo($"EXTRACTION FAILED with {TotalErrors} error{(TotalErrors == 1 ? "" : "s")} in {stopWatch.Elapsed}"); else Logger.LogInfo($"EXTRACTION SUCCEEDED in {stopWatch.Elapsed}"); compilationTrapFile?.Dispose(); } /// /// Number of errors encountered during extraction. /// private int ExtractorErrors => ExtractionContext?.Errors ?? 0; /// /// Number of errors encountered by the compiler. /// public int CompilationErrors { get; set; } /// /// Total number of errors reported. /// public int TotalErrors => CompilationErrors + ExtractorErrors; /// /// Logs information about the extractor. /// public void LogExtractorInfo() { Logger.LogInfo($" Extractor: {Environment.GetCommandLineArgs()[0]}"); Logger.LogInfo($" Extractor version: {Version}"); Logger.LogInfo($" Current working directory: {Directory.GetCurrentDirectory()}"); } private static string Version { get { // the attribute for the git information are always attached to the entry assembly by our build system var assembly = Assembly.GetEntryAssembly(); var versionString = assembly?.GetCustomAttribute(); if (versionString == null) { return "unknown (not built from internal bazel workspace)"; } return versionString.InformationalVersion; } } } }