using Microsoft.CodeAnalysis.Text; using Semmle.Util.Logging; using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Linq; using System.Management.Automation.Language; using System.Net.Http; using System.Threading.Tasks; using Semmle.Extraction.PowerShell.Entities; using File = System.IO.File; /* * Test */ namespace Semmle.Extraction.PowerShell { /// /// Encapsulates a PowerShell analysis task. /// public class Analyser : IDisposable { protected Extraction.Extractor? extractor; protected Layout? layout; protected CommonOptions? options; private readonly object progressMutex = new object(); // The bulk of the extraction work, potentially executed in parallel. protected readonly List extractionTasks = new List(); private int taskCount = 0; private readonly Stopwatch stopWatch = new Stopwatch(); private readonly IProgressMonitor progressMonitor; public ILogger Logger { get; } protected readonly bool addAssemblyTrapPrefix; public PathTransformer PathTransformer { get; } public Analyser(IProgressMonitor pm, ILogger logger, bool addAssemblyTrapPrefix, PathTransformer pathTransformer, CommonOptions options) { Logger = logger; this.addAssemblyTrapPrefix = addAssemblyTrapPrefix; Logger.Log(Severity.Info, "EXTRACTION STARTING at {0}", DateTime.Now); stopWatch.Start(); progressMonitor = pm; PathTransformer = pathTransformer; extractor = new StandaloneExtractor(Logger, PathTransformer); this.options = options; layout = new Layout(); LogExtractorInfo(Extraction.Extractor.Version); } /// /// Perform an analysis on a source file/syntax tree. /// /// Syntax tree to analyse. public void QueueAnalyzeScriptTask(CompiledScript script) { extractionTasks.Add(() => DoExtractScript(script)); } #nullable disable warnings private Microsoft.CodeAnalysis.Location GetCodeAnalysisLocationForToken(string path, Token token) { return Microsoft.CodeAnalysis.Location.Create(path, new TextSpan(token.Extent.StartOffset, token.Extent.EndOffset - token.Extent.StartOffset), new LinePositionSpan(new LinePosition(token.Extent.StartLineNumber, token.Extent.StartColumnNumber), new LinePosition(token.Extent.EndLineNumber, token.Extent.EndColumnNumber))); } private void DoExtractScript(CompiledScript script) { try { Stopwatch stopwatch = new Stopwatch(); stopwatch.Start(); string sourcePath = script.Location; PathTransformer.ITransformedPath transformedSourcePath = PathTransformer.Transform(sourcePath); Layout.SubProject projectLayout = layout.LookupProjectOrNull(transformedSourcePath); bool excluded = projectLayout is null; string trapPath = excluded ? "" : projectLayout!.GetTrapPath(Logger, transformedSourcePath, options.TrapCompression); bool upToDate = false; if (!excluded) { using TrapWriter trapWriter = projectLayout!.CreateTrapWriter(Logger, transformedSourcePath, options.TrapCompression, discardDuplicates: false); upToDate = options.Fast && FileIsUpToDate(sourcePath, trapWriter.TrapFile); if (!upToDate) { PowerShellContext cx = new PowerShellContext(extractor, script, trapWriter, addAssemblyTrapPrefix); // Ensure that the file itself is populated in case the source file is totally empty Entities.File.Create(cx, sourcePath); // Parse any comments foreach (var token in script.Tokens.Where(x => x.Kind == TokenKind.Comment)) { CommentEntity.Create(cx, token); } // Parse the AST contained in script.ParseResult script.ParseResult.Visit(new PowerShellVisitor2(cx)); cx.PopulateAll(); } } ReportProgress(sourcePath, trapPath, stopwatch.Elapsed, excluded ? AnalysisAction.Excluded : upToDate ? AnalysisAction.UpToDate : AnalysisAction.Extracted); } catch (Exception ex) // lgtm[cs/catch-of-all-exceptions] { extractor.Message(new Message($"Unhandled exception processing syntax tree. {ex.Message}", script.Location, null, ex.StackTrace)); } } #nullable restore warnings private static bool FileIsUpToDate(string src, string dest) { return File.Exists(dest) && File.GetLastWriteTime(dest) >= File.GetLastWriteTime(src); } private void ReportProgress(string src, string output, TimeSpan time, AnalysisAction action) { lock (progressMutex) progressMonitor.Analysed(++taskCount, extractionTasks.Count, src, output, time, action); } /// /// Run all extraction tasks. /// /// The number of threads to use. public void PerformExtractionTasks(int numberOfThreads) { Parallel.Invoke( new ParallelOptions { MaxDegreeOfParallelism = numberOfThreads }, extractionTasks.ToArray()); } public virtual void Dispose() { stopWatch.Stop(); Logger.Log(Severity.Info, " Peak working set = {0} MB", Process.GetCurrentProcess().PeakWorkingSet64 / (1024 * 1024)); if (TotalErrors > 0) Logger.Log(Severity.Info, "EXTRACTION FAILED with {0} error{1} in {2}", TotalErrors, TotalErrors == 1 ? "" : "s", stopWatch.Elapsed); else Logger.Log(Severity.Info, "EXTRACTION SUCCEEDED in {0}", stopWatch.Elapsed); Logger.Dispose(); } /// /// Number of errors encountered during extraction. /// private int ExtractorErrors => extractor?.Errors ?? 0; /// /// Number of errors encountered by the compiler. /// public int CompilationErrors { get; set; } /// /// Total number of errors reported. /// public int TotalErrors => CompilationErrors + ExtractorErrors; /// /// Logs information about the extractor. /// public void LogExtractorInfo(string extractorVersion) { Logger.Log(Severity.Info, " Extractor: {0}", Environment.GetCommandLineArgs().First()); Logger.Log(Severity.Info, " Extractor version: {0}", extractorVersion); Logger.Log(Severity.Info, " Current working directory: {0}", Directory.GetCurrentDirectory()); } } }