using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Linq; using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.CSharp; using Semmle.Extraction.CSharp.DependencyFetching; using Semmle.Util; using Semmle.Util.Logging; namespace Semmle.Extraction.CSharp.Standalone { public static class Extractor { private static IEnumerable GetResolvedReferencesStandalone(IEnumerable referencePaths, BlockingCollection references) { return referencePaths.Select(path => () => { var reference = MetadataReference.CreateFromFile(path); references.Add(reference); }); } private static void AnalyseStandalone( StandaloneAnalyser analyser, ExtractionInput extractionInput, CommonOptions options, IProgressMonitor progressMonitor, Stopwatch stopwatch) { var output = FileUtils.CreateTemporaryFile(".dll", out var shouldCleanUpContainingFolder); try { CSharp.Extractor.Analyse(stopwatch, analyser, options, references => GetResolvedReferencesStandalone(extractionInput.References, references), (analyser, syntaxTrees) => CSharp.Extractor.ReadSyntaxTrees(extractionInput.Sources, analyser, null, null, syntaxTrees), (syntaxTrees, references) => CSharpCompilation.Create( output.Name, syntaxTrees, references, new CSharpCompilationOptions(OutputKind.ConsoleApplication, allowUnsafe: true) ), (compilation, options) => analyser.Initialize(output.FullName, extractionInput.CompilationInfos, compilation, options), () => { foreach (var type in analyser.ExtractionContext!.MissingNamespaces) { progressMonitor.MissingNamespace(type); } foreach (var type in analyser.ExtractionContext!.MissingTypes) { progressMonitor.MissingType(type); } progressMonitor.MissingSummary(analyser.ExtractionContext!.MissingTypes.Count(), analyser.ExtractionContext!.MissingNamespaces.Count()); // If extracting a base database, we need to create an empty metadata file. if (EnvironmentVariables.GetBaseMetaDataOutPath() is string baseMetaDataOutPath) { try { analyser.Logger.LogInfo($"Creating base metadata file at {baseMetaDataOutPath}"); File.WriteAllText(baseMetaDataOutPath, string.Empty); } catch (Exception ex) { analyser.Logger.LogError($"Failed to create base metadata file: {ex.Message}"); } } }); } finally { try { if (shouldCleanUpContainingFolder) { FileUtils.TryDelete(output.FullName); } } catch { } } } private class ExtractionProgress : IProgressMonitor { public ExtractionProgress(ILogger output) { logger = output; } private readonly ILogger logger; public void Analysed(int item, int total, string source, string output, TimeSpan time, AnalysisAction action) { var extra = action switch { AnalysisAction.Extracted => time.ToString(), AnalysisAction.Excluded => "excluded", AnalysisAction.UpToDate => "up to date", _ => "unknown action" }; logger.LogDebug($"[{item}/{total}] {source} ({extra})"); } public void Started(int item, int total, string source) { logger.LogDebug($"[{item}/{total}] {source} (processing started)"); } public void MissingType(string type) { logger.LogDebug($"Missing type {type}"); } public void MissingNamespace(string @namespace) { logger.LogInfo($"Missing namespace {@namespace}"); } public void MissingSummary(int missingTypes, int missingNamespaces) { if (missingTypes > 0 || missingNamespaces > 0) { logger.LogInfo($"Failed to resolve {missingTypes} types in {missingNamespaces} namespaces"); } } } public record ExtractionInput(IEnumerable Sources, IEnumerable References, IEnumerable<(string, string)> CompilationInfos); public static ExitCode Run(Options options) { var overallStopwatch = new Stopwatch(); overallStopwatch.Start(); using var logger = new ConsoleLogger(options.Verbosity, logThreadId: true); logger.Log(Severity.Info, "Extracting C# with build-mode set to 'none'"); using var dependencyManager = new DependencyManager(options.SrcDir, logger); if (!dependencyManager.NonGeneratedSourcesFiles.Any()) { logger.Log(Severity.Error, "No source files found"); return ExitCode.Errors; } using var fileLogger = CSharp.Extractor.MakeLogger(options.Verbosity, false); logger.Log(Severity.Info, ""); logger.Log(Severity.Info, "Extracting..."); var analyzerStopwatch = new Stopwatch(); analyzerStopwatch.Start(); var canonicalPathCache = CanonicalPathCache.Create(fileLogger, 1000); var pathTransformer = new PathTransformer(canonicalPathCache); var progressMonitor = new ExtractionProgress(logger); var overlayInfo = OverlayInfoFactory.Make(logger, options.SrcDir); using var analyser = new StandaloneAnalyser(progressMonitor, fileLogger, pathTransformer, canonicalPathCache, overlayInfo, false); try { var extractionInput = new ExtractionInput(dependencyManager.AllSourceFiles, dependencyManager.ReferenceFiles, dependencyManager.CompilationInfos); AnalyseStandalone(analyser, extractionInput, options, progressMonitor, analyzerStopwatch); } catch (Exception ex) // lgtm[cs/catch-of-all-exceptions] { fileLogger.LogError($" Unhandled exception: {ex}"); } logger.Log(Severity.Info, $"Extraction completed in {analyzerStopwatch.Elapsed}"); logger.Log(Severity.Info, $"Total time: {overallStopwatch.Elapsed}"); return ExitCode.Ok; } } }