Merge pull request #8203 from michaelnebel/csharp/extractor-option-buildless

C#: Refactoring - Move some of the standalone extractor code to the Standalone project.

csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs

@@ -0,0 +1,149 @@
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using Microsoft.CodeAnalysis;
+using Microsoft.CodeAnalysis.CSharp;
+using Semmle.Util;
+using Semmle.Util.Logging;
+
+namespace Semmle.Extraction.CSharp.Standalone
+{
+    public static class Extractor
+    {
+
+        private static IEnumerable<Action> GetResolvedReferencesStandalone(IEnumerable<string> referencePaths, BlockingCollection<MetadataReference> references)
+        {
+            return referencePaths.Select<string, Action>(path => () =>
+            {
+                var reference = MetadataReference.CreateFromFile(path);
+                references.Add(reference);
+            });
+        }
+
+        private static void AnalyseStandalone(
+            StandaloneAnalyser analyser,
+            IEnumerable<string> sources,
+            IEnumerable<string> referencePaths,
+            CommonOptions options,
+            IProgressMonitor progressMonitor,
+            Stopwatch stopwatch)
+        {
+            CSharp.Extractor.Analyse(stopwatch, analyser, options,
+                references => GetResolvedReferencesStandalone(referencePaths, references),
+                (analyser, syntaxTrees) => CSharp.Extractor.ReadSyntaxTrees(sources, analyser, null, null, syntaxTrees),
+                (syntaxTrees, references) => CSharpCompilation.Create("csharp.dll", syntaxTrees, references),
+                (compilation, options) => analyser.Initialize(compilation, options),
+                () => { },
+                _ => { },
+                () =>
+                {
+                    foreach (var type in analyser.MissingNamespaces)
+                    {
+                        progressMonitor.MissingNamespace(type);
+                    }
+
+                    foreach (var type in analyser.MissingTypes)
+                    {
+                        progressMonitor.MissingType(type);
+                    }
+
+                    progressMonitor.MissingSummary(analyser.MissingTypes.Count(), analyser.MissingNamespaces.Count());
+                });
+        }
+
+        private static void ExtractStandalone(
+            IEnumerable<string> sources,
+            IEnumerable<string> referencePaths,
+            IProgressMonitor pm,
+            ILogger logger,
+            CommonOptions options)
+        {
+            var stopwatch = new Stopwatch();
+            stopwatch.Start();
+
+            var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
+            var pathTransformer = new PathTransformer(canonicalPathCache);
+
+            using var analyser = new StandaloneAnalyser(pm, logger, false, pathTransformer);
+            try
+            {
+                AnalyseStandalone(analyser, sources, referencePaths, options, pm, stopwatch);
+            }
+            catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
+            {
+                analyser.Logger.Log(Severity.Error, "  Unhandled exception: {0}", ex);
+            }
+        }
+
+        private class ExtractionProgress : IProgressMonitor
+        {
+            public ExtractionProgress(ILogger output)
+            {
+                logger = output;
+            }
+
+            private readonly ILogger logger;
+
+            public void Analysed(int item, int total, string source, string output, TimeSpan time, AnalysisAction action)
+            {
+                logger.Log(Severity.Info, "[{0}/{1}] {2} ({3})", item, total, source,
+                    action == AnalysisAction.Extracted
+                        ? time.ToString()
+                        : action == AnalysisAction.Excluded
+                            ? "excluded"
+                            : "up to date");
+            }
+
+            public void MissingType(string type)
+            {
+                logger.Log(Severity.Debug, "Missing type {0}", type);
+            }
+
+            public void MissingNamespace(string @namespace)
+            {
+                logger.Log(Severity.Info, "Missing namespace {0}", @namespace);
+            }
+
+            public void MissingSummary(int missingTypes, int missingNamespaces)
+            {
+                logger.Log(Severity.Info, "Failed to resolve {0} types in {1} namespaces", missingTypes, missingNamespaces);
+            }
+        }
+
+        public static ExitCode Run(Options options)
+        {
+            var stopwatch = new Stopwatch();
+            stopwatch.Start();
+
+            using var logger = new ConsoleLogger(options.Verbosity);
+            logger.Log(Severity.Info, "Running C# standalone extractor");
+            using var a = new Analysis(logger, options);
+            var sourceFileCount = a.Extraction.Sources.Count;
+
+            if (sourceFileCount == 0)
+            {
+                logger.Log(Severity.Error, "No source files found");
+                return ExitCode.Errors;
+            }
+
+            if (!options.SkipExtraction)
+            {
+                using var fileLogger = CSharp.Extractor.MakeLogger(options.Verbosity, false);
+
+                logger.Log(Severity.Info, "");
+                logger.Log(Severity.Info, "Extracting...");
+                ExtractStandalone(
+                    a.Extraction.Sources,
+                    a.References,
+                    new ExtractionProgress(logger),
+                    fileLogger,
+                    options);
+                logger.Log(Severity.Info, $"Extraction completed in {stopwatch.Elapsed}");
+            }
+
+            return ExitCode.Ok;
+        }
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp.Standalone/Program.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using Semmle.BuildAnalyser;
 using Semmle.Util.Logging;
 
@@ -52,11 +53,10 @@ namespace Semmle.Extraction.CSharp.Standalone
     {
         public static int Main(string[] args)
         {
-            Extractor.SetInvariantCulture();
+            CSharp.Extractor.SetInvariantCulture();
 
             var options = Options.Create(args);
             // options.CIL = true;  // To do: Enable this
-            using var output = new ConsoleLogger(options.Verbosity);
 
             if (options.Help)
             {
@@ -67,69 +67,7 @@ namespace Semmle.Extraction.CSharp.Standalone
             if (options.Errors)
                 return 1;
 
-            var start = DateTime.Now;
-
-            output.Log(Severity.Info, "Running C# standalone extractor");
-            using var a = new Analysis(output, options);
-            var sourceFileCount = a.Extraction.Sources.Count;
-
-            if (sourceFileCount == 0)
-            {
-                output.Log(Severity.Error, "No source files found");
-                return 1;
-            }
-
-            if (!options.SkipExtraction)
-            {
-                using var fileLogger = new FileLogger(options.Verbosity, Extractor.GetCSharpLogPath());
-
-                output.Log(Severity.Info, "");
-                output.Log(Severity.Info, "Extracting...");
-                Extractor.ExtractStandalone(
-                    a.Extraction.Sources,
-                    a.References,
-                    new ExtractionProgress(output),
-                    fileLogger,
-                    options);
-                output.Log(Severity.Info, $"Extraction completed in {DateTime.Now - start}");
-            }
-
-            return 0;
-        }
-
-        private class ExtractionProgress : IProgressMonitor
-        {
-            public ExtractionProgress(ILogger output)
-            {
-                logger = output;
-            }
-
-            private readonly ILogger logger;
-
-            public void Analysed(int item, int total, string source, string output, TimeSpan time, AnalysisAction action)
-            {
-                logger.Log(Severity.Info, "[{0}/{1}] {2} ({3})", item, total, source,
-                    action == AnalysisAction.Extracted
-                        ? time.ToString()
-                        : action == AnalysisAction.Excluded
-                            ? "excluded"
-                            : "up to date");
-            }
-
-            public void MissingType(string type)
-            {
-                logger.Log(Severity.Debug, "Missing type {0}", type);
-            }
-
-            public void MissingNamespace(string @namespace)
-            {
-                logger.Log(Severity.Info, "Missing namespace {0}", @namespace);
-            }
-
-            public void MissingSummary(int missingTypes, int missingNamespaces)
-            {
-                logger.Log(Severity.Info, "Failed to resolve {0} types in {1} namespaces", missingTypes, missingNamespaces);
-            }
+            return (int)Extractor.Run(options);
         }
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.Standalone/StandaloneAnalyser.cs

@@ -11,7 +11,7 @@ namespace Semmle.Extraction.CSharp
         {
         }
 
-        public void InitializeStandalone(CSharpCompilation compilationIn, CommonOptions options)
+        public void Initialize(CSharpCompilation compilationIn, CommonOptions options)
         {
             compilation = compilationIn;
             layout = new Layout();

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs

@@ -16,15 +16,15 @@ using System.Threading;
 
 namespace Semmle.Extraction.CSharp
 {
+    public enum ExitCode
+    {
+        Ok,         // Everything worked perfectly
+        Errors,     // Trap was generated but there were processing errors
+        Failed      // Trap could not be generated
+    }
+
     public static class Extractor
     {
-        public enum ExitCode
-        {
-            Ok,         // Everything worked perfectly
-            Errors,     // Trap was generated but there were processing errors
-            Failed      // Trap could not be generated
-        }
-
         private class LogProgressMonitor : IProgressMonitor
         {
             private readonly ILogger logger;
@@ -69,6 +69,14 @@ namespace Semmle.Extraction.CSharp
             Thread.CurrentThread.CurrentUICulture = culture;
         }
 
+        public static ILogger MakeLogger(Verbosity verbosity, bool includeConsole)
+        {
+            var fileLogger = new FileLogger(verbosity, GetCSharpLogPath());
+            return includeConsole
+                ? new CombinedLogger(new ConsoleLogger(verbosity), fileLogger)
+                : (ILogger)fileLogger;
+        }
+
         /// <summary>
         /// Command-line driver for the extractor.
         /// </summary>
@@ -89,10 +97,7 @@ namespace Semmle.Extraction.CSharp
             var options = Options.CreateWithEnvironment(args);
             Entities.Compilation.Settings = (Directory.GetCurrentDirectory(), options.CompilerArguments.ToArray());
 
-            var fileLogger = new FileLogger(options.Verbosity, GetCSharpLogPath());
-            using var logger = options.Console
-                ? new CombinedLogger(new ConsoleLogger(options.Verbosity), fileLogger)
-                : (ILogger)fileLogger;
+            using var logger = MakeLogger(options.Verbosity, options.Console);
 
             if (Environment.GetEnvironmentVariable("SEMMLE_CLRTRACER") == "1" && !options.ClrTracer)
             {
@@ -276,7 +281,7 @@ namespace Semmle.Extraction.CSharp
         /// The constructed syntax trees will be added (thread-safely) to the supplied
         /// list <paramref name="ret"/>.
         /// </summary>
-        private static IEnumerable<Action> ReadSyntaxTrees(IEnumerable<string> sources, Analyser analyser, CSharpParseOptions? parseOptions, Encoding? encoding, IList<SyntaxTree> ret)
+        public static IEnumerable<Action> ReadSyntaxTrees(IEnumerable<string> sources, Analyser analyser, CSharpParseOptions? parseOptions, Encoding? encoding, IList<SyntaxTree> ret)
         {
             return sources.Select<string, Action>(path => () =>
             {
@@ -298,31 +303,7 @@ namespace Semmle.Extraction.CSharp
             });
         }
 
-        public static void ExtractStandalone(
-            IEnumerable<string> sources,
-            IEnumerable<string> referencePaths,
-            IProgressMonitor pm,
-            ILogger logger,
-            CommonOptions options)
-        {
-            var stopwatch = new Stopwatch();
-            stopwatch.Start();
-
-            var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
-            var pathTransformer = new PathTransformer(canonicalPathCache);
-
-            using var analyser = new StandaloneAnalyser(pm, logger, false, pathTransformer);
-            try
-            {
-                AnalyseStandalone(analyser, sources, referencePaths, options, pm, stopwatch);
-            }
-            catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
-            {
-                analyser.Logger.Log(Severity.Error, "  Unhandled exception: {0}", ex);
-            }
-        }
-
-        private static ExitCode Analyse(Stopwatch stopwatch, Analyser analyser, CommonOptions options,
+        public static ExitCode Analyse(Stopwatch stopwatch, Analyser analyser, CommonOptions options,
             Func<BlockingCollection<MetadataReference>, IEnumerable<Action>> getResolvedReferenceTasks,
             Func<Analyser, List<SyntaxTree>, IEnumerable<Action>> getSyntaxTreeTasks,
             Func<IEnumerable<SyntaxTree>, IEnumerable<MetadataReference>, CSharpCompilation> getCompilation,
@@ -395,37 +376,6 @@ namespace Semmle.Extraction.CSharp
             return analyser.TotalErrors == 0 ? ExitCode.Ok : ExitCode.Errors;
         }
 
-        private static void AnalyseStandalone(
-            StandaloneAnalyser analyser,
-            IEnumerable<string> sources,
-            IEnumerable<string> referencePaths,
-            CommonOptions options,
-            IProgressMonitor progressMonitor,
-            Stopwatch stopwatch)
-        {
-            Analyse(stopwatch, analyser, options,
-                references => GetResolvedReferencesStandalone(referencePaths, references),
-                (analyser, syntaxTrees) => ReadSyntaxTrees(sources, analyser, null, null, syntaxTrees),
-                (syntaxTrees, references) => CSharpCompilation.Create("csharp.dll", syntaxTrees, references),
-                (compilation, options) => analyser.InitializeStandalone(compilation, options),
-                () => { },
-                _ => { },
-                () =>
-                {
-                    foreach (var type in analyser.MissingNamespaces)
-                    {
-                        progressMonitor.MissingNamespace(type);
-                    }
-
-                    foreach (var type in analyser.MissingTypes)
-                    {
-                        progressMonitor.MissingType(type);
-                    }
-
-                    progressMonitor.MissingSummary(analyser.MissingTypes.Count(), analyser.MissingNamespaces.Count());
-                });
-        }
-
         private static ExitCode AnalyseTracing(
             TracingAnalyser analyser,
             CSharpCommandLineArguments compilerArguments,
@@ -468,15 +418,6 @@ namespace Semmle.Extraction.CSharp
                 () => { });
         }
 
-        private static IEnumerable<Action> GetResolvedReferencesStandalone(IEnumerable<string> referencePaths, BlockingCollection<MetadataReference> references)
-        {
-            return referencePaths.Select<string, Action>(path => () =>
-            {
-                var reference = MetadataReference.CreateFromFile(path);
-                references.Add(reference);
-            });
-        }
-
         /// <summary>
         /// Gets the path to the `csharp.log` file written to by the C# extractor.
         /// </summary>