Merge pull request #15827 from tamasvajk/buildless/impr-progress-reporting

C#: Improve buildless progress reporting

csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs

@@ -60,10 +60,9 @@ namespace Semmle.Extraction.CSharp.Standalone
             {
                 try
                 {
-                    FileUtils.TryDelete(output.FullName);
                     if (shouldCleanUpContainingFolder)
                     {
-                        output.Directory?.Delete(true);
+                        FileUtils.TryDelete(output.FullName);
                     }
                 }
                 catch
@@ -105,12 +104,19 @@ namespace Semmle.Extraction.CSharp.Standalone
 
             public void Analysed(int item, int total, string source, string output, TimeSpan time, AnalysisAction action)
             {
-                logger.Log(Severity.Info, "[{0}/{1}] {2} ({3})", item, total, source,
-                    action == AnalysisAction.Extracted
-                        ? time.ToString()
-                        : action == AnalysisAction.Excluded
-                            ? "excluded"
-                            : "up to date");
+                var extra = action switch
+                {
+                    AnalysisAction.Extracted => time.ToString(),
+                    AnalysisAction.Excluded => "excluded",
+                    AnalysisAction.UpToDate => "up to date",
+                    _ => "unknown action"
+                };
+                logger.LogInfo($"[{item}/{total}] {source} ({extra})");
+            }
+
+            public void Started(int item, int total, string source)
+            {
+                logger.LogInfo($"[{item}/{total}] {source} (processing started)");
             }
 
             public void MissingType(string type)

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
+using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.CSharp;
@@ -23,8 +24,6 @@ namespace Semmle.Extraction.CSharp
         private protected Entities.Compilation? compilationEntity;
         private IDisposable? compilationTrapFile;
 
-        private readonly object progressMutex = new object();
-
         // The bulk of the extraction work, potentially executed in parallel.
         protected readonly List<Action> extractionTasks = new List<Action>();
         private int taskCount = 0;
@@ -131,6 +130,9 @@ namespace Semmle.Extraction.CSharp
 
                 var skipExtraction = options.Cache && File.Exists(trapWriter.TrapFile);
 
+                var currentTaskId = IncrementTaskCount();
+                ReportProgressTaskStarted(currentTaskId, assemblyPath);
+
                 if (!skipExtraction)
                 {
                     /* Note on parallel builds:
@@ -167,7 +169,7 @@ namespace Semmle.Extraction.CSharp
                     }
                 }
 
-                ReportProgress(assemblyPath, trapWriter.TrapFile, stopwatch.Elapsed, skipExtraction ? AnalysisAction.UpToDate : AnalysisAction.Extracted);
+                ReportProgressTaskDone(currentTaskId, assemblyPath, trapWriter.TrapFile, stopwatch.Elapsed, skipExtraction ? AnalysisAction.UpToDate : AnalysisAction.Extracted);
             }
             catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
             {
@@ -177,11 +179,13 @@ namespace Semmle.Extraction.CSharp
 
         private void DoExtractCIL(PortableExecutableReference r)
         {
+            var currentTaskId = IncrementTaskCount();
+            ReportProgressTaskStarted(currentTaskId, r.FilePath);
             var stopwatch = new Stopwatch();
             stopwatch.Start();
             CIL.Analyser.ExtractCIL(r.FilePath!, Logger, options, out var trapFile, out var extracted);
             stopwatch.Stop();
-            ReportProgress(r.FilePath, trapFile, stopwatch.Elapsed, extracted ? AnalysisAction.Extracted : AnalysisAction.UpToDate);
+            ReportProgressTaskDone(currentTaskId, r.FilePath, trapFile, stopwatch.Elapsed, extracted ? AnalysisAction.Extracted : AnalysisAction.UpToDate);
         }
 
         private void DoExtractTree(SyntaxTree tree)
@@ -201,6 +205,9 @@ namespace Semmle.Extraction.CSharp
 
                 upToDate = options.Fast && FileIsUpToDate(sourcePath, trapWriter.TrapFile);
 
+                var currentTaskId = IncrementTaskCount();
+                ReportProgressTaskStarted(currentTaskId, sourcePath);
+
                 if (!upToDate)
                 {
                     var cx = new Context(extractor, compilation.Clone(), trapWriter, new SourceScope(tree), addAssemblyTrapPrefix);
@@ -221,7 +228,7 @@ namespace Semmle.Extraction.CSharp
                     cx.PopulateAll();
                 }
 
-                ReportProgress(sourcePath, trapPath, stopwatch.Elapsed, upToDate ? AnalysisAction.UpToDate : AnalysisAction.Extracted);
+                ReportProgressTaskDone(currentTaskId, sourcePath, trapPath, stopwatch.Elapsed, upToDate ? AnalysisAction.UpToDate : AnalysisAction.Extracted);
             }
             catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
             {
@@ -234,6 +241,11 @@ namespace Semmle.Extraction.CSharp
             try
             {
                 var assemblyPath = extractor.OutputPath;
+                var stopwatch = new Stopwatch();
+                stopwatch.Start();
+                var currentTaskId = IncrementTaskCount();
+                ReportProgressTaskStarted(currentTaskId, assemblyPath);
+
                 var transformedAssemblyPath = PathTransformer.Transform(assemblyPath);
                 var assembly = compilation.Assembly;
                 var trapWriter = transformedAssemblyPath.CreateTrapWriter(Logger, options.TrapCompression, discardDuplicates: false);
@@ -243,6 +255,8 @@ namespace Semmle.Extraction.CSharp
                 compilationEntity = Entities.Compilation.Create(cx);
 
                 extractor.CompilationInfos.ForEach(ci => trapWriter.Writer.compilation_info(compilationEntity, ci.key, ci.value));
+
+                ReportProgressTaskDone(currentTaskId, assemblyPath, trapWriter.TrapFile, stopwatch.Elapsed, AnalysisAction.Extracted);
             }
             catch (Exception ex)  // lgtm[cs/catch-of-all-exceptions]
             {
@@ -279,10 +293,19 @@ namespace Semmle.Extraction.CSharp
             }
         }
 
-        private void ReportProgress(string src, string output, TimeSpan time, AnalysisAction action)
+        private int IncrementTaskCount()
         {
-            lock (progressMutex)
-                progressMonitor.Analysed(++taskCount, extractionTasks.Count, src, output, time, action);
+            return Interlocked.Increment(ref taskCount);
+        }
+
+        private void ReportProgressTaskStarted(int currentCount, string src)
+        {
+            progressMonitor.Started(currentCount, extractionTasks.Count, src);
+        }
+
+        private void ReportProgressTaskDone(int currentCount, string src, string output, TimeSpan time, AnalysisAction action)
+        {
+            progressMonitor.Analysed(currentCount, extractionTasks.Count, src, output, time, action);
         }
 
         /// <summary>

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs

@@ -47,6 +47,8 @@ namespace Semmle.Extraction.CSharp
                 }
             }
 
+            public void Started(int item, int total, string source) { }
+
             public void MissingNamespace(string @namespace) { }
 
             public void MissingSummary(int types, int namespaces) { }

csharp/extractor/Semmle.Extraction.CSharp/Extractor/IProgressMonitor.cs

@@ -19,6 +19,11 @@ namespace Semmle.Extraction.CSharp
         /// <param name="action">What action was taken for the file.</param>
         void Analysed(int item, int total, string source, string output, TimeSpan time, AnalysisAction action);
 
+        /// <summary>
+        /// Callback that processing of a particular item has been started.
+        /// </summary>
+        void Started(int item, int total, string source);
+
         /// <summary>
         /// A "using namespace" directive was seen but the given
         /// namespace could not be found.