C#: Add aggregated compiler and extractor message counts to extraction telemetry query

csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/Compilation.cs

@@ -3,6 +3,7 @@ using System.Collections.Concurrent;
 using System.IO;
 using System.Linq;
 using Microsoft.CodeAnalysis;
+using Semmle.Extraction.Entities;
 using Semmle.Util;
 
 namespace Semmle.Extraction.CSharp.Entities
@@ -89,13 +90,21 @@ namespace Semmle.Extraction.CSharp.Entities
             trapFile.compilation_finished(this, (float)p.Total.Cpu.TotalSeconds, (float)p.Total.Elapsed.TotalSeconds);
         }
 
+        public void PopulateAggregatedMessages()
+        {
+            ExtractionMessage.groupedMessageCounts.ForEach(pair =>
+            {
+                Context.TrapWriter.Writer.compilation_info(this, $"Extractor message count for group '{pair.Key}'", pair.Value.ToString());
+            });
+        }
+
         public override void WriteId(EscapingTextWriter trapFile)
         {
             trapFile.Write(hashCode);
             trapFile.Write(";compilation");
         }
 
-        public override Location ReportingLocation => throw new NotImplementedException();
+        public override Microsoft.CodeAnalysis.Location ReportingLocation => throw new NotImplementedException();
 
         public override bool NeedsPopulation => Context.IsAssemblyScope;
 

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs

@@ -250,6 +250,8 @@ namespace Semmle.Extraction.CSharp
 
         public void LogPerformance(Entities.PerformanceMetrics p) => compilationEntity.PopulatePerformance(p);
 
+        public void ExtractAggregatedMessages() => compilationEntity.PopulateAggregatedMessages();
+
 #nullable restore warnings
 
         /// <summary>

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs

@@ -458,6 +458,7 @@ namespace Semmle.Extraction.CSharp
 
             sw.Restart();
             analyser.PerformExtraction(options.Threads);
+            analyser.ExtractAggregatedMessages();
             sw.Stop();
             var cpuTime2 = currentProcess.TotalProcessorTime;
             var userTime2 = currentProcess.UserProcessorTime;

csharp/extractor/Semmle.Extraction/BUILD.bazel

@@ -26,6 +26,7 @@ codeql_csharp_library(
         ],
         "//conditions:default": [],
     }),
+    internals_visible_to = ["Semmle.Extraction.CSharp"],
     visibility = ["//csharp:__subpackages__"],
     deps = [
         "//csharp/extractor/Semmle.Util",

csharp/extractor/Semmle.Extraction/Entities/ExtractionMessage.cs

@@ -1,4 +1,5 @@
-using System.IO;
+using System.Collections.Concurrent;
+using System.IO;
 using System.Threading;
 using Semmle.Util;
 
@@ -7,6 +8,8 @@ namespace Semmle.Extraction.Entities
     internal class ExtractionMessage : FreshEntity
     {
         private static readonly int limit = EnvironmentVariables.TryGetExtractorNumberOption<int>("MESSAGE_LIMIT") ?? 10000;
+
+        internal static readonly ConcurrentDictionary<string, int> groupedMessageCounts = [];
         private static int messageCount = 0;
 
         private readonly Message msg;
@@ -25,6 +28,10 @@ namespace Semmle.Extraction.Entities
 
         protected override void Populate(TextWriter trapFile)
         {
+            // For the time being we're counting the number of messages per severity, we could introduce other groupings in the future
+            var key = msg.Severity.ToString();
+            groupedMessageCounts.AddOrUpdate(key, 1, (_, c) => c + 1);
+
             if (!bypassLimit)
             {
                 var val = Interlocked.Increment(ref messageCount);

csharp/extractor/Semmle.Extraction/Semmle.Extraction.csproj

@@ -5,6 +5,8 @@
 	</PropertyGroup>
 	<ItemGroup>
 		<ProjectReference Include="..\Semmle.Util\Semmle.Util.csproj" />
+
+		<InternalsVisibleTo Include="Semmle.Extraction.CSharp" />
 	</ItemGroup>
   <Import Project="..\..\.paket\Paket.Restore.targets" />
 </Project>