Merge pull request #11585 from jcogs33/jcogs33/mad-metrics-query

Java: add MaD metrics query

java/ql/src/Metrics/Summaries/GeneratedVsManualCoverage.ql

@@ -0,0 +1,76 @@
+/**
+ * @id java/summary/generated-vs-manual-coverage
+ * @name Metrics of generated versus manual MaD coverage
+ * @description Expose metrics for the number of API endpoints covered by generated versus manual MaD models.
+ * @kind table
+ * @tags summary
+ */
+
+import java
+import semmle.code.java.dataflow.FlowSummary
+import utils.modelgenerator.internal.CaptureModels
+
+/**
+ * Returns the number of `DataFlowTargetApi`s with Summary MaD models
+ * for a given package and provenance.
+ */
+bindingset[package]
+private int getNumMadModeledApis(string package, string provenance) {
+  provenance in ["generated", "manual", "both"] and
+  result =
+    count(SummarizedCallable sc |
+      package = sc.asCallable().getCompilationUnit().getPackage().getName() and
+      sc.asCallable() instanceof DataFlowTargetApi and
+      (
+        // "auto-only"
+        sc.isAutoGenerated() and
+        provenance = "generated"
+        or
+        // "manual-only"
+        sc.hasProvenance(false) and
+        not sc.hasProvenance(true) and
+        provenance = "manual"
+        or
+        // "both"
+        sc.hasProvenance(false) and
+        sc.hasProvenance(true) and
+        provenance = "both"
+      )
+    )
+}
+
+/** Returns the total number of `DataFlowTargetApi`s for a given package. */
+private int getNumApis(string package) {
+  result =
+    strictcount(DataFlowTargetApi dataFlowTargApi |
+      package = dataFlowTargApi.getCompilationUnit().getPackage().getName()
+    )
+}
+
+from
+  string package, int generatedOnly, int both, int manualOnly, int generated, int manual, int non,
+  int all, float coverage, float generatedCoverage, float manualCoverage,
+  float manualCoveredByGenerated, float generatedCoveredByManual, float match
+where
+  // count the number of APIs with generated-only, both, and manual-only MaD models for each package
+  generatedOnly = getNumMadModeledApis(package, "generated") and
+  both = getNumMadModeledApis(package, "both") and
+  manualOnly = getNumMadModeledApis(package, "manual") and
+  // calculate the total generated and total manual numbers
+  generated = generatedOnly + both and
+  manual = manualOnly + both and
+  // count the total number of `DataFlowTargetApi`s for each package
+  all = getNumApis(package) and
+  non = all - (generatedOnly + both + manualOnly) and
+  // Proportion of coverage
+  coverage = (generatedOnly + both + manualOnly).(float) / all and
+  generatedCoverage = generated.(float) / all and
+  manualCoverage = manual.(float) / all and
+  // Proportion of manual models covered by generated ones
+  manualCoveredByGenerated = both.(float) / (both + manualOnly) and
+  // Proportion of generated models covered by manual ones
+  generatedCoveredByManual = both.(float) / (both + generatedOnly) and
+  // Proportion of data points that match
+  match = (both.(float) + non) / all
+select package, generatedOnly, both, manualOnly, non, all, coverage, generatedCoverage,
+  manualCoverage, manualCoveredByGenerated, generatedCoveredByManual, match order by package

java/ql/src/change-notes/2022-12-12-java-mad-metrics-query.md

@@ -0,0 +1,4 @@
+---
+category: newQuery
+---
+* Added a new query, `java/summary/generated-vs-manual-coverage`, to expose metrics for the number of API endpoints covered by generated versus manual MaD models.