Merge pull request #16810 from smowton/smowton/feature/java-low-db-quality-query

Java: add diagnostic query indicating low database quality

.github/workflows/compile-queries.yml

@@ -29,6 +29,8 @@ jobs:
           key: all-queries
       - name: check formatting
         run: find shared */ql -type f \( -name "*.qll" -o -name "*.ql" \) -print0 | xargs -0 -n 3000 -P 10 codeql query format -q --check-only
+      - name: Omit DatabaseQualityDiagnostics.ql from compile checking # Remove me once CodeQL 2.18.0 is released!
+        run: mv java/ql/src/Telemetry/DatabaseQualityDiagnostics.ql{,.hidden}
       - name: compile queries - check-only
         # run with --check-only if running in a PR (github.sha != main)
         if : ${{ github.event_name == 'pull_request' }}
@@ -39,3 +41,6 @@ jobs:
         if : ${{ github.event_name != 'pull_request' }}
         shell: bash
         run: codeql query compile -q -j0 */ql/{src,examples} --keep-going --warnings=error --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" --compilation-cache-size=500
+      - name: Restore DatabaseQualityDiagnostics.ql after compile checking # Remove me once CodeQL 2.18.0 is released
+        run: mv java/ql/src/Telemetry/DatabaseQualityDiagnostics.ql{.hidden,}
+

java/ql/integration-tests/all-platforms/java/buildless-erroneous/DatabaseQualityDiagnostics.expected

@@ -0,0 +1,6 @@
+diagnosticAttributes
+| Scanning Java code completed successfully, but the scan encountered issues. This may be caused by problems identifying dependencies or use of generated source code, among other reasons -- see other CodeQL diagnostics reported on the CodeQL status page for more details of possible causes. Addressing these warnings is advisable to avoid false-positive or missing results. If they cannot be addressed, consider scanning Java using either the `autobuild` or `manual` [build modes](https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages#comparison-of-the-build-modes). | visibilityCliSummaryTable | true |
+| Scanning Java code completed successfully, but the scan encountered issues. This may be caused by problems identifying dependencies or use of generated source code, among other reasons -- see other CodeQL diagnostics reported on the CodeQL status page for more details of possible causes. Addressing these warnings is advisable to avoid false-positive or missing results. If they cannot be addressed, consider scanning Java using either the `autobuild` or `manual` [build modes](https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages#comparison-of-the-build-modes). | visibilityStatusPage | true |
+| Scanning Java code completed successfully, but the scan encountered issues. This may be caused by problems identifying dependencies or use of generated source code, among other reasons -- see other CodeQL diagnostics reported on the CodeQL status page for more details of possible causes. Addressing these warnings is advisable to avoid false-positive or missing results. If they cannot be addressed, consider scanning Java using either the `autobuild` or `manual` [build modes](https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages#comparison-of-the-build-modes). | visibilityTelemetry | true |
+#select
+| Scanning Java code completed successfully, but the scan encountered issues. This may be caused by problems identifying dependencies or use of generated source code, among other reasons -- see other CodeQL diagnostics reported on the CodeQL status page for more details of possible causes. Addressing these warnings is advisable to avoid false-positive or missing results. If they cannot be addressed, consider scanning Java using either the `autobuild` or `manual` [build modes](https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages#comparison-of-the-build-modes). | Scanning Java code completed successfully, but the scan encountered issues. This may be caused by problems identifying dependencies or use of generated source code, among other reasons -- see other CodeQL diagnostics reported on the CodeQL status page for more details of possible causes. Addressing these warnings is advisable to avoid false-positive or missing results. If they cannot be addressed, consider scanning Java using either the `autobuild` or `manual` [build modes](https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages#comparison-of-the-build-modes). | 1 |

java/ql/integration-tests/all-platforms/java/buildless-erroneous/DatabaseQualityDiagnostics.qlref

@@ -0,0 +1 @@
+Telemetry/DatabaseQualityDiagnostics.ql

java/ql/lib/change-notes/2024-06-25-java-tools-status.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* Java build-mode `none` analyses now only report a warning on the CodeQL status page when there are significant analysis problems-- defined as 5% of expressions lacking a type, or 5% of call targets being unknown. Other messages reported on the status page are downgraded from warnings to notes and so are less prominent, but are still available for review.

java/ql/src/Telemetry/DatabaseQuality.qll

@@ -0,0 +1,65 @@
+/**
+ * Provides database quality statistics that are reported by java/telemetry/extractor-information
+ * and perhaps warned about by java/diagnostics/database-quality.
+ */
+
+import java
+
+signature module StatsSig {
+  int getNumberOfOk();
+
+  int getNumberOfNotOk();
+
+  string getOkText();
+
+  string getNotOkText();
+}
+
+module ReportStats<StatsSig Stats> {
+  predicate numberOfOk(string key, int value) {
+    value = Stats::getNumberOfOk() and
+    key = "Number of " + Stats::getOkText()
+  }
+
+  predicate numberOfNotOk(string key, int value) {
+    value = Stats::getNumberOfNotOk() and
+    key = "Number of " + Stats::getNotOkText()
+  }
+
+  predicate percentageOfOk(string key, float value) {
+    value = Stats::getNumberOfOk() * 100.0 / (Stats::getNumberOfOk() + Stats::getNumberOfNotOk()) and
+    key = "Percentage of " + Stats::getOkText()
+  }
+}
+
+module CallTargetStats implements StatsSig {
+  int getNumberOfOk() { result = count(Call c | exists(c.getCallee())) }
+
+  int getNumberOfNotOk() { result = count(Call c | not exists(c.getCallee())) }
+
+  string getOkText() { result = "calls with call target" }
+
+  string getNotOkText() { result = "calls with missing call target" }
+}
+
+private class SourceExpr extends Expr {
+  SourceExpr() { this.getFile().isSourceFile() }
+}
+
+private predicate hasGoodType(Expr e) {
+  exists(e.getType()) and not e.getType() instanceof ErrorType
+}
+
+module ExprTypeStats implements StatsSig {
+  int getNumberOfOk() { result = count(SourceExpr e | hasGoodType(e)) }
+
+  int getNumberOfNotOk() { result = count(SourceExpr e | not hasGoodType(e)) }
+
+  string getOkText() { result = "expressions with known type" }
+
+  string getNotOkText() { result = "expressions with unknown type" }
+}
+
+module CallTargetStatsReport = ReportStats<CallTargetStats>;
+
+module ExprTypeStatsReport = ReportStats<ExprTypeStats>;

java/ql/src/Telemetry/DatabaseQualityDiagnostics.ql

@@ -0,0 +1,44 @@
+/**
+ * @name Low Java analysis quality
+ * @description Low Java analysis quality
+ * @kind diagnostic
+ * @id java/diagnostic/database-quality
+ */
+
+import java
+import DatabaseQuality
+
+private newtype TDbQualityDiagnostic =
+  TTheDbQualityDiagnostic() {
+    exists(float percentageGood |
+      CallTargetStatsReport::percentageOfOk(_, percentageGood)
+      or
+      ExprTypeStatsReport::percentageOfOk(_, percentageGood)
+    |
+      percentageGood < 95
+    )
+  }
+
+class DbQualityDiagnostic extends TDbQualityDiagnostic {
+  string toString() {
+    result =
+      "Scanning Java code completed successfully, but the scan encountered issues. " +
+        "This may be caused by problems identifying dependencies or use of generated source code, among other reasons -- "
+        +
+        "see other CodeQL diagnostics reported on the CodeQL status page for more details of possible causes. "
+        +
+        "Addressing these warnings is advisable to avoid false-positive or missing results. If they cannot be addressed, consider scanning Java "
+        +
+        "using either the `autobuild` or `manual` [build modes](https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages#comparison-of-the-build-modes)."
+  }
+}
+
+query predicate diagnosticAttributes(DbQualityDiagnostic e, string key, string value) {
+  e = e and // Quieten warning about unconstrained 'e'
+  key = ["visibilityCliSummaryTable", "visibilityTelemetry", "visibilityStatusPage"] and
+  value = "true"
+}
+
+from DbQualityDiagnostic d
+select d, d.toString(), 1
+/* Warning severity */

java/ql/src/Telemetry/ExtractorInformation.ql

@@ -8,6 +8,7 @@
 
 import java
 import semmle.code.java.Diagnostics
+import DatabaseQuality
 
 extensible predicate extractorInformationSkipKey(string key);
 
@@ -85,65 +86,6 @@ predicate extractorTotalDiagnostics(string key, int value) {
   )
 }
 
-signature module StatsSig {
-  int getNumberOfOk();
-
-  int getNumberOfNotOk();
-
-  string getOkText();
-
-  string getNotOkText();
-}
-
-module ReportStats<StatsSig Stats> {
-  predicate numberOfOk(string key, int value) {
-    value = Stats::getNumberOfOk() and
-    key = "Number of " + Stats::getOkText()
-  }
-
-  predicate numberOfNotOk(string key, int value) {
-    value = Stats::getNumberOfNotOk() and
-    key = "Number of " + Stats::getNotOkText()
-  }
-
-  predicate percentageOfOk(string key, float value) {
-    value = Stats::getNumberOfOk() * 100.0 / (Stats::getNumberOfOk() + Stats::getNumberOfNotOk()) and
-    key = "Percentage of " + Stats::getOkText()
-  }
-}
-
-module CallTargetStats implements StatsSig {
-  int getNumberOfOk() { result = count(Call c | exists(c.getCallee())) }
-
-  int getNumberOfNotOk() { result = count(Call c | not exists(c.getCallee())) }
-
-  string getOkText() { result = "calls with call target" }
-
-  string getNotOkText() { result = "calls with missing call target" }
-}
-
-private class SourceExpr extends Expr {
-  SourceExpr() { this.getFile().isSourceFile() }
-}
-
-private predicate hasGoodType(Expr e) {
-  exists(e.getType()) and not e.getType() instanceof ErrorType
-}
-
-module ExprTypeStats implements StatsSig {
-  int getNumberOfOk() { result = count(SourceExpr e | hasGoodType(e)) }
-
-  int getNumberOfNotOk() { result = count(SourceExpr e | not hasGoodType(e)) }
-
-  string getOkText() { result = "expressions with known type" }
-
-  string getNotOkText() { result = "expressions with unknown type" }
-}
-
-module CallTargetStatsReport = ReportStats<CallTargetStats>;
-
-module ExprTypeStatsReport = ReportStats<ExprTypeStats>;
-
 from string key, int value
 where
   not exists(string pattern | extractorInformationSkipKey(pattern) and key.matches(pattern)) and