Merge pull request #17892 from github/calumgrant/telemetry

C++: Initial telemetry queries

cpp/ql/src/Telemetry/CompilerErrors.ql

@@ -0,0 +1,13 @@
+/**
+ * @name Compiler errors
+ * @description A count of all compiler errors, grouped by error text.
+ * @kind metric
+ * @tags summary telemetry
+ * @id cpp/telemetry/compiler-errors
+ */
+
+import Metrics
+
+from CppMetrics::ErrorCount m
+where RankMetric<CppMetrics::ErrorCount>::getRank(m) <= 50
+select m.toString(), m.getValue()

cpp/ql/src/Telemetry/DatabaseQuality.ql

@@ -0,0 +1,12 @@
+/**
+ * @name Database quality
+ * @description Metrics that indicate the quality of the database.
+ * @kind metric
+ * @tags summary telemetry
+ * @id cpp/telemetry/database-quality
+ */
+
+import Metrics
+
+from QualityMetric m
+select m.toString(), m.getValue()

cpp/ql/src/Telemetry/Diagnostics.qll

@@ -0,0 +1,29 @@
+import cpp
+
+/**
+ * A syntax error.
+ */
+class SyntaxError extends CompilerError {
+  SyntaxError() {
+    this.getTag().matches("exp_%") or
+    this.getTag() =
+      [
+        "bad_data_member_initialization", "bad_pure_specifier", "bad_return", "bad_uuid_string",
+        "literal_without_initializer", "missing_class_definition", "missing_exception_declaration",
+        "nonstd_const_member_decl_not_allowed", "operator_name_not_allowed",
+        "wide_string_invalid_in_asm"
+      ]
+  }
+}
+
+/**
+ * A cannot open file error.
+ * Typically this is due to a missing include.
+ */
+class CannotOpenFileError extends CompilerError {
+  CannotOpenFileError() { this.hasTag(["cannot_open_file", "cannot_open_file_reason"]) }
+
+  string getIncludedFile() {
+    result = this.getMessage().regexpCapture("cannot open source file '([^']+)'", 1)
+  }
+}

cpp/ql/src/Telemetry/ExtractionMetrics.ql

@@ -0,0 +1,12 @@
+/**
+ * @name Extraction metrics
+ * @description Raw metrics relating to extraction.
+ * @kind metric
+ * @tags summary telemetry
+ * @id cpp/telemetry/extraction-metrics
+ */
+
+import Metrics
+
+from ExtractionMetric m
+select m.toString(), m.getValue()

cpp/ql/src/Telemetry/Metrics.qll

@@ -0,0 +1,269 @@
+import cpp
+import Diagnostics
+
+/**
+ * A metric is a string with a value.
+ */
+abstract class Metric extends string {
+  bindingset[this]
+  Metric() { any() }
+}
+
+/**
+ * A metric that we want to report in cpp/telemetry/extraction-metrics
+ */
+abstract class ExtractionMetric extends Metric {
+  bindingset[this]
+  ExtractionMetric() { any() }
+
+  /** Gets the value of this metric. */
+  abstract int getValue();
+}
+
+/**
+ * A metric that provides a baseline for a SuccessMetric.
+ */
+abstract class BaseMetric extends ExtractionMetric {
+  bindingset[this]
+  BaseMetric() { any() }
+}
+
+/**
+ * A metric that is relative to another metric,
+ * so can be used to calculate percentages.
+ *
+ * For clarity, metrics should express success,
+ * so higher values means better.
+ */
+abstract class SuccessMetric extends ExtractionMetric {
+  bindingset[this]
+  SuccessMetric() { any() }
+
+  /** Gets the metric this is relative to. */
+  abstract BaseMetric getBaseline();
+}
+
+/**
+ * A metric used to report database quality.
+ */
+class QualityMetric extends Metric {
+  BaseMetric baseMetric;
+  SuccessMetric relativeMetric;
+
+  QualityMetric() {
+    baseMetric = relativeMetric.getBaseline() and this = "Percentage of " + relativeMetric
+  }
+
+  float getValue() {
+    baseMetric.getValue() > 0 and
+    result = 100.0 * relativeMetric.getValue() / baseMetric.getValue()
+  }
+}
+
+signature class RankedMetric extends Metric {
+  int getValue();
+}
+
+module RankMetric<RankedMetric M> {
+  int getRank(M s) { s = rank[result](M m | | m order by m.getValue() desc) }
+}
+
+/** Various metrics we want to report. */
+module CppMetrics {
+  class Compilations extends BaseMetric {
+    Compilations() { this = "compilations" }
+
+    override int getValue() { result = count(Compilation c) }
+  }
+
+  class SourceAndHeaderFiles extends BaseMetric {
+    SourceAndHeaderFiles() { this = "source/header files" }
+
+    override int getValue() { result = count(File f | f.fromSource()) }
+  }
+
+  class SourceAndHeaderFilesWithoutErrors extends SuccessMetric {
+    SourceAndHeaderFilesWithoutErrors() { this = "source/header files without errors" }
+
+    override int getValue() {
+      result = count(File f | f.fromSource() and not exists(CompilerError e | f = e.getFile()))
+    }
+
+    override SourceAndHeaderFiles getBaseline() { any() }
+  }
+
+  class CompilationsWithoutErrors extends SuccessMetric {
+    CompilationsWithoutErrors() { this = "compilations without errors" }
+
+    override int getValue() {
+      result = count(Compilation c | not exists(Diagnostic d | d.getFile() = c.getAFileCompiled()))
+    }
+
+    override Compilations getBaseline() { any() }
+  }
+
+  class Expressions extends BaseMetric {
+    Expressions() { this = "expressions" }
+
+    override int getValue() { result = count(Expr e) }
+  }
+
+  class SucceededExpressions extends SuccessMetric {
+    SucceededExpressions() { this = "non-error expressions" }
+
+    override int getValue() { result = count(Expr e) - count(ErrorExpr e) }
+
+    override Expressions getBaseline() { any() }
+  }
+
+  class TypedExpressions extends SuccessMetric {
+    TypedExpressions() { this = "expressions with a known type" }
+
+    override int getValue() { result = count(Expr e | not e.getType() instanceof ErroneousType) }
+
+    override Expressions getBaseline() { any() }
+  }
+
+  class Calls extends BaseMetric {
+    Calls() { this = "calls" }
+
+    override int getValue() { result = count(Call c) }
+  }
+
+  class CallsWithExplicitTarget extends SuccessMetric {
+    CallsWithExplicitTarget() { this = "calls with an explicit target" }
+
+    override int getValue() {
+      result = count(Call c | not c.getTarget().getADeclarationEntry().isImplicit())
+    }
+
+    override Calls getBaseline() { any() }
+  }
+
+  class Variables extends BaseMetric {
+    Variables() { this = "variables" }
+
+    override int getValue() { result = count(Variable v) }
+  }
+
+  class VariablesKnownType extends SuccessMetric {
+    VariablesKnownType() { this = "variables with a known type" }
+
+    override int getValue() {
+      result = count(Variable v | not v.getType() instanceof ErroneousType)
+    }
+
+    override Variables getBaseline() { any() }
+  }
+
+  class LinesOfText extends BaseMetric {
+    LinesOfText() { this = "lines of text" }
+
+    override int getValue() { result = sum(File f | | f.getMetrics().getNumberOfLines()) }
+  }
+
+  class LinesOfCode extends BaseMetric {
+    LinesOfCode() { this = "lines of code" }
+
+    override int getValue() { result = sum(File f | | f.getMetrics().getNumberOfLinesOfCode()) }
+  }
+
+  private predicate errorLine(File file, int line) {
+    exists(Locatable l, Location loc |
+      loc = l.getLocation() and
+      loc.getFile() = file and
+      line in [loc.getStartLine() .. loc.getEndLine()]
+    |
+      l instanceof Diagnostic
+      or
+      l instanceof ErrorExpr
+    )
+  }
+
+  class SucceededLines extends SuccessMetric {
+    SucceededLines() { this = "lines of code without errors" }
+
+    override int getValue() {
+      result =
+        sum(File f | | f.getMetrics().getNumberOfLinesOfCode()) -
+          count(File f, int line | errorLine(f, line))
+    }
+
+    override LinesOfCode getBaseline() { any() }
+  }
+
+  class Functions extends BaseMetric {
+    Functions() { this = "functions" }
+
+    override int getValue() { result = count(Function f) }
+  }
+
+  class SucceededFunctions extends SuccessMetric {
+    SucceededFunctions() { this = "functions without errors" }
+
+    override int getValue() { result = count(Function f | not f.hasErrors()) }
+
+    override Functions getBaseline() { any() }
+  }
+
+  class Includes extends BaseMetric {
+    Includes() { this = "#include directives" }
+
+    override int getValue() { result = count(Include i) + count(CannotOpenFileError e) }
+  }
+
+  class SucceededIncludes extends SuccessMetric {
+    SucceededIncludes() { this = "successfully resolved #include directives" }
+
+    override int getValue() { result = count(Include i) }
+
+    override Includes getBaseline() { any() }
+  }
+
+  class SucceededIncludeCount extends Metric {
+    string includeText;
+
+    SucceededIncludeCount() {
+      exists(Include i |
+        i.getIncludeText() = includeText and
+        exists(i.getFile().getRelativePath()) // Only report includes from the repo
+      ) and
+      this = "Successfully included " + includeText
+    }
+
+    int getValue() { result = count(Include i | i.getIncludeText() = includeText) }
+
+    string getIncludeText() { result = includeText }
+  }
+
+  class MissingIncludeCount extends Metric {
+    string includeText;
+
+    MissingIncludeCount() {
+      exists(CannotOpenFileError e | e.getIncludedFile() = includeText) and
+      this = "Failed to include '" + includeText + "'"
+    }
+
+    int getValue() { result = count(CannotOpenFileError e | e.getIncludedFile() = includeText) }
+
+    string getIncludeText() { result = includeText }
+  }
+
+  class CompilerErrors extends ExtractionMetric {
+    CompilerErrors() { this = "compiler errors" }
+
+    override int getValue() { result = count(CompilerError e) }
+  }
+
+  class ErrorCount extends Metric {
+    ErrorCount() { exists(CompilerError e | e.getMessage() = this) }
+
+    int getValue() { result = count(CompilerError e | e.getMessage() = this) }
+  }
+
+  class SyntaxErrorCount extends ExtractionMetric {
+    SyntaxErrorCount() { this = "syntax errors" }
+
+    override int getValue() { result = count(SyntaxError e) }
+  }
+}

cpp/ql/src/Telemetry/MissingIncludes.ql

@@ -0,0 +1,13 @@
+/**
+ * @name Failed to include header file
+ * @description A count of all failed includes, grouped by filename.
+ * @kind metric
+ * @tags summary telemetry
+ * @id cpp/telemetry/failed-includes
+ */
+
+import Metrics
+
+from CppMetrics::MissingIncludeCount e
+where RankMetric<CppMetrics::MissingIncludeCount>::getRank(e) <= 50
+select e.getIncludeText(), e.getValue()

cpp/ql/src/Telemetry/SucceededIncludes.ql

@@ -0,0 +1,13 @@
+/**
+ * @name Successfully included header files
+ * @description A count of all succeeded includes, grouped by filename.
+ * @kind metric
+ * @tags summary telemetry
+ * @id cpp/telemetry/succeeded-includes
+ */
+
+import Metrics
+
+from CppMetrics::SucceededIncludeCount m
+where RankMetric<CppMetrics::SucceededIncludeCount>::getRank(m) <= 50
+select m.getIncludeText(), m.getValue()

cpp/ql/test/library-tests/extraction_errors/CompilerErrors.expected

@@ -0,0 +1,10 @@
+| 'this' may only be used inside a nonstatic member function | 1 |
+| There was an error during this compilation | 1 |
+| expected a ')' | 1 |
+| expected a ';' | 1 |
+| expected an expression | 1 |
+| identifier 'no_such_function' is undefined | 1 |
+| identifier 'nsf2' is undefined | 1 |
+| identifier 'so_is_this' is undefined | 1 |
+| identifier 'uint32_t' is undefined | 1 |
+| too few arguments in function call | 1 |

cpp/ql/test/library-tests/extraction_errors/CompilerErrors.qlref

@@ -0,0 +1 @@
+Telemetry/CompilerErrors.ql

cpp/ql/test/library-tests/extraction_errors/DatabaseQuality.expected

@@ -0,0 +1,9 @@
+| Percentage of calls with an explicit target | 50.0 |
+| Percentage of compilations without errors | 50.0 |
+| Percentage of expressions with a known type | 30.0 |
+| Percentage of functions without errors | 75.0 |
+| Percentage of lines of code without errors | 63.1578947368421 |
+| Percentage of non-error expressions | 30.0 |
+| Percentage of source/header files without errors | 66.66666666666667 |
+| Percentage of successfully resolved #include directives | 100.0 |
+| Percentage of variables with a known type | 90.0 |

cpp/ql/test/library-tests/extraction_errors/DatabaseQuality.qlref

@@ -0,0 +1 @@
+Telemetry/DatabaseQuality.ql

cpp/ql/test/library-tests/extraction_errors/ErrorExprs.expected

@@ -0,0 +1,7 @@
+| test.cpp:6:13:6:31 | initializer for x | test.cpp:6:13:6:31 | <error expr> |
+| test.cpp:7:5:7:8 | ExprStmt | file://:0:0:0:0 | <error expr> |
+| test.cpp:8:5:8:23 | ExprStmt | file://:0:0:0:0 | <error expr> |
+| test.cpp:9:5:9:21 | ExprStmt | file://:0:0:0:0 | <error expr> |
+| test.cpp:11:5:11:8 | ExprStmt | file://:0:0:0:0 | <error expr> |
+| test.cpp:15:5:15:8 | ExprStmt | file://:0:0:0:0 | <error expr> |
+| test.cpp:16:5:16:16 | ExprStmt | file://:0:0:0:0 | <error expr> |

cpp/ql/test/library-tests/extraction_errors/ErrorExprs.ql

@@ -0,0 +1,4 @@
+import cpp
+
+from ErrorExpr e
+select e.getParent(), e

cpp/ql/test/library-tests/extraction_errors/ExtractionMetrics.expected

@@ -0,0 +1,20 @@
+| #include directives | 2 |
+| calls | 2 |
+| calls with an explicit target | 1 |
+| compilations | 2 |
+| compilations without errors | 1 |
+| compiler errors | 10 |
+| expressions | 10 |
+| expressions with a known type | 3 |
+| functions | 8 |
+| functions without errors | 6 |
+| lines of code | 19 |
+| lines of code without errors | 12 |
+| lines of text | 24 |
+| non-error expressions | 3 |
+| source/header files | 3 |
+| source/header files without errors | 2 |
+| successfully resolved #include directives | 2 |
+| syntax errors | 3 |
+| variables | 10 |
+| variables with a known type | 9 |

cpp/ql/test/library-tests/extraction_errors/ExtractionMetrics.qlref

@@ -0,0 +1 @@
+Telemetry/ExtractionMetrics.ql

cpp/ql/test/library-tests/extraction_errors/SucceededIncludes.expected

@@ -0,0 +1 @@
+| "test.h" | 2 |

cpp/ql/test/library-tests/extraction_errors/SucceededIncludes.qlref

@@ -0,0 +1 @@
+Telemetry/SucceededIncludes.ql

cpp/ql/test/library-tests/extraction_errors/diags.expected

@@ -0,0 +1,10 @@
+| file://:0:0:0:0 | There was an error during this compilation |
+| test.cpp:6:14:6:14 | identifier 'no_such_function' is undefined |
+| test.cpp:9:14:9:14 | identifier 'nsf2' is undefined |
+| test.cpp:11:7:11:7 | too few arguments in function call |
+| test.cpp:14:1:14:1 | identifier 'uint32_t' is undefined |
+| test.cpp:15:5:15:5 | 'this' may only be used inside a nonstatic member function |
+| test.cpp:15:10:15:10 | expected a ';' |
+| test.cpp:16:5:16:5 | identifier 'so_is_this' is undefined |
+| test.cpp:16:16:16:16 | expected a ')' |
+| test.cpp:16:16:16:16 | expected an expression |

cpp/ql/test/library-tests/extraction_errors/diags.ql

@@ -0,0 +1,4 @@
+import cpp
+
+from Diagnostic d
+select d

cpp/ql/test/library-tests/extraction_errors/error_variables.expected

@@ -0,0 +1 @@
+| test.cpp:6:10:6:10 | x | This variable does not have a type. |

cpp/ql/test/library-tests/extraction_errors/error_variables.ql

@@ -0,0 +1,5 @@
+import cpp
+
+from Variable v
+where v.getType() instanceof ErroneousType or not exists(v.getType())
+select v, "This variable does not have a type."

cpp/ql/test/library-tests/extraction_errors/functions.expected

@@ -0,0 +1,6 @@
+| test.c:3:6:3:6 | g |
+| test.c:4:13:4:13 | no_such_function |
+| test.cpp:5:6:5:25 | function_with_errors |
+| test.cpp:14:10:14:12 | fn2 |
+| test.h:2:5:2:5 | f |
+| test.h:2:5:2:5 | f |

cpp/ql/test/library-tests/extraction_errors/functions.ql

@@ -0,0 +1,5 @@
+import cpp
+
+from Function fn
+where fn.fromSource()
+select fn

cpp/ql/test/library-tests/extraction_errors/test.c

@@ -0,0 +1,5 @@
+#include "test.h"
+
+void g() {
+    int x = no_such_function();
+}

cpp/ql/test/library-tests/extraction_errors/test.cpp

@@ -0,0 +1,17 @@
+// semmle-extractor-options: --expect_errors
+
+#include "test.h"
+
+void function_with_errors() {
+    auto x = no_such_function();
+    x+2;
+    no_such_function();
+    ADD(x+1, nsf2());
+    f(1);
+    f();
+}
+
+uint32_t fn2() {
+    this is a syntax error;
+    so_is_this(;
+}

cpp/ql/test/library-tests/extraction_errors/test.h

@@ -0,0 +1,2 @@
+#define ADD(A,B) ((A)+(B))
+int f(int);