C++: Initial telemetry queries and tests

cpp/ql/src/Telemetry/CompilerErrors.ql

@@ -0,0 +1,12 @@
+/**
+ * @name Compiler errors
+ * @description A count of all compiler errors, grouped by error text.
+ * @kind metric
+ * @tags summary telemetry
+ * @id cpp/telemetry/compiler-errors
+ */
+
+import Metrics
+
+from CppMetrics::ErrorCount m
+select m, m.getValue() as c order by c desc

cpp/ql/src/Telemetry/DatabaseQuality.ql

@@ -0,0 +1,12 @@
+/**
+ * @name Database quality
+ * @description Metrics that indicate the quality of the database.
+ * @kind metric
+ * @tags summary telemetry
+ * @id cpp/telemetry/database-quality
+ */
+
+import Metrics
+
+from QualityMetric m
+select m, m.getValue() order by m

cpp/ql/src/Telemetry/Diagnostics.qll

@@ -0,0 +1,32 @@
+import cpp
+
+/**
+ * A syntax error.
+ */
+class SyntaxError extends CompilerError {
+  SyntaxError() { this.getTag().matches("exp_%") }
+}
+
+/**
+ * A cannot open file error.
+ * Typically this is due to a missing include.
+ */
+class CannotOpenFile extends CompilerError {
+  CannotOpenFile() { this.hasTag("cannot_open_file") }
+
+  string getIncludedFile() {
+    result = this.getMessage().regexpCapture("cannot open source file '([^']+)'", 1)
+  }
+}
+
+/**
+ * An undefined identifier error.
+ * Currently unused.
+ */
+class UndefinedIdentifier extends CompilerError {
+  UndefinedIdentifier() { this.hasTag("undefined_identifier") }
+
+  string getIdentifier() {
+    result = this.getMessage().regexpCapture("identifier '([^']+)' is undefined", 1)
+  }
+}

cpp/ql/src/Telemetry/ExtractionMetrics.ql

@@ -0,0 +1,12 @@
+/**
+ * @name Extraction metrics
+ * @description Raw metrics relating to extraction.
+ * @kind metric
+ * @tags summary telemetry
+ * @id cpp/telemetry/extraction-metrics
+ */
+
+import Metrics
+
+from ExtractionMetric m
+select m, m.getValue() order by m

cpp/ql/src/Telemetry/Metrics.qll

@@ -0,0 +1,260 @@
+import cpp
+import Diagnostics
+
+/**
+ * A metric is a string with a value.
+ */
+abstract class Metric extends string {
+  bindingset[this]
+  Metric() { any() }
+
+  /** Gets the value of this metric. */
+  abstract float getValue();
+}
+
+/**
+ * A metric that we want to report in cpp/telemetry/extraction-metrics
+ */
+abstract class ExtractionMetric extends Metric {
+  bindingset[this]
+  ExtractionMetric() { any() }
+}
+
+/**
+ * A metric that provides a baseline for a SuccessMetric.
+ */
+abstract class BaseMetric extends ExtractionMetric {
+  bindingset[this]
+  BaseMetric() { any() }
+}
+
+/**
+ * A metric that is relative to another metric,
+ * so can be used to calculate percentages.
+ *
+ * For clarity, metrics should express success,
+ * so higher values means better.
+ */
+abstract class SuccessMetric extends ExtractionMetric {
+  bindingset[this]
+  SuccessMetric() { any() }
+
+  /** Gets the metric this is relative to. */
+  abstract BaseMetric getBaseline();
+}
+
+/**
+ * A metric used to report database quality.
+ */
+class QualityMetric extends Metric {
+  BaseMetric base_metric;
+  SuccessMetric relative_metric;
+
+  QualityMetric() {
+    base_metric = relative_metric.getBaseline() and this = "Percentage of " + relative_metric
+  }
+
+  override float getValue() { result = 100 * relative_metric.getValue() / base_metric.getValue() }
+}
+
+/** Various metrics we want to report. */
+module CppMetrics {
+  class CompilationUnits extends BaseMetric {
+    CompilationUnits() { this = "compilation units" }
+
+    override float getValue() { result = count(Compilation c) }
+  }
+
+  class SourceFiles extends BaseMetric {
+    SourceFiles() { this = "source files" }
+
+    override float getValue() { result = count(File f | f.fromSource()) }
+  }
+
+  class SourceFilesWithoutErrors extends SuccessMetric {
+    SourceFilesWithoutErrors() { this = "source files without errors" }
+
+    override float getValue() {
+      result = count(File f | f.fromSource() and not exists(CompilerError e | f = e.getFile()))
+    }
+
+    override SourceFiles getBaseline() { any() }
+  }
+
+  class CompilationUnitsWithoutErrors extends SuccessMetric {
+    CompilationUnitsWithoutErrors() { this = "compilation units without errors" }
+
+    override float getValue() {
+      result = count(Compilation c | not exists(Diagnostic d | d.getFile() = c.getAFileCompiled()))
+    }
+
+    override CompilationUnits getBaseline() { any() }
+  }
+
+  class Expressions extends BaseMetric {
+    Expressions() { this = "expressions" }
+
+    override float getValue() { result = count(Expr e) }
+  }
+
+  class SucceededExpressions extends SuccessMetric {
+    SucceededExpressions() { this = "non-error expressions" }
+
+    override float getValue() { result = count(Expr e) - count(ErrorExpr e) }
+
+    override Expressions getBaseline() { any() }
+  }
+
+  class TypedExpressions extends SuccessMetric {
+    TypedExpressions() { this = "expressions with a known type" }
+
+    override float getValue() { result = count(Expr e | not e.getType() instanceof ErroneousType) }
+
+    override Expressions getBaseline() { any() }
+  }
+
+  class Calls extends BaseMetric {
+    Calls() { this = "calls" }
+
+    override float getValue() { result = count(Call c) }
+  }
+
+  class SucceededCalls extends SuccessMetric {
+    SucceededCalls() { this = "calls with a target" }
+
+    override float getValue() {
+      result = count(Call c | not c.getTarget().getADeclarationEntry().isImplicit())
+    }
+
+    override Calls getBaseline() { any() }
+  }
+
+  class Variables extends BaseMetric {
+    Variables() { this = "variables" }
+
+    override float getValue() { result = count(Variable v) }
+  }
+
+  class VariablesKnownType extends SuccessMetric {
+    VariablesKnownType() { this = "variables with a known type" }
+
+    override float getValue() {
+      result = count(Variable v | not v.getType() instanceof ErroneousType)
+    }
+
+    override Variables getBaseline() { any() }
+  }
+
+  class LinesOfText extends BaseMetric {
+    LinesOfText() { this = "lines of text" }
+
+    override float getValue() { result = sum(File f | | f.getMetrics().getNumberOfLines()) }
+  }
+
+  class LinesOfCode extends BaseMetric {
+    LinesOfCode() { this = "lines of code" }
+
+    override float getValue() { result = sum(File f | | f.getMetrics().getNumberOfLinesOfCode()) }
+  }
+
+  private predicate errorLine(File file, int line) {
+    exists(Locatable l, Location loc |
+      loc = l.getLocation() and
+      loc.getFile() = file and
+      line in [loc.getStartLine() .. loc.getEndLine()]
+    |
+      l instanceof Diagnostic
+      or
+      l instanceof ErrorExpr
+    )
+  }
+
+  class SucceededLines extends SuccessMetric {
+    SucceededLines() { this = "lines of code without errors" }
+
+    override float getValue() {
+      result =
+        sum(File f | | f.getMetrics().getNumberOfLinesOfCode()) -
+          count(File file, int line | errorLine(file, line))
+    }
+
+    override LinesOfCode getBaseline() { any() }
+  }
+
+  class Functions extends BaseMetric {
+    Functions() { this = "functions" }
+
+    override float getValue() { result = count(Function f) }
+  }
+
+  class SucceededFunctions extends SuccessMetric {
+    SucceededFunctions() { this = "functions without errors" }
+
+    override float getValue() { result = count(Function f | not f.hasErrors()) }
+
+    override Functions getBaseline() { any() }
+  }
+
+  class Includes extends BaseMetric {
+    Includes() { this = "#include directives" }
+
+    override float getValue() { result = count(Include i) + count(CannotOpenFile e) }
+  }
+
+  class SucceededIncludes extends SuccessMetric {
+    SucceededIncludes() { this = "successfully resolved #include directives" }
+
+    override float getValue() { result = count(Include i) }
+
+    override Includes getBaseline() { any() }
+  }
+
+  class SucceededIncludeCount extends Metric {
+    string include_text;
+
+    SucceededIncludeCount() {
+      exists(Include i |
+        i.getIncludeText() = include_text and
+        exists(i.getFile().getRelativePath())  // Only report includes from the repo
+      ) and
+      this = "Successfully included " + include_text
+    }
+
+    override float getValue() { result = count(Include i | i.getIncludeText() = include_text) }
+
+    string getIncludeText() { result = include_text }
+  }
+
+  class MissingIncludeCount extends Metric {
+    string include_text;
+
+    MissingIncludeCount() {
+      exists(CannotOpenFile e | e.getIncludedFile() = include_text) and
+      this = "Failed to include '" + include_text + "'"
+    }
+
+    override float getValue() {
+      result = count(CannotOpenFile e | e.getIncludedFile() = include_text)
+    }
+
+    string getIncludeText() { result = include_text }
+  }
+
+  class CompilerErrors extends ExtractionMetric {
+    CompilerErrors() { this = "compiler errors" }
+
+    override float getValue() { result = count(CompilerError e) }
+  }
+
+  class ErrorCount extends Metric {
+    ErrorCount() { exists(CompilerError e | e.getMessage() = this) }
+
+    override float getValue() { result = count(CompilerError e | e.getMessage() = this) }
+  }
+
+  class SyntaxErrorCount extends ExtractionMetric {
+    SyntaxErrorCount() { this = "syntax errors" }
+
+    override float getValue() { result = count(SyntaxError e) }
+  }
+}

cpp/ql/src/Telemetry/MissingIncludes.ql

@@ -0,0 +1,12 @@
+/**
+ * @name Failed to include header file
+ * @description A count of all failed includes, grouped by filename.
+ * @kind metric
+ * @tags summary telemetry
+ * @id cpp/telemetry/failed-includes
+ */
+
+import Metrics
+
+from CppMetrics::MissingIncludeCount e
+select e.getIncludeText(), e.getValue() as c order by c desc

cpp/ql/src/Telemetry/SucceededIncludes.ql

@@ -0,0 +1,12 @@
+/**
+ * @name Successfully included header files
+ * @description A count of all succeeded includes, grouped by filename.
+ * @kind metric
+ * @tags summary telemetry
+ * @id cpp/telemetry/succeeded-includes
+ */
+
+import Metrics
+
+from CppMetrics::SucceededIncludeCount m
+select m.getIncludeText(), m.getValue()

cpp/ql/test/library-tests/extraction_errors/CompilerErrors.expected

@@ -0,0 +1,10 @@
+| 'this' may only be used inside a nonstatic member function | 1.0 |
+| There was an error during this compilation | 1.0 |
+| expected a ')' | 1.0 |
+| expected a ';' | 1.0 |
+| expected an expression | 1.0 |
+| identifier 'no_such_function' is undefined | 1.0 |
+| identifier 'nsf2' is undefined | 1.0 |
+| identifier 'so_is_this' is undefined | 1.0 |
+| identifier 'uint32_t' is undefined | 1.0 |
+| too few arguments in function call | 1.0 |

cpp/ql/test/library-tests/extraction_errors/CompilerErrors.qlref

@@ -0,0 +1 @@
+Telemetry/CompilerErrors.ql

cpp/ql/test/library-tests/extraction_errors/DatabaseQuality.expected

@@ -0,0 +1,9 @@
+| Percentage of calls with a target | 50.0 |
+| Percentage of compilation units without errors | 50.0 |
+| Percentage of expressions with a known type | 30.0 |
+| Percentage of functions without errors | 75.0 |
+| Percentage of lines of code without errors | 63.1578947368421 |
+| Percentage of non-error expressions | 30.0 |
+| Percentage of source files without errors | 66.66666666666667 |
+| Percentage of successfully resolved #include directives | 100.0 |
+| Percentage of variables with a known type | 90.0 |

cpp/ql/test/library-tests/extraction_errors/DatabaseQuality.qlref

@@ -0,0 +1 @@
+Telemetry/DatabaseQuality.ql

cpp/ql/test/library-tests/extraction_errors/ExtractionMetrics.expected

@@ -0,0 +1,20 @@
+| #include directives | 2.0 |
+| calls | 2.0 |
+| calls with a target | 1.0 |
+| compilation units | 2.0 |
+| compilation units without errors | 1.0 |
+| compiler errors | 10.0 |
+| expressions | 10.0 |
+| expressions with a known type | 3.0 |
+| functions | 8.0 |
+| functions without errors | 6.0 |
+| lines of code | 19.0 |
+| lines of code without errors | 12.0 |
+| lines of text | 26.0 |
+| non-error expressions | 3.0 |
+| source files | 3.0 |
+| source files without errors | 2.0 |
+| successfully resolved #include directives | 2.0 |
+| syntax errors | 3.0 |
+| variables | 10.0 |
+| variables with a known type | 9.0 |

cpp/ql/test/library-tests/extraction_errors/ExtractionMetrics.qlref

@@ -0,0 +1 @@
+Telemetry/ExtractionMetrics.ql

cpp/ql/test/library-tests/extraction_errors/MissingIncludes.qlref

@@ -0,0 +1 @@
+Telemetry/MissingIncludes.ql

cpp/ql/test/library-tests/extraction_errors/PrintAst.expected

@@ -0,0 +1,79 @@
+#-----| [CopyAssignmentOperator] __va_list_tag& __va_list_tag::operator=(__va_list_tag const&)
+#-----|   <params>: 
+#-----|     getParameter(0): [Parameter] (unnamed parameter 0)
+#-----|         Type = [LValueReferenceType] const __va_list_tag &
+#-----| [MoveAssignmentOperator] __va_list_tag& __va_list_tag::operator=(__va_list_tag&&)
+#-----|   <params>: 
+#-----|     getParameter(0): [Parameter] (unnamed parameter 0)
+#-----|         Type = [RValueReferenceType] __va_list_tag &&
+test.c:
+#    3| [TopLevelFunction] void g()
+#    3|   <params>: 
+#    3|   getEntryPoint(): [BlockStmt] { ... }
+#    4|     getStmt(0): [DeclStmt] declaration
+#    4|       getDeclarationEntry(0): [VariableDeclarationEntry] definition of x
+#    4|           Type = [IntType] int
+#    4|         getVariable().getInitializer(): [Initializer] initializer for x
+#    4|           getExpr(): [FunctionCall] call to no_such_function
+#    4|               Type = [IntType] int
+#    4|               ValueCategory = prvalue
+#    5|     getStmt(1): [ReturnStmt] return ...
+#    4| [TopLevelFunction] int no_such_function()
+#    4|   <params>: 
+test.cpp:
+#    5| [TopLevelFunction] void function_with_errors()
+#    5|   <params>: 
+#    5|   getEntryPoint(): [BlockStmt] { ... }
+#    6|     getStmt(0): [DeclStmt] declaration
+#    6|       getDeclarationEntry(0): [VariableDeclarationEntry] definition of x
+#    6|           Type = [ErroneousType] error
+#    6|         getVariable().getInitializer(): [Initializer] initializer for x
+#    6|           getExpr(): [ErrorExpr] <error expr>
+#    6|               Type = [ErroneousType] error
+#    6|               ValueCategory = prvalue
+#    7|     getStmt(1): [ExprStmt] ExprStmt
+#-----|       getExpr(): [ErrorExpr] <error expr>
+#-----|           Type = [ErroneousType] error
+#-----|           ValueCategory = prvalue
+#    8|     getStmt(2): [ExprStmt] ExprStmt
+#-----|       getExpr(): [ErrorExpr] <error expr>
+#-----|           Type = [ErroneousType] error
+#-----|           ValueCategory = prvalue
+#    9|     getStmt(3): [ExprStmt] ExprStmt
+#-----|       getExpr(): [ErrorExpr] <error expr>
+#-----|           Type = [ErroneousType] error
+#-----|           ValueCategory = prvalue
+#   10|     getStmt(4): [ExprStmt] ExprStmt
+#   10|       getExpr(): [FunctionCall] call to f
+#   10|           Type = [IntType] int
+#   10|           ValueCategory = prvalue
+#   10|         getArgument(0): [Literal] 1
+#   10|             Type = [IntType] int
+#   10|             Value = [Literal] 1
+#   10|             ValueCategory = prvalue
+#   11|     getStmt(5): [ExprStmt] ExprStmt
+#-----|       getExpr(): [ErrorExpr] <error expr>
+#-----|           Type = [ErroneousType] error
+#-----|           ValueCategory = prvalue
+#   12|     getStmt(6): [ReturnStmt] return ...
+#   14| [TopLevelFunction] error fn2()
+#   14|   <params>: 
+#   14|   getEntryPoint(): [BlockStmt] { ... }
+#   15|     getStmt(0): [ExprStmt] ExprStmt
+#-----|       getExpr(): [ErrorExpr] <error expr>
+#-----|           Type = [ErroneousType] error
+#-----|           ValueCategory = prvalue
+#   16|     getStmt(1): [ExprStmt] ExprStmt
+#-----|       getExpr(): [ErrorExpr] <error expr>
+#-----|           Type = [ErroneousType] error
+#-----|           ValueCategory = prvalue
+#   17|     getStmt(2): [ReturnStmt] return ...
+test.h:
+#    4| [TopLevelFunction] int f(int)
+#    4|   <params>: 
+#    4|     getParameter(0): [Parameter] (unnamed parameter 0)
+#    4|         Type = [IntType] int
+#    4| [TopLevelFunction] int f(int)
+#    4|   <params>: 
+#    4|     getParameter(0): [Parameter] (unnamed parameter 0)
+#    4|         Type = [IntType] int

cpp/ql/test/library-tests/extraction_errors/PrintAst.qlref

@@ -0,0 +1 @@
+semmle/code/cpp/PrintAST.ql

cpp/ql/test/library-tests/extraction_errors/SucceededIncludes.expected

@@ -0,0 +1 @@
+| "test.h" | 2.0 |

cpp/ql/test/library-tests/extraction_errors/SucceededIncludes.qlref

@@ -0,0 +1 @@
+Telemetry/SucceededIncludes.ql

cpp/ql/test/library-tests/extraction_errors/diags.expected

@@ -0,0 +1,10 @@
+| file://:0:0:0:0 | There was an error during this compilation |
+| test.cpp:6:14:6:14 | identifier 'no_such_function' is undefined |
+| test.cpp:9:14:9:14 | identifier 'nsf2' is undefined |
+| test.cpp:11:7:11:7 | too few arguments in function call |
+| test.cpp:14:1:14:1 | identifier 'uint32_t' is undefined |
+| test.cpp:15:5:15:5 | 'this' may only be used inside a nonstatic member function |
+| test.cpp:15:10:15:10 | expected a ';' |
+| test.cpp:16:5:16:5 | identifier 'so_is_this' is undefined |
+| test.cpp:16:16:16:16 | expected a ')' |
+| test.cpp:16:16:16:16 | expected an expression |

cpp/ql/test/library-tests/extraction_errors/diags.ql

@@ -0,0 +1,4 @@
+import cpp
+
+from Diagnostic d
+select d

cpp/ql/test/library-tests/extraction_errors/error_fn_returns.expected

@@ -0,0 +1 @@
+| test.cpp:14:10:14:12 | fn2 |

cpp/ql/test/library-tests/extraction_errors/error_fn_returns.ql

@@ -0,0 +1,5 @@
+import cpp
+
+from Function fn
+where fn.getType() instanceof ErroneousType or not exists(fn.getType())
+select fn

cpp/ql/test/library-tests/extraction_errors/error_types.expected

@@ -0,0 +1 @@
+| test.cpp:6:13:6:31 | <error expr> |

cpp/ql/test/library-tests/extraction_errors/error_types.ql

@@ -0,0 +1,6 @@
+import cpp
+
+from Expr e
+where e.getType() instanceof ErroneousType
+and e.fromSource()
+select e

cpp/ql/test/library-tests/extraction_errors/error_variables.expected

@@ -0,0 +1 @@
+| test.cpp:6:10:6:10 | x | This variable does not have a type. |

cpp/ql/test/library-tests/extraction_errors/error_variables.ql

@@ -0,0 +1,5 @@
+import cpp
+
+from Variable v
+where v.getType() instanceof ErroneousType or not exists(v.getType())
+select v, "This variable does not have a type."

cpp/ql/test/library-tests/extraction_errors/functions.expected

@@ -0,0 +1,6 @@
+| test.c:3:6:3:6 | g |
+| test.c:4:13:4:13 | no_such_function |
+| test.cpp:5:6:5:25 | function_with_errors |
+| test.cpp:14:10:14:12 | fn2 |
+| test.h:4:5:4:5 | f |
+| test.h:4:5:4:5 | f |

cpp/ql/test/library-tests/extraction_errors/functions.ql

@@ -0,0 +1,5 @@
+import cpp
+
+from Function fn
+where fn.fromSource()
+select fn

cpp/ql/test/library-tests/extraction_errors/test.c

@@ -0,0 +1,5 @@
+#include "test.h"
+
+void g() {
+    int x = no_such_function();
+}

cpp/ql/test/library-tests/extraction_errors/test.cpp

@@ -0,0 +1,17 @@
+// semmle-extractor-options: --expect_errors
+
+#include "test.h"
+
+void function_with_errors() {
+    auto x = no_such_function();
+    x+2;
+    no_such_function();
+    ADD(x+1, nsf2());
+    f(1);
+    f();
+}
+
+uint32_t fn2() {
+    this is a syntax error;
+    so_is_this(;
+}

cpp/ql/test/library-tests/extraction_errors/test.h

@@ -0,0 +1,4 @@
+
+#define ADD(A,B) ((A)+(B))
+
+int f(int);