diff --git a/cpp/ql/src/Telemetry/CompilerErrors.ql b/cpp/ql/src/Telemetry/CompilerErrors.ql new file mode 100644 index 00000000000..0f5166f9e9f --- /dev/null +++ b/cpp/ql/src/Telemetry/CompilerErrors.ql @@ -0,0 +1,13 @@ +/** + * @name Compiler errors + * @description A count of all compiler errors, grouped by error text. + * @kind metric + * @tags summary telemetry + * @id cpp/telemetry/compiler-errors + */ + +import Metrics + +from CppMetrics::ErrorCount m +where RankMetric::getRank(m) <= 50 +select m.toString(), m.getValue() diff --git a/cpp/ql/src/Telemetry/DatabaseQuality.ql b/cpp/ql/src/Telemetry/DatabaseQuality.ql new file mode 100644 index 00000000000..af8e340507b --- /dev/null +++ b/cpp/ql/src/Telemetry/DatabaseQuality.ql @@ -0,0 +1,12 @@ +/** + * @name Database quality + * @description Metrics that indicate the quality of the database. + * @kind metric + * @tags summary telemetry + * @id cpp/telemetry/database-quality + */ + +import Metrics + +from QualityMetric m +select m.toString(), m.getValue() diff --git a/cpp/ql/src/Telemetry/Diagnostics.qll b/cpp/ql/src/Telemetry/Diagnostics.qll new file mode 100644 index 00000000000..cc7dcb3a36d --- /dev/null +++ b/cpp/ql/src/Telemetry/Diagnostics.qll @@ -0,0 +1,29 @@ +import cpp + +/** + * A syntax error. + */ +class SyntaxError extends CompilerError { + SyntaxError() { + this.getTag().matches("exp_%") or + this.getTag() = + [ + "bad_data_member_initialization", "bad_pure_specifier", "bad_return", "bad_uuid_string", + "literal_without_initializer", "missing_class_definition", "missing_exception_declaration", + "nonstd_const_member_decl_not_allowed", "operator_name_not_allowed", + "wide_string_invalid_in_asm" + ] + } +} + +/** + * A cannot open file error. + * Typically this is due to a missing include. + */ +class CannotOpenFileError extends CompilerError { + CannotOpenFileError() { this.hasTag(["cannot_open_file", "cannot_open_file_reason"]) } + + string getIncludedFile() { + result = this.getMessage().regexpCapture("cannot open source file '([^']+)'", 1) + } +} diff --git a/cpp/ql/src/Telemetry/ExtractionMetrics.ql b/cpp/ql/src/Telemetry/ExtractionMetrics.ql new file mode 100644 index 00000000000..968bf456ecb --- /dev/null +++ b/cpp/ql/src/Telemetry/ExtractionMetrics.ql @@ -0,0 +1,12 @@ +/** + * @name Extraction metrics + * @description Raw metrics relating to extraction. + * @kind metric + * @tags summary telemetry + * @id cpp/telemetry/extraction-metrics + */ + +import Metrics + +from ExtractionMetric m +select m.toString(), m.getValue() diff --git a/cpp/ql/src/Telemetry/Metrics.qll b/cpp/ql/src/Telemetry/Metrics.qll new file mode 100644 index 00000000000..b3c90af8450 --- /dev/null +++ b/cpp/ql/src/Telemetry/Metrics.qll @@ -0,0 +1,269 @@ +import cpp +import Diagnostics + +/** + * A metric is a string with a value. + */ +abstract class Metric extends string { + bindingset[this] + Metric() { any() } +} + +/** + * A metric that we want to report in cpp/telemetry/extraction-metrics + */ +abstract class ExtractionMetric extends Metric { + bindingset[this] + ExtractionMetric() { any() } + + /** Gets the value of this metric. */ + abstract int getValue(); +} + +/** + * A metric that provides a baseline for a SuccessMetric. + */ +abstract class BaseMetric extends ExtractionMetric { + bindingset[this] + BaseMetric() { any() } +} + +/** + * A metric that is relative to another metric, + * so can be used to calculate percentages. + * + * For clarity, metrics should express success, + * so higher values means better. + */ +abstract class SuccessMetric extends ExtractionMetric { + bindingset[this] + SuccessMetric() { any() } + + /** Gets the metric this is relative to. */ + abstract BaseMetric getBaseline(); +} + +/** + * A metric used to report database quality. + */ +class QualityMetric extends Metric { + BaseMetric baseMetric; + SuccessMetric relativeMetric; + + QualityMetric() { + baseMetric = relativeMetric.getBaseline() and this = "Percentage of " + relativeMetric + } + + float getValue() { + baseMetric.getValue() > 0 and + result = 100.0 * relativeMetric.getValue() / baseMetric.getValue() + } +} + +signature class RankedMetric extends Metric { + int getValue(); +} + +module RankMetric { + int getRank(M s) { s = rank[result](M m | | m order by m.getValue() desc) } +} + +/** Various metrics we want to report. */ +module CppMetrics { + class Compilations extends BaseMetric { + Compilations() { this = "compilations" } + + override int getValue() { result = count(Compilation c) } + } + + class SourceAndHeaderFiles extends BaseMetric { + SourceAndHeaderFiles() { this = "source/header files" } + + override int getValue() { result = count(File f | f.fromSource()) } + } + + class SourceAndHeaderFilesWithoutErrors extends SuccessMetric { + SourceAndHeaderFilesWithoutErrors() { this = "source/header files without errors" } + + override int getValue() { + result = count(File f | f.fromSource() and not exists(CompilerError e | f = e.getFile())) + } + + override SourceAndHeaderFiles getBaseline() { any() } + } + + class CompilationsWithoutErrors extends SuccessMetric { + CompilationsWithoutErrors() { this = "compilations without errors" } + + override int getValue() { + result = count(Compilation c | not exists(Diagnostic d | d.getFile() = c.getAFileCompiled())) + } + + override Compilations getBaseline() { any() } + } + + class Expressions extends BaseMetric { + Expressions() { this = "expressions" } + + override int getValue() { result = count(Expr e) } + } + + class SucceededExpressions extends SuccessMetric { + SucceededExpressions() { this = "non-error expressions" } + + override int getValue() { result = count(Expr e) - count(ErrorExpr e) } + + override Expressions getBaseline() { any() } + } + + class TypedExpressions extends SuccessMetric { + TypedExpressions() { this = "expressions with a known type" } + + override int getValue() { result = count(Expr e | not e.getType() instanceof ErroneousType) } + + override Expressions getBaseline() { any() } + } + + class Calls extends BaseMetric { + Calls() { this = "calls" } + + override int getValue() { result = count(Call c) } + } + + class CallsWithExplicitTarget extends SuccessMetric { + CallsWithExplicitTarget() { this = "calls with an explicit target" } + + override int getValue() { + result = count(Call c | not c.getTarget().getADeclarationEntry().isImplicit()) + } + + override Calls getBaseline() { any() } + } + + class Variables extends BaseMetric { + Variables() { this = "variables" } + + override int getValue() { result = count(Variable v) } + } + + class VariablesKnownType extends SuccessMetric { + VariablesKnownType() { this = "variables with a known type" } + + override int getValue() { + result = count(Variable v | not v.getType() instanceof ErroneousType) + } + + override Variables getBaseline() { any() } + } + + class LinesOfText extends BaseMetric { + LinesOfText() { this = "lines of text" } + + override int getValue() { result = sum(File f | | f.getMetrics().getNumberOfLines()) } + } + + class LinesOfCode extends BaseMetric { + LinesOfCode() { this = "lines of code" } + + override int getValue() { result = sum(File f | | f.getMetrics().getNumberOfLinesOfCode()) } + } + + private predicate errorLine(File file, int line) { + exists(Locatable l, Location loc | + loc = l.getLocation() and + loc.getFile() = file and + line in [loc.getStartLine() .. loc.getEndLine()] + | + l instanceof Diagnostic + or + l instanceof ErrorExpr + ) + } + + class SucceededLines extends SuccessMetric { + SucceededLines() { this = "lines of code without errors" } + + override int getValue() { + result = + sum(File f | | f.getMetrics().getNumberOfLinesOfCode()) - + count(File f, int line | errorLine(f, line)) + } + + override LinesOfCode getBaseline() { any() } + } + + class Functions extends BaseMetric { + Functions() { this = "functions" } + + override int getValue() { result = count(Function f) } + } + + class SucceededFunctions extends SuccessMetric { + SucceededFunctions() { this = "functions without errors" } + + override int getValue() { result = count(Function f | not f.hasErrors()) } + + override Functions getBaseline() { any() } + } + + class Includes extends BaseMetric { + Includes() { this = "#include directives" } + + override int getValue() { result = count(Include i) + count(CannotOpenFileError e) } + } + + class SucceededIncludes extends SuccessMetric { + SucceededIncludes() { this = "successfully resolved #include directives" } + + override int getValue() { result = count(Include i) } + + override Includes getBaseline() { any() } + } + + class SucceededIncludeCount extends Metric { + string includeText; + + SucceededIncludeCount() { + exists(Include i | + i.getIncludeText() = includeText and + exists(i.getFile().getRelativePath()) // Only report includes from the repo + ) and + this = "Successfully included " + includeText + } + + int getValue() { result = count(Include i | i.getIncludeText() = includeText) } + + string getIncludeText() { result = includeText } + } + + class MissingIncludeCount extends Metric { + string includeText; + + MissingIncludeCount() { + exists(CannotOpenFileError e | e.getIncludedFile() = includeText) and + this = "Failed to include '" + includeText + "'" + } + + int getValue() { result = count(CannotOpenFileError e | e.getIncludedFile() = includeText) } + + string getIncludeText() { result = includeText } + } + + class CompilerErrors extends ExtractionMetric { + CompilerErrors() { this = "compiler errors" } + + override int getValue() { result = count(CompilerError e) } + } + + class ErrorCount extends Metric { + ErrorCount() { exists(CompilerError e | e.getMessage() = this) } + + int getValue() { result = count(CompilerError e | e.getMessage() = this) } + } + + class SyntaxErrorCount extends ExtractionMetric { + SyntaxErrorCount() { this = "syntax errors" } + + override int getValue() { result = count(SyntaxError e) } + } +} diff --git a/cpp/ql/src/Telemetry/MissingIncludes.ql b/cpp/ql/src/Telemetry/MissingIncludes.ql new file mode 100644 index 00000000000..6ff58729dd2 --- /dev/null +++ b/cpp/ql/src/Telemetry/MissingIncludes.ql @@ -0,0 +1,13 @@ +/** + * @name Failed to include header file + * @description A count of all failed includes, grouped by filename. + * @kind metric + * @tags summary telemetry + * @id cpp/telemetry/failed-includes + */ + +import Metrics + +from CppMetrics::MissingIncludeCount e +where RankMetric::getRank(e) <= 50 +select e.getIncludeText(), e.getValue() diff --git a/cpp/ql/src/Telemetry/SucceededIncludes.ql b/cpp/ql/src/Telemetry/SucceededIncludes.ql new file mode 100644 index 00000000000..aa08acd9fdb --- /dev/null +++ b/cpp/ql/src/Telemetry/SucceededIncludes.ql @@ -0,0 +1,13 @@ +/** + * @name Successfully included header files + * @description A count of all succeeded includes, grouped by filename. + * @kind metric + * @tags summary telemetry + * @id cpp/telemetry/succeeded-includes + */ + +import Metrics + +from CppMetrics::SucceededIncludeCount m +where RankMetric::getRank(m) <= 50 +select m.getIncludeText(), m.getValue() diff --git a/cpp/ql/test/library-tests/extraction_errors/CompilerErrors.expected b/cpp/ql/test/library-tests/extraction_errors/CompilerErrors.expected new file mode 100644 index 00000000000..1e4e4c8e3c6 --- /dev/null +++ b/cpp/ql/test/library-tests/extraction_errors/CompilerErrors.expected @@ -0,0 +1,10 @@ +| 'this' may only be used inside a nonstatic member function | 1 | +| There was an error during this compilation | 1 | +| expected a ')' | 1 | +| expected a ';' | 1 | +| expected an expression | 1 | +| identifier 'no_such_function' is undefined | 1 | +| identifier 'nsf2' is undefined | 1 | +| identifier 'so_is_this' is undefined | 1 | +| identifier 'uint32_t' is undefined | 1 | +| too few arguments in function call | 1 | diff --git a/cpp/ql/test/library-tests/extraction_errors/CompilerErrors.qlref b/cpp/ql/test/library-tests/extraction_errors/CompilerErrors.qlref new file mode 100644 index 00000000000..fd0c287c00d --- /dev/null +++ b/cpp/ql/test/library-tests/extraction_errors/CompilerErrors.qlref @@ -0,0 +1 @@ +Telemetry/CompilerErrors.ql diff --git a/cpp/ql/test/library-tests/extraction_errors/DatabaseQuality.expected b/cpp/ql/test/library-tests/extraction_errors/DatabaseQuality.expected new file mode 100644 index 00000000000..6d9cea2d734 --- /dev/null +++ b/cpp/ql/test/library-tests/extraction_errors/DatabaseQuality.expected @@ -0,0 +1,9 @@ +| Percentage of calls with an explicit target | 50.0 | +| Percentage of compilations without errors | 50.0 | +| Percentage of expressions with a known type | 30.0 | +| Percentage of functions without errors | 75.0 | +| Percentage of lines of code without errors | 63.1578947368421 | +| Percentage of non-error expressions | 30.0 | +| Percentage of source/header files without errors | 66.66666666666667 | +| Percentage of successfully resolved #include directives | 100.0 | +| Percentage of variables with a known type | 90.0 | diff --git a/cpp/ql/test/library-tests/extraction_errors/DatabaseQuality.qlref b/cpp/ql/test/library-tests/extraction_errors/DatabaseQuality.qlref new file mode 100644 index 00000000000..b2c536f00d7 --- /dev/null +++ b/cpp/ql/test/library-tests/extraction_errors/DatabaseQuality.qlref @@ -0,0 +1 @@ +Telemetry/DatabaseQuality.ql diff --git a/cpp/ql/test/library-tests/extraction_errors/ErrorExprs.expected b/cpp/ql/test/library-tests/extraction_errors/ErrorExprs.expected new file mode 100644 index 00000000000..3f437a7fd26 --- /dev/null +++ b/cpp/ql/test/library-tests/extraction_errors/ErrorExprs.expected @@ -0,0 +1,7 @@ +| test.cpp:6:13:6:31 | initializer for x | test.cpp:6:13:6:31 | | +| test.cpp:7:5:7:8 | ExprStmt | file://:0:0:0:0 | | +| test.cpp:8:5:8:23 | ExprStmt | file://:0:0:0:0 | | +| test.cpp:9:5:9:21 | ExprStmt | file://:0:0:0:0 | | +| test.cpp:11:5:11:8 | ExprStmt | file://:0:0:0:0 | | +| test.cpp:15:5:15:8 | ExprStmt | file://:0:0:0:0 | | +| test.cpp:16:5:16:16 | ExprStmt | file://:0:0:0:0 | | diff --git a/cpp/ql/test/library-tests/extraction_errors/ErrorExprs.ql b/cpp/ql/test/library-tests/extraction_errors/ErrorExprs.ql new file mode 100644 index 00000000000..25c2118e010 --- /dev/null +++ b/cpp/ql/test/library-tests/extraction_errors/ErrorExprs.ql @@ -0,0 +1,4 @@ +import cpp + +from ErrorExpr e +select e.getParent(), e diff --git a/cpp/ql/test/library-tests/extraction_errors/ExtractionMetrics.expected b/cpp/ql/test/library-tests/extraction_errors/ExtractionMetrics.expected new file mode 100644 index 00000000000..ee2cc0f9963 --- /dev/null +++ b/cpp/ql/test/library-tests/extraction_errors/ExtractionMetrics.expected @@ -0,0 +1,20 @@ +| #include directives | 2 | +| calls | 2 | +| calls with an explicit target | 1 | +| compilations | 2 | +| compilations without errors | 1 | +| compiler errors | 10 | +| expressions | 10 | +| expressions with a known type | 3 | +| functions | 8 | +| functions without errors | 6 | +| lines of code | 19 | +| lines of code without errors | 12 | +| lines of text | 24 | +| non-error expressions | 3 | +| source/header files | 3 | +| source/header files without errors | 2 | +| successfully resolved #include directives | 2 | +| syntax errors | 3 | +| variables | 10 | +| variables with a known type | 9 | diff --git a/cpp/ql/test/library-tests/extraction_errors/ExtractionMetrics.qlref b/cpp/ql/test/library-tests/extraction_errors/ExtractionMetrics.qlref new file mode 100644 index 00000000000..80547fdfd98 --- /dev/null +++ b/cpp/ql/test/library-tests/extraction_errors/ExtractionMetrics.qlref @@ -0,0 +1 @@ +Telemetry/ExtractionMetrics.ql \ No newline at end of file diff --git a/cpp/ql/test/library-tests/extraction_errors/SucceededIncludes.expected b/cpp/ql/test/library-tests/extraction_errors/SucceededIncludes.expected new file mode 100644 index 00000000000..13536ce172c --- /dev/null +++ b/cpp/ql/test/library-tests/extraction_errors/SucceededIncludes.expected @@ -0,0 +1 @@ +| "test.h" | 2 | diff --git a/cpp/ql/test/library-tests/extraction_errors/SucceededIncludes.qlref b/cpp/ql/test/library-tests/extraction_errors/SucceededIncludes.qlref new file mode 100644 index 00000000000..055b6af49a7 --- /dev/null +++ b/cpp/ql/test/library-tests/extraction_errors/SucceededIncludes.qlref @@ -0,0 +1 @@ +Telemetry/SucceededIncludes.ql diff --git a/cpp/ql/test/library-tests/extraction_errors/diags.expected b/cpp/ql/test/library-tests/extraction_errors/diags.expected new file mode 100644 index 00000000000..68e5c9c154c --- /dev/null +++ b/cpp/ql/test/library-tests/extraction_errors/diags.expected @@ -0,0 +1,10 @@ +| file://:0:0:0:0 | There was an error during this compilation | +| test.cpp:6:14:6:14 | identifier 'no_such_function' is undefined | +| test.cpp:9:14:9:14 | identifier 'nsf2' is undefined | +| test.cpp:11:7:11:7 | too few arguments in function call | +| test.cpp:14:1:14:1 | identifier 'uint32_t' is undefined | +| test.cpp:15:5:15:5 | 'this' may only be used inside a nonstatic member function | +| test.cpp:15:10:15:10 | expected a ';' | +| test.cpp:16:5:16:5 | identifier 'so_is_this' is undefined | +| test.cpp:16:16:16:16 | expected a ')' | +| test.cpp:16:16:16:16 | expected an expression | diff --git a/cpp/ql/test/library-tests/extraction_errors/diags.ql b/cpp/ql/test/library-tests/extraction_errors/diags.ql new file mode 100644 index 00000000000..3fa864748e1 --- /dev/null +++ b/cpp/ql/test/library-tests/extraction_errors/diags.ql @@ -0,0 +1,4 @@ +import cpp + +from Diagnostic d +select d diff --git a/cpp/ql/test/library-tests/extraction_errors/error_variables.expected b/cpp/ql/test/library-tests/extraction_errors/error_variables.expected new file mode 100644 index 00000000000..d5b216dba71 --- /dev/null +++ b/cpp/ql/test/library-tests/extraction_errors/error_variables.expected @@ -0,0 +1 @@ +| test.cpp:6:10:6:10 | x | This variable does not have a type. | diff --git a/cpp/ql/test/library-tests/extraction_errors/error_variables.ql b/cpp/ql/test/library-tests/extraction_errors/error_variables.ql new file mode 100644 index 00000000000..e3867462b56 --- /dev/null +++ b/cpp/ql/test/library-tests/extraction_errors/error_variables.ql @@ -0,0 +1,5 @@ +import cpp + +from Variable v +where v.getType() instanceof ErroneousType or not exists(v.getType()) +select v, "This variable does not have a type." diff --git a/cpp/ql/test/library-tests/extraction_errors/functions.expected b/cpp/ql/test/library-tests/extraction_errors/functions.expected new file mode 100644 index 00000000000..db80ede0655 --- /dev/null +++ b/cpp/ql/test/library-tests/extraction_errors/functions.expected @@ -0,0 +1,6 @@ +| test.c:3:6:3:6 | g | +| test.c:4:13:4:13 | no_such_function | +| test.cpp:5:6:5:25 | function_with_errors | +| test.cpp:14:10:14:12 | fn2 | +| test.h:2:5:2:5 | f | +| test.h:2:5:2:5 | f | diff --git a/cpp/ql/test/library-tests/extraction_errors/functions.ql b/cpp/ql/test/library-tests/extraction_errors/functions.ql new file mode 100644 index 00000000000..ac52eec7309 --- /dev/null +++ b/cpp/ql/test/library-tests/extraction_errors/functions.ql @@ -0,0 +1,5 @@ +import cpp + +from Function fn +where fn.fromSource() +select fn diff --git a/cpp/ql/test/library-tests/extraction_errors/test.c b/cpp/ql/test/library-tests/extraction_errors/test.c new file mode 100644 index 00000000000..e247964f71c --- /dev/null +++ b/cpp/ql/test/library-tests/extraction_errors/test.c @@ -0,0 +1,5 @@ +#include "test.h" + +void g() { + int x = no_such_function(); +} diff --git a/cpp/ql/test/library-tests/extraction_errors/test.cpp b/cpp/ql/test/library-tests/extraction_errors/test.cpp new file mode 100644 index 00000000000..b2ca86bcbaa --- /dev/null +++ b/cpp/ql/test/library-tests/extraction_errors/test.cpp @@ -0,0 +1,17 @@ +// semmle-extractor-options: --expect_errors + +#include "test.h" + +void function_with_errors() { + auto x = no_such_function(); + x+2; + no_such_function(); + ADD(x+1, nsf2()); + f(1); + f(); +} + +uint32_t fn2() { + this is a syntax error; + so_is_this(; +} diff --git a/cpp/ql/test/library-tests/extraction_errors/test.h b/cpp/ql/test/library-tests/extraction_errors/test.h new file mode 100644 index 00000000000..a3941657c0e --- /dev/null +++ b/cpp/ql/test/library-tests/extraction_errors/test.h @@ -0,0 +1,2 @@ +#define ADD(A,B) ((A)+(B)) +int f(int);